From cbec2b091196ad6ea1ba6f0828f1e66c81b2b125 Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:36:09 +0000 Subject: [PATCH] amd-kernels: tensor parallel 2 for MiniMax-M3.1-Flash on MI355X The kernels vLLM runs for MiniMax-M3.1-Flash now also serve tensor parallel 2 (per GPU: 32 q / 2 KV / 2 index heads, QKV N 4992, o_proj K 4096, MoE I 1536, dense I 6144, lm_head N 100032, two-rank collectives over one xGMI link). New per-shape packages for the TP2 shapes and two-rank collectives; the head-count-generic packages (sparse attention, indexer, inserts, SWA decode, embedding gathers) gain tuned two-KV-head / two-rank paths. TP4 dispatch is unchanged except where a shared shape measured faster at every served row. Also: a split-K decode kernel kind for mid-M MXFP8 GEMMs, the full MoE shared expert on every TP2 rank with an FP4-only sequence-parallel gather and a second stream, and the SDMA transport's agents keyed by PCI domain. Co-Authored-By: Claude Opus 5.5 (1M context) --- amd_kernels/README.md | 89 +- .../swiglu_oai_mxfp8_i6144/README.md | 47 + .../swiglu_oai_mxfp8_i6144/__init__.py | 15 + .../activation/swiglu_oai_mxfp8_i6144/host.py | 101 + .../swiglu_oai_mxfp8_i6144/kernel.py | 287 +++ .../swiglu_oai_mxfp8_i6144/reference.py | 58 + .../README.md | 49 + .../__init__.py | 8 + .../host.py | 247 ++ .../kernel.py | 542 ++++ .../packing.py | 36 + .../reference.py | 48 + .../norm_rope_fp8_insert_d128_rot64/README.md | 12 +- .../README.md | 32 +- .../host.py | 75 +- .../kernel_prefill_q64_kv4_iq4.py | 7 +- .../kernel_q32_kv2_iq2_vsplit.py | 442 ++++ .../qkv.py | 22 +- .../splitk_reduce_kernel_q64_kv4_iq4.py | 18 +- .../README.md | 48 +- ...bf16mfma_gqa16_d128_p128_top16_prefetch.py | 14 +- ...a_gqa16_d128_p128_top16_prefetch_select.py | 25 +- ...bf16mfma_gqa16_d128_p128_top16_q8_fixed.py | 14 +- ...a_gqa16_d128_p128_top16_q8_fixed_select.py | 25 +- ...f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py | 14 +- ..._gqa16_d128_p128_top16_q8_rowxcd_select.py | 23 +- ...f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py | 15 +- ..._gqa16_d128_p128_top16_q8_rowxcd_select.py | 23 +- .../fp8/kernel.py | 13 +- ...fp8mfma_gqa16_d128_p128_top16_q8_rowxcd.py | 11 +- .../host.py | 183 +- .../index_decode.py | 27 +- .../kernel.py | 9 +- .../kernel_select.py | 32 +- .../README.md | 21 +- .../layer.py | 23 +- .../README.md | 27 +- .../host.py | 44 +- .../swa_decode_fp8kv_gqa16_d128/README.md | 20 +- .../bf16mfma_gqa16_d128_p128_ring.py | 705 +++++ .../swa_decode_fp8kv_gqa16_d128/host.py | 82 +- .../oproj_input.py | 7 +- .../all_gather_i64_c7_ranks2/README.md | 46 + .../all_gather_i64_c7_ranks2/__init__.py | 4 + .../all_gather_i64_c7_ranks2/host.py | 28 + .../all_gather_i64_c7_ranks2/kernel.py | 54 + .../all_gather_i64_c7_ranks2/reference.py | 14 + .../README.md | 48 + .../__init__.py | 4 + .../all_gather_sampler_i64_c8_ranks2/host.py | 74 + .../kernel.py | 101 + .../reference.py | 35 + .../all_to_all_bf16_n100032_ranks2/README.md | 47 + .../__init__.py | 5 + .../all_to_all_bf16_n100032_ranks2/host.py | 202 ++ .../all_to_all_bf16_n100032_ranks2/kernel.py | 95 + .../reference.py | 23 + .../stage_publish.py | 76 + .../pull_collectives_ranks2/README.md | 60 + .../pull_collectives_ranks2/__init__.py | 16 + .../all_gather_fence_h6144.py | 38 + .../pull_collectives_ranks2/gather_host.py | 73 + .../pull_collectives_ranks2/host.py | 240 ++ .../pull_collectives_ranks2/kernel.py | 167 ++ .../pull_collectives_ranks2/lossless_x3.py | 185 ++ .../norm_gather_h6144.py | 244 ++ .../pull_unpack_q4096_shuffled.py | 234 ++ .../pull_collectives_ranks2/reference.py | 99 + .../pull_collectives_ranks2/slots4_host.py | 206 ++ .../slots4_stage_pull.py | 150 ++ ...lots4_stage_pull_sum2_add_rmsnorm_mxfp8.py | 311 +++ .../slots4_stage_rows.py | 61 + .../slots4_stage_rows_x3.py | 57 + .../pull_collectives_ranks2/stage_pull.py | 270 ++ .../sum2_add_rmsnorm_mxfp8.py | 253 ++ .../sum2_x3_add_rmsnorm_mxfp8.py | 110 + .../collectives/sdma_push_ranks2/README.md | 44 + .../collectives/sdma_push_ranks2/__init__.py | 7 + .../sdma_push_ranks2/hsa_runtime.py | 123 + .../collectives/sdma_push_ranks2/ipc.py | 74 + .../collectives/sdma_push_ranks2/transport.py | 193 ++ .../collectives/sdma_push_ranks4/README.md | 1 + .../sdma_push_ranks4/hsa_runtime.py | 15 +- .../README.md | 60 + .../__init__.py | 27 + .../handoffs.py | 407 +++ .../handoffs_sdma.py | 419 +++ .../mxfp8_pack_q4096.py | 174 ++ .../reference.py | 47 + .../dense_mxfp8_h6144_i6144_ranks2/README.md | 46 + .../__init__.py | 13 + .../dense_mxfp8_h6144_i6144_ranks2/host.py | 216 ++ .../dense_mxfp8_h6144_i6144_ranks2/mlp.py | 75 + .../norm_linear.py | 47 + .../reference.py | 46 + .../vocab_gather_gemma_mxfp8_h6144/README.md | 44 +- .../vocab_gather_gemma_mxfp8_h6144/host.py | 6 +- .../vocab_gather_gemma_mxfp8_h6144/kernel.py | 14 +- .../kernel_fourwave.py | 14 +- .../reference.py | 5 +- .../embedding/vocab_gather_h6144/README.md | 42 +- .../embedding/vocab_gather_h6144/host.py | 11 +- .../embedding/vocab_gather_h6144/ipc.py | 6 +- .../embedding/vocab_gather_h6144/kernel.py | 7 +- .../embedding/vocab_gather_h6144/reference.py | 2 +- .../bf16_concat_linear_n3072_k30720/README.md | 37 + .../__init__.py | 5 + .../bf16_concat_linear_n3072_k30720/comm.py | 100 + .../bf16_concat_linear_n3072_k30720/flags.py | 46 + .../bf16_concat_linear_n3072_k30720/host.py | 92 + .../bf16_concat_linear_n3072_k30720/kernel.py | 30 + .../kernel_lds.py | 219 ++ .../kernel_staged_a_eightwave.py | 104 + .../reduce_gather.py | 81 + .../reference.py | 16 + amd_kernels/gemm/bf16_n100032_k6144/README.md | 57 + .../gemm/bf16_n100032_k6144/__init__.py | 7 + amd_kernels/gemm/bf16_n100032_k6144/host.py | 220 ++ .../gemm/bf16_n100032_k6144/indexed.py | 247 ++ amd_kernels/gemm/bf16_n100032_k6144/kernel.py | 225 ++ amd_kernels/gemm/bf16_n100032_k6144/large.py | 245 ++ .../gemm/bf16_n100032_k6144/large_indexed.py | 284 ++ .../gemm/bf16_n100032_k6144/publish.py | 35 + .../gemm/bf16_n100032_k6144/reference.py | 19 + amd_kernels/gemm/bf16_n200064_k256/README.md | 32 +- amd_kernels/gemm/bf16_n200064_k256/host.py | 15 +- .../gemm/bf16_n200064_k256/kernel_m128.py | 84 + .../gemm/bf16_n200064_k256/reference.py | 4 +- amd_kernels/gemm/mxfp8_common/README.md | 38 +- amd_kernels/gemm/mxfp8_common/gemm.py | 154 +- .../gemm/mxfp8_common/mxfp8_gemm_decode.py | 6 +- .../mxfp8_common/mxfp8_gemm_decode_splitk.py | 305 +++ .../gemm/mxfp8_common/mxfp8_gemm_tuned.py | 138 +- .../README.md | 39 + .../__init__.py | 7 + .../gateup_swiglu_bf16.py | 605 +++++ .../gateup_swiglu_bf16_onewave.py | 579 +++++ .../host.py | 59 + amd_kernels/gemm/mxfp8_n4992_k6144/README.md | 52 + .../gemm/mxfp8_n4992_k6144/__init__.py | 3 + amd_kernels/gemm/mxfp8_n4992_k6144/host.py | 206 ++ .../gemm/mxfp8_n4992_k6144/persistent.py | 740 ++++++ amd_kernels/gemm/mxfp8_n6144_k4096/README.md | 41 + .../gemm/mxfp8_n6144_k4096/__init__.py | 3 + amd_kernels/gemm/mxfp8_n6144_k4096/host.py | 173 ++ .../prefill_256x192_one_tile.py | 842 ++++++ .../gemm/mxfp8_n6144_k4096/tile_128x192.py | 228 ++ .../indexer_nvfp4_d128_p128_top16/README.md | 47 +- .../decode_top16.py | 72 +- .../prefill_top16.py | 6 +- .../score_decode/columns.py | 14 + .../score_decode/host.py | 118 +- .../score_firstpage_d128_p128_h2.py | 181 ++ .../score_decode/score_page_d128_p128_h2.py | 186 ++ .../score_decode/score_runs_d128_p128_h2.py | 439 ++++ .../score_prefill/host.py | 7 +- .../split2/__init__.py | 4 + .../split2/decode_score.py | 689 +++++ .../split2/host.py | 157 ++ .../split2/kernel.py | 50 + .../split2/select_merge.py | 164 ++ .../moe/mxfp4_h6144_i1536_e128_top4/README.md | 60 + .../mxfp4_h6144_i1536_e128_top4/__init__.py | 41 + .../a16w4_utils.py | 188 ++ .../moe/mxfp4_h6144_i1536_e128_top4/chain.py | 297 +++ .../common/__init__.py | 3 + .../common/epilogue.py | 69 + .../common/launch.py | 14 + .../common/loaders.py | 433 ++++ .../common/mfma_order.py | 10 + .../common/reduce_slots_shared.py | 41 + .../common/scale_sort_tile_map.py | 281 ++ .../common/sort.py | 65 + .../common/sort_count_all.py | 202 ++ .../common/sort_scales_all.py | 305 +++ .../common/sort_tile_map.py | 221 ++ .../common/tile_map.py | 26 + .../common/utils.py | 77 + .../experts_a4w4.py | 167 ++ .../experts_a4w4_bm128.py | 171 ++ .../experts_a4w4_small.py | 131 + .../gemm12_a4w4.py | 162 ++ .../gemm12_a4w4_shared_a8w8.py | 171 ++ .../gemm1_a4w4_bm128_half_rows.py | 931 +++++++ .../gemm1_a4w4_bm256_half_rows.py | 931 +++++++ .../gemm1_a4w4_id_rows.py | 226 ++ .../gemm1_a4w4_inline_sort.py | 217 ++ .../gemm1_a4w4_small.py | 217 ++ .../gemm1_prefill_h6144_i1536.py | 898 +++++++ .../gemm2_a4w4_bm128.py | 795 ++++++ .../gemm2_a4w4_bm128_adaptive.py | 801 ++++++ .../gemm2_a4w4_bm128_epilogue_groups.py | 808 ++++++ .../gemm2_a4w4_bm256_epilogue_groups.py | 776 ++++++ .../gemm2_a4w4_bm256_half_rows.py | 771 ++++++ .../gemm2_a4w4_id_rows.py | 124 + .../gemm2_a4w4_inline_sort.py | 134 + .../gemm2_a4w4_small.py | 126 + .../i3072/__init__.py | 5 + .../gemm1_prefill_h6144_i3072_persistent.py | 237 ++ .../moe/mxfp4_h6144_i1536_e128_top4/layout.py | 48 + .../mxfp4_from_mxfp8.py | 134 + .../mxfp8/__init__.py | 5 + .../mxfp8/gemm1_prefill_h6144_i3072.py | 218 ++ .../mxfp8/shared_dense_4wave.py | 2284 +++++++++++++++++ .../prefill_h6144_i1536.py | 45 + .../prepared_input.py | 234 ++ .../moe/mxfp4_h6144_i1536_e128_top4/quant.py | 106 + .../mxfp4_h6144_i1536_e128_top4/reference.py | 235 ++ .../router_gemm.py | 467 ++++ .../router_gemm_direct.py | 138 + .../router_gemm_nblocks.py | 325 +++ .../router_gemm_nblocks_lines.py | 242 ++ .../router_gemm_topk_mxfp4_mxfp8_quant.py | 60 + .../router_mxfp4_mxfp8_quant_line_layout.py | 453 ++++ .../router_mxfp4_mxfp8_quant_row_scales.py | 449 ++++ .../router_mxfp4_quant.py | 104 + .../router_mxfp4_quant_line_layout.py | 417 +++ .../router_topk.py | 272 ++ .../router_topk_mxfp4_mxfp8_quant.py | 324 +++ ...ter_topk_mxfp4_mxfp8_quant_sorted_tasks.py | 320 +++ .../router_topk_sorted.py | 196 ++ .../router_topk_vec2.py | 173 ++ .../routing_ids.py | 89 + .../shared_dense.py | 218 ++ .../shared_expert.py | 231 ++ .../shared_gemm1_a8w8.py | 186 ++ .../shared_gemm1_a8w8_epilogue_early.py | 26 + .../shared_gemm1_a8w8_single_tile.py | 25 + .../shared_gemm1_a8w8_write_through.py | 42 + .../shared_gemm2_a8w8.py | 87 + .../shared_quant.py | 86 + .../sidecar_pack.py | 61 + .../sidecar_unpack.py | 67 + .../mxfp4_h6144_i1536_e128_top4/sort_a16w4.py | 242 ++ .../README.md | 47 + .../__init__.py | 5 + .../common.py | 106 + .../host.py | 185 ++ .../kernel.py | 246 ++ .../norm_math.py | 137 + .../quant_math.py | 89 + .../reference.py | 40 + .../norm/gemma_add_rmsnorm_h6144/README.md | 45 +- .../norm/gemma_add_rmsnorm_h6144/__init__.py | 4 +- .../allreduce_add_rmsnorm_ranks2/__init__.py | 180 ++ .../allreduce_add_rmsnorm_ranks2/common.py | 161 ++ .../allreduce_add_rmsnorm_ranks2/host.py | 247 ++ .../lossless_x3.py | 72 + .../allreduce_add_rmsnorm_ranks2/oneshot.py | 272 ++ .../norm/gemma_add_rmsnorm_h6144/reference.py | 16 +- .../README.md | 58 + .../__init__.py | 10 + .../host.py | 465 ++++ .../reference.py | 48 + .../sampling/top_p_mask_f32_v200064/README.md | 53 +- .../top_p_mask_f32_v200064/__init__.py | 9 +- .../sampling/top_p_mask_f32_v200064/host.py | 66 +- .../top_p_mask_f32_v200064/kernel_rows.py | 900 +++++++ .../top_p_mask_f32_v200064/reference.py | 198 ++ .../top_p_mask_f32_v200064/sigma_table.py | 27 + .../benchmark_swiglu_oai_mxfp8.py | 98 + .../benchmark_chain.py | 175 ++ .../benchmark_norm_rope_insert.py | 13 +- .../benchmark_norm_rope_insert.py | 44 +- .../benchmark_index_decode.py | 140 + .../benchmark_index_decode_ranks2.py | 293 +++ .../benchmark_oproj_input.py | 161 ++ .../benchmark_sparse_decode.py | 28 +- .../benchmark_sparse_prefill.py | 10 +- .../sweep_tiles.py | 87 + .../benchmark_decode.py | 8 +- .../benchmark_oproj_input.py | 144 ++ .../benchmark_gather.py | 88 + .../benchmark_gather.py | 101 + .../benchmark_exchange.py | 104 + .../sweep_pull.py | 80 + .../benchmark_collectives.py | 299 +++ .../pull_collectives_ranks2/benchmark_link.py | 133 + .../sdma_push_ranks2/bench_engines.py | 91 + .../sdma_push_ranks2/benchmark_push.py | 121 + .../benchmark_sp_layer.py | 307 +++ .../benchmark_draft_forward.py | 276 ++ .../benchmark_gather_norm.py | 17 +- .../vocab_gather_h6144/benchmark_gather.py | 14 +- .../vocab_gather_h6144/benchmark_stock.py | 95 + .../benchmark_comm.py | 85 + .../benchmark_concat_linear.py | 119 + .../gemm/bf16_n100032_k6144/benchmark_gemm.py | 94 + .../gemm/bf16_n200064_k256/benchmark_gemm.py | 2 +- .../mxfp8_common/benchmark_decode_splitk.py | 134 + .../gemm/mxfp8_common/benchmark_table.py | 43 + benchmarks/gemm/mxfp8_common/sweep_rows.py | 310 +++ .../benchmark_gateup_swiglu_bf16.py | 117 + .../benchmark_n4992_k6144.py | 208 ++ .../mxfp8_n4992_k6144/sweep_n4992_k6144.py | 204 ++ .../benchmark_n6144_k4096.py | 128 + .../gemm/mxfp8_n6144_k4096/sweep_rows.py | 152 ++ .../benchmark_index_decode_h2.py | 152 ++ .../benchmark_indexer.py | 5 +- .../sweep_decode_h2.py | 113 + .../sweep_prefill_plan.py | 78 + .../benchmark_baseline_moe.py | 115 + .../benchmark_prepared.py | 75 + .../benchmark_shared_full.py | 78 + .../benchmark_shared_moe.py | 132 + .../benchmark_sidecar.py | 72 + .../benchmark_allreduce_ranks2.py | 143 ++ .../benchmark_ar_add_rmsnorm_ranks2.py | 200 ++ .../benchmark_block.py | 183 ++ .../benchmark_top_p_mask.py | 14 +- .../swiglu_oai_mxfp8_i6144/__init__.py | 2 + .../test_swiglu_oai_mxfp8.py | 178 ++ .../__init__.py | 2 + .../test_compile.py | 76 + .../test_counters.py | 132 + .../test_isa.py | 55 + .../test_packing.py | 58 + .../test_qkv_insert.py | 409 +++ .../test_compile.py | 2 + .../test_entry_warm_up.py | 12 +- .../test_norm_rope_insert.py | 15 +- .../test_compile.py | 91 +- .../test_norm_rope_insert.py | 145 +- .../test_qkv_chain_tp2.py | 103 + .../plan.py | 44 +- .../test_fp8_decode.py | 2 +- .../test_index_decode.py | 29 +- .../test_index_decode_ranks2.py | 74 + .../test_index_decode_warm_up.py | 31 +- .../test_index_select.py | 58 +- .../test_oproj_input.py | 59 +- .../test_sparse_decode.py | 67 +- .../test_warm_up_entry.py | 18 +- .../test_init_warm_up.py | 66 +- .../test_layer.py | 43 +- .../cases.py | 3 +- .../test_compile.py | 14 +- .../test_sparse_prefill.py | 22 +- .../test_tp2_tiles.py | 119 + .../test_warm_up_entry.py | 17 +- .../test_compile.py | 6 + .../test_decode.py | 93 +- .../test_oproj_input.py | 73 +- .../test_oproj_input_entry.py | 26 +- .../all_gather_i64_c7_ranks2/__init__.py | 2 + .../test_i64_c7_gather.py | 43 + .../__init__.py | 2 + .../test_sampler_gather.py | 100 + .../__init__.py | 2 + .../test_exchange.py | 134 + .../pull_collectives_ranks2/__init__.py | 2 + .../pull_collectives_ranks2/served.py | 25 + .../test_all_gather.py | 186 ++ .../test_gather_odd_rows.py | 80 + .../test_norm_gather_h6144.py | 153 ++ .../test_slots4_reduce_scatter.py | 356 +++ .../test_unpack_q4096_shuffled.py | 105 + .../collectives/sdma_push_ranks2/__init__.py | 2 + .../sdma_push_ranks2/test_transport.py | 64 + .../sdma_push_ranks4/test_agents.py | 68 + .../__init__.py | 2 + .../test_coded_tail.py | 55 + .../test_dbo.py | 99 + .../test_handoffs.py | 371 +++ .../test_pack.py | 91 + .../test_shard.py | 41 + .../test_warm_up.py | 102 + .../workload.py | 49 + .../__init__.py | 2 + .../test_decoder_ranks2.py | 269 ++ .../test_mlp_norm_linear.py | 195 ++ .../test_gather_norm.py | 15 +- .../vocab_gather_h6144/test_gather.py | 12 +- .../__init__.py | 2 + .../test_compile.py | 35 + .../test_concat_linear.py | 107 + .../test_gather.py | 46 + tests/gemm/bf16_n100032_k6144/__init__.py | 2 + tests/gemm/bf16_n100032_k6144/test_gemm.py | 194 ++ tests/gemm/bf16_n100032_k6144/test_publish.py | 139 + tests/gemm/bf16_n200064_k256/test_gemm.py | 23 +- tests/gemm/mxfp8_common/test_compile.py | 11 +- .../mxfp8_common/test_ctx_kv_rows_n2560.py | 74 + tests/gemm/mxfp8_common/test_decode_splitk.py | 339 +++ tests/gemm/mxfp8_common/test_dispatch.py | 6 +- .../mxfp8_common/test_ffn_qkv_decode_rows.py | 7 +- .../mxfp8_common/test_gateup_decode_rows.py | 96 - tests/gemm/mxfp8_common/test_gemm.py | 6 + .../gemm/mxfp8_common/test_oproj_out_k4096.py | 67 + tests/gemm/mxfp8_common/test_shape_rows.py | 285 ++ .../__init__.py | 2 + .../test_gateup_swiglu_bf16.py | 130 + tests/gemm/mxfp8_n4992_k6144/__init__.py | 2 + tests/gemm/mxfp8_n4992_k6144/test_isa.py | 54 + .../mxfp8_n4992_k6144/test_n4992_k6144.py | 225 ++ tests/gemm/mxfp8_n4992_k6144/test_warmup.py | 47 + tests/gemm/mxfp8_n6144_k4096/__init__.py | 2 + .../mxfp8_n6144_k4096/test_kernels_k4096.py | 68 + .../mxfp8_n6144_k4096/test_oproj_k4096.py | 147 ++ .../gemm/mxfp8_n6144_k4096/test_out_k4096.py | 38 + .../test_compile.py | 5 +- .../test_compile_entry.py | 17 +- .../test_compile_prefill_entry.py | 20 +- .../test_compile_rows_entry.py | 18 +- .../test_prefill_top16.py | 16 +- .../test_score_top16_h2.py | 193 ++ .../test_split2_ranks2.py | 158 ++ .../mxfp4_h6144_i1536_e128_top4/__init__.py | 2 + .../test_a4w4_small.py | 62 + .../test_a4w4_variants.py | 43 + .../test_compile_entry.py | 88 + .../mxfp4_h6144_i1536_e128_top4/test_entry.py | 223 ++ .../test_gather_ranks2.py | 117 + .../test_gemm12_a4w4.py | 120 + .../test_gemm2_adaptive.py | 61 + .../test_gemm2_bm256_epilogue_groups.py | 111 + .../test_gemm2_epilogue_groups.py | 75 + .../test_gemm2_row_probes.py | 33 + .../test_gemm_half_rows.py | 141 + .../test_id_rows_experts.py | 150 ++ .../test_prepared_input.py | 139 + .../mxfp4_h6144_i1536_e128_top4/test_quant.py | 57 + .../test_router_dual_quant_line_layout.py | 107 + .../test_router_dual_quant_tasks.py | 65 + .../test_router_gemm_dual_quant_tasks.py | 76 + .../test_router_gemm_nblocks.py | 126 + .../test_router_gemm_nblocks_lines.py | 53 + .../test_router_quant_line_layout.py | 53 + .../test_router_rounds.py | 87 + .../test_router_token_partials.py | 63 + .../test_router_topk_sorted_tasks.py | 99 + .../test_router_topk_tpw.py | 93 + .../test_router_topk_vec2.py | 105 + .../test_scale_sort_tile_map.py | 88 + .../test_shared_epilogue_early.py | 115 + .../test_shared_expert.py | 343 +++ .../test_shared_full.py | 136 + .../test_shared_persistent.py | 98 + .../test_shared_side_stream.py | 133 + .../test_sidecar.py | 165 ++ .../test_sort_a16w4.py | 87 + .../test_sort_count_all.py | 84 + .../test_sort_scales_all.py | 121 + .../test_sort_tile_map.py | 101 + .../test_sp_prefill_warm_up.py | 79 + .../test_unsupported_reason.py | 37 + .../test_warm_up_full_shared.py | 100 + .../test_warm_up_modes.py | 83 + .../mxfp4_h6144_i1536_e128_top4/workload.py | 170 ++ tests/multi_gpu.py | 61 +- .../test_allreduce_ranks2.py | 191 ++ .../test_compile_ranks2.py | 44 + .../test_ranks2_compile.py | 75 + .../test_ranks2_decoder_tail.py | 185 ++ .../test_ranks2_slots4.py | 204 ++ .../test_ranks2_staged.py | 354 +++ .../test_ranks2_unsupported_reason.py | 12 + .../__init__.py | 2 + .../test_batch_sharded.py | 301 +++ .../test_top_p_mask_rows.py | 162 ++ tools/ci/targets.tsv | 28 + 461 files changed, 59371 insertions(+), 1110 deletions(-) create mode 100644 amd_kernels/activation/swiglu_oai_mxfp8_i6144/README.md create mode 100644 amd_kernels/activation/swiglu_oai_mxfp8_i6144/__init__.py create mode 100644 amd_kernels/activation/swiglu_oai_mxfp8_i6144/host.py create mode 100644 amd_kernels/activation/swiglu_oai_mxfp8_i6144/kernel.py create mode 100644 amd_kernels/activation/swiglu_oai_mxfp8_i6144/reference.py create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/README.md create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/host.py create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/kernel.py create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/packing.py create mode 100644 amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/reference.py create mode 100644 amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_q32_kv2_iq2_vsplit.py create mode 100644 amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/bf16mfma_gqa16_d128_p128_ring.py create mode 100644 amd_kernels/collectives/all_gather_i64_c7_ranks2/README.md create mode 100644 amd_kernels/collectives/all_gather_i64_c7_ranks2/__init__.py create mode 100644 amd_kernels/collectives/all_gather_i64_c7_ranks2/host.py create mode 100644 amd_kernels/collectives/all_gather_i64_c7_ranks2/kernel.py create mode 100644 amd_kernels/collectives/all_gather_i64_c7_ranks2/reference.py create mode 100644 amd_kernels/collectives/all_gather_sampler_i64_c8_ranks2/README.md create mode 100644 amd_kernels/collectives/all_gather_sampler_i64_c8_ranks2/__init__.py create mode 100644 amd_kernels/collectives/all_gather_sampler_i64_c8_ranks2/host.py create mode 100644 amd_kernels/collectives/all_gather_sampler_i64_c8_ranks2/kernel.py create mode 100644 amd_kernels/collectives/all_gather_sampler_i64_c8_ranks2/reference.py create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/README.md create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/__init__.py create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/host.py create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/kernel.py create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/reference.py create mode 100644 amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/stage_publish.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/README.md create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/__init__.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/all_gather_fence_h6144.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/gather_host.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/host.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/kernel.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/lossless_x3.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/norm_gather_h6144.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/pull_unpack_q4096_shuffled.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/reference.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/slots4_host.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_pull.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_pull_sum2_add_rmsnorm_mxfp8.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows_x3.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/stage_pull.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/sum2_add_rmsnorm_mxfp8.py create mode 100644 amd_kernels/collectives/pull_collectives_ranks2/sum2_x3_add_rmsnorm_mxfp8.py create mode 100644 amd_kernels/collectives/sdma_push_ranks2/README.md create mode 100644 amd_kernels/collectives/sdma_push_ranks2/__init__.py create mode 100644 amd_kernels/collectives/sdma_push_ranks2/hsa_runtime.py create mode 100644 amd_kernels/collectives/sdma_push_ranks2/ipc.py create mode 100644 amd_kernels/collectives/sdma_push_ranks2/transport.py create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/README.md create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs.py create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs_sdma.py create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/mxfp8_pack_q4096.py create mode 100644 amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/reference.py create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/README.md create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/host.py create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/mlp.py create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/norm_linear.py create mode 100644 amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/reference.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/README.md create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/__init__.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/comm.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/flags.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/host.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel_lds.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel_staged_a_eightwave.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/reduce_gather.py create mode 100644 amd_kernels/gemm/bf16_concat_linear_n3072_k30720/reference.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/README.md create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/__init__.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/host.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/indexed.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/kernel.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/large.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/large_indexed.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/publish.py create mode 100644 amd_kernels/gemm/bf16_n100032_k6144/reference.py create mode 100644 amd_kernels/gemm/bf16_n200064_k256/kernel_m128.py create mode 100644 amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode_splitk.py create mode 100644 amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/README.md create mode 100644 amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py create mode 100644 amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/gateup_swiglu_bf16.py create mode 100644 amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/gateup_swiglu_bf16_onewave.py create mode 100644 amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/host.py create mode 100644 amd_kernels/gemm/mxfp8_n4992_k6144/README.md create mode 100644 amd_kernels/gemm/mxfp8_n4992_k6144/__init__.py create mode 100644 amd_kernels/gemm/mxfp8_n4992_k6144/host.py create mode 100644 amd_kernels/gemm/mxfp8_n4992_k6144/persistent.py create mode 100644 amd_kernels/gemm/mxfp8_n6144_k4096/README.md create mode 100644 amd_kernels/gemm/mxfp8_n6144_k4096/__init__.py create mode 100644 amd_kernels/gemm/mxfp8_n6144_k4096/host.py create mode 100644 amd_kernels/gemm/mxfp8_n6144_k4096/prefill_256x192_one_tile.py create mode 100644 amd_kernels/gemm/mxfp8_n6144_k4096/tile_128x192.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/columns.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_firstpage_d128_p128_h2.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_page_d128_p128_h2.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_runs_d128_p128_h2.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/__init__.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/decode_score.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/host.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/kernel.py create mode 100644 amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/select_merge.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/README.md create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/__init__.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/a16w4_utils.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/chain.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/__init__.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/epilogue.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/launch.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/loaders.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/mfma_order.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/reduce_slots_shared.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/scale_sort_tile_map.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_count_all.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_scales_all.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_tile_map.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/tile_map.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/utils.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_bm128.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_small.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4_shared_a8w8.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_bm128_half_rows.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_bm256_half_rows.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_id_rows.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_inline_sort.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_small.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_prefill_h6144_i1536.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_adaptive.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_epilogue_groups.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_epilogue_groups.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_half_rows.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_id_rows.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_inline_sort.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_small.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/__init__.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/gemm1_prefill_h6144_i3072_persistent.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/layout.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp4_from_mxfp8.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/__init__.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/gemm1_prefill_h6144_i3072.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/shared_dense_4wave.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prefill_h6144_i1536.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prepared_input.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/quant.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/reference.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_direct.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks_lines.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_topk_mxfp4_mxfp8_quant.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_mxfp8_quant_line_layout.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_mxfp8_quant_row_scales.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant_line_layout.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_mxfp4_mxfp8_quant.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_mxfp4_mxfp8_quant_sorted_tasks.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_sorted.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_vec2.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/routing_ids.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_dense.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_expert.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_epilogue_early.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_single_tile.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_write_through.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm2_a8w8.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_quant.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_pack.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_unpack.py create mode 100644 amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sort_a16w4.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/README.md create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/__init__.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/common.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/host.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/kernel.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/norm_math.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/quant_math.py create mode 100644 amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/reference.py create mode 100644 amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/__init__.py create mode 100644 amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/common.py create mode 100644 amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/host.py create mode 100644 amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/lossless_x3.py create mode 100644 amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/oneshot.py create mode 100644 amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/README.md create mode 100644 amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py create mode 100644 amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/host.py create mode 100644 amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/reference.py create mode 100644 amd_kernels/sampling/top_p_mask_f32_v200064/kernel_rows.py create mode 100644 amd_kernels/sampling/top_p_mask_f32_v200064/sigma_table.py create mode 100644 benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py create mode 100644 benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py create mode 100644 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py create mode 100644 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py create mode 100644 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py create mode 100644 benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/sweep_tiles.py create mode 100644 benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py create mode 100644 benchmarks/collectives/all_gather_i64_c7_ranks2/benchmark_gather.py create mode 100644 benchmarks/collectives/all_gather_sampler_i64_c8_ranks2/benchmark_gather.py create mode 100644 benchmarks/collectives/all_to_all_bf16_n100032_ranks2/benchmark_exchange.py create mode 100644 benchmarks/collectives/all_to_all_bf16_n100032_ranks2/sweep_pull.py create mode 100644 benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py create mode 100644 benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py create mode 100644 benchmarks/collectives/sdma_push_ranks2/bench_engines.py create mode 100644 benchmarks/collectives/sdma_push_ranks2/benchmark_push.py create mode 100644 benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py create mode 100644 benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py create mode 100644 benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py create mode 100644 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py create mode 100644 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py create mode 100644 benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py create mode 100644 benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py create mode 100644 benchmarks/gemm/mxfp8_common/benchmark_table.py create mode 100644 benchmarks/gemm/mxfp8_common/sweep_rows.py create mode 100644 benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py create mode 100644 benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py create mode 100644 benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py create mode 100644 benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py create mode 100644 benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py create mode 100644 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py create mode 100644 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_decode_h2.py create mode 100644 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_prefill_plan.py create mode 100644 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_baseline_moe.py create mode 100644 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_prepared.py create mode 100644 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_full.py create mode 100644 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_moe.py create mode 100644 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_sidecar.py create mode 100644 benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py create mode 100644 benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py create mode 100644 benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py create mode 100644 tests/activation/swiglu_oai_mxfp8_i6144/__init__.py create mode 100644 tests/activation/swiglu_oai_mxfp8_i6144/test_swiglu_oai_mxfp8.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_compile.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_counters.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_isa.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_packing.py create mode 100644 tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_qkv_insert.py create mode 100644 tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_qkv_chain_tp2.py create mode 100644 tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks2.py create mode 100644 tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_tp2_tiles.py create mode 100644 tests/collectives/all_gather_i64_c7_ranks2/__init__.py create mode 100644 tests/collectives/all_gather_i64_c7_ranks2/test_i64_c7_gather.py create mode 100644 tests/collectives/all_gather_sampler_i64_c8_ranks2/__init__.py create mode 100644 tests/collectives/all_gather_sampler_i64_c8_ranks2/test_sampler_gather.py create mode 100644 tests/collectives/all_to_all_bf16_n100032_ranks2/__init__.py create mode 100644 tests/collectives/all_to_all_bf16_n100032_ranks2/test_exchange.py create mode 100644 tests/collectives/pull_collectives_ranks2/__init__.py create mode 100644 tests/collectives/pull_collectives_ranks2/served.py create mode 100644 tests/collectives/pull_collectives_ranks2/test_all_gather.py create mode 100644 tests/collectives/pull_collectives_ranks2/test_gather_odd_rows.py create mode 100644 tests/collectives/pull_collectives_ranks2/test_norm_gather_h6144.py create mode 100644 tests/collectives/pull_collectives_ranks2/test_slots4_reduce_scatter.py create mode 100644 tests/collectives/pull_collectives_ranks2/test_unpack_q4096_shuffled.py create mode 100644 tests/collectives/sdma_push_ranks2/__init__.py create mode 100644 tests/collectives/sdma_push_ranks2/test_transport.py create mode 100644 tests/collectives/sdma_push_ranks4/test_agents.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_coded_tail.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_dbo.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_handoffs.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_pack.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_shard.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_warm_up.py create mode 100644 tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/workload.py create mode 100644 tests/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py create mode 100644 tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_decoder_ranks2.py create mode 100644 tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_mlp_norm_linear.py create mode 100644 tests/gemm/bf16_concat_linear_n3072_k30720/__init__.py create mode 100644 tests/gemm/bf16_concat_linear_n3072_k30720/test_compile.py create mode 100644 tests/gemm/bf16_concat_linear_n3072_k30720/test_concat_linear.py create mode 100644 tests/gemm/bf16_concat_linear_n3072_k30720/test_gather.py create mode 100644 tests/gemm/bf16_n100032_k6144/__init__.py create mode 100644 tests/gemm/bf16_n100032_k6144/test_gemm.py create mode 100644 tests/gemm/bf16_n100032_k6144/test_publish.py create mode 100644 tests/gemm/mxfp8_common/test_ctx_kv_rows_n2560.py create mode 100644 tests/gemm/mxfp8_common/test_decode_splitk.py delete mode 100644 tests/gemm/mxfp8_common/test_gateup_decode_rows.py create mode 100644 tests/gemm/mxfp8_common/test_oproj_out_k4096.py create mode 100644 tests/gemm/mxfp8_common/test_shape_rows.py create mode 100644 tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py create mode 100644 tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/test_gateup_swiglu_bf16.py create mode 100644 tests/gemm/mxfp8_n4992_k6144/__init__.py create mode 100644 tests/gemm/mxfp8_n4992_k6144/test_isa.py create mode 100644 tests/gemm/mxfp8_n4992_k6144/test_n4992_k6144.py create mode 100644 tests/gemm/mxfp8_n4992_k6144/test_warmup.py create mode 100644 tests/gemm/mxfp8_n6144_k4096/__init__.py create mode 100644 tests/gemm/mxfp8_n6144_k4096/test_kernels_k4096.py create mode 100644 tests/gemm/mxfp8_n6144_k4096/test_oproj_k4096.py create mode 100644 tests/gemm/mxfp8_n6144_k4096/test_out_k4096.py create mode 100644 tests/indexer/indexer_nvfp4_d128_p128_top16/test_score_top16_h2.py create mode 100644 tests/indexer/indexer_nvfp4_d128_p128_top16/test_split2_ranks2.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/__init__.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_small.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_variants.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_compile_entry.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_entry.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gather_ranks2.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm12_a4w4.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_adaptive.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_bm256_epilogue_groups.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_epilogue_groups.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_row_probes.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm_half_rows.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_id_rows_experts.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_prepared_input.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_quant.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_line_layout.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_tasks.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_dual_quant_tasks.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks_lines.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_quant_line_layout.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_rounds.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_token_partials.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_sorted_tasks.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_tpw.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_vec2.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_scale_sort_tile_map.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_epilogue_early.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_expert.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_full.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_persistent.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_side_stream.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sidecar.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_a16w4.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_count_all.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_scales_all.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_tile_map.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_sp_prefill_warm_up.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_unsupported_reason.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_full_shared.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_modes.py create mode 100644 tests/moe/mxfp4_h6144_i1536_e128_top4/workload.py create mode 100644 tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_allreduce_ranks2.py create mode 100644 tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_compile_ranks2.py create mode 100644 tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_compile.py create mode 100644 tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_decoder_tail.py create mode 100644 tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_slots4.py create mode 100644 tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_staged.py create mode 100644 tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_unsupported_reason.py create mode 100644 tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py create mode 100644 tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/test_batch_sharded.py create mode 100644 tests/sampling/top_p_mask_f32_v200064/test_top_p_mask_rows.py diff --git a/amd_kernels/README.md b/amd_kernels/README.md index bbc9b9a..bb4572d 100644 --- a/amd_kernels/README.md +++ b/amd_kernels/README.md @@ -5,60 +5,82 @@ One line per kernel package. Each package's README has what it does, its entry p ## GEMM - [`gemm/mxfp8_common`](gemm/mxfp8_common/README.md): MXFP8 dense GEMM entries (`mxfp8_linear`, `mxfp8_gemm`, ...), - the shape -> kernel selection and the shared kernels. -- [`gemm/mxfp8_n2560_k6144`](gemm/mxfp8_n2560_k6144/README.md): the QKV projection. -- [`gemm/mxfp8_n6144_k2048`](gemm/mxfp8_n6144_k2048/README.md): the attention output projection. + the shape -> kernel selection (TP4 and TP2 shards) and the shared kernels. +- [`gemm/mxfp8_n2560_k6144`](gemm/mxfp8_n2560_k6144/README.md): the QKV projection at TP4. +- [`gemm/mxfp8_n4992_k6144`](gemm/mxfp8_n4992_k6144/README.md): the QKV projection at TP2. +- [`gemm/mxfp8_n6144_k2048`](gemm/mxfp8_n6144_k2048/README.md): the attention output projection at TP4. +- [`gemm/mxfp8_n6144_k4096`](gemm/mxfp8_n6144_k4096/README.md): the attention output projection at TP2 (decode and + every prefill chunk). - [`gemm/mxfp8_n6144_k8192`](gemm/mxfp8_n6144_k8192/README.md): the full output projection of the sequence-parallel prefill. - [`gemm/mxfp8_gateup_swiglu_n6144_k6144_i3072`](gemm/mxfp8_gateup_swiglu_n6144_k6144_i3072/README.md): dense MLP - gate_up + SwiGLU at 1-8 rows. -- [`gemm/bf16_n50016_k6144`](gemm/bf16_n50016_k6144/README.md): the lm_head vocabulary shard, with an optional row map. -- [`gemm/bf16_n200064_k256`](gemm/bf16_n200064_k256/README.md): the draft's Markov projection over selected rows. + gate_up + SwiGLU at 1-8 rows (TP4). +- [`gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144`](gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/README.md): dense MLP + gate_up + SwiGLU at 1-16 rows (TP2). +- [`gemm/bf16_n50016_k6144`](gemm/bf16_n50016_k6144/README.md): the TP4 lm_head vocabulary shard, with an optional row + map. +- [`gemm/bf16_n100032_k6144`](gemm/bf16_n100032_k6144/README.md): the TP2 lm_head vocabulary shard, with an optional + row map, a strided output and the exchange publish. +- [`gemm/bf16_n200064_k256`](gemm/bf16_n200064_k256/README.md): the draft's Markov projection over 0-128 selected rows. - [`gemm/bf16_concat_linear_n1536_k30720`](gemm/bf16_concat_linear_n1536_k30720/README.md): the draft's input fc over - five taps, and the four-rank gather. + five taps at TP4, and the four-rank gather. +- [`gemm/bf16_concat_linear_n3072_k30720`](gemm/bf16_concat_linear_n3072_k30720/README.md): the draft's input fc over + five taps at TP2, and the two-rank exchange. ## MoE -- [`moe/mxfp4_h6144_i768_e128_top4`](moe/mxfp4_h6144_i768_e128_top4/README.md): routed MXFP4 experts (A4W4) with the - MXFP8 shared expert. +- [`moe/mxfp4_h6144_i768_e128_top4`](moe/mxfp4_h6144_i768_e128_top4/README.md): the TP4 MoE block (I 768 per rank): + routed MXFP4 experts (A4W4) with the MXFP8 shared expert. +- [`moe/mxfp4_h6144_i1536_e128_top4`](moe/mxfp4_h6144_i1536_e128_top4/README.md): the TP2 MoE block (I 1536 per rank): + routed MXFP4 experts (A4W4) with the MXFP8 shared expert (a TP shard, or the full copy: its shard read in place, + the whole expert on a sequence-parallel rank's own rows). ## Attention - [`attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16`](attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/README.md): - sparse prefill attention over the NVFP4 KV cache (bf16, fp16 or fp8 compute). + sparse prefill attention over the NVFP4 KV cache, 1 or 2 KV heads per GPU (TP4 / TP2; bf16, fp16 or fp8 compute). - [`attention/nvfp4_sparse_decode_gqa16_d128_p128_top16`](attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/README.md): - sparse decode attention over the NVFP4 KV cache (bf16, fp16 or fp8 compute). + sparse decode attention over the NVFP4 KV cache, 1 or 2 KV heads per GPU (TP4 / TP2; bf16, fp16 or fp8 compute). - [`attention/norm_rope_nvfp4_insert_d128_rot64_p128`](attention/norm_rope_nvfp4_insert_d128_rot64_p128/README.md): - q / k norm + partial RoPE + NVFP4 K / V / index-K inserts in one launch. + q / k norm + partial RoPE + NVFP4 K / V / index-K inserts in one launch (q16 kv1 iq1 at TP4, q32 kv2 iq2 at TP2). - [`attention/mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128`](attention/mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128/README.md): - the QKV GEMM, its reduce and the norm / RoPE / inserts, fused for small batches. + the TP4 QKV GEMM, its reduce and the norm / RoPE / inserts, fused for small batches. +- [`attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128`](attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/README.md): + the TP2 QKV GEMM, its reduce and the norm / RoPE / inserts: one launch at 1-16 rows, the deferred chain to 768. - [`attention/nvfp4_sparse_layer_gqa16_d128_p128_top16`](attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/README.md): - one NVFP4 sparse attention layer at its served dtypes: the indexer, the warm-up, the decode / prefill calls. + one NVFP4 sparse attention layer at its served dtypes, TP4 and TP2: the indexer, the warm-up, the decode / prefill + calls. - [`attention/swa_decode_fp8kv_gqa16_d128`](attention/swa_decode_fp8kv_gqa16_d128/README.md): the draft's - sliding-window decode attention over an fp8 KV cache. + sliding-window decode attention over an fp8 KV cache, 1 or 2 KV heads per GPU (TP4 / TP2). - [`attention/norm_rope_fp8_insert_d128_rot64`](attention/norm_rope_fp8_insert_d128_rot64/README.md): the draft's - norm + RoPE + fp8 KV insert. + norm + RoPE + fp8 KV insert, 1 or 2 KV heads per GPU (TP4 / TP2). ## Indexer - [`indexer/indexer_nvfp4_d128_p128_top16`](indexer/indexer_nvfp4_d128_p128_top16/README.md): the sparse-attention - indexer: block scores over NVFP4 index pages and the top-16 blocks, decode and prefill, one or four GPUs. + indexer: block scores over NVFP4 index pages and the top-16 blocks, decode and prefill, one GPU at 1 / 2 index heads + (TP4 / TP2), or two or four GPUs at decode. ## Embedding and activation -- [`embedding/vocab_gather_h6144`](embedding/vocab_gather_h6144/README.md): vocabulary lookup over four shards without - an all-reduce. +- [`embedding/vocab_gather_h6144`](embedding/vocab_gather_h6144/README.md): vocabulary lookup over two or four shards + without an all-reduce. - [`embedding/vocab_gather_gemma_mxfp8_h6144`](embedding/vocab_gather_gemma_mxfp8_h6144/README.md): the same lookup + Gemma norm + MXFP8 quant in one launch. - [`activation/swiglu_oai_mxfp8_i3072`](activation/swiglu_oai_mxfp8_i3072/README.md): SwiGLU + MXFP8 quant for the - down GEMM. + TP4 down GEMM (I 3072). +- [`activation/swiglu_oai_mxfp8_i6144`](activation/swiglu_oai_mxfp8_i6144/README.md): SwiGLU + MXFP8 quant for the + TP2 down GEMM (I 6144). ## Norms - [`norm/gemma_add_rmsnorm_h6144`](norm/gemma_add_rmsnorm_h6144/README.md): add + Gemma RMSNorm to bf16 or MXFP8, with - the four-rank all-reduce fused in front; a decoder layer's tail (norm, FFN, the next norm). + the four-rank (`allreduce_add_rmsnorm`) or two-rank (`allreduce_add_rmsnorm_ranks2`) all-reduce fused in front; a + decoder layer's tail (norm, FFN, the next norm). - [`norm/allreduce_gemma_bf16_mxfp8_h6144`](norm/allreduce_gemma_bf16_mxfp8_h6144/README.md): four-rank all-reduce + Gemma norm with bf16-rounded MXFP8. +- [`norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2`](norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/README.md): two-rank + all-reduce + Gemma norm with bf16-rounded MXFP8. - [`norm/gemma_rmsnorm_bf16_mxfp8_h6144`](norm/gemma_rmsnorm_bf16_mxfp8_h6144/README.md): Gemma RMSNorm to bf16-rounded MXFP8. @@ -77,21 +99,42 @@ One line per kernel package. Each package's README has what it does, its entry p - [`collectives/all_gather_i64_c7_ranks4`](collectives/all_gather_i64_c7_ranks4/README.md): draft token ids into batch order. +## Collectives (two ranks) + +- [`collectives/pull_collectives_ranks2`](collectives/pull_collectives_ranks2/README.md): the unpacking all-to-all, + all-gather, fixed-order reduce-scatter and SP norm tails over the one xGMI link of a GPU pair. +- [`collectives/seq_parallel_handoffs_h6144_q4096_ranks2`](collectives/seq_parallel_handoffs_h6144_q4096_ranks2/README.md): + the sequence-parallel prefill's layer hand-offs (pull collectives, an SDMA form for DBO; the shared expert local to + its rows' owner on a side stream). +- [`collectives/sdma_push_ranks2`](collectives/sdma_push_ranks2/README.md): pushes on one SDMA engine, GPU-ordered, + without CUs. +- [`collectives/all_to_all_bf16_n100032_ranks2`](collectives/all_to_all_bf16_n100032_ranks2/README.md): logits rows + into the final vocabulary layout. +- [`collectives/all_gather_sampler_i64_c8_ranks2`](collectives/all_gather_sampler_i64_c8_ranks2/README.md): the + sampler output gather. +- [`collectives/all_gather_i64_c7_ranks2`](collectives/all_gather_i64_c7_ranks2/README.md): draft token ids into batch + order. + ## Sampling - [`sampling/batch_sharded_markov_v200064_h6144_r256_ranks4`](sampling/batch_sharded_markov_v200064_h6144_r256_ranks4/README.md): batch-sharded sampling on four ranks: logits exchange, output gather and the Markov draft chain. +- [`sampling/batch_sharded_markov_v200064_h6144_r256_ranks2`](sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/README.md): + batch-sharded sampling on two ranks: owned-row logits (exchange, or locally on a copy of the peer shard), output + gather and the Markov draft chain. - [`sampling/gumbel_partials_bias_bf16_v200064`](sampling/gumbel_partials_bias_bf16_v200064/README.md): Gumbel noise and block partials of one Markov draft step. - [`sampling/gumbel_combine_markov_embed_v200064_r256`](sampling/gumbel_combine_markov_embed_v200064_r256/README.md): the end of one Markov draft step: combine, draft id, next row. -- [`sampling/top_p_mask_f32_v200064`](sampling/top_p_mask_f32_v200064/README.md): top-p mask of <= 64 fp32 logits - rows in one launch (vLLM's split top-p chain, bit for bit). +- [`sampling/top_p_mask_f32_v200064`](sampling/top_p_mask_f32_v200064/README.md): vLLM's top-p without top-k on fp32 + logits rows, bit for bit: the split chain (<= 64 rows) in one launch, the one-program-per-row kernel above. ## Decoder blocks - [`decoder/dense_mxfp8_h6144_i3072_ranks4`](decoder/dense_mxfp8_h6144_i3072_ranks4/README.md): the draft's dense decoder layers around their attention, four ranks. +- [`decoder/dense_mxfp8_h6144_i6144_ranks2`](decoder/dense_mxfp8_h6144_i6144_ranks2/README.md): the draft's dense + decoder layers around their attention, two ranks (and the TP2 target's dense MLP). ## Shared code diff --git a/amd_kernels/activation/swiglu_oai_mxfp8_i6144/README.md b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/README.md new file mode 100644 index 0000000..f5aa805 --- /dev/null +++ b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/README.md @@ -0,0 +1,47 @@ +# SwiGLU-OAI + MXFP8 quant, I 6144 (gfx950) + +## What it does + +One launch for the SwiGLU-OAI activation of a split-layout gate_up row (`[m, 2 x 6144]` bf16, gate | up, unit column +stride, any row stride) and the MXFP8 quant of the result in the layout the down GEMM (N 6144, K 6144) reads: e4m3 +codes row-major `[m, 6144]` or in the decode GEMM's (16, 16) tiles with rows padded to the tile, E8M0 scales +`[ceil(rows / 32) * 32, 192]` in AITER's shuffled layout (padding rows and scale padding unwritten; the GEMM masks +them). Caller: the M3.1 dense FFN at TP2 (I = 6144 per rank): the DSpark draft's 5 layers and the target's dense +layers 0-2, above 16 rows (1-16 rows run `gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144`). Numerics (`reference.py`): +`form` is the caller's, never chosen by row count. `bf16_chain` = the draft's aten chain (`clamp`, `sigmoid`, `mul`, +`add`, each rounded to bf16); `fp32` = the target's `swiglu_oai_split` (fp32 math, one bf16 rounding); both use +`1 / (1 + exp(-x))` with the accurate exp and an IEEE divide. Then the Triton MXFP8 quantizer on the bf16 value: scale +`clamp(ceil(log2(amax / 448)) + 127, 0, 254)` per 32 columns, e4m3 codes of `act * 2^(127 - scale)`. Codes and scales +equal the reference bit for bit on finite inputs, for every workgroup shape. + +## Entry points + +- `swiglu_oai_mxfp8`: the down GEMM's codes and scales of a gate_up row block, row-major or in (16, 16) tiles; the + caller takes the tile from `gemm.mxfp8_common.gemm.prequant_code_tile_rows(m, 6144, 6144)` and runs the down GEMM + with `gemm.mxfp8_common.gemm.mxfp8_gemm_prequant`. The output buffers may be the caller's. +- `warmup_swiglu_oai_mxfp8`: compiles all 8 variants (2 workgroup shapes x 2 forms x 2 code layouts; the row count + is a runtime argument). Call it before serving or graph capture. + +`host.SHAPES` picks the workgroup shape (pieces per row, threads) by row count. + +## Performance + +One MI355X, graph replay, 100 distinct inputs per graph, us per call, against what vLLM runs at TP2 without it: the +aten chain + quantizer (`bf16_chain`) or Triton `swiglu_oai_split` + quantizer (`fp32`). HBM floor at 6.5 TB/s. + +| Rows | Code tile (0: row-major) | bf16_chain baseline | bf16_chain | fp32 baseline | fp32 | HBM floor | fraction (fp32) | +|---|---|---|---|---|---|---|---| +| 7 | 16 | 17.79 | 2.49 | 3.76 | 2.38 | 0.03 | 1% | +| 64 | 64 | 23.51 | 2.78 | 4.31 | 2.70 | 0.30 | 11% | +| 128 | 64 | 27.90 | 3.66 | 5.04 | 3.51 | 0.61 | 17% | +| 256 | 128 | 33.25 | 4.60 | 7.03 | 4.33 | 1.22 | 28% | +| 512 | 128 | 40.20 | 6.74 | 10.40 | 6.20 | 2.43 | 39% | +| 1024 | 0 | 65.17 | 10.64 | 15.40 | 9.71 | 4.87 | 50% | +| 1792 | 0 | 99.54 | 16.90 | 24.12 | 15.06 | 8.52 | 57% | + +## Tests + +```bash +python3 -m pytest tests/activation/swiglu_oai_mxfp8_i6144 -q +python3 benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py [--form fp32] [--sweep] +``` diff --git a/amd_kernels/activation/swiglu_oai_mxfp8_i6144/__init__.py b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/__init__.py new file mode 100644 index 0000000..29d2f2f --- /dev/null +++ b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/__init__.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SwiGLU-OAI of a split-layout gate_up + MXFP8 quant of the result, I = 6144 output columns, FlyDSL +(gfx950). The torch contract is ``reference.py``.""" + + +def __getattr__(name): + if name in __all__: + from . import host + + return getattr(host, name) + raise AttributeError(name) + + +__all__ = ["swiglu_oai_mxfp8", "warmup_swiglu_oai_mxfp8"] diff --git a/amd_kernels/activation/swiglu_oai_mxfp8_i6144/host.py b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/host.py new file mode 100644 index 0000000..2337490 --- /dev/null +++ b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/host.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Compile cache + launch wrapper of the SwiGLU-OAI + MXFP8 quant kernel.""" + +import functools +import struct + +import torch + +from . import kernel as _kernel + +I = _kernel.I + +# (max rows, pieces, threads): 12 one-wave pieces per row up to 704 rows, then one 12-wave workgroup per row +# (benchmark_swiglu_oai_mxfp8.py --sweep, both forms) +SHAPES = ((704, 12, 64), (1 << 31, 1, 768)) + + +def _run_compiled(exe, *args): + """First call compiles and runs (``flyc.compile``); later calls dispatch the cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +@functools.cache +def _get_kernel(form: str, shuffled: bool, pieces: int, threads: int): + return _kernel.compile_swiglu_oai_mxfp8(form, shuffled, pieces, threads) + + +def _shape(m: int) -> tuple[int, int]: + return next((p, t) for top, p, t in SHAPES if m <= top) + + +def _f32_bits(v: float) -> int: + return struct.unpack(" tuple[torch.Tensor, torch.Tensor]: + """(codes, scales) of ``quantize(swiglu(gate_up))`` for the down GEMM: codes row-major [m, I] + (``code_tile_rows`` 0) or in the decode GEMM's (16, 16) layout, rows padded to the tile (padding rows + unwritten); e8m0 scales [ceil(rows / 32) * 32, I / 32] in AITER's shuffled layout (padding unwritten). + ``form``: the SwiGLU numerics, the caller's (``bf16_chain`` = the draft's aten chain, ``fp32`` = the + target's ``swiglu_oai_split``). ``shape`` (pieces, threads) overrides the row-count choice (sweeps only).""" + m = gate_up.shape[0] + assert gate_up.dtype == torch.bfloat16 and gate_up.shape[1] == 2 * I and gate_up.stride(1) == 1 + shuffled = code_tile_rows > 0 + if shuffled: + assert code_tile_rows % 16 == 0 + rows = (m + code_tile_rows - 1) // code_tile_rows * code_tile_rows + else: + rows = m + q = q_out if q_out is not None else torch.empty((rows, I), dtype=torch.float8_e4m3fn, device=gate_up.device) + s = ( + scale_out + if scale_out is not None + else torch.empty(((rows + 31) // 32 * 32, I // 32), dtype=torch.uint8, device=gate_up.device) + ) + assert q.shape[0] >= rows and q.shape[1] == I and q.is_contiguous() + assert s.shape[0] >= (rows + 31) // 32 * 32 and s.shape[1] == I // 32 and s.is_contiguous() + if m: + _run_compiled( + _get_kernel(form, shuffled, *(shape or _shape(m))), + gate_up.data_ptr(), + q.data_ptr(), + s.data_ptr(), + q.numel() * q.element_size(), + s.numel(), + m, + gate_up.stride(0), + _f32_bits(alpha), + _f32_bits(beta), + _f32_bits(limit), + torch.cuda.current_stream(), + ) + return q, s + + +def warmup_swiglu_oai_mxfp8(device: torch.device | None = None, forms=("bf16_chain", "fp32")) -> None: + """Compile every variant (each row-count shape x form x row-major / tiled codes; the row count is a runtime + argument) before serving or graph capture (README rule 4).""" + for top, _, _ in SHAPES: + m = min(top, 2048) + x = torch.zeros((m, 2 * I), dtype=torch.bfloat16, device=device or "cuda") + for form in forms: + for tile in (0, 16): + swiglu_oai_mxfp8(x, 1.0, 0.0, 1.0, tile, form) diff --git a/amd_kernels/activation/swiglu_oai_mxfp8_i6144/kernel.py b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/kernel.py new file mode 100644 index 0000000..0834999 --- /dev/null +++ b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/kernel.py @@ -0,0 +1,287 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SwiGLU-OAI of a split-layout gate_up row + MXFP8 (1x32, E8M0) quant of the result, FlyDSL (gfx950), +I = 6144 output columns per row (the M3.1 dense FFN at TP2: gate_up N 12288 = [gate 6144 | up 6144]). + +``form`` (compile time, the caller's numerics; never chosen by row count): + ``bf16_chain``: the aten chain of ``SiluAndMulWithClamp.forward_native`` (each op in fp32, rounded to + bf16): g = min(gate, limit); u = clamp(up, -limit, limit); t1 = bf(alpha * g); t2 = bf(1 / (1 + exp(-t1))); + t3 = bf(g * t2); t4 = bf(u + beta); act = bf(t3 * t4) (the DSpark draft). + ``fp32``: ``swiglu_oai_split``'s form: act = bf((g * (1 / (1 + exp(-alpha * g)))) * (u + beta)), fp32 + throughout, one rounding (the target's dense layers 0-2). +The quant then reads the bf16 ``act`` exactly as the Triton MXFP8 quantizer (``_mxfp8_quantize_kernel``) +does: amax over 32 columns, scale byte ``clamp(ceil(log2(amax / 448)) + 127, 0, 254)``, codes by +``v_cvt_scalef32_pk_fp8_f32`` of ``act * 2^(127 - byte)`` with MODE.fp16_ovfl. Codes row-major [m, I] or in the +decode GEMM's (16, 16) layout; scales in AITER's shuffled layout. + +Layout (compile time): a row splits into ``pieces`` workgroups of ``threads`` lanes (items spread over XCDs in +contiguous runs); a thread holds I / (8 * pieces * threads) chunks of 8 adjacent output columns, chunk c of +thread t at 8-column index piece * threads * chunks + c * threads + t, so a 32-column MX group is 4 adjacent lanes +(quad DPP amax). All chunk loads are issued before the math. About 49 VALU instructions per element (bf16_chain; +accurate exp, IEEE divide), so above ~1000 rows the kernel is VALU-bound, not HBM-bound.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import arith as _arith +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import const_expr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +I = 6144 # output columns per row +SCALE_K1 = I // 256 # AITER scale layout: 256-column blocks per row +TINY = 0x00800000 # fp32 tiny, the amax floor +HWREG_MODE_FP16_OVFL = 1 | (23 << 6) # hwreg(HW_REG_MODE, 23, 1) +FORMS = ("bf16_chain", "fp32") + +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} + + +def _f32(bits): + return fx.Int32(bits).bitcast(fx.Float32) + + +def _bits(x): + return fx.Float32(x).bitcast(fx.Int32) + + +def _maxnum(a, b): + return fx.Float32(_arith.MaxNumFOp(fx.as_ir_value(a), fx.as_ir_value(b)).result) + + +def _minnum(a, b): + return fx.Float32(_arith.MinNumFOp(fx.as_ir_value(a), fx.as_ir_value(b)).result) + + +def _max_dpp(a, ctrl): + b = fx.Int32( + _rocdl_ops.update_dpp( + T.i32, _llvm.mlir_undef(T.i32), fx.as_ir_value(_bits(a)), ctrl, 0xF, 0xF, True + ) + ).bitcast(fx.Float32) + return _maxnum(a, b) + + +def _swap_xor1(words): + return [ + fx.Int32( + _rocdl_ops.update_dpp( + T.i32, _llvm.mlir_undef(T.i32), fx.as_ir_value(w), 0xB1, 0xF, 0xF, True + ) + ) + for w in words + ] + + +def _unpack8(v): + """8 fp32 of 4 dwords of bf16 (low half first).""" + out = [] + for i in range(4): + w = fx.Int32(v[i]) + out += [(w << 16).bitcast(fx.Float32), (w & -65536).bitcast(fx.Float32)] + return out + + +def _pk(a, b): + """Two fp32 -> two bf16 in a dword (v_cvt_pk_bf16_f32, RNE), ``a`` low.""" + return fx.Int32(fx.rocdl.cvt_pk_bf16_f32(fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b)))) + + +def _bf(vals): + """Round 8 fp32 to bf16 (RNE) and back to fp32.""" + return _unpack8([_pk(vals[2 * i], vals[2 * i + 1]) for i in range(4)]) + + +def _set_fp16_ovfl(): + _llvm.call_intrinsic( + None, + "llvm.amdgcn.s.setreg", + [fx.Int32(HWREG_MODE_FP16_OVFL).ir_value(), fx.Int32(1).ir_value()], + [], + [], + ) + + +def _cvt_pk_fp8(old, a, b, hi: bool): + v2i16 = _ir.VectorType.get([2], _ir.IntegerType.get_signless(16)) + res = _llvm.call_intrinsic( + v2i16, + "llvm.amdgcn.cvt.scalef32.pk.fp8.f32", + [ + _llvm.bitcast(v2i16, fx.as_ir_value(old)), + fx.as_ir_value(a), + fx.as_ir_value(b), + fx.as_ir_value(fx.Float32(1.0)), + fx.Boolean(hi).ir_value(), + ], + [], + [], + ) + return fx.Int32(_llvm.bitcast(T.i32, res)) + + +def _e8m0(y): + """(scale byte as fp32, 2^(127 - byte)) of the 32-column group whose 4 lanes hold ``y``.""" + a = fx.math.absf(y[0]) + for v in y[1:]: + a = _maxnum(a, fx.math.absf(v)) + a = _max_dpp(a, 0x4E) # quad_perm [2, 3, 0, 1] + a = _maxnum(_max_dpp(a, 0xB1), _f32(TINY)) # quad_perm [1, 0, 3, 2] + d = a / fx.Float32(448.0) + small = d < _f32(TINY) + lg = fx.Float32( + _rocdl_ops.log(T.f32, fx.as_ir_value(small.select(d * fx.Float32(2.0**32), d))) + ) - small.select(fx.Float32(32.0), fx.Float32(0.0)) + sb = _minnum(_maxnum(fx.math.ceil(lg) + fx.Float32(127.0), fx.Float32(0.0)), fx.Float32(254.0)) + rescale = fx.Float32(_rocdl_ops.exp2(T.f32, fx.as_ir_value(fx.Float32(127.0) - sb))) + return sb, rescale + + +def _codes(y, rescale): + q = [v * rescale for v in y] + lo = _cvt_pk_fp8(_cvt_pk_fp8(fx.Int32(0), q[0], q[1], False), q[2], q[3], True) + hi = _cvt_pk_fp8(_cvt_pk_fp8(fx.Int32(0), q[4], q[5], False), q[6], q[7], True) + return [lo, hi] + + +def _scale_off(m, g): + """Byte of (row, group) in AITER's shuffled scale layout.""" + return ( + ((((m // 32) * SCALE_K1 + g // 8) * 4 + g % 4) * 16 + m % 16) * 2 + (g % 8) // 4 + ) * 2 + (m % 32) // 16 + + +def _code_off(m, p): + """Byte of (row, 16-column piece) in the (16, 16) code layout.""" + return (m // 16) * (16 * I) + (p // 4) * 1024 + (p % 4) * 256 + (m % 16) * 16 + + +def _xcd_row(bid, m): + x, k = bid % 8, bid // 8 + q8, r8 = m // 8, m % 8 + extra = x < r8 + row = x * q8 + extra.select(x, r8) + k + return row, k < q8 + extra.select(fx.Int32(1), fx.Int32(0)) + + +def _xcd_grid(m): + return (m + 7) // 8 * 8 + + +def _swiglu_bf16_chain(gate, up, alpha, beta, limit): + g = [_minnum(v, limit) for v in gate] + u = [_minnum(_maxnum(v, -limit), limit) for v in up] + t1 = _bf([alpha * v for v in g]) + t2 = _bf([fx.Float32(1.0) / (fx.Float32(1.0) + fx.math.exp(-v)) for v in t1]) + t3 = _bf([a * b for a, b in zip(g, t2)]) + t4 = _bf([v + beta for v in u]) + return _bf([a * b for a, b in zip(t3, t4)]) + + +def _swiglu_fp32(gate, up, alpha, beta, limit): + """swiglu_oai_split's form: fp32 math, one bf16 rounding of the result.""" + out = [] + for gv, uv in zip(gate, up): + g = _minnum(gv, limit) + u = _minnum(_maxnum(uv, -limit), limit) + sig = fx.Float32(1.0) / (fx.Float32(1.0) + fx.math.exp(-(alpha * g))) + out.append((g * sig) * (u + beta)) + return _bf(out) + + +def _chunk(form, shuffled, qr, sr, row, tx, c8, gw, uw, alpha, beta, limit): + """SwiGLU, quant and stores of one 8-column chunk (8-column index ``c8``).""" + gv, uv = _unpack8(gw), _unpack8(uw) + if form == "bf16_chain": + y = _swiglu_bf16_chain(gv, uv, alpha, beta, limit) + else: + y = _swiglu_fp32(gv, uv, alpha, beta, limit) + sb, rescale = _e8m0(y) + codes = _codes(y, rescale) + if shuffled: + recv = _swap_xor1(codes) + piece = Vec.from_elements([codes[0], codes[1], recv[0], recv[1]], fx.Int32) + buffer_ops.buffer_store(piece, qr, _code_off(row, c8 // 2) // 4, mask=(tx % 2) == 0) + else: + buffer_ops.buffer_store(Vec.from_elements(codes, fx.Int32), qr, row * (I // 4) + c8 * 2) + buffer_ops.buffer_store( + fx.Int32(sb.to(fx.Int32)).to(fx.Int8), sr, _scale_off(row, c8 // 4), mask=(tx % 4) == 0, offset_is_bytes=True + ) + + +def compile_swiglu_oai_mxfp8(form: str = "bf16_chain", shuffled: bool = False, pieces: int = 1, threads: int = 768): + assert form in FORMS, form + chunks = I // 8 // pieces // threads + assert chunks * 8 * pieces * threads == I and threads % 64 == 0 + name = f"swiglu_oai_p{pieces}t{threads}_{form}_mxfp8_{'tiled' if shuffled else 'rowmajor'}_i{I}" + + @flyc.kernel(name=name, known_block_size=[threads, 1, 1]) + def kernel( + i32_m: fx.Int32, + i32_sx: fx.Int32, # gate_up row stride (elements) + arg_x: fx.Int64, # [m, 2I] bf16 + arg_q: fx.Int64, # codes e4m3 + arg_s: fx.Int64, # scales u8, AITER layout + i64_q_bytes: fx.Int64, + i64_s_bytes: fx.Int64, + i32_alpha: fx.Int32, # fp32 bits + i32_beta: fx.Int32, + i32_limit: fx.Int32, + ): + _set_fp16_ovfl() + tx = fx.thread_idx.x + if const_expr(pieces == 1): + row, live = _xcd_row(fx.block_idx.x, i32_m) + base = tx + else: + item, live = _xcd_row(fx.block_idx.x, i32_m * pieces) + row = item // pieces + base = tx + (item % pieces) * (threads * chunks) + if live: + xr = buffer_ops.create_buffer_resource_from_addr( + arg_x + fx.Int64(row) * fx.Int64(i32_sx * 2), num_records_bytes=4 * I + ) + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=i64_q_bytes) + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=i64_s_bytes) + cols = [base + c * threads for c in range(chunks)] + raw = [] + for c8 in cols: + raw.append(( + Vec(buffer_ops.buffer_load(xr, c8 * 4, vec_width=4, dtype=fx.Int32)), + Vec(buffer_ops.buffer_load(xr, I // 2 + c8 * 4, vec_width=4, dtype=fx.Int32)), + )) + alpha = i32_alpha.bitcast(fx.Float32) + beta = i32_beta.bitcast(fx.Float32) + limit = i32_limit.bitcast(fx.Float32) + for c8, gw, uw in [(c, g, u) for c, (g, u) in zip(cols, raw)]: + _chunk(form, shuffled, qr, sr, row, tx, c8, gw, uw, alpha, beta, limit) + + @flyc.jit + def launch_swiglu_oai_mxfp8( + arg_x: fx.Int64, + arg_q: fx.Int64, + arg_s: fx.Int64, + i64_q_bytes: fx.Int64, + i64_s_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sx: fx.Int32, + i32_alpha: fx.Int32, + i32_beta: fx.Int32, + i32_limit: fx.Int32, + stream: fx.Stream, + ): + kernel( + i32_m, i32_sx, arg_x, arg_q, arg_s, i64_q_bytes, i64_s_bytes, i32_alpha, i32_beta, i32_limit + ).launch(grid=(fx.Int64(_xcd_grid(i32_m * pieces)), 1, 1), block=(threads, 1, 1), stream=stream) + + launch_swiglu_oai_mxfp8.kernel_name = name + launch_swiglu_oai_mxfp8.compile_hints = PRELOAD_HINTS + return launch_swiglu_oai_mxfp8 diff --git a/amd_kernels/activation/swiglu_oai_mxfp8_i6144/reference.py b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/reference.py new file mode 100644 index 0000000..57500ad --- /dev/null +++ b/amd_kernels/activation/swiglu_oai_mxfp8_i6144/reference.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The torch contract of ``swiglu_oai_mxfp8``. + +``bf16_chain`` (the DSpark draft's ``SiluAndMulWithClamp.forward_native`` on a bf16 tensor; every aten op +computes in fp32 and rounds to bf16): + + g = clamp(gate, max=limit); u = clamp(up, -limit, limit) + act = g * sigmoid(alpha * g) * (u + beta) (bf16 after every op) + +then the Triton MXFP8 quantizer of the bf16 ``act`` (1 x 32 blocks, E8M0 scale +``clamp(ceil(log2(amax / 448)) + 127, 0, 254)``, e4m3 codes of ``act * 2^(127 - scale)``), in the layout the +down GEMM reads (row-major or (16, 16) tiles; scales in AITER's shuffled layout). + +``fp32`` (the target's dense layers 0-2, ``swiglu_oai_split``): ``swiglu_fp32`` (fp32 math, one bf16 +rounding), then the same quantizer. + +Tolerance: the kernel's codes and scales equal this reference bit for bit (the same bf16 ``act``, the same +quantizer) on finite inputs; the tests check that, so a change of the SwiGLU's order or rounding shows up.""" + +import torch + + +def swiglu_bf16_chain(gate_up: torch.Tensor, alpha: float, beta: float, limit: float) -> torch.Tensor: + d = gate_up.shape[-1] // 2 + gate = torch.clamp(gate_up[..., :d], max=limit) + up = torch.clamp(gate_up[..., d:], min=-limit, max=limit) + return gate * torch.sigmoid(alpha * gate) * (up + beta) + + +def swiglu_fp32(gate_up: torch.Tensor, alpha: float, beta: float, limit: float) -> torch.Tensor: + """``swiglu_oai_split``'s form: fp32 math, one bf16 rounding.""" + d = gate_up.shape[-1] // 2 + gate = torch.clamp(gate_up[..., :d].float(), max=limit) + up = torch.clamp(gate_up[..., d:].float(), min=-limit, max=limit) + return (gate * torch.sigmoid(alpha * gate) * (up + beta)).bfloat16() + + +def quantize_mxfp8(act: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Row-major e4m3 codes [m, K] and e8m0 scale bytes [m, K / 32] of a bf16 [m, K] (the Triton + quantizer's arithmetic).""" + m, k = act.shape + x = act.float().view(m, k // 32, 32) + amax = x.abs().amax(-1).clamp_min(torch.finfo(torch.float32).tiny) + sb = torch.clamp(torch.ceil(torch.log2(amax / 448.0)) + 127.0, 0.0, 254.0) + q = (x * torch.exp2(127.0 - sb)[..., None]).view(m, k).to(torch.float8_e4m3fn) + return q, sb.to(torch.uint8) + + +def shuffle_scales(sb: torch.Tensor) -> torch.Tensor: + """e8m0 bytes [m, K / 32] -> AITER's shuffled layout [ceil(m / 32) * 32, K / 32] (padding zero).""" + from aiter.ops.shuffle import shuffle_scale_a16w4 + + m, g = sb.shape + rows = (m + 31) // 32 * 32 + pad = torch.zeros((rows, g), dtype=torch.uint8, device=sb.device) + pad[:m] = sb + return shuffle_scale_a16w4(pad, 1, False).reshape(rows, g) diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/README.md b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/README.md new file mode 100644 index 0000000..6932018 --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/README.md @@ -0,0 +1,49 @@ +# QKV MXFP8 GEMM + norm / RoPE + NVFP4 page inserts, N 4992, K 6144, rotary 64, pages 128 (TP2) + +## What it does + +The QKV chain of a MiniMax-M3.1-Flash TP2 sparse attention layer in one entry: the MXFP8 QKV GEMM (N 4992 = 32 q + 2 +k + 2 v + 2 index-q + 1 replicated index-k heads of 128, K 6144), then per head the Gemma RMS norm and partial NeoX +RoPE (rotary 64), bf16 q `[m, 4096]`, e4m3 index-q `[m, 256]` (row stride divisible by 4), and the NVFP4 K / V / +index-K inserts into `[blocks, 4, 128, 72]` main and `[blocks, 128, 72]` index pages (bytes of skipped and unwritten +tokens stay). Input: the producer's e4m3 codes in the (16, 16) tile (`prequant_code_tile_rows(m, 4992, 6144)` rows, +1-768), shuffled E8M0 scales and the `preshuffle_mxfp8_weight` weight; heads 32 / 2 / 2, `eps >= 2^-126`. Bytes: bit +for bit the deferred chain at every row count (`gemm.mxfp8_common.mxfp8_gemm_shuffled(defer_reduce=True)` + +`splitk_reduce_norm_rope_nvfp4_insert`, or the bf16 GEMM + `norm_rope_nvfp4_insert`). One launch at 1-16 rows; at +17-768 the deferred chain's two launches: split-K partials at 17-48, the split-K decode GEMM (its K split reduced in +the kernel) at 49-256, the GEMM's bf16 stores non-temporal at 257-448 rows (the insert reads them at once). Contract: +`reference.py` (the GEMM reference, then the insert's). + +## Entry points + +- `mxfp8_qkv_norm_rope_nvfp4_insert`: the whole chain into the caller's q / index-q outputs and pages (slot -1 + skips); M 0 is a no-op, rows outside the shuffled-code tile's 1-768 raise `ValueError`. An `amd_kernels.warm_up` + entry: the first call compiles rows 1-768 and must be eager (capture raises). + +Usual caller: `mxfp8_gemm_qkv_norm_rope_nvfp4_insert` of +[`norm_rope_nvfp4_insert_d128_rot64_p128`](../norm_rope_nvfp4_insert_d128_rot64_p128/README.md), which routes the +TP2 heads here (`one_launch=False` keeps its two launches). Counters: one set per eager stream or per (capture ID, +stream), from a pool of 512 sets (2.4 MiB) per device. `host.py` picks the one launch, the chain and its store policy +by row count. + +## Performance + +Graph replay, us per call, layer 39 TP2 rank 0 weights in 14 cold copies, up to 100 inputs (decode rows: requests of +8 tokens), one MI355X. Deferred chain: the same GEMM row + the TP2 insert in two launches. Stock: vLLM's TP2 path +(Triton `tl.dot_scaled` + vLLM's FlyDSL insert). Roofline: 6.5 TB/s over weight, codes and insert bytes, or 5 PF/s. + +| M | entry | deferred chain | stock | roofline | +|---|---|---|---|---| +| 1 / 8 / 16 (one launch) | 9.01 / 9.26 / 9.52 | 9.34 / 10.17 / 10.27 | 12.94 / 13.56 / 14.12 | 4.87 / 4.88 / 4.90 | +| 32 (chain) | 11.03 | 11.03 | 15.00 | 4.94 | +| 64 / 96 / 128 / 176 / 256 (chain, split-K decode GEMM) | 12.79 / 13.96 / 14.97 / 17.34 / 20.32 | 12.80 / 13.95 / 14.97 / 17.36 / 20.32 | 18.35 / 23.45 / 25.10 / 27.42 / 41.02 | 5.02 / 5.09 / 5.17 / 5.28 / 5.47 | +| 288 / 352 / 384 / 416 (chain, nt GEMM stores) | 24.01 / 25.02 / 25.43 / 28.01 | 24.34 / 25.61 / 26.01 / 28.16 | 37.89 / 39.01 / 40.92 / 63.59 | 5.54 / 5.69 / 5.77 / 5.84 | +| 512 / 768 (chain) | 30.46 / 36.18 | 30.42 / 36.19 | 70.95 / 72.91 | 6.28 / 9.42 | + +## Tests + +```bash +python3 -m pytest tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 -q +python3 benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py \ + --checkpoint --out /tmp/qkv_fused_tp2.json +``` diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py new file mode 100644 index 0000000..c207767 --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV MXFP8 GEMM (N 4992, K 6144) + split-K reduce + q / k / index norm + RoPE (rotary 64) + NVFP4 K / V / +index-K inserts into 128-token pages, one launch at 1-768 rows (FlyDSL, gfx950).""" + +from .host import mxfp8_qkv_norm_rope_nvfp4_insert + +__all__ = ["mxfp8_qkv_norm_rope_nvfp4_insert"] diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/host.py b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/host.py new file mode 100644 index 0000000..5cce0e9 --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/host.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV MXFP8 GEMM + norm / RoPE / NVFP4 inserts, bitwise the deferred chain: ``mxfp8_gemm_shuffled( +defer_reduce=True)``, then ``splitk_reduce_norm_rope_nvfp4_insert`` (split rows) or ``norm_rope_nvfp4_insert``. + +One launch (``kernel``) at 1-16 rows; the deferred chain's two launches at 17-768, its GEMM storing non-temporally +on the 160 / 192 / 224 x 48 rows (257-448). Counter storage is separated by stream and graph capture. +""" + +import ctypes +import functools +import struct +import threading + +import torch + +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( + norm_rope_nvfp4_insert, + splitk_reduce_norm_rope_nvfp4_insert, +) +from amd_kernels.common.tensor_shim import _run_compiled +from amd_kernels.gemm.mxfp8_common import gemm as _gemm +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import W_CACHE_MOD, compile_mxfp8_gemm_decode +from amd_kernels.warm_up import entry + +from . import kernel as _kernel + +N, K = _kernel.N, _kernel.K +MAX_ROWS = 768 # the shuffled-code producer's range (``prequant_code_tile_rows``) +# one launch to here: -4..-11% at 1-16 rows against the deferred chain, slower above (benchmark_chain.py) +FUSED_MAX_ROWS = 16 + + +@functools.cache +def _plan(m: int): + """(RT, NI, KW, KS, LA) of the deferred GEMM row at ``m`` rows: the one-launch kernel's tile and K split.""" + table = _gemm._select_prequant_config(m, N, K, torch.bfloat16) + config = _gemm._deferred_qkv_config(m, N, K, table) + assert config.kernel == _gemm._DECODE_KERNEL and config.split_k > 1 and m <= config.tile_m, (m, config) + la = (config.la_w, config.la_a) if config.la_w else None + return config.tile_m // 16, config.tile_n // 16, config.waves_m, config.split_k, la + + +@functools.cache +def _fused_launcher(plan): + return _kernel.compile_qkv_fused(*plan) + + +# decode tiles whose bf16 QKV goes non-temporal (same bytes): chain -0.1..-0.6 us at 257-448, slower elsewhere +# (benchmark_chain.py) +NT_OUT_TILES = ((160, 48), (192, 48), (224, 48)) + + +@functools.cache +def _nt_gemm_launcher(config): + host = _gemm._shape_host(N, K) + return compile_mxfp8_gemm_decode( + N=N, K=K, RT=config.tile_m // 16, NI=config.tile_n // 16, KW=config.waves_m, KS=1, + LA=(config.la_w, config.la_a) if config.la_w else None, sched_barriers=bool(config.waves_n), + OUT_CM=W_CACHE_MOD, GROUPED=config in host.GROUPED_ROWS, + ) + + +@functools.cache +def _nt_config(m): + """The decode row at ``m`` rows if its output goes non-temporal, else None.""" + config = _gemm._select_prequant_config(m, N, K, torch.bfloat16) + if config.kernel != _gemm._DECODE_KERNEL or config.split_k != 1: + return None + return config if (config.tile_m, config.tile_n) in NT_OUT_TILES else None + + +_COUNTER_SETS = 512 # eager streams and retained graph captures per device; each graph shares its set serially +_COUNTER_WORDS = _kernel.COUNTERS +_COUNTER_LOCK = threading.Lock() +_POOL: dict[torch.device, torch.Tensor] = {} +_COUNTERS: dict[tuple[torch.device, tuple], torch.Tensor] = {} + + +@functools.cache +def _capture_info_api(): + """The HIP capture ID distinguishes graphs built on the same stream (stdlib ctypes, no device launch).""" + lib = ctypes.CDLL("libamdhip64.so") + fn = lib.hipStreamGetCaptureInfo + fn.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_ulonglong)] + fn.restype = ctypes.c_int + return lib, fn + + +def _counter_key(device): + stream = torch.cuda.current_stream(device).cuda_stream + if not torch.cuda.is_current_stream_capturing(): + return device, ("stream", stream) + _, fn = _capture_info_api() + status, capture_id = ctypes.c_int(), ctypes.c_ulonglong() + rc = fn(ctypes.c_void_p(stream), ctypes.byref(status), ctypes.byref(capture_id)) + if rc or status.value != 1: + raise RuntimeError(f"hipStreamGetCaptureInfo failed: rc={rc}, status={status.value}") + return device, ("capture", capture_id.value, stream) + + +def _counters(device: torch.device) -> torch.Tensor: + """One counter set per eager stream or (capture ID, stream), zeroed once outside capture: graphs captured on one + stream may replay concurrently on others, and each call adds ``EPOCH`` to every counter it uses. Pool + initialization and slice reservation are serialized across host threads (tensor ops may release the GIL).""" + key = _counter_key(device) + cnt = _COUNTERS.get(key) + if cnt is not None: + return cnt + with _COUNTER_LOCK: + cnt = _COUNTERS.get(key) + if cnt is not None: + return cnt + pool = _POOL.get(device) + if pool is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("mxfp8_qkv_norm_rope_nvfp4_insert: first call inside CUDA graph capture; call it " + "once eagerly on the device first (the entry's warm-up does)") + pool = torch.zeros(_COUNTER_SETS * _COUNTER_WORDS, dtype=torch.int32, device=device) + ready = torch.cuda.Event() + ready.record(torch.cuda.current_stream(device)) + ready.synchronize() + _POOL[device] = pool + used = sum(1 for d, _ in _COUNTERS if d == device) + if used >= _COUNTER_SETS: + raise RuntimeError(f"mxfp8_qkv_norm_rope_nvfp4_insert: more than {_COUNTER_SETS} stream/capture " + f"counter sets on {device}") + cnt = _COUNTERS[key] = pool[used * _COUNTER_WORDS:(used + 1) * _COUNTER_WORDS] + return cnt + + +def _fused(activation, activation_scale, weight, weight_scale, m, q_norm_weight, k_norm_weight, + index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, slot_mapping, index_slot_mapping, + kv_cache, index_cache, k_scale, v_scale, q_out, index_q_out, eps): + plan = _plan(m) + slab = torch.empty((plan[3], m, N), dtype=torch.float32, device=activation.device) + _run_compiled( + _fused_launcher(plan), + activation.data_ptr(), weight.data_ptr(), activation_scale.data_ptr(), weight_scale.data_ptr(), + slab.data_ptr(), _counters(activation.device).data_ptr(), q_out.data_ptr(), index_q_out.data_ptr(), + q_norm_weight.data_ptr(), k_norm_weight.data_ptr(), index_q_norm_weight.data_ptr(), + index_k_norm_weight.data_ptr(), cos_sin_cache.data_ptr(), positions.data_ptr(), slot_mapping.data_ptr(), + index_slot_mapping.data_ptr(), kv_cache.data_ptr(), index_cache.data_ptr(), k_scale.data_ptr(), + v_scale.data_ptr(), m, activation_scale.numel() * activation_scale.element_size(), index_q_out.stride(0), + kv_cache.stride(0), index_cache.stride(0), cos_sin_cache.shape[0], + struct.unpack(" None: + """``mxfp8_gemm_shuffled(defer_reduce=True)`` + the split-K reduce-insert (or the bf16 GEMM + + ``norm_rope_nvfp4_insert``), bit for bit; one launch at 1-16 rows.""" + if m == 0: + return + assert (num_heads, num_kv_heads, num_index_heads) == (_kernel.NQ, _kernel.NKV, _kernel.NIQ) + assert tuple(weight.shape) == (N, K) and activation.shape[1] == K + assert cos_sin_cache.shape[1] == _kernel.ROT + if not 1 <= m <= MAX_ROWS or not _gemm.prequant_code_tile_rows(m, N, K, torch.bfloat16): + raise ValueError(f"no shuffled-code QKV row at M {m}") + ins = (q_norm_weight, k_norm_weight, index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, + slot_mapping, index_slot_mapping, kv_cache, index_cache, k_scale, v_scale, q_out, index_q_out) + if m <= FUSED_MAX_ROWS: + assert eps >= 2.0**-126 # the kernel's rsqrt skips ocml's denormal scaling + assert activation.shape[0] >= 16 and activation.is_contiguous() and weight.is_contiguous() + assert q_out.dtype == torch.bfloat16 and q_out.is_contiguous() + assert index_q_out.dtype == torch.float8_e4m3fn and index_q_out.stride(1) == 1 + assert index_q_out.stride(0) % 4 == 0 + assert kv_cache.shape[1:] == (2 * _kernel.NKV, 128, 72) and kv_cache.stride(1) == 128 * 72 + assert kv_cache.stride(3) == 1 + assert index_cache.shape[1:] == (128, 72) and index_cache.stride(1) == 72 and index_cache.stride(2) == 1 + _fused(activation, activation_scale, weight, weight_scale, m, *ins, eps) + return + kw = dict(num_heads=num_heads, num_kv_heads=num_kv_heads, num_index_heads=num_index_heads, eps=eps) + config = _nt_config(m) + if config is not None: + assert activation.shape[0] == -(-m // config.tile_m) * config.tile_m + qkv = torch.empty((m, N), dtype=torch.bfloat16, device=activation.device) + n_mb = -(-m // config.tile_m) + u8 = lambda t: t.view(torch.uint8).view(-1) # noqa: E731 + _run_compiled(_nt_gemm_launcher(config), u8(activation), u8(weight), u8(qkv), u8(activation_scale), + u8(weight_scale), m, activation_scale.numel() * activation_scale.element_size(), n_mb, + n_mb * (N // config.tile_n), torch.cuda.current_stream()) + norm_rope_nvfp4_insert(qkv, *ins, **kw) + return + qkv = _gemm.mxfp8_gemm_shuffled(activation, activation_scale, weight, weight_scale, torch.bfloat16, m, + defer_reduce=True) + if isinstance(qkv, _gemm.SkPartials): + splitk_reduce_norm_rope_nvfp4_insert(qkv.tmp, qkv.split_k, *ins, **kw) + else: + norm_rope_nvfp4_insert(qkv, *ins, **kw) + + +def _mode(*args, **kwargs): + """One mode: the variants depend on the token count only (``amd_kernels.warm_up``).""" + return () + + +def _sizes(mode): + """Every token count the entry serves (each picks its kernel by the count).""" + return range(1, MAX_ROWS + 1) + + +def _inputs(mode, n, activation, activation_scale, weight, weight_scale, m, q_norm_weight, k_norm_weight, + index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, slot_mapping, index_slot_mapping, + kv_cache, index_cache, k_scale, v_scale, q_out, index_q_out, **kw): + """One warm-up call at ``n`` tokens: zero codes in the producer's tile, every slot -1 (no page is written).""" + dev = weight.device + rows = _gemm.prequant_code_tile_rows(n, N, K, torch.bfloat16) + rows = (n + rows - 1) // rows * rows + codes = torch.zeros((rows, K), dtype=activation.dtype, device=dev) + scales = torch.full(((rows + 31) // 32 * 32, K // 32), 127, dtype=activation_scale.dtype, device=dev) + zeros = torch.zeros(n, dtype=torch.int64, device=dev) + skip = torch.full((n,), -1, dtype=torch.int64, device=dev) + q = torch.empty((n, q_out.shape[1]), dtype=q_out.dtype, device=dev) + iq = torch.empty((n, index_q_out.shape[1]), dtype=index_q_out.dtype, device=dev) + return ((codes, scales, weight, weight_scale, n, q_norm_weight, k_norm_weight, index_q_norm_weight, + index_k_norm_weight, cos_sin_cache, zeros, skip, skip, kv_cache, index_cache, k_scale, v_scale, q, iq), + kw) + + +mxfp8_qkv_norm_rope_nvfp4_insert = entry("attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128", + mode=_mode, sizes=_sizes, inputs=_inputs)(mxfp8_qkv_norm_rope_nvfp4_insert) diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/kernel.py b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/kernel.py new file mode 100644 index 0000000..14f095f --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/kernel.py @@ -0,0 +1,542 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV MXFP8 GEMM (N 4992, K 6144) with the split-K reduce + norm / RoPE + NVFP4 inserts in its tail, one +launch (gfx950). It builds for every deferred split row (1-48 rows); the host serves 1-16, where it wins. + +GEMM: ``gemm/mxfp8_common``'s decode kernel on the deferred row's own tile (``gemm/mxfp8_n4992_k6144`` +``DEFERRED_ROWS``: RT 16-row tiles, all rows in one block, x NI 16-column tiles, KW K-waves, KS K splits over +workgroups), so every output is the same KS * KW sequential MFMA chains over equal K ranges, the KW chains of a split +summed in wave order (parked in LDS, the decode kernel's park-all sums). Each split stores its fp32 slab +``[KS, m, 4992]`` at agent scope (sc1), and the tail sums the slabs ``((0 + s0) + s1) + ...``, rounds to bf16 and runs +``splitk_reduce_kernel_q64_kv4_iq4``'s arithmetic: the deferred chain's bytes. + +Hand-off: a 48 / 64-column tile overlaps one or two heads; a head's counter collects its KS x (3 or 4) tiles. Once a +workgroup's slab stores are acknowledged (``vmcnt(0)``, a barrier), wave 0 adds to the counter of each head its tile +covers: the split-0 first tile of the head adds ``EPOCH - count + 1``, every other tile 1, so only the last arrival +sees the counter reach the epoch (epoch counters: each call adds ``EPOCH`` to every counter it uses; no reset, i32 wrap +included). The last arrival runs the head for every row. No workgroup waits for another. + +Tail: 8 waves. The tail waves stage every slab-independent input in LDS while the K-waves run the GEMM (cos / sin +rows, main and index slots of the rows and 8 tokens past them, the norm weights, the exact NVFP4 scale reciprocals of +the K, V and index-K scales: ``_quant4_lds``); after a barrier each wave reads its token pairs' inputs and the V octets' +old bytes before the hand-off, so after it only the slab loads remain. Wave w takes pairs w, w + 8, ...; a V head's +waves quantize their rows into LDS, and after a barrier the first token of each octet run packs the run from LDS +(``packing.py``). + +Head columns (of 128): q 0-31, k 32-33, v 34-35, index-q 36-37, index-k 38. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import ( + PRELOAD_HINTS, + W_CACHE_MOD, + _lookahead, + _mfma, + _pack8, + _pin_accumulators, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128.kernel import ( + BLK, + F448, + F448N, + RCP6, + D, + _bf16x4, + _e2m1x2, + _e4m3_byte, + _e4m3_value, + _f32, + _gptr, + _iq_fp8, + _k_scale_off, + _maxnum, + _minnum, + _norm_rope4, + _pk_bf16, + _xmax, +) +from amd_kernels.warm_up import launcher +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import BFloat16, Float32, T +from flydsl.expr.typing import Vector as Vec + +from .packing import _v_expand, _v_expanded_words + +N, K = 4992, 6144 +NQ, NKV, NIQ, ROT = 32, 2, 2, 64 +HEADS = N // D # 39 +HK, HV, HIQ, HIK = NQ, NQ + NKV, NQ + 2 * NKV, NQ + 2 * NKV + NIQ # first k / v / index-q head, the index-k head +NWAVE = 8 +EPOCH = 16 # a power of 2 >= every head's contributor count (KS 3 x 4 tiles = 12) +SC1 = 0x10 # agent-scope (cross-XCD) stores / loads of the slabs +HALF = ROT // 2 +SIDE = NKV * BLK * 72 +CNT_STRIDE = 32 # one counter per 128-byte line: atomics of neighbouring heads do not share a line +COUNTERS = HEADS * CNT_STRIDE +TABLE = 260 # dwords of one reciprocal table (256 codes + 1 / gs) + + +def _global_atomic_add(addr, value): + """``old = *addr; *addr += value`` on a global i32 (agent scope, relaxed).""" + ptr = fx.inttoptr(fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), fx.Int64(addr)) + return fx.Int32(_llvm.AtomicRMWOp(_llvm.AtomicBinOp.add, ptr.llvm_ptr, fx.Int32(value).ir_value(), + _llvm.AtomicOrdering.monotonic, syncscope="agent", alignment=4).result) + + +def _lds_ptr(addr): + return _llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), fx.as_ir_value(fx.Int64(addr))) + + +def _lds_i32(addr): + return fx.Int32(_llvm.load(T.i32, _lds_ptr(addr), alignment=4)) + + +def _lds_vec(addr, n): + return Vec(_llvm.load(T.vec(n, T.i32), _lds_ptr(addr), alignment=4 * n)) + + +def _lds_store(addr, v, align): + _llvm.StoreOp(fx.as_ir_value(v), _lds_ptr(addr), alignment=align) + + +def _quant4_lds(xs, table): + """``_quant4`` with its divisions read from the LDS table at byte address ``table``: entries 0-255 hold the same + correctly rounded 1 / (gs * e4m3(byte)) (0 for a zero scale), entry 256 the same 1 / gs, so the codes and scale + bytes are ``_quant4``'s.""" + a = [fx.math.absf(x) for x in xs] + m = _maxnum(_maxnum(a[0], a[1]), _maxnum(a[2], a[3])) + m = _xmax(_xmax(m, 1), 2) + inv_gs = _lds_i32(table + 256 * 4).bitcast(fx.Float32) + sf = _minnum((m * _f32(RCP6)) * inv_gs, _f32(F448)) + byte = _e4m3_byte(sf) + quant = _lds_i32(table + byte * 4).bitcast(fx.Float32) + y = [x * quant for x in xs] + return _e2m1x2(y[2], y[3], _e2m1x2(y[0], y[1], fx.Int32(0), 0), 1), byte + + +@launcher +def compile_qkv_fused(RT: int, NI: int, KW: int, KS: int, LA: tuple | None = None): + """The one-launch chain on a deferred GEMM row's tile; every row in one block (m <= 16 * RT). + + Args: + RT, NI, KW, KS: 16-row tiles, 16-column tiles, K-waves and K splits of the deferred row (its bytes). + LA: (W steps, A blocks) in flight ahead of the MFMAs (default: the deepest that fits). + """ + KSTEPS = K // 128 + assert KSTEPS % (KS * KW) == 0 and (KSTEPS // (KS * KW)) % 2 == 0 and KW < NWAVE + SW = KSTEPS // (KS * KW) + BM = 16 * RT + C = 16 * NI + NB = N // C + assert N % C == 0 and C <= D + SCB = K // 256 + RP = max(1, RT // 2) + if RT > 1 and RT % 2 == 1: + RP = (RT + 1) // 2 + CP = NI if NI % 2 else NI // 2 + LA_W, LA_A = LA or _lookahead(SW, RT, NI, RP, CP) + NA = SW * RT + TW = NWAVE - KW + NT = RT * NI # accumulator tiles of a K-wave + PPW = BM // 2 // NWAVE # token pairs per wave + PARK = KW * NT * 1024 if KW > 1 else 0 + CS_OFF = PARK # [BM, 32] dwords: cos | sin of each row's position + SL_OFF = CS_OFF + BM * 128 # main slots of tokens 0 .. BM + 7 (-1 past the batch) + ISL_OFF = SL_OFF + (BM + 8) * 4 + W_OFF = ISL_OFF + BM * 4 # norm weights: q, k, index-q, index-k + RCP_OFF = W_OFF + 4 * 64 * 4 # reciprocal tables: K, V, index-K + VROW_OFF = RCP_OFF + 3 * TABLE * 4 # per head slot, BM + 8 V rows quantized: (expanded codes, scale byte) + VROW_BYTES = (BM + 8) * 32 * 8 + BUF_BYTES = VROW_OFF + 2 * VROW_BYTES + assert BUF_BYTES <= 160 * 1024, BUF_BYTES + NSL = -(-(BM + 8) // 64) + name = (f"mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128_rt{RT}_ni{NI}_kw{KW}_ks{KS}" + f"_la{LA_W}x{LA_A}") + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, BUF_BYTES, 16] + flags: fx.Array[fx.Int32, 4] # the tile's two heads: the last arrival is here + + @flyc.kernel(name=name, known_block_size=[64 * NWAVE, 1, 1]) + def kernel( + i32_m: fx.Int32, + a_scale_bytes: fx.Int32, + arg_w: fx.Int64, + arg_ws: fx.Int64, + arg_as: fx.Int64, + arg_a: fx.Int64, + arg_slab: fx.Int64, # [KS, m, N] fp32 partials (workspace) + arg_cnt: fx.Int64, # [COUNTERS] i32 head counters, a multiple of EPOCH between calls + i32_cs_rows: fx.Int32, + arg_pos: fx.Int64, + arg_slot: fx.Int64, + arg_islot: fx.Int64, + arg_cs: fx.Int64, + arg_qw: fx.Int64, + arg_kw: fx.Int64, + arg_iqw: fx.Int64, + arg_ikw: fx.Int64, + arg_q: fx.Int64, + arg_iq: fx.Int64, + arg_kv: fx.Int64, + arg_idx: fx.Int64, + arg_ks: fx.Int64, + arg_vs: fx.Int64, + i64_kv_st: fx.Int64, + i64_idx_st: fx.Int64, + i32_iq_st: fx.Int32, + i32_eps: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + l16, q16 = lane % 16, lane // 16 + kz, nb = bx // NB, bx % NB + slab_bytes = i32_m * (N * 4 * KS) + slab = buffer_ops.create_buffer_resource_from_addr(arg_slab, num_records_bytes=slab_bytes) + lds = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + hl = lane % 32 + hi16 = (hl & 16) != 0 + dim0 = hl * 4 + eps = i32_eps.bitcast(fx.Float32) + first = dim0 < HALF + in_rope = dim0 < ROT + ib = first.select(dim0, dim0 - HALF) + h0 = (nb * C) // D + heads = [h0, h0 + 1] + has1 = (nb * C + (C - 1)) // D > h0 # the tile reaches a second head + is_vs = [(h >= HV) & (h < HIQ) for h in heads] + + def gemm_tile(kwave): + """The decode kernel's K-wave ``kwave``: its RT x NI accumulators (fp32, pinned).""" + k_base = kwave * (SW * 2048) + a_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_a, num_records_bytes=BM * K) + w_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K) + as_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_as, num_records_bytes=a_scale_bytes) + ws_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_ws, num_records_bytes=(N // 32) * SCB * 256) + lane_off = q16 * 256 + l16 * 16 + n16 = [nb * NI + ni for ni in range_constexpr(NI)] + w_dw = [(n16[ni] * (16 * K) + k_base + lane_off) // 4 for ni in range_constexpr(NI)] + a_dw = [(rt * (16 * K) + k_base + lane_off) // 4 for rt in range_constexpr(RT)] + sc_lane = q16 * 16 + l16 + sc_k = kwave * (SW // 2 * 64) + ws_dw = [((n16[cp if NI % 2 else 2 * cp] // 2) * SCB) * 64 + sc_k + sc_lane for cp in range_constexpr(CP)] + as_dw = [(rp * SCB) * 64 + sc_k + sc_lane for rp in range_constexpr(RP)] + w_shift = [(n16[ni] % 2) * 8 for ni in range_constexpr(NI)] if NI % 2 else None + + def load_pair(rsrc, dw, s, nt): + kw = dict(vec_width=4, dtype=fx.Int32) + if const_expr(nt): + kw["cache_modifier"] = W_CACHE_MOD + return _pack8(buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048, **kw), + buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048 + 1024, **kw)) + + def load_scales(rsrc, dws, s, shift): + vals = [fx.Int32(buffer_ops.buffer_load(rsrc, dw, vec_width=1, dtype=fx.Int32, + soffset_bytes=(s // 2) * 256)) for dw in dws] + if const_expr(shift is None): + return vals + return [v.shrui(sh) for v, sh in zip(vals, shift)] + + def load_w_step(s): + bb = [load_pair(w_rsrc, w_dw[ni], s, True) for ni in range_constexpr(NI)] + if const_expr(s % 2 == 0): + return bb, (load_scales(ws_rsrc, ws_dw, s, w_shift), load_scales(as_rsrc, as_dw, s, None)) + return bb, None + + def load_a(t): + rt, s = t % RT, t // RT + return load_pair(a_rsrc, a_dw[rt], s, False) + + acc = [[None] * NI for _ in range(RT)] + w_q = [None] * SW + a_q = [None] * NA + for s in range_constexpr(LA_W): + w_q[s] = load_w_step(s) + for t in range_constexpr(LA_A): + a_q[t] = load_a(t) + sw = sa = None + for s in range_constexpr(SW): + if const_expr(s + LA_W < SW): + w_q[s + LA_W] = load_w_step(s + LA_W) + bb, sc = w_q[s] + if const_expr(s % 2 == 0): + sw, sa = sc + for rt in range_constexpr(RT): + t = s * RT + rt + if const_expr(t + LA_A < NA): + a_q[t + LA_A] = load_a(t + LA_A) + a_op = a_q[t] + a_q[t] = None + for ni in range_constexpr(NI): + acc[rt][ni] = _mfma(bb[ni], a_op, acc[rt][ni], sw[ni if NI % 2 else ni // 2], sa[rt // 2], + 0 if NI % 2 else ni % 2, rt % 2, s % 2) + w_q[s] = None + return _pin_accumulators([acc[rt][ni] for rt in range(RT) for ni in range(NI)]) + + def store_tile(i, v): + """Accumulator tile ``i`` of this split's slab, agent scope.""" + row = (i // NI) * 16 + l16 + col = (nb * NI + i % NI) * 16 + q16 * 4 + buffer_ops.buffer_store(v.bitcast(fx.Int32), slab, ((kz * i32_m + row) * N + col) * 4, + offset_is_bytes=True, mask=row < i32_m, cache_modifier=SC1) + + def preload(w0): + """The tail's slab-independent inputs into LDS (tail wave ``w0`` of TW).""" + pos_r = buffer_ops.create_buffer_resource_from_addr(arg_pos, num_records_bytes=fx.Int64(i32_m) * 8) + sl_r = buffer_ops.create_buffer_resource_from_addr(arg_slot, num_records_bytes=fx.Int64(i32_m) * 8) + isl_r = buffer_ops.create_buffer_resource_from_addr(arg_islot, num_records_bytes=fx.Int64(i32_m) * 8) + cs_r = buffer_ops.create_buffer_resource_from_addr(arg_cs, + num_records_bytes=fx.Int64(i32_cs_rows) * (ROT * 2)) + toks = [] # 8 rows of cos / sin per wave load + for k in range_constexpr(-(-(BM // 8) // TW)): + c = w0 + k * TW + toks.append((c < BM // 8).select(c, fx.Int32(BM // 8 - 1)) * 8 + lane // 8) + poss = [fx.Int32(buffer_ops.buffer_load(pos_r, t * 2, vec_width=1, dtype=fx.Int32, mask=t < i32_m)) + for t in toks] + entries = [] + for k in range_constexpr(-(-NSL // TW)): + e = w0 + k * TW + entries.append((e < NSL).select(e, fx.Int32(NSL - 1)) * 64 + lane) + live = [e < i32_m for e in entries] + slots = [fx.Int32(buffer_ops.buffer_load(sl_r, e * 2, vec_width=1, dtype=fx.Int32, mask=ok)) + for e, ok in zip(entries, live)] + islots = [fx.Int32(buffer_ops.buffer_load(isl_r, e * 2, vec_width=1, dtype=fx.Int32, mask=ok)) + for e, ok in zip(entries, live)] + ws = [] + for a in (arg_qw, arg_kw, arg_iqw, arg_ikw): + wr = buffer_ops.create_buffer_resource_from_addr(a, num_records_bytes=D * 2) + ws.append(fx.Int32(buffer_ops.buffer_load(wr, lane, vec_width=1, dtype=fx.Int32))) + ks = fx.Float32(_gptr(arg_ks, T.f32, 4)[0]) + vs = fx.Float32(_gptr(arg_vs, T.f32, 4)[0]) + css = [Vec(buffer_ops.buffer_load(cs_r, p * (ROT // 2) + (lane % 8) * 4, vec_width=4, dtype=fx.Int32)) + for p in poss] + for k, gs in enumerate((ks, vs, fx.Float32(1.0))): + table = lds + RCP_OFF + k * (TABLE * 4) + for i in range_constexpr(4): + sv = _e4m3_value(lane + i * 64) + quant = (sv != 0.0).select(fx.Float32(1.0) / (gs * sv), fx.Float32(0.0)) + _lds_store(table + (lane + i * 64) * 4, quant.bitcast(fx.Int32), 4) + _lds_store(table + 256 * 4, (fx.Float32(1.0) / gs).bitcast(fx.Int32), 4) + for e, ok, v, iv in zip(entries, live, slots, islots): + if e < BM + 8: + _lds_store(lds + SL_OFF + e * 4, ok.select(v, fx.Int32(-1)), 4) + if e < BM: + _lds_store(lds + ISL_OFF + e * 4, ok.select(iv, fx.Int32(-1)), 4) + for k in range_constexpr(4): + _lds_store(lds + W_OFF + (k * 64 + lane) * 4, ws[k], 4) + for t, v in zip(toks, css): + _lds_store(lds + CS_OFF + t * 128 + (lane % 8) * 16, v, 16) + + if wave < KW: + flat = gemm_tile(kz * KW + wave) + if const_expr(KW == 1): + for i in range_constexpr(NT): + store_tile(i, Vec(flat[i])) + else: + for i in range_constexpr(NT): + _lds_store(lds + (wave * NT + i) * 1024 + lane * 16, Vec(flat[i]).bitcast(fx.Int32), 16) + else: + preload(wave - KW) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if const_expr(KW > 1): + # the K-waves' partials summed in wave order (the decode kernel's park-all sums), tiles over all waves + for j in range_constexpr(-(-NT // NWAVE)): + i = wave + j * NWAVE + if i < NT: + v = None + for w in range_constexpr(KW): + p = _lds_vec(lds + (w * NT + i) * 1024 + lane * 16, 4).bitcast(fx.Float32) + v = p if v is None else v + p + store_tile(i, v) + + # this wave's token pairs: the V run each one starts and its octet's old bytes, cos / sin, weights + pairs = [] + for k in range_constexpr(PPW): + tok = (wave + k * NWAVE) * 2 + lane // 32 + live = tok < i32_m + slot = _lds_i32(lds + SL_OFF + tok * 4) + prev = (tok > 0).select(_lds_i32(lds + SL_OFF + (tok - 1) * 4), fx.Int32(-1)) + near_t = tok - 7 + hl + near = ((hl < 16) & (near_t >= 0)).select(_lds_i32(lds + SL_OFF + near_t * 4), fx.Int32(-1)) + base_o = slot - slot % 8 + in_oct = (near >= base_o) & (near < base_o + 8) + votes = fx.Int64(fx.rocdl.ballot(T.i64, in_oct)) + mine = (votes >> fx.Int64((lane // 32) * 32)).to(fx.Int32) & 0xFFFF + n_run = fx.Int32(fx.math.ctpop(mine)) + go = live & (slot >= 0) & ((prev < 0) | ((prev >> 3) != (slot >> 3))) + wslot = go.select(slot, fx.Int32(0)) + tp_ = wslot & (BLK - 1) + olds = [] + for j in range_constexpr(2): + vh = is_vs[j].select(heads[j] - HV, fx.Int32(0)) + v_side = arg_kv + fx.Int64(wslot >> 7) * i64_kv_st + SIDE + v_data = v_side + fx.Int64(vh * (BLK * 64) + (tp_ // 8) * (D * 4)) + v_sf = v_side + fx.Int64(NKV * BLK * 64 + vh * (BLK * 8) + tp_ - tp_ % 8) + dp = _gptr(v_data + fx.Int64(dim0 * 4), T.i32, 4) + sp = _gptr(v_sf + fx.Int64((hl // 4) * BLK), T.i32, 4) + old = [fx.Int32(dp[i]) for i in range_constexpr(4)] + [fx.Int32(sp[i]) for i in range_constexpr(2)] + olds.append((dp, sp, old)) + cs_row = lds + CS_OFF + tok * 128 + (ib // 2) * 4 + cos = _bf16x4(_lds_vec(cs_row, 2)) + sin = _bf16x4(_lds_vec(cs_row + 64, 2)) + pairs.append((tok, live, n_run, go, tp_, olds, cos, sin)) + wts = [] + for h in heads: + ty = (h < HK).select(fx.Int32(0), ((h >= HK) & (h < HV)).select( + fx.Int32(1), (h == HIK).select(fx.Int32(3), fx.Int32(2)))) + wts.append(_bf16x4(_lds_vec(lds + W_OFF + (ty * 64 + hl * 2) * 4, 2))) + + # every slab store is acknowledged before the counters move + fx.rocdl.s_waitcnt(vmcnt=0) + if tx < 4: + smem.flags.ptr[tx] = fx.Int32(0) + fx.gpu.barrier() + if wave == 0: + h = h0 + lane + t0 = (h * D) // C + count = ((h * D + (D - 1)) // C - t0 + 1) * KS + inc = ((kz == 0) & (nb == t0)).select(EPOCH + 1 - count, fx.Int32(1)) + if (lane == 0) | ((lane == 1) & has1): + old = _global_atomic_add(arg_cnt + fx.Int64(h) * (4 * CNT_STRIDE), inc) + if ((old & (EPOCH - 1)) + inc) == EPOCH: + smem.flags.ptr[lane] = fx.Int32(1) + fx.gpu.barrier() + done = [fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.Int32(smem.flags.ptr[j]))) != 0 for j in range(2)] + + def row4(tok, h, live): + """This lane's 4 slab vectors at (token, head).""" + return [Vec(buffer_ops.buffer_load(slab, (k * i32_m + tok) * N + h * D + dim0, vec_width=4, + dtype=Float32, mask=live, cache_modifier=SC1)) + for k in range_constexpr(KS)] + + def sum4(raw): + """The slabs summed in the reduce's order, rounded to bf16, as 4 fp32.""" + acc = Vec.filled(4, 0.0, Float32) + for k in range_constexpr(KS): + acc = acc + raw[k] + e = acc.to(BFloat16).to(Float32) + return [e[i] for i in range_constexpr(4)] + + raws = [[row4(tok, heads[j], live & done[j]) for tok, live, *_ in pairs] for j in range(2)] + q_r = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=fx.Int64(i32_m) * (NQ * D * 2)) + iq_r = buffer_ops.create_buffer_resource_from_addr(arg_iq, + num_records_bytes=fx.Int64(i32_m) * fx.Int64(i32_iq_st)) + vtab = lds + RCP_OFF + TABLE * 4 + for j in range_constexpr(2): + h = heads[j] + vrow = lds + VROW_OFF + j * VROW_BYTES + if done[j] & is_vs[j]: + for (tok, *_), raw in zip(pairs, raws[j]): + nib, sf = _quant4_lds(sum4(raw), vtab) + _lds_store(vrow + (tok * 32 + hl) * 8, + Vec.from_elements([_v_expand(fx.Int32(nib)), fx.Int32(sf)], fx.Int32), 8) + if done[j] & ~is_vs[j]: + is_q = h < HK + is_k = (h >= HK) & (h < HV) + is_ik = h == HIK + for (tok, live, n_run, go, tp_, olds, cos, sin), raw in zip(pairs, raws[j]): + out = _norm_rope4(sum4(raw), (cos, sin), wts[j], eps, hi16, first, in_rope, HALF // 4) + qv = Vec.from_elements([_pk_bf16(out[0], out[1]), _pk_bf16(out[2], out[3])], fx.Int32) + if is_q: + buffer_ops.buffer_store(qv, q_r, tok * (NQ * D // 2) + h * (D // 2) + hl * 2, mask=live, + cache_modifier=2) + if (h >= HIQ) & (h < HIK): + f = [_minnum(_maxnum(xv, _f32(F448N)), _f32(F448)) for xv in _bf16x4(qv)] + word = _iq_fp8(f[2], f[3], _iq_fp8(f[0], f[1], fx.Int32(0), False), True) + buffer_ops.buffer_store(word, iq_r, tok * i32_iq_st + (h - HIQ) * D + dim0, mask=live, + offset_is_bytes=True, cache_modifier=2) + if is_k | is_ik: + kslot = _lds_i32(is_k.select(lds + SL_OFF, lds + ISL_OFF) + tok * 4) + table = lds + RCP_OFF + is_k.select(fx.Int32(0), fx.Int32(2 * TABLE * 4)) + packed, ksf = _quant4_lds(_bf16x4(qv), table) + kh = h - HK + grp = hl // 4 + ktp = kslot & (BLK - 1) + page = is_k.select(arg_kv + fx.Int64(kslot >> 7) * i64_kv_st, + arg_idx + fx.Int64(kslot >> 7) * i64_idx_st) + d_off = is_k.select(kh * (BLK * 64), fx.Int32(0)) + ktp * 64 + hl * 2 + s_off = is_k.select(NKV * BLK * 64 + kh * (BLK * 8) + _k_scale_off(ktp, grp), + BLK * 64 + ktp * 8 + grp) + d_ptr = _gptr(page + fx.Int64(d_off), T.i16, 2) + s_ptr = _gptr(page + fx.Int64(s_off), T.i8, 1) + if live & (kslot >= 0): + d_ptr[0] = packed.to(fx.Int16) + if hl % 4 == 0: + s_ptr[0] = ksf.to(fx.Int8) + if (done[0] & is_vs[0]) | (done[1] & is_vs[1]): + fx.gpu.barrier() + for j in range_constexpr(2): + vrow = lds + VROW_OFF + j * VROW_BYTES + for tok, live, n_run, go, tp_, olds, cos, sin in pairs: + dp, sp, oldv = olds[j] + for run_rows, take in ((4, go & done[j] & is_vs[j] & (n_run <= 4)), + (8, go & done[j] & is_vs[j] & (n_run > 4))): + if take: + nibbles = [] + scales = fx.Uint64(0) + for r in range_constexpr(run_rows): + got = _lds_vec(vrow + ((tok + r) * 32 + hl) * 8, 2) + ok = (tok + r < i32_m) & (r < n_run) + nibbles.append(ok.select(got[0], fx.Int32(0))) + scales = scales | (ok.select(got[1], fx.Int32(0)).to(fx.Uint64) << (8 * r)) + words = _v_expanded_words(nibbles) + p0 = tp_ % 8 + mask = (fx.Uint32(0xFFFFFFFF) >> ((8 - n_run) * 4)) << (p0 * 4) + for i in range_constexpr(4): + dp[i] = (words[i] << (p0 * 4)) | (oldv[i] & ~mask.to(fx.Int32)) + smask = (fx.Uint64(0xFFFFFFFFFFFFFFFF) >> ((8 - n_run) * 8)) << (p0 * 8) + scales = scales << (p0 * 8) + sc0 = scales.to(fx.Int32) | (oldv[4] & ~smask.to(fx.Int32)) + sc1 = (scales >> 32).to(fx.Int32) | (oldv[5] & ~(smask >> 32).to(fx.Int32)) + if hl % 4 == 0: + sp[0] = sc0 + sp[1] = sc1 + + @flyc.jit + def launch( + arg_a: fx.Int64, + arg_w: fx.Int64, + arg_as: fx.Int64, + arg_ws: fx.Int64, + arg_slab: fx.Int64, + arg_cnt: fx.Int64, + arg_q: fx.Int64, + arg_iq: fx.Int64, + arg_qw: fx.Int64, + arg_kw: fx.Int64, + arg_iqw: fx.Int64, + arg_ikw: fx.Int64, + arg_cs: fx.Int64, + arg_pos: fx.Int64, + arg_slot: fx.Int64, + arg_islot: fx.Int64, + arg_kv: fx.Int64, + arg_idx: fx.Int64, + arg_ks: fx.Int64, + arg_vs: fx.Int64, + i32_m: fx.Int32, + a_scale_bytes: fx.Int32, + i32_iq_st: fx.Int32, + i64_kv_st: fx.Int64, + i64_idx_st: fx.Int64, + i32_cs_rows: fx.Int32, + i32_eps: fx.Int32, + stream: fx.Stream, + ): + kernel( + i32_m, a_scale_bytes, arg_w, arg_ws, arg_as, arg_a, arg_slab, arg_cnt, i32_cs_rows, arg_pos, arg_slot, + arg_islot, arg_cs, arg_qw, arg_kw, arg_iqw, arg_ikw, arg_q, arg_iq, arg_kv, arg_idx, arg_ks, arg_vs, + i64_kv_st, i64_idx_st, i32_iq_st, i32_eps, + ).launch(grid=(KS * NB, 1, 1), block=(64 * NWAVE, 1, 1), stream=stream) + + launch.kernel_name = name + launch.compile_hints = PRELOAD_HINTS + return launch diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/packing.py b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/packing.py new file mode 100644 index 0000000..49b1956 --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/packing.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Pack four or eight rows of four FP4 nibbles into four token-major V page words (the TP4 entry's ``packing.py``, +expanded form only). + +Each row's four nibbles are first spread to the low nibble of each byte (``_v_expand``); pairs of rows are merged into +bytes and the bytes transposed with byte permutes. The four output words contain the same nibbles as inserting each +row's bits one row at a time. +""" + +import flydsl.expr as fx + + +def _v_perm(hi, lo, select): + """Select four bytes from (hi:lo); the native byte-permute instruction implements the exact integer mapping.""" + return fx.Int32(fx.rocdl.perm_b32(fx.Int32(hi), fx.Int32(lo), fx.Int32(select))) + + +def _v_expand(nibbles): + """Four low-16-bit nibbles -> the low nibble of each byte, before the LDS join.""" + value = (nibbles | (nibbles << 8)) & 0x00FF00FF + return (value | (value << 4)) & 0x0F0F0F0F + + +def _v_expanded_words(nibbles): + """Four/eight expanded rows -> page words, each holding four/eight tokens of one dimension (high bits zero for + four tokens).""" + pairs = [nibbles[i] | (nibbles[i + 1] << 4) for i in range(0, len(nibbles), 2)] + x = _v_perm(pairs[1], pairs[0], 0x05010400) + y = _v_perm(pairs[1], pairs[0], 0x07030602) + if len(nibbles) == 4: + return [x & 0xFFFF, (x >> 16) & 0xFFFF, y & 0xFFFF, (y >> 16) & 0xFFFF] + u = _v_perm(pairs[3], pairs[2], 0x05010400) + v = _v_perm(pairs[3], pairs[2], 0x07030602) + return [_v_perm(u, x, 0x05040100), _v_perm(u, x, 0x07060302), + _v_perm(v, y, 0x05040100), _v_perm(v, y, 0x07060302)] diff --git a/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/reference.py b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/reference.py new file mode 100644 index 0000000..6054ab0 --- /dev/null +++ b/amd_kernels/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/reference.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The torch contract of ``mxfp8_qkv_norm_rope_nvfp4_insert`` (root README, rule 3): the QKV GEMM's contract +(``amd_kernels/gemm/mxfp8_common/reference.py``) followed by the norm / RoPE / NVFP4 insert's +(``amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/reference.py``), both imported unchanged. + +1. ``qkv = A @ W^T``: the exact values of the MXFP8 activation (row-major E4M3 codes + E8M0 exponents, per 1x32 + block along K) and weight, accumulated in fp32, rounded once to bf16 (``mxfp8_gemm_reference``). +2. ``norm_rope_nvfp4_insert(qkv, ...)``: per token and head the Gemma RMS norm, partial NeoX RoPE (rotary 64), bf16 + q, e4m3 index-q, NVFP4 K / V / index-K into 128-token pages (that reference's docstring: layouts, rounding). + +Bytes (the entry's README): bit for bit the deferred chain (``gemm.mxfp8_gemm_shuffled(defer_reduce=True)`` + +``splitk_reduce_norm_rope_nvfp4_insert``) at every row count; against this reference the GEMM's K split and +summation order move the bf16 QKV in its last bit now and then, so the tests hold the insert reference's +tolerances (``COS_MIN``, ``REL_L2``, ``PAGE_REL_L2``, ``CODE_MISMATCH``, re-exported here), except index-q: its e4m3 +(3 mantissa bits) turns a moved bf16 input into a flip of 1/8 now and then, so it holds ``INDEX_Q_REL_L2`` +(measured on the tests' inputs: relative L2 <= 1.1e-3 at 1-256 rows, the deferred chain's bytes). +""" + +import torch + +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import reference as _insert +from amd_kernels.gemm.mxfp8_common.reference import mxfp8_gemm_reference + +COS_MIN = _insert.COS_MIN +REL_L2 = _insert.REL_L2 +PAGE_REL_L2 = _insert.PAGE_REL_L2 +CODE_MISMATCH = _insert.CODE_MISMATCH +INDEX_Q_REL_L2 = 5e-3 + + +def mxfp8_qkv_norm_rope_nvfp4_insert( + codes: torch.Tensor, # [m, 6144] e4m3, row-major + exponents: torch.Tensor, # [m, 192] e8m0, row-major + weight_codes: torch.Tensor, # [4992, 6144] e4m3, row-major (not preshuffled) + weight_exponents: torch.Tensor, # [4992, 192] e8m0 + q_norm_weight, k_norm_weight, index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, + slot_mapping, index_slot_mapping, kv_cache, index_cache, k_scale, v_scale, q_out, index_q_out, + *, num_heads: int, num_kv_heads: int, num_index_heads: int, eps: float, +) -> None: + """The op's contract: writes ``q_out``, ``index_q_out`` and the pages in place.""" + if codes.shape[0] == 0: + return + qkv = mxfp8_gemm_reference(codes, exponents, weight_codes, weight_exponents, torch.bfloat16) + _insert.norm_rope_nvfp4_insert( + qkv, q_norm_weight, k_norm_weight, index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, + slot_mapping, index_slot_mapping, kv_cache, index_cache, k_scale, v_scale, q_out, index_q_out, + num_heads=num_heads, num_kv_heads=num_kv_heads, num_index_heads=num_index_heads, eps=eps) diff --git a/amd_kernels/attention/norm_rope_fp8_insert_d128_rot64/README.md b/amd_kernels/attention/norm_rope_fp8_insert_d128_rot64/README.md index 7817472..708d6e8 100644 --- a/amd_kernels/attention/norm_rope_fp8_insert_d128_rot64/README.md +++ b/amd_kernels/attention/norm_rope_fp8_insert_d128_rot64/README.md @@ -7,11 +7,11 @@ over dims 0-63 of 128 (halves of 32), and the e4m3 K / V insert into the cache p Caller: the MiniMax-M3.1 DSpark draft (5 layers): the query block's fused qkv row and the context rows' fused K / V row, which feeds all 5 layers' caches. Cache `[pages, kv_heads, page, 256]` e4m3 (128 K bytes, then 128 V bytes per token), e4m3 of `value / scale` saturated at +-448, per-tensor fp32 scales; `cos_sin_cache [rows, 64]` bf16 (cos | -sin). Served: TP4 16 q / 1 KV heads, page 128. Contract: `reference.py`. The qkv norm takes `rstd` and the products -in fp64 (as Inductor compiles the Triton norm); the multi-layer norm stays in fp32 (the eager Triton norm), so the two -K paths can differ by one bf16 ulp. Every output is within one bf16 ulp / e4m3 step of the reference or `ROW_ATOL` -(2^-6) of its row maximum, at least `EXACT_MIN` (99.9%) bit-equal; cache bytes outside the written slots stay -unchanged. +sin). Served: TP4 16 q / 1 KV and TP2 32 q / 2 KV heads, page 128. Contract: `reference.py`. The qkv norm takes +`rstd` and the products in fp64 (as Inductor compiles the Triton norm); the multi-layer norm stays in fp32 (the eager +Triton norm), so the two K paths can differ by one bf16 ulp. Every output is within one bf16 ulp / e4m3 step of the +reference or `ROW_ATOL` (2^-6) of its row maximum, at least `EXACT_MIN` (99.9%) bit-equal; cache bytes outside the +written slots stay unchanged. ## Entry points @@ -38,7 +38,9 @@ Bitwise the Triton / C++ chain of the same ops. | Kernel | Heads | Page | Rows -> us | |---|---|---|---| | qkv | 16 q / 1 KV | 128 | 4 / 64 / 256 / 1024 -> 2.00 / 2.65 / 3.37 / 7.42 | +| qkv | 32 q / 2 KV | 128 | 4 / 64 / 256 / 1024 / 2048 -> 1.95 / 2.85 / 4.67 / 12.38 / 21.83 | | multi-layer, 5 layers | 1 KV | 128 | 4 / 384 / 4096 -> 1.95 / 2.91 / 12.93 | +| multi-layer, 5 layers | 2 KV | 128 | 4 / 384 / 4096 / 8192 -> 1.94 / 3.88 / 24.47 / 46.29 | ## Tests diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/README.md b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/README.md index 0235521..9ab2100 100644 --- a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/README.md +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/README.md @@ -5,11 +5,11 @@ The attention input of a MiniMax-M3.1 sparse attention layer with an NVFP4 KV cache, in one launch: per token the Gemma RMS norm and partial NeoX RoPE of the q, k, index-q and index-k heads in fp32, q to bf16, index-q to e4m3, and the NVFP4 inserts of K, V (not normed) and index-K. Input `qkv [n, (nq + 2 nkv + niq + 1) * 128]` bf16 (q, k, v, -index-q, index-k heads) or the QKV GEMM's fp32 split-K partials; heads q16 kv1 iq1, rotary 64 (TP4). -128-token pages: main `[blocks, 2 nkv, 128, 72]` u8, index `[blocks, 128, 72]` u8 (64 code + 8 e4m3 scale bytes per -128 values, V token-paired, free block stride); unwritten tokens keep their bytes. Contract: `reference.py`; bitwise -the HIP norm / RoPE + Triton insert chain. Needs `eps >= 2^-126`, index-q row stride % 4 == 0, padding at the -batch tail only (a -1 slot inside a run of mapped tokens in one V octet races: strict xfail). +index-q, index-k heads) or the QKV GEMM's fp32 split-K partials; heads q16 kv1 iq1 (TP4, row 2560) or q32 kv2 iq2 +(TP2, row 4992), rotary 64. 128-token pages: main `[blocks, 2 nkv, 128, 72]` u8, index `[blocks, 128, 72]` u8 (64 code ++ 8 e4m3 scale bytes per 128 values, V token-paired, free block stride); unwritten tokens keep their bytes. Contract: +`reference.py`; bitwise the HIP norm / RoPE + Triton insert chain. Needs `eps >= 2^-126`, index-q row stride % 4 == 0, +padding at the batch tail only (a -1 slot inside a run of mapped tokens in one V octet races: strict xfail). ## Entry points @@ -21,27 +21,33 @@ batch tail only (a -1 slot inside a run of mapped tokens in one V octet races: s - `splitk_reduce_norm_rope_nvfp4_insert`: the same from the QKV GEMM's fp32 split-K slab (1-8 splits), summed `((0 + s0) + s1) + ...` and rounded once to bf16 as the split-K reduce stores it. - `mxfp8_gemm_qkv_norm_rope_nvfp4_insert`: from the QKV GEMM's MXFP8 input: the one-launch entry - `mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128` where it applies, else the GEMM and the insert (bitwise - either way). A caller that captures each layer segment on its own (breakable PIECEWISE) turns the one launch off. -- `warmup_prefill_kernels`: none of these is a `warm_up` entry. Call it at init: graph capture reaches only the main - kernel, so a >= 1537-token step would compile on the forward path. On a layer `insert_blocker` rejects it compiles - nothing and returns the reason. + `mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128` (TP4) or `..._n4992_k6144_rot64_p128` (TP2) where it + applies, else the GEMM and the insert (bitwise either way). A caller that captures each layer segment on its own + (breakable PIECEWISE) turns the one launch off. +- `warmup_prefill_kernels`: none of these is a `warm_up` entry. Call it at init: graph capture reaches only the decode + tiles, so a prefill step would compile on the forward path. On a layer `insert_blocker` rejects it compiles nothing + and returns the reason. -`host.py` picks the kernel (main, streaming q stores, prefill) and the token pairs per wave by row count. +`host.py` picks the kernel and the token pairs per wave by heads and row count. ## Performance -Graph replay, us per call; the chain is the QKV GEMM + this insert (layer 39 rank 0), the insert alone HBM-cold. +Graph replay, us per call; the chain is the QKV GEMM + this insert (layer 39 rank 0), the insert alone HBM-cold. TP2 +baseline: vLLM's FlyDSL `minimax_m3_norm_rope_nvfp4_insert` (the stock path at TP2); TP2 decode rows are DSpark-7 +requests of 8 rows. | Shape | Rows -> us | |---|---| | TP4 chain | 289 / 384 / 512 / 1024 / 1536 / 2048 / 4096 / 8192 / 16384 / 32768 -> 18.19 / 19.28 / 21.35 / 28.76 / 37.66 / 44.24 / 66.68 / 124.94 / 227.37 / 415.28 | -| Insert alone | 1 / 4 / 64 / 172 / 256 -> 2.21 / 2.42 / 2.75 / 3.03 / 3.31; 32768 -> 61.76 | +| TP4 insert | 1 / 4 / 64 / 172 / 256 -> 2.21 / 2.42 / 2.75 / 3.03 / 3.31; 32768 -> 61.76 | +| TP2 insert | 8 / 64 / 128 / 256 / 320 / 384 / 416 / 512 / 2048 / 8192 / 32768 -> 2.68 / 3.31 / 3.63 / 4.25 / 4.72 / 4.75 / 4.85 / 5.89 / 10.59 / 30.32 / 127.21 | +| TP2 baseline | the same rows -> 2.96 / 3.89 / 4.40 / 5.20 / 5.51 / 5.97 / 6.35 / 7.38 / 15.35 / 49.80 / 188.93 | ## Tests ```bash python3 -m pytest tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128 -q python3 benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py # also --reduce +python3 benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py --heads 32 2 2 --decode-run 8 # also --baseline python3 benchmarks/gemm/mxfp8_n2560_k6144/benchmark_qkv_insert_stores.py --checkpoint --baseline ``` diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/host.py b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/host.py index 0f7b3b4..dbba654 100644 --- a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/host.py +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/host.py @@ -9,6 +9,7 @@ from . import kernel as _kernel from . import kernel_q16_kv1_iq1_streaming as _kernel_streaming +from . import kernel_q32_kv2_iq2_vsplit as _kernel_vsplit from . import kernel_prefill_q64_kv4_iq4 as _kernel_prefill from . import splitk_reduce_kernel_q64_kv4_iq4 as _reduce_kernel @@ -24,8 +25,10 @@ def _run_compiled(exe, *args): cf(*args) -# (q heads, KV heads, index heads, rotary dim) of the layer at TP4, the one head set the entries take -HEADS = (16, 1, 1, 64) +ROT = 64 +TP4 = (16, 1, 1) # (q heads, KV heads, index heads) per rank +TP2 = (32, 2, 2) +SERVED = (TP4, TP2) def insert_blocker( @@ -41,23 +44,23 @@ def insert_blocker( cos_sin_cache: torch.Tensor | None = None, ) -> str | None: """Why a sparse attention layer (this rank's heads) does not fit the insert and its one-call QKV forms (None: it - fits): 16 query, 1 KV and 1 index head of 128 dims (TP4), NeoX RoPE on dims 0-63, an e4m3 index-q, a contiguous - bf16 cos / sin cache [rows, 64]. The entries assert the same.""" - if (num_heads, num_kv_heads, num_index_heads) != HEADS[:3]: + fits): 16 query, 1 KV and 1 index head (TP4) or 32, 2 and 2 (TP2) of 128 dims, NeoX RoPE on dims 0-63, an e4m3 + index-q, a contiguous bf16 cos / sin cache [rows, 64]. The entries assert the same.""" + if (num_heads, num_kv_heads, num_index_heads) not in SERVED: return ( f"{num_heads} query / {num_kv_heads} KV / {num_index_heads} index heads per rank " - f"(q{HEADS[0]} kv{HEADS[1]} iq{HEADS[2]}, TP4)" + f"(q16 kv1 iq1 at TP4, q32 kv2 iq2 at TP2)" ) if (head_dim, index_head_dim) != (_kernel.D, _kernel.D): return f"head dim {head_dim}, index head dim {index_head_dim} ({_kernel.D})" - if rotary_dim != HEADS[3] or not neox: - return f"RoPE on {rotary_dim} dims{'' if neox else ', not NeoX'} (NeoX on {HEADS[3]})" + if rotary_dim != ROT or not neox: + return f"RoPE on {rotary_dim} dims{'' if neox else ', not NeoX'} (NeoX on {ROT})" if index_q_dtype != torch.float8_e4m3fn: return f"index-q {index_q_dtype} (e4m3)" c = cos_sin_cache - if c is not None and (c.dtype != torch.bfloat16 or c.dim() != 2 or c.shape[1] != HEADS[3] + if c is not None and (c.dtype != torch.bfloat16 or c.dim() != 2 or c.shape[1] != ROT or not c.is_contiguous()): - return f"cos / sin cache {tuple(c.shape)} {c.dtype} (contiguous bf16 [rows, {HEADS[3]}])" + return f"cos / sin cache {tuple(c.shape)} {c.dtype} (contiguous bf16 [rows, {ROT}])" return None @@ -81,6 +84,11 @@ def _get_streaming_q16_kernel(): return _kernel_streaming.compile_streaming_q16_kv1_iq1() +@functools.cache +def _get_vsplit_q32_kernel(): + return _kernel_vsplit.compile_norm_rope_insert_q32_kv2_iq2() + + def _token_pairs_per_wave(n: int) -> int: return 1 if n <= 512 else 4 @@ -111,6 +119,32 @@ def _prefill_token_pairs(n): return _main_token_pairs(n) +def _tp4_launcher(n: int, prefill: bool): + tp = _main_token_pairs(n) + if _use_prefill(n, prefill, tp): + return _get_prefill_kernel(*TP4, ROT, _prefill_token_pairs(n)) + if 289 <= n <= 384: + return _get_streaming_q16_kernel() + return _get_kernel(*TP4, ROT, tp) + + +def _tp2_launcher(n: int, prefill: bool): + # V-split tile at 8-512 rows (QKV chain -0.3..-1.7 us vs one pair, +0.1 us at 1 row); above 4 prefill pairs -2..-12% + if n < 8 and not prefill: + return _get_kernel(*TP2, ROT, 1) + if n <= 512 and not prefill: + return _get_vsplit_q32_kernel() + return _get_prefill_kernel(*TP2, ROT, 4) + + +def _launcher(n: int, heads: tuple, prefill: bool = False, split_k: int = 0): + """The compiled launcher a call of ``n`` rows takes (``split_k``: the split-K form).""" + if split_k: + pairs = _reduce_token_pairs(n, split_k) if heads == TP4 else _tp2_reduce_token_pairs(n) + return _get_reduce_kernel(*heads, ROT, pairs, split_k) + return _tp4_launcher(n, prefill) if heads == TP4 else _tp2_launcher(n, prefill) + + def norm_rope_nvfp4_insert( qkv: torch.Tensor, # [n, (nq + 2 nkv + niq + 1) * 128] bf16 q_norm_weight: torch.Tensor, @@ -155,15 +189,8 @@ def norm_rope_nvfp4_insert( assert kv_cache.stride(1) == _kernel.BLK * 72 and kv_cache.stride(3) == 1 assert index_cache.shape[1:] == (_kernel.BLK, 72) and index_cache.stride(2) == 1 assert index_cache.stride(1) == 72 - tp = _main_token_pairs(n) - if _use_prefill(n, prefill, tp): - exe = _get_prefill_kernel(nq, nkv, niq, rotary_dim, _prefill_token_pairs(n)) - elif 289 <= n <= 384: - exe = _get_streaming_q16_kernel() - else: - exe = _get_kernel(nq, nkv, niq, rotary_dim, tp) _run_compiled( - exe, + _launcher(n, (nq, nkv, niq), prefill), qkv.data_ptr(), q_out.data_ptr(), index_q_out.data_ptr(), @@ -201,6 +228,11 @@ def _reduce_token_pairs(n, split_k): return 1 if n <= 32 else (2 if n <= 512 else 4) +def _tp2_reduce_token_pairs(n): + # two pairs from 161 rows: -1..-11% at 192-768 rows (four pairs win nowhere by more than noise) + return 1 if n <= 160 else 2 + + def splitk_reduce_norm_rope_nvfp4_insert( slab: torch.Tensor, # [sk, n, (nq + 2 nkv + niq + 1) * 128] fp32 QKV partials split_k: int, @@ -245,9 +277,8 @@ def splitk_reduce_norm_rope_nvfp4_insert( assert kv_cache.stride(1) == _kernel.BLK * 72 and kv_cache.stride(3) == 1 assert index_cache.shape[1:] == (_kernel.BLK, 72) and index_cache.stride(2) == 1 assert index_cache.stride(1) == 72 - token_pairs = _reduce_token_pairs(n, split_k) _run_compiled( - _get_reduce_kernel(nq, nkv, niq, rotary_dim, token_pairs, split_k), + _launcher(n, (nq, nkv, niq), split_k=split_k), slab.data_ptr(), q_out.data_ptr(), index_q_out.data_ptr(), @@ -280,7 +311,7 @@ def warmup_prefill_kernels( rotary_dim: int = 64, device=None, ) -> str | None: - """Compiles the prefill variants at init (root README rule 4: graph capture reaches only the main kernel) by + """Compiles the prefill variants at init (root README rule 4: graph capture reaches only the decode tiles) by launching each tile on a zero batch with slots -1, which writes no page. A layer ``insert_blocker`` rejects compiles nothing: returns that reason (the caller keeps its stock path), else None.""" device = device or torch.device("cuda") @@ -288,7 +319,7 @@ def warmup_prefill_kernels( why = insert_blocker(nq, nkv, niq, rotary_dim) if why is not None: return why - for n in (PREFILL_TOKENS, STREAMING_PREFILL_RANGE[0]): + for n in (PREFILL_TOKENS, STREAMING_PREFILL_RANGE[0]) if (nq, nkv, niq) == TP4 else (513,): row = (nq + 2 * nkv + niq + 1) * _kernel.D qkv = torch.zeros((n, row), dtype=torch.bfloat16, device=device) w = torch.zeros(_kernel.D, dtype=torch.bfloat16, device=device) diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_prefill_q64_kv4_iq4.py b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_prefill_q64_kv4_iq4.py index d2da5fd..a436116 100644 --- a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_prefill_q64_kv4_iq4.py +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_prefill_q64_kv4_iq4.py @@ -37,6 +37,9 @@ ) +NT_Q_TILES = ((16, 1, 1, 64, 4), (16, 1, 1, 64, 8), (32, 2, 2, 64, 4)) # (nq, nkv, niq, rotary_dim, pairs) + + def _half_lane(v, k, lane): """Lane k (lanes < 32) / lane 32 + k (lanes >= 32) of the i32 ``v``: the value the half-wave's own lane k holds.""" @@ -78,8 +81,8 @@ def compile_norm_rope_insert(nq: int, nkv: int, niq: int, rotary_dim: int, tp: i HALF = rotary_dim // 2 SIDE = nkv * BLK * 72 # one side of a main page (bytes) TPW = 2 * tp # tokens per wave - # nt q / index-q stores on the q16 bf16 tiles: QKV chain 2048 / 3072 / 6144 tokens -2.7 / -3.3 / -1.4% - Q_STORE_CM = 2 if (nq, nkv, niq, rotary_dim) == (16, 1, 1, 64) and tp in (4, 8) else 0 + # nt q / index-q stores: q16 QKV chain -1.4..-3.3% at 2048-6144 tokens, q32 insert -2..-12% at 513-32768 + Q_STORE_CM = 2 if (nq, nkv, niq, rotary_dim, tp) in NT_Q_TILES else 0 name = ( f"norm_rope_nvfp4_insert_prefill_d128_p128_q{nq}_kv{nkv}_iq{niq}" f"_rot{rotary_dim}_pairs{tp}" diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_q32_kv2_iq2_vsplit.py b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_q32_kv2_iq2_vsplit.py new file mode 100644 index 0000000..2fcd8e2 --- /dev/null +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/kernel_q32_kv2_iq2_vsplit.py @@ -0,0 +1,442 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``kernel.py``'s main kernel for q32 kv2 iq2 (TP2) with nt q / index-q stores and the V run split over a wave's +two halves: the same bytes. + +A V wave holds two tokens, one per 32-lane half; the half whose token starts a run of tokens in one octet writes the +run. With DSpark-7 decode (8-token requests) most runs are 5-8 tokens, and quantizing them row after row bounds the +kernel. Here a half that starts no run of its own quantizes rows 4 .. 7 of the other half's run while that half does +rows 0 .. 3; the writer ORs the helper's nibbles and scale bytes (v_permlane32_swap) into its own, which touch disjoint +bits, then writes the octet as ``kernel.py`` does. Where both halves start a run, the long one does all its rows. +Every lane loads its octet's old bytes before the quantization and the quantization has no branch around it: inside +a branch the compiler issues those loads after it (+0.3 us at 256 DSpark-7 rows). Runs of <= 4 tokens pay for the +speculative rows (+0.05..0.15 us in the QKV chain at 8-256 rows). +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .kernel import ( + BLK, + F448, + F448N, + NW, + PRELOAD_HINTS, + D, + _bf16x4, + _f32, + _gptr, + _iq_fp8, + _k_scale_off, + _maxnum, + _minnum, + _norm_rope_out4, + _pk_bf16, + _quant4, + _v_row, +) + + +def _other_half(v, upper): + """The i32 that lane l ^ 32 holds (v_permlane32_swap).""" + pair_ty = _ir.Type.parse("!llvm.struct<(i32, i32)>") + res = _rocdl_ops.permlane32_swap(pair_ty, fx.as_ir_value(fx.Int32(v)), fx.as_ir_value(fx.Int32(v)), False, False) + lo = fx.Int32(_llvm.extractvalue(T.i32, res, [0])) + hi = fx.Int32(_llvm.extractvalue(T.i32, res, [1])) + return upper.select(lo, hi) + + +def compile_norm_rope_insert_q32_kv2_iq2(): + """The launcher (one token pair per wave); arguments as ``kernel.compile_norm_rope_insert``'s launcher's.""" + nq, nkv, niq, rotary_dim, tp = 32, 2, 2, 64, 1 + NR = nq + nkv + 1 + niq # slots that norm + rope + S = NR + nkv * tp # waves per 2 * tp tokens + ROW = (nq + 2 * nkv + niq + 1) * D # qkv row (elements) + HALF = rotary_dim // 2 + SIDE = nkv * BLK * 72 # one side of a main page (bytes) + TPW = 2 * tp # tokens per wave + Q_STORE_CM = 2 # nt q / index-q stores (kernel.NT_Q_TILES) + name = f"norm_rope_nvfp4_insert_d128_p128_q{nq}_kv{nkv}_iq{niq}_rot{rotary_dim}_pairs{tp}_vsplit_qnt" + + # the first loads' arguments lead: the first 56 bytes arrive preloaded in SGPRs + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + i32_n: fx.Int32, + i32_cs_rows: fx.Int32, + arg_qkv: fx.Int64, # [n, ROW] bf16 + arg_pos: fx.Int64, # positions [n] i64 + arg_slot: fx.Int64, # main slots [n] i64, -1 skips + arg_islot: fx.Int64, # index slots [n] i64, -1 skips + arg_cs: fx.Int64, # cos_sin_cache [i32_cs_rows, rotary_dim] bf16 + arg_qw: fx.Int64, # norm weights [128] bf16: q, k, index-q, index-k + arg_kw: fx.Int64, + arg_iqw: fx.Int64, + arg_ikw: fx.Int64, + arg_q: fx.Int64, # q_out [n, nq * 128] bf16 + arg_iq: fx.Int64, # index_q_out rows of niq * 128 e4m3, row stride i32_iq_st + arg_kv: fx.Int64, # main pages [blocks, 2 * nkv, 128, 72] u8, stride i64_kv_st + arg_idx: fx.Int64, # index pages [blocks, 128, 72] u8, stride i64_idx_st + arg_ks: fx.Int64, # k / v global scales, fp32 scalars + arg_vs: fx.Int64, + i64_kv_st: fx.Int64, # bytes + i64_idx_st: fx.Int64, # bytes + i32_iq_st: fx.Int32, # bytes + i32_eps: fx.Int32, # fp32 bits + ): + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wid = bx * NW + wave + s = wid % S + hl = lane % 32 + hi16 = (hl & 16) != 0 + dim0 = hl * 4 + toks = [(wid // S) * TPW + t * 2 + lane // 32 for t in range_constexpr(tp)] + lives = [tk < i32_n for tk in toks] + + qkv = buffer_ops.create_buffer_resource_from_addr( + arg_qkv, num_records_bytes=fx.Int64(i32_n) * (ROW * 2) + ) + sl_r = buffer_ops.create_buffer_resource_from_addr( + arg_slot, num_records_bytes=fx.Int64(i32_n) * 8 + ) + + if s < NR: + is_q = s < nq + is_k = (s >= nq) & (s < nq + nkv) + is_ik = s == nq + nkv + is_kq = is_k | is_ik + # the head's column (in heads) of the qkv row + col = (s <= nq + nkv).select( + is_ik.select(fx.Int32(nq + 2 * nkv + niq), s), + s - (nq + nkv + 1) + (nq + 2 * nkv), + ) + w_addr = is_q.select( + arg_qw, is_k.select(arg_kw, is_ik.select(arg_ikw, arg_iqw)) + ) + wr = buffer_ops.create_buffer_resource_from_addr( + w_addr, num_records_bytes=D * 2 + ) + pos_r = buffer_ops.create_buffer_resource_from_addr( + arg_pos, num_records_bytes=fx.Int64(i32_n) * 8 + ) + isl_r = buffer_ops.create_buffer_resource_from_addr( + arg_islot, num_records_bytes=fx.Int64(i32_n) * 8 + ) + cs_r = buffer_ops.create_buffer_resource_from_addr( + arg_cs, num_records_bytes=fx.Int64(i32_cs_rows) * (rotary_dim * 2) + ) + xws, poss, slots = [], [], [] + for t in range_constexpr(tp): + xws.append( + Vec( + buffer_ops.buffer_load( + qkv, + toks[t] * (ROW // 2) + col * (D // 2) + hl * 2, + vec_width=2, + dtype=fx.Int32, + mask=lives[t], + ) + ) + ) + poss.append( + fx.Int32( + buffer_ops.buffer_load( + pos_r, + toks[t] * 2, + vec_width=1, + dtype=fx.Int32, + mask=lives[t], + ) + ) + ) + kslot = fx.Int32( + buffer_ops.buffer_load( + sl_r, + toks[t] * 2, + vec_width=1, + dtype=fx.Int32, + mask=lives[t] & is_k, + ) + ) + islot = fx.Int32( + buffer_ops.buffer_load( + isl_r, + toks[t] * 2, + vec_width=1, + dtype=fx.Int32, + mask=lives[t] & is_ik, + ) + ) + slots.append(is_k.select(kslot, islot)) + w = _bf16x4( + Vec(buffer_ops.buffer_load(wr, hl * 2, vec_width=2, dtype=fx.Int32)) + ) + first = dim0 < HALF + in_rope = dim0 < rotary_dim + ib = first.select(dim0, dim0 - HALF) + css = [] + for t in range_constexpr(tp): + cs_off = (poss[t] * rotary_dim + ib) // 2 + css.append( + ( + _bf16x4( + Vec( + buffer_ops.buffer_load( + cs_r, + cs_off, + vec_width=2, + dtype=fx.Int32, + mask=lives[t] & in_rope, + ) + ) + ), + _bf16x4( + Vec( + buffer_ops.buffer_load( + cs_r, + cs_off + HALF // 2, + vec_width=2, + dtype=fx.Int32, + mask=lives[t] & in_rope, + ) + ) + ), + ) + ) + eps = i32_eps.bitcast(fx.Float32) + qvs = [] # every pair first, so the scheduler can interleave them + for t in range_constexpr(tp): + out = _norm_rope_out4( + xws[t], css[t], w, eps, hi16, first, in_rope, HALF // 4 + ) + lo, hi = _pk_bf16(out[0], out[1]), _pk_bf16(out[2], out[3]) + qvs.append(Vec.from_elements([lo, hi], fx.Int32)) + if is_q: + q_r = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=fx.Int64(i32_n) * (nq * D * 2) + ) + for t in range_constexpr(tp): + buffer_ops.buffer_store( + qvs[t], + q_r, + toks[t] * (nq * D // 2) + s * (D // 2) + hl * 2, + mask=lives[t], + cache_modifier=Q_STORE_CM, + ) + if s > nq + nkv: + iq_r = buffer_ops.create_buffer_resource_from_addr( + arg_iq, num_records_bytes=fx.Int64(i32_n) * fx.Int64(i32_iq_st) + ) + iq_col = (s - (nq + nkv + 1)) * D + for t in range_constexpr(tp): + f = [ + _minnum(_maxnum(x, _f32(F448N)), _f32(F448)) + for x in _bf16x4(qvs[t]) + ] + word = _iq_fp8( + f[2], f[3], _iq_fp8(f[0], f[1], fx.Int32(0), False), True + ) + buffer_ops.buffer_store( + word, + iq_r, + toks[t] * i32_iq_st + iq_col + dim0, + mask=lives[t], + offset_is_bytes=True, + cache_modifier=Q_STORE_CM, + ) + if is_kq: + gs = is_k.select( + fx.Float32(_gptr(arg_ks, T.f32, 4)[0]), fx.Float32(1.0) + ) + inv_gs = fx.Float32(1.0) / gs + quants = [ + _quant4(_bf16x4(qvs[t]), gs, inv_gs) + for t in range_constexpr(tp) + ] + h = s - nq + grp = hl // 4 + for t in range_constexpr(tp): + slot = slots[t] + tp_ = slot & (BLK - 1) + page = is_k.select( + arg_kv + fx.Int64(slot >> 7) * i64_kv_st, + arg_idx + fx.Int64(slot >> 7) * i64_idx_st, + ) + d_off = ( + is_k.select(h * (BLK * 64), fx.Int32(0)) + tp_ * 64 + hl * 2 + ) + s_off = is_k.select( + nkv * BLK * 64 + h * (BLK * 8) + _k_scale_off(tp_, grp), + BLK * 64 + tp_ * 8 + grp, + ) + d_ptr = _gptr(page + fx.Int64(d_off), T.i16, 2) + s_ptr = _gptr(page + fx.Int64(s_off), T.i8, 1) + packed = quants[t][0].to(fx.Int16) + sf8 = quants[t][1].to(fx.Int8) + if lives[t] & (slot >= 0): + d_ptr[0] = packed + if hl % 4 == 0: + s_ptr[0] = sf8 + else: + vh = (s - NR) // tp + tok = (wid // S) * TPW + ((s - NR) % tp) * 2 + lane // 32 + live = tok < i32_n + vcol = (nq + nkv + vh) * (D // 2) # V head column (dwords) + vs = fx.Float32(_gptr(arg_vs, T.f32, 4)[0]) + near_t = tok - 7 + hl + has_near = (hl < 16) & (near_t >= 0) & (near_t < i32_n) + near = has_near.select( + fx.Int32( + buffer_ops.buffer_load( + sl_r, near_t * 2, vec_width=1, dtype=fx.Int32, mask=has_near + ) + ), + fx.Int32(-1), + ) + slot = fx.Int32( + buffer_ops.buffer_load( + sl_r, tok * 2, vec_width=1, dtype=fx.Int32, mask=live + ) + ) + has_prev = live & (tok > 0) + prev = has_prev.select( + fx.Int32( + buffer_ops.buffer_load( + sl_r, tok * 2 - 2, vec_width=1, dtype=fx.Int32, mask=has_prev + ) + ), + fx.Int32(-1), + ) + # the rows a run starting here or at the other half's token can cover, before the slots are known + rows = [ + Vec( + buffer_ops.buffer_load( + qkv, + (tok + r) * (ROW // 2) + vcol + hl * 2, + vec_width=2, + dtype=fx.Int32, + mask=tok + r < i32_n, + ) + ) + for r in range_constexpr(9) + ] + base = slot - slot % 8 + in_oct = (near >= base) & (near < base + 8) + votes = fx.Int64(fx.rocdl.ballot(T.i64, in_oct)) + mine = (votes >> fx.Int64((lane // 32) * 32)).to(fx.Int32) & 0xFFFF + n_run = fx.Int32(fx.math.ctpop(mine)) + run_first = (prev < 0) | ((prev >> 3) != (slot >> 3)) + tp_ = slot & (BLK - 1) + # every lane loads its octet's old bytes before the quantization; a lane without a slot reads block 0 + v_side = arg_kv + fx.Int64((slot >= 0).select(slot, fx.Int32(0)) >> 7) * i64_kv_st + SIDE + v_data = v_side + fx.Int64(vh * (BLK * 64) + (tp_ // 8) * (D * 4)) + v_sf = v_side + fx.Int64(nkv * BLK * 64 + vh * (BLK * 8) + tp_ - tp_ % 8) + dp = _gptr(v_data + fx.Int64(dim0 * 4), T.i32, 4) + sp = _gptr(v_sf + fx.Int64((hl // 4) * BLK), T.i32, 4) + old = [fx.Int32(dp[i]) for i in range_constexpr(4)] + old_sc = [fx.Int32(sp[i]) for i in range_constexpr(2)] + go = live & (slot >= 0) & run_first + inv_vs = fx.Float32(1.0) / vs + upper = lane >= 32 + go_votes = fx.Int64(fx.rocdl.ballot(T.i64, go)) + other_go = upper.select( + (go_votes & fx.Int64(0xFFFFFFFF)) != fx.Int64(0), (go_votes >> fx.Int64(32)) != fx.Int64(0) + ) + o_n_run = _other_half(n_run, upper) + o_p0 = _other_half(tp_ % 8, upper) + # a half that starts no run quantizes rows 4 .. 7 of the other half's run (its own rows 3 .. 6 or 5 .. 8) + helps = ~go & other_go & (o_n_run > 4) + alone = go & other_go & (n_run > 4) + r_n = helps.select(o_n_run, go.select(n_run, fx.Int32(0))) + r_p0 = helps.select(o_p0, tp_ % 8) + r_off = helps.select(fx.Int32(4), fx.Int32(0)) + st = [fx.Int32(0)] * 9 + for r in range_constexpr(4): + row = [ + helps.select(upper.select(fx.Int32(rows[3 + r][i]), fx.Int32(rows[5 + r][i])), fx.Int32(rows[r][i])) + for i in range_constexpr(2) + ] + st = _v_row(st, row, r_off + r, r_n, r_p0, vs, inv_vs) + # the helper's nibbles and scale bytes, disjoint from the writer's + merge = go & ~other_go + st = [x | merge.select(_other_half(x, upper), fx.Int32(0)) for x in st] + for take, rest in ((go & ~alone, False), (alone, True)): + if take: + full = st + if const_expr(rest): # both halves start a run: this one quantizes all of its rows + for r in range_constexpr(4, 8): + full = _v_row(full, rows[r], r, n_run, tp_ % 8, vs, inv_vs) + keep = ~full[4] + for i in range_constexpr(4): + dp[i] = full[i] | (old[i] & keep) + sc0 = full[5] | (old_sc[0] & ~full[7]) + sc1 = full[6] | (old_sc[1] & ~full[8]) + if hl % 4 == 0: + sp[0] = sc0 + sp[1] = sc1 + + @flyc.jit + def launch( + arg_qkv: fx.Int64, + arg_q: fx.Int64, + arg_iq: fx.Int64, + arg_qw: fx.Int64, + arg_kw: fx.Int64, + arg_iqw: fx.Int64, + arg_ikw: fx.Int64, + arg_cs: fx.Int64, + arg_pos: fx.Int64, + arg_slot: fx.Int64, + arg_islot: fx.Int64, + arg_kv: fx.Int64, + arg_idx: fx.Int64, + arg_ks: fx.Int64, + arg_vs: fx.Int64, + i32_n: fx.Int32, + i32_iq_st: fx.Int32, + i64_kv_st: fx.Int64, + i64_idx_st: fx.Int64, + i32_cs_rows: fx.Int32, + i32_eps: fx.Int32, + stream: fx.Stream, + ): + waves = (i32_n + (TPW - 1)) // TPW * S + kernel( + i32_n, + i32_cs_rows, + arg_qkv, + arg_pos, + arg_slot, + arg_islot, + arg_cs, + arg_qw, + arg_kw, + arg_iqw, + arg_ikw, + arg_q, + arg_iq, + arg_kv, + arg_idx, + arg_ks, + arg_vs, + i64_kv_st, + i64_idx_st, + i32_iq_st, + i32_eps, + ).launch( + grid=(fx.Int64((waves + (NW - 1)) // NW), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch.kernel_name = name + launch.compile_hints = PRELOAD_HINTS + return launch diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/qkv.py b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/qkv.py index b50afd1..a49ed9f 100644 --- a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/qkv.py +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/qkv.py @@ -5,8 +5,9 @@ - ``qkv_norm_rope_nvfp4_insert``: from the bf16 QKV rows (any projection). - ``mxfp8_gemm_qkv_norm_rope_nvfp4_insert``: from the MXFP8 codes of the QKV GEMM's input (``gemm.mxfp8_common``'s - pre-quantized layout): the one-launch entry ``mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128`` where it - applies (N 2560, K 6144, 16 / 1 / 1 heads, rotary 64, bf16, 1-1536 rows in the shuffled-code tile) and + pre-quantized layout): the one-launch entry ``mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128`` (N 2560, + 16 / 1 / 1 heads, 1-1536 rows) or ``..._n4992_k6144_rot64_p128`` (N 4992, 32 / 2 / 2 heads, 1-768 rows) where it + applies (K 6144, rotary 64, bf16, rows in the shuffled-code tile) and ``one_launch`` allows it; else the GEMM (where the codes are in the shuffled tile, its deferred split-K partials go to ``splitk_reduce_norm_rope_nvfp4_insert``) and the insert. Bitwise either way. @@ -14,6 +15,8 @@ device); a caller that captures every layer's segment on its own (breakable PIECEWISE graphs) passes False. """ +import importlib + import torch from .host import norm_rope_nvfp4_insert, splitk_reduce_norm_rope_nvfp4_insert @@ -59,12 +62,17 @@ def qkv_norm_rope_nvfp4_insert( return q, index_q -def _one_launch_entry(): +_ONE_LAUNCH = { + (2560, 6144): "amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128.host", + (4992, 6144): "amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128.host", +} + + +def _one_launch_entry(n, k): try: - from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n2560_k6144_rot64_p128 import host - except ImportError: + return importlib.import_module(_ONE_LAUNCH[(n, k)]) + except (KeyError, ImportError): return None - return host def mxfp8_gemm_qkv_norm_rope_nvfp4_insert( @@ -103,7 +111,7 @@ def mxfp8_gemm_qkv_norm_rope_nvfp4_insert( ins = (q_norm_weight, k_norm_weight, index_q_norm_weight, index_k_norm_weight, cos_sin_cache, positions, slot_mapping, index_slot_mapping, kv_cache, index_cache, k_scale, v_scale) kw = dict(num_heads=num_heads, num_kv_heads=num_kv_heads, num_index_heads=num_index_heads, eps=eps) - f = _one_launch_entry() if one_launch else None + f = _one_launch_entry(n, k) if one_launch else None if ( f is not None and out_dtype == torch.bfloat16 diff --git a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/splitk_reduce_kernel_q64_kv4_iq4.py b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/splitk_reduce_kernel_q64_kv4_iq4.py index 07c26c6..baf00cc 100644 --- a/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/splitk_reduce_kernel_q64_kv4_iq4.py +++ b/amd_kernels/attention/norm_rope_nvfp4_insert_d128_rot64_p128/splitk_reduce_kernel_q64_kv4_iq4.py @@ -35,6 +35,11 @@ ) +# measured on the insert alone at q32: early slab loads -1..-2% at 192-768 rows, nt q / index-q stores -4..-11% at 8-768 +EARLY_NKV = (1, 2) # KV heads whose V waves load the slabs early +NT_Q_TILES = ((32, 2, 2, 64, 1), (32, 2, 2, 64, 2)) # (nq, nkv, niq, rotary_dim, pairs) + + def compile_reduce_insert( nq: int, nkv: int, niq: int, rotary_dim: int, tp: int, sk: int ): @@ -52,10 +57,15 @@ def compile_reduce_insert( HALF = rotary_dim // 2 SIDE = nkv * BLK * 72 # one side of a main page (bytes) TPW = 2 * tp # tokens per wave + # >= 2 pairs: V waves issue every slab load a run may need before the slot loads (split-K consumer) + EARLY = tp >= 2 and nkv in EARLY_NKV + Q_STORE_CM = 2 if (nq, nkv, niq, rotary_dim, tp) in NT_Q_TILES else 0 name = ( f"splitk_reduce_norm_rope_nvfp4_insert_d128_p128_q{nq}_kv{nkv}_iq{niq}" f"_rot{rotary_dim}_pairs{tp}_s{sk}" ) + if Q_STORE_CM: + name += "_qnt" # the first loads' arguments lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) @@ -236,6 +246,7 @@ def row4(tok, col_elem, live): q_r, toks[t] * (nq * D // 2) + s * (D // 2) + hl * 2, mask=lives[t], + cache_modifier=Q_STORE_CM, ) if s > nq + nkv: iq_r = buffer_ops.create_buffer_resource_from_addr( @@ -256,6 +267,7 @@ def row4(tok, col_elem, live): toks[t] * i32_iq_st + iq_col + dim0, mask=lives[t], offset_is_bytes=True, + cache_modifier=Q_STORE_CM, ) if is_kq: gs = is_k.select( @@ -292,9 +304,7 @@ def row4(tok, col_elem, live): tok = (wid // S) * TPW + ((s - NR) % tp) * 2 + lane // 32 live = tok < i32_n vcol = (nq + nkv + vh) * D # V head column (elements) - # one V head, >= 2 pairs: every slab load a run may need before the slot loads (split-K consumer) - early = tp >= 2 and nkv == 1 - vfr = [load4(tok + r, vcol, tok + r < i32_n) for r in range_constexpr(8)] if early else [] + vfr = [load4(tok + r, vcol, tok + r < i32_n) for r in range_constexpr(8)] if EARLY else [] vs = fx.Float32(_gptr(arg_vs, T.f32, 4)[0]) near_t = tok - 7 + hl has_near = (hl < 16) & (near_t >= 0) & (near_t < i32_n) @@ -323,7 +333,7 @@ def row4(tok, col_elem, live): # the rows a run starting here can cover, before its slot is known rows = ( [sum4(vfr[r]) for r in range_constexpr(8)] - if early + if EARLY else [row4(tok + r, vcol, tok + r < i32_n) for r in range_constexpr(8)] ) base = slot - slot % 8 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/README.md b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/README.md index e6fe3ca..b5884d1 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/README.md +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/README.md @@ -5,42 +5,52 @@ MiniMax-M3.1's sparse decode attention over the NVFP4 K / V cache: each query row attends over the 16 128-token blocks the indexer picked (`topk_idx [Hkv, rows, topk]` int32 logical blocks, -1 none, the own block causal), the pages `[pages, 2 Hkv, 128, 72]` u8 (`reference.py`; free page stride) dequantized in registers; bf16 `[rows, H, 128]` -or the o_proj's MXFP8 input out. Served: TP4 1 KV / 16 q heads, 1 / 4 / 8 rows per request, 1-256 requests, compute -bf16, fp16 (fastest, below) or fp8. Contract `sparse_decode_reference` (fp64, `REL_L2` 5e-3, `COS` 0.99999, splits in -split order): bf16 K / V `e4m3_rn_satfinite(e2m1 x scale)` (NV's; rel L2 to exact K / V 3.1-3.3e-2), fp16 exact -(1.6-1.7e-3), fp8 NV Q8KV4, P per 32-key chunk; MXFP8 = vLLM's quantizer bytes. +or the o_proj's MXFP8 input out. Served: TP4 1 KV / 16 q heads (1-256 requests), TP2 2 KV / 32 q heads (1-128 +requests), 1 / 4 / 8 rows per request, compute bf16, fp16 (fastest, below) or fp8. Contract `sparse_decode_reference` +(fp64, `REL_L2` 5e-3, `COS` 0.99999, splits in split order): bf16 K / V `e4m3_rn_satfinite(e2m1 x scale)` (NV's; rel +L2 to exact K / V 3.1-3.3e-2), fp16 exact (1.6-1.7e-3), fp8 NV Q8KV4, P per 32-key chunk; MXFP8 = vLLM's quantizer +bytes. ## Entry points - `nvfp4_sparse_decode`: the attention of a step's decode rows (row i is row i % ql of request i // ql) into bf16 or the o_proj's MXFP8 input (row-major or in the decode GEMM's 16-row tiles); given the indexer's scores it also - writes `topk_idx` (bitwise `nvfp4_index_score_top16`, selected inside the attention at 1 KV head, bf16 / fp16, - <= 256 rows). An `amd_kernels.warm_up` entry: a mode's first call compiles 1-256 requests. Graph-safe. + writes `topk_idx` (bitwise `nvfp4_index_score_top16`, selected inside the attention at 1 / 2 KV heads up to 256 / + 1024 rows, index head h = KV head h, bf16 / fp16). An `amd_kernels.warm_up` entry: a mode's first call compiles + 1-256 requests. Graph-safe. - `host.warm_up_decode`: compiles both outputs at init (else a mixed step's bf16-output mode compiles late). - `index_decode.nvfp4_index_sparse_decode`: a sparse layer's decode block in one call (index top-16, then the - attention); returns its route, bitwise equal on every route: `split4` (four GPUs), `fused` (one GPU, bf16 / fp16), - `chain`. `index_decode.warm_up` compiles it. + attention); returns its route, bitwise equal on every route: `split4` (four GPUs), `split2` (two GPUs), `fused` (one + GPU, bf16 / fp16), `chain`. `index_decode.warm_up` compiles it. -`host.py` picks the variant, splits and grid by rows (`decode_splits`, the tail grid); `index_decode.route` the -route by request count. +`host.py` picks the variant, splits and grid by rows and KV heads (`_tuned_splits`, `decode_splits`, the tail grid); +`index_decode.route` the route by request count. ## Performance -TP4 (1 KV head), graph replay, 100 calls with own pages (K / V cold), 60K-100K, 25 distinct picks per 4 rows, us per -call, MXFP8 out, one process, two rounds (fp16 plain: fp16 without its small-batch schedule); bf16 out -0.2-0.6 us less: +Graph replay, 100 calls with own pages (K / V cold), 60K-100K, 25 distinct picks per 4 rows, us per call, MXFP8 out, +one process, two rounds (fp16 plain: fp16 without its small-batch schedule; TP2 untuned: the 2-KV-head launch without +its schedule, the general kernel); bf16 out 0.2-0.6 us less. TP2 at N requests = TP4 at 2N (the same (row, KV head) +pairs); vs AITER's Gluon sparse PA: 0.55-0.73x the time: | ql x requests | 1 | 2 | 4 | 8 | 16 | 32 | 64 | 128 | |---|---|---|---|---|---|---|---|---| -| 4: bf16 / fp16 plain / fp16 / fp8 | 8.46 / 10.35 / 8.15 / 8.78 | 8.90 / 10.59 / 8.59 / 9.28 | 9.55 / 11.43 / 9.24 / 9.87 | 10.19 / 9.82 / 9.85 / 10.52 | 11.03 / 10.39 / 10.42 / 11.46 | 13.57 / 12.45 / 12.44 / 13.85 | 18.18 / 16.58 / 16.58 / 18.55 | 29.03 / 26.92 / 26.97 / 28.49 | -| 8: bf16 / fp16 plain / fp16 / fp8 | 8.35 / 10.62 / 8.20 / 8.46 | 8.72 / 11.44 / 8.55 / 8.84 | 9.29 / 12.35 / 9.19 / 9.38 | 10.39 / 10.33 / 10.35 / 10.48 | 12.63 / 12.17 / 12.15 / 12.49 | 17.04 / 15.82 / 15.81 / 16.84 | 28.01 / 25.58 / 25.60 / 26.52 | | +| TP4 4: bf16 / fp16 plain / fp16 / fp8 | 8.46 / 10.35 / 8.15 / 8.78 | 8.90 / 10.59 / 8.59 / 9.28 | 9.55 / 11.43 / 9.24 / 9.87 | 10.19 / 9.82 / 9.85 / 10.52 | 11.03 / 10.39 / 10.42 / 11.46 | 13.57 / 12.45 / 12.44 / 13.85 | 18.18 / 16.58 / 16.58 / 18.55 | 29.03 / 26.92 / 26.97 / 28.49 | +| TP4 8: bf16 / fp16 plain / fp16 / fp8 | 8.35 / 10.62 / 8.20 / 8.46 | 8.72 / 11.44 / 8.55 / 8.84 | 9.29 / 12.35 / 9.19 / 9.38 | 10.39 / 10.33 / 10.35 / 10.48 | 12.63 / 12.17 / 12.15 / 12.49 | 17.04 / 15.82 / 15.81 / 16.84 | 28.01 / 25.58 / 25.60 / 26.52 | | +| TP2 4: fp16 untuned / fp16 | 10.63 / 8.21 | 11.41 / 8.56 | 12.59 / 9.49 | 10.42 / 10.38 | 12.44 / 12.32 | 16.89 / 16.74 | 27.74 / 27.25 | 47.18 / 47.34 | +| TP2 8: fp16 untuned / fp16 | 11.42 / 8.50 | 12.34 / 9.12 | 12.19 / 10.60 | 12.19 / 12.13 | 15.89 / 15.82 | 25.62 / 25.84 | 44.88 / 43.39 | 80.62 / 81.17 | -bf16 out, fp16 vs bf16: ql 8 x 8 +0.00..+0.08 us (three runs), elsewhere faster; with `index_score`: -0.04..-1.90 us -MXFP8 out, bf16 out ql 4 x 1-8 within +0.08 us. vs AITER's Gluon sparse PA: 0.55-0.73x the time. +TP2 ql 8 between the whole CU rounds (the tail grid): 24 / 40 / 52 requests untuned 23.25 / 34.13 / 44.17, fp16 +22.22 / 31.33 / 39.18 us. bf16 out, fp16 vs bf16: ql 8 x 8 +0.00..+0.08 us (three runs), elsewhere faster; with +`index_score`: -0.04..-1.90 us MXFP8 out, bf16 out ql 4 x 1-8 within +0.08 us. ## Tests -bf16 / fp16 / fp8; strict xfails: mixed steps without `warm_up_decode`, the MXFP8 subnormal scale byte. +bf16 / fp16 / fp8, 1 and 2 KV heads; strict xfails: mixed steps without `warm_up_decode`, the MXFP8 subnormal scale +byte. ```bash -python3 -m pytest tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16 -q # *_ranks4: 4 GPUs +python3 -m pytest tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16 -q # *_ranks4: 4 GPUs, *_ranks2: 2 python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_sparse_decode.py +python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py --hkv 2 +python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py --hkv 2 +python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py --ql 8 # 2 GPUs ``` diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch.py index 058580e..e492816 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``kernel.py`` at 8 rows per request, 16 query heads, 1 KV head, 8 XCDs, first loads before the table barrier. +"""``kernel.py`` at 8 rows per request, 16 query heads per KV head, 1 or 2 KV heads (``hkv``), 8 XCDs, first loads +before the table barrier. The request arithmetic is folded for that shape; splits 1, 2 or 4. The host takes it when the grid is at most one workgroup per CU. The numerics and every sum order are kernel.py's. @@ -20,13 +21,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 1, 2 or 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits in (1, 2, 4) SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_gqa16_d128_p128_top16_s{SPLITS}_xcd8_prefetch" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") ) @@ -134,10 +138,10 @@ def kernel( i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, ): - # the served shape: 8 rows per request, 16 query heads, 1 KV head + # the served shape: 8 rows per request, 16 query heads per KV head, HKV KV heads i32_ql = fx.Int32(8) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) smem = fx.SharedAllocator().allocate(Shared).peek() tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch_select.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch_select.py index 6b2e90e..f74a0c5 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch_select.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_prefetch_select.py @@ -27,13 +27,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch_select(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch_select(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 1, 2 or 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits in (1, 2, 4) SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_gqa16_d128_p128_top16_s{SPLITS}_xcd8_prefetch" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") + "_select16" ) @@ -117,6 +120,12 @@ def store_q(arg_qc, arg_qs, i32_rows, i32_H, i32_tile, row, kvh, lane, acc, inv) sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=row < i32_rows ) + def score_row(arg_score, i64_s_row, rowc, kvh): + """The address of row ``rowc``'s index scores of index head ``kvh``.""" + if HKV == 1: + return arg_score + rowc.to(fx.Int64) * i64_s_row * 4 + return arg_score + (rowc.to(fx.Int64) * (i64_s_row & 0xFFFFFFFF) + kvh.to(fx.Int64) * (i64_s_row >> 32)) * 4 + # the arguments of the first (dependent) loads lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) def kernel( @@ -145,14 +154,14 @@ def kernel( f32_c: fx.Float32, # sm_scale * log2(e) i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, - arg_score: fx.Int64, # [1, >= rows, >= width] fp32 index scores - i64_s_row: fx.Int64, # their row stride (elements) + arg_score: fx.Int64, # [Hkv, >= rows, >= width] fp32 index scores + i64_s_row: fx.Int64, # their row stride (elements); at 2 KV heads | the head stride << 32 i32_width: fx.Int32, # score_width(max_seq_len) of the indexer ): - # the served shape: 8 rows per request, 16 query heads, 1 KV head + # the served shape: 8 rows per request, 16 query heads per KV head, HKV KV heads i32_ql = fx.Int32(8) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) smem = fx.SharedAllocator().allocate(Shared).peek() tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y @@ -196,11 +205,11 @@ def tab(sc): # lane = top-k slot: its logical block and page (-1: none) slot_ok = lane < TOPK - # the row's top-16 from its index scores (one KV head = one index head), stored as the selector kernel stores + # the row's top-16 from its index scores (index head = KV head), stored as the selector kernel stores # it; padding rows redo row 0's id_row = arg_topk + (kvh * i32_topk_hs + rowc * TOPK).to(fx.Int64) * 4 fast = row_top16_prologue_ids( - tx, arg_score + rowc.to(fx.Int64) * i64_s_row * 4, i32_width, arg_seq, + tx, score_row(arg_score, i64_s_row, rowc, kvh), i32_width, arg_seq, rowc, i32_ql, smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, id_row, ) blk_lds = fx.Int32(smem.tmp.ptr[lane % TOPK]) diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed.py index 58f0c0e..84b4f41 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``kernel.py`` at 8 rows per request, 16 query heads, 1 KV head, 8 XCDs, with the request arithmetic folded. +"""``kernel.py`` at 8 rows per request, 16 query heads per KV head, 1 or 2 KV heads (``hkv``), 8 XCDs, with the request +arithmetic folded. Splits 2 or 4 and kernel.py's prologue (the table barrier before the first loads); the host takes it at <= 192 rows where the prefetch variant does not fit. The numerics and every sum order are kernel.py's. @@ -20,13 +21,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 2 or 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits in (2, 4) SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_gqa16_d128_p128_top16_s{SPLITS}_xcd8_q8_fixed" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") ) @@ -134,10 +138,10 @@ def kernel( i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, ): - # the served shape: 8 rows per request, 16 query heads, 1 KV head + # the served shape: 8 rows per request, 16 query heads per KV head, HKV KV heads i32_ql = fx.Int32(8) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) smem = fx.SharedAllocator().allocate(Shared).peek() tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed_select.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed_select.py index fa93db8..24c4f51 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed_select.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_fixed_select.py @@ -27,13 +27,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed_select(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed_select(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 2 or 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits in (2, 4) SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_gqa16_d128_p128_top16_s{SPLITS}_xcd8_q8_fixed" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") + "_select16" ) @@ -117,6 +120,12 @@ def store_q(arg_qc, arg_qs, i32_rows, i32_H, i32_tile, row, kvh, lane, acc, inv) sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=row < i32_rows ) + def score_row(arg_score, i64_s_row, rowc, kvh): + """The address of row ``rowc``'s index scores of index head ``kvh``.""" + if HKV == 1: + return arg_score + rowc.to(fx.Int64) * i64_s_row * 4 + return arg_score + (rowc.to(fx.Int64) * (i64_s_row & 0xFFFFFFFF) + kvh.to(fx.Int64) * (i64_s_row >> 32)) * 4 + # the arguments of the first (dependent) loads lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) def kernel( @@ -145,14 +154,14 @@ def kernel( f32_c: fx.Float32, # sm_scale * log2(e) i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, - arg_score: fx.Int64, # [1, >= rows, >= width] fp32 index scores - i64_s_row: fx.Int64, # their row stride (elements) + arg_score: fx.Int64, # [Hkv, >= rows, >= width] fp32 index scores + i64_s_row: fx.Int64, # their row stride (elements); at 2 KV heads | the head stride << 32 i32_width: fx.Int32, # score_width(max_seq_len) of the indexer ): - # the served shape: 8 rows per request, 16 query heads, 1 KV head + # the served shape: 8 rows per request, 16 query heads per KV head, HKV KV heads i32_ql = fx.Int32(8) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) smem = fx.SharedAllocator().allocate(Shared).peek() tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y @@ -196,11 +205,11 @@ def tab(sc): # lane = top-k slot: its logical block and page (-1: none) slot_ok = lane < TOPK - # the row's top-16 from its index scores (one KV head = one index head), stored as the selector kernel stores + # the row's top-16 from its index scores (index head = KV head), stored as the selector kernel stores # it; padding rows redo row 0's id_row = arg_topk + (kvh * i32_topk_hs + rowc * TOPK).to(fx.Int64) * 4 fast = row_top16_prologue_ids( - tx, arg_score + rowc.to(fx.Int64) * i64_s_row * 4, i32_width, arg_seq, + tx, score_row(arg_score, i64_s_row, rowc, kvh), i32_width, arg_seq, rowc, i32_ql, smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, id_row, ) blk_lds = fx.Int32(smem.tmp.ptr[lane % TOPK]) diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd.py index 071f156..70d7a46 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``kernel.py`` at 8 rows per request, 16 query heads, 1 KV head, 4 splits, rows spread over the 8 XCDs. +"""``kernel.py`` at 8 rows per request, 16 query heads per KV head, 1 or 2 KV heads (``hkv``), 4 splits, rows spread +over the 8 XCDs. Row bx runs on XCD bx % 8 (grid.x padded to 8, not 8 x ql), so a request's rows spread over the XCDs and a row's splits still share one; the first block's loads issue before the table barrier. The host takes it at <= 32 rows. The numerics @@ -21,13 +22,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits == 4 SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_sparse_gqa16_d128_p128_top{TOPK}_q8_rowxcd_s{SPLITS}_xcd{XCDS}" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") ) @@ -136,9 +140,9 @@ def kernel( i32_tpx: fx.Int32, ): smem = fx.SharedAllocator().allocate(Shared).peek() - # the served shape: 16 query heads, 1 KV head (8 rows per request below) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + # the served shape: 16 query heads per KV head, HKV KV heads (8 rows per request below) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py index e01ad25..6a6b857 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py @@ -27,13 +27,16 @@ ) -def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits: int, quant: bool): +def compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits == 4 SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_bf16mfma_sparse_gqa16_d128_p128_top{TOPK}_q8_rowxcd_s{SPLITS}_xcd{XCDS}" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") + "_select16" ) @@ -117,6 +120,12 @@ def store_q(arg_qc, arg_qs, i32_rows, i32_H, i32_tile, row, kvh, lane, acc, inv) sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=row < i32_rows ) + def score_row(arg_score, i64_s_row, rowc, kvh): + """The address of row ``rowc``'s index scores of index head ``kvh``.""" + if HKV == 1: + return arg_score + rowc.to(fx.Int64) * i64_s_row * 4 + return arg_score + (rowc.to(fx.Int64) * (i64_s_row & 0xFFFFFFFF) + kvh.to(fx.Int64) * (i64_s_row >> 32)) * 4 + # the arguments of the first (dependent) loads lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) def kernel( @@ -146,13 +155,13 @@ def kernel( i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, arg_score: fx.Int64, # [Hkv, >= rows, >= width] fp32 index scores - i64_s_row: fx.Int64, # their row stride (elements) + i64_s_row: fx.Int64, # their row stride (elements); at 2 KV heads | the head stride << 32 i32_width: fx.Int32, # score_width(max_seq_len) of the indexer ): smem = fx.SharedAllocator().allocate(Shared).peek() - # the served shape: 16 query heads, 1 KV head (8 rows per request below) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + # the served shape: 16 query heads per KV head, HKV KV heads (8 rows per request below) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 @@ -192,11 +201,11 @@ def tab(sc): # lane = top-k slot: its logical block and page (-1: none) slot_ok = lane < TOPK - # the row's top-16 from its index scores (one KV head = one index head), stored as the selector kernel stores + # the row's top-16 from its index scores (index head = KV head), stored as the selector kernel stores # it; padding rows redo row 0's id_row = arg_topk + (kvh * i32_topk_hs + rowc * TOPK).to(fx.Int64) * 4 fast = row_top16_prologue_ids( - tx, arg_score + rowc.to(fx.Int64) * i64_s_row * 4, i32_width, arg_seq, + tx, score_row(arg_score, i64_s_row, rowc, kvh), i32_width, arg_seq, rowc, fx.Int32(8), smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, id_row, ) blk_lds = fx.Int32(smem.tmp.ptr[lane % TOPK]) diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py index 938ae30..06ca205 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``kernel.py``'s fp16 path (exact K / V, fp16 Q and P) at 8 rows per request, 16 query heads, 1 KV head, 4 splits, -rows spread over the 8 XCDs. +"""``kernel.py``'s fp16 path (exact K / V, fp16 Q and P) at 8 rows per request, 16 query heads per KV head, 1 or 2 KV +heads (``hkv``), 4 splits, rows spread over the 8 XCDs. Row bx runs on XCD bx % 8 (grid.x padded to 8, not 8 x ql), so a request's rows spread over the XCDs and a row's splits still share one. The host takes it at <= 32 rows. The numerics and every sum order are kernel.py's fp16 path. @@ -20,13 +20,16 @@ ) -def compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits: int, quant: bool): +def compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits == 4 SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_f16mfma_sparse_gqa16_d128_p128_top{TOPK}_q8_rowxcd_s{SPLITS}_xcd{XCDS}" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") ) @@ -132,9 +135,9 @@ def kernel( i32_tpx: fx.Int32, ): smem = fx.SharedAllocator().allocate(Shared).peek() - # the served shape: 16 query heads, 1 KV head (8 rows per request below) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + # the served shape: 16 query heads per KV head, HKV KV heads (8 rows per request below) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py index 5f4fd0b..a659c2e 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select.py @@ -27,13 +27,16 @@ ) -def compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits: int, quant: bool): +def compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits: int, quant: bool, hkv: int = 1): """The launcher (splits 4).""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits == 4 SPLITS, TOPK, XCDS = splits, 16, 8 PART_SCOPE = 0 name = ( f"nvfp4_f16mfma_sparse_gqa16_d128_p128_top{TOPK}_q8_rowxcd_s{SPLITS}_xcd{XCDS}" + + ("_kv2" if HKV == 2 else "") + ("_mxfp8" if quant else "") + "_select16" ) @@ -114,6 +117,12 @@ def store_q(arg_qc, arg_qs, i32_rows, i32_H, i32_tile, row, kvh, lane, acc, inv) sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=row < i32_rows ) + def score_row(arg_score, i64_s_row, rowc, kvh): + """The address of row ``rowc``'s index scores of index head ``kvh``.""" + if HKV == 1: + return arg_score + rowc.to(fx.Int64) * i64_s_row * 4 + return arg_score + (rowc.to(fx.Int64) * (i64_s_row & 0xFFFFFFFF) + kvh.to(fx.Int64) * (i64_s_row >> 32)) * 4 + # the arguments of the first (dependent) loads lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) def kernel( @@ -143,13 +152,13 @@ def kernel( i32_heavy: fx.Int32, # (kernel.py's tail-grid arguments, unused) i32_tpx: fx.Int32, arg_score: fx.Int64, # [Hkv, >= rows, >= width] fp32 index scores - i64_s_row: fx.Int64, # their row stride (elements) + i64_s_row: fx.Int64, # their row stride (elements); at 2 KV heads | the head stride << 32 i32_width: fx.Int32, # score_width(max_seq_len) of the indexer ): smem = fx.SharedAllocator().allocate(Shared).peek() - # the served shape: 16 query heads, 1 KV head (8 rows per request below) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + # the served shape: 16 query heads per KV head, HKV KV heads (8 rows per request below) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 @@ -179,11 +188,11 @@ def kernel( # lane = top-k slot: its logical block and page (-1: none) slot_ok = lane < TOPK - # the row's top-16 from its index scores (one KV head = one index head), stored as the selector kernel stores + # the row's top-16 from its index scores (index head = KV head), stored as the selector kernel stores # it; padding rows redo row 0's id_row = arg_topk + (kvh * i32_topk_hs + rowc * TOPK).to(fx.Int64) * 4 fast = row_top16_prologue_ids( - tx, arg_score + rowc.to(fx.Int64) * i64_s_row * 4, i32_width, arg_seq, + tx, score_row(arg_score, i64_s_row, rowc, kvh), i32_width, arg_seq, rowc, fx.Int32(8), smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, id_row, ) blk_lds = fx.Int32(smem.tmp.ptr[lane % TOPK]) diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8/kernel.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8/kernel.py index b2f9555..c976405 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8/kernel.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8/kernel.py @@ -115,17 +115,21 @@ def compile_nvfp4_fp8mfma_sparse_decode( tail: bool = False, prefetch: bool = False, q8: bool = False, + hkv: int = 1, ): """The launcher of one variant; the other arguments as ``kernel.compile_nvfp4_sparse_decode``'s. Args: prefetch: issue the first block's loads before the table barrier (the host sets it at <= one workgroup per CU; with more workgroups the overlap lost). - q8: fold the request arithmetic for 8 rows per request, 16 query heads, 1 KV head (the host checks it). + q8: fold the request arithmetic for 8 rows per request, 16 query heads per KV head, ``hkv`` (1 or 2) KV heads + (the host checks it). """ assert 1 <= topk <= 64 and splits >= 1 assert not tail or xcds > 0, "the tail grid places a pair's splits by XCD" SPLITS, TOPK, XCDS, TAIL, PREFETCH, Q8 = splits, topk, xcds, tail, prefetch, q8 + assert hkv in (1, 2) and (Q8 or hkv == 1), (hkv, q8) + HKV = hkv # Pair order removes 16 K interleave permutations per block. At <= one workgroup per CU the V table reads are # batched before their first consumer; otherwise LLVM serializes them and hides the instruction saving. PAIR = Q8 @@ -139,6 +143,7 @@ def compile_nvfp4_fp8mfma_sparse_decode( + ("_tail" if tail else "") + ("_prefetch" if prefetch else "") + ("_q8" if q8 else "") + + ("_kv2" if HKV == 2 else "") ) name += "_vlut" @@ -248,10 +253,10 @@ def kernel( ): smem = fx.SharedAllocator().allocate(Shared).peek() if const_expr(Q8): - # the served shape: 8 rows per request, 16 query heads, 1 KV head + # the served shape: 8 rows per request, 16 query heads per KV head, HKV KV heads i32_ql = fx.Int32(8) - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8mfma_gqa16_d128_p128_top16_q8_rowxcd.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8mfma_gqa16_d128_p128_top16_q8_rowxcd.py index e240150..db72739 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8mfma_gqa16_d128_p128_top16_q8_rowxcd.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/fp8mfma_gqa16_d128_p128_top16_q8_rowxcd.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``fp8/kernel.py`` at 8 rows per request, 16 query heads, 1 KV head, 4 splits, rows spread over the 8 XCDs. +"""``fp8/kernel.py`` at 8 rows per request, 16 query heads per KV head, 1 or 2 KV heads (``hkv``), 4 splits, rows +spread over the 8 XCDs. What ``bf16mfma_gqa16_d128_p128_top16_q8_rowxcd.py`` is to kernel.py: row bx runs on XCD bx % 8 and a row's splits share it; the first block's loads issue before the table barrier, and the V table words are consumed together. The Q / K / V / @@ -28,8 +29,11 @@ def compile_nvfp4_fp8mfma_gqa16_d128_p128_top16_q8_rowxcd( xcds: int = 0, quant: bool = False, tail: bool = False, + hkv: int = 1, ): """The launcher; the host's ``fp8/kernel.py`` key at splits 4, topk 16, xcds 8, no tail.""" + assert hkv in (1, 2), hkv + HKV = hkv assert splits == 4 and topk == 16 and xcds == 8 and not tail assert 1 <= topk <= 64 and splits >= 1 assert not tail or xcds > 0, "the tail grid places a pair's splits by XCD" @@ -42,6 +46,7 @@ def compile_nvfp4_fp8mfma_gqa16_d128_p128_top16_q8_rowxcd( + (f"_xcd{XCDS}" if XCDS else "") + ("_mxfp8" if quant else "") + ("_tail" if tail else "") + + ("_kv2" if HKV == 2 else "") ) name += "_vlut" @@ -150,8 +155,8 @@ def kernel( i32_tpx: fx.Int32, # tail: tail pairs per XCD ): smem = fx.SharedAllocator().allocate(Shared).peek() - i32_H = fx.Int32(16) - i32_hkv = fx.Int32(1) + i32_H = fx.Int32(16 * HKV) + i32_hkv = fx.Int32(HKV) tx = fx.thread_idx.x bx, kvh = fx.block_idx.x, fx.block_idx.y lane = tx % 64 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/host.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/host.py index 7f31473..31d9d92 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/host.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/host.py @@ -48,85 +48,90 @@ def _get_kernel( row_xcd: bool = False, prefetch: bool = False, fixed: bool = False, + hkv: int = 1, + row_spread: bool = False, ): + """``hkv``: the KV heads the q8 variants (row_xcd, prefetch, fixed, fp8 q8) fold; kernel.py takes them at run + time. ``row_spread``: kernel.py's row-spread grid.""" if fixed and not fp8: from .bf16mfma_gqa16_d128_p128_top16_q8_fixed import compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed(splits, quant, hkv) if prefetch and not fp8: from .bf16mfma_gqa16_d128_p128_top16_prefetch import ( compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch, ) - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch(splits, quant, hkv) if row_xcd and fp8: from .fp8mfma_gqa16_d128_p128_top16_q8_rowxcd import ( compile_nvfp4_fp8mfma_gqa16_d128_p128_top16_q8_rowxcd, ) return compile_nvfp4_fp8mfma_gqa16_d128_p128_top16_q8_rowxcd( - splits, topk, xcds, quant, tail + splits, topk, xcds, quant, tail, hkv ) if row_xcd and exact_fp16: from .f16mfma_gqa16_d128_p128_top16_q8_rowxcd import compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd - return compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits, quant) + return compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits, quant, hkv) if row_xcd: from .bf16mfma_gqa16_d128_p128_top16_q8_rowxcd import ( compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd, ) - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd(splits, quant, hkv) if fp8: from .fp8 import kernel as _kernel_fp8 return _kernel_fp8.compile_nvfp4_fp8mfma_sparse_decode( - splits, topk, xcds, quant, tail, prefetch, fixed + splits, topk, xcds, quant, tail, prefetch, fixed, hkv if fixed else 1 ) return _kernel.compile_nvfp4_sparse_decode( - splits, topk, xcds, quant, exact_fp16, tail + splits, topk, xcds, quant, exact_fp16, tail, row_spread ) @functools.cache -def _get_kernel_select(splits: int, quant: bool, exact_fp16: bool): +def _get_kernel_select(splits: int, quant: bool, exact_fp16: bool, hkv: int = 1): if exact_fp16: from .f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select import ( compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select, ) - return compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits, quant) + return compile_nvfp4_f16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits, quant, hkv) from .bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select import ( compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select, ) - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_rowxcd_select(splits, quant, hkv) @functools.cache def _get_kernel_select_generic( - splits: int, topk: int, xcds: int, quant: bool, exact_fp16: bool, tail: bool + splits: int, topk: int, xcds: int, quant: bool, exact_fp16: bool, tail: bool, hkv: int = 1, + row_spread: bool = False, ): from .kernel_select import compile_nvfp4_sparse_decode_select - return compile_nvfp4_sparse_decode_select(splits, topk, xcds, quant, exact_fp16, tail) + return compile_nvfp4_sparse_decode_select(splits, topk, xcds, quant, exact_fp16, tail, hkv, row_spread) @functools.cache -def _get_kernel_select_prefetch(splits: int, quant: bool): +def _get_kernel_select_prefetch(splits: int, quant: bool, hkv: int = 1): from .bf16mfma_gqa16_d128_p128_top16_prefetch_select import ( compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch_select, ) - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch_select(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_prefetch_select(splits, quant, hkv) @functools.cache -def _get_kernel_select_fixed(splits: int, quant: bool): +def _get_kernel_select_fixed(splits: int, quant: bool, hkv: int = 1): from .bf16mfma_gqa16_d128_p128_top16_q8_fixed_select import ( compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed_select, ) - return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed_select(splits, quant) + return compile_nvfp4_bf16mfma_gqa16_d128_p128_top16_q8_fixed_select(splits, quant, hkv) def _select_ids(index_score, seq_lens, topk_idx, rows, query_len, max_seq_len): @@ -158,6 +163,8 @@ def _unit_scale(device: torch.device) -> torch.Tensor: return torch.ones(1, dtype=torch.float32, device=device) +# tail grid at 2 KV heads up to this leftover of a CU round: -1..-18% at 1/16-1/2, +0.4..+18% at 5/8-7/8 (--tail-splits) +KV2_TAIL_MAX_LEFT = 0.5 # (ql, requests) where the padded one-split grid beats the tail grid beyond A/A noise (up to 10%) _PADDED_S1 = {6: (8, 29, 40), 8: (8, 14, 22, 30, 38, 46)} # (ql, requests) where the tail grid's split count adds a CU round: ql 6 x 50 requests 4 splits 55.17 us, 1 split 51.36 @@ -181,6 +188,19 @@ def _tp4_q8_splits(rows: int) -> int: return 1 +def _tp2_splits(ql: int, rows: int) -> int | None: + """Splits of the padded grid at 2 KV heads, top-16, 8 XCDs (partial CU rounds past the first take the tail grid, + ``_use_tail``); None: ``decode_splits``. Measured where the cost model misses (``benchmark_sparse_decode.py + --splits``, 1-128 requests: ql 4 x 24 requests 1 split 16.0 vs 4 splits 17.2 us).""" + if ql == 8: + return 4 if rows <= 32 else 2 if rows <= 64 else 1 + if ql == 4: + return 4 if rows <= 32 else 2 if rows <= 64 else 1 if rows <= 128 else None + if ql == 1: + return 8 if rows <= 4 else 4 if rows <= 32 else 2 if rows <= 64 else 1 if rows <= 128 else None + return None + + def _tuned_splits( exact_fp16: bool, quant: bool, @@ -198,6 +218,8 @@ def _tuned_splits( and xcds == 8 ): return _tp4_q8_splits(reqs * decode_query_len) + if num_kv_heads == 2 and topk == 16 and xcds == 8: + return _tp2_splits(decode_query_len, reqs * decode_query_len) if not (exact_fp16 and quant and num_kv_heads == 4 and topk == 16 and xcds == 8): return None # (q = 6 and 8 take the tail grid, ``_use_tail``, except these requests.) @@ -214,9 +236,17 @@ def _use_tail( topk: int, xcd_count: int, reqs: int, + cus: int, ) -> bool: """Whether an automatic launch takes the tail grid (``tail_schedule``): 4 KV heads, exact fp16, MXFP8 output, - ql 6 or 8, top-16, 8 XCDs, except the ``_PADDED_S1`` batches.""" + ql 6 or 8, top-16, 8 XCDs, except the ``_PADDED_S1`` batches; 2 KV heads, exact fp16, top-16, 8 XCDs, past one CU + round of (row, KV head) pairs with a leftover of at most ``KV2_TAIL_MAX_LEFT`` of a round.""" + if num_kv_heads == 2: + pairs = reqs * decode_query_len * num_kv_heads + return ( + exact_fp16 and topk == 16 and xcd_count == 8 and pairs > cus + and 0 < pairs % cus <= KV2_TAIL_MAX_LEFT * cus + ) return ( reqs not in _PADDED_S1.get(decode_query_len, ()) and exact_fp16 @@ -269,6 +299,18 @@ def _cu_count(device: torch.device) -> int: return torch.cuda.get_device_properties(device).multi_processor_count +# the KV head counts the q8 variants (rowxcd, prefetch, fixed, fp8 q8; ql 8, top-16, 8 XCDs) fold +Q8_KV_HEADS = (1, 2) +# rows up to which the rowxcd kernels serve, per KV heads +ROW_XCD_MAX_ROWS = {1: 32, 2: 32} +# fp16 at more KV heads than this drops the padded XCD grid for <= 4 multi-row requests (below) +PLAIN_GRID_MIN_KV = 2 +# rows up to which kernel.py's row-spread grid serves ql < 8: 2 KV heads ql 4 x 1-6 requests -0.1..-0.5 us, 8: +0.2 +ROW_SPREAD_MAX_ROWS = {2: 24} + +# in-kernel top-16 through these rows (2 KV heads: -1.0..-2.6 us vs the selector launch, benchmark_index_decode.py) +SELECT_MAX_ROWS = {1: 256, 2: 1024} + # ``quant_out`` from this many rows: attention + o_proj -1.0..-11.4% at 8-512 rows, +1.5% at 4 (bench_attn_oproj_mxfp8) MXFP8_OUT_MIN_ROWS = 8 @@ -305,9 +347,9 @@ def nvfp4_sparse_decode( splits: None: the host's schedule (the tail grid for the ``_use_tail`` shapes); a count keeps the padded grid. tail_splits: the tail grid with that many tail splits (sweeps). index_score: the indexer's scores (``nvfp4_index_score_rows``; ``index_max_seq_len`` its host max): the call - writes ``topk_idx`` (``nvfp4_index_score_top16``'s ids, bit for bit) before reading it. At 1 KV head, - top-16, bf16 and <= 256 rows the attention workgroups select themselves (the ``*_select`` kernels); - elsewhere the selector launches first. + writes ``topk_idx`` (``nvfp4_index_score_top16``'s ids, bit for bit) before reading it. At 1 or 2 KV + heads (index head h feeds KV head h), top-16, bf16 / fp16 and <= ``SELECT_MAX_ROWS`` rows the attention + workgroups select themselves (the ``*_select`` kernels); elsewhere the selector launches first. """ rows, num_heads, head_dim = q.shape if rows == 0: @@ -391,26 +433,31 @@ def nvfp4_sparse_decode( topk, xcds, seq_lens.numel(), + _cu_count(q.device), ) ) - # <= 4 requests of 8 rows: rows spread over the XCDs, a row's 4 splits on one of them (grid.x a multiple of 8, + # few requests of 8 rows: rows spread over the XCDs, a row's 4 splits on one of them (grid.x a multiple of 8, # so the partials stay XCD-local) row_xcd = ( - not tail and num_kv_heads == 1 - and decode_query_len == 8 and topk == 16 and xcds == 8 and rows <= 32 + not tail and num_kv_heads in Q8_KV_HEADS and num_kv_heads in ROW_XCD_MAX_ROWS + and decode_query_len == 8 and topk == 16 and xcds == 8 and rows <= ROW_XCD_MAX_ROWS[num_kv_heads] and (splits is None or splits == 4) ) heavy = tpx = 0 - # no padded XCD grid for <= 4 multi-row requests in fp16 at 4 KV heads (measured at ql 4 and 8; at 1 KV head - # it wins: ql 4 x 1 request 10.34 -> 8.04 us) + # fp16, <= 4 multi-row requests: the padded XCD grid loses at 4 KV heads, wins at 1 / 2 (ql 4 x 1: -2.3 / -1.6 us) if ( not tail and exact_fp16 - and num_kv_heads > 1 + and num_kv_heads > PLAIN_GRID_MIN_KV and decode_query_len > 1 and rows <= 4 * decode_query_len ): xcds = 0 + # (not at 8 rows per request: the q8 variants fold the padded grid's row permutation) + row_spread = ( + not tail and not row_xcd and not fp8 and xcds > 0 and 1 < decode_query_len < 8 + and rows <= ROW_SPREAD_MAX_ROWS.get(num_kv_heads, 0) + ) if tail: assert xcds > 0 and splits is None, "the tail grid needs the XCD grid" # (Automatic tail grid => the serving gate of ``_use_tail`` holds.) @@ -422,7 +469,7 @@ def nvfp4_sparse_decode( rows * num_kv_heads, topk, _cu_count(q.device), xcds, tail_splits ) elif xcds: - group = xcds if row_xcd else xcds * decode_query_len + group = xcds if row_xcd or row_spread else xcds * decode_query_len grid_x = -(-rows // group) * group else: grid_x = rows @@ -430,6 +477,8 @@ def nvfp4_sparse_decode( wg_rows = grid_x * num_kv_heads if wg_rows > MAX_SPLIT_ROWS: splits = 1 + elif row_xcd: + splits = 4 else: tuned = _tuned_splits( exact_fp16, @@ -445,32 +494,27 @@ def nvfp4_sparse_decode( else: splits = decode_splits(rows * num_kv_heads, topk, _cu_count(q.device)) assert tail or splits == 1 or grid_x * num_kv_heads <= MAX_SPLIT_ROWS + # the q8 variants' grid bounds count (row, KV head) pairs + q8 = num_kv_heads in Q8_KV_HEADS and decode_query_len == 8 and topk == 16 and xcds == 8 + pairs_x = grid_x * num_kv_heads # (fp16 keeps kernel.py here: its folded prefetch variant measured 64 rows 9.87 vs 9.76 us, 256 rows -0.4%) prefetch = ( - not tail and not row_xcd and not exact_fp16 and not fp8 - and num_kv_heads == 1 and decode_query_len == 8 and topk == 16 and xcds == 8 - and splits in (1, 2, 4) and grid_x * splits <= _cu_count(q.device) + not tail and not row_xcd and not exact_fp16 and not fp8 and q8 + and splits in (1, 2, 4) and pairs_x * splits <= _cu_count(q.device) ) # fp8 per row, the same first-block overlap and grid bound: -3.3..-3.9% at 40-128 rows, -0.8% at 256 fp8_prefetch = ( - fp8 and not tail and not row_xcd - and num_kv_heads == 1 and decode_query_len == 8 and topk == 16 and xcds == 8 - and splits in (1, 2, 4) and grid_x * splits <= _cu_count(q.device) - ) - fp8_q8 = ( - fp8 and not row_xcd - and num_kv_heads == 1 and decode_query_len == 8 and topk == 16 and xcds == 8 - and splits in (1, 2, 4) + fp8 and not tail and not row_xcd and q8 + and splits in (1, 2, 4) and pairs_x * splits <= _cu_count(q.device) ) + fp8_q8 = fp8 and not row_xcd and q8 and splits in (1, 2, 4) fixed = ( - not tail and not row_xcd and not prefetch and not exact_fp16 and not fp8 - and num_kv_heads == 1 and decode_query_len == 8 and topk == 16 and xcds == 8 - and splits in (2, 4) and grid_x <= 192 + not tail and not row_xcd and not prefetch and not exact_fp16 and not fp8 and q8 + and splits in (2, 4) and pairs_x <= 192 ) - # (measured through 256 rows; larger batches keep the selector launch) generic = not row_xcd and not prefetch and not fixed and not fp8 select = ( - index_score is not None and num_kv_heads == 1 and topk == 16 and rows <= 256 + index_score is not None and topk == 16 and rows <= SELECT_MAX_ROWS.get(num_kv_heads, 0) and ((row_xcd and not fp8) or prefetch or fixed or generic) ) if index_score is not None and not select: @@ -482,28 +526,37 @@ def nvfp4_sparse_decode( c = float(np.float32(sm_scale) * np.float32(_kernel.LOG2E)) extra = () if select: - extra = (index_score.data_ptr(), index_score.stride(1), width) + s_row = index_score.stride(1) + if num_kv_heads == 2: + assert 0 <= s_row < 2**32 and 0 <= index_score.stride(0) < 2**31 + s_row |= index_score.stride(0) << 32 + extra = (index_score.data_ptr(), s_row, width) + # the q8 compile keys carry hkv only at 2 KV heads + hk = (2,) if q8 and num_kv_heads == 2 else () + if select: + exe = ( + _get_kernel_select(splits, quant, exact_fp16, *hk) if row_xcd + else _get_kernel_select_prefetch(splits, quant, *hk) if prefetch + else _get_kernel_select_fixed(splits, quant, *hk) if fixed + else _get_kernel_select_generic(splits, topk, xcds, quant, exact_fp16, tail, + hkv=2 if num_kv_heads == 2 else 1, row_spread=row_spread) + ) + elif row_xcd: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, fp8, True, *((False, False) + hk if hk else ())) + elif prefetch: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, False, False, True, + *((False,) + hk if hk else ())) + elif fixed: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, False, False, False, True, *hk) + elif fp8: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, fp8=True, prefetch=fp8_prefetch, fixed=fp8_q8, + **({"hkv": 2} if hk else {})) + elif row_spread: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, False, False, False, False, 1, True) + else: + exe = _get_kernel(splits, topk, xcds, quant, exact_fp16, tail) _run_compiled( - ( - _get_kernel_select(splits, quant, exact_fp16) - if select and row_xcd - else _get_kernel_select_prefetch(splits, quant) - if select and prefetch - else _get_kernel_select_fixed(splits, quant) - if select and fixed - else _get_kernel_select_generic(splits, topk, xcds, quant, exact_fp16, tail) - if select - else _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, fp8, True) - if row_xcd - else _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, False, False, True) - if prefetch - else _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, False, False, False, True) - if fixed - else _get_kernel(splits, topk, xcds, quant, exact_fp16, tail, fp8=True, - prefetch=fp8_prefetch, fixed=fp8_q8) - if fp8 - else _get_kernel(splits, topk, xcds, quant, exact_fp16, tail) - ), + exe, q.data_ptr(), kv_cache.data_ptr(), topk_idx.data_ptr(), @@ -538,7 +591,7 @@ def nvfp4_sparse_decode( # ---- warm-up (root README rule 4) # the request counts the first-call warm-up walks (``amd_kernels.warm_up``): the variant follows from the row and -# request counts alone, so every count up to the TP4 batch limit; ``warm_up_decode`` walks a caller's own limit +# request counts alone, so every count up to the largest served batch; ``warm_up_decode`` walks a caller's own limit WARM_MAX_REQS = 256 diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/index_decode.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/index_decode.py index 4db4f97..851844e 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/index_decode.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/index_decode.py @@ -7,9 +7,12 @@ - ``split4``: the four-GPU scorer + top-16 (``decode_top16.IndexDecodeTop16``, from ``decode_top16.SPLIT_MIN_REQS`` requests), then the attention on its ids; -- ``fused`` (one GPU, <= ``FUSED_MAX_ROWS`` rows, one index head per KV head, bf16 or fp16): the scorer +- ``split2``: the two-GPU scorer + top-16 (TP2, from ``decode_top16.SPLIT2_MIN_REQS`` requests), then the attention on + its ids; +- ``fused`` (one GPU, <= ``host.SELECT_MAX_ROWS`` rows, one index head per KV head, bf16 or fp16): the scorer (``nvfp4_index_score_rows``), then the attention with ``index_score=``, whose workgroups select the top-16 - themselves: -0.84..-1.43 us per layer at 4-32 rows, -0.7..-2.2 us at 40-256 rows (32K-192K, bf16) against ``chain``; + themselves: -0.84..-1.43 us per layer at 4-32 rows, -0.7..-2.2 us at 40-256 rows (1 KV head, 32K-192K, bf16), -1.0.. + -2.0 us at 8-1024 rows (2 KV heads, fp16) against ``chain``; - ``chain`` (one GPU, otherwise): ``nvfp4_index_score_top16`` (scorer + selector), then the attention. ``topk_idx`` holds the ids afterwards on every route (a layer that reuses the previous layer's top-16 reads it). @@ -23,21 +26,19 @@ nvfp4_index_score_top16, ) -from .host import nvfp4_sparse_decode, warm_up_decode - -# the in-attention selection was measured through 256 rows; larger steps keep the scorer + selector -FUSED_MAX_ROWS = 256 +from .host import SELECT_MAX_ROWS, nvfp4_sparse_decode, warm_up_decode def route(index, num_reqs: int, query_len: int, num_index_heads: int, num_kv_heads: int, compute_type: torch.dtype = torch.bfloat16, split: bool = True) -> str: - """``"split4"``, ``"fused"`` or ``"chain"`` for a decode step (module doc); ``index``: the step's + """``"split4"``, ``"split2"``, ``"fused"`` or ``"chain"`` for a decode step (module doc); ``index``: the step's ``IndexDecodeTop16`` or None (one GPU).""" - if index is not None and index.route(num_reqs, query_len, num_index_heads, split) == "split4": - return "split4" + multi = "one_gpu" if index is None else index.route(num_reqs, query_len, num_index_heads, split) + if multi != "one_gpu": + return multi if ( num_index_heads == num_kv_heads - and num_reqs * query_len <= FUSED_MAX_ROWS + and num_reqs * query_len <= SELECT_MAX_ROWS.get(num_kv_heads, 0) and compute_type in (torch.bfloat16, torch.float16) ): return "fused" @@ -45,7 +46,7 @@ def route(index, num_reqs: int, query_len: int, num_index_heads: int, num_kv_hea def nvfp4_index_sparse_decode( - index, # decode_top16.IndexDecodeTop16 (the four-GPU route's buffers) or None + index, # decode_top16.IndexDecodeTop16 (the multi-GPU route's buffers) or None iq: torch.Tensor, # [rows, index heads, 128] e4m3 index_cache: torch.Tensor, # NVFP4 index pages [num_blocks, 128, 72] uint8 index_block_table: torch.Tensor, # [reqs, max_blocks] int32, the index cache's pages @@ -69,7 +70,7 @@ def nvfp4_index_sparse_decode( compute_type: torch.dtype = torch.bfloat16, ) -> str: """Index top-16 + sparse decode of the decode rows (module doc); returns the route taken. ``split`` False keeps - the step on one GPU (e.g. DBO micro-batches, where no collective orders the four-GPU push); ``compute_type``: + the step on one GPU (e.g. DBO micro-batches, where no collective orders the multi-GPU exchange); ``compute_type``: ``host.nvfp4_sparse_decode``'s. Graph-safe.""" rows, heads, _ = iq.shape num_reqs = seq_lens.shape[0] @@ -77,7 +78,7 @@ def nvfp4_index_sparse_decode( r = route(index, num_reqs, decode_query_len, heads, num_kv_heads, compute_type, split) topk = topk_idx[:, :rows] extra = {} - if r == "split4": + if r in ("split4", "split2"): index(iq, index_cache, index_block_table, index_seq_lens, init_blocks, local_blocks, topk, query_len=decode_query_len, max_seq_len=max_seq_len, split=split) elif r == "fused": diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel.py index bb1486a..7941e26 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel.py @@ -253,6 +253,7 @@ def compile_nvfp4_sparse_decode( quant: bool = False, exact_fp16: bool = False, tail: bool = False, + row_spread: bool = False, ): """The launcher of one variant. @@ -265,10 +266,13 @@ def compile_nvfp4_sparse_decode( exact_fp16: exact e2m1 x scale K / V with fp16 Q / P. tail: (needs ``xcds``) a 1-D grid: the whole CU rounds of (row, KV head) pairs at one split each, then the leftover pairs at ``splits`` each. + row_spread: (needs ``xcds``) row bx on XCD bx % xcds, grid.x padded to ``xcds`` only: a request's rows spread + over the XCDs, a row's splits share one. """ assert 1 <= topk <= 64 and splits >= 1 assert not tail or xcds > 0, "the tail grid places a pair's splits by XCD" - SPLITS, TOPK, XCDS, TAIL = splits, topk, xcds, tail + assert not row_spread or (xcds > 0 and not tail) + SPLITS, TOPK, XCDS, TAIL, ROW_SPREAD = splits, topk, xcds, tail, row_spread PART_SCOPE = 0 if XCDS else AGENT_SCOPE mfma = "f16" if exact_fp16 else "bf16" # the MFMA operand dtype name = ( @@ -276,6 +280,7 @@ def compile_nvfp4_sparse_decode( + (f"_xcd{XCDS}" if XCDS else "") + ("_mxfp8" if quant else "") + ("_tail" if tail else "") + + ("_rowspread" if row_spread else "") ) @fx.struct @@ -430,7 +435,7 @@ def tab(sc): row = fx.Int32(fx.rocdl.readfirstlane(T.i32, pair // i32_hkv)) kvh = fx.Int32(fx.rocdl.readfirstlane(T.i32, pair % i32_hkv)) else: - if const_expr(XCDS > 0): + if const_expr(XCDS > 0 and not ROW_SPREAD): # grid.x is padded to XCDS x ql: a permutation that puts each request's ql rows on one XCD (padding # rows map past i32_rows) xcd = bx % XCDS diff --git a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel_select.py b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel_select.py index 7e07828..b18c69d 100644 --- a/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel_select.py +++ b/amd_kernels/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/kernel_select.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``kernel.py`` with the indexer's top-16 selector as prologue (one KV head = one index head, top-16). +"""``kernel.py`` with the indexer's top-16 selector as prologue (index head h = KV head h, top-16). Each workgroup first selects its row's 16 blocks from the index scores (``row_top16_prologue_ids``, the indexer's plain-order four-wave selector), stores them to ``topk_idx`` (every split of a row stores the same bytes) and attends @@ -36,11 +36,17 @@ def compile_nvfp4_sparse_decode_select( quant: bool = False, exact_fp16: bool = False, tail: bool = False, + hkv: int = 1, + row_spread: bool = False, ): - """The launcher; the arguments of ``kernel.compile_nvfp4_sparse_decode``.""" + """The launcher; the arguments of ``kernel.compile_nvfp4_sparse_decode``; ``hkv`` 2: KV head h selects from index + head h's scores (``i64_s_row`` packs the row stride and, in its high word, the head stride).""" assert 1 <= topk <= 64 and splits >= 1 + assert hkv in (1, 2), hkv + HKV = hkv assert not tail or xcds > 0, "the tail grid places a pair's splits by XCD" - SPLITS, TOPK, XCDS, TAIL = splits, topk, xcds, tail + assert not row_spread or (xcds > 0 and not tail) + SPLITS, TOPK, XCDS, TAIL, ROW_SPREAD = splits, topk, xcds, tail, row_spread PART_SCOPE = 0 if XCDS else AGENT_SCOPE mfma = "f16" if exact_fp16 else "bf16" # the MFMA operand dtype name = ( @@ -48,6 +54,8 @@ def compile_nvfp4_sparse_decode_select( + (f"_xcd{XCDS}" if XCDS else "") + ("_mxfp8" if quant else "") + ("_tail" if tail else "") + + ("_rowspread" if row_spread else "") + + ("_kv2" if HKV == 2 else "") + "_select16" ) @@ -130,6 +138,12 @@ def store_q(arg_qc, arg_qs, i32_rows, i32_H, i32_tile, row, kvh, lane, acc, inv) sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=row < i32_rows ) + def score_row(arg_score, i64_s_row, rowc, kvh): + """The address of row ``rowc``'s index scores of index head ``kvh``.""" + if HKV == 1: + return arg_score + rowc.to(fx.Int64) * i64_s_row * 4 + return arg_score + (rowc.to(fx.Int64) * (i64_s_row & 0xFFFFFFFF) + kvh.to(fx.Int64) * (i64_s_row >> 32)) * 4 + # the arguments of the first (dependent) loads lead: the first 56 bytes arrive preloaded in SGPRs @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) def kernel( @@ -158,8 +172,8 @@ def kernel( f32_c: fx.Float32, # sm_scale * log2(e) i32_heavy: fx.Int32, # tail: whole-round pairs (a multiple of XCDS) i32_tpx: fx.Int32, # tail: tail pairs per XCD - arg_score: fx.Int64, # [1, >= rows, >= width] fp32 index scores - i64_s_row: fx.Int64, # their row stride (elements) + arg_score: fx.Int64, # [Hkv, >= rows, >= width] fp32 index scores + i64_s_row: fx.Int64, # their row stride (elements); at 2 KV heads | the head stride << 32 i32_width: fx.Int32, # score_width(max_seq_len) of the indexer ): smem = fx.SharedAllocator().allocate(Shared).peek() @@ -210,7 +224,7 @@ def tab(sc): row = fx.Int32(fx.rocdl.readfirstlane(T.i32, pair // i32_hkv)) kvh = fx.Int32(fx.rocdl.readfirstlane(T.i32, pair % i32_hkv)) else: - if const_expr(XCDS > 0): + if const_expr(XCDS > 0 and not ROW_SPREAD): # grid.x is padded to XCDS x ql: a permutation that puts each request's ql rows on one XCD (padding # rows map past i32_rows) xcd = bx % XCDS @@ -230,11 +244,11 @@ def tab(sc): # lane = top-k slot: its logical block and page (-1: none) slot_ok = lane < TOPK - # the row's top-16 from its index scores (one KV head = one index head), stored as the selector kernel stores - # it; padding rows redo row 0's + # the row's top-16 from its index scores (index head = KV head), stored as the selector kernel stores it; + # padding rows redo row 0's id_row = arg_topk + (kvh * i32_topk_hs + rowc * TOPK).to(fx.Int64) * 4 fast = row_top16_prologue_ids( - tx, arg_score + rowc.to(fx.Int64) * i64_s_row * 4, i32_width, arg_seq, + tx, score_row(arg_score, i64_s_row, rowc, kvh), i32_width, arg_seq, rowc, i32_ql, smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, id_row, ) blk_lds = fx.Int32(smem.tmp.ptr[lane % TOPK]) diff --git a/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/README.md b/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/README.md index 0694c64..308895c 100644 --- a/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/README.md +++ b/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/README.md @@ -2,22 +2,23 @@ ## What it does -The kernels of one NVFP4 sparse (MSA) attention layer of MiniMax-M3.1, as a server calls them: the decode indexer -(`IndexDecodeTop16`, one or four GPUs), the warm-up of every variant the layer launches, and the insert, decode and -prefill entries with the served compute dtypes bound (decode fp16, prefill fp16 with the lazy rescale: exact K / V; -`DECODE_COMPUTE`, `PREFILL_COMPUTE`, `PREFILL_LAZY`). The caller passes tensors and its limits (heads, rows per -request, requests, context). No kernel of its own. Contract: each call is the call of its entry with those dtypes, -bit for bit (the entries' `reference.py`). +The kernels of one NVFP4 sparse (MSA) attention layer of MiniMax-M3.1, as a server calls them, at TP4 (16 q / 1 KV / 1 +index head per GPU) and TP2 (32 / 2 / 2): the decode indexer (`IndexDecodeTop16`), the warm-up of every variant the +layer launches, and the insert, decode and prefill entries with the served compute dtypes bound (decode fp16, prefill +fp16 with the lazy rescale: exact K / V; `DECODE_COMPUTE`, `PREFILL_COMPUTE`, `PREFILL_LAZY`). The caller passes tensors +and its limits (heads, rows per request, requests, context). No kernel of its own. Contract: each call is the call of +its entry with those dtypes, bit for bit (the entries' `reference.py`). ## Entry points -- `init`: before serving, builds the decode indexer (with the four ranks' group a collective on every rank, in order) - and runs the entries' warm-ups for the server's limits: insert, decode, prefill, prefill indexer. Returns the - indexer. +- `init`: before serving, builds the decode indexer (with the ranks' group, where the indexer has a multi-GPU route, + a collective on every rank, in order) and runs the entries' warm-ups for the server's limits and heads: insert, + decode, prefill, prefill indexer. Returns the indexer. - `sparse_decode`, `index_sparse_decode`, `sparse_prefill`: `nvfp4_sparse_decode`, `index_decode.nvfp4_index_sparse_decode` and `nvfp4_sparse_prefill` with the served dtypes bound. - `quant_out_buffers`: the decode's MXFP8 output buffers in the layout of the MXFP8 GEMM it feeds; None below - `QUANT_OUT_MIN_ROWS` = 8 rows, where the bf16 output and the GEMM's own quant are faster. + `QUANT_OUT_MIN_ROWS[k]` rows of the o_proj K (2048: 8, where the bf16 output and the GEMM's own quant are faster; + 4096: 1). - `qkv_norm_rope_nvfp4_insert`, `mxfp8_gemm_qkv_norm_rope_nvfp4_insert`: re-exported from `norm_rope_nvfp4_insert_d128_rot64_p128`. diff --git a/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/layer.py b/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/layer.py index 827b319..c50cd48 100644 --- a/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/layer.py +++ b/amd_kernels/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/layer.py @@ -16,10 +16,11 @@ from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import prefill_top16 from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.decode_top16 import IndexDecodeTop16 -DECODE_COMPUTE = torch.float16 # exact K / V, the fastest decode at TP4 (decode README, Performance) +DECODE_COMPUTE = torch.float16 # exact K / V, the fastest decode at TP4 and TP2 (decode README, Performance) PREFILL_COMPUTE = "fp16" # exact K / V PREFILL_LAZY = True # fp16 P re-based past 2^8: the fastest prefill at every T (prefill README, Performance) -QUANT_OUT_MIN_ROWS = decode_host.MXFP8_OUT_MIN_ROWS +# MXFP8 decode output from this many rows, per o_proj K (4096: -8% at 1-32 rows, decode benchmark_oproj_input.py) +QUANT_OUT_MIN_ROWS = {2048: decode_host.MXFP8_OUT_MIN_ROWS, 4096: 1} sparse_decode = functools.partial(decode_host.nvfp4_sparse_decode, compute_type=DECODE_COMPUTE) index_sparse_decode = functools.partial(index_decode.nvfp4_index_sparse_decode, compute_type=DECODE_COMPUTE) @@ -29,8 +30,8 @@ def quant_out_buffers(rows: int, n: int, k: int, device) -> tuple[torch.Tensor, torch.Tensor, int] | None: """``quant_out`` of a ``rows``-row decode whose output feeds an MXFP8 GEMM with an ``[n, k]`` weight, in that - GEMM's pre-quantized layout (``gemm.prequant_buffers``); None below ``QUANT_OUT_MIN_ROWS`` (bf16 output).""" - if rows < QUANT_OUT_MIN_ROWS: + GEMM's pre-quantized layout (``gemm.prequant_buffers``); None below ``QUANT_OUT_MIN_ROWS[k]`` (bf16 output).""" + if rows < QUANT_OUT_MIN_ROWS.get(k, decode_host.MXFP8_OUT_MIN_ROWS): return None return gemm.prequant_buffers(rows, n, k, device) @@ -51,11 +52,12 @@ def init( rank: int | None = None, group=None, ) -> IndexDecodeTop16 | None: - """Before serving: the decode indexer (``index``; with ``group`` the four-GPU route, a collective on every rank in - order) and every variant the layer launches, compiled: the insert's prefill tiles (``insert``), the decode of - 1-``max_reqs`` requests x ``query_len`` rows with the bf16 output, the MXFP8 output from ``QUANT_OUT_MIN_ROWS`` - rows (``quant_out``) and the in-kernel top-16 (``index``), the prefill (``attention``) and the prefill indexer - (``index``: ``prefill_top16.index_prefill_top16``). Returns the decode indexer.""" + """Before serving: the decode indexer (``index``; with ``group`` its multi-GPU route where the indexer has one, a + collective on every rank in order) and every variant the layer launches, compiled: the insert's prefill tiles + (``insert``), the decode of 1-``max_reqs`` requests x ``query_len`` rows with the bf16 output, the MXFP8 output from + ``QUANT_OUT_MIN_ROWS`` rows of the o_proj K = ``num_heads`` x 128 (``quant_out``) and the in-kernel top-16 + (``index``), the prefill (``attention``) and the prefill indexer (``index``: ``prefill_top16.index_prefill_top16``). + Returns the decode indexer.""" device = torch.device(device) indexer = None if index: @@ -66,8 +68,9 @@ def init( if insert: warmup_prefill_kernels(num_heads, num_kv_heads, num_index_heads, device=device) if attention: + quant_min = QUANT_OUT_MIN_ROWS.get(num_heads * 128, decode_host.MXFP8_OUT_MIN_ROWS) decode_host.warm_up_decode(device, num_heads=num_heads, num_kv_heads=num_kv_heads, query_len=query_len, - max_reqs=max_reqs, quant_min_rows=QUANT_OUT_MIN_ROWS if quant_out else None, + max_reqs=max_reqs, quant_min_rows=quant_min if quant_out else None, index_select=index, compute_type=DECODE_COMPUTE) prefill_host.warm_up_prefill(device, num_heads=num_heads, num_kv_heads=num_kv_heads, compute_dtype=PREFILL_COMPUTE, lazy_rescale=PREFILL_LAZY) diff --git a/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/README.md b/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/README.md index aa9c938..4350271 100644 --- a/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/README.md +++ b/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/README.md @@ -7,9 +7,9 @@ Inputs: Q `[T, 16 Hkv, 128]` (bf16, fp16 for fp16 compute, or prepacked e4m3 u8 `[pages, 2 Hkv, 128, 72]` u8 with free page stride, logical picks `[Hkv, T, 16]` i32 (-1 none), block tables, request ids and positions. Tiles may span requests. Output: bf16, MXFP8 codes + AITER-shuffled scales for o_proj, or sequence-parallel all-to-all rows (Hkv 1); the fused outputs leave the bf16 output and the padding rows untouched. -TP4: Hkv 1, up to 65536 tokens. TP4 serves fp16 with the lazy rescale (fp16 P against a reference re-based only past -2^8, P <= 256; the fastest at every T); bf16, plain fp16 and fp8 stay available. FP8 uses NV Q8KV4 with P rounded per -32-key chunk; gate it with the NV KL check. +Served: Hkv 1 (TP4) and Hkv 2 (TP2), up to 65536 tokens, fp16 with the lazy rescale (fp16 P against a reference +re-based only past 2^8, P <= 256; the fastest at every T and both Hkv); bf16, plain fp16 and fp8 stay available. FP8 +uses NV Q8KV4 with P rounded per 32-key chunk; gate it with the NV KL check. Contract: `reference.py` (`REL_L2_TOL`, `COS_MIN`); fused outputs equal quantization of the kernel's own bf16 output. Strict xfail: bf16 TQ4 can propagate a NaN V scale past the sequence end through zero P. @@ -17,29 +17,30 @@ Strict xfail: bf16 TQ4 can propagate a NaN V scale past the sequence end through - `nvfp4_sparse_prefill`: the attention of a prefill step into bf16, or fused into the MXFP8 o_proj input or the sequence-parallel all-to-all rows; graph-safe. An `amd_kernels.warm_up` entry: the first call of each mode warms - every variant at 8 / 512 / 1024 / 1536 / 2048 tokens. + every variant at 8 / 512 / 1024 / 1536 / 2048 tokens (Hkv 2: 8 / 256 / 384 / 1024). - `nvfp4_sparse_prefill_bf16`, `nvfp4_sparse_prefill_fp16`, `nvfp4_sparse_prefill_fp8`: the same with the compute precision bound. Build the fp8 table outside graph capture. - `host.warm_up_prefill`: compiles every bf16-output variant of one compute precision now. - `supports_nvfp4_sparse_prefill`: whether a model fits (gfx950, GQA16, D128, top-16 blocks of 128). -`host.py` `nvfp4_sparse_prefill` picks the kernel by precision, token count and grid size. +`host.py` `nvfp4_sparse_prefill` picks the kernel by precision, KV heads, token count and grid size (`_local_tile`). ## Performance -MI355X, Hkv 1, bf16 Q, 16K prefix, 100 cold inputs, 4 alternated graph-replay rounds, us. -Tokens: 256 / 512 / 1024 / 1536 / 4096 / 8192 / 16384 / 32768. FP8 includes Q packing. +MI355X, fp16 lazy (served), bf16 Q and output (what vLLM calls), 16K prefix, 100 cold inputs, 2 alternated +graph-replay rounds, us. Hkv 2 baseline: no small tiles, the four-query kernel at every count (one round below 769 +tokens; the served kernel from 1024). -| Compute | quant | pack | -|---|---|---| -| bf16 | 15.7 / 22.2 / 32.6 / 42.8 / 95.1 / 180.2 / 355.9 / 708.7 | 15.6 / 22.0 / 32.5 / 42.5 / 95.0 / 179.6 / 355.0 / 711.3 | -| fp16 | 14.7 / 22.1 / 31.8 / 41.7 / 92.6 / 176.2 / 348.8 / 696.4 | 14.5 / 21.9 / 31.6 / 41.5 / 92.4 / 175.6 / 348.1 / 694.6 | -| fp8 | 17.2 / 24.2 / 34.8 / 44.8 / 99.3 / 187.0 / 361.6 / 712.1 | 17.0 / 23.8 / 34.5 / 44.6 / 98.9 / 185.8 / 360.4 / 708.8 | -| fp16 lazy | 14.7 / 21.0 / 29.8 / 38.8 / 85.0 / 162.9 / 322.4 / 644.0 | 14.6 / 20.7 / 29.7 / 38.6 / 84.6 / 161.9 / 321.2 / 641.7 | +| Tokens | 128 | 256 | 512 | 768 | 1024 | 2048 | 4096 | 8192 | 16384 | 32768 | +|---|---|---|---|---|---|---|---|---|---|---| +| Hkv 1 (TP4) | 14.1 | 14.5 | 21.0 | 29.1 | 29.7 | 47.0 | 87.4 | 167.4 | 329.5 | 658.3 | +| Hkv 2 (TP2) | 14.5 | 20.7 | 29.8 | 39.8 | 47.5 | 87.7 | 170.2 | 328.7 | 654.8 | 1315.1 | +| Hkv 2, baseline | 38.5 | 38.1 | 38.1 | 45.9 | 47.9 | 88.9 | 170.5 | 335.1 | 667.3 | - | ## Tests ```bash python3 -m pytest tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16 -q python3 benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py --fp8-bf16q --fp16-lazy +python3 benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py --fp16-lazy --only 2:128,2:256,2:512,2:768,2:2048,2:8192 ``` diff --git a/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/host.py b/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/host.py index c601804..53994e7 100644 --- a/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/host.py +++ b/amd_kernels/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/host.py @@ -131,6 +131,23 @@ def _on_gfx950() -> bool: ).gcnArchName.startswith("gfx950") +# Hkv 2 (TP2): (most tokens, queries per workgroup, shared Q), the fastest tile at every count (sweep_tiles.py --hkv 2) +HKV2_TILES = ((128, 1, False), (256, 2, True), (384, 3, False), (512, 2, True), (768, 3, False)) + + +def _local_tile(n_tok: int, num_kv_heads: int) -> tuple[int, bool] | None: + """(queries per workgroup, shared Q staging) of the one- / two- / three-query tiles, or None (four-query kernels). + Hkv 1: one query to 256 tokens, two to 1025 (shared Q above one two-query workgroup per CU, 513-1025), three to + 1536; Hkv 2: ``HKV2_TILES``.""" + if num_kv_heads == 1: + if n_tok > 1536: + return None + return (1 if n_tok <= 256 else 2 if n_tok <= 1025 else 3), 512 < n_tok <= 1025 + if num_kv_heads == 2: + return next(((tq, shared) for most, tq, shared in HKV2_TILES if n_tok <= most), None) + return None + + def supports_nvfp4_sparse_prefill( num_heads: int, num_kv_heads: int, head_dim: int, topk_blocks: int, block_size: int ) -> bool: @@ -248,20 +265,19 @@ def nvfp4_sparse_prefill( n_wg = (n_tok + 3) // 4 * num_kv_heads rounds = 2 if n_wg > _num_cus(q.device) else 1 fp8_args = (_fp8_lut(q.device).data_ptr(),) if compute_dtype == "fp8" else () - # Hkv 1 small tiles by the measured crossovers: bf16 / fp16 (not with the fp16 scheduling ports) one query to - # 256 tokens, two to 1025, three to 1536; fp8 one to 512 + # small tiles (bf16 / fp16, not with the fp16 scheduling ports) by the measured crossovers (``_local_tile``); fp8 + # one query to 512 tokens at Hkv 1 use_sched = sched and switches.PREFILL_MSA_SCHED and compute_dtype != "fp8" - local = (compute_dtype == "bf16" or (compute_dtype == "fp16" and not use_sched)) and num_kv_heads == 1 \ - and n_tok <= 1536 + tile = _local_tile(n_tok, num_kv_heads) if compute_dtype == "bf16" or ( + compute_dtype == "fp16" and not use_sched) else None q_bf16 = compute_dtype == "fp8" and q.dtype == torch.bfloat16 # the lazy rescale runs on the two- / three-query tiles and the two-round kernel; the default call keeps its arguments - lazy_local = {"lazy": True} if lazy_rescale and n_tok > 256 else {} + lazy_local = {"lazy": True} if lazy_rescale and tile is not None and tile[0] > 1 else {} lazy_grid = {"lazy": True} if lazy_rescale and rounds == 2 else {} - # Shared Q conversion wins above one two-query workgroup per CU (513-1025 tokens; see the README). - shared_local = {"q_shared": True} if 512 < n_tok <= 1025 else {} - exe = (_get_bf16_local(1 if n_tok <= 256 else 2 if n_tok <= 1025 else 3, fused_pack, fused_quant, compute_dtype == "fp16", + shared_local = {"q_shared": True} if tile is not None and tile[1] else {} + exe = (_get_bf16_local(tile[0], fused_pack, fused_quant, compute_dtype == "fp16", q.dtype == torch.float16, **lazy_local, **shared_local) - if local else _get_kernel( + if tile is not None else _get_kernel( compute_dtype, fused_pack, rounds, @@ -307,9 +323,13 @@ def nvfp4_sparse_prefill( # tiles; fp8 at one KV head also has the 513-1024 and 1088-1536 tiles WARM_TOKENS = (8, 512, 1024, 1536, 2048) WARM_TOKENS_FP8 = (8, 512, 1024, 1536, 2048) +# two KV heads: the ``HKV2_TILES`` tiles, the one-round (fp8, scheduling ports) and two-round kernels +WARM_TOKENS_HKV2 = (8, 256, 384, 1024) -def _warm_tokens(compute_dtype: str) -> tuple[int, ...]: +def _warm_tokens(compute_dtype: str, num_kv_heads: int = 1) -> tuple[int, ...]: + if num_kv_heads == 2: + return WARM_TOKENS_HKV2 return WARM_TOKENS_FP8 if compute_dtype == "fp8" else WARM_TOKENS @@ -322,7 +342,7 @@ def _prefill_mode(q, kv_cache, topk, block_table, query_req_id, query_abs_pos, n def _prefill_sizes(mode): - return _warm_tokens(mode[2]) + return _warm_tokens(mode[2], mode[1]) def _prefill_inputs(mode, n, q, *args, **kwargs): @@ -358,7 +378,7 @@ def warm_up_prefill(device: torch.device, *, num_heads: int, num_kv_heads: int, with torch.inference_mode(): kv = torch.zeros((2, 2 * num_kv_heads, BLK, 72), dtype=torch.uint8, device=dev) one = torch.ones((), dtype=torch.float32, device=dev) - for n in _warm_tokens(compute_dtype): + for n in _warm_tokens(compute_dtype, num_kv_heads): q = torch.zeros((n, num_heads, D), dtype=torch.bfloat16, device=dev) idx = torch.full((num_kv_heads, n, TOPK), -1, dtype=torch.int32, device=dev) idx[..., 0] = 0 diff --git a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/README.md b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/README.md index b646d79..8bc01c1 100644 --- a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/README.md +++ b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/README.md @@ -3,11 +3,11 @@ ## What it does Decode attention of a sliding-window layer over an e4m3 KV cache (per-tensor K / V scales), 1-8 query rows per -request. Caller: the MiniMax-M3.1 DSpark draft (5 sliding layers, window 4096): TP4 16 q / 1 KV heads, 128-token -pages, 3 or 7 rows per request, causal or not (the upstream draft is non-causal), up to 256 requests. -`q [rows, 16 Hkv, 128]` bf16; cache `[pages, Hkv, page, 256]` e4m3, 128 K then 128 V bytes per token, page a multiple -of 32. A request's rows are the last positions of its sequence; row p attends `p - window < k < seq_len` (causal: -`k <= p`); requests with `seq_len < rows` (graph padding) write nothing. Contract: +request. Caller: the MiniMax-M3.1 DSpark draft (5 sliding layers, window 4096): TP4 16 q / 1 KV heads and TP2 32 q / +2 KV heads, 128-token pages, 3 or 7 rows per request, causal or not (the upstream draft is non-causal), up to 256 +requests. `q [rows, 16 Hkv, 128]` bf16; cache `[pages, Hkv, page, 256]` e4m3, 128 K then 128 V bytes per token, page a +multiple of 32. A request's rows are the last positions of its sequence; row p attends `p - window < k < seq_len` +(causal: `k <= p`); requests with `seq_len < rows` (graph padding) write nothing. Contract: `reference.py`: exact bf16 K / V, bf16 Q and P, fp32 softmax and sums, bf16 output, splits combined in split order (deterministic, but the split count follows the request count); the MXFP8 output equals the o_proj quantizer's bytes. @@ -22,12 +22,13 @@ of 32. A request's rows are the last positions of its sequence; row p attends `p row's layout (`oproj_codes`) from 8 rows of 3- or 7-row blocks, else the bf16 output. An `amd_kernels.warm_up` entry: the first call of a mode (rows, window, causal, heads, page, o_proj N) compiles 1-256 requests. -`host.py` picks the kernel and the splits (`plan_splits`: one round of CUs, a request's splits on one XCD); -`oproj_input.mxfp8_out` picks the output. +`host.py` picks the kernel and the splits (TP4: `plan_splits`, one round of CUs, a request's splits on one XCD, and the +query tiles; TP2: `_TP2_PLANS`); `oproj_input.mxfp8_out` picks the output. ## Performance Graph replay of 100 launches with their own queries, block tables and lengths, K / V cold (full windows), us per call. +TP2 baseline: the request-wide `kernel.py` with its split plan (`plan_splits`). | Case | Requests -> us | |---|---| @@ -35,10 +36,15 @@ Graph replay of 100 launches with their own queries, block tables and lengths, K | TP4 7 rows causal, bf16 | 1 / 2 / 4 / 16 / 32 / 64 -> 9.83 / 10.22 / 11.85 / 23.35 / 33.65 / 46.78 | | TP4 3 rows non-causal | 1 / 4 / 16 / 32 / 48 / 64 -> bf16 15.90 / 16.80 / 17.54 / 20.22 / 21.91 / 24.25, MXFP8 16.30 / 17.20 / 17.93 / 20.59 / 22.22 / 24.66 | | TP4 4 rows causal, bf16 | 1 / 16 / 64 / 128 / 256 -> 20.55 / 22.83 / 28.86 / 39.75 / 65.70 | +| TP2 7 rows non-causal, bf16 | 1 / 2 / 4 / 8 / 16 / 24 / 32 / 40 / 48 / 64 / 96 / 128 / 256 -> 9.97 / 11.70 / 14.85 / 18.90 / 27.13 / 30.81 / 34.39 / 37.57 / 42.49 / 47.94 / 69.82 / 81.55 / 163.53 vs baseline 37.57 / 38.37 / 39.08 / 39.34 / 43.87 / 44.96 / 49.48 / 53.26 / 69.46 / 70.40 / 115.10 / 116.11 / 231.14 | +| TP2 7 rows causal, bf16 | 1 / 4 / 16 / 32 / 48 / 64 / 128 -> 9.95 / 14.86 / 27.36 / 34.32 / 42.52 / 47.85 / 82.12 | +| TP2 3 rows non-causal, bf16 | 1 / 4 / 16 / 32 / 48 / 64 / 128 -> 12.73 / 14.21 / 17.31 / 21.23 / 26.44 / 32.51 / 52.95 vs baseline 16.11 / 17.07 / 20.03 / 24.18 / 30.50 / 34.06 / 54.73 | ## Tests ```bash python3 -m pytest tests/attention/swa_decode_fp8kv_gqa16_d128 -q python3 benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py --case tp4_dspark7_noncausal_p128_bf16 +python3 benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py --case tp2_dspark7_noncausal_p128_bf16 +python3 benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py ``` diff --git a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/bf16mfma_gqa16_d128_p128_ring.py b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/bf16mfma_gqa16_d128_p128_ring.py new file mode 100644 index 0000000..c36188b --- /dev/null +++ b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/bf16mfma_gqa16_d128_p128_ring.py @@ -0,0 +1,705 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""FP8-KV sliding-window attention over 128-token pages staged through an LDS ring, up to 8 query rows per request. + +A workgroup is one (request, KV head, split). Its waves are 4 key lanes x row halves of ``rows_per_half`` rows (one +half up to that many rows). Key lane j of every half attends keys 32 j .. 32 j + 31 of each page, so the waves of a +SIMD read the same chunk, each for its own rows (<= 4 rows of O: 128 registers, two waves per SIMD). + +A step is one page of the split's contiguous page range. Every wave DMAs its 1 KB pieces of the page +(``buffer_load ... lds``, lane-linear) into a ``STAGES``-deep LDS ring, ``STAGES - 1`` pages ahead; a step waits for +its page's pieces (``vmcnt``), meets the other waves at a barrier (which also frees the previous step's slot) and +issues the next DMA. The DMAs are inline asm: LLVM would make every LDS read wait for all of them. Token t's 16 B +column c sits at column ``c ^ swz(t)`` (``_swz``), so a K read (16 tokens per 16 lanes) and a V transpose read (16 +tokens per 32 lanes) hit distinct banks. Bytes past the sequence end are out of the DMA's range and land as zeros (an +unwritten slot may hold an e4m3 NaN). + +Per chunk: S^T = K Q^T (v_mfma_f32_16x16x32_bf16, 16 keys x 16 heads; Q^T in LDS), online softmax in fp32 with keys +8 g .. 8 g + 7 in lane group g, O^T += V^T P^T straight from S^T. O^T tile dt is dims 16 dt .. 16 dt + 15, so its +V^T operand comes from one ``ds_read_b64_tr_b8`` (no byte transposes) and lane (n, g) holds dims 16 dt + 4 g + e of +head n; an MXFP8 group of 32 dims spans the 4 lanes of a head. The running max is lazy: a row keeps its reference max +until a score passes it by 2^TAU_LOG2, so the rescale branch is rare and most chunks are one basic block; +P = exp2(s c - m) <= 2^TAU_LOG2 is rounded to bf16 (the same relative rounding), sums stay fp32. + +After the key loop the waves of a half merge its rows through LDS (the ring is free), and with splits > 1 the last +split of a request (a ticket) combines the partials in split order, so the result does not depend on arrival order. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from . import CH, MAX_QL +from .kernel import ( + AGENT_SCOPE, + D, + HG, + TINY_F32, + NDT, + NKS, + PART_F, + PRELOAD_HINTS, + TOK_B, + _as_f32, + _bf16x8_e4m3, + _bits, + _exp2, + _global_atomic_add, + _global_i32_ptr, + _maxf_nn, + _mfma, + _permlane_swap, + _pk_bf16, + _pk_fp8, + _xlane4, +) + +PAGE = 128 +PAGE_B = PAGE * TOK_B # one (page, KV head): 32 KB +STAGES = 4 +TAU_LOG2 = 8.0 +KEY_LANES = 4 +ROWS_PER_HALF = 4 +DMA_B = 1024 # bytes per wave per DMA instruction (64 lanes x 16 B) + + +def _swz(t): + """The 16 B column XOR of token ``t``: column bits (t0, t1, t2 ^ t4, t3), a bijection over the 16 tokens of a + K read (bits 0, 1, 3, 4; bit 2 fixed) and of a V transpose read (bits 0-3; bit 4 fixed).""" + return (t & 3) | (((t >> 2) ^ (t >> 4)) & 1) << 2 | (t & 8) + + +def _lds_tr8(addr): + """``ds_read_b64_tr_b8`` at LDS byte ``addr``: lane 16 G + 8 h + k gets byte k of the 8 bytes addressed by + lanes 16 G + 2 j + h, j = 0 .. 7.""" + ptr = _llvm.IntToPtrOp(_ir.Type.parse("!llvm.ptr<3>"), fx.as_ir_value(fx.Int32(addr))).result + return Vec(_rocdl_ops.ds_read_tr8_b64(Vec.make_type(2, fx.Int32), ptr).result) + + +def _imax(a, b): + return (a > b).select(a, b) + + +def _mxfp8_group8(words): + """MXFP8 of a 32-value group held 8 per lane by the 4 lanes n, n + 16, n + 32, n + 48: ``words`` are this lane's + 4 packed-bf16 dwords. Returns its 2 code dwords and the group's e8m0 scale (fp32, 0..254), as + ``kernel._mxfp8_group`` computes them (the amax an integer max over the magnitude bits, so a NaN stays NaN).""" + mags = [] + for wd in words: + mags += [wd & fx.Int32(0x7FFF), (wd >> 16) & fx.Int32(0x7FFF)] + while len(mags) > 1: + mags = [_imax(mags[i], mags[i + 1]) for i in range(0, len(mags), 2)] + mag = mags[0] + a, b = _permlane_swap(_rocdl.permlane32_swap, mag, mag) + mag = _imax(a, b) + a, b = _permlane_swap(_rocdl.permlane16_swap, mag, mag) + mag = _imax(a, b) + amax = fx.arith.maximumf(fx.Float32(TINY_F32), _as_f32(mag << 16)) + sb = fx.math.ceil(fx.math.log2(amax / fx.Float32(448.0))) + fx.Float32(127.0) + sb = fx.arith.minimumf(fx.arith.maximumf(sb, fx.Float32(0.0)), fx.Float32(254.0)) + rs = fx.Float32(fx.math.exp2(fx.Float32(127.0) - sb)) + dws = [] + for i in range_constexpr(2): + w0, w1 = words[2 * i], words[2 * i + 1] + lo = _pk_fp8(_as_f32(w0 << 16) * rs, _as_f32(w0 & fx.Int32(-65536)) * rs, fx.Int32(0), False) + dws.append(_pk_fp8(_as_f32(w1 << 16) * rs, _as_f32(w1 & fx.Int32(-65536)) * rs, lo, True)) + return dws, sb + + +def _dma16(m0, voff, rsrc): + """``buffer_load_dwordx4 ... lds``: 16 B per lane to LDS byte m0 + 16 lane (inline asm: no LLVM waitcnt).""" + _llvm.inline_asm( + None, + [fx.as_ir_value(fx.Int32(m0)), fx.as_ir_value(fx.Int32(voff)), fx.as_ir_value(rsrc)], + "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, 0 offen lds", + "s,v,s,~{m0}", + has_side_effects=True, + ) + + +def _sync(): + """LDS writes done, then ``s_barrier`` (an intrinsic: MLIR erases back-to-back ``gpu.barrier`` ops, and the two + halves' bodies must meet the same number of barriers).""" + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.rocdl.s_barrier() + + +def row_halves(ql: int, rows_per_half: int = ROWS_PER_HALF) -> tuple[int, ...]: + """Rows per half: (ql,) up to ``rows_per_half`` rows, else (rows_per_half, ql - rows_per_half).""" + return (ql,) if ql <= rows_per_half else (rows_per_half, ql - rows_per_half) + + +def compile_swa_fp8kv_bf16mfma_gqa16_d128_p128_ring( + ql: int, + splits: int, + window: int, + causal: bool, + xcds: int, + quant: bool = False, + rows_per_half: int = ROWS_PER_HALF, +): + """``ql`` query rows per request (1..8), ``splits`` workgroups per (request, KV head) over contiguous page ranges, + 128-token pages; ``xcds`` > 0: a request's splits on one XCD; ``quant``: the MXFP8 o_proj-input epilogue.""" + assert 1 <= ql <= MAX_QL and splits >= 1 + assert 0 < window <= 64 * CH * 4 - max(ql, 6) - CH + 2 + QL, S, W, XCDS = ql, splits, window, xcds + HALVES = row_halves(ql, rows_per_half) + NH = len(HALVES) + NW = KEY_LANES * NH + NDMA = PAGE_B // (NW * DMA_B) # DMA instructions per wave and page + PART_SCOPE = 0 if XCDS else AGENT_SCOPE + QUANT = quant + QSZ = QL * NKS * 64 * 4 # Q^T operand words + RING_I = STAGES * PAGE_B // 4 + MR = 2 # rows of a half merged per round + O_I = NH * MR * KEY_LANES * NDT * 64 * 4 # merge slots, in the ring after the key loop + assert O_I <= RING_I and NW * QL * 2 * 64 <= QSZ # (max, sum) per (wave, row, lane) in the dead Q^T + name = ( + f"swa_fp8kv_bf16mfma_gqa16_d128_ring_r{QL}_s{S}_w{W}_p{PAGE}_n{NW}" + + (f"_h{HALVES[0]}" if rows_per_half != ROWS_PER_HALF else "") + + ("_c" if causal else "") + + (f"_x{XCDS}" if XCDS else "") + + ("_mxfp8" if quant else "") + ) + + @fx.struct + class Shared: + ring: fx.Array[fx.Int32, RING_I, 16] + q: fx.Array[fx.Int32, QSZ, 16] + ticket: fx.Array[fx.Int32, 4] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + i32_reqs: fx.Int32, + i32_bt_s: fx.Int32, # block-table row stride (elements) + i32_page_b: fx.Int32, # cache page stride (bytes) + i32_head_b: fx.Int32, # cache KV-head stride (bytes) + i32_H: fx.Int32, # query heads + i32_hkv: fx.Int32, + arg_seq: fx.Int64, # [reqs] int32 + arg_qsl: fx.Int64, # [reqs + 1] int32 + arg_bt: fx.Int64, # [reqs, stride] int32 + arg_q: fx.Int64, # [rows, H, 128] bf16 + arg_cache: fx.Int64, # [pages, Hkv, 128, 256] e4m3 + arg_out: fx.Int64, # [rows, H, 128] bf16 + arg_ks: fx.Int64, # fp32 K scale + arg_vs: fx.Int64, # fp32 V scale + arg_part: fx.Int64, # splits: [grid requests * Hkv, S, QL] records of 64 PART_F fp32: O^T, then (max, sum) + arg_cnt: fx.Int64, # splits: [grid requests * Hkv] int32 tickets, zero + i64_part_bytes: fx.Int64, + i32_rows: fx.Int32, + f32_c: fx.Float32, # sm_scale * log2(e) + arg_qc: fx.Int64, # quant: [>= ceil(rows / 16) * 16, H * 128] e4m3 codes + arg_qs: fx.Int64, # quant: [>= ceil(rows / 32) * 32, H * 4] e8m0 scales + i32_tile: fx.Int32, # quant: 0 row-major codes, > 0 the (16, 16) tile layout + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx = fx.thread_idx.x + bx, kvh = fx.block_idx.x, fx.block_idx.y + lane = tx % 64 + w = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + n, g = lane % 16, lane // 16 + if const_expr(XCDS > 0): + # grid.x is padded to XCDS x S: a request's splits share an XCD + xcd = bx % XCDS + slot = bx // XCDS + req = (slot // S) * XCDS + xcd + split = slot % S + else: + req = bx // S + split = bx % S + req_ok = req < i32_reqs + reqc = req_ok.select(req, fx.Int32(0)) + seq_p = _global_i32_ptr(arg_seq) + qsl_p = _global_i32_ptr(arg_qsl) + bt = _global_i32_ptr(arg_bt) + f32p = fx.PointerType.get(T.f32, address_space=fx.AddressSpace.Global, alignment=4) + # (the scales are made uniform here: a use inside the key loop would wait for every DMA in flight) + ks_ld = fx.Float32(fx.inttoptr(f32p, fx.Int64(arg_ks))[0]) + vs_ld = fx.Float32(fx.inttoptr(f32p, fx.Int64(arg_vs))[0]) + seq_ld = fx.Int32(seq_p[reqc]) + q0_ld = fx.Int32(qsl_p[reqc]) + q1_ld = fx.Int32(qsl_p[reqc + 1]) + c = _as_f32(fx.rocdl.readfirstlane(T.i32, _bits(f32_c * ks_ld))) + inv_c = fx.Float32(1.0) / c + v_scale = _as_f32(fx.rocdl.readfirstlane(T.i32, _bits(vs_ld))) + neg_inf = fx.Float32(float("-inf")) + + seq_len = fx.Int32(fx.rocdl.readfirstlane(T.i32, seq_ld)) + q0 = fx.Int32(fx.rocdl.readfirstlane(T.i32, q0_ld)) + q1 = fx.Int32(fx.rocdl.readfirstlane(T.i32, q1_ld)) + qlen = req_ok.select(q1 - q0, fx.Int32(0)) + qlen = (qlen > QL).select(fx.Int32(QL), qlen) + qlen = (seq_len < qlen).select(fx.Int32(0), qlen) # graph padding: no keys + ctx = seq_len - qlen + k_lo = ctx - (W - 1) + k_lo = (k_lo < 0).select(fx.Int32(0), k_lo) + p_first = k_lo // PAGE + n_pg = (qlen > 0).select((seq_len - 1) // PAGE - p_first + 1, fx.Int32(0)) + v_beg = (split * n_pg) // S + n_st = fx.Int32(fx.rocdl.readfirstlane(T.i32, ((split + 1) * n_pg) // S - v_beg)) + pb = p_first + v_beg # the split's first page (logical block) + head_off = kvh * i32_head_b + # lane l = the page of step l + lane_ok = lane < n_st + pg_lane = fx.Int32(bt[reqc * i32_bt_s + pb + lane_ok.select(lane, fx.Int32(0))]) + pg_lane = lane_ok.select(pg_lane, fx.Int32(0)) + + # Q^T into LDS in operand order: row t, k-step ks = 2 h + s, lane (n, g) holds dims 64 h + 16 g + 8 s .. + 8 + # of head n (the Q^T stores wait for page 0's DMA too) + lds_atom = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + q_lds = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.q.ptr), fx.make_layout(QSZ, 1)), + fx.make_layout(4, 1), + ) + qr = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=fx.Int64(i32_rows) * fx.Int64(i32_H) * (D * 2) + ) + NQI = (QL * HG * 16 + 64 * NW - 1) // (64 * NW) + q_items = [] + for k in range_constexpr(NQI): + item = tx + k * (64 * NW) # (row, head, 16-byte piece j of the head) + t_i = item // (HG * 16) + h_i = (item // 16) % HG + jp = item % 16 + row = q0 + (t_i < qlen).select(t_i, fx.Int32(0)) + qv = fx.Vector( + buffer_ops.buffer_load( + qr, ((row * i32_H + kvh * HG + h_i) * D + 8 * jp) // 2, vec_width=4, dtype=fx.Int32 + ) + ) + q_items.append((item, t_i, h_i, jp, qv)) + + # DMA: wave w's piece k of a page is LDS bytes (k NW + w) KB of the slot; lane l writes 16 B unit + # u = (k NW + w) 64 + l = token u // 16, column u % 16, read from column (u % 16) ^ swz(token) + ring_base = fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.Int32(fx.ptrtoint(smem.ring.ptr)))) + voffs = [] + for k in range_constexpr(NDMA): + tok = (k * NW + w) * 4 + lane // 16 + voffs.append(tok * TOK_B + ((lane % 16) ^ _swz(tok)) * 16) + + def dma_page(step, stage): + """The DMA of step ``step`` (past the split's pages: an empty range, zeros) into ring slot ``stage``.""" + ok = step < n_st + stc = ok.select(step, fx.Int32(0)) + pg = fx.Int32(fx.rocdl.readlane(T.i32, pg_lane, stc)) + valid = seq_len - (pb + stc) * PAGE + valid = (valid > PAGE).select(fx.Int32(PAGE), valid) + valid = (valid < 0).select(fx.Int32(0), valid) + valid = ok.select(valid, fx.Int32(0)) + valid = fx.Int32(fx.rocdl.readfirstlane(T.i32, valid)) + rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_cache + fx.Int64(pg) * fx.Int64(i32_page_b) + fx.Int64(head_off), + num_records_bytes=valid * TOK_B, + ) + for k in range_constexpr(NDMA): + _dma16(ring_base + stage * PAGE_B + (k * NW + w) * DMA_B, voffs[k], rsrc) + + fx.rocdl.sched_barrier(0) # the Q loads go out before the page lookup wait + dma_page(fx.Int32(0), fx.Int32(0)) + for k in range_constexpr(NQI): + item, t_i, h_i, jp, qv = q_items[k] + ks_i = 2 * (jp // 8) + jp % 2 + g_i = (jp % 8) // 2 + if item < QL * HG * 16: + r_q = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r_q.store(qv) + fx.copy(lds_atom, r_q, fx.slice(q_lds, (None, (t_i * NKS + ks_i) * 64 + g_i * 16 + h_i))) + for s_ in range_constexpr(1, STAGES - 1): + dma_page(fx.Int32(s_), fx.Int32(s_)) + + ring16 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.ring.ptr), fx.make_layout(RING_I, 1)), + fx.make_layout(4, 1), + ) + jl = w % KEY_LANES + # K: lane (n, g), tile st, half h: token 32 j + 8 (n // 4) + 4 st + n % 4, column 4 h + g + k_units = [] + for st in range_constexpr(2): + tk = jl * CH + 8 * (n // 4) + 4 * st + n % 4 + for h in range_constexpr(2): + k_units.append(tk * 16 + ((4 * h + g) ^ _swz(tk))) + # V by transpose reads: as a supplier, lane (n, g) addresses token 8 g + n // 2, bytes 128 + 16 dt + 8 (n % 2) + # .. + 8, so lane (n, g) receives dim 16 dt + n of tokens 8 g .. 8 g + 7: the V^T operand of O^T tile dt + tv = jl * CH + 8 * g + n // 2 + v_bytes = [tv * TOK_B + ((8 + dt) ^ _swz(tv)) * 16 + 8 * (n % 2) for dt in range_constexpr(NDT)] + + def read_chunk(stage): + """This lane's 16 K dwords (the global loads' order) and per O^T tile dt the 2 dwords of its V^T + operand (8 e4m3 of tokens 8 g .. 8 g + 7) of its key lane's chunk in ring slot ``stage``.""" + raw = [] + for i in range_constexpr(4): + r_k = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(lds_atom, fx.slice(ring16, (None, stage * (PAGE_B // 16) + k_units[i])), r_k) + kv = Vec(r_k.load()) + raw += [kv[e] for e in range_constexpr(4)] + sb_ = ring_base + stage * PAGE_B + for dt in range_constexpr(NDT): + vv = _lds_tr8(sb_ + v_bytes[dt]) + raw += [vv[0], vv[1]] + return raw + + # per-row key bounds: row t at position ctx + t sees [ctx + t - W + 1, hi) + klo = [ctx + (t - (W - 1)) for t in range_constexpr(QL)] + khi = [(ctx + (t + 1)) if causal else seq_len for t in range_constexpr(QL)] + last_t = (qlen > 0).select(qlen - 1, fx.Int32(0)) + full_lo = ctx + last_t - (W - 1) + full_hi = (ctx + 1) if causal else seq_len + zero4 = Vec.filled(4, 0.0, fx.Float32) + + def q_frag(t, ks): + r_f = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(lds_atom, fx.slice(q_lds, (None, (t * NKS + ks) * 64 + lane)), r_f) + return Vec(r_f.load()) + + def widen_v(vraw): + """V^T operands vf[dt] (bf16): dim 16 dt + n of tokens 8 g .. 8 g + 7.""" + return [_bf16x8_e4m3(vraw[2 * dt], vraw[2 * dt + 1]) for dt in range_constexpr(NDT)] + + outr = buffer_ops.create_buffer_resource_from_addr( + arg_out, num_records_bytes=fx.Int64(i32_rows) * fx.Int64(i32_H) * (D * 2) + ) + + def store_out(t, acc, inv): + """Row t's O (``acc`` x ``inv``) in bf16: lane (n, g) holds dims 16 dt + 4 g + e of head n.""" + o_row = ((q0 + t) * i32_H + kvh * HG + n) * (D // 2) + g * 2 + for dt in range_constexpr(NDT): + val = Vec.from_elements( + [_pk_bf16(acc[dt][0] * inv, acc[dt][1] * inv), _pk_bf16(acc[dt][2] * inv, acc[dt][3] * inv)], + fx.Int32, + ) + buffer_ops.buffer_store(val, outr, o_row + dt * 8, mask=t < qlen) + + def store_q(t, acc, inv): + """Row t's o_proj input: the MXFP8 codes of O rounded to bf16 (row-major, or the (16, 16) tile layout + when ``i32_tile`` > 0) and the e8m0 scales; group b (dims 32 b .. 32 b + 31 of head n, tiles 2 b and + 2 b + 1) spans lanes g = 0 .. 3, and lane g writes group g's scale.""" + row = q0 + t + k_row = i32_H * D + head = kvh * HG + n + rows16 = (i32_rows + 15) // 16 * 16 + qcr = buffer_ops.create_buffer_resource_from_addr( + arg_qc, num_records_bytes=fx.Int64(rows16) * fx.Int64(k_row) + ) + shuffled = i32_tile > 0 + sbs = [] + for b in range_constexpr(4): + words = [ + _pk_bf16(acc[2 * b + i][2 * j] * inv, acc[2 * b + i][2 * j + 1] * inv) + for i in range_constexpr(2) + for j in range_constexpr(2) + ] + dws, sb = _mxfp8_group8(words) + sbs.append(sb) + for i in range_constexpr(2): + k = head * D + 32 * b + 16 * i + 4 * g + off_t = ( + (row // 16) * (16 * k_row) + (k // 64) * 1024 + ((k % 64) // 16) * 256 + (row % 16) * 16 + + k % 16 + ) // 4 + off_l = (row * k_row + k) // 4 + buffer_ops.buffer_store(dws[i], qcr, shuffled.select(off_t, off_l), mask=t < qlen) + sb = (g == 0).select(sbs[0], (g == 1).select(sbs[1], (g == 2).select(sbs[2], sbs[3]))) + gi = head * 4 + g + s_off = ( + ((((row // 32) * (k_row // 256) + gi // 8) * 4 + gi % 4) * 16 + row % 16) * 2 + (gi % 8) // 4 + ) * 2 + (row % 32) // 16 + rows32 = (i32_rows + 31) // 32 * 32 + qsr = buffer_ops.create_buffer_resource_from_addr( + arg_qs, num_records_bytes=fx.Int64(rows32) * fx.Int64(k_row // 32) + ) + buffer_ops.buffer_store(sb.to(fx.Int32).to(fx.Int8), qsr, s_off, mask=t < qlen) + + def store_row(t, acc, inv): + if const_expr(QUANT): + store_q(t, acc, inv) + else: + store_out(t, acc, inv) + + tid = req * i32_hkv + kvh + pr = buffer_ops.create_buffer_resource_from_addr(arg_part, num_records_bytes=i64_part_bytes) + o_lds = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.ring.ptr), fx.make_layout(O_I, 1)), + fx.make_layout(4, 1), + ) + ml = fx.recast_iter(fx.Float32, smem.q.ptr) + + def body(R0, NR, jl): + """The key loop and the row merge of the half whose rows are R0 .. R0 + NR - 1 (static).""" + + def scores(raw, k0): + """S^T of the half's rows for one chunk (keys k0 ..), masked at the window's edges: value 4 st + e + of a lane is key k0 + 8 g + 4 st + e.""" + s_all = [[zero4, zero4] for _ in range_constexpr(NR)] + for ks in range_constexpr(NKS): + kf = [ + _bf16x8_e4m3( + raw[st * 8 + (ks // 2) * 4 + 2 * (ks % 2)], + raw[st * 8 + (ks // 2) * 4 + 2 * (ks % 2) + 1], + ) + for st in range_constexpr(2) + ] + for r in range_constexpr(NR): + qv = q_frag(R0 + r, ks) + s_all[r] = [_mfma(kf[st], qv, s_all[r][st]) for st in range_constexpr(2)] + edge = (k0 < full_lo) | (k0 + CH > full_hi) + if edge: + key = k0 + 8 * g + s_all = [ + [ + Vec.from_elements( + [ + ((key + 4 * st + e >= klo[R0 + r]) & (key + 4 * st + e < khi[R0 + r])).select( + s_all[r][st][e], neg_inf + ) + for e in range_constexpr(4) + ], + fx.Float32, + ) + for st in range_constexpr(2) + ] + for r in range_constexpr(NR) + ] + return s_all + + def accumulate(s_all, vraw, m_t, l_t, th_t, mn_t, o_t): + """A chunk's scores and V into the half's rows' state: reference max m (scaled units), sum l, the + raw-score threshold th = (m + TAU_LOG2) / c that triggers a rescale, mn = -m (0 while m is -inf), + and O.""" + lmax = [] + for r in range_constexpr(NR): + a = [s_all[r][i // 4][i % 4] for i in range_constexpr(8)] + mx = a[0] + for i in range_constexpr(1, 8): + mx = _maxf_nn(mx, a[i]) + lmax.append(mx) + need = lmax[0] > th_t[0] + for r in range_constexpr(1, NR): + need = need | (lmax[r] > th_t[r]) + if fx.Int64(fx.rocdl.ballot(T.i64, need)) != fx.Int64(0): + m_n = [_maxf_nn(m_t[r], _xlane4(lmax[r], _maxf_nn) * c) for r in range_constexpr(NR)] + mu_n = [(m_n[r] == neg_inf).select(fx.Float32(0.0), m_n[r]) for r in range_constexpr(NR)] + alphas = [_exp2(m_t[r] - mu_n[r]) for r in range_constexpr(NR)] + o_t = [ + [o_t[r][dt] * Vec.filled(4, alphas[r], fx.Float32) for dt in range_constexpr(NDT)] + for r in range_constexpr(NR) + ] + l_t = [l_t[r] * alphas[r] for r in range_constexpr(NR)] + th_t = [(m_n[r] + fx.Float32(TAU_LOG2)) * inv_c for r in range_constexpr(NR)] + mn_t = [-mu_n[r] for r in range_constexpr(NR)] + m_t = m_n + c4 = Vec.filled(4, c, fx.Float32) + pfs = [] + l_o = [] + for r in range_constexpr(NR): + mn4 = Vec.filled(4, mn_t[r], fx.Float32) + p = [] + for st in range_constexpr(2): + x4 = Vec(fx.math.fma(s_all[r][st], c4, mn4)) + p += [_exp2(x4[e]) for e in range_constexpr(4)] + q4 = Vec.from_elements(p[0:4], fx.Float32) + Vec.from_elements(p[4:8], fx.Float32) + q2 = Vec.from_elements([q4[0], q4[1]], fx.Float32) + Vec.from_elements([q4[2], q4[3]], fx.Float32) + l_o.append(l_t[r] + (q2[0] + q2[1])) + pfs.append( + Vec.from_elements([_pk_bf16(p[2 * i], p[2 * i + 1]) for i in range_constexpr(4)], fx.Int32) + ) + vf = widen_v(vraw) + o_o = [ + [_mfma(vf[dt], pfs[r], o_t[r][dt]) for dt in range_constexpr(NDT)] for r in range_constexpr(NR) + ] + return m_t, l_o, th_t, mn_t, o_o + + NS = 4 * NR # scalar state: m, l, th, mn per row + init = [neg_inf for _ in range_constexpr(NR)] + init += [fx.Float32(0.0) for _ in range_constexpr(NR)] + init += [neg_inf for _ in range_constexpr(NR)] + init += [fx.Float32(0.0) for _ in range_constexpr(NR)] + init += [zero4 for _ in range_constexpr(NR * NDT)] + results = init + for it, st_ in range( # type: ignore[call-overload] + fx.Index(0), fx.Index(n_st), fx.Index(1), init=[fx.as_ir_value(x) for x in init] + ): + sc = [[fx.Float32(st_[e * NR + r]) for r in range_constexpr(NR)] for e in range_constexpr(4)] + o_t = [[Vec(st_[NS + r * NDT + dt]) for dt in range_constexpr(NDT)] for r in range_constexpr(NR)] + v = fx.Int32(it) + stage = v % STAGES + # this step's page has landed (each wave waits for its own pieces, the barrier for the others') + # and every wave is done with the previous step's slot + fx.rocdl.s_waitcnt(vmcnt=(STAGES - 2) * NDMA, lgkmcnt=0) + fx.rocdl.s_barrier() + dma_page(v + (STAGES - 1), (v + (STAGES - 1)) % STAGES) + raw = read_chunk(stage) + k0 = (pb + v) * PAGE + jl * CH + m_t, l_t, th_t, mn_t, o_t = accumulate(scores(raw, k0), raw[16:32], sc[0], sc[1], sc[2], sc[3], o_t) + state = m_t + l_t + th_t + mn_t + [o for row in o_t for o in row] + results = yield [fx.as_ir_value(x) for x in state] + + # the ring is free once every DMA has landed and every wave has left the loop + fx.rocdl.s_waitcnt(vmcnt=0) + _sync() + for r in range_constexpr(NR): + for e in range_constexpr(2): + ml[((w * QL + R0 + r) * 2 + e) * 64 + lane] = fx.Float32(results[e * NR + r]) + hs = R0 // HALVES[0] # this half's merge slot + + def put_o(r, slot): + for dt in range_constexpr(NDT): + r_w = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r_w.store(Vec(results[NS + r * NDT + dt]).bitcast(fx.Int32)) + fx.copy(lds_atom, r_w, fx.slice(o_lds, (None, ((slot * KEY_LANES + jl) * NDT + dt) * 64 + lane))) + + def merge_row(t, slot): + """Row t's key lanes merged from the half's LDS slot: (acc, max, sum).""" + w0 = hs * KEY_LANES + ms = [fx.Float32(ml[(((w0 + kl) * QL + t) * 2) * 64 + lane]) for kl in range_constexpr(KEY_LANES)] + big = ms[0] + for kl in range_constexpr(1, KEY_LANES): + big = _maxf_nn(big, ms[kl]) + big_raw = big + big = (big == neg_inf).select(fx.Float32(0.0), big) + wt = [fx.Float32(_exp2(ms[kl] - big)) for kl in range_constexpr(KEY_LANES)] + l_tot = fx.Float32(0.0) + for kl in range_constexpr(KEY_LANES): + l_w = fx.Float32(ml[(((w0 + kl) * QL + t) * 2 + 1) * 64 + lane]) + l_tot = l_tot + l_w * wt[kl] + l_tot = _xlane4(l_tot, lambda x, y: x + y) + acc = [zero4 for _ in range_constexpr(NDT)] + for kl in range_constexpr(KEY_LANES): + wv = Vec.filled(4, wt[kl], fx.Float32) + for dt in range_constexpr(NDT): + r_c = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + src = fx.slice(o_lds, (None, ((slot * KEY_LANES + kl) * NDT + dt) * 64 + lane)) + fx.copy(lds_atom, src, r_c) + acc[dt] = acc[dt] + Vec(r_c.load()).bitcast(fx.Float32) * wv + return acc, big_raw, l_tot + + # MR rows of each half per round, merged by key lanes 0 .. MR - 1 (every half takes the first half's + # rounds: the barriers match) + for k in range_constexpr((HALVES[0] + MR - 1) // MR): + if const_expr(k > 0): + _sync() # the previous round's reads are done + for i in range_constexpr(MR): + if const_expr(k * MR + i < NR): + put_o(k * MR + i, hs * MR + i) + _sync() + for i in range_constexpr(MR): + if const_expr(k * MR + i < NR): + t = R0 + k * MR + i + if jl == i: + acc, big_raw, l_tot = merge_row(t, hs * MR + i) + if const_expr(S > 1): + rec = ((tid * S + split) * QL + t) * (64 * PART_F) + for dt in range_constexpr(NDT): + buffer_ops.buffer_store( + acc[dt].bitcast(fx.Int32), pr, rec + dt * 64 * 4 + lane * 4, + cache_modifier=PART_SCOPE, + ) + buffer_ops.buffer_store( + Vec.from_elements([_bits(big_raw), _bits(l_tot)], fx.Int32), + pr, + rec + NDT * 64 * 4 + lane * 2, + cache_modifier=PART_SCOPE, + ) + else: + inv = (l_tot > fx.Float32(0.0)).select(v_scale / l_tot, fx.Float32(0.0)) + store_row(t, acc, inv) + + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() # Q^T is in LDS + if const_expr(NH == 1): + body(0, HALVES[0], jl) + else: + if w < KEY_LANES: + body(0, HALVES[0], jl) + if w >= KEY_LANES: + body(HALVES[0], HALVES[1], jl) + + if const_expr(S > 1): + # every row's partial is stored before the ticket is taken + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + smem.ticket.ptr[0] = _global_atomic_add(arg_cnt + fx.Int64(tid) * 4, 1) + fx.gpu.barrier() + last = fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.Int32(smem.ticket.ptr[0]))) + if last == S - 1: + for batch in range_constexpr((QL + NW - 1) // NW): + t_f = w + batch * NW + if t_f < QL: + recs = [((tid * S + sp) * QL + t_f) * (64 * PART_F) for sp in range_constexpr(S)] + mls = [ + Vec( + buffer_ops.buffer_load( + pr, rc + NDT * 64 * 4 + lane * 2, vec_width=2, dtype=fx.Int32, + cache_modifier=PART_SCOPE, + ) + ) + for rc in recs + ] + bigs = [_as_f32(x[0]) for x in mls] + top = bigs[0] + for sp in range_constexpr(1, S): + top = _maxf_nn(top, bigs[sp]) + top = (top == neg_inf).select(fx.Float32(0.0), top) + ws = [fx.Float32(_exp2(b - top)) for b in bigs] + l_all = fx.Float32(0.0) + for sp in range_constexpr(S): + l_all = l_all + _as_f32(mls[sp][1]) * ws[sp] + inv_all = (l_all > fx.Float32(0.0)).select(v_scale / l_all, fx.Float32(0.0)) + acc_all = [zero4 for _ in range_constexpr(NDT)] + for sp in range_constexpr(S): + wv = Vec.filled(4, ws[sp], fx.Float32) + for dt in range_constexpr(NDT): + part = Vec( + buffer_ops.buffer_load( + pr, recs[sp] + dt * 64 * 4 + lane * 4, vec_width=4, dtype=fx.Int32, + cache_modifier=PART_SCOPE, + ) + ).bitcast(fx.Float32) + acc_all[dt] = acc_all[dt] + part * wv + store_row(t_f, acc_all, inv_all) + if (last == S - 1) & (tx == 0): + _global_i32_ptr(arg_cnt + fx.Int64(tid) * 4)[0] = fx.Int32(0) + + @flyc.jit + def launch( + arg_q: fx.Int64, + arg_cache: fx.Int64, + arg_bt: fx.Int64, + arg_seq: fx.Int64, + arg_qsl: fx.Int64, + arg_out: fx.Int64, + arg_ks: fx.Int64, + arg_vs: fx.Int64, + arg_part: fx.Int64, + arg_cnt: fx.Int64, + i64_part_bytes: fx.Int64, + i32_reqs: fx.Int32, + i32_rows: fx.Int32, + i32_H: fx.Int32, + i32_hkv: fx.Int32, + i32_bt_s: fx.Int32, + i32_page_b: fx.Int32, + i32_head_b: fx.Int32, + f32_c: fx.Float32, + arg_qc: fx.Int64, + arg_qs: fx.Int64, + i32_tile: fx.Int32, + i32_grid_x: fx.Int32, + stream: fx.Stream, + ): + kernel( + i32_reqs, i32_bt_s, i32_page_b, i32_head_b, i32_H, i32_hkv, arg_seq, arg_qsl, arg_bt, arg_q, arg_cache, + arg_out, arg_ks, arg_vs, arg_part, arg_cnt, i64_part_bytes, i32_rows, f32_c, arg_qc, arg_qs, i32_tile, + ).launch(grid=(i32_grid_x, i32_hkv, 1), block=(64 * NW, 1, 1), stream=stream) + + launch.kernel_name = name + launch.compile_hints = {**PRELOAD_HINTS, "waves_per_eu": 2 if NW == 8 else 1} + return launch diff --git a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/host.py b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/host.py index 31f81b4..c90ccb4 100644 --- a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/host.py +++ b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/host.py @@ -9,6 +9,7 @@ from . import kernel as _kernel from . import max_window +from .bf16mfma_gqa16_d128_p128_ring import compile_swa_fp8kv_bf16mfma_gqa16_d128_p128_ring from .device import initialize_xcd_topology, on_gfx950 MAX_SPLITS = 6 @@ -95,6 +96,51 @@ def _tp4_q7_rows(reqs: int) -> int: return 0 +@functools.cache +def _get_ring_kernel(ql: int, splits: int, window: int, causal: bool, xcds: int, quant: bool, rows_per_half: int): + return compile_swa_fp8kv_bf16mfma_gqa16_d128_p128_ring( + ql, splits, window, causal, xcds, quant=quant, rows_per_half=rows_per_half + ) + + +# TP2 (2 KV heads per rank, window 4096, 128-token pages, 8 XCDs): per rows per request, (last request count, kernel, +# splits); kernels: tq1 / tq2 / tq3 the query tiles, ring4 / ring2 the LDS-ring kernel with 4 / 2 rows per wave, +# kernel the request-wide kernel. Graph-timed with cold K/V (benchmark_decode.py; causal within 2% of non-causal). +_TP2_PLANS = { + 7: ((1, "tq1", 6), (2, "tq1", 5), (4, "tq1", 3), (6, "tq2", 5), (10, "tq2", 3), (16, "tq2", 2), + (32, "ring4", 4), (40, "ring4", 3), (64, "ring4", 2), (None, "ring4", 1)), + 3: ((4, "ring2", 12), (8, "ring2", 8), (16, "ring2", 6), (24, "ring2", 4), (32, "ring2", 3), (64, "ring2", 2), + (128, "ring2", 1), (None, "kernel", 1)), +} + + +def _tp2_plan(num_reqs: int, max_query_len: int) -> tuple[str, int]: + for last, kind, splits in _TP2_PLANS[max_query_len]: + if last is None or num_reqs <= last: + return kind, splits + + +def _plan(num_reqs, num_kv_heads, block_size, max_query_len, window, causal, xcds, cus, waves, splits): + """(splits, ring rows per wave (0: not the ring kernel), batch rows, row groups) of a launch: the TP2 table, else + the request-wide split plan with the TP4 query tiles.""" + if (splits is None and num_kv_heads == 2 and max_query_len in _TP2_PLANS and window == 4096 + and block_size == 128 and xcds == 8 and waves == 4): + kind, splits = _tp2_plan(num_reqs, max_query_len) + if kind.startswith("ring"): + return splits, int(kind[4:]), False, 0 + return splits, 0, False, int(kind[2:]) if kind.startswith("tq") else 0 + splits = plan_splits(num_reqs, num_kv_heads, xcds, cus, splits) + # the query tiles and the 2-row batch kernel run split windows only (the TP4 draft + # at <= 128 requests); one split takes the request-wide kernel + group_rows = 0 + if (num_kv_heads == 1 and max_query_len == 7 and window == 4096 + and block_size == 128 and (causal or NONCAUSAL_Q7_TILES) and xcds == 8 + and waves == 4 and splits > 1): + group_rows = _tp4_q7_rows(num_reqs) + batch_rows = num_kv_heads == 1 and max_query_len == 7 and splits > 1 and waves == 4 + return splits, 0, batch_rows, group_rows + + @functools.cache def _split_buffers(device: torch.device): """The splits' partials and tickets (zero; the last split of a request resets @@ -143,12 +189,14 @@ def plan_splits( return splits -def launch_splits(q, kv_cache, num_reqs, splits=None, xcd_count=None) -> int: - """``plan_splits`` with the device's topology, as ``swa_decode_fp8kv`` uses.""" +def launch_splits(q, kv_cache, num_reqs, splits=None, xcd_count=None, max_query_len=None, window=4096) -> int: + """The split count ``swa_decode_fp8kv`` launches with on this device (``max_query_len`` None: the request-wide + plan).""" if xcd_count is None: xcd_count = initialize_xcd_topology() xcds = xcd_count if xcd_count and xcd_count > 1 else 0 - return plan_splits(num_reqs, kv_cache.shape[1], xcds, _cu_count(q.device), splits) + return _plan(num_reqs, kv_cache.shape[1], kv_cache.shape[2], max_query_len, window, True, xcds, + _cu_count(q.device), 4, splits)[0] def supports_swa_decode_fp8kv( @@ -235,29 +283,17 @@ def swa_decode_fp8kv( xcds = xcd_count if xcd_count and xcd_count > 1 else 0 group = xcds if xcds else 1 grid_reqs = -(-num_reqs // group) * group - splits = plan_splits(num_reqs, num_kv_heads, xcds, _cu_count(q.device), splits) - # the query tiles and the 2-row batch kernel run split windows only (the TP4 draft - # at <= 128 requests); one split takes the request-wide kernel - group_rows = 0 - if (num_kv_heads == 1 and max_query_len == 7 and window == 4096 - and block_size == 128 and (causal or NONCAUSAL_Q7_TILES) and xcds == 8 - and waves == 4 and splits > 1): - group_rows = _tp4_q7_rows(num_reqs) + splits, ring_rows, batch_rows, group_rows = _plan(num_reqs, num_kv_heads, block_size, max_query_len, window, + causal, xcds, _cu_count(q.device), waves, splits) part, cnt = _split_buffers(q.device) c = float(np.float32(sm_scale) * np.float32(_kernel.LOG2E)) + if ring_rows: + exe = _get_ring_kernel(max_query_len, splits, window, causal, xcds, quant_out is not None, ring_rows) + else: + exe = _get_kernel(max_query_len, splits, window, causal, block_size, xcds, waves, quant_out is not None, + batch_rows, group_rows) _run_compiled( - _get_kernel( - max_query_len, - splits, - window, - causal, - block_size, - xcds, - waves, - quant_out is not None, - num_kv_heads == 1 and max_query_len == 7 and splits > 1 and waves == 4, - group_rows, - ), + exe, q.data_ptr(), kv_cache.data_ptr(), block_table.data_ptr(), diff --git a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/oproj_input.py b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/oproj_input.py index 2dac9f8..16de61e 100644 --- a/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/oproj_input.py +++ b/amd_kernels/attention/swa_decode_fp8kv_gqa16_d128/oproj_input.py @@ -10,14 +10,13 @@ from . import host as _host -# The MXFP8 epilogue as o_proj's input pays from 8 rows: attention + o_proj per layer, TP4 o_proj 6144 x 2048, graph -# replay, cold K/V and weights (benchmarks/attention/bench_attn_oproj_mxfp8.py): 7-row query -# blocks 14-28 rows -2.2 / -6.7%, 49-448 rows -2.6..-6.1%, 7 rows +1.3%. +# The MXFP8 epilogue as o_proj's input pays from 8 rows: attention + o_proj -2..-7% at 12-448 rows, -0.1..+5% below 8 +# (TP4: benchmarks/attention/bench_attn_oproj_mxfp8.py, TP2: swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py). MXFP8_MIN_ROWS = 8 # The query blocks whose MXFP8 epilogue the tests cover (tests/.../test_oproj_input.py, 1-64 requests): the 7-row and # the 3-row block (the 3-row kernels need the native fptrunc bf16 pack). MXFP8_QUERY_ROWS = (3, 7) -# The request counts the warm-up walks (the TP4 draft serves up to 256 requests per rank). +# The request counts the warm-up walks (the draft serves up to 256 requests per rank). MAX_REQS = 256 diff --git a/amd_kernels/collectives/all_gather_i64_c7_ranks2/README.md b/amd_kernels/collectives/all_gather_i64_c7_ranks2/README.md new file mode 100644 index 0000000..7a4562f --- /dev/null +++ b/amd_kernels/collectives/all_gather_i64_c7_ranks2/README.md @@ -0,0 +1,46 @@ +# Two-rank all-gather of int64 rows [requests, 7] + +## What it does + +`I64C7AllGather` gathers the DSpark draft-token rows of MiniMax-M3.1-Flash TP2 from two ranks straight into a +contiguous int64 `[requests, 7]` output, in one FlyDSL launch, in place of a padded all-gather plus `index_select` +in the batch-sharded draft sampler's draft-id gather. Contract (`reference.py`): integer-exact, every int64 bit pattern +kept; `src[i]` decodes as `(rank, local_row) = divmod(src[i], src_stride)`; only the referenced local rows must be +valid. The communicator is `SamplerAllGather`'s (`collectives/all_gather_sampler_i64_c8_ranks2`): the same IPC pool of +two 22528-byte slots (this entry uses 256 x 56 bytes of each), the same two-slot device sequence and system-scope store +/ load rules, with its own kernel and its own sequence counter. Atomics only poll the sequence flags; output rows use +plain stores. + +## Entry points + +- `I64C7AllGather`: a collective on the two ranks of a gloo group; allocates and maps the pool. +- `I64C7AllGather.warmup`: a collective before graph capture; compiles the one variant (0-256 local and output rows). +- `I64C7AllGather.__call__`: gathers the referenced int64 `[*, 7]` rows (any row stride) into a contiguous output on + the communicator's device; returns it. The caller passes `local_draft_ids[:num_local_reqs]`, + `gathered_src[:out.shape[0]]` and `num_reqs_padded` as the source stride. +- `I64C7AllGather.close`: a collective, after every graph that uses the communicator is retired. + +Both ranks call in the same order, empty senders and receivers included; one communicator per stream. Shapes are +runtime arguments, fixed inside a captured graph; payloads and maps may change on every replay. The call allocates +nothing. + +## Performance + +Two GPUs of one MI355X node. Graphs of 100 calls on 100 distinct payloads, ten replays per round, five alternated +rounds, MAX over ranks, median us, dense owners. Baseline: RCCL all-gather of the padded rows + `index_select`, in the +same benchmark. + +| requests | 10 | 16 | 32 | 40 | 48 | 52 | 64 | 128 | 256 | +|---|---|---|---|---|---|---|---|---|---| +| baseline | 10.25 | 11.91 | 8.50 | 8.55 | 8.55 | 8.58 | 8.60 | 9.85 | 10.08 | +| this entry | 3.98 | 4.01 | 4.06 | 4.09 | 4.11 | 4.10 | 4.11 | 4.41 | 5.19 | + +## Tests + +```bash +python3 -m pytest tests/collectives/all_gather_i64_c7_ranks2 -q # 2 GPUs +python3 benchmarks/collectives/all_gather_i64_c7_ranks2/benchmark_gather.py # 2 GPUs +``` + +Two spawned ranks: graph replay with changed maps and payloads, strided rows, unequal owners, empty ranks, all rows on +one rank, odd counts. The shared protocol's sequence wrap and 1000 slot reuses are tested by the sampler gather. diff --git a/amd_kernels/collectives/all_gather_i64_c7_ranks2/__init__.py b/amd_kernels/collectives/all_gather_i64_c7_ranks2/__init__.py new file mode 100644 index 0000000..774e21c --- /dev/null +++ b/amd_kernels/collectives/all_gather_i64_c7_ranks2/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +from .host import I64C7AllGather +__all__=['I64C7AllGather'] diff --git a/amd_kernels/collectives/all_gather_i64_c7_ranks2/host.py b/amd_kernels/collectives/all_gather_i64_c7_ranks2/host.py new file mode 100644 index 0000000..73be315 --- /dev/null +++ b/amd_kernels/collectives/all_gather_i64_c7_ranks2/host.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Reuse the sampler gather's IPC allocation and teardown, with a separate sequence.""" +import functools +import torch +from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2.host import SamplerAllGather,_run +from .kernel import compile_i64_c7_gather + +@functools.cache +def _exe():return compile_i64_c7_gather() + +class I64C7AllGather(SamplerAllGather): + def __call__(self,x,src,src_stride,out): + assert not self.closed + assert x.ndim==2 and x.shape[1]==7 and x.dtype==torch.int64 and x.stride(1)==1 + assert src.ndim==1 and src.dtype==torch.int64 and src.is_contiguous() + assert out.shape==(src.numel(),7) and out.dtype==torch.int64 and out.is_contiguous() + assert x.device==src.device==out.device==self.device + assert 0<=x.shape[0]<=256 and 0<=src.numel()<=256 and 1<=src_stride<=256 + _run(_exe(),x.data_ptr(),src.data_ptr(),out.data_ptr(),*self.ptrs,*self.flags,self.counter, + x.shape[0],src.numel(),src_stride,x.stride(0),self.rank,torch.cuda.current_stream(self.device)) + return out + + def warmup(self): + dev=self.device + x=torch.zeros(1,7,dtype=torch.int64,device=dev) + src=torch.tensor([self.rank],dtype=torch.int64,device=dev) + self(x,src,1,torch.empty_like(x));torch.cuda.synchronize(dev) diff --git a/amd_kernels/collectives/all_gather_i64_c7_ranks2/kernel.py b/amd_kernels/collectives/all_gather_i64_c7_ranks2/kernel.py new file mode 100644 index 0000000..5c557de --- /dev/null +++ b/amd_kernels/collectives/all_gather_i64_c7_ranks2/kernel.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Single-workgroup seven-column int64 all-gather, two device-sequenced slots.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2.kernel import _read_flag,_sel2,SLOT_BYTES,SYS +NT=256 + +def compile_i64_c7_gather(): + @flyc.kernel(name='all_gather_i64_c7_ranks2',known_block_size=[NT,1,1]) + def kernel(arg_x:fx.Int64,arg_src:fx.Int64,arg_out:fx.Int64, + p0:fx.Int64,p1:fx.Int64,f0:fx.Int64,f1:fx.Int64,counter:fx.Tensor, + nlocal:fx.Int32,nout:fx.Int32,src_stride:fx.Int32,x_stride:fx.Int32,rank:fx.Int32): + tx=fx.thread_idx.x + seq=fx.Int32(counter[0])+1;slot=fx.Int64((seq-1)&1)*SLOT_BYTES + dst=buffer_ops.create_buffer_resource_from_addr(_sel2(rank,p0,p1)+slot,num_records_bytes=SLOT_BYTES) + if tx>32).to(fx.Int32)],fx.Int32), + dst,(tx*FIELDS+col)*2,cache_modifier=SYS) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx==0: + fr=buffer_ops.create_buffer_resource_from_addr(_sel2(rank,f0,f1),num_records_bytes=4) + buffer_ops.buffer_store(seq,fr,0,cache_modifier=SYS) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx<2: + flag=_sel2(tx,f0,f1) + val=_read_flag(flag) + while (val-seq)<0:val=_read_flag(flag) + fx.gpu.barrier() + llvm.FenceOp(llvm.AtomicOrdering.acquire,syncscope='') + if tx= own rows, 200064]` bf16 output; + the counts are a CUDA int32 `[2]`, the same on every rank, summing to the rows (device values, not checked). +- `RowAllToAll.slot`: this rank's send slot (pool address, slot bytes, device counter, flag address) for a producer + that writes and publishes it itself (`gemm/bf16_n100032_k6144`'s publish); `RowAllToAll.pull` then receives. +- `RowAllToAll.close`: a collective, after every graph that uses the communicator is retired. + +Both ranks call every exchange in the same order, empty receivers included; one communicator per stream. Counts may +change between graph replays (device sequence, no host state). `host.py` picks the stage grid by rows. + +## Performance + +Two GPUs (one xGMI link) of one MI355X node. `benchmark_exchange.py`: graph replay, inputs rotating over 256 MB, five +alternating rounds, MAX over ranks, median us, dense owners, stage + pull; baseline RCCL `all_to_all_single` + +permute. Link rate = the larger owner's peer bytes / time; the link floor is about 55.5 GB/s. + +| requests (8 rows each) | baseline | this entry | link GB/s | draft (7 rows): baseline | this entry | +|---:|---:|---:|---:|---:|---:| +| 1 | 35.46 | 35.43 | 45.2 | 32.25 | 31.30 | +| 4 | 89.90 | 68.70 | 46.6 | 80.15 | 60.30 | +| 10 | 191.70 | 158.09 | 50.6 | 170.32 | 139.35 | +| 16 | 285.56 | 247.86 | 51.7 | 252.09 | 217.76 | +| 32 | 538.99 | 473.16 | 54.1 | 473.09 | 414.67 | +| 52 | 856.63 | 762.86 | 54.5 | 752.62 | 668.75 | +| 128 | 2062.83 | 1854.25 | 55.2 | 1808.71 | 1622.06 | + +## Tests + +```bash +python3 -m pytest tests/collectives/all_to_all_bf16_n100032_ranks2 -q # 2 GPUs +python3 benchmarks/collectives/all_to_all_bf16_n100032_ranks2/benchmark_exchange.py # 2 GPUs +``` diff --git a/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/__init__.py b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/__init__.py new file mode 100644 index 0000000..cdd0137 --- /dev/null +++ b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +from .host import RowAllToAll + +__all__ = ["RowAllToAll"] diff --git a/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/host.py b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/host.py new file mode 100644 index 0000000..df79fae --- /dev/null +++ b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/host.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One stream per communicator; construct collectively before capture. + +Two slots suffice: publishing phase c+1 follows the pull of phase c, so all +peers have read slot c before anyone reuses it at phase c+2. Every rank, +including empty receivers, participates in the peer wait. +""" +import ctypes +import functools +import torch +import torch.distributed as dist +from amd_kernels.common import hip_runtime as ops +from amd_kernels.common.hip_runtime import load_hip_runtime +from . import kernel +from .stage_publish import compile_stage_publish + +N, WORLD, V = kernel.N, kernel.WORLD, kernel.N * kernel.WORLD +STAGES = (64, 128, 512) +PULL = (16, 4) # (remote, local) workgroups: best at 4-512 owned rows, 37-55 GB/s (sweep_pull.py in benchmarks/) + + +def _stage_wgs(rows: int) -> int: + return 64 if rows <= 128 else (128 if rows <= 512 else 512) + + +class _IpcHandle(ctypes.Structure): + _fields_ = [("reserved", ctypes.c_char * 64)] + + +@functools.cache +def _hip(): + lib = load_hip_runtime() + lib.hipMalloc.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t] + lib.hipIpcGetMemHandle.argtypes = [ctypes.POINTER(_IpcHandle), ctypes.c_void_p] + lib.hipIpcOpenMemHandle.argtypes = [ + ctypes.POINTER(ctypes.c_void_p), + _IpcHandle, + ctypes.c_uint, + ] + return lib + + +class _DevicePointer: + """A device allocation as torch sees it (``torch.as_tensor`` wraps it).""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "data": (ptr, False), + "shape": (nbytes,), + "typestr": "|u1", + "strides": None, + "version": 3, + } + + +def _alloc_pool(nbytes: int, device: torch.device) -> tuple[int, bytes, torch.Tensor]: + """A hipMalloc pool (coarse-grained, freed by collective close) -> (pointer, IPC handle bytes, the pool as a uint8 tensor).""" + hip = _hip() + ptr = ctypes.c_void_p() + with torch.cuda.device(device): + rc = hip.hipMalloc(ctypes.byref(ptr), nbytes) + assert rc == 0, f"hipMalloc({nbytes}) failed: {rc}" + handle = _IpcHandle() + rc = hip.hipIpcGetMemHandle(ctypes.byref(handle), ptr) + assert rc == 0, f"hipIpcGetMemHandle failed: {rc}" + pool = torch.as_tensor(_DevicePointer(ptr.value, nbytes), device=device) + assert pool.data_ptr() == ptr.value + # the raw 64 bytes (a c_char field reads back only up to its first NUL) + raw = ctypes.string_at(ctypes.addressof(handle), ctypes.sizeof(handle)) + return ptr.value, raw, pool + + +def _open_pool(handle: bytes, device: torch.device) -> int: + hip = _hip() + ptr = ctypes.c_void_p() + h = _IpcHandle() + assert len(handle) == ctypes.sizeof(h) + ctypes.memmove(ctypes.addressof(h), handle, len(handle)) + with torch.cuda.device(device): + rc = hip.hipIpcOpenMemHandle(ctypes.byref(ptr), h, 1) # lazy peer access + assert rc == 0, f"hipIpcOpenMemHandle failed: {rc}" + return ptr.value + + +def _exchange(obj, rank: int, world: int, group) -> list: + """Every rank's ``obj`` (broadcasts: ``all_gather_object`` is unavailable on + gloo under inference mode, which the profile run enters).""" + out = [[None] for _ in range(world)] + out[rank][0] = obj + ranks = sorted(dist.get_process_group_ranks(group=group)) + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +def _run(exe, *args): + import flydsl.compiler as flyc + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +class RowAllToAll: + def __init__(self, rank, group, device, max_rows=2048): + assert dist.get_world_size(group) == WORLD + assert rank == dist.get_rank(group) and 0 <= rank < WORLD + assert 0 < max_rows <= 2048 + self.rank, self.device = rank, torch.device(device) + self.group = group + self.closed = False + self.max_rows = max_rows + self.slot_bytes = max_rows * N * 2 + ptr, handle, self.pool = _alloc_pool(2 * self.slot_bytes, device) + flag, fh = ops.allocate_shared_buffer_and_handle(256) + handles = _exchange((handle, fh), rank, WORLD, group) + self.ptrs = [ptr if r == rank else _open_pool(h, device) for r, (h, _) in enumerate(handles)] + self.flags = [flag if r == rank else ops.open_mem_handle(h) for r, (_, h) in enumerate(handles)] + self.counter = torch.zeros(1, dtype=torch.int32, device=device) + self.stages = {w: compile_stage_publish(w) for w in STAGES} + self.pull_exe = kernel.compile_pull(*PULL) + + def __call__(self, x, counts, out): + """Exchange owner-sorted x into preallocated out, bit for bit. + + counts is CUDA int32 [2], identical on all ranks, nonnegative, + sum(counts) == x.shape[0]; counts[rank] <= out.shape[0]. These device + values are caller preconditions; checking them would synchronize. + Counts may change between replays. One stream uses a communicator; + independent streams need independent communicators. Every rank calls + every exchange, including ranks receiving zero rows. + """ + assert not self.closed + assert x.device == self.counter.device == counts.device == out.device + assert x.dtype == torch.bfloat16 and x.ndim == 2 and x.shape[1] == N and x.is_contiguous() + assert x.shape[0] <= self.max_rows + assert counts.shape == (WORLD,) and counts.dtype == torch.int32 and counts.is_contiguous() + assert out.dtype == x.dtype and out.ndim == 2 and out.shape[1] == V and out.is_contiguous() + stream = torch.cuda.current_stream(self.device) + rows = x.shape[0] + _run(self.stages[_stage_wgs(rows)], x.data_ptr(), self.ptrs[self.rank], self.counter, + x.numel() // 2, self.slot_bytes, self.flags[self.rank], stream) + return self.pull(rows, counts, out) + + def slot(self): + """This rank's send slot for a producer that writes and publishes it itself (the lm_head GEMM's publish + epilogue, bf16_gemm_n100032_k6144(..., publish=comm.slot())): (pool address, slot bytes, device counter, + flag address). The producer writes slot counter[0] & 1 with system-scope stores and its last workgroup + publishes, as stage_publish does; then call pull. Graph safe: the slot is chosen on the device.""" + assert not self.closed + return (self.ptrs[self.rank], self.slot_bytes, self.counter, self.flags[self.rank]) + + def pull(self, rows, counts, out): + """The receive half of __call__, after this rank's ``rows`` owner-sorted rows were published (by + __call__'s stage or by a producer through slot()). Same preconditions as __call__.""" + assert not self.closed + assert counts.device == out.device == self.counter.device and 0 <= rows <= self.max_rows + assert counts.shape == (WORLD,) and counts.dtype == torch.int32 and counts.is_contiguous() + assert out.dtype == torch.bfloat16 and out.ndim == 2 and out.shape[1] == V and out.is_contiguous() + stream = torch.cuda.current_stream(self.device) + _run(self.pull_exe, out.data_ptr(), *self.ptrs, *self.flags, self.counter, counts, + self.slot_bytes, self.rank, stream) + return out + + def warmup(self): + """Collective: compile every stage grid the row range reaches and the pull, eagerly. + + Every rank calls this before capture. The buffers are temporary; the + communicator retains only its IPC pool, flags and device counter. + """ + rows_list = sorted({min(r, self.max_rows) for r in (4, 256, self.max_rows)}) + assert {_stage_wgs(r) for r in range(1, self.max_rows + 1)} <= {_stage_wgs(r) for r in rows_list} + for rows in rows_list: + counts = [rows // WORLD + int(r < rows % WORLD) for r in range(WORLD)] + x = torch.zeros(rows, N, dtype=torch.bfloat16, device=self.device) + cs = torch.tensor(counts, dtype=torch.int32, device=self.device) + out = torch.empty(counts[self.rank], V, dtype=torch.bfloat16, device=self.device) + self(x, cs, out) + torch.cuda.synchronize(self.device) + + def close(self): + """Collective teardown after all graphs using this object are retired.""" + if self.closed: + return + torch.cuda.synchronize(self.device) + dist.barrier(group=self.group) + lib = load_hip_runtime() + lib.hipIpcCloseMemHandle.argtypes = [ctypes.c_void_p] + lib.hipFree.argtypes = [ctypes.c_void_p] + for r in range(WORLD): + if r != self.rank: + for ptr in (self.ptrs[r], self.flags[r]): + rc = lib.hipIpcCloseMemHandle(ctypes.c_void_p(ptr)) + assert rc == 0, (r, rc) + dist.barrier(group=self.group) + for ptr in (self.ptrs[self.rank], self.flags[self.rank]): + rc = lib.hipFree(ctypes.c_void_p(ptr)) + assert rc == 0, rc + self.pool = None + self.closed = True diff --git a/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/kernel.py b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/kernel.py new file mode 100644 index 0000000..73bda63 --- /dev/null +++ b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/kernel.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-safe owner row exchange into its final vocabulary layout, two ranks. + +The protocol of all_to_all_bf16_n50016_ranks4: two IPC slots, an uncached sequence flag, peer system loads, a device +counter (graph replay never reuses a host sequence). With one peer over one xGMI link, the grid is split unevenly: +the first PR workgroups pull the peer's rows over the link, the other PL copy this rank's own rows from its slot. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +N = 100032 +WORLD = 2 +NT = 256 +UNROLL = 8 +STEP = NT * 4 * UNROLL +SYS = 0x11 + + +def _global_i32_ptr(addr_i64): + ptr_ty = fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4) + return fx.inttoptr(ptr_ty, fx.Int64(addr_i64)) + + +def _flag_read(addr_i64): + """The flag's current value: an atomic add of 0 at system scope (never hoisted out of the spin loop, served by + the owner's uncached memory).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _global_i32_ptr(addr_i64).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +def _sel2(p, a0, a1): + return (p == 0).select(a0, a1) + + +def compile_pull(remote_wgs: int, local_wgs: int): + PR, PL = remote_wgs, local_wgs + + @flyc.kernel(name=f"all_to_all_bf16_n100032_ranks2_pull_r{PR}_l{PL}", known_block_size=[NT, 1, 1]) + def kernel(arg_out: fx.Int64, arg_s0: fx.Int64, arg_s1: fx.Int64, arg_f0: fx.Int64, arg_f1: fx.Int64, + counter: fx.Tensor, counts: fx.Tensor, i64_slot_bytes: fx.Int64, i32_rank: fx.Int32): + tx, bid = fx.thread_idx.x, fx.block_idx.x + remote = bid < PR + p = remote.select(1 - i32_rank, i32_rank) + wid = remote.select(bid, bid - PR) + nw = remote.select(fx.Int32(PR), fx.Int32(PL)) + seq = fx.Int32(counter[0]) + if (tx == 0) & remote: + flag = _sel2(p, arg_f0, arg_f1) + val = _flag_read(flag) + # Modular int32 comparison remains valid across counter wrap. + # Two slots bound the peer lead/lag to much less than 2**31. + while (val - seq) < 0: + val = _flag_read(flag) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="") + lo = (i32_rank > 0).select(fx.Int32(counts[0]), fx.Int32(0)) + rows = fx.Int32(counts[i32_rank]) + ndw = rows * (N // 2) + slot_off = fx.Int64((seq - 1) & 1) * i64_slot_bytes + src = buffer_ops.create_buffer_resource_from_addr( + _sel2(p, arg_s0, arg_s1) + slot_off + fx.Int64(lo) * (N * 2), num_records_bytes=ndw * 4) + dst = buffer_ops.create_buffer_resource_from_addr(arg_out, num_records_bytes=ndw * WORLD * 4) + off = wid * STEP + tx * 4 + while off < ndw: + vals = [Vec(buffer_ops.buffer_load(src, off + u * NT * 4, vec_width=4, dtype=fx.Int32, + cache_modifier=SYS)) for u in range_constexpr(UNROLL)] + for u in range_constexpr(UNROLL): + o = off + u * NT * 4 + row = o // (N // 2) + col = o % (N // 2) + buffer_ops.buffer_store(vals[u], dst, row * N + p * (N // 2) + col, mask=o < ndw) + off = off + nw * STEP + + @flyc.jit + def launch_all_to_all_bf16_n100032_ranks2_pull(arg_out: fx.Int64, arg_s0: fx.Int64, arg_s1: fx.Int64, + arg_f0: fx.Int64, arg_f1: fx.Int64, counter: fx.Tensor, + counts: fx.Tensor, i64_slot_bytes: fx.Int64, i32_rank: fx.Int32, + stream: fx.Stream): + kernel(arg_out, arg_s0, arg_s1, arg_f0, arg_f1, counter, counts, i64_slot_bytes, i32_rank).launch( + grid=(PR + PL, 1, 1), block=(NT, 1, 1), stream=stream) + return launch_all_to_all_bf16_n100032_ranks2_pull diff --git a/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/reference.py b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/reference.py new file mode 100644 index 0000000..c039c84 --- /dev/null +++ b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/reference.py @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Bitwise row exchange: each input [sum(counts), 100032] is sorted by owner. + +Rank r receives rows [sum(counts[:r]):sum(counts[:r+1])] from each of the two sources, concatenated along the +vocabulary into bf16 [counts[r], 200064]. There is no arithmetic, including on NaN payloads. Counts are common across +ranks. +""" +import torch + +N = 100032 +WORLD = 2 + + +def all_to_all_reference(inputs: list[torch.Tensor], counts: list[int], rank: int) -> torch.Tensor: + lo = sum(counts[:rank]) + hi = lo + counts[rank] + return torch.cat([x[lo:hi].to(inputs[rank].device) for x in inputs], dim=1) + + +def merge_received_reference(received: torch.Tensor) -> torch.Tensor: + """[source, owned row, 100032] -> [owned row, source vocabulary].""" + return received.permute(1, 0, 2).reshape(received.shape[1], WORLD * N).contiguous() diff --git a/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/stage_publish.py b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/stage_publish.py new file mode 100644 index 0000000..4f1fba5 --- /dev/null +++ b/amd_kernels/collectives/all_to_all_bf16_n100032_ranks2/stage_publish.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The send half of the exchange: copy owner-sorted rows into this rank's slot (counter[0] & 1) with system-scope +stores (no dirty XCD L2 lines for the peer's system loads); the last workgroup to take a ticket on flag word 1 +increments the device counter and stores it to flag word 0.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +NT = 256 +UNROLL = 8 +STEP = NT * 4 * UNROLL +SYS = 0x11 + +def _global_i32_ptr(addr_i64): + ptr_ty = fx.PointerType.get( + T.i32, address_space=fx.AddressSpace.Global, alignment=4 + ) + return fx.inttoptr(ptr_ty, fx.Int64(addr_i64)) + + +def _flag_add(addr_i64, v): + """Atomic add at system scope on an uncached word; returns the old value.""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _global_i32_ptr(addr_i64).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +def compile_stage_publish(stage_wgs): + SW = stage_wgs # 64 / 128 / 512, chosen by a measured sweep + assert SW in (64, 128, 512) + @flyc.kernel(name=f"all_to_all_bf16_n100032_ranks2_stage_publish_w{SW}", known_block_size=[NT, 1, 1]) + def kernel(arg_x: fx.Int64, arg_pool: fx.Int64, counter: fx.Tensor, + i32_ndw: fx.Int32, i64_slot_bytes: fx.Int64, arg_flag: fx.Int64): + slot = fx.Int32(counter[0]) & 1 + src = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=i32_ndw * 4) + dst = buffer_ops.create_buffer_resource_from_addr(arg_pool + fx.Int64(slot) * i64_slot_bytes, + num_records_bytes=i32_ndw * 4) + off = fx.block_idx.x * STEP + fx.thread_idx.x * 4 + while off < i32_ndw: + vals = [Vec(buffer_ops.buffer_load(src, off + u * NT * 4, vec_width=4, dtype=fx.Int32)) + for u in range_constexpr(UNROLL)] + for u in range_constexpr(UNROLL): + buffer_ops.buffer_store(vals[u], dst, off + u * NT * 4, cache_modifier=SYS) + off = off + SW * STEP + + # All producing waves complete their system stores before lane 0 + # takes this workgroup's ticket. SYS stores avoid dirty XCD L2 lines. + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if fx.thread_idx.x == 0: + ticket = _flag_add(arg_flag + 4, 1) + if ticket == SW - 1: + _flag_add(arg_flag + 4, -SW) + seq = fx.Int32(counter[0]) + 1 + counter[0] = seq + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=64) + buffer_ops.buffer_store(seq, fr, 0, cache_modifier=SYS) + + @flyc.jit + def launch_all_to_all_bf16_n100032_ranks2_stage_publish(arg_x: fx.Int64, arg_pool: fx.Int64, counter: fx.Tensor, + i32_ndw: fx.Int32, i64_slot_bytes: fx.Int64, arg_flag: fx.Int64, stream: fx.Stream): + kernel(arg_x, arg_pool, counter, i32_ndw, i64_slot_bytes, arg_flag).launch( + grid=(SW, 1, 1), block=(NT, 1, 1), stream=stream) + return launch_all_to_all_bf16_n100032_ranks2_stage_publish diff --git a/amd_kernels/collectives/pull_collectives_ranks2/README.md b/amd_kernels/collectives/pull_collectives_ranks2/README.md new file mode 100644 index 0000000..76cd0d7 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/README.md @@ -0,0 +1,60 @@ +# Pull collectives over IPC-mapped pools, two ranks + +## What it does + +The SP-prefill collectives of MiniMax-M3.1-Flash TP2 in FlyDSL instead of RCCL, over the one xGMI link of a GPU pair: +the all-to-all with the MXFP8 unpack into o_proj_full's input (4096 code columns per rank), all-gathers of row-major +fields, the reduce-scatter, and the SP norm tails fused into them (callers: the two-rank `seq_parallel_handoffs`, MoE +`gather_mxfp4_moe` of `moe/mxfp4_h6144_i1536_e128_top4`). Shard rows R = T / 2, T = 4096..32768 tokens. IPC pools, two +slots and uncached ticket words per rank; stagers and pullers share one launch and hand off part by part. Bit for bit +(`reference.py`): gathers in rank order; `unpack_reference`; reduce-scatter `bf16((+0 + b0) + b1)` in fp32; slot +combine `(((s0+s1)+s2)+s3)+shared` in fp32, shared `[2R, H]` on every row or `[R, H]` (owner form) on the own rows; +norms per `norm/gemma_add_rmsnorm_h6144` (quant of the fp32 norm). The MoE tail's peer sums cross the link in +`lossless_x3.py`'s exact 12-bit code (9600 B per row instead of 12288). + +## Entry points + +All are built collectively (gloo, same sizes on both ranks), warmed up collectively on the current stream before +capture, one per ordered stream or DBO microbatch, called in the same order on both ranks; pools live for the process; +a communicator's grid is fixed for its lifetime. `Pools`, `Collectives` and `FencedAllGatherComm` keep their sequence +on the host (eager only); the others keep device counters, so eager calls and graph replays interleave. + +- `amd_kernels.collectives.pull_collectives_ranks2.Pools`: a rank's IPC pools and flag words, one set per microbatch. +- `...Collectives`: one phase's slot of a set, bound to its stream; `alloc` gives a slot view the producer (the pack) + writes and the all-to-all reads in place. +- `...FencedAllGatherComm.all_to_all_unpack_shuffled`: the attention rows' all-to-all into o_proj_full's MXFP8 codes + and scales in `mxfp8_gemm`'s preshuffled layout. +- `...AllGatherComm.all_gather`: gathers contiguous 16 B-aligned row-major fields in rank order (the routed or compact + MoE payload, codes + scales, bf16 rows). +- `...SlotsReduceScatterComm.slots4_reduce_scatter_add_rmsnorm_mxfp8`: the MoE tail (slot combine, shared add of + either form, picked by its rows, reduce-scatter, residual add, norm, quant) into codes, shuffled scales, residual; + `slots4_reduce_scatter` and `reduce_scatter` stop after the sum; the SDMA hand-offs' coded tail is + `slots4_host._stage_rows_x3` + push + `_sum_norm_x3`. +- `...NormAllGatherComm.norm_all_gather`: the dense norm + quant fused into the gather (codes, scales, residual). + +Each communicator's host file picks its kernels (`slots4_host.py`, `gather_host.py`, `all_gather_fence_h6144.py`, +`norm_gather_h6144.py`). + +## Performance + +Graph replay (the all-to-all eager), rank max, us per call at T tokens: ours / RCCL at world 2 (the same bytes, the +non-collective steps as separate launches); in brackets the fraction of the link roofline (R x row bytes at 61.2 GB/s, +the measured rate of one xGMI link with both directions busy; CU pulls top out at 57.4 GB/s, 93.8%). Bytes per row: +MoE tail 9600 (coded, owner-form shared; every-row shared within 0.5%), RS 12288, MoE gather 3296 (routed; the 7328 B +compact one: 276.2 / 538.7 / 1053.8 / 2101.7 us), norm gather 6336, a2a 4224. + +| entry | T = 4096 | 8192 | 16384 | 32768 | +|---|---:|---:|---:|---:| +| MoE tail | 373.3 / 594.8 (86.1%) | 733.5 / 1147.7 (87.6%) | 1451.8 / 2209.4 (88.5%) | 2889.7 / 4305.7 (88.9%) | +| RS | 451.8 / 545.2 (91.0%) | 892.7 / 1061.4 (92.1%) | 1776.6 / 2072.3 (92.6%) | 3536.5 / 4040.1 (93.0%) | +| MoE gather | 128.1 / 143.5 (86.1%) | 250.4 / 276.4 (88.1%) | 480.9 / 518.1 (91.7%) | 951.3 / 1008.5 (92.8%) | +| norm gather | 238.1 / 296.4 (89.0%) | 465.6 / 555.0 (91.1%) | 923.9 / 1076.2 (91.8%) | 1834.2 / 2119.9 (92.5%) | +| a2a unpack | 162.7 / 240.2 (86.9%) | 314.8 / 420.8 (89.8%) | 616.3 / 791.4 (91.7%) | 1224.8 / 1535.7 (92.3%) | + +## Tests + +```bash +python3 -m pytest tests/collectives/pull_collectives_ranks2 -q # 2 GPUs +python3 benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py r.json +python3 benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py +``` diff --git a/amd_kernels/collectives/pull_collectives_ranks2/__init__.py b/amd_kernels/collectives/pull_collectives_ranks2/__init__.py new file mode 100644 index 0000000..94fe406 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/__init__.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""2-rank pull collectives over IPC-mapped pools, FlyDSL (gfx950): the sequence-parallel prefill's hand-off transport +over the one xGMI link of a GPU pair. The torch contract is ``reference.py``. ``FencedAllGatherComm`` unpacks the +attention output's all-to-all into o_proj_full's input; ``AllGatherComm`` gathers row-major fields with device-owned +sequence counters; ``SlotsReduceScatterComm`` adds the graph-safe MoE slot combine + reduce-scatter (+ shard +norm/quant) and the dense reduce-scatter; ``NormAllGatherComm`` fuses the dense norm/quant into a gather.""" + +from .host import WORLD, Collectives, Pools +from .slots4_host import SlotsReduceScatterComm +from .all_gather_fence_h6144 import FencedAllGatherComm +from .gather_host import AllGatherComm +from .norm_gather_h6144 import NormAllGatherComm + +__all__ = ["WORLD", "Collectives", "Pools", "SlotsReduceScatterComm", "AllGatherComm", "FencedAllGatherComm", + "NormAllGatherComm"] diff --git a/amd_kernels/collectives/pull_collectives_ranks2/all_gather_fence_h6144.py b/amd_kernels/collectives/pull_collectives_ranks2/all_gather_fence_h6144.py new file mode 100644 index 0000000..c2ea56f --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/all_gather_fence_h6144.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The sequence-parallel attention phase's unpacking all-to-all on two ranks, as a per-phase-stream communicator: the +packed 4096-code / 128-scale attention rows of both ranks -> this rank's o_proj_full input, the scales in the GEMM's +layout (``pull_unpack_q4096_shuffled``: the release / ticket / publish in the unpack launch up to +``FENCE_MAX_TOKENS``, a separate publication above). The payload never uses an atomic output add; only flags / +tickets use atomics. +""" +import torch + +from .host import Collectives, Pools, WORLD + + +class FencedAllGatherComm: + """Per-phase-stream communicator on the caller's ``Pools``, constructed before serving. Eager-only sequence + state (the phase's ``Collectives``); no host synchronization.""" + + def __init__(self, pools: Pools): + self.p = pools + + def all_to_all_unpack_shuffled(self, coll: Collectives, packed: torch.Tensor): + """Packed 4096-code / 128-scale attention rows ``[2 rows, 4224]`` -> o_proj_full's inputs: codes + ``[rows, 8192]`` and scales in its GEMM layout (``mxfp8_gemm(..., activation_scale_preshuffled=True)``).""" + from .pull_unpack_q4096_shuffled import all_to_all_unpack_q4096_shuffled + + return all_to_all_unpack_q4096_shuffled(coll, packed) + + def warm_up(self): + """Compile both publication forms of the unpack before eager serving.""" + from .pull_unpack_q4096_shuffled import FENCE_MAX_TOKENS + + stream = torch.cuda.current_stream(self.p.device) + for rows in (32, FENCE_MAX_TOKENS // WORLD + 32): + c = Collectives(self.p, 0).bind(stream) + packed = c.alloc((WORLD * rows, 4224), torch.uint8) + packed.zero_() + self.all_to_all_unpack_shuffled(c, packed) + torch.cuda.synchronize(self.p.device) diff --git a/amd_kernels/collectives/pull_collectives_ranks2/gather_host.py b/amd_kernels/collectives/pull_collectives_ranks2/gather_host.py new file mode 100644 index 0000000..5804d2a --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/gather_host.py @@ -0,0 +1,73 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-safe, lossless gather of row-major tensor fields on two ranks (``stage_pull.compile_gather_stage_pull``: one +launch per six fields).""" +import functools + +import torch + +from amd_kernels.common import hip_runtime as ops +from .host import Pools, WORLD, _exchange, _run_compiled +from .stage_pull import FIELDS, PART_BYTES, PARTS, PULLERS, STAGERS, TICKET_BYTES, UNROLL, compile_gather_stage_pull + + +@functools.cache +def _gather_kernel(*grid): + return compile_gather_stage_pull(*grid) + + +class AllGatherComm: + """Gather a list of contiguous tensor shards, preserving every byte. + + Construct collectively with the same slot capacity on both ranks. Each field has a positive byte length divisible + by 16 and 16-byte-aligned input/output addresses. Fields occupy 256-byte-aligned regions of a slot. Outputs + concatenate ranks along their first axis. One communicator per ordered stream/microbatch. Eager calls and graph + replay can interleave; both ranks must issue the same shapes and operation order. Inputs and outputs must not + alias this communicator's IPC pool. ``grid`` (pullers, stagers, parts, unroll, part bytes) is fixed for the + communicator's lifetime (its tickets count the launches). + """ + + def __init__(self, rank, cpu_group, device, slot_bytes, grid=(PULLERS, STAGERS, PARTS, UNROLL, PART_BYTES)): + assert 0 < slot_bytes < 2**31 and slot_bytes % 256 == 0 + self.device = torch.device(device) + self.pools = Pools(rank, WORLD, cpu_group, self.device, slot_bytes, sets=1) + self.counter = torch.zeros(1, dtype=torch.int32, device=self.device) + ptr, handle = ops.allocate_shared_buffer_and_handle(TICKET_BYTES) + handles = _exchange(handle, rank, WORLD, cpu_group) + self.tickets = [ptr if r == rank else ops.open_mem_handle(h) for r, h in enumerate(handles)] + self.kernel = _gather_kernel(*grid) + + def all_gather(self, outs, ins): + """Stage, publish and pull every field, six per launch.""" + assert len(outs) == len(ins) and len(ins) > 0 + for i in range(0, len(ins), FIELDS): + self._launch(outs[i:i + FIELDS], ins[i:i + FIELDS]) + + def _launch(self, outs, ins): + p = self.pools + offsets, offset = [], 0 + for out, x in zip(outs, ins): + assert x.ndim > 0 and x.is_contiguous() and out.is_contiguous() + assert x.data_ptr() % 16 == 0 and out.data_ptr() % 16 == 0 + assert x.device == self.device and out.device == self.device and out.dtype == x.dtype + assert tuple(out.shape) == (WORLD * x.shape[0], *x.shape[1:]) + n = x.numel() * x.element_size() + assert n > 0 and n % 16 == 0 + offset = (offset + 255) // 256 * 256 + assert offset + n <= p.slot_bytes + for t in (x, out): + assert not (p.pool.data_ptr() <= t.data_ptr() < p.pool.data_ptr() + p.pool.numel()) + offsets.append((offset, n)) + offset += n + pad = FIELDS - len(ins) + _run_compiled(self.kernel, *(x.data_ptr() for x in ins), *[0] * pad, *(o.data_ptr() for o in outs), *[0] * pad, + *(off for off, _ in offsets), *[0] * pad, *(n for _, n in offsets), *[0] * pad, + *p.pool_ptrs, *self.tickets, self.counter, p.slot_bytes, p.rank, + torch.cuda.current_stream(self.device)) + + def warm_up(self): + """Collective: compile the byte-count-independent transport before capture.""" + x = torch.zeros((1, 16), device=self.device, dtype=torch.uint8) + out = torch.empty((WORLD, 16), device=self.device, dtype=torch.uint8) + self.all_gather([out], [x]) + torch.cuda.synchronize(self.device) diff --git a/amd_kernels/collectives/pull_collectives_ranks2/host.py b/amd_kernels/collectives/pull_collectives_ranks2/host.py new file mode 100644 index 0000000..db6f39c --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/host.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The pools and the phase slots the pull collectives read (``kernel.py``). + +``Pools`` is created once per rank (a collective: both ranks of the group call it): a data pool of ``sets x 2`` slots +in coarse-grained device memory, mapped on the peer by IPC, and a flag word per set in uncached memory. A set belongs +to a DBO microbatch (its collectives are issued on its own comm stream hand-offs, so each set keeps its own sequence); +without DBO only set 0 is used. + +A phase takes one slot (alternating, ``Collectives``): its collectives copy their inputs into the slot, publish, wait +for the peer and pull. Two slots per set suffice: a rank writes slot ``k`` for phase ``c + 2`` only after its phase +``c + 1`` completed, i.e. after the peer published phase ``c + 1``, which it does after its own pull of phase ``c`` +(the previous use of slot ``k``) finished. +""" + +import ctypes +import functools +import logging +import math + +import torch +import torch.distributed as dist + +from amd_kernels.common import hip_runtime as ops +from . import kernel as _kernel +from amd_kernels.common.hip_runtime import load_hip_runtime + +logger = logging.getLogger(__name__) + +WORLD = _kernel.WORLD +SLOTS = 2 +_ALIGN = 256 + + +def _run_compiled(exe, *args): + """First call compiles and runs (``flyc.compile``); later calls dispatch the + cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +@functools.cache +def _kernels(): + k = _kernel + return k.compile_publish(), k.compile_fence_publish(), k.compile_sum2() + + +class _IpcHandle(ctypes.Structure): + _fields_ = [("reserved", ctypes.c_char * 64)] + + +@functools.cache +def _hip(): + lib = load_hip_runtime() + lib.hipMalloc.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t] + lib.hipIpcGetMemHandle.argtypes = [ctypes.POINTER(_IpcHandle), ctypes.c_void_p] + lib.hipIpcOpenMemHandle.argtypes = [ + ctypes.POINTER(ctypes.c_void_p), + _IpcHandle, + ctypes.c_uint, + ] + return lib + + +class _DevicePointer: + """A device allocation as torch sees it (``torch.as_tensor`` wraps it).""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "data": (ptr, False), + "shape": (nbytes,), + "typestr": "|u1", + "strides": None, + "version": 3, + } + + +def _alloc_pool(nbytes: int, device: torch.device) -> tuple[int, bytes, torch.Tensor]: + """A hipMalloc pool (coarse-grained, never freed: it lives as long as the + process) -> (pointer, IPC handle bytes, the pool as a uint8 tensor).""" + hip = _hip() + ptr = ctypes.c_void_p() + with torch.cuda.device(device): + rc = hip.hipMalloc(ctypes.byref(ptr), nbytes) + assert rc == 0, f"hipMalloc({nbytes}) failed: {rc}" + handle = _IpcHandle() + rc = hip.hipIpcGetMemHandle(ctypes.byref(handle), ptr) + assert rc == 0, f"hipIpcGetMemHandle failed: {rc}" + pool = torch.as_tensor(_DevicePointer(ptr.value, nbytes), device=device) + assert pool.data_ptr() == ptr.value + # the raw 64 bytes (a c_char field reads back only up to its first NUL) + raw = ctypes.string_at(ctypes.addressof(handle), ctypes.sizeof(handle)) + return ptr.value, raw, pool + + +def _open_pool(handle: bytes, device: torch.device) -> int: + hip = _hip() + ptr = ctypes.c_void_p() + h = _IpcHandle() + assert len(handle) == ctypes.sizeof(h) + ctypes.memmove(ctypes.addressof(h), handle, len(handle)) + with torch.cuda.device(device): + rc = hip.hipIpcOpenMemHandle(ctypes.byref(ptr), h, 1) # lazy peer access + assert rc == 0, f"hipIpcOpenMemHandle failed: {rc}" + return ptr.value + + +def _exchange(obj, rank: int, world: int, group) -> list: + """Every rank's ``obj`` (broadcasts: ``all_gather_object`` is unavailable on + gloo under inference mode, which the profile run enters).""" + out = [[None] for _ in range(world)] + out[rank][0] = obj + ranks = sorted(dist.get_process_group_ranks(group=group)) + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +class Pools: + """One rank's pools and flags, with the peer's mapped. ``slot_bytes`` is the + largest phase (the inputs of its collectives, 256 B aligned each).""" + + def __init__( + self, + rank: int, + world: int, + group, + device: torch.device, + slot_bytes: int, + sets: int = 2, + ): + assert world == WORLD, world + self.rank, self.world, self.device = rank, world, device + self.sets, self.slot_bytes = sets, slot_bytes + nbytes = sets * SLOTS * slot_bytes + ptr, handle, self.pool = _alloc_pool(nbytes, device) + flag_ptr, flag_handle = ops.allocate_shared_buffer_and_handle(_ALIGN) + peers = _exchange((handle, flag_handle), rank, world, group) + self.pool_ptrs = [ + ptr if r == rank else _open_pool(h, device) + for r, (h, _) in enumerate(peers) + ] + self.flag_ptrs = [ + flag_ptr if r == rank else ops.open_mem_handle(h) + for r, (_, h) in enumerate(peers) + ] + # per set: the sequence (the flag's next value) and the phase count + self.seq = [0] * sets + self.phase = [0] * sets + publish, self.fence_publish, _ = _kernels() + stream = torch.cuda.current_stream(device) + # the flag word and the ticket counter of every set start at 0 + for s in range(sets): + base = self.flag_ptrs[rank] + _kernel.FLAG_STRIDE * s + _run_compiled(publish, base, 0, stream) + _run_compiled(publish, base + 4, 0, stream) + torch.cuda.synchronize(device) + dist.barrier(group=group) + logger.info( + "pull_collectives_ranks2: %d sets x %d slots x %.0f MB pools", + sets, + SLOTS, + slot_bytes / 2**20, + ) + + def slot(self, set_id: int) -> tuple[int, int]: + """Begin a phase of ``set_id``: (byte offset of its slot in every pool, + the set's flag offset).""" + k = self.phase[set_id] % SLOTS + self.phase[set_id] += 1 + return (set_id * SLOTS + k) * self.slot_bytes, _kernel.FLAG_STRIDE * set_id + + def next_seq(self, set_id: int) -> int: + self.seq[set_id] += 1 + return self.seq[set_id] + + +class Collectives: + """One phase in the next slot of ``set_id`` (the unpacking all-to-all of + ``pull_unpack_q4096_shuffled`` reads it). Created before the compute that + produces the phase's inputs: ``alloc`` hands the producers views of the slot, + which the collective then reads in place; other inputs are copied in + (``_stage``). Bound to the comm stream when the phase runs (``bind``).""" + + def __init__(self, pools: Pools, set_id: int): + self.p = pools + self.set_id = set_id + self.base, self.flag_off = pools.slot(set_id) + self.off = 0 + self.stream: torch.cuda.Stream | None = None + + def bind(self, stream: torch.cuda.Stream) -> "Collectives": + self.stream = stream + return self + + def alloc(self, shape: tuple[int, ...], dtype: torch.dtype) -> torch.Tensor | None: + """A ``shape`` tensor of ``dtype`` in the slot for a producer to write + (the collective it feeds reads it in place); None when it does not fit.""" + n = math.prod(shape) * dtype.itemsize + off = self._take(n) + if off is None: + return None + return self.p.pool[off : off + n].view(dtype).view(shape) + + def _take(self, nbytes: int) -> int | None: + """Reserve ``nbytes`` of the slot (256 B aligned); None when it does not fit.""" + off = (self.off + _ALIGN - 1) // _ALIGN * _ALIGN + if off + nbytes > self.p.slot_bytes: + return None + self.off = off + nbytes + return self.base + off + + def _stage(self, x: torch.Tensor) -> int | None: + """``x``'s byte offset in every pool: where it is when ``alloc`` placed it + (the same offset on every rank: the producers allocate in one order), + else after a copy into the slot; None when it does not fit.""" + n = x.numel() * x.element_size() + start = self.p.pool.data_ptr() + self.base + if start <= x.data_ptr() < start + self.p.slot_bytes: + assert x.data_ptr() + n <= start + self.p.slot_bytes + return x.data_ptr() - self.p.pool.data_ptr() + off = self._take(n) + if off is None: + return None + dst = self.p.pool[off : off + n].view(x.dtype).view(x.shape) + dst.copy_(x) + return off + + def _publish_wait(self) -> None: + """Fence + publish this rank's slot; the pull that follows waits for the + peer's flag itself.""" + p = self.p + self.seq = p.next_seq(self.set_id) + me_flag = p.flag_ptrs[p.rank] + self.flag_off + _run_compiled(p.fence_publish, me_flag, self.seq, self.stream) diff --git a/amd_kernels/collectives/pull_collectives_ranks2/kernel.py b/amd_kernels/collectives/pull_collectives_ranks2/kernel.py new file mode 100644 index 0000000..20702a6 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/kernel.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""FlyDSL collectives of the sequence-parallel prefill on two ranks (MI355X, gfx950): the shared flag helpers and +constants, and the launches every communicator builds on. + +Every rank owns a data pool (coarse-grained device memory the peer maps by IPC) and uncached flag / ticket words the +peer maps. The eager all-to-all publishes with ``fence_publish`` (every workgroup writes its XCD's L2 back -- a +coarse-grained buffer's dirty lines are not written back for the peer's xGMI reads at a kernel boundary -- and the +last arriver on the ticket stores the sequence number into the flag); the graph-safe collectives stage with +system-coherent stores and hand off part by part in the same launch (``stage_pull.py``). Pullers read with +system-coherent loads (``SC0 | SC1``, no stale lines from the local L2). + +Two ranks share one direct xGMI link, so every puller serves the one peer; the own block never crosses it. Few, +long-lived workgroups matter: next to the compute stream's kernels a CU slot is rare. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +WORLD = 2 +NT = 256 # threads per workgroup +UNROLL = 8 # 16 B per thread per step +STEP = NT * 4 * UNROLL # dwords per workgroup per loop step +SYS = 0x11 # sc0 | sc1: system-coherent loads / stores +# MI355X: workgroup w runs on XCD w % 8, each XCD has its own L2; 8 workgroups per XCD +FENCE_WGS = 64 +SUM_WGS = 1024 +# per microbatch set: the flag word, then the fence ticket counter (bytes) +FLAG_STRIDE = 16 + + +def _global_i32_ptr(addr_i64): + ptr_ty = fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4) + return fx.inttoptr(ptr_ty, fx.Int64(addr_i64)) + + +def _flag_read(addr_i64): + """The flag's current value: an atomic add of 0 at system scope (never hoisted out of the spin loop, served by the + owner's uncached memory).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _global_i32_ptr(addr_i64).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +def _flag_add(addr_i64, v): + """Atomic add at system scope on an uncached word; returns the old value.""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _global_i32_ptr(addr_i64).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +def _sel2(p, a0, a1): + return (p == 0).select(a0, a1) + + +def _as_f32(i): + return fx.Int32(i).bitcast(fx.Float32) + + +def _pk_bf16(a, b): + return fx.Int32(fx.rocdl.cvt_pk_bf16_f32(fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b)))) + + +def compile_publish(): + """The flag word <- value (a system-coherent store): the init's zeroing of the flags and ticket counters.""" + + @flyc.kernel(name="pull_collectives_ranks2_publish", known_block_size=[64, 1, 1]) + def kernel(arg_flag: fx.Int64, i32_seq: fx.Int32): + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=64) + buffer_ops.buffer_store(i32_seq, fr, 0, mask=fx.thread_idx.x == 0, cache_modifier=SYS) + + @flyc.jit + def launch(arg_flag: fx.Int64, i32_seq: fx.Int32, stream: fx.Stream): + kernel(arg_flag, i32_seq).launch(grid=(1, 1, 1), block=(64, 1, 1), stream=stream) + + launch.kernel_name = "pull_collectives_ranks2_publish" + return launch + + +def compile_fence_publish(): + """Fence + publish in one launch: every workgroup releases its XCD's L2 (the producers' dirty lines reach memory), + takes a ticket on the counter word next to the flag; the last arriver resets the counter and stores ``seq`` into + the flag. Run on the stream that produced the data.""" + + @flyc.kernel(name="pull_collectives_ranks2_fence_publish", known_block_size=[64, 1, 1]) + def kernel(arg_flag: fx.Int64, i32_seq: fx.Int32): + _llvm.FenceOp(_llvm.AtomicOrdering.release, syncscope="") + if fx.thread_idx.x == 0: + t = _flag_add(arg_flag + 4, 1) + if t == FENCE_WGS - 1: + _flag_add(arg_flag + 4, -FENCE_WGS) + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=64) + buffer_ops.buffer_store(i32_seq, fr, 0, cache_modifier=SYS) + + @flyc.jit + def launch(arg_flag: fx.Int64, i32_seq: fx.Int32, stream: fx.Stream): + kernel(arg_flag, i32_seq).launch(grid=(FENCE_WGS, 1, 1), block=(64, 1, 1), stream=stream) + + launch.kernel_name = "pull_collectives_ranks2_fence_publish" + return launch + + +def compile_sum2(): + """The reduce-scatter's sum after the pull: out = bf16((+0 + b0) + b1) in fp32, ``b_p`` rank p's bf16 block (one + is the own pool block read in place, the other the pulled staging).""" + + @flyc.kernel(name="pull_collectives_ranks2_sum2", known_block_size=[NT, 1, 1]) + def kernel(arg_out: fx.Int64, arg_s0: fx.Int64, arg_s1: fx.Int64, i64_block: fx.Int64): + tx = fx.thread_idx.x + bid = fx.block_idx.x + srcs = [buffer_ops.create_buffer_resource_from_addr(a, num_records_bytes=i64_block) for a in (arg_s0, arg_s1)] + out = buffer_ops.create_buffer_resource_from_addr(arg_out, num_records_bytes=i64_block) + n_dw = fx.Int32(i64_block // 4) + per_block = (n_dw + (SUM_WGS - 1)) // SUM_WGS + per_block = (per_block + (STEP - 1)) // STEP * STEP + start = bid * per_block + end = start + per_block + off = start + tx * 4 + while off < end: + words = [] + for u in range_constexpr(UNROLL): + o = off + u * NT * 4 + words.append( + [ + Vec(buffer_ops.buffer_load(srcs[r], o, vec_width=4, dtype=fx.Int32, mask=o < n_dw)) + for r in range_constexpr(WORLD) + ] + ) + for u in range_constexpr(UNROLL): + o = off + u * NT * 4 + packed = [] + for i in range_constexpr(4): + lo = fx.Float32(0.0) + hi = fx.Float32(0.0) + for r in range_constexpr(WORLD): + w = fx.Int32(words[u][r][i]) + lo = lo + _as_f32(w << 16) + hi = hi + _as_f32(w & fx.Int32(-65536)) + packed.append(_pk_bf16(lo, hi)) + buffer_ops.buffer_store(Vec.from_elements(packed, fx.Int32), out, o, mask=o < n_dw) + off = off + STEP + + @flyc.jit + def launch(arg_out: fx.Int64, arg_s0: fx.Int64, arg_s1: fx.Int64, i64_block: fx.Int64, stream: fx.Stream): + kernel(arg_out, arg_s0, arg_s1, i64_block).launch(grid=(SUM_WGS, 1, 1), block=(NT, 1, 1), stream=stream) + + launch.kernel_name = "pull_collectives_ranks2_sum2" + return launch diff --git a/amd_kernels/collectives/pull_collectives_ranks2/lossless_x3.py b/amd_kernels/collectives/pull_collectives_ranks2/lossless_x3.py new file mode 100644 index 0000000..e17ad49 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/lossless_x3.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Exact BF16 rows of 6144 values in 9600 bytes (0.781x) for the link: the two-rank decode all-reduce's 12-bit code. + +Adapted from amd_kernels norm/gemma_add_rmsnorm_h6144 (allreduce_add_rmsnorm/lossless_bf16.py, the codec, and +allreduce_add_rmsnorm_ranks2/lossless_x3.py, the record layout). + +Code lane t holds columns 8 t .. 8 t + 7; a virtual wave is 64 code lanes (512 values). Per 32 values a group +exponent; per value the sign, 7 mantissa bits and a 4-bit exponent delta 0..14; delta 15 is a signed zero or an index +into the virtual wave's six exception values. A lane with several exceptions, or a virtual wave with more than six, +stores its 1 KB raw in the row's raw slot (a bf16 row: the values at their own offsets). Lane t's 8 sign/mantissa +bytes and 8 nibbles are a 12-byte record at byte 12 t, then 12 headers of 32 B at ``HDR`` (16 group exponent bytes, +the count word, 6 exception values). Every 16-bit pattern round-trips. + +``dec16`` reads with the norm lane map (thread tx holds columns 16 tx .. 16 tx + 15, code lanes 2 tx and 2 tx + 1): +a hardware wave then spans two virtual waves, so the zero / exception / raw paths branch per lane. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .kernel import SYS + +H = 6144 +ROWB = H * 2 # a raw (bf16) row +CODED = 9600 # a coded row +HDR = 9216 # its headers +VWAVES = 12 +NIBPOS = [4 * (v // 2) + 16 * (v % 2) for v in range(8)] +RAWF, SPECF, NEXC = 0x100, 0x200, 6 +SIGN2 = -2147450880 # 0x80008000 + + +def _dpp_i(v, ctrl): + return fx.Int32(_rocdl_ops.update_dpp( + T.i32, _llvm.mlir_undef(T.i32), fx.as_ir_value(fx.Int32(v)), ctrl, 0xF, 0xF, True + )) + + +def _imax(a, b): + return (a > b).select(a, b) + + +@flyc.jit +def enc(value, lane): + """Code one lane's eight BF16 values; return the wave header and payload.""" + one, zero = fx.Int32(1), fx.Int32(0) + rd = [fx.Int32(value[t]) for t in range_constexpr(4)] + us = [] + for t in range_constexpr(4): + us.append(rd[t] & 0xFFFF) + us.append((rd[t] >> 16) & 0xFFFF) + ex = [(u >> 7) & 0xFF for u in us] + gmax = ex[0] + for t in range_constexpr(1, 8): + gmax = _imax(gmax, ex[t]) + gmax = _imax(gmax, _dpp_i(gmax, 0xB1)) # quad_perm [1, 0, 3, 2] + gmax = _imax(gmax, _dpp_i(gmax, 0x4E)) # quad_perm [2, 3, 0, 1] + dd = [gmax - ex[t] for t in range_constexpr(8)] + big = [dd[t] > 14 for t in range_constexpr(8)] + exc = [big[t] & ((us[t] & 0x7FFF) != 0) for t in range_constexpr(8)] + nib = fx.Int32(0) + nbig = fx.Int32(0) + nexc = fx.Int32(0) + ev = fx.Int32(0) + for t in range_constexpr(8): + nib = nib | (big[t].select(fx.Int32(15), dd[t]) << NIBPOS[t]) + nbig = nbig + big[t].select(one, zero) + nexc = nexc + exc[t].select(one, zero) + ev = ev | exc[t].select(us[t], zero) + has = nexc > 0 + vsp = fx.Int64(fx.rocdl.ballot(T.i64, nbig > 0)) + v1 = fx.Int64(fx.rocdl.ballot(T.i64, has)) + av = [((us[t] >> 8) & 0x80) | (us[t] & 0x7F) for t in range_constexpr(8)] + a0 = av[0] | (av[1] << 8) | (av[2] << 16) | (av[3] << 24) + a1 = av[4] | (av[5] << 8) | (av[6] << 16) | (av[7] << 24) + cnt = zero + rawi = zero + idx = one + if v1 != fx.Int64(0): + # an exception in this wave (wave-uniform, rare): list indices, or a raw wave + v2 = fx.Int64(fx.rocdl.ballot(T.i64, nexc > 1)) + cnt = fx.Int64(fx.math.ctpop(v1)).to(fx.Int32) + rawi = ((v2 != fx.Int64(0)) | (cnt > NEXC)).select(one, zero) + below = v1 & ((fx.Int64(1) << fx.Int64(lane)) - fx.Int64(1)) + idx = fx.Int64(fx.math.ctpop(below)).to(fx.Int32) + 1 + ax = [exc[t].select(((us[t] >> 8) & 0x80) | idx, av[t]) for t in range_constexpr(8)] + a0 = ax[0] | (ax[1] << 8) | (ax[2] << 16) | (ax[3] << 24) + a1 = ax[4] | (ax[5] << 8) | (ax[6] << 16) | (ax[7] << 24) + rawb = rawi != 0 + cwv = rawb.select(fx.Int32(RAWF), cnt | (vsp != fx.Int64(0)).select(fx.Int32(SPECF), zero)) + return a0, a1, nib, gmax, cwv, ev, idx, has, rawi + + +@flyc.jit +def enc_store(value, base, raw, tid): + """Write code lane ``tid``'s 8 values (4 packed dwords) of the coded row at ``base`` (or its virtual wave's raw + fallback at ``raw``) with system-coherent stores, without publishing; every lane of the hardware wave codes one + virtual wave.""" + lane, wave = tid % 64, tid // 64 + a0, a1, nib, gmax, cwv, ev, idx, has, rawi = enc(value, lane) + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=CODED) + rr = buffer_ops.create_buffer_resource_from_addr(raw, num_records_bytes=ROWB) + buffer_ops.buffer_store(value, rr, tid * 4, mask=rawi != 0, cache_modifier=SYS) + buffer_ops.buffer_store(Vec.from_elements([a0, a1, nib], fx.Int32), rs, tid * 3, cache_modifier=SYS) + buffer_ops.buffer_store(gmax.to(fx.Int8), rs, HDR + 32 * wave + lane // 4, mask=lane % 4 == 0, + cache_modifier=SYS, offset_is_bytes=True) + buffer_ops.buffer_store(cwv, rs, HDR // 4 + 8 * wave + 4, mask=lane == 0, cache_modifier=SYS) + buffer_ops.buffer_store(ev.to(fx.Int16), rs, HDR + 32 * wave + 20 + 2 * (idx - 1), + mask=has & (rawi == 0), cache_modifier=SYS, offset_is_bytes=True) + + +def _dec_fast(pw, pb, hw, lane): + """4 packed bf16 pairs of a lane from its planes and its wave's header words (no zero / exception).""" + q4 = lane // 16 + hs = (q4 == 0).select(hw[0], (q4 == 1).select(hw[1], (q4 == 2).select(hw[2], hw[3]))) + gE = (hs >> (((lane // 4) % 4) * 8)) & 0xFF + e16 = gE | (gE << 16) + out = [] + for p in range_constexpr(4): + w = pw[p // 2] + sh = 16 * (p % 2) + a16 = ((w >> sh) & 0xFF) | (((w >> (sh + 8)) & 0xFF) << 16) + d16 = (pb >> (4 * p)) & 0x000F000F + out.append(((a16 << 8) & SIGN2) | (((e16 - d16) << 7) | (a16 & 0x007F007F))) + return out + + +def _dec_fix(pairs, pw, pb, hw): + """The pairs with zeros and exceptions applied (the per-value selects).""" + zero = fx.Int32(0) + e0, e1, e2 = hw[5], hw[6], hw[7] + v16 = [] + for t in range_constexpr(8): + a = (pw[t // 4] >> (8 * (t % 4))) & 0xFF + d = (pb >> NIBPOS[t]) & 15 + isx = (d == 15) & ((a & 0x7F) != 0) + sl = isx.select((a & 0x7F) - 1, zero) + h = sl // 2 + dw = (h == 0).select(e0, (h == 1).select(e1, e2)) + xv = (dw >> ((sl % 2) * 16)) & 0xFFFF + normal = (pairs[t // 2] >> (16 * (t % 2))) & 0xFFFF + v16.append(isx.select(xv, (d == 15).select((a & 0x80) << 8, normal))) + return [v16[2 * p] | (v16[2 * p + 1] << 16) for p in range_constexpr(4)] + + +def load16(base, tx, mask, cm=SYS): + """Thread tx's two records and its virtual wave's 8 header words of the coded row at ``base``; issue every load + of a row before decoding; ``cm`` as ``dec16``.""" + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=CODED) + recs = [] + for k in range_constexpr(2): + r = Vec(buffer_ops.buffer_load(rs, tx * 6 + k * 3, vec_width=3, dtype=fx.Int32, mask=mask, + cache_modifier=cm)) + recs.append([fx.Int32(r[t]) for t in range_constexpr(3)]) + hw = [] + for k in range_constexpr(2): + h = Vec(buffer_ops.buffer_load(rs, HDR // 4 + 8 * (tx // 32) + 4 * k, vec_width=4, dtype=fx.Int32, + mask=mask, cache_modifier=cm)) + hw += [fx.Int32(h[t]) for t in range_constexpr(4)] + return recs, hw + + +def _dec8(rec, hw, raw, tid, mask, cm): + """Code lane ``tid``'s 4 packed bf16 pairs. Zeros / exceptions by selects and the raw load by a lane mask: one + hardware wave holds two virtual waves, so no wave-uniform branch.""" + pw, pb = rec[:2], rec[2] + d = _dec_fix(_dec_fast(pw, pb, hw, tid % 64), pw, pb, hw) + is_raw = (hw[4] & RAWF) != 0 + rr = buffer_ops.create_buffer_resource_from_addr(raw, num_records_bytes=ROWB) + r = Vec(buffer_ops.buffer_load(rr, tid * 4, vec_width=4, dtype=fx.Int32, mask=mask & is_raw, cache_modifier=cm)) + return Vec.from_elements([is_raw.select(fx.Int32(r[t]), d[t]) for t in range_constexpr(4)], fx.Int32) + + +def dec16(recs, hw, raw, tx, mask, cm=SYS): + """Thread tx's 16 values (columns 16 tx ..) as two 4-dword packs from ``load16``'s words; a raw virtual wave + reads the row's raw slot at ``raw``. ``cm``: SYS where the rows were written in the same launch or by the peer's + CUs; 0 (cached) for an SDMA receive region, whose lines a system-coherent load left stale in L2 for the next + kernel's plain loads (test_handoffs.py, SDMA reduce-scatter after the coded tail).""" + return [_dec8(recs[k], hw, raw, 2 * tx + k, mask, cm) for k in range_constexpr(2)] diff --git a/amd_kernels/collectives/pull_collectives_ranks2/norm_gather_h6144.py b/amd_kernels/collectives/pull_collectives_ranks2/norm_gather_h6144.py new file mode 100644 index 0000000..e91c513 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/norm_gather_h6144.py @@ -0,0 +1,244 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Dense Gemma add/RMSNorm/FP32 MXFP8 quant and chunk-ready all-gather on two ranks. + +Eight-wave arithmetic is the dense tiled norm's at every row count. Code rows +are row-major; each rank's scale block uses AITER's shuffled layout. Persistent +stagers write the codes through L2 into the pool slot (for the peer) and into the +own block of the output, then add to the chunk's monotonic ticket without waiting +(every call adds MAX_STAGERS to every chunk's ticket: stager 0 adds the launch's +missing count). Every puller serves the peer and owns whole chunks (dealt +round-robin): it copies a chunk once its ticket reaches (call) x MAX_STAGERS. +Only tickets use atomic arithmetic. + +Each communicator owns two staging slots and a device generation. A peer can +read slot c%2 until call c+1 publishes; call c+2 cannot start before that peer +completed c. No peer reads the producer's input, only communicator staging. +""" +import functools +import struct + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import ( + H, PRELOAD_HINTS, bf16x8, pk_bf16, seq_sum_sq, wave_sum_lane63, rsqrt_rn, +) +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.add_rmsnorm_mxfp8 import ( + _block_sum, _codes, _e8m0, _scale_off, _set_fp16_ovfl, +) +from .host import WORLD, _alloc_pool, _exchange, _open_pool, _run_compiled, ops +from .kernel import SYS, _flag_add, _flag_read, _sel2 + +NW = 8 +NT = NW * 64 +PULLERS = 16 +STAGERS = 192 +MAX_STAGERS = 192 # every call adds this much to every part's ticket, whatever the launch's stager count +PARTS = 32 +FLAG_STRIDE = 128 +MAX_PARTS = 32 +END = MAX_PARTS * FLAG_STRIDE +FLAG_BYTES = END + FLAG_STRIDE + + +def compile_norm_gather_h6144(pullers=PULLERS, stagers=STAGERS, parts=PARTS): + assert 1 <= parts <= MAX_PARTS + assert pullers > 0 and stagers % 8 == 0 and stagers <= MAX_STAGERS and pullers + stagers <= 256 + step = NT * 4 * 4 + name = f'gemma_add_rmsnorm_mxfp8_gather_ranks2_h6144_p{pullers}_n{stagers}_c{parts}' + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, NW] + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + arg_x: fx.Int64, arg_r: fx.Int64, arg_w: fx.Int64, arg_res: fx.Int64, + arg_q: fx.Int64, arg_s: fx.Int64, + arg_p0: fx.Int64, arg_p1: fx.Int64, arg_f0: fx.Int64, arg_f1: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i64_scale_off: fx.Int64, + i32_m: fx.Int32, i32_sx: fx.Int32, i32_sr: fx.Int32, + i32_eps: fx.Int32, i32_me: fx.Int32, + ): + tx, bid = fx.thread_idx.x, fx.block_idx.x + seq = fx.Int32(counter[0]) + 1 + slot = fx.Int64((seq - 1) & 1) * i64_stride + own = _sel2(i32_me, arg_f0, arg_f1) + peer = 1 - i32_me + if bid >= pullers: + _set_fp16_ovfl() + sid = bid - pullers + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + shared = fx.SharedAllocator().allocate(Shared).peek() + part = shared.part.ptr + pool = _sel2(i32_me, arg_p0, arg_p1) + slot + qr = buffer_ops.create_buffer_resource_from_addr(pool, num_records_bytes=fx.Int64(i32_m) * H) + sr = buffer_ops.create_buffer_resource_from_addr(pool + i64_scale_off, num_records_bytes=fx.Int64(i32_m) * (H//32)) + qo = buffer_ops.create_buffer_resource_from_addr(arg_q+fx.Int64(i32_me)*fx.Int64(i32_m)*H, num_records_bytes=fx.Int64(i32_m) * H) + so = buffer_ops.create_buffer_resource_from_addr(arg_s+fx.Int64(i32_me)*fx.Int64(i32_m)*(H//32), num_records_bytes=fx.Int64(i32_m) * (H//32)) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=H*2) + has1 = tx < 256 + offs = [tx*4, tx*4+2048] + masks = [None, has1] + wv = [bf16x8(Vec(buffer_ops.buffer_load(wr, offs[c], vec_width=4, dtype=fx.Int32, mask=masks[c]))) for c in range_constexpr(2)] + k = fx.Int32(0) + while k < parts: + first = (k * (i32_m//32) // parts) * 32 + last = ((k+1) * (i32_m//32) // parts) * 32 + row = first + sid + while row < last: + xr = buffer_ops.create_buffer_resource_from_addr(arg_x + fx.Int64(row)*fx.Int64(i32_sx*2), num_records_bytes=H*2) + rr = buffer_ops.create_buffer_resource_from_addr(arg_r + fx.Int64(row)*fx.Int64(i32_sr*2), num_records_bytes=H*2) + resr = buffer_ops.create_buffer_resource_from_addr(arg_res + fx.Int64(row)*(H*2), num_records_bytes=H*2) + xv = [bf16x8(Vec(buffer_ops.buffer_load(xr, offs[c], vec_width=4, dtype=fx.Int32, mask=masks[c]))) for c in range_constexpr(2)] + rv = [bf16x8(Vec(buffer_ops.buffer_load(rr, offs[c], vec_width=4, dtype=fx.Int32, mask=masks[c]))) for c in range_constexpr(2)] + vals = [[xv[c][i]+rv[c][i] for i in range_constexpr(8)] for c in range_constexpr(2)] + for c in range_constexpr(2): + packed = Vec.from_elements([pk_bf16(vals[c][2*i], vals[c][2*i+1]) for i in range_constexpr(4)], fx.Int32) + buffer_ops.buffer_store(packed,resr,offs[c],mask=masks[c]) + acc = wave_sum_lane63(seq_sum_sq(vals[0]+vals[1])) + if lane == 63: + part[wave] = acc + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + inv = rsqrt_rn(_block_sum([part[i] for i in range_constexpr(NW)]), i32_eps.bitcast(fx.Float32)) + for c in range_constexpr(2): + y = [(vals[c][i]*inv)*(wv[c][i]+fx.Float32(1.0)) for i in range_constexpr(8)] + sb, rescale = _e8m0(y) + words = _codes(y,rescale) + cw = Vec.from_elements(words,fx.Int32) + sw = sb.to(fx.Int32).to(fx.Int8) + smask = ((tx%4)==0) & ((tx<256) if c else (tx<512)) + buffer_ops.buffer_store(cw,qr,row*(H//4)+tx*2+c*1024,mask=masks[c],cache_modifier=SYS) + buffer_ops.buffer_store(sw,sr,_scale_off(row,tx//4+c*128),mask=smask,offset_is_bytes=True,cache_modifier=SYS) + buffer_ops.buffer_store(cw,qo,row*(H//4)+tx*2+c*1024,mask=masks[c]) + buffer_ops.buffer_store(sw,so,_scale_off(row,tx//4+c*128),mask=smask,offset_is_bytes=True) + # All waves must finish reading the previous row's LDS sums. + fx.gpu.barrier() + row = row + stagers + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + _flag_add(own+fx.Int64(k*FLAG_STRIDE+4),(sid == 0).select(fx.Int32(1+MAX_STAGERS-stagers),fx.Int32(1))) + k = k+1 + else: + flags = _sel2(peer,arg_f0,arg_f1) + src = _sel2(peer,arg_p0,arg_p1)+slot + qsrc = buffer_ops.create_buffer_resource_from_addr(src,num_records_bytes=fx.Int64(i32_m)*H) + ssrc = buffer_ops.create_buffer_resource_from_addr(src+i64_scale_off,num_records_bytes=fx.Int64(i32_m)*(H//32)) + qdst = buffer_ops.create_buffer_resource_from_addr(arg_q+fx.Int64(peer)*fx.Int64(i32_m)*H,num_records_bytes=fx.Int64(i32_m)*H) + sdst = buffer_ops.create_buffer_resource_from_addr(arg_s+fx.Int64(peer)*fx.Int64(i32_m)*(H//32),num_records_bytes=fx.Int64(i32_m)*(H//32)) + j = bid + while j < parts: + if tx == 0: + fw = flags+fx.Int64(j*FLAG_STRIDE+4) + ready = _flag_read(fw) + while ready - seq*MAX_STAGERS < 0: + ready = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire,syncscope='') + r0 = (j*(i32_m//32)//parts)*32 + r1 = ((j+1)*(i32_m//32)//parts)*32 + off = r0*(H//4)+tx*4 + while off < r1*(H//4): + qs = [Vec(buffer_ops.buffer_load(qsrc,off+u*NT*4,vec_width=4,dtype=fx.Int32,mask=off+u*NT*4> sh) & 0xFF) | (((b[2 * h] >> sh) & 0xFF) << 8) + w = w | (((a[2 * h + 1] >> sh) & 0xFF) << 16) | (((b[2 * h + 1] >> sh) & 0xFF) << 24) + off = ((m1 * K1 + k1) * 4 + kl) * 16 + mlane + buffer_ops.buffer_store(w, scales, off) + t = t + per_src * NT + + @flyc.jit + def launch_unpack_q4096_shuffled( + arg_codes: fx.Int64, + arg_scales: fx.Int64, + arg_s0: fx.Int64, + arg_s1: fx.Int64, + arg_f0: fx.Int64, + arg_f1: fx.Int64, + i64_src_off: fx.Int64, + i32_seq: fx.Int32, + i32_me: fx.Int32, + i32_rows: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_codes, arg_scales, arg_s0, arg_s1, arg_f0, arg_f1, + i64_src_off, i32_seq, i32_me, i32_rows, + ).launch(grid=(grid, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_unpack_q4096_shuffled.kernel_name = name + return launch_unpack_q4096_shuffled + + +@functools.cache +def _kernel(wgs: int, fence: bool): + return compile_pull_unpack_q4096_shuffled(wgs, fence) + + +# packed rows (tokens) up to which the release/publish rides in the unpack launch +# (the measured split for the row-major unpack) +FENCE_MAX_TOKENS = 8192 + + +def all_to_all_unpack_q4096_shuffled(coll: Collectives, x: torch.Tensor, wgs: int | None = None, + fence: bool | None = None): + """The all-to-all of packed rows ``x`` ``[2 rows, 4096 + 128]`` (row block r to rank r), unpacked + (``reference.unpack_reference``), with the scales in o_proj_full's GEMM layout: + (codes ``[rows, 8192]`` e4m3, scales ``[ceil(rows / 32) * 32, 256]`` u8, AITER's + shuffle), or None when ``x`` does not fit the slot (the caller falls back). + ``fence``: force the publication form (None: ``FENCE_MAX_TOKENS``). ``wgs`` pullers (None: ``WGS``, the best of + 16 / 32 / 64 in both forms at 4096..32768 tokens).""" + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("SP unpack uses eager Collectives sequence state") + x = x.contiguous() + rows = x.shape[0] // WORLD + assert x.dtype == torch.uint8 and x.shape == (WORLD * rows, D + D // 32) and rows > 0 + off = coll._stage(x) + if off is None: + return None + p = coll.p + if fence is None: + fence = x.shape[0] <= FENCE_MAX_TOKENS + if wgs is None: + wgs = WGS + if fence: + coll.seq = p.next_seq(coll.set_id) + else: + coll._publish_wait() + codes = torch.empty((rows, WORLD * D), dtype=torch.uint8, device=x.device) + scales = torch.empty(((rows + 31) // 32 * 32, WORLD * D // 32), dtype=torch.uint8, device=x.device) + block = x.numel() // WORLD + _run_compiled( + _kernel(wgs, fence), + codes.data_ptr(), + scales.data_ptr(), + *[ptr + off for ptr in p.pool_ptrs], + *[f + coll.flag_off for f in p.flag_ptrs], + p.rank * block, + coll.seq, + p.rank, + rows, + coll.stream, + ) + return codes.view(torch.float8_e4m3fn), scales diff --git a/amd_kernels/collectives/pull_collectives_ranks2/reference.py b/amd_kernels/collectives/pull_collectives_ranks2/reference.py new file mode 100644 index 0000000..24981f1 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/reference.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The torch contract of the 2-rank pull collectives (root README, rule 3). + +``xs[p]`` is rank p's input (p = 0, 1), ``r`` this rank, a row block is half of the rows: + +- all-gather (``AllGatherComm.all_gather(outs, ins)``): each output is ``cat(in_0, in_1)`` along the rows, the bytes + as they are. +- all-to-all: row block p of the output is row block r of ``x_p``. +- unpacking all-to-all (``FencedAllGatherComm.all_to_all_unpack_shuffled``, d 4096): the all-to-all of packed rows + ``[2 rows, d + d/32]`` (``d`` code bytes, then ``d / 32`` scale bytes per row), unpacked on the way: + ``codes[i, p d : (p + 1) d]`` / ``scales[i, p d / 32 : (p + 1) d / 32]`` are the code / scale bytes of row i of the + block that came from rank p (the whole o_proj input of this rank's rows); the kernel writes the scales in the GEMM's + shuffled layout. +- reduce-scatter (``SlotsReduceScatterComm.reduce_scatter(x)``): ``out = bf16((+0 + b_0) + b_1)`` with ``b_p`` row + block r of ``x_p`` in fp32: rank order from +0 (a sum of -0.0 is +0.0), denormals kept, one RNE rounding. + +The MoE combine variant is ``slots4_reduce_scatter_reference`` below (slots4: the MoE top-4 slots). Its norm/quant +variant applies ``norm/gemma_add_rmsnorm_h6144/reference.py`` steps 2-4 to this rank's shard, with row-major codes +and shuffled scales. The shard residual and rank sum are bitwise; the norm follows that reference's FP64 tolerances +and matches the existing FlyDSL norm's output bits. + +Tolerances: none for the communication, every collective is bit for bit. +""" + +import torch + +WORLD = 2 + + +def all_gather_reference(ins: list[torch.Tensor]) -> torch.Tensor: + """The 2 ranks' ``[rows, ...]`` inputs -> ``[2 rows, ...]``.""" + assert len(ins) == WORLD + return torch.cat(ins) + + +def all_to_all_reference(xs: list[torch.Tensor], rank: int) -> torch.Tensor: + """Rank ``rank``'s output of the all-to-all of the 2 ranks' ``xs``.""" + return torch.cat([x.reshape(WORLD, -1)[rank] for x in xs]).view(xs[0].dtype).view(xs[0].shape) + + +def unpack_reference(recv: torch.Tensor, rows: int, d: int) -> tuple[torch.Tensor, torch.Tensor]: + """``[2 rows, d + d / 32]`` received packed rows (block p from rank p) -> (codes ``[rows, 2 d]``, scales + ``[rows, 2 d / 32]``), uint8.""" + blocks = recv.view(torch.uint8).view(WORLD, rows, d + d // 32).transpose(0, 1) + return ( + blocks[:, :, :d].reshape(rows, WORLD * d), + blocks[:, :, d:].reshape(rows, WORLD * (d // 32)), + ) + + +def reduce_scatter_reference(xs: list[torch.Tensor], rank: int) -> torch.Tensor: + """Rank ``rank``'s ``[rows, H]`` bf16 sums of the 2 ranks' ``[2 rows, H]`` bf16 partials (module docstring).""" + rows = xs[0].shape[0] // WORLD + blk = slice(rank * rows, (rank + 1) * rows) + acc = torch.zeros((rows, *xs[0].shape[1:]), dtype=torch.float32, device=xs[0].device) + for x in xs: + acc = acc + x[blk].float() + return acc.bfloat16() + + +def slots4_reduce_scatter_reference(slots: list[torch.Tensor], shared: list[torch.Tensor], rank: int) -> torch.Tensor: + """Weighted BF16 [M,4,H] slots -> BF16 [M/2,H]. + + ``shared[p]``: rank p's BF16 shared expert term, [M, H] (its partial on every row) or [M/2, H] (owner form: the + whole shared expert output of rank p's own row block p, which only rank p adds). + Local order on rank p: (((s0+s1)+s2)+s3) [+ shared] in FP32, one BF16 RNE rounding (owner form: the shared term on + block p's rows only; elsewhere none, the same bits as adding +0 up to a zero's sign, which the rank sum drops). + Rank order: +0, rank0, rank1 in FP32, one BF16 RNE. + Denormals are retained. Exact bytes are the contract; a reduce-then-add-shared path rounds twice locally and + therefore has a different numerical contract. + """ + xs = [] + for p, (s, h) in enumerate(zip(slots, shared)): + acc = s[:, 0].float() + for k in range(1, 4): + acc = acc + s[:, k].float() + if h.shape[0] == s.shape[0]: + acc = acc + h.float() + else: + rows = h.shape[0] + acc[p * rows : (p + 1) * rows] = acc[p * rows : (p + 1) * rows] + h.float() + xs.append(acc.bfloat16()) + return reduce_scatter_reference(xs, rank) + + +def norm_all_gather_reference(xs, residuals, weights, eps=1e-6): + """Dense SP norm + all-gather contract, two BF16 [rows,6144] shards. + + Each shard follows norm/reference.py's FP64 add/RMSNorm contract. Quantize the FP32 norm result (not its BF16 + rounding) at every M. The exact shader oracle is gemma_add_rmsnorm_mxfp8, untiled, with its eight-wave SSQ tree. + Codes concatenate row-major. Scale bytes concatenate two independent AITER-shuffled blocks; shards must have a + multiple of 32 rows. Returns FP64 norm shards and BF16 residual shards for compare_mxfp8; tests additionally + require byte equality to the standalone shader oracle. + """ + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import add_rmsnorm_reference + + assert len(xs) == len(residuals) == len(weights) == WORLD + return [add_rmsnorm_reference(x, r, w, eps) for x, r, w in zip(xs, residuals, weights)] diff --git a/amd_kernels/collectives/pull_collectives_ranks2/slots4_host.py b/amd_kernels/collectives/pull_collectives_ranks2/slots4_host.py new file mode 100644 index 0000000..3c44d60 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/slots4_host.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-safe MoE combine + reduce-scatter on two MI355X GPUs. + +One communicator per stream/microbatch. Construct collectively and warm up before graph capture. Buffers and counters +live for the process lifetime, as in Pools. Neither inputs nor outputs may alias the communicator's pool. +""" +import functools +import struct + +import torch + +from amd_kernels.common import hip_runtime as ops +from .host import Pools, WORLD, _exchange, _run_compiled +from .kernel import compile_sum2 +from .lossless_x3 import CODED +from .slots4_stage_pull_sum2_add_rmsnorm_mxfp8 import ( + NORMERS, PARTS, PULLERS, STAGERS, UNROLL, compile_slots4_stage_pull_sum2_add_rmsnorm_mxfp8) +from .slots4_stage_pull import FLAG_BYTES, compile_slots4_stage_pull +from .slots4_stage_rows import compile_slots4_stage_rows +from .stage_pull import PARTS as RS_PARTS, PULLERS as RS_PULLERS, STAGERS as RS_STAGERS, UNROLL as RS_UNROLL +from .stage_pull import TICKET_BYTES, compile_rs_stage_pull +from .sum2_add_rmsnorm_mxfp8 import compile_sum2_add_rmsnorm_mxfp8 + +H = 6144 +MAX_ROWS = 32768 + + +@functools.cache +def _slots4_kernels(owner=False): + return compile_slots4_stage_pull(owner=owner), compile_sum2() + + +@functools.cache +def _rs_kernel(*grid): + return compile_rs_stage_pull(*grid) + + +class SlotsReduceScatterComm: + """Weighted slots + shared -> this rank's BF16 rows, fixed FP32 sum order. + + ``shared`` comes in two forms, told apart by its rows: [2 m, H] (every token: each rank's partial of the shared + expert, added to every row's slot sum) or [m, H] (owner form: this rank's whole shared expert output of its own m + rows, added to its own rows' slot sum only; the peer's rows carry its slot sum alone). Sum order per rank p: + ``P_p = bf16((((s0 + s1) + s2) + s3) [+ shared])``, then ``bf16((+0 + P_0) + P_1)``. + + The slot sum + shared stage overlaps the pull of the peer's sums (the own block is staged locally). Supports total + rows divisible by two, 2..32768. The one-launch tail moves the peer's sums in ``lossless_x3``'s exact code + (``coded``, 0.781x the bytes), the other entries BF16; no lossy communication and no atomic output adds. + Calls on different ranks must use the same shape and order. Concurrent streams need distinct communicators. Graph + replays and eager calls may be interleaved. ``tail`` (the one-launch tail's pullers, stagers, norm workers, parts, + unroll) and ``rs`` (the dense reduce-scatter's pullers, stagers, parts, unroll) are fixed for the communicator's + lifetime (their tickets count the launches). + """ + + def __init__(self, rank, cpu_group, device, max_rows=MAX_ROWS, tail=(PULLERS, STAGERS, NORMERS, PARTS, UNROLL), + rs=(RS_PULLERS, RS_STAGERS, RS_PARTS, RS_UNROLL), coded=True): + assert WORLD <= max_rows <= MAX_ROWS and max_rows % WORLD == 0 + self.device = torch.device(device) + self.max_rows = max_rows + self.pools = Pools(rank, WORLD, cpu_group, self.device, max_rows * H * 2, sets=1) + self.counter = torch.zeros(1, dtype=torch.int32, device=self.device) + # the stage + pull launch's per-chunk flags (uncached, zeroed, mapped by the peer) + ptr, handle = ops.allocate_shared_buffer_and_handle(FLAG_BYTES) + handles = _exchange(handle, rank, WORLD, cpu_group) + self.chunk_flags = [ptr if r == rank else ops.open_mem_handle(h) for r, h in enumerate(handles)] + ptr, handle = ops.allocate_shared_buffer_and_handle(TICKET_BYTES) + handles = _exchange(handle, rank, WORLD, cpu_group) + self.rs_tickets = [ptr if r == rank else ops.open_mem_handle(h) for r, h in enumerate(handles)] + self.rs_calls = torch.zeros(1, dtype=torch.int32, device=self.device) + self.stage_pull, self.sum2 = _slots4_kernels() + self.stage_pull_owner = _slots4_kernels(True)[0] + self.rs = _rs_kernel(*rs) + self.coded = coded + self.tail = _fused_tail(*tail, coded) + self.tail_owner = _fused_tail(*tail, coded, True) + self.tail_calls = torch.zeros(1, dtype=torch.int32, device=self.device) + + def _check_inputs(self, slots, shared): + """-> (every token's rows, whether ``shared`` is the owner form).""" + rows = slots.shape[0] + assert WORLD <= rows <= self.max_rows and rows % WORLD == 0 + assert slots.shape == (rows, 4, H) and shared.shape in ((rows, H), (rows // WORLD, H)), ( + slots.shape, shared.shape) + for x in (slots, shared): + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.device == self.device + return rows, shared.shape[0] != rows + + def _stage_slots(self, slots, shared): + """Slot sum + shared of the peer's rows into the pool, overlapped with the pull of the peer's sums of this + rank's rows -> the bf16 staging ``[rows, H]`` of both ranks' sums of this rank's rows (rank 0's block first).""" + rows, owner = self._check_inputs(slots, shared) + p = self.pools + staging = torch.empty(rows * H * 2, dtype=torch.uint8, device=self.device) + _run_compiled(self.stage_pull_owner if owner else self.stage_pull, slots.data_ptr(), shared.data_ptr(), + staging.data_ptr(), + *p.pool_ptrs, *self.chunk_flags, self.counter, p.slot_bytes, rows, p.rank, + torch.cuda.current_stream(self.device)) + return staging + + def _slabs(self, staging, rows): + block = rows // WORLD * H * 2 + return staging.data_ptr(), staging.data_ptr() + block + + def _sum(self, staging, rows, out): + if out is None: + out = torch.empty((rows // WORLD, H), dtype=torch.bfloat16, device=self.device) + assert out.shape == (rows // WORLD, H) and out.dtype == torch.bfloat16 + assert out.is_contiguous() and out.device == self.device + _run_compiled(self.sum2, out.data_ptr(), *self._slabs(staging, rows), rows // WORLD * H * 2, + torch.cuda.current_stream(self.device)) + return out + + def slots4_reduce_scatter(self, slots, shared, out=None): + """Slot sum and shared add fused into BF16 reduce-scatter publication (``shared`` either form).""" + staging = self._stage_slots(slots, shared) + return self._sum(staging, slots.shape[0], out) + + def reduce_scatter(self, x, out=None): + """Dense BF16 RS, same transport, graph-safe device sequence counters.""" + rows = x.shape[0] + assert WORLD <= rows <= self.max_rows and rows % WORLD == 0 + assert x.shape == (rows, H) and x.dtype == torch.bfloat16 + assert x.is_contiguous() and x.device == self.device + if out is None: + out = torch.empty((rows // WORLD, H), dtype=torch.bfloat16, device=self.device) + assert out.shape == (rows // WORLD, H) and out.dtype == torch.bfloat16 + assert out.is_contiguous() and out.device == self.device + p = self.pools + _run_compiled(self.rs, x.data_ptr(), out.data_ptr(), *p.pool_ptrs, *self.rs_tickets, self.counter, + self.rs_calls, p.slot_bytes, rows // WORLD * H * 2, p.rank, + torch.cuda.current_stream(self.device)) + return out + + def slots4_reduce_scatter_add_rmsnorm_mxfp8(self, slots, shared, residual, weight, eps, fused=True): + """Slot sum + shared (either form) + BF16 RS + shard norm/quant: one launch (``fused``), else the stage + pull + and the norm launch (the same bits; the SDMA hand-offs' tail kernel). + + Returns (row-major codes, shuffled scales, residual_out). + """ + rows, owner = self._check_inputs(slots, shared) + rows //= WORLD + assert residual.shape == (rows, H) and residual.dtype == torch.bfloat16 + assert residual.is_contiguous() and residual.device == self.device + assert weight.shape == (H,) and weight.dtype == torch.bfloat16 + assert weight.is_contiguous() and weight.device == self.device + q = torch.empty((rows, H), dtype=torch.float8_e4m3fn, device=self.device) + scale = torch.empty(((rows + 31) // 32 * 32, H // 32), dtype=torch.uint8, device=self.device) + res_out = torch.empty_like(residual) + eps_bits = struct.unpack(' slot c % 2) in ``parts`` chunks, each published by a ticket and a flag +word (the last of the stagers stores c+1 into it), then the own block straight into its rows of the local staging +``[rows, H]``. Stores are system-coherent (SC0 | SC1, through the L2). After a chunk every wave waits for its stores +(vmcnt 0), the workgroup meets at a barrier and lane 0 takes the chunk's ticket. + +Pullers: every one serves the peer. Per part j its lane 0 spins on the peer's flag word j, the workgroup acquires, +then copies its stripes of the part with system-coherent 16-byte loads into the peer's rows of the staging. The last +workgroup of the launch to finish stores c+1 into the device counter. + +Flag words: one uncached buffer per rank, mapped by the peer: flags at word j (j < parts), tickets at byte +TICKETS + 4 j, the launch's end ticket at END. A rank rewrites slot c % 2 only in call c+2, after the peer finished +the pull of call c (the peer's call c+1 launch started after it). + +``owner``: ``shared`` is [rows / 2, H], this rank's shared expert output of its own rows, added to the own block only +(masked loads: the peer's rows add +0, which only drops a zero's sign; the rank-order sum from +0 drops it anyway). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import bf16x8, pk_bf16 +from .kernel import SYS, _flag_add, _flag_read, _sel2 + +H = 6144 +NT = 256 +UNROLL = 4 +PULLERS = 16 +STAGERS = 192 +PARTS = 16 +TICKETS = 256 # byte offsets in the chunk-flag buffer +END = 1024 +FLAG_BYTES = 4096 + + +def compile_slots4_stage_pull(pullers: int = PULLERS, stagers: int = STAGERS, parts: int = PARTS, + unroll: int = UNROLL, owner: bool = False): + assert pullers + stagers <= 256 and 4 * parts <= TICKETS + step = NT * 4 * unroll # dwords per puller per loop step + nwg = pullers + stagers + name = (f"slots4_shared_stage_pull_h6144_ranks2_p{pullers}_s{stagers}_c{parts}_u{unroll}" + + ("_owner" if owner else "")) + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel(arg_slots: fx.Int64, arg_shared: fx.Int64, arg_dst: fx.Int64, + arg_p0: fx.Int64, arg_p1: fx.Int64, arg_c0: fx.Int64, arg_c1: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32): + tx = fx.thread_idx.x + bid = fx.block_idx.x + i32_seq = fx.Int32(counter[0]) + 1 + slot_off = fx.Int64((i32_seq - 1) & 1) * i64_stride + br = i32_rows // 2 + peer = 1 - i32_me + my_c = _sel2(i32_me, arg_c0, arg_c1) + if bid >= pullers: + sid = bid - pullers + pool = _sel2(i32_me, arg_p0, arg_p1) + slot_off + hr = buffer_ops.create_buffer_resource_from_addr( + arg_shared, num_records_bytes=fx.Int64(br if owner else i32_rows) * H * 2) + k = fx.Int32(0) + while k < parts + 1: + # k < parts: part k of the peer's block; k == parts: the whole own block + last = k == parts + r0 = last.select(i32_me * br, peer * br + k * br // parts) + r1 = last.select((i32_me + 1) * br, peer * br + (k + 1) * br // parts) + dst = buffer_ops.create_buffer_resource_from_addr( + last.select(arg_dst, pool), num_records_bytes=fx.Int64(i32_rows) * H * 2) + piece = sid + while piece < (r1 - r0) * 3: + row = r0 + piece // 3 + col = (piece % 3) * (NT * 4) + tx * 4 + off = row * (H // 2) + col + sr = buffer_ops.create_buffer_resource_from_addr( + arg_slots + fx.Int64(row) * (4 * H * 2), num_records_bytes=4 * H * 2) + vals = [bf16x8(Vec(buffer_ops.buffer_load(sr, col + q * (H // 2), vec_width=4, dtype=fx.Int32))) + for q in range_constexpr(4)] + if const_expr(owner): + shared = bf16x8(Vec(buffer_ops.buffer_load(hr, off - i32_me * br * (H // 2), vec_width=4, + dtype=fx.Int32, mask=last))) + else: + shared = bf16x8(Vec(buffer_ops.buffer_load(hr, off, vec_width=4, dtype=fx.Int32))) + acc = [(((vals[0][e] + vals[1][e]) + vals[2][e]) + vals[3][e]) + shared[e] + for e in range_constexpr(8)] + packed = [pk_bf16(acc[2 * e], acc[2 * e + 1]) for e in range_constexpr(4)] + buffer_ops.buffer_store(Vec.from_elements(packed, fx.Int32), dst, off, cache_modifier=SYS) + piece = piece + stagers + if k < parts: + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + tk = my_c + TICKETS + fx.Int64(k * 4) + if _flag_add(tk, 1) == stagers - 1: + _flag_add(tk, -stagers) + fr = buffer_ops.create_buffer_resource_from_addr(my_c, num_records_bytes=TICKETS) + buffer_ops.buffer_store(i32_seq, fr, k, cache_modifier=SYS) + k = k + 1 + else: + fa = _sel2(peer, arg_c0, arg_c1) + block = fx.Int64(br) * (H * 2) + srs = buffer_ops.create_buffer_resource_from_addr( + _sel2(peer, arg_p0, arg_p1) + slot_off + fx.Int64(i32_me * br) * (H * 2), num_records_bytes=block) + drs = buffer_ops.create_buffer_resource_from_addr(arg_dst + fx.Int64(peer) * block, num_records_bytes=block) + j = fx.Int32(0) + while j < parts: + if tx == 0: + fw = fa + fx.Int64(j * 4) + f = _flag_read(fw) + while f < i32_seq: + f = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="") + d1 = ((j + 1) * br // parts) * (H // 2) + off = (j * br // parts) * (H // 2) + bid * step + tx * 4 + while off < d1: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(srs, o, vec_width=4, dtype=fx.Int32, mask=o < d1, + cache_modifier=SYS))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], drs, o, mask=o < d1) + off = off + step * pullers + j = j + 1 + fx.gpu.barrier() + if tx == 0: + if _flag_add(my_c + END, 1) == nwg - 1: + _flag_add(my_c + END, -nwg) + counter[0] = i32_seq + + @flyc.jit + def launch_slots4_stage_pull( + arg_slots: fx.Int64, arg_shared: fx.Int64, arg_dst: fx.Int64, + arg_p0: fx.Int64, arg_p1: fx.Int64, arg_c0: fx.Int64, arg_c1: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32, stream: fx.Stream): + kernel(arg_slots, arg_shared, arg_dst, arg_p0, arg_p1, arg_c0, arg_c1, counter, i64_stride, i32_rows, + i32_me).launch(grid=(nwg, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_slots4_stage_pull.kernel_name = name + return launch_slots4_stage_pull diff --git a/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_pull_sum2_add_rmsnorm_mxfp8.py b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_pull_sum2_add_rmsnorm_mxfp8.py new file mode 100644 index 0000000..8c8c194 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_pull_sum2_add_rmsnorm_mxfp8.py @@ -0,0 +1,311 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-rank MoE tail in one launch: slot combine + shared add, the reduce-scatter, residual add, Gemma RMSNorm and +MXFP8 quant (row-major codes, AITER-shuffled scales). Three roles share the grid (512 threads per workgroup); this +rank's rows are cut into ``parts`` parts of ``ceil(m / parts)`` rows, and every role but the norm owns whole parts +(dealt round-robin), so one workgroup's wait never stalls the others. + +Stagers: per owned part, the slot-order FP32 sum + shared of the peer's rows of that part, rounded once to BF16, into +the own pool slot (counter c -> slot c % 2). A staged row is half-major: 16-byte piece v * 384 + t holds columns +16 t + 8 v .. 16 t + 8 v + 7, the norm lane map's halves, so every wave's stores and loads of a row cover whole lines +(uncached accesses in the natural order use half of every line). ``coded`` (the default): instead every wave codes one +virtual wave (512 columns of a row) per step in ``lossless_x3``'s exact 12-bit code, row r of the part at r x 9600 of +the slot, its raw virtual waves at m x 9600 + r x 12288 (inside the bf16 slot's bytes). Then every wave waits for its +system-coherent stores (vmcnt 0), the workgroup meets at a barrier and lane 0 adds 1 to the part's ticket without +waiting for the result. + +Pullers: per owned part, lane 0 spins until the peer's ticket of the part counts this launch, the workgroup acquires +and copies the part (system-coherent loads over the link, system-coherent stores) into a local staging block, then the +same vmcnt / barrier / ticket publishes it locally. + +Norm workers: rows strided over the workers; per row lane 0 spins on the local ticket of the row's part, then the +pulled peer row from the staging (coded: decoded, a raw virtual wave read from the peer's slot over the link), the +own slot sum + shared of the row computed in place (rounded once, as the peer's stagers round theirs), the rank-order +sum ``(+0 + b0) + b1`` in FP32 and its BF16 rounding, then +``sum2_add_rmsnorm_mxfp8``'s residual add, norm and quant with its lane map (thread tx holds columns 16 tx .. +16 tx + 15, tx < 384), so the output bits equal the two-launch path's. + +Tickets: one uncached buffer per rank, mapped by the peer: the stagers' at byte PEER_TICKETS + 4 j, the pullers' at +LOCAL_TICKETS + 4 j; every launch adds exactly 1 to each, counted against ``calls`` (both ranks launch in the same +order; int32 differences, so the counts may wrap). The launch's end ticket is at END: its last workgroup stores c+1 +into the device counter and the launch number into ``calls``. A rank rewrites slot c % 2 only in call c+2, after the +peer finished the pull of call c (the peer's call c+1 launch started after it). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.add_rmsnorm_mxfp8 import NW, _block_sum, _set_fp16_ovfl +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import ( + PRELOAD_HINTS, H, bf16x8, pk_bf16, rsqrt_rn, wave_sum_lane63, +) +from .kernel import SYS, _flag_add, _flag_read, _sel2 +from .lossless_x3 import CODED, ROWB, VWAVES, dec16, enc_store, load16 +from .slots4_stage_pull import END +from .sum2_add_rmsnorm_mxfp8 import _tail_rowq + +NT = 64 * NW +PULLERS = 16 +STAGERS = 64 +NORMERS = 176 +PARTS = 256 +UNROLL = 4 +PEER_TICKETS = 2048 # byte offsets in the flag buffer, 4 bytes per part +LOCAL_TICKETS = 3072 +HALF = H // 4 # dwords per half-major half row + + +def _slot_sum(arg_slots, arg_shared, row, tx, has, shared_row=None): + """This thread's 16 columns of row ``row``: bf16((((s0 + s1) + s2) + s3) + shared) as 8 packed dwords; the shared + row ``shared_row`` (default ``row``), none if ``arg_shared`` is None.""" + sr = buffer_ops.create_buffer_resource_from_addr(arg_slots + fx.Int64(row) * (4 * H * 2), + num_records_bytes=4 * H * 2) + if const_expr(arg_shared is not None): + hr = buffer_ops.create_buffer_resource_from_addr( + arg_shared + fx.Int64(row if shared_row is None else shared_row) * (H * 2), num_records_bytes=H * 2) + out = [] + for v in range_constexpr(2): + vals = [bf16x8(Vec(buffer_ops.buffer_load(sr, q * (H // 2) + tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, + mask=has))) + for q in range_constexpr(4)] + acc = [((vals[0][e] + vals[1][e]) + vals[2][e]) + vals[3][e] for e in range_constexpr(8)] + if const_expr(arg_shared is not None): + sh = bf16x8(Vec(buffer_ops.buffer_load(hr, tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, mask=has))) + acc = [acc[e] + sh[e] for e in range_constexpr(8)] + out += [pk_bf16(acc[2 * e], acc[2 * e + 1]) for e in range_constexpr(4)] + return out + + +def _slot_sum8(arg_slots, arg_shared, row, tid, shared_row=None, shared_mask=None): + """Code lane ``tid``'s 8 columns of row ``row``, the same sum and rounding as ``_slot_sum``, as 4 packed dwords; + ``shared_mask``: rows without it add +0 (the owner form's peer rows: only a zero's sign differs, which the + rank-order sum from +0 drops).""" + sr = buffer_ops.create_buffer_resource_from_addr(arg_slots + fx.Int64(row) * (4 * H * 2), + num_records_bytes=4 * H * 2) + vals = [bf16x8(Vec(buffer_ops.buffer_load(sr, q * (H // 2) + tid * 4, vec_width=4, dtype=fx.Int32))) + for q in range_constexpr(4)] + acc = [((vals[0][e] + vals[1][e]) + vals[2][e]) + vals[3][e] for e in range_constexpr(8)] + if const_expr(arg_shared is not None): + hr = buffer_ops.create_buffer_resource_from_addr( + arg_shared + fx.Int64(row if shared_row is None else shared_row) * (H * 2), num_records_bytes=H * 2) + sh = bf16x8(Vec(buffer_ops.buffer_load(hr, tid * 4, vec_width=4, dtype=fx.Int32, mask=shared_mask))) + acc = [acc[e] + sh[e] for e in range_constexpr(8)] + return Vec.from_elements([pk_bf16(acc[2 * e], acc[2 * e + 1]) for e in range_constexpr(4)], fx.Int32) + + +def compile_slots4_stage_pull_sum2_add_rmsnorm_mxfp8(pullers: int = PULLERS, stagers: int = STAGERS, + normers: int = NORMERS, parts: int = PARTS, unroll: int = UNROLL, + coded: bool = True, owner: bool = False): + """``owner``: ``shared`` is [m, H], this rank's shared expert output of its own rows, added to its own rows' slot + sums only (the peer's rows carry none); else [2 m, H], added to every row.""" + assert min(pullers, stagers, normers) > 0 and pullers + stagers + normers <= 256 + assert 4 * parts <= LOCAL_TICKETS - PEER_TICKETS + nwg = pullers + stagers + normers + step = NT * 4 * unroll + row_dw = CODED // 4 if coded else H // 2 # dwords per staged row + name = (f"slots4_stage_pull_sum2_add_rmsnorm_mxfp8_h6144_ranks2_p{pullers}_s{stagers}_n{normers}_c{parts}" + f"_u{unroll}{'_x3' if coded else ''}{'_owner' if owner else ''}") + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, NW] + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + arg_slots: fx.Int64, # [rows, 4, H] bf16, every token + arg_shared: fx.Int64, # [rows, H] bf16 ([m, H]: owner) + arg_stage: fx.Int64, # the peer's sums of this rank's rows: [m, CODED] coded, else [m, H] bf16 half-major + arg_p0: fx.Int64, + arg_p1: fx.Int64, + arg_c0: fx.Int64, + arg_c1: fx.Int64, + counter: fx.Tensor, + calls: fx.Tensor, # this kernel's launches so far (its monotonic tickets count them) + i64_stride: fx.Int64, + i32_rows: fx.Int32, # every token's rows (2 m) + i32_me: fx.Int32, + arg_r: fx.Int64, # residual [m, H] bf16 + arg_w: fx.Int64, # norm weight [H] bf16 + arg_res: fx.Int64, # residual out [m, H] bf16 + arg_q: fx.Int64, # codes e4m3 [m, H] row-major + arg_s: fx.Int64, # scales u8, AITER layout + i64_s_bytes: fx.Int64, + i32_eps: fx.Int32, # fp32 bits + ): + tx = fx.thread_idx.x + bid = fx.block_idx.x + has = tx < 384 + i32_seq = fx.Int32(counter[0]) + 1 + slot_off = fx.Int64((i32_seq - 1) & 1) * i64_stride + m = i32_rows // 2 + peer = 1 - i32_me + my_c = _sel2(i32_me, arg_c0, arg_c1) + nb = H * 2 + call = fx.Int32(calls[0]) + 1 + c = (m + (parts - 1)) // parts # rows per part: part j holds rows [min(j c, m), min(j c + c, m)) + if bid < pullers: + fa = _sel2(peer, arg_c0, arg_c1) + block = fx.Int64(m) * (row_dw * 4) + src = _sel2(peer, arg_p0, arg_p1) + slot_off + if const_expr(not coded): + src = src + fx.Int64(i32_me) * block + srs = buffer_ops.create_buffer_resource_from_addr(src, num_records_bytes=block) + drs = buffer_ops.create_buffer_resource_from_addr(arg_stage, num_records_bytes=block) + j = bid + while j < parts: + if tx == 0: + fw = fa + PEER_TICKETS + fx.Int64(j * 4) + f = _flag_read(fw) + while f - call < 0: + f = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="") + r1 = (j * c + c < m).select(j * c + c, m) + d1 = r1 * row_dw + off = (j * c < m).select(j * c, m) * row_dw + tx * 4 + while off < d1: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(srs, o, vec_width=4, dtype=fx.Int32, mask=o < d1, + cache_modifier=SYS))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], drs, o, mask=o < d1, cache_modifier=SYS) + off = off + step + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + _flag_add(my_c + LOCAL_TICKETS + fx.Int64(j * 4), 1) + j = j + pullers + if (bid >= pullers) & (bid < pullers + stagers): + sid = bid - pullers + own_pool = _sel2(i32_me, arg_p0, arg_p1) + slot_off + dst = buffer_ops.create_buffer_resource_from_addr(own_pool, num_records_bytes=fx.Int64(i32_rows) * nb) + k = sid + while k < parts: + if const_expr(coded): + slane = tx % 64 + it = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + r0 = (k * c < m).select(k * c, m) + n_it = ((k * c + c < m).select(k * c + c, m) - r0) * VWAVES + while it < n_it: + row = r0 + it // VWAVES + tid = (it % VWAVES) * 64 + slane + enc_store(_slot_sum8(arg_slots, None if owner else arg_shared, peer * m + row, tid), + own_pool + fx.Int64(row) * CODED, + own_pool + fx.Int64(m) * CODED + fx.Int64(row) * ROWB, tid) + it = it + NW + else: + row = peer * m + (k * c < m).select(k * c, m) + r1 = peer * m + (k * c + c < m).select(k * c + c, m) + while row < r1: + packed = _slot_sum(arg_slots, None if owner else arg_shared, row, tx, has) + for v in range_constexpr(2): + buffer_ops.buffer_store(Vec.from_elements(packed[v * 4:v * 4 + 4], fx.Int32), dst, + row * (H // 2) + v * HALF + tx * 4, mask=has, + cache_modifier=SYS) + row = row + 1 + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + _flag_add(my_c + PEER_TICKETS + fx.Int64(k * 4), 1) + k = k + stagers + if bid >= pullers + stagers: + _set_fp16_ovfl() + nid = bid - pullers - stagers + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + smem = fx.SharedAllocator().allocate(Shared).peek() + part = smem.part.ptr + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=nb) + wv = [] + for v in range_constexpr(2): + wv += bf16x8(Vec(buffer_ops.buffer_load(wr, tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, mask=has))) + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=fx.Int64(m) * H) + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=i64_s_bytes) + row = nid + while row < m: + if tx == 0: + fw = my_c + LOCAL_TICKETS + fx.Int64((row // c) * 4) + f = _flag_read(fw) + while f - call < 0: + f = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="workgroup") + if const_expr(coded): + recs, hw = load16(arg_stage + fx.Int64(row) * CODED, tx, has) + else: + pr = buffer_ops.create_buffer_resource_from_addr(arg_stage + fx.Int64(row) * nb, + num_records_bytes=nb) + pv = [Vec(buffer_ops.buffer_load(pr, v * HALF + tx * 4, vec_width=4, dtype=fx.Int32, mask=has, + cache_modifier=SYS)) for v in range_constexpr(2)] + own = _slot_sum(arg_slots, arg_shared, i32_me * m + row, tx, has, shared_row=row if owner else None) + if const_expr(coded): + pv = dec16(recs, hw, _sel2(peer, arg_p0, arg_p1) + slot_off + fx.Int64(m) * CODED + + fx.Int64(row) * ROWB, tx, has) + rr = buffer_ops.create_buffer_resource_from_addr(arg_r + fx.Int64(row) * nb, num_records_bytes=nb) + rv = [] + for v in range_constexpr(2): + rv += bf16x8(Vec(buffer_ops.buffer_load(rr, tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, + mask=has))) + first = i32_me == 0 + b0 = [] + b1 = [] + for v in range_constexpr(2): + for i in range_constexpr(4): + o = fx.Int32(own[v * 4 + i]) + p = fx.Int32(pv[v][i]) + b0.append(first.select(o, p)) + b1.append(first.select(p, o)) + x0 = bf16x8(b0[:4]) + bf16x8(b0[4:]) + x1 = bf16x8(b1[:4]) + bf16x8(b1[4:]) + acc = [(fx.Float32(0.0) + x0[i]) + x1[i] for i in range_constexpr(16)] + red = [pk_bf16(acc[2 * i], acc[2 * i + 1]) for i in range_constexpr(8)] + xv = bf16x8(red[:4]) + bf16x8(red[4:]) + s = [xv[i] + rv[i] for i in range_constexpr(16)] + res = [pk_bf16(s[2 * i], s[2 * i + 1]) for i in range_constexpr(8)] + resr = buffer_ops.create_buffer_resource_from_addr(arg_res + fx.Int64(row) * nb, + num_records_bytes=nb) + for v in range_constexpr(2): + buffer_ops.buffer_store(Vec.from_elements(res[v * 4:v * 4 + 4], fx.Int32), resr, + tx * 8 + v * 4, mask=has) + fma = lambda a, b2: fx.Float32(fx.math.fma(a, a, b2 * b2)) # noqa: E731 + e = [fma(s[4 * i], s[4 * i + 2]) for i in range_constexpr(4)] + o = [fma(s[4 * i + 1], s[4 * i + 3]) for i in range_constexpr(4)] + ssq = ((e[0] + e[1]) + (e[2] + e[3])) + ((o[0] + o[1]) + (o[2] + o[3])) + acc2 = wave_sum_lane63(ssq) + if lane == 63: + part[wave] = acc2 + fx.gpu.barrier() + rstd = rsqrt_rn(_block_sum([part[i] for i in range_constexpr(NW)]), i32_eps.bitcast(fx.Float32)) + _tail_rowq(tx, row, lane, s, wv, rstd, has, qr, sr) + # every wave has read this row's partial sums before the next row writes them + fx.gpu.barrier() + row = row + normers + fx.gpu.barrier() + if tx == 0: + if _flag_add(my_c + END, 1) == nwg - 1: + _flag_add(my_c + END, -nwg) + counter[0] = i32_seq + calls[0] = call + + @flyc.jit + def launch_slots4_stage_pull_sum2_add_rmsnorm_mxfp8( + arg_slots: fx.Int64, arg_shared: fx.Int64, arg_stage: fx.Int64, arg_p0: fx.Int64, arg_p1: fx.Int64, + arg_c0: fx.Int64, arg_c1: fx.Int64, counter: fx.Tensor, calls: fx.Tensor, i64_stride: fx.Int64, + i32_rows: fx.Int32, + i32_me: fx.Int32, arg_r: fx.Int64, arg_w: fx.Int64, arg_res: fx.Int64, arg_q: fx.Int64, arg_s: fx.Int64, + i64_s_bytes: fx.Int64, i32_eps: fx.Int32, stream: fx.Stream, + ): + kernel(arg_slots, arg_shared, arg_stage, arg_p0, arg_p1, arg_c0, arg_c1, counter, calls, i64_stride, i32_rows, + i32_me, arg_r, arg_w, arg_res, arg_q, arg_s, i64_s_bytes, i32_eps).launch( + grid=(nwg, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_slots4_stage_pull_sum2_add_rmsnorm_mxfp8.kernel_name = name + launch_slots4_stage_pull_sum2_add_rmsnorm_mxfp8.compile_hints = PRELOAD_HINTS + return launch_slots4_stage_pull_sum2_add_rmsnorm_mxfp8 diff --git a/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows.py b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows.py new file mode 100644 index 0000000..e3bbb74 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Slot-order FP32 sum + shared output into a stage buffer (the SDMA hand-offs' reduce-scatter source). + +256 threads x 8 BF16 values per thread, a grid-stride loop over 256 workgroups. The 16-byte pieces never cross a +6144-column row. The device counter's parity selects one of two slots ``i64_stride`` apart (0: one buffer). +``owner``: ``shared`` is [rows / 2, H], rank ``i32_me``'s shared expert output of its own block, added there only (the +other rows add +0 by a masked load). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import Vector as Vec +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import bf16x8, pk_bf16 + +H = 6144 +NT = 256 +WGS = 256 # the winner of a 128/256/512/1024/2048 row-stage sweep + + +def compile_slots4_stage_rows(owner: bool = False): + name = "slots4_shared_stage_rows_h6144_ranks2" + ("_owner" if owner else "") + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel(arg_slots: fx.Int64, arg_shared: fx.Int64, arg_pool: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32): + call = fx.Int32(counter[0]) + dst = buffer_ops.create_buffer_resource_from_addr( + arg_pool + fx.Int64(call & 1) * i64_stride, + num_records_bytes=fx.Int64(i32_rows) * H * 2) + m = i32_rows // 2 + hr = buffer_ops.create_buffer_resource_from_addr( + arg_shared, num_records_bytes=fx.Int64(m if owner else i32_rows) * H * 2) + piece = fx.block_idx.x + while piece < i32_rows * 3: + row = piece // 3 + col = (piece % 3) * (NT * 4) + fx.thread_idx.x * 4 + off = piece * (NT * 4) + fx.thread_idx.x * 4 + sr = buffer_ops.create_buffer_resource_from_addr( + arg_slots + fx.Int64(row) * (4 * H * 2), num_records_bytes=4 * H * 2) + vals = [bf16x8(Vec(buffer_ops.buffer_load( + sr, col + k * (H // 2), vec_width=4, dtype=fx.Int32))) + for k in range_constexpr(4)] + if const_expr(owner): + shared = bf16x8(Vec(buffer_ops.buffer_load(hr, off - i32_me * m * (H // 2), vec_width=4, + dtype=fx.Int32, mask=row // m == i32_me))) + else: + shared = bf16x8(Vec(buffer_ops.buffer_load(hr, off, vec_width=4, dtype=fx.Int32))) + acc = [(((vals[0][j] + vals[1][j]) + vals[2][j]) + vals[3][j]) + shared[j] + for j in range_constexpr(8)] + packed = [pk_bf16(acc[2*j], acc[2*j+1]) for j in range_constexpr(4)] + buffer_ops.buffer_store(Vec.from_elements(packed, fx.Int32), dst, off) + piece = piece + WGS + + @flyc.jit + def launch_slots4_stage_rows(arg_slots: fx.Int64, arg_shared: fx.Int64, arg_pool: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32, stream: fx.Stream): + kernel(arg_slots, arg_shared, arg_pool, counter, i64_stride, i32_rows, i32_me).launch( + grid=(WGS, 1, 1), block=(NT, 1, 1), stream=stream) + launch_slots4_stage_rows.kernel_name = name + return launch_slots4_stage_rows diff --git a/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows_x3.py b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows_x3.py new file mode 100644 index 0000000..fe542e8 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/slots4_stage_rows_x3.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Slot-order FP32 sum + shared output of every token, rounded once to BF16 (``slots4_stage_rows``'s sum): this rank's +rows as BF16 into ``own``, the peer's rows in ``lossless_x3``'s code into ``coded`` (the SDMA push source), their raw +virtual waves straight into ``raw`` (the peer's receive region, system-coherent stores over the link). + +One wave per virtual wave (512 columns of a row) and step, a grid-stride loop over 256 workgroups of 4 waves. +``owner``: ``shared`` is [rows / 2, H], this rank's shared expert output of its own rows, added to them only (the +peer's rows add +0 by a masked load). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr +from flydsl.expr.typing import T + +from .lossless_x3 import CODED, ROWB, VWAVES, enc_store +from .slots4_stage_pull_sum2_add_rmsnorm_mxfp8 import _slot_sum8 + +NT = 256 +WGS = 256 + + +def compile_slots4_stage_rows_x3(owner: bool = False): + name = "slots4_shared_stage_rows_x3_h6144_ranks2" + ("_owner" if owner else "") + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel(arg_slots: fx.Int64, arg_shared: fx.Int64, arg_own: fx.Int64, arg_coded: fx.Int64, + arg_raw: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32): + m = i32_rows // 2 + lane = fx.thread_idx.x % 64 + it = fx.block_idx.x * (NT // 64) + fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.thread_idx.x // 64)) + own = buffer_ops.create_buffer_resource_from_addr(arg_own, num_records_bytes=fx.Int64(m) * ROWB) + while it < i32_rows * VWAVES: + row = it // VWAVES + tid = (it % VWAVES) * 64 + lane + r = row - (row >= m).select(m, fx.Int32(0)) + mine = (row >= m) == (i32_me == 1) + if const_expr(owner): + val = _slot_sum8(arg_slots, arg_shared, row, tid, shared_row=r, shared_mask=mine) + else: + val = _slot_sum8(arg_slots, arg_shared, row, tid) + if mine: + buffer_ops.buffer_store(val, own, r * (ROWB // 4) + tid * 4) + else: + enc_store(val, arg_coded + fx.Int64(r) * CODED, arg_raw + fx.Int64(r) * ROWB, tid) + it = it + WGS * (NT // 64) + + @flyc.jit + def launch_slots4_stage_rows_x3(arg_slots: fx.Int64, arg_shared: fx.Int64, arg_own: fx.Int64, + arg_coded: fx.Int64, arg_raw: fx.Int64, i32_rows: fx.Int32, i32_me: fx.Int32, + stream: fx.Stream): + kernel(arg_slots, arg_shared, arg_own, arg_coded, arg_raw, i32_rows, i32_me).launch( + grid=(WGS, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_slots4_stage_rows_x3.kernel_name = name + return launch_slots4_stage_rows_x3 diff --git a/amd_kernels/collectives/pull_collectives_ranks2/stage_pull.py b/amd_kernels/collectives/pull_collectives_ranks2/stage_pull.py new file mode 100644 index 0000000..c93b75f --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/stage_pull.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank all-gather and dense reduce-scatter in one launch each: the stage, the publication and the pull overlap +part by part. + +Gather (up to six fields): stagers copy each field's input into the own pool slot (counter c -> slot c % 2, the +field's 256 B-aligned offset) and into the own block of the field's output, in chunks of about ``PART_BYTES`` (at +most ``parts`` per field: every chunk costs a hand-off); pullers copy the peer's chunks into the peer's block of the +outputs as soon as they are staged. + +Reduce-scatter (bf16 [2 m, H]): stagers copy only the peer's rows of the input into the pool; pullers read the peer's +staged copy of this rank's rows and this rank's own rows of the input, add them in rank order ``(+0 + b0) + b1`` in +fp32 and store the bf16 sum. + +Hand-off: after a chunk every stager wave waits for its system-coherent stores (vmcnt 0), the workgroup meets at a +barrier and lane 0 adds 1 to the chunk's monotonic ticket in the uncached ticket buffer the peer maps, without waiting +for the result (a gather field's unused tickets advance by a whole launch, so every ticket counts every launch); a +puller workgroup owns whole chunks (dealt round-robin, so the link stays busy while one waits): its lane 0 spins +until the peer's ticket reaches (call number) x stagers (int32 differences, so the counts may wrap; a communicator +keeps one grid for its lifetime), the workgroup acquires and copies with system-coherent loads. The last workgroup of +the launch stores the call number into the device counter (graph-safe: eager calls and replays interleave). A rank +rewrites slot c % 2 only in call c + 2, after the peer finished its pull of call c (the peer's call c + 1 launch +started after it). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +from .kernel import NT, SYS, _as_f32, _flag_add, _flag_read, _pk_bf16, _sel2 + +FIELDS = 6 +PULLERS = 16 +STAGERS = 64 +PARTS = 64 # at most, per field +PART_BYTES = 1 << 19 # a gather field's parts: one per PART_BYTES, 1..PARTS +UNROLL = 4 +TICKETS = 0 # byte offset of the chunk tickets in the ticket buffer: field f, part k at TICKETS + 4 (f PARTS + k) +END = 2048 +TICKET_BYTES = 4096 + + +def _field_parts(n, parts, part_bytes): + """A field's parts: one per ``part_bytes``, 1..``parts``.""" + q = n // part_bytes + return (q < 1).select(fx.Int32(1), (q > parts).select(fx.Int32(parts), q)) + + +def compile_gather_stage_pull(pullers: int = PULLERS, stagers: int = STAGERS, parts: int = PARTS, + unroll: int = UNROLL, part_bytes: int = PART_BYTES): + assert pullers + stagers <= 256 and 4 * FIELDS * parts <= END + step = NT * 4 * unroll # dwords per workgroup per loop step + nwg = pullers + stagers + name = f"gather6_stage_pull_ranks2_p{pullers}_s{stagers}_c{parts}_u{unroll}_b{part_bytes}" + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + arg_x0: fx.Int64, arg_x1: fx.Int64, arg_x2: fx.Int64, arg_x3: fx.Int64, arg_x4: fx.Int64, arg_x5: fx.Int64, + arg_o0: fx.Int64, arg_o1: fx.Int64, arg_o2: fx.Int64, arg_o3: fx.Int64, arg_o4: fx.Int64, arg_o5: fx.Int64, + off0: fx.Int64, off1: fx.Int64, off2: fx.Int64, off3: fx.Int64, off4: fx.Int64, off5: fx.Int64, + n0: fx.Int32, n1: fx.Int32, n2: fx.Int32, n3: fx.Int32, n4: fx.Int32, n5: fx.Int32, + arg_p0: fx.Int64, arg_p1: fx.Int64, arg_t0: fx.Int64, arg_t1: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_me: fx.Int32, + ): + tx = fx.thread_idx.x + bid = fx.block_idx.x + call = fx.Int32(counter[0]) + 1 + slot = fx.Int64((call - 1) & 1) * i64_stride + peer = 1 - i32_me + my_t = _sel2(i32_me, arg_t0, arg_t1) + xs = (arg_x0, arg_x1, arg_x2, arg_x3, arg_x4, arg_x5) + outs = (arg_o0, arg_o1, arg_o2, arg_o3, arg_o4, arg_o5) + offs = (off0, off1, off2, off3, off4, off5) + ns = (n0, n1, n2, n3, n4, n5) + if bid >= pullers: + sid = bid - pullers + pool = _sel2(i32_me, arg_p0, arg_p1) + slot + for f in range_constexpr(FIELDS): + n = ns[f] + n16 = n // 16 + np_ = _field_parts(n, parts, part_bytes) + src = buffer_ops.create_buffer_resource_from_addr(xs[f], num_records_bytes=n) + dst = buffer_ops.create_buffer_resource_from_addr(pool + offs[f], num_records_bytes=n) + own = buffer_ops.create_buffer_resource_from_addr(outs[f] + fx.Int64(i32_me) * fx.Int64(n), + num_records_bytes=n) + k = fx.Int32(0) + while k < np_: + d1 = ((k + 1) * n16 // np_) * 4 + off = (k * n16 // np_) * 4 + sid * step + tx * 4 + while off < d1: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(src, o, vec_width=4, dtype=fx.Int32, mask=o < d1))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], dst, o, mask=o < d1, cache_modifier=SYS) + buffer_ops.buffer_store(vals[u], own, o, mask=o < d1) + off = off + step * stagers + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + _flag_add(my_t + TICKETS + fx.Int64((f * parts + k) * 4), 1) + k = k + 1 + # every ticket counts every launch: the field's unused parts advance by a whole launch's worth + if (sid == 0) & (tx == 0): + k = np_ + while k < parts: + _flag_add(my_t + TICKETS + fx.Int64((f * parts + k) * 4), stagers) + k = k + 1 + else: + pt = _sel2(peer, arg_t0, arg_t1) + pool = _sel2(peer, arg_p0, arg_p1) + slot + base = fx.Int32(0) # parts of the fields before f: the parts are dealt round-robin over all fields + for f in range_constexpr(FIELDS): + n = ns[f] + n16 = n // 16 + np_ = _field_parts(n, parts, part_bytes) + src = buffer_ops.create_buffer_resource_from_addr(pool + offs[f], num_records_bytes=n) + dst = buffer_ops.create_buffer_resource_from_addr(outs[f] + fx.Int64(peer) * fx.Int64(n), + num_records_bytes=n) + k = (bid + pullers * FIELDS * parts - base) % pullers + while k < np_: + d0 = (k * n16 // np_) * 4 + d1 = ((k + 1) * n16 // np_) * 4 + if tx == 0: + fw = pt + TICKETS + fx.Int64((f * parts + k) * 4) + t = _flag_read(fw) + while t - call * stagers < 0: + t = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="") + off = d0 + tx * 4 + while off < d1: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(src, o, vec_width=4, dtype=fx.Int32, mask=o < d1, + cache_modifier=SYS))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], dst, o, mask=o < d1) + off = off + step + k = k + pullers + base = base + np_ + fx.gpu.barrier() + if tx == 0: + if _flag_add(my_t + END, 1) == nwg - 1: + _flag_add(my_t + END, -nwg) + counter[0] = call + + @flyc.jit + def launch_gather_stage_pull( + arg_x0: fx.Int64, arg_x1: fx.Int64, arg_x2: fx.Int64, arg_x3: fx.Int64, arg_x4: fx.Int64, arg_x5: fx.Int64, + arg_o0: fx.Int64, arg_o1: fx.Int64, arg_o2: fx.Int64, arg_o3: fx.Int64, arg_o4: fx.Int64, arg_o5: fx.Int64, + off0: fx.Int64, off1: fx.Int64, off2: fx.Int64, off3: fx.Int64, off4: fx.Int64, off5: fx.Int64, + n0: fx.Int32, n1: fx.Int32, n2: fx.Int32, n3: fx.Int32, n4: fx.Int32, n5: fx.Int32, + arg_p0: fx.Int64, arg_p1: fx.Int64, arg_t0: fx.Int64, arg_t1: fx.Int64, + counter: fx.Tensor, i64_stride: fx.Int64, i32_me: fx.Int32, stream: fx.Stream, + ): + kernel(arg_x0, arg_x1, arg_x2, arg_x3, arg_x4, arg_x5, arg_o0, arg_o1, arg_o2, arg_o3, arg_o4, arg_o5, + off0, off1, off2, off3, off4, off5, n0, n1, n2, n3, n4, n5, arg_p0, arg_p1, arg_t0, arg_t1, + counter, i64_stride, i32_me).launch(grid=(nwg, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_gather_stage_pull.kernel_name = name + return launch_gather_stage_pull + + +def compile_rs_stage_pull(pullers: int = PULLERS, stagers: int = STAGERS, parts: int = PARTS, unroll: int = UNROLL): + """The dense bf16 reduce-scatter of ``[2 m, H]`` rows (``i64_block`` bytes = m H 2 per rank block): the pool slot + follows the communicator's ``counter``, the tickets count this kernel's own launches (``calls``).""" + assert pullers + stagers <= 256 and 4 * parts <= END + step = NT * 4 * unroll + nwg = pullers + stagers + name = f"rs_stage_pull_sum2_ranks2_p{pullers}_s{stagers}_c{parts}_u{unroll}" + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel(arg_x: fx.Int64, arg_out: fx.Int64, arg_p0: fx.Int64, arg_p1: fx.Int64, arg_t0: fx.Int64, + arg_t1: fx.Int64, counter: fx.Tensor, calls: fx.Tensor, i64_stride: fx.Int64, i64_block: fx.Int64, + i32_me: fx.Int32): + tx = fx.thread_idx.x + bid = fx.block_idx.x + seq = fx.Int32(counter[0]) + 1 + slot = fx.Int64((seq - 1) & 1) * i64_stride + call = fx.Int32(calls[0]) + 1 + peer = 1 - i32_me + my_t = _sel2(i32_me, arg_t0, arg_t1) + n16 = fx.Int32(i64_block // 16) + if bid >= pullers: + sid = bid - pullers + src = buffer_ops.create_buffer_resource_from_addr(arg_x + fx.Int64(peer) * i64_block, + num_records_bytes=i64_block) + dst = buffer_ops.create_buffer_resource_from_addr(_sel2(i32_me, arg_p0, arg_p1) + slot, + num_records_bytes=i64_block) + k = fx.Int32(0) + while k < parts: + d1 = ((k + 1) * n16 // parts) * 4 + off = (k * n16 // parts) * 4 + sid * step + tx * 4 + while off < d1: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(src, o, vec_width=4, dtype=fx.Int32, mask=o < d1))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], dst, o, mask=o < d1, cache_modifier=SYS) + off = off + step * stagers + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if tx == 0: + _flag_add(my_t + TICKETS + fx.Int64(k * 4), 1) + k = k + 1 + else: + pt = _sel2(peer, arg_t0, arg_t1) + rs = buffer_ops.create_buffer_resource_from_addr(_sel2(peer, arg_p0, arg_p1) + slot, + num_records_bytes=i64_block) + ls = buffer_ops.create_buffer_resource_from_addr(arg_x + fx.Int64(i32_me) * i64_block, + num_records_bytes=i64_block) + out = buffer_ops.create_buffer_resource_from_addr(arg_out, num_records_bytes=i64_block) + first = i32_me == 0 + k = bid + while k < parts: + d0 = (k * n16 // parts) * 4 + d1 = ((k + 1) * n16 // parts) * 4 + if tx == 0: + fw = pt + TICKETS + fx.Int64(k * 4) + t = _flag_read(fw) + while t - call * stagers < 0: + t = _flag_read(fw) + fx.gpu.barrier() + _llvm.FenceOp(_llvm.AtomicOrdering.acquire, syncscope="") + off = d0 + tx * 4 + while off < d1: + rv = [] + lv = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + rv.append(Vec(buffer_ops.buffer_load(rs, o, vec_width=4, dtype=fx.Int32, mask=o < d1, + cache_modifier=SYS))) + lv.append(Vec(buffer_ops.buffer_load(ls, o, vec_width=4, dtype=fx.Int32, mask=o < d1))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + packed = [] + for i in range_constexpr(4): + a = first.select(fx.Int32(lv[u][i]), fx.Int32(rv[u][i])) + b = first.select(fx.Int32(rv[u][i]), fx.Int32(lv[u][i])) + lo = (fx.Float32(0.0) + _as_f32(a << 16)) + _as_f32(b << 16) + hi = (fx.Float32(0.0) + _as_f32(a & fx.Int32(-65536))) + _as_f32(b & fx.Int32(-65536)) + packed.append(_pk_bf16(lo, hi)) + buffer_ops.buffer_store(Vec.from_elements(packed, fx.Int32), out, o, mask=o < d1) + off = off + step + k = k + pullers + fx.gpu.barrier() + if tx == 0: + if _flag_add(my_t + END, 1) == nwg - 1: + _flag_add(my_t + END, -nwg) + counter[0] = seq + calls[0] = call + + @flyc.jit + def launch_rs_stage_pull(arg_x: fx.Int64, arg_out: fx.Int64, arg_p0: fx.Int64, arg_p1: fx.Int64, + arg_t0: fx.Int64, arg_t1: fx.Int64, counter: fx.Tensor, calls: fx.Tensor, + i64_stride: fx.Int64, i64_block: fx.Int64, i32_me: fx.Int32, stream: fx.Stream): + kernel(arg_x, arg_out, arg_p0, arg_p1, arg_t0, arg_t1, counter, calls, i64_stride, i64_block, i32_me).launch( + grid=(nwg, 1, 1), block=(NT, 1, 1), stream=stream) + + launch_rs_stage_pull.kernel_name = name + return launch_rs_stage_pull diff --git a/amd_kernels/collectives/pull_collectives_ranks2/sum2_add_rmsnorm_mxfp8.py b/amd_kernels/collectives/pull_collectives_ranks2/sum2_add_rmsnorm_mxfp8.py new file mode 100644 index 0000000..6500d7f --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/sum2_add_rmsnorm_mxfp8.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank RS sum + residual + Gemma RMSNorm + MXFP8 (row-major codes, AITER-shuffled scales) in the final local +launch. + +The first load sums the two BF16 rank slabs (each its own pointer: the staged own block and the received peer +block), with the reduce-scatter's sum order and BF16 rounding. The norm and quant are +Triton's SHUFFLE_Q=False kernel's lane map, thread tx holds the 16 contiguous columns +16 tx .. 16 tx + 15 (tx < 384), and the sum of squares is its packed tree with FMA leaves, ssq = E + O, +E = (fma(q0, q0, q2^2) + fma(q4, q4, q6^2)) + (fma(q8, q8, q10^2) + fma(q12, q12, q14^2)) (O: odd columns), then +the same wave tree and a cross-warp fold equal to ``_block_sum`` with commuted operands (bitwise equal). One +16-byte code piece per thread; a 32-column group is 2 lanes (quad_perm [1, 0, 3, 2] pair max). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.add_rmsnorm_mxfp8 import ( + NW, + TINY, + _block_sum, + _cvt_pk_fp8, + _f32, + _scale_off, + _set_fp16_ovfl, +) +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import ( + PRELOAD_HINTS, + H, + bf16x8, + max_dpp, + maxnum, + minnum, + pk_bf16, + rsqrt_rn, + wave_sum_lane63, + xcd_grid, + xcd_row, +) + + +def _e8m0_rowq(y): + """(scale byte as fp32, 2^(127 - byte)) of a 32-column group whose 2 lanes hold 16 values each: the tiled + variant's amax / log2 / ceil / clamp sequence.""" + a = fx.math.absf(y[0]) + for v in y[1:]: + a = maxnum(a, fx.math.absf(v)) + a = maxnum(max_dpp(a, 0xB1), _f32(TINY)) # quad_perm [1, 0, 3, 2] + d = a / fx.Float32(448.0) + small = d < _f32(TINY) + lg = fx.Float32( + _rocdl_ops.log(T.f32, fx.as_ir_value(small.select(d * fx.Float32(2.0**32), d))) + ) - small.select(fx.Float32(32.0), fx.Float32(0.0)) + sb = minnum( + maxnum(fx.math.ceil(lg) + fx.Float32(127.0), fx.Float32(0.0)), + fx.Float32(254.0), + ) + rescale = fx.Float32(_rocdl_ops.exp2(T.f32, fx.as_ir_value(fx.Float32(127.0) - sb))) + return sb, rescale + + +def _codes16(y, rescale): + """16 e4m3 codes of y * rescale as 4 dwords (column order, low byte first).""" + q = [v * rescale for v in y] + return [ + _cvt_pk_fp8( + _cvt_pk_fp8(fx.Int32(0), q[4 * d], q[4 * d + 1], False), + q[4 * d + 2], + q[4 * d + 3], + True, + ) + for d in range_constexpr(4) + ] + + +def _tail_rowq(tx, row, lane, s, wv, rstd, has, qr, sr): + """Row-major codes (one 16-byte piece per thread) + AITER scale bytes.""" + y = [(s[i] * rstd) * (wv[i] + fx.Float32(1.0)) for i in range_constexpr(16)] + sb, rescale = _e8m0_rowq(y) + codes = _codes16(y, rescale) + buffer_ops.buffer_store( + Vec.from_elements(codes, fx.Int32), + qr, + row * (H // 4) + tx * 4, + mask=has, + ) + # the even lane of each pair stores group tx // 2's scale byte (both compute it) + sbi = fx.Int32(sb.to(fx.Int32)) + buffer_ops.buffer_store( + sbi.to(fx.Int8), + sr, + _scale_off(row, tx // 2), + mask=has & ((tx & 1) == 0), + offset_is_bytes=True, + ) + + +def _slab_acc(slabs, tx, row, n_dwords, base_dword, mask): + """The two ranks' BF16 rows, FP32 accumulation from +0 in rank order.""" + acc = [fx.Float32(0.0)] * n_dwords + for rank in range_constexpr(2): + xr = buffer_ops.create_buffer_resource_from_addr( + slabs[rank] + fx.Int64(row) * (H * 2), + num_records_bytes=H * 2, + ) + for v in range_constexpr(n_dwords // 8): + vals = bf16x8(Vec(buffer_ops.buffer_load( + xr, base_dword // 2 + v * 4, vec_width=4, dtype=fx.Int32, mask=mask))) + for j in range_constexpr(8): + acc[v * 8 + j] = acc[v * 8 + j] + vals[j] + return acc + + +def _pre_rowq(slabs, tx, row, rr, wr, resr): + """The Triton row-major kernel's lane map: thread tx holds the 16 + contiguous columns 16 tx .. 16 tx + 15 (tx < 384). Returns (thread ssq, s, + wv) and stores the residual-out.""" + has = tx < 384 + + def ld(rs, v): + return bf16x8( + Vec( + buffer_ops.buffer_load( + rs, tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, mask=has + ) + ) + ) + + rv = ld(rr, 0) + ld(rr, 1) + wv = ld(wr, 0) + ld(wr, 1) + acc = _slab_acc(slabs, tx, row, 16, tx * 16, has) + # the bf16 dwords the plain kernel loads from the reduced GEMM output + packed = [pk_bf16(acc[2 * i], acc[2 * i + 1]) for i in range_constexpr(8)] + xv = bf16x8(packed[:4]) + bf16x8(packed[4:]) + s = [xv[i] + rv[i] for i in range_constexpr(16)] + res = [pk_bf16(s[2 * i], s[2 * i + 1]) for i in range_constexpr(8)] + for v in range_constexpr(2): + buffer_ops.buffer_store( + Vec.from_elements(res[v * 4 : v * 4 + 4], fx.Int32), + resr, + tx * 8 + v * 4, + mask=has, + ) + # Triton's packed per-thread tree (FMA leaves; see the module docstring) + fma = lambda a, b2: fx.Float32(fx.math.fma(a, a, b2 * b2)) # noqa: E731 + e = [fma(s[4 * i], s[4 * i + 2]) for i in range_constexpr(4)] + o = [fma(s[4 * i + 1], s[4 * i + 3]) for i in range_constexpr(4)] + ssq = ((e[0] + e[1]) + (e[2] + e[3])) + ((o[0] + o[1]) + (o[2] + o[3])) + return wave_sum_lane63(ssq), s, wv, has + + +def compile_sum2_add_rmsnorm_mxfp8(): + name = "sum2_add_rmsnorm_mxfp8_rowmajor_h6144_ranks2" + _pre = _pre_rowq + _tail = _tail_rowq + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, NW] + + # the first loads' arguments lead (the first 56 bytes arrive preloaded) + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + i32_m: fx.Int32, + i32_sr: fx.Int32, + i32_so: fx.Int32, + arg_s0: fx.Int64, # rank 0's [m, H] BF16 rows + arg_s1: fx.Int64, # rank 1's + arg_r: fx.Int64, # residual [m, H] bf16, row stride i32_sr + arg_w: fx.Int64, # norm weight [H] bf16 + arg_res: fx.Int64, # residual out [m, H] bf16, row stride i32_so + arg_q: fx.Int64, # codes e4m3 [m, H] row-major + arg_s: fx.Int64, # scales u8, AITER layout + i64_q_bytes: fx.Int64, + i64_s_bytes: fx.Int64, + i32_eps: fx.Int32, # fp32 bits + ): + _set_fp16_ovfl() + tx = fx.thread_idx.x + row, live = xcd_row(fx.block_idx.x, i32_m) + if live: + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + smem = fx.SharedAllocator().allocate(Shared).peek() + part = smem.part.ptr + nb = H * 2 + rr = buffer_ops.create_buffer_resource_from_addr( + arg_r + fx.Int64(row) * fx.Int64(i32_sr * 2), num_records_bytes=nb + ) + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=nb + ) + resr = buffer_ops.create_buffer_resource_from_addr( + arg_res + fx.Int64(row) * fx.Int64(i32_so * 2), num_records_bytes=nb + ) + qr = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=i64_q_bytes + ) + sr = buffer_ops.create_buffer_resource_from_addr( + arg_s, num_records_bytes=i64_s_bytes + ) + acc2, s, wv, has = _pre((arg_s0, arg_s1), tx, row, rr, wr, resr) + if lane == 63: + part[wave] = acc2 + fx.gpu.barrier() + rstd = rsqrt_rn( + _block_sum([part[i] for i in range_constexpr(NW)]), + i32_eps.bitcast(fx.Float32), + ) + _tail(tx, row, lane, s, wv, rstd, has, qr, sr) + + @flyc.jit + def launch_sum2_add_rmsnorm_mxfp8( + arg_s0: fx.Int64, + arg_s1: fx.Int64, + arg_r: fx.Int64, + arg_w: fx.Int64, + arg_q: fx.Int64, + arg_s: fx.Int64, + arg_res: fx.Int64, + i64_q_bytes: fx.Int64, + i64_s_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sr: fx.Int32, + i32_so: fx.Int32, + i32_eps: fx.Int32, + stream: fx.Stream, + ): + kernel( + i32_m, + i32_sr, + i32_so, + arg_s0, + arg_s1, + arg_r, + arg_w, + arg_res, + arg_q, + arg_s, + i64_q_bytes, + i64_s_bytes, + i32_eps, + ).launch( + grid=(fx.Int64(xcd_grid(i32_m)), 1, 1), block=(64 * NW, 1, 1), stream=stream + ) + + launch_sum2_add_rmsnorm_mxfp8.kernel_name = name + launch_sum2_add_rmsnorm_mxfp8.compile_hints = PRELOAD_HINTS + return launch_sum2_add_rmsnorm_mxfp8 diff --git a/amd_kernels/collectives/pull_collectives_ranks2/sum2_x3_add_rmsnorm_mxfp8.py b/amd_kernels/collectives/pull_collectives_ranks2/sum2_x3_add_rmsnorm_mxfp8.py new file mode 100644 index 0000000..daa6471 --- /dev/null +++ b/amd_kernels/collectives/pull_collectives_ranks2/sum2_x3_add_rmsnorm_mxfp8.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``sum2_add_rmsnorm_mxfp8`` with the peer's slab in ``lossless_x3``'s code: this rank's BF16 rows, the peer's coded +rows and their raw virtual waves (``slots4_stage_rows_x3``), decoded in the norm lane map, then the same rank-order +sum, residual add, norm and quant: the same output bits.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.add_rmsnorm_mxfp8 import NW, _block_sum, _set_fp16_ovfl +from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.common import ( + PRELOAD_HINTS, H, bf16x8, pk_bf16, rsqrt_rn, wave_sum_lane63, xcd_grid, xcd_row, +) +from .lossless_x3 import CODED, ROWB, dec16, load16 +from .sum2_add_rmsnorm_mxfp8 import _tail_rowq + + +def _ld(rs, tx, v, has): + """Thread tx's 8 columns 16 tx + 8 v .. of a bf16 row as 4 dwords.""" + return Vec(buffer_ops.buffer_load(rs, tx * 8 + v * 4, vec_width=4, dtype=fx.Int32, mask=has)) + + +def compile_sum2_x3_add_rmsnorm_mxfp8(): + name = "sum2_x3_add_rmsnorm_mxfp8_rowmajor_h6144_ranks2" + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, NW] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + i32_m: fx.Int32, + i32_me: fx.Int32, + arg_own: fx.Int64, # this rank's [m, H] BF16 rows + arg_coded: fx.Int64, # the peer's [m, CODED] coded rows + arg_raw: fx.Int64, # their raw virtual waves, [m, H] BF16 slots + arg_r: fx.Int64, # residual [m, H] bf16 + arg_w: fx.Int64, # norm weight [H] bf16 + arg_res: fx.Int64, # residual out [m, H] bf16 + arg_q: fx.Int64, # codes e4m3 [m, H] row-major + arg_s: fx.Int64, # scales u8, AITER layout + i64_s_bytes: fx.Int64, + i32_eps: fx.Int32, # fp32 bits + ): + _set_fp16_ovfl() + tx = fx.thread_idx.x + row, live = xcd_row(fx.block_idx.x, i32_m) + if live: + has = tx < 384 + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + smem = fx.SharedAllocator().allocate(Shared).peek() + part = smem.part.ptr + nb = H * 2 + recs, hw = load16(arg_coded + fx.Int64(row) * CODED, tx, has, 0) + ownr = buffer_ops.create_buffer_resource_from_addr(arg_own + fx.Int64(row) * nb, num_records_bytes=nb) + rr = buffer_ops.create_buffer_resource_from_addr(arg_r + fx.Int64(row) * nb, num_records_bytes=nb) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=nb) + own = [_ld(ownr, tx, v, has) for v in range_constexpr(2)] + rv = bf16x8(_ld(rr, tx, 0, has)) + bf16x8(_ld(rr, tx, 1, has)) + wv = bf16x8(_ld(wr, tx, 0, has)) + bf16x8(_ld(wr, tx, 1, has)) + peer = dec16(recs, hw, arg_raw + fx.Int64(row) * ROWB, tx, has, 0) + first = i32_me == 0 + b0 = [] + b1 = [] + for v in range_constexpr(2): + for i in range_constexpr(4): + o = fx.Int32(own[v][i]) + p = fx.Int32(peer[v][i]) + b0.append(first.select(o, p)) + b1.append(first.select(p, o)) + x0 = bf16x8(b0[:4]) + bf16x8(b0[4:]) + x1 = bf16x8(b1[:4]) + bf16x8(b1[4:]) + acc = [(fx.Float32(0.0) + x0[i]) + x1[i] for i in range_constexpr(16)] + red = [pk_bf16(acc[2 * i], acc[2 * i + 1]) for i in range_constexpr(8)] + xv = bf16x8(red[:4]) + bf16x8(red[4:]) + s = [xv[i] + rv[i] for i in range_constexpr(16)] + res = [pk_bf16(s[2 * i], s[2 * i + 1]) for i in range_constexpr(8)] + resr = buffer_ops.create_buffer_resource_from_addr(arg_res + fx.Int64(row) * nb, num_records_bytes=nb) + for v in range_constexpr(2): + buffer_ops.buffer_store(Vec.from_elements(res[v * 4:v * 4 + 4], fx.Int32), resr, tx * 8 + v * 4, + mask=has) + fma = lambda a, b2: fx.Float32(fx.math.fma(a, a, b2 * b2)) # noqa: E731 + e = [fma(s[4 * i], s[4 * i + 2]) for i in range_constexpr(4)] + o = [fma(s[4 * i + 1], s[4 * i + 3]) for i in range_constexpr(4)] + ssq = ((e[0] + e[1]) + (e[2] + e[3])) + ((o[0] + o[1]) + (o[2] + o[3])) + acc2 = wave_sum_lane63(ssq) + if lane == 63: + part[wave] = acc2 + fx.gpu.barrier() + rstd = rsqrt_rn(_block_sum([part[i] for i in range_constexpr(NW)]), i32_eps.bitcast(fx.Float32)) + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=fx.Int64(i32_m) * H) + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=i64_s_bytes) + _tail_rowq(tx, row, lane, s, wv, rstd, has, qr, sr) + + @flyc.jit + def launch_sum2_x3_add_rmsnorm_mxfp8( + arg_own: fx.Int64, arg_coded: fx.Int64, arg_raw: fx.Int64, arg_r: fx.Int64, arg_w: fx.Int64, + arg_q: fx.Int64, arg_s: fx.Int64, arg_res: fx.Int64, i64_s_bytes: fx.Int64, i32_m: fx.Int32, + i32_me: fx.Int32, i32_eps: fx.Int32, stream: fx.Stream, + ): + kernel(i32_m, i32_me, arg_own, arg_coded, arg_raw, arg_r, arg_w, arg_res, arg_q, arg_s, i64_s_bytes, + i32_eps).launch(grid=(fx.Int64(xcd_grid(i32_m)), 1, 1), block=(64 * NW, 1, 1), stream=stream) + + launch_sum2_x3_add_rmsnorm_mxfp8.kernel_name = name + launch_sum2_x3_add_rmsnorm_mxfp8.compile_hints = PRELOAD_HINTS + return launch_sum2_x3_add_rmsnorm_mxfp8 diff --git a/amd_kernels/collectives/sdma_push_ranks2/README.md b/amd_kernels/collectives/sdma_push_ranks2/README.md new file mode 100644 index 0000000..c5f0f9e --- /dev/null +++ b/amd_kernels/collectives/sdma_push_ranks2/README.md @@ -0,0 +1,44 @@ +# Two-rank pushes on the SDMA engines, ordered by the GPU stream + +## What it does + +`SdmaPush` moves bytes between the two ranks of a GPU pair with a copy (SDMA) engine instead of CU kernels: no +workgroup reads the peer or spins on a flag, so a hand-off can run next to another stream's or DBO microbatch's compute +without taking its CUs. The GPU orders it: the copies start once the caller's stream reaches the push, and the +receiver's stream waits with a command-processor wait. The host blocks only to recycle a signal-ring slot `RING` (32) +pushes later. Bytes are copied unchanged (no arithmetic, no `reference.py`). Caller: `SdmaSeqParallelHandoffs` +(`collectives/seq_parallel_handoffs_h6144_q4096_ranks2`), the DBO microbatches of the two-rank SP prefill; push / wait +take the signatures of `sdma_push_ranks4`. + +## Entry points + +- `amd_kernels.collectives.sdma_push_ranks2.SdmaPush`: a pair's receive regions (one IPC pool) and flag words, built + collectively on a gloo group. +- `SdmaPush.push`: copies this rank's items into the peer's region once the stream reaches the push. +- `SdmaPush.wait`: the stream waits until the peer's push of a sequence number has landed. +- `SdmaPush.drain`: the host waits for every push's copies and drops the held sources. + +Eager only; both ranks push and wait on a region in the same order, the source and region lifetimes are the caller's +(`transport.py` docstring, which also holds the engine policy and the per-push steps). + +## Performance + +One xGMI link per pair: one engine reaches 61.2 GB/s per direction with both directions busy at 256 MB; 2 to 8 +engines striped to the one peer gain <= 1% one way and lose up to 1.3% both ways (`bench_engines.py`), so `ENGINES` is +one engine. Push + wait of an SP MoE layer's hand-offs (R = T / 2 rows x 30176 B, both ranks at once, eager) vs RCCL +all-gather of the same bytes, us; a bf16 GEMM loop on another stream slows by at most 1% while the pushes run: + +| T (tokens) | MB per rank | `SdmaPush` | RCCL all-gather | +|---:|---:|---:|---:| +| 4096 | 61.8 | 1042.5 | 1165.6 | +| 8192 | 123.6 | 2044.3 | 2279.8 | +| 16384 | 247.2 | 4057.9 | 4510.5 | +| 32768 | 494.4 | 8082.3 | 8942.3 | + +## Tests + +```bash +python3 -m pytest tests/collectives/sdma_push_ranks2 -q # 2 GPUs +python3 benchmarks/collectives/sdma_push_ranks2/bench_engines.py 8 64 256 +python3 benchmarks/collectives/sdma_push_ranks2/benchmark_push.py +``` diff --git a/amd_kernels/collectives/sdma_push_ranks2/__init__.py b/amd_kernels/collectives/sdma_push_ranks2/__init__.py new file mode 100644 index 0000000..4cc7264 --- /dev/null +++ b/amd_kernels/collectives/sdma_push_ranks2/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank pushes on the SDMA engines, ordered by the GPU stream (``transport.SdmaPush``).""" + +from .transport import ENGINES, RING, WORLD, SdmaPush + +__all__ = ["ENGINES", "RING", "WORLD", "SdmaPush"] diff --git a/amd_kernels/collectives/sdma_push_ranks2/hsa_runtime.py b/amd_kernels/collectives/sdma_push_ranks2/hsa_runtime.py new file mode 100644 index 0000000..e1ddfd4 --- /dev/null +++ b/amd_kernels/collectives/sdma_push_ranks2/hsa_runtime.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""ctypes binding to the HSA runtime HIP itself loaded (the SDMA copies), and to HIP's stream memory operations. + +The binding must reach the runtime that owns this process's device context: an image can ship several +``libhsa-runtime64.so`` copies, as it ships several HIP runtimes (``amd_kernels.common.hip_runtime``), so the +library is taken from the process's own mappings after torch has initialized HIP. +""" + +import ctypes +import functools +import os + +import torch + +from amd_kernels.common.hip_runtime import load_hip_runtime + +HSA_AGENT_INFO_DEVICE = 17 +HSA_DEVICE_TYPE_GPU = 1 +HSA_AMD_AGENT_INFO_BDFID = 0xA006 +HSA_AMD_AGENT_INFO_DOMAIN = 0xA00F +HSA_AMD_SIGNAL_AMD_GPU_ONLY = 1 # the value lives in device memory, where a HIP stream can write it +HSA_SIGNAL_CONDITION_LT = 2 +HSA_WAIT_STATE_ACTIVE = 1 +HIP_STREAM_WAIT_VALUE_GTE = 0 + + +class Agent(ctypes.Structure): + _fields_ = [("handle", ctypes.c_uint64)] + + +class Signal(ctypes.Structure): + _fields_ = [("handle", ctypes.c_uint64)] + + +def _mapped(name: str) -> str | None: + """The path of a shared library already mapped into this process.""" + with open("/proc/self/maps") as f: + for line in f: + path = line.split()[-1] + if os.path.basename(path).startswith(name): + return path + return None + + +@functools.cache +def hsa() -> ctypes.CDLL: + torch.cuda.init() + lib = ctypes.CDLL(_mapped("libhsa-runtime64.so") or "libhsa-runtime64.so.1") + rc = lib.hsa_init() # reference counted: HIP has already initialized it + assert rc == 0, f"hsa_init failed: {rc:#x}" + visit = ctypes.CFUNCTYPE(ctypes.c_int, Agent, ctypes.c_void_p) + lib.hsa_iterate_agents.argtypes = [visit, ctypes.c_void_p] + lib.hsa_agent_get_info.argtypes = [Agent, ctypes.c_int, ctypes.c_void_p] + lib.hsa_amd_signal_create.argtypes = [ctypes.c_int64, ctypes.c_uint32, ctypes.c_void_p, ctypes.c_uint64, + ctypes.POINTER(Signal)] # fmt: skip + lib.hsa_signal_store_relaxed.argtypes = [Signal, ctypes.c_int64] + lib.hsa_signal_load_relaxed.argtypes = [Signal] + lib.hsa_signal_load_relaxed.restype = ctypes.c_int64 + lib.hsa_signal_wait_scacquire.argtypes = [Signal, ctypes.c_int, ctypes.c_int64, ctypes.c_uint64, ctypes.c_int] + lib.hsa_signal_wait_scacquire.restype = ctypes.c_int64 + lib.hsa_amd_signal_value_pointer.argtypes = [Signal, ctypes.POINTER(ctypes.c_void_p)] + lib.hsa_amd_memory_copy_engine_status.argtypes = [Agent, Agent, ctypes.POINTER(ctypes.c_uint32)] + lib.hsa_amd_memory_async_copy_on_engine.argtypes = [ + ctypes.c_void_p, Agent, ctypes.c_void_p, Agent, ctypes.c_size_t, ctypes.c_uint32, ctypes.POINTER(Signal), + Signal, ctypes.c_uint32, ctypes.c_bool, + ] # fmt: skip + lib._visit_type = visit + return lib + + +@functools.cache +def hip() -> ctypes.CDLL: + lib = load_hip_runtime() + lib.hipStreamWriteValue64.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint] + lib.hipStreamWaitValue64.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint64, ctypes.c_uint, + ctypes.c_uint64] # fmt: skip + return lib + + +@functools.cache +def gpu_agents() -> dict[int, Agent]: + """Every GPU agent of the process by its PCI domain / bus / device (the function bits cleared): a node can have + several PCI domains with the same bus / device numbers.""" + lib, out = hsa(), {} + + def visit(agent, _): + kind = ctypes.c_uint32() + lib.hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, ctypes.byref(kind)) + if kind.value == HSA_DEVICE_TYPE_GPU: + bdf = ctypes.c_uint32() + lib.hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_BDFID, ctypes.byref(bdf)) + domain = ctypes.c_uint32() + lib.hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_DOMAIN, ctypes.byref(domain)) + key = (domain.value << 16) | (bdf.value & ~0x7) + assert key not in out, f"two GPU agents at PCI key {key:#x}" + out[key] = Agent(agent.handle) + return 0 + + callback = lib._visit_type(visit) + rc = lib.hsa_iterate_agents(callback, None) + assert rc == 0, f"hsa_iterate_agents failed: {rc:#x}" + return out + + +def pci_key(device: torch.device) -> int: + """The key of ``gpu_agents`` for a torch device.""" + p = torch.cuda.get_device_properties(device) + return (p.pci_domain_id << 16) | (p.pci_bus_id << 8) | (p.pci_device_id << 3) + + +def signal(value: int = 1) -> Signal: + s = Signal() + rc = hsa().hsa_amd_signal_create(value, 0, None, HSA_AMD_SIGNAL_AMD_GPU_ONLY, ctypes.byref(s)) + assert rc == 0, f"hsa_amd_signal_create failed: {rc:#x}" + return s + + +def value_pointer(s: Signal) -> int: + p = ctypes.c_void_p() + rc = hsa().hsa_amd_signal_value_pointer(s, ctypes.byref(p)) + assert rc == 0, f"hsa_amd_signal_value_pointer failed: {rc:#x}" + return p.value diff --git a/amd_kernels/collectives/sdma_push_ranks2/ipc.py b/amd_kernels/collectives/sdma_push_ranks2/ipc.py new file mode 100644 index 0000000..735b590 --- /dev/null +++ b/amd_kernels/collectives/sdma_push_ranks2/ipc.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The IPC pool of the receive regions: a coarse-grained hipMalloc the peer maps, and the handle exchange.""" + +import ctypes +import functools + +import torch +import torch.distributed as dist + +from amd_kernels.common.hip_runtime import load_hip_runtime + + +class _IpcHandle(ctypes.Structure): + _fields_ = [("reserved", ctypes.c_char * 64)] + + +@functools.cache +def _hip(): + lib = load_hip_runtime() + lib.hipMalloc.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.c_size_t] + lib.hipIpcGetMemHandle.argtypes = [ctypes.POINTER(_IpcHandle), ctypes.c_void_p] + lib.hipIpcOpenMemHandle.argtypes = [ctypes.POINTER(ctypes.c_void_p), _IpcHandle, ctypes.c_uint] + return lib + + +class _DevicePointer: + """A device allocation as torch sees it (``torch.as_tensor`` wraps it).""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "data": (ptr, False), + "shape": (nbytes,), + "typestr": "|u1", + "strides": None, + "version": 3, + } + + +def alloc_pool(nbytes: int, device: torch.device) -> tuple[int, bytes, torch.Tensor]: + """A hipMalloc pool (coarse-grained, never freed) -> (pointer, IPC handle bytes, the pool as a uint8 tensor).""" + hip = _hip() + ptr = ctypes.c_void_p() + with torch.cuda.device(device): + rc = hip.hipMalloc(ctypes.byref(ptr), nbytes) + assert rc == 0, f"hipMalloc({nbytes}) failed: {rc}" + handle = _IpcHandle() + rc = hip.hipIpcGetMemHandle(ctypes.byref(handle), ptr) + assert rc == 0, f"hipIpcGetMemHandle failed: {rc}" + pool = torch.as_tensor(_DevicePointer(ptr.value, nbytes), device=device) + # the raw 64 bytes (a c_char field reads back only up to its first NUL) + return ptr.value, ctypes.string_at(ctypes.addressof(handle), ctypes.sizeof(handle)), pool + + +def open_pool(handle: bytes, device: torch.device) -> int: + hip = _hip() + ptr = ctypes.c_void_p() + h = _IpcHandle() + assert len(handle) == ctypes.sizeof(h) + ctypes.memmove(ctypes.addressof(h), handle, len(handle)) + with torch.cuda.device(device): + rc = hip.hipIpcOpenMemHandle(ctypes.byref(ptr), h, 1) # lazy peer access + assert rc == 0, f"hipIpcOpenMemHandle failed: {rc}" + return ptr.value + + +def exchange(obj, rank: int, world: int, group) -> list: + """Every rank's ``obj`` (broadcasts: ``all_gather_object`` is unavailable on gloo under inference mode).""" + out = [[None] for _ in range(world)] + out[rank][0] = obj + ranks = sorted(dist.get_process_group_ranks(group=group)) + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] diff --git a/amd_kernels/collectives/sdma_push_ranks2/transport.py b/amd_kernels/collectives/sdma_push_ranks2/transport.py new file mode 100644 index 0000000..ecf3925 --- /dev/null +++ b/amd_kernels/collectives/sdma_push_ranks2/transport.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Pushes between the two ranks of a GPU pair on the SDMA engines, ordered by the GPU (no host waits, no CUs). + +Every rank owns receive regions (one IPC pool, coarse-grained) that its peer maps. ``push(region, items)`` copies this +rank's bytes into the peer's copy of a region: + +1. (host, enqueued at once) ``hsa_amd_memory_async_copy_on_engine`` copies of the items on the push engines (an item + of at least ``STRIPE_MIN`` bytes is cut into one piece per engine), every copy depending on the push's trigger + signal (HSA, GPU_ONLY attribute: its value lives where a stream can write it); +2. an 8-byte copy of the push's sequence number into the peer's flag word of the region, after every data copy + completed (one done signal counts them down); +3. (stream) the sequence number into the push's device word, then 0 into the trigger: the copies start once the + stream's work ahead of the push is done. + +``wait(region, seq)`` puts a ``hipStreamWaitValue64`` (flag >= seq) on the stream: the work after it sees the peer's +bytes. The command processor waits; no workgroup spins. Both ranks issue the same pushes and waits on a region in the +same order. Flag words are uncached: the peer's SDMA write lands in HBM, and the waiting command processor must not +read a stale L2 line. + +Lifetimes, the caller's part: an SDMA engine reads a source after the push's stream position, outside stream order, +so a source tensor must stay alive until those reads are done. ``push(..., keep=...)`` holds the tensors; +``wait(..., release=True)`` drops the ones of every push but the latest: correct when, as in a chain of all-to-all +phases, the peer triggers its next push only after its wait for this rank's previous one (that wait saw this rank's +flag, which follows the data copies). A region's bytes are overwritten by the next push into it: the consumer of a +region must be ahead (in stream order on its rank) of that rank's next push that the peer waits for before pushing +into the region again. + +Engines: ``ENGINES`` (``bench_engines.py``: one link per pair, see the README), restricted to those +``hsa_amd_memory_copy_engine_status`` reports free. +""" + +import ctypes +import logging + +import torch + +from amd_kernels.common import hip_runtime as ops + +from . import hsa_runtime as rt +from .ipc import alloc_pool, exchange, open_pool + +logger = logging.getLogger(__name__) + +WORLD = 2 +RING = 32 # pushes in flight per transport; a slot is reused only after its copies are done +FLAG_STRIDE = 64 # bytes per (region, writer) flag word +REGION_ALIGN = 4096 +ENGINES = (2,) +STRIPE_MIN = 1 << 20 # items from 1 MB are cut into one piece per engine +PIECE_ALIGN = 4096 + + +def _round_up(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +class _Slot: + __slots__ = ("trigger", "done", "fdone", "used") + + def __init__(self): + self.trigger = rt.signal() + self.done = rt.signal() + self.fdone = rt.signal() + self.used = False # whether this slot's last push armed its signals + + +class SdmaPush: + """Receive regions of ``region_bytes[k]`` bytes on both ranks, the flags, the SDMA engines and the signal ring. + Collective: the two ranks of ``cpu_group`` (gloo) construct it at the same point. ``engines`` overrides + ``ENGINES``.""" + + def __init__(self, rank: int, cpu_group, device: torch.device, region_bytes, engines=None): + device = torch.device(device) + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + assert rank in (0, 1) + self.rank, self.peer, self.device = rank, 1 - rank, device + self.offsets, off = [], 0 + for n in region_bytes: + self.offsets.append(off) + off += _round_up(n, REGION_ALIGN) + self.region_bytes = list(region_bytes) + ptr, handle, self.pool = alloc_pool(max(off, REGION_ALIGN), device) + nflags = _round_up(len(self.offsets) * WORLD * FLAG_STRIDE, 4096) + flag_ptr, flag_handle = ops.allocate_shared_buffer_and_handle(nflags) + # the sequence words the flag copies read (uncached: a stream write must reach HBM before an engine reads it) + self.seq_words, _ = ops.allocate_shared_buffer_and_handle(RING * 8) + peers = exchange((handle, flag_handle, rt.pci_key(device)), rank, WORLD, cpu_group) + self.peer_pool = open_pool(peers[self.peer][0], device) + self.flag_ptrs = [flag_ptr if r == rank else ops.open_mem_handle(h) for r, (_, h, _) in enumerate(peers)] + agents = rt.gpu_agents() + self.me_agent, self.peer_agent = agents[peers[rank][2]], agents[peers[self.peer][2]] + self.engines = self._pick_engines(ENGINES if engines is None else engines) + self.ring = [_Slot() for _ in range(RING)] + self.pushes = 0 + self.seq = [0] * len(self.offsets) + self._held: list[tuple] = [] + self._hip, self._hsa = rt.hip(), rt.hsa() + logger.info("sdma_push_ranks2: rank %d, %d regions, %.0f MB, engines %s", rank, len(self.offsets), + off / 2**20, [e.bit_length() - 1 for e in self.engines]) # fmt: skip + + def _pick_engines(self, wanted) -> list[int]: + mask = ctypes.c_uint32() + rc = rt.hsa().hsa_amd_memory_copy_engine_status(self.peer_agent, self.me_agent, ctypes.byref(mask)) + free = [e for e in range(32) if mask.value >> e & 1] if rc == 0 else [] + picked = [e for e in wanted if not free or e in free] or free[:1] + assert picked, "no SDMA engine to the peer" + return [1 << e for e in picked] + + def region(self, k: int) -> torch.Tensor: + """This rank's receive region ``k`` (uint8).""" + return self.pool[self.offsets[k] : self.offsets[k] + self.region_bytes[k]] + + def _flag_off(self, k: int, writer: int) -> int: + return (k * WORLD + writer) * FLAG_STRIDE + + def _pieces(self, items): + """(src, offset, bytes, engine) copies: an item from ``STRIPE_MIN`` bytes cut into one piece per engine.""" + out, n_eng, i = [], len(self.engines), 0 + for src, off, n in items: + if n_eng == 1 or n < STRIPE_MIN: + out.append((src, off, n, self.engines[i % n_eng])) + i += 1 + continue + part = _round_up(-(-n // n_eng), PIECE_ALIGN) + for j in range(n_eng): + lo = j * part + if lo < n: + out.append((src + lo, off + lo, min(part, n - lo), self.engines[j])) + return out + + def push(self, k: int, items, stream=None, keep=()) -> int: + """Copy ``items`` = [(peer, src address, byte offset in the peer's region k, bytes)] once the stream's preceding + work is done; returns region k's sequence number for ``wait``.""" + stream = torch.cuda.current_stream(self.device) if stream is None else stream + self.seq[k] += 1 + seq = self.seq[k] + slot = self.ring[self.pushes % RING] + word = self.seq_words + (self.pushes % RING) * 8 + self.pushes += 1 + lib = self._hsa + if slot.used: # the slot's previous copies must be done before its signals are reset + for s in (slot.done, slot.fdone): + while lib.hsa_signal_wait_scacquire(s, rt.HSA_SIGNAL_CONDITION_LT, 1, 1 << 40, + rt.HSA_WAIT_STATE_ACTIVE) > 0: # fmt: skip + pass + for p, _, off, n in items: + assert p == self.peer and 0 <= off and off + n <= self.region_bytes[k] and n > 0, (p, off, n) + pieces = self._pieces([item[1:] for item in items]) + lib.hsa_signal_store_relaxed(slot.trigger, 1) + trigger = (rt.Signal * 1)(slot.trigger) + lib.hsa_signal_store_relaxed(slot.done, len(pieces)) + lib.hsa_signal_store_relaxed(slot.fdone, 1) + dst = self.peer_pool + self.offsets[k] + for src, off, n, engine in pieces: + rc = lib.hsa_amd_memory_async_copy_on_engine(dst + off, self.peer_agent, src, self.me_agent, n, 1, trigger, + slot.done, engine, True) # fmt: skip + assert rc == 0, f"SDMA copy to the peer failed: {rc:#x}" + dep = (rt.Signal * 1)(slot.done) + rc = lib.hsa_amd_memory_async_copy_on_engine(self.flag_ptrs[self.peer] + self._flag_off(k, self.rank), + self.peer_agent, word, self.me_agent, 8, 1, dep, slot.fdone, + self.engines[0], True) # fmt: skip + assert rc == 0, f"SDMA flag copy to the peer failed: {rc:#x}" + slot.used = True + s = ctypes.c_void_p(stream.cuda_stream) + assert self._hip.hipStreamWriteValue64(s, word, seq, 0) == 0 + assert self._hip.hipStreamWriteValue64(s, rt.value_pointer(slot.trigger), 0, 0) == 0 + self._held.append(tuple(keep)) + return seq + + def wait(self, k: int, seq: int, peers=None, stream=None, release: bool = True) -> None: + """The stream waits for the peer's push ``seq`` into this rank's region k (``peers``: the ranks4 signature; + only the peer counts).""" + stream = torch.cuda.current_stream(self.device) if stream is None else stream + s = ctypes.c_void_p(stream.cuda_stream) + if peers is None or self.peer in peers: + rc = self._hip.hipStreamWaitValue64(s, self.flag_ptrs[self.rank] + self._flag_off(k, self.peer), seq, + rt.HIP_STREAM_WAIT_VALUE_GTE, 0xFFFFFFFFFFFFFFFF) # fmt: skip + assert rc == 0, f"hipStreamWaitValue64 failed: {rc}" + if release: + del self._held[:-1] + + def drain(self) -> None: + """Host: wait until every push's copies are done, and drop the held sources.""" + lib = self._hsa + for slot in self.ring: + if slot.used: + for s in (slot.done, slot.fdone): + while lib.hsa_signal_wait_scacquire(s, rt.HSA_SIGNAL_CONDITION_LT, 1, 1 << 40, + rt.HSA_WAIT_STATE_ACTIVE) > 0: # fmt: skip + pass + self._held.clear() diff --git a/amd_kernels/collectives/sdma_push_ranks4/README.md b/amd_kernels/collectives/sdma_push_ranks4/README.md index e93bb4c..33f107d 100644 --- a/amd_kernels/collectives/sdma_push_ranks4/README.md +++ b/amd_kernels/collectives/sdma_push_ranks4/README.md @@ -46,3 +46,4 @@ torchrun --nproc-per-node 4 benchmarks/collectives/sdma_push_ranks4/bench_engine ``` `test_transport.py`: gather / reduce-scatter-shaped pushes, 80 rounds, a slowed producer, sources dropped early. +`test_agents.py`: every rank's HSA agent is its GPU (keys unique across PCI domains, checked by agent UUID). diff --git a/amd_kernels/collectives/sdma_push_ranks4/hsa_runtime.py b/amd_kernels/collectives/sdma_push_ranks4/hsa_runtime.py index 182b9c3..f1d388f 100644 --- a/amd_kernels/collectives/sdma_push_ranks4/hsa_runtime.py +++ b/amd_kernels/collectives/sdma_push_ranks4/hsa_runtime.py @@ -18,6 +18,7 @@ HSA_AGENT_INFO_DEVICE = 17 HSA_DEVICE_TYPE_GPU = 1 HSA_AMD_AGENT_INFO_BDFID = 0xA006 +HSA_AMD_AGENT_INFO_DOMAIN = 0xA00F HSA_AMD_SIGNAL_AMD_GPU_ONLY = 1 # the value lives in device memory, where a HIP stream can write it HSA_SIGNAL_CONDITION_LT = 2 HSA_WAIT_STATE_ACTIVE = 1 @@ -79,8 +80,9 @@ def hip() -> ctypes.CDLL: @functools.cache def gpu_agents() -> dict[int, Agent]: - """Every GPU agent of the process by its PCI bus / device id (the function bits cleared).""" - lib, out = hsa(), {} + """Every GPU agent of the process by its PCI domain / bus / device (the function bits cleared): a node can have + several PCI domains with the same bus / device numbers.""" + lib, out, keys = hsa(), {}, [] def visit(agent, _): kind = ctypes.c_uint32() @@ -88,19 +90,24 @@ def visit(agent, _): if kind.value == HSA_DEVICE_TYPE_GPU: bdf = ctypes.c_uint32() lib.hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_BDFID, ctypes.byref(bdf)) - out[bdf.value & ~0x7] = Agent(agent.handle) + domain = ctypes.c_uint32() + lib.hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_DOMAIN, ctypes.byref(domain)) + keys.append((domain.value << 16) | (bdf.value & ~0x7)) + out[keys[-1]] = Agent(agent.handle) return 0 callback = lib._visit_type(visit) rc = lib.hsa_iterate_agents(callback, None) assert rc == 0, f"hsa_iterate_agents failed: {rc:#x}" + # checked here, not in visit: ctypes prints and drops an exception raised in a callback + assert len(out) == len(keys), f"two GPU agents at one PCI key: {sorted(map(hex, keys))}" return out def pci_key(device: torch.device) -> int: """The key of ``gpu_agents`` for a torch device.""" p = torch.cuda.get_device_properties(device) - return (p.pci_bus_id << 8) | (p.pci_device_id << 3) + return (p.pci_domain_id << 16) | (p.pci_bus_id << 8) | (p.pci_device_id << 3) def signal(value: int = 1) -> Signal: diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/README.md b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/README.md new file mode 100644 index 0000000..c32d0a8 --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/README.md @@ -0,0 +1,60 @@ +# Sequence-parallel layer hand-offs, hidden 6144, 4096 attention columns per rank, two ranks + +## What it does + +`SeqParallelHandoffs` runs the chains of a sequence-parallel (SP) decoder layer of MiniMax-M3.1-Flash TP2 prefill +between the head-sharded parts (qkv, attention, the FFN of every token) and the token-sharded residual stream, with the +four `pull_collectives_ranks2` communicators and the warm-up; the caller keeps the layer loop, attention, the first +and final norms and the DBO gating. o_proj_full MXFP8 `[6144, 8192]` (`gemm/mxfp8_common`), the MoE of +`moe/mxfp4_h6144_i1536_e128_top4`. Rank r holds `shard_rows(t)` tokens (a multiple of 32), 2048 <= t <= 32768. +Contract (`reference.py`): `pack_reference` bit for bit (the Triton `mxfp8_quant_pack` but for blocks with a maximum +below 2^-119); every phase output bit for bit its kernels' outputs on the rows the collectives deliver (gathers and +fp32 reduce-scatter sums in rank order); no atomics, no quantized reduction; the MoE tail's reduce-scatter crosses the +link in `pull_collectives_ranks2`'s exact 12-bit code (`coded_tail`). With the full shared expert copy each rank runs +it on its own rows (a side stream, joined by an event before the tail) and adds it to its own rows' slot sum; only +the routed payload is gathered (3296 B/token; a shard copy keeps the 7328 B compact one). Eager only (host-side +sequence state), on the current stream, the same calls on both ranks. + +## Entry points + +- `SeqParallelHandoffs`: a collective; builds the transport for a batch-token limit (clamped to 2048..32768) and + compiles it; `supports` says whether a token count fits. +- `SeqParallelHandoffs.warm_up`: local, before serving: every compute variant at 2048-32768 tokens. +- `SeqParallelHandoffs.input_gather`: the first layer's normed bf16 shard -> MXFP8 codes + AITER-shuffled scales of + every token. +- `SeqParallelHandoffs.attention_moe`: this rank's heads of every token -> the routed-only `MoeInput` of every token, + the new residual shard and the `SharedPending` shared expert of the own rows; `attention_dense`: the same o_proj, + then the dense MLP's codes + scales of every token. +- `SeqParallelHandoffs.ffn_norm_gather`: the FFN output of every token (MoE slots + `SharedPending` or a shared + partial, or a dense partial) -> the next layer's codes, scales and residual shard. +- `SharedPending`: the shared expert's output and the event `wait` joins; `OWNER_SHARED`: the feature flag. +- `SeqParallelHandoffs.ffn_reduce_scatter`: after the last layer, this rank's summed rows; `gather`: every token's. +- `shard` / `shard_rows`: a rank's zero-padded rows of a full tensor / their count (a multiple of 32). +- `mxfp8_pack_q4096` (also `pack`): bf16 `[M, 4096]` -> uint8 `[M, 4224]` (4096 codes, 128 scale bytes per row). +- `SdmaSeqParallelHandoffs`: the same phases and bits on SDMA pushes (`sdma_push_ranks2`); each phase takes a hook + that runs after a push and before its wait (DBO: the yield); one instance per microbatch; `drain` waits on the host + for every push. + +The phase order and kernels of each chain are in `handoffs.py` and `handoffs_sdma.py`. + +## Performance + +A MoE layer from the attention output to the next layer's QKV output (attention core not run), eager, rank max, median +of 7 rounds, us (`benchmark_sp_layer.py`); "shard copy": the shared expert's TP shard on every token (the compact +gather; under DBO on the caller's stream). TP: o_proj on every token + all-reduce (AITER custom up to 64 MiB, else +RCCL) + norm, MoE, all-reduce + norm/quant. DBO: two microbatches on the SDMA form, vLLM's turn-taking. Ideal: link +bytes (23456 B per shard row) at 61.2 GB/s plus (DBO: max with) the compute ideal. DBO and pull are even at 8192. + +| T tokens | 2048 | 4096 | 8192 | 16384 | 32768 | +|---|---:|---:|---:|---:|---:| +| SP pull (shard copy) | 1141 (1230) | 1750 (1942) | 3009 (3392) | 5510 (6335) | 10783 (12549) | +| SP SDMA DBO (shard copy) | - | 2178 (2355) | 2983 (3199) | 4778 (5185) | 8695 (9377) | +| TP2 (best all-reduce) | 1584 | 2814 | 5262 | 10048 | 19885 | +| best SP / ideal | 67% | 70% | 53% | 66% | 72% | + +## Tests + +```bash +python3 -m pytest tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2 -q # 2 GPUs; test_warm_up.py alone +python3 benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py [out.json] +``` diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py new file mode 100644 index 0000000..3355d66 --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The hand-offs of a sequence-parallel decoder layer on two ranks, hidden 6144, 4096 attention-output columns per +rank (``handoffs.py``): the attention output's MXFP8 pack + unpacking all-to-all + o_proj_full + post-attention norm +(+ MoE input preparation, the routed payload's gather and the full shared expert of the own rows on a side stream), +the FFN output's reduce-scatter + next input norm / quant + gather, owned with their communicators, pool slots and +warm-up; ``handoffs_sdma.py`` the same phases on SDMA pushes. The torch contract is ``reference.py``.""" + +from .handoffs import ( + H, + MAX_TOKENS, + MIN_TOKENS, + OWNER_SHARED, + Q, + ROW_ALIGN, + WORLD, + SeqParallelHandoffs, + SharedPending, + pack, + shard, + shard_rows, +) +from .handoffs_sdma import SdmaSeqParallelHandoffs +from .mxfp8_pack_q4096 import mxfp8_pack_q4096 + +__all__ = ["H", "MAX_TOKENS", "MIN_TOKENS", "OWNER_SHARED", "Q", "ROW_ALIGN", "WORLD", "SeqParallelHandoffs", + "SdmaSeqParallelHandoffs", "SharedPending", "pack", "shard", "shard_rows", "mxfp8_pack_q4096"] diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs.py b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs.py new file mode 100644 index 0000000..b55f2e8 --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs.py @@ -0,0 +1,407 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The hand-offs of a sequence-parallel decoder layer on two ranks (hidden 6144, 4096 attention-output columns per +rank): the multi-kernel chains between the head-sharded parts of a layer (qkv, attention, the FFN of every token) and +the token-sharded residual stream (rank r holds ``rows`` consecutive tokens, ``rows`` a multiple of 32). + +``SeqParallelHandoffs`` owns the four communicators of ``pull_collectives_ranks2`` (and the pools of the unpacking +all-to-all), the per-phase pool slots, the MXFP8 pack of the attention output +(``mxfp8_pack_q4096``) and the warm-up of every kernel the phases launch (``amd_kernels.warm_up.entry``). The caller +passes tensors: this rank's rows, the layer's weights, epsilons. Per layer: + +1. ``input_gather(x)``: the first layer's normed bf16 rows -> MXFP8 codes + AITER-shuffled scales of every token + (``gemm/mxfp8_common``'s quantizer on the shard, then ``AllGatherComm``). +2. ``attention_moe(attn_output, o_weight, o_scale, residual, norm_weight, eps, moe_weights)``: this rank's heads of + every token -> the pack into the pool slot, ``FencedAllGatherComm.all_to_all_unpack_shuffled`` (the whole o_proj + input of this rank's rows, scales in the GEMM's layout), o_proj_full on ``mxfp8_gemm``, ``gemma_add_rmsnorm`` + (bf16), ``prepare_mxfp4_moe``; its routed operands gathered (``AllGatherComm``, 3296 B/token) -> the routed-only + ``MoeInput`` of every token for ``mxfp4_moe(..., return_partials=True)``, the new residual shard and the + ``SharedPending`` full shared expert of this rank's rows (``shared_expert_full`` on the instance's side stream, + forked after the preparation, enqueued after the gather: it overlaps the gather and the routed experts). + ``attention_dense``: the same o_proj, then ``NormAllGatherComm`` (add + Gemma RMSNorm + MXFP8 quant of the fp32 + norm + gather in one launch) -> the dense MLP's codes + scales of every token, the residual shard. +3. ``ffn_norm_gather(ffn_out, residual, norm_weight, eps)``: the FFN output of every token (MoE: ``(slots, + shared)``, ``shared`` the ``SharedPending`` (joined first: the current stream waits on its event) or a [2 rows, H] + partial; dense: the bf16 partial) -> the reduce-scatter (fp32 in rank order; the owner's shared output added to + its own rows' slot sum), the next layer's input norm + MXFP8 quant on this rank's rows, gathered: + ``SlotsReduceScatterComm.slots4_reduce_scatter_add_rmsnorm_mxfp8`` + ``AllGatherComm``, or the dense + reduce-scatter + ``NormAllGatherComm``. + After the last layer ``ffn_reduce_scatter(ffn_out)`` gives this rank's summed rows for the model's final norm, and + ``gather(x)`` gathers bf16 rows (the normed output, an aux hidden state). + +Every collective moves bf16 partial sums or quantized next-GEMM inputs (no quantized reduction, no atomics). The +phases run on the caller's current stream, eagerly (the pool slots carry host-side sequence state; not inside CUDA +graph capture); every rank calls them with the same shapes in the same order. The shared expert's side stream joins +by an event on its exact launch before the reduce-scatter's first kernel, whose norm workers spin on in-launch +tickets: the persistent shared GEMM never runs next to them, and it waits on nothing itself (next to the gather's +spinning pullers it only delays them). Its inputs and buffers belong to the current stream's pool and are not +recorded on the side stream: the ``SharedPending`` holds them until the join, so they are freed in current-stream +order and the pool does not grow while the host runs layers ahead. + +Warm-up (root README rule 4): the communicators compile their transport kernels at construction (a collective); +``warm_up(o_weight, o_scale, moe_weights)`` makes the first call of each compute mode (the quantizer, o_proj, o_proj + +norm + MoE preparation, the pack, the routed-only partials mode of ``mxfp4_moe``, ``shared_expert_full``), each of +which compiles every variant of its rows range first (``ROWS``; ``amd_kernels.warm_up``). The compute modes run no +collective, so a rank's warm-up never waits for another. +""" + +from dataclasses import dataclass, field + +import torch + +from amd_kernels.warm_up import entry + +from .mxfp8_pack_q4096 import ROW_BYTES, mxfp8_pack_q4096 + +WORLD = 2 +H = 6144 +Q = 4096 # attention output per rank: 32 heads x 128 +ROW_ALIGN = 32 # shard rows: whole 32-row scale groups (NormAllGatherComm, the GEMMs' scale layout) +# 16384 rows per rank: gather_mxfp4_moe's and NormAllGatherComm's bound at two ranks. +MAX_TOKENS = 32768 +MAX_ROWS = MAX_TOKENS // WORLD +# The served range starts at 2048 tokens (1024 rows per rank; a MoE layer vs the TP path: -20% at 2048 tokens, even at +# 1024, benchmark_sp_layer.py): the warm-up covers MIN_ROWS..MAX_ROWS. +MIN_TOKENS = 2048 +MIN_ROWS = MIN_TOKENS // WORLD +_FP8 = torch.float8_e4m3fn + +# the routed-only MoeInput's fields in gather order: (columns, dtype), 3296 B/token +_FP4 = ( + (H // 2, torch.uint8), # routed MXFP4 codes + (H // 32, torch.uint8), # their e8m0 scales + (4, torch.float32), # top-4 weights (routed scale included) + (4, torch.int32), # top-4 ids +) +# the full shared expert's output stays on its rows' owner (no link bytes), added in the MoE tail's reduce-scatter +OWNER_SHARED = True + + +@dataclass +class SharedPending: + """The full shared expert's bf16 output of this rank's rows [rows, H], computed on a hand-offs instance's side + stream, the event recorded after its launch there, the stream whose pool its buffers came from and the tensors + the side stream reads or writes (released by the join).""" + + out: torch.Tensor + event: torch.cuda.Event + origin: torch.cuda.Stream | None = None + keep: list = field(default_factory=list) + + def wait(self) -> None: + """The current stream (and the pool's stream, if another) waits for the shared expert, then the held + buffers are released.""" + stream = torch.cuda.current_stream(self.out.device) + stream.wait_event(self.event) + if self.origin is not None and self.origin != stream: + self.origin.wait_event(self.event) + self.out.record_stream(stream) + self.keep.clear() + + def __del__(self): + # never joined (an exception between fork and join): the held blocks must not return to the pool before + # the side stream is done with them + if self.keep and self.origin is not None: + self.origin.wait_event(self.event) + + +# up to here the full shared expert runs on the current stream after the gather: on the side stream it collides with +# the routed experts instead of hiding under the gather (SP pull layer, benchmark_sp_layer.py: 2048 tokens +6%, 2304 +# +6%, 2560 even, 2816 -2.4%, 4096 -4%, 8192 -5%) +SHARED_SERIAL_MAX_ROWS = 1152 + + +def _fork_shared(side, local, moe_weights, after) -> SharedPending: + """``shared_expert_full`` of the prepared ``local`` rows on ``side`` once the current stream's event ``after`` (the + preparation) completed (``side`` None or at most ``SHARED_SERIAL_MAX_ROWS`` rows: on the current stream); its + buffers come from the current stream's pool.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import shared_expert_full + + current = torch.cuda.current_stream() + if side is None or local.shared_codes.shape[0] <= SHARED_SERIAL_MAX_ROWS: + side = current + else: + side.wait_event(after) + keep = [] + out = shared_expert_full(local.shared_codes, local.shared_scales, moe_weights.shared_weights, stream=side, + keep=keep) + done = torch.cuda.Event() + done.record(side) + return SharedPending(out, done, current, keep) + + +def _ffn_parts(ffn_out): + """``(slots, shared)`` with a ``SharedPending`` joined (the current stream waits) and replaced by its output.""" + slots, shared = ffn_out + if isinstance(shared, SharedPending): + shared.wait() + shared = shared.out + return slots, shared + + +def _full_shared(moe_weights) -> bool: + return bool(getattr(moe_weights.shared_weights, "full", False)) + + +def _round_up(x: int, m: int) -> int: + return (x + m - 1) // m * m + + +def shard_rows(num_tokens: int) -> int: + """Rows per rank of a ``num_tokens`` call (``ROW_ALIGN``-aligned, the last ranks' rows padded with zeros).""" + return _round_up(-(-num_tokens // WORLD), ROW_ALIGN) + + +def shard(x: torch.Tensor, rank: int) -> torch.Tensor: + """Rank ``rank``'s ``shard_rows`` rows of every token's ``x`` ``[t, ...]``, zero padded past ``t``.""" + rows = shard_rows(x.shape[0]) + start = rank * rows + n = min(rows, max(0, x.shape[0] - start)) + if n == rows: + return x[start : start + n].contiguous() + out = x.new_zeros((rows, *x.shape[1:])) + if n: + out[:n].copy_(x[start : start + n]) + return out + + +# The shard compute of the phases (no collective): one registered entry, a mode per kind. + + +def _shard_compute(kind, x, scales=None, o_weight=None, o_scale=None, residual=None, norm_weight=None, eps=0.0, + moe_weights=None): # fmt: skip + """``input``: bf16 rows ``x`` -> MXFP8 codes + shuffled scales. ``dense``: o_proj_full of the unpacked codes ``x`` + + ``scales`` -> bf16. ``moe``: that o_proj, the bf16 add + Gemma RMSNorm with ``residual``, the MoE preparation -> + (``MoeInput``, new residual).""" + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_e4m3_quantize_flydsl, mxfp8_gemm + + if kind == "input": + return mxfp8_e4m3_quantize_flydsl(x) + h = mxfp8_gemm(x, scales, o_weight, o_scale, torch.bfloat16, activation_scale_preshuffled=True) + if kind == "dense": + return h + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import prepare_mxfp4_moe + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.host import gemma_add_rmsnorm + + y, r = gemma_add_rmsnorm(h, residual, norm_weight, eps) + return prepare_mxfp4_moe(y, moe_weights), r + + +def _shard_mode(kind, *args, moe_weights=None, **kwargs): + """What a call fixes about its kernels besides the row count (``amd_kernels.warm_up``).""" + if kind != "moe": + return (kind,) + return kind, moe_weights.shared_weights is not None, moe_weights.router_weight_lines is not None + + +def _gemm_rows() -> tuple[int, ...]: + """One row count per o_proj_full variant (``mxfp8_gemm``'s table row and its four-wave remainder plan) over the + served rows: what ``warmup_mxfp8_gemm_weight`` walks, for this one shape and row-major codes.""" + from amd_kernels.gemm.mxfp8_common.gemm import _FOURWAVE_KERNELS, _fourwave_plan, _select_config + + seen = {} + for m in range(MIN_ROWS, MAX_ROWS + 1, ROW_ALIGN): + config = _select_config(m, H, WORLD * Q, torch.bfloat16, False) + key: tuple = (config,) + if config.kernel in _FOURWAVE_KERNELS: + key += tuple(tiles for tiles, _, _ in _fourwave_plan(m, H, config)) + seen.setdefault(key, m) + return tuple(sorted(seen.values())) + + +def _shard_sizes(mode) -> tuple[int, ...]: + # the quantizer's variant is the same from 1024 rows (rows per wave by row count) + return (MIN_ROWS,) if mode[0] == "input" else _gemm_rows() + + +def _shard_inputs(mode, n, kind, x, scales=None, o_weight=None, o_scale=None, residual=None, norm_weight=None, + eps=0.0, moe_weights=None): # fmt: skip + """One warm-up call of the mode at ``n`` rows (zeros: the dispatch depends on shapes only).""" + dev = x.device + if kind == "input": + return ("input", torch.zeros((n, H), dtype=torch.bfloat16, device=dev)), {} + codes = torch.zeros((n, WORLD * Q), dtype=_FP8, device=dev) + sc = torch.full((_round_up(n, 32), WORLD * Q // 32), 127, dtype=torch.uint8, device=dev) + if kind == "dense": + return (kind, codes, sc, o_weight, o_scale), {} + res = torch.zeros((n, H), dtype=torch.bfloat16, device=dev) + return (kind, codes, sc, o_weight, o_scale, res, norm_weight, eps), {"moe_weights": moe_weights} + + +shard_compute = entry("collectives/seq_parallel_handoffs_h6144_q4096_ranks2", mode=_shard_mode, sizes=_shard_sizes, + inputs=_shard_inputs)(_shard_compute) # fmt: skip + + +def _pack_mode(x, out=None, **kwargs): + return tuple(sorted(kwargs.items())) + + +def _pack_inputs(mode, n, x, out=None, **kwargs): + return (torch.zeros((n, Q), dtype=torch.bfloat16, device=x.device),), kwargs + + +# one variant (rows are a runtime argument) +pack = entry("collectives/seq_parallel_handoffs_h6144_q4096_ranks2:pack", mode=_pack_mode, sizes=lambda mode: (32,), + inputs=_pack_inputs)(mxfp8_pack_q4096) # fmt: skip + + +class SeqParallelHandoffs: + """The four communicators of the phases, one of each per rank, sized for ``max_tokens`` tokens per call (a + server's batch-token limit, clamped to ``MIN_TOKENS``..``MAX_TOKENS``). Collective: both ranks of the two-rank + ``cpu_group`` (gloo) constructs it at the same point; it compiles the transport kernels (``warm_up`` of each + communicator, real collectives on zero payloads). ``coded_tail``: the MoE tail's reduce-scatter in + ``lossless_x3``'s exact code (the same bits, 0.781x its link bytes), else BF16.""" + + def __init__(self, rank: int, cpu_group, device: torch.device, max_tokens: int = MAX_TOKENS, + coded_tail: bool = True, shared_stream: bool = True): + """``shared_stream`` False: the full shared expert on the current stream after the gather (benchmarks).""" + from amd_kernels.collectives.pull_collectives_ranks2 import ( + AllGatherComm, + FencedAllGatherComm, + NormAllGatherComm, + Pools, + SlotsReduceScatterComm, + ) + from amd_kernels.collectives.pull_collectives_ranks2.pull_unpack_q4096_shuffled import FENCE_MAX_TOKENS + + max_tokens = min(max(max_tokens, MIN_TOKENS), MAX_TOKENS) + device = torch.device(device) + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + self.device = device + self.max_tokens = max_tokens + self.rows = shard_rows(max_tokens) + self.padded = WORLD * self.rows + # the transport warm-up's larger unpack (FENCE_MAX_TOKENS // 2 + 32 rows per rank) + a2a_tokens = max(self.padded, FENCE_MAX_TOKENS + WORLD * 32) + self.pools = Pools(rank, WORLD, cpu_group, device, _round_up(a2a_tokens * ROW_BYTES, 256), sets=1) + self.a2a = FencedAllGatherComm(self.pools) + # the widest field set: one bf16 [rows, H] (aux / final output); the MoE payload is 3296 B/row in four 256 B + # aligned fields (7328 in six with a shard copy of the shared expert) + self.ag = AllGatherComm(rank, cpu_group, device, _round_up(self.rows * 2 * H + 6 * 256, 256)) + self.rs = SlotsReduceScatterComm(rank, cpu_group, device, max_rows=self.padded, coded=coded_tail) + self.norm_gather = NormAllGatherComm(rank, cpu_group, device, max_rows=self.rows) + # the full shared expert's stream (``attention_moe``) + self.side = torch.cuda.Stream(device=device) if shared_stream else None + for c in (self.a2a, self.ag, self.rs, self.norm_gather): + c.warm_up() + torch.accelerator.synchronize() + + def supports(self, num_tokens: int) -> bool: + """Whether a ``num_tokens`` call fits the communicators.""" + return 0 < num_tokens <= self.max_tokens + + def warm_up(self, o_weight: torch.Tensor, o_scale: torch.Tensor, moe_weights=None) -> None: + """Compile every compute variant the phases launch, before serving (module docstring): ``o_weight`` / + ``o_scale`` a layer's preshuffled o_proj_full (``gemm.mxfp8_common``), ``moe_weights`` a MoE layer's + ``MoeWeights``, with the full shared expert (``SharedWeights.full``; a shard copy warms the compact payload + form) (the dispatch depends on shapes only, so one layer serves all).""" + dev = self.device + x = torch.zeros((MIN_ROWS, H), dtype=torch.bfloat16, device=dev) + shard_compute("input", x) + codes = torch.zeros((MIN_ROWS, WORLD * Q), dtype=_FP8, device=dev) + sc = torch.full((MIN_ROWS, WORLD * Q // 32), 127, dtype=torch.uint8, device=dev) + shard_compute("dense", codes, sc, o_weight, o_scale) + pack(torch.zeros((ROW_ALIGN, Q), dtype=torch.bfloat16, device=dev)) + if moe_weights is not None: + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe, shared_expert_full + + w = torch.zeros(H, dtype=torch.bfloat16, device=dev) + local, _ = shard_compute("moe", codes, sc, o_weight, o_scale, x, w, 1e-6, moe_weights=moe_weights) + if _full_shared(moe_weights): + # the consumers: the routed-only partials mode of the gathered payload, the full shared expert + mxfp4_moe(local.routed(), moe_weights, return_partials=True) + shared_expert_full(local.shared_codes, local.shared_scales, moe_weights.shared_weights) + else: + mxfp4_moe(local.compact(), moe_weights, return_partials=True) + torch.accelerator.synchronize() + + def _gathered(self, rows: int, cols: int, dtype: torch.dtype) -> torch.Tensor: + return torch.empty((WORLD * rows, cols), dtype=dtype, device=self.device) + + def input_gather(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """This rank's normed bf16 rows ``[rows, H]`` -> MXFP8 codes ``[2 rows, H]`` e4m3 + AITER-shuffled scales + ``[2 rows, H / 32]`` of every token (the quantizer of the non-SP first layer, gathered).""" + rows = x.shape[0] + codes, scales = self._gathered(rows, H, _FP8), self._gathered(rows, H // 32, torch.uint8) + q, s = shard_compute("input", x) + self.ag.all_gather([codes, scales], [q, s]) + return codes, scales + + def _unpacked(self, attn_output: torch.Tensor, rows: int) -> tuple[torch.Tensor, torch.Tensor]: + """This rank's heads of every token ``[t, Q]`` -> the o_proj input of this rank's rows: codes ``[rows, 2 Q]``, + scales in o_proj_full's GEMM layout.""" + from amd_kernels.collectives.pull_collectives_ranks2 import Collectives + + t, d = attn_output.shape + padded = WORLD * rows + assert d == Q and t <= padded, (attn_output.shape, rows) + coll = Collectives(self.pools, 0) + coll.bind(torch.cuda.current_stream(self.device)) + send = coll.alloc((padded, ROW_BYTES), torch.uint8) + assert send is not None, "the all-to-all slot is sized for every call up to max_tokens" + pack(attn_output, out=send[:t]) + send[t:].zero_() + return self.a2a.all_to_all_unpack_shuffled(coll, send) + + def attention_moe(self, attn_output, o_weight, o_scale, residual, norm_weight, eps, moe_weights): + """The o_proj, the post-attention norm and the MoE input preparation of this rank's rows (``residual`` its + ``[rows, H]`` residual shard) -> (the routed-only ``MoeInput`` of every token, the new residual shard, the + ``SharedPending`` full shared expert of this rank's rows). With a shard copy of the shared expert + (``SharedWeights.full`` False): (``MoeInputCompact`` of every token, the residual shard, None).""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, MoeInputCompact + + rows = residual.shape[0] + xq, xs = self._unpacked(attn_output, rows) + local, r = shard_compute("moe", xq, xs, o_weight, o_scale, residual, norm_weight, eps, moe_weights=moe_weights) + if not _full_shared(moe_weights): + compact = local.compact() + outs = [self._gathered(rows, t.shape[1], t.dtype) for t in compact.tensors()] + self.ag.all_gather(outs, list(compact.tensors())) + return MoeInputCompact(*outs), r, None + prepared = torch.cuda.Event() + prepared.record() + outs = [self._gathered(rows, cols, dtype) for cols, dtype in _FP4] + self.ag.all_gather(outs, list(local.routed().tensors()[:4])) + shared = _fork_shared(self.side, local, moe_weights, prepared) + return MoeInput(*outs), r, shared + + def attention_dense(self, attn_output, o_weight, o_scale, residual, norm_weight, eps): + """The o_proj and the post-attention norm + MXFP8 quant of this rank's rows -> (codes ``[2 rows, H]``, scales, + the new residual shard): the dense MLP's input of every token.""" + rows = residual.shape[0] + codes, scales = self._gathered(rows, H, _FP8), self._gathered(rows, H // 32, torch.uint8) + xq, xs = self._unpacked(attn_output, rows) + h = shard_compute("dense", xq, xs, o_weight, o_scale) + _, _, r = self.norm_gather.norm_all_gather(h, residual, norm_weight, eps, q_out=codes, scale_out=scales) + return codes, scales, r + + def ffn_norm_gather(self, ffn_out, residual, norm_weight, eps): + """The FFN output of every token (MoE: ``(slots [2 rows, 4, H], shared)``, ``shared`` the ``SharedPending`` of + ``attention_moe`` or a [2 rows, H] partial; dense: the bf16 partial ``[2 rows, H]``) -> reduce-scatter, the + next layer's input norm + MXFP8 quant on this rank's rows, gathered: (codes ``[2 rows, H]``, scales, the new + residual shard).""" + if isinstance(ffn_out, tuple): + ffn_out = _ffn_parts(ffn_out) + rows = residual.shape[0] + codes, scales = self._gathered(rows, H, _FP8), self._gathered(rows, H // 32, torch.uint8) + if isinstance(ffn_out, tuple): + q, s, r = self.rs.slots4_reduce_scatter_add_rmsnorm_mxfp8(*ffn_out, residual, norm_weight, eps) + self.ag.all_gather([codes, scales], [q, s]) + else: + _, _, r = self.norm_gather.norm_all_gather( + self.rs.reduce_scatter(ffn_out), residual, norm_weight, eps, q_out=codes, scale_out=scales + ) + return codes, scales, r + + def ffn_reduce_scatter(self, ffn_out) -> torch.Tensor: + """The FFN output of every token (as ``ffn_norm_gather``) -> this rank's summed bf16 rows ``[rows, H]``.""" + if isinstance(ffn_out, tuple): + return self.rs.slots4_reduce_scatter(*_ffn_parts(ffn_out)) + return self.rs.reduce_scatter(ffn_out) + + def gather(self, x: torch.Tensor) -> torch.Tensor: + """This rank's bf16 rows ``[rows, H]`` -> every token's ``[2 rows, H]``.""" + out = self._gathered(x.shape[0], x.shape[1], x.dtype) + self.ag.all_gather([out], [x]) + return out diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs_sdma.py b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs_sdma.py new file mode 100644 index 0000000..2dfccbb --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/handoffs_sdma.py @@ -0,0 +1,419 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``SeqParallelHandoffs`` with the bytes moved by SDMA pushes (``collectives/sdma_push_ranks2``): the same phases and +the same outputs bit for bit, with no collective on the CUs. Every phase method takes ``between``, called after a +push is armed and before its wait: a DBO microbatch passes its yield, so the other microbatch computes while the +engines copy; a single batch passes nothing and waits at once. + +Per layer, as ``SeqParallelHandoffs`` (``handoffs.py``), with the same local kernels: + +1. ``input_gather``: the quantizer on this rank's rows, pushed into the peer's codes region. +2. ``attention_moe`` / ``attention_dense``: the pack (``mxfp8_pack_q4096``) into a send buffer, the peer's block + pushed to its all-to-all region; the unpack (``pull_unpack_q4096_shuffled`` without its publication, reading the + own block from the send buffer and the peer's from the local region; its flag reads see an always-ready word); + o_proj_full, the norm (+ the MoE preparation, its routed payload pushed, then the full shared expert on the + instance's side stream, both before ``between``; dense: ``gemma_add_rmsnorm_mxfp8_row_major`` writes this rank's + slot of the codes region, pushed from there). +3. ``ffn_norm_gather``: MoE: the shared expert joined, ``slots4_stage_rows_x3`` (slot sum + shared) writes this rank's + rows as BF16 and the + peer's in ``lossless_x3``'s exact code (0.781x the bytes) into a send buffer, their raw virtual waves straight into + the peer's reduce-scatter region after the coded rows; the coded rows pushed to the start of that region; + ``sum2_x3_add_rmsnorm_mxfp8`` decodes the peer's rows and sums in rank order (the same bits as the BF16 slabs), + writing this rank's slot of the codes region, pushed from there. Dense: the partial's peer block pushed, the + two-pointer ``sum2``, the norm into the codes slot, pushed. +4. ``ffn_reduce_scatter`` and ``gather``: as ``handoffs.py``; ``gather`` returns a copy (its regions are reused). + +Region reuse (``SdmaPush``'s contract): a region's consumer on this rank runs before this rank's next push, and +the peer waits for that push before it pushes into the region again, since every phase gathers from both ranks. The +coded tail's raw stores into the peer's region follow the same order: the stage kernel runs after this rank's wait +for the peer's previous push. +Two pushes into one region need a phase in between (its push follows the first one's consumer): ``gather`` alternates +between two regions of its own, since two gathers come in a row (the final output and an aux hidden state) and the +last layer's reduce-scatter comes right before them (a shared region races: a peer's gather can land before this +rank's sum reads it). + +Memory per instance (``max_tokens`` 16384, 8192 rows per rank): ~0.9 GB of regions, plus the stage buffer of the +step. One instance per DBO microbatch (their regions must differ; each owns its shared-expert side stream, so a +microbatch's join waits for its own shared expert only); both ranks call the phases of an instance in the same +order. +""" + +import struct + +import torch + +from .handoffs import (_FP4, H, MAX_TOKENS, MIN_ROWS, Q, ROW_ALIGN, WORLD, _ffn_parts, _fork_shared, _full_shared, + _round_up, pack, shard_compute, shard_rows) +from .mxfp8_pack_q4096 import ROW_BYTES + +CODED = 9600 # a row in pull_collectives_ranks2's lossless_x3 code + +_FP8 = torch.float8_e4m3fn +A2A, COMPACT, RS, CODES, GATHER0, GATHER1 = range(6) +_UNPACK_WGS = 32 +_READY = 0x7FFFFFFF # the unpack's flag words: always at or past any sequence number + + +def _field_bytes(cols, dtype) -> int: + return cols * torch.empty((), dtype=dtype).element_size() + + +# a shard copy's compact MoE payload (MoeInputCompact's fields) +_COMPACT = ((H // 32, torch.uint8), (4, torch.float32), (4, torch.int32), (H, torch.uint8), (H // 32, torch.uint8), + (H // 8, torch.uint8)) + + +def _compact_offsets(rows: int, fields=_FP4) -> list[int]: + offs, off = [], 0 + for cols, dtype in fields: + offs.append(off) + off = _round_up(off + WORLD * rows * _field_bytes(cols, dtype), 256) + return offs + [off] + + +def _codes_offsets(rows: int) -> tuple[int, int]: + """(scales offset, total) of the codes region at ``rows``: codes [2 rows, H] e4m3, scales [2 rows, H / 32].""" + s = _round_up(WORLD * rows * H, 256) + return s, s + WORLD * rows * (H // 32) + + +def _noop(): + pass + + +class SdmaSeqParallelHandoffs: + """The phases of ``SeqParallelHandoffs`` (same signatures, plus ``between``) on SDMA pushes, sized for + ``max_tokens`` tokens per call. Collective: both ranks of the two-rank ``cpu_group`` construct it at the same + point; it compiles the local kernels the transport adds (no collective in their warm-up). ``coded_tail`` as + ``SeqParallelHandoffs``.""" + + def __init__(self, rank: int, cpu_group, device: torch.device, max_tokens: int = MAX_TOKENS, + coded_tail: bool = True, compact: bool = False, shared_stream: bool = True): + """``compact``: also size the MoE payload region for a shard copy of the shared expert (``MoeInputCompact``, + 7328 B/token; else the routed payload's 3296). ``shared_stream`` as ``SeqParallelHandoffs``.""" + from amd_kernels.collectives.sdma_push_ranks2 import SdmaPush + + assert 0 < max_tokens <= MAX_TOKENS, max_tokens + device = torch.device(device) + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + self.device, self.rank = device, rank + self.coded_tail = coded_tail + self.max_tokens = max_tokens + self.rows = shard_rows(max_tokens) + self.padded = WORLD * self.rows + r = self.rows + sizes = [0] * 6 + sizes[A2A] = WORLD * r * ROW_BYTES + sizes[COMPACT] = max(_compact_offsets(r)[-1], _compact_offsets(r, _COMPACT)[-1] if compact else 0) + sizes[RS] = WORLD * r * H * 2 + sizes[CODES] = _codes_offsets(r)[1] + sizes[GATHER0] = sizes[GATHER1] = WORLD * r * H * 2 + self.t = SdmaPush(rank, cpu_group, device, sizes) + # this microbatch's full shared expert (``attention_moe``) + self.side = torch.cuda.Stream(device=device) if shared_stream else None + self.ready = torch.full((64,), _READY, dtype=torch.int32, device=device) + self.zero_counter = torch.zeros(1, dtype=torch.int32, device=device) + self._gathers = 0 + self._warm_transport() + torch.accelerator.synchronize() + + def supports(self, num_tokens: int) -> bool: + return 0 < num_tokens <= self.max_tokens + + # -- local kernels ------------------------------------------------------------------------------------------- + + @staticmethod + def _kernels(): + from amd_kernels.collectives.pull_collectives_ranks2.host import _kernels + from amd_kernels.collectives.pull_collectives_ranks2.pull_unpack_q4096_shuffled import _kernel + from amd_kernels.collectives.pull_collectives_ranks2.slots4_host import ( + _stage_rows, _stage_rows_x3, _sum_norm, _sum_norm_x3) + + return (_kernel(_UNPACK_WGS, False), _stage_rows(), _sum_norm(), _kernels()[-1], _stage_rows_x3(), + _sum_norm_x3(), _stage_rows(True), _stage_rows_x3(True)) + + def _warm_transport(self) -> None: + """Compile the local kernels the SDMA phases launch (zero payloads, no collective).""" + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + + dev, rows = self.device, ROW_ALIGN + send = torch.zeros((WORLD * rows, ROW_BYTES), dtype=torch.uint8, device=dev) + self._unpack([send[p * rows : (p + 1) * rows].data_ptr() for p in range(WORLD)], rows) + slots = torch.zeros((WORLD * rows, 4, H), dtype=torch.bfloat16, device=dev) + stage = self._stage(slots, slots[:, 0].contiguous()) + self._stage(slots, slots[:rows, 0].contiguous()) + self._stage_x3(slots, slots[:rows, 0].contiguous(), stage.data_ptr()) + res = torch.zeros((rows, H), dtype=torch.bfloat16, device=dev) + w = torch.zeros(H, dtype=torch.bfloat16, device=dev) + q = torch.empty((rows, H), dtype=_FP8, device=dev) + s = torch.empty((rows, H // 32), dtype=torch.uint8, device=dev) + slabs = [stage.data_ptr() + p * rows * H * 2 for p in range(WORLD)] + self._sum_norm(slabs, res, w, 1e-6, q, s) + own, coded = self._stage_x3(slots, slots[:, 0].contiguous(), stage.data_ptr()) + self._sum_norm_x3(own, coded.data_ptr(), stage.data_ptr(), res, w, 1e-6, q, s) + out = self._sum2(slabs, rows) + gemma_add_rmsnorm_mxfp8_row_major(out, res, w, 1e-6) + + def _unpack(self, srcs: list[int], rows: int): + """Codes ``[rows, 2 Q]`` + scales in o_proj_full's layout from the two ranks' packed blocks at ``srcs``.""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + unpack = self._kernels()[0] + codes = torch.empty((rows, WORLD * Q), dtype=torch.uint8, device=self.device) + scales = torch.empty((_round_up(rows, 32), WORLD * Q // 32), dtype=torch.uint8, device=self.device) + f = self.ready.data_ptr() + _run_compiled(unpack, codes.data_ptr(), scales.data_ptr(), *srcs, f, f, 0, 1, self.rank, rows, + torch.cuda.current_stream(self.device)) # fmt: skip + return codes.view(_FP8), scales + + def _owner(self, slots, shared) -> bool: + """Whether ``shared`` is the owner form ([rows, H]: this rank's rows only) or [2 rows, H].""" + total = slots.shape[0] + assert slots.shape == (total, 4, H) and shared.shape in ((total, H), (total // WORLD, H)) and total % WORLD == 0 + for x in (slots, shared): + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.device == self.device + return shared.shape[0] != total + + def _stage(self, slots, shared) -> torch.Tensor: + """Slot sum + shared of every token -> a bf16 ``[2 rows, H]`` stage buffer (``slots4_stage_rows``).""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + owner = self._owner(slots, shared) + total = slots.shape[0] + stage = torch.empty((total, H), dtype=torch.bfloat16, device=self.device) + _run_compiled(self._kernels()[6 if owner else 1], slots.data_ptr(), shared.data_ptr(), stage.data_ptr(), + self.zero_counter, 0, total, self.rank, torch.cuda.current_stream(self.device)) # fmt: skip + return stage + + def _stage_x3(self, slots, shared, raw: int): + """``slots4_stage_rows_x3``: (this rank's bf16 ``[rows, H]`` sums, the peer's ``[rows, CODED]`` coded sums); + raw virtual waves to ``raw`` (``[rows, H]`` bf16 slots).""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + owner = self._owner(slots, shared) + total = slots.shape[0] + rows = total // WORLD + own = torch.empty((rows, H), dtype=torch.bfloat16, device=self.device) + coded = torch.empty((rows, CODED), dtype=torch.uint8, device=self.device) + _run_compiled(self._kernels()[7 if owner else 4], slots.data_ptr(), shared.data_ptr(), own.data_ptr(), + coded.data_ptr(), raw, total, self.rank, torch.cuda.current_stream(self.device)) # fmt: skip + return own, coded + + def _sum_norm_x3(self, own, coded: int, raw: int, residual, weight, eps, q, scale): + """``sum2_x3_add_rmsnorm_mxfp8``: this rank's bf16 rows ``own``, the peer's coded rows at ``coded`` (raw + waves at ``raw``) -> codes into ``q``, scales into ``scale``; returns the new residual.""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + rows = residual.shape[0] + assert residual.shape == (rows, H) and residual.dtype == torch.bfloat16 and residual.is_contiguous() + assert own.shape == (rows, H) and weight.shape == (H,) and weight.dtype == torch.bfloat16 + assert q.is_contiguous() and q.shape == (rows, H) + res_out = torch.empty_like(residual) + _run_compiled(self._kernels()[5], own.data_ptr(), coded, raw, residual.data_ptr(), weight.data_ptr(), + q.data_ptr(), scale.data_ptr(), res_out.data_ptr(), scale.numel(), rows, self.rank, + struct.unpack(" codes into + ``q``, scales into ``scale``; returns the new residual.""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + rows = residual.shape[0] + assert residual.shape == (rows, H) and residual.dtype == torch.bfloat16 and residual.is_contiguous() + assert weight.shape == (H,) and weight.dtype == torch.bfloat16 + res_out = torch.empty_like(residual) + _run_compiled(self._kernels()[2], *srcs, residual.data_ptr(), weight.data_ptr(), q.data_ptr(), + scale.data_ptr(), res_out.data_ptr(), q.numel(), scale.numel(), rows, H, H, + struct.unpack(" torch.Tensor: + """The rank-order sum of the two bf16 ``[rows, H]`` slabs at ``srcs`` (the reduce-scatter's ``sum2``).""" + from amd_kernels.collectives.pull_collectives_ranks2.host import _run_compiled + + out = torch.empty((rows, H), dtype=torch.bfloat16, device=self.device) + _run_compiled(self._kernels()[3], out.data_ptr(), *srcs, rows * H * 2, torch.cuda.current_stream(self.device)) + return out + + # -- pushes -------------------------------------------------------------------------------------------------- + + def _region_view(self, k, off, shape, dtype) -> torch.Tensor: + n = 1 + for d in shape: + n *= d + nbytes = n * _field_bytes(1, dtype) + return self.t.region(k)[off : off + nbytes].view(dtype).view(shape) + + def _peers(self): + return [p for p in range(WORLD) if p != self.rank] + + def _gather_push(self, k, fields, rows, between, keep=()): + """Push this rank's ``fields`` = [(src address, field offset in region k, bytes per row)] to the peer's + slot ``rank`` of each field, call ``between``, wait.""" + items = [(p, src, off + self.rank * rows * n, rows * n) for p in self._peers() for src, off, n in fields] + seq = self.t.push(k, items, keep=keep) + between() + self.t.wait(k, seq) + + def _codes_slot(self, rows): + """This rank's slot of the codes region: (codes [rows, H], scales [rows, H / 32]).""" + s_off, _ = _codes_offsets(rows) + me = slice(self.rank * rows, (self.rank + 1) * rows) + codes = self._region_view(CODES, 0, (WORLD * rows, H), _FP8) + scales = self._region_view(CODES, s_off, (WORLD * rows, H // 32), torch.uint8) + return codes[me], scales[me] + + # -- phases -------------------------------------------------------------------------------------------------- + + def input_gather(self, x: torch.Tensor, between=_noop): + rows = x.shape[0] + q, s = shard_compute("input", x) + return self._codes_gather_from(rows, q, s, between) + + def _codes_gather_from(self, rows, q, s, between): + """Push this rank's codes ``q`` / scales ``s`` (a tensor elsewhere) as the codes region's slot.""" + s_off, _ = _codes_offsets(rows) + codes = self._region_view(CODES, 0, (WORLD * rows, H), _FP8) + scales = self._region_view(CODES, s_off, (WORLD * rows, H // 32), torch.uint8) + me = slice(self.rank * rows, (self.rank + 1) * rows) + codes[me].copy_(q) + scales[me].copy_(s) + self._gather_push(CODES, [(q.data_ptr(), 0, H), (s.data_ptr(), s_off, H // 32)], rows, between, keep=(q, s)) + return codes, scales + + def _codes_gather_slot(self, rows, between): + """Push the codes region's own slot, already written by a local kernel.""" + s_off, _ = _codes_offsets(rows) + mine_c, mine_s = self._codes_slot(rows) + self._gather_push(CODES, [(mine_c.data_ptr(), 0, H), (mine_s.data_ptr(), s_off, H // 32)], rows, between) + codes = self._region_view(CODES, 0, (WORLD * rows, H), _FP8) + scales = self._region_view(CODES, s_off, (WORLD * rows, H // 32), torch.uint8) + return codes, scales + + def _unpacked(self, attn_output, rows, between): + t, d = attn_output.shape + padded = WORLD * rows + assert d == Q and t <= padded, (attn_output.shape, rows) + send = torch.empty((padded, ROW_BYTES), dtype=torch.uint8, device=self.device) + pack(attn_output, out=send[:t]) + send[t:].zero_() + block = rows * ROW_BYTES + items = [(p, send.data_ptr() + p * block, self.rank * block, block) for p in self._peers()] + seq = self.t.push(A2A, items, keep=(send,)) + between() + self.t.wait(A2A, seq) + region = self.t.region(A2A).data_ptr() + srcs = [send.data_ptr() + p * block if p == self.rank else region + p * block for p in range(WORLD)] + return self._unpack(srcs, rows) + + def attention_moe(self, attn_output, o_weight, o_scale, residual, norm_weight, eps, moe_weights, between=_noop): + """``SeqParallelHandoffs.attention_moe``: the routed payload pushed, the full shared expert launched on this + instance's side stream, then ``between``, then the wait for the peer's payload.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, MoeInputCompact + + rows = residual.shape[0] + xq, xs = self._unpacked(attn_output, rows, between) + local, r = shard_compute("moe", xq, xs, o_weight, o_scale, residual, norm_weight, eps, moe_weights=moe_weights) + full = _full_shared(moe_weights) + if full: + prepared = torch.cuda.Event() + prepared.record() + fields, spec = list(local.routed().tensors()[:4]), _FP4 + else: + fields, spec = list(local.compact().tensors()), _COMPACT + offs = _compact_offsets(rows, spec) + assert offs[-1] <= self.t.region_bytes[COMPACT], "a shard copy of the shared expert needs compact=True" + outs, items = [], [] + me = slice(self.rank * rows, (self.rank + 1) * rows) + for (cols, dtype), off, x in zip(spec, offs, fields): + assert x.shape == (rows, cols) and x.dtype == dtype and x.is_contiguous(), (x.shape, x.dtype) + out = self._region_view(COMPACT, off, (WORLD * rows, cols), dtype) + out[me].copy_(x) + outs.append(out) + items.append((x.data_ptr(), off, _field_bytes(cols, dtype))) + pushes = [(p, src, off + self.rank * rows * n, rows * n) for p in self._peers() for src, off, n in items] + seq = self.t.push(COMPACT, pushes, keep=tuple(fields)) + shared = _fork_shared(self.side, local, moe_weights, prepared) if full else None + between() + self.t.wait(COMPACT, seq) + return (MoeInput(*outs) if full else MoeInputCompact(*outs)), r, shared + + def attention_dense(self, attn_output, o_weight, o_scale, residual, norm_weight, eps, between=_noop): + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + + rows = residual.shape[0] + xq, xs = self._unpacked(attn_output, rows, between) + h = shard_compute("dense", xq, xs, o_weight, o_scale) + q, s = self._codes_slot(rows) + _, _, r = gemma_add_rmsnorm_mxfp8_row_major(h, residual, norm_weight, eps, q_out=q, scale_out=s) + codes, scales = self._codes_gather_slot(rows, between) + return codes, scales, r + + def _reduce_scatter_push(self, full: torch.Tensor, rows, between): + """The peer's block of this rank's bf16 ``[2 rows, H]`` ``full`` -> slot ``rank`` of the peer's + reduce-scatter region; returns both slabs' addresses in rank order, this rank's own block read from + ``full``.""" + block = rows * H * 2 + items = [(p, full.data_ptr() + p * block, self.rank * block, block) for p in self._peers()] + seq = self.t.push(RS, items, keep=(full,)) + between() + self.t.wait(RS, seq) + region = self.t.region(RS).data_ptr() + return [full.data_ptr() + p * block if p == self.rank else region + p * block for p in range(WORLD)] + + def ffn_norm_gather(self, ffn_out, residual, norm_weight, eps, between=_noop): + if isinstance(ffn_out, tuple): + ffn_out = _ffn_parts(ffn_out) + rows = residual.shape[0] + q, s = self._codes_slot(rows) + if isinstance(ffn_out, tuple) and self.coded_tail: + # the peer's raw waves land after its coded rows in this rank's region, the coded rows at its start + peer_rs = self.t.peer_pool + self.t.offsets[RS] + own, coded = self._stage_x3(*ffn_out, peer_rs + rows * CODED) + seq = self.t.push(RS, [(p, coded.data_ptr(), 0, rows * CODED) for p in self._peers()], keep=(coded,)) + between() + self.t.wait(RS, seq) + region = self.t.region(RS).data_ptr() + r = self._sum_norm_x3(own, region, region + rows * CODED, residual, norm_weight, eps, q, s) + elif isinstance(ffn_out, tuple): + stage = self._stage(*ffn_out) + r = self._sum_norm(self._reduce_scatter_push(stage, rows, between), residual, norm_weight, eps, q, s) + else: + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + + assert ffn_out.dtype == torch.bfloat16 and ffn_out.is_contiguous() and ffn_out.shape == (WORLD * rows, H) + summed = self._sum2(self._reduce_scatter_push(ffn_out, rows, between), rows) + _, _, r = gemma_add_rmsnorm_mxfp8_row_major(summed, residual, norm_weight, eps, q_out=q, scale_out=s) + codes, scales = self._codes_gather_slot(rows, between) + return codes, scales, r + + def ffn_reduce_scatter(self, ffn_out, between=_noop) -> torch.Tensor: + if isinstance(ffn_out, tuple): + full = self._stage(*_ffn_parts(ffn_out)) + else: + full = ffn_out + assert full.dtype == torch.bfloat16 and full.is_contiguous() and full.shape[1] == H + rows = full.shape[0] // WORLD + return self._sum2(self._reduce_scatter_push(full, rows, between), rows) + + def gather(self, x: torch.Tensor, between=_noop) -> torch.Tensor: + rows, cols = x.shape + assert cols == H and x.dtype == torch.bfloat16 and x.is_contiguous() + k = GATHER0 if self._gathers % 2 == 0 else GATHER1 + self._gathers += 1 + out = self._region_view(k, 0, (WORLD * rows, H), torch.bfloat16) + out[self.rank * rows : (self.rank + 1) * rows].copy_(x) + self._gather_push(k, [(x.data_ptr(), 0, H * 2)], rows, between, keep=(x,)) + return out.clone() + + def warm_up(self, o_weight, o_scale, moe_weights=None) -> None: + """The compute variants (``SeqParallelHandoffs.warm_up``: the same registered entries).""" + from .handoffs import SeqParallelHandoffs + + SeqParallelHandoffs.warm_up(self, o_weight, o_scale, moe_weights) + + def drain(self) -> None: + """Host: every push done; the held sources dropped.""" + self.t.drain() diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/mxfp8_pack_q4096.py b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/mxfp8_pack_q4096.py new file mode 100644 index 0000000..780f71f --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/mxfp8_pack_q4096.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP8 pack of the attention output for the two-rank sequence-parallel all-to-all: bf16 ``[M, 4096]`` row-major -> +uint8 ``[M, 4224]``, each row its 4096 e4m3 codes, then its 128 e8m0 scale bytes (row-major, one per 32 values). The +layout ``pull_collectives_ranks2``'s unpacking all-to-all reads (``unpack_reference``). + +A wave per row pair (its whole 4096 values), 4 waves per workgroup: lanes 0-31 row 2j, 32-63 row 2j + 1, a lane one +16-byte piece (8 values) of each 512-byte block, all sixteen loads issued up front; per block the 32-value maximum on +the packed bf16 patterns, completed over the quad with two DPP shuffles, the quantizer's scale byte from the bf16 +pattern of the maximum and 8 codes per lane from ``v_cvt_scalef32_pk_fp8_bf16`` (the arithmetic of +``gemm/mxfp8_common/mxfp8_quant.py``, copied below), stored as 8 bytes per lane (non-temporal). One lane per group +drops its scale byte into the wave's 256 LDS bytes; the wave then stores both rows' 128 scale bytes in one dword store +(no barrier: only the wave reads its LDS bytes). Rows beyond M read zero and store nowhere (buffer resource bounds). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +D = 4096 # values per row +ROW_BYTES = D + D // 32 # 4096 codes + 128 scale bytes +KB = D // 256 # 256-value blocks per row +WAVES = 4 # one row pair each: 8 rows per workgroup +NT_STORE = 2 # non-temporal code stores: the peer pulls the slot from memory +QUAD_SWAP_ADJ = 0xB1 # dpp quad_perm [1, 0, 3, 2] +QUAD_SWAP_PAIR = 0x4E # dpp quad_perm [2, 3, 0, 1] + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _i32_max(a, b): + return (a > b).select(a, b) + + +def _pk_max_u16(a, b): + """per-16-bit-half unsigned max (v_pk_max_u16): on sign-cleared bf16 patterns that is the value order.""" + va = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(a))).result + vb = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(b))).result + return fx.Int32(llvm.BitcastOp(T.i32, llvm.intr_umax(va, vb)).result) + + +def _amax_bf16(dwords): + """max |x| over the 2 * len(dwords) bf16 values, as a 16-bit bf16 pattern in an i32.""" + m = None + for d in dwords: + a = fx.Int32(d) & fx.Int32(0x7FFF7FFF) + m = a if m is None else _pk_max_u16(m, a) + lo, hi = m & fx.Int32(0xFFFF), m >> 16 + return (lo > hi).select(lo, hi) + + +def _dpp(v, ctrl): + return fx.Int32(fx.rocdl.update_dpp(T.i32, _raw(fx.Int32(0)), _raw(fx.Int32(v)), ctrl, 0xF, 0xF, True)) + + +def _quad_max(v): + """max over the 4 lanes of a quad, in every lane of it""" + m = _i32_max(v, _dpp(v, QUAD_SWAP_ADJ)) + return _i32_max(m, _dpp(m, QUAD_SWAP_PAIR)) + + +def _e8m0_bf16(amax_bits): + """(scale byte, conversion scale) of a block from the bf16 pattern of its maximum: clamp(ceil(log2(max(amax, + tiny) / 448)) + 127, 0, 254), i.e. the exponent field - 8 + [mantissa > 0x60] in one carry; zero, denormal and + tiny maxima land on 0. Byte 0 would make the conversion scale a denormal, so such a block (maximum below 2^-119) + is converted with 2^-126 (reference.py).""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(8)) - fx.Int32(8) + scale = ((_i32_max(t, fx.Int32(9)) - fx.Int32(8)) << 23).bitcast(fx.Float32) + return sb, scale + + +def _quant_bf16(dwords, scale): + """dwords of bf16 pairs -> half as many dwords of fp8 codes: x / scale, round to nearest even.""" + out = [] + for i in range(0, len(dwords), 2): + pair0 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i]))).result + pair1 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i + 1]))).result + # the first conversion's other half is overwritten by the second + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16(T.i16x2, llvm.mlir_undef(T.i16x2), pair0, _raw(scale), False) + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16(T.i16x2, w, pair1, _raw(scale), True) + out.append(fx.Int32(llvm.BitcastOp(T.i32, w).result)) + return Vec.from_elements(out, fx.Int32) + + +@launcher +def compile_mxfp8_pack_q4096(waves: int = WAVES): + name = f"seq_parallel_mxfp8_pack_d4096_w{waves}" + + @fx.struct + class Shared: + sc: fx.Array[fx.Uint8, 256 * waves, 16] + + @flyc.kernel(name=name, known_block_size=[64 * waves, 1, 1]) + def kernel(X: fx.Tensor, P: fx.Tensor, i32_m: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + half, piece = lane // 32, lane % 32 # row of the pair, 16-byte piece of a 512-byte block + group = piece // 4 # 32-value group of the 256-value block (0..7) + x_rsrc = buffer_ops.create_buffer_resource(X, max_size=False, num_records_bytes=i32_m * (2 * D)) + p_rsrc = buffer_ops.create_buffer_resource(P, max_size=False, num_records_bytes=i32_m * ROW_BYTES) + row = (bx * waves + wave) * 2 + half + x_dw = (row * (2 * D) + piece * 16) // 4 + raw = [ + buffer_ops.buffer_load(x_rsrc, x_dw, vec_width=4, dtype=fx.Int32, soffset_bytes=kb * 512) # fmt: skip + for kb in range_constexpr(KB) + ] + out = row * ROW_BYTES + # this wave's two rows of scale bytes (128 each) in LDS: written and read by this wave only + lds_wave = fx.Int32(fx.ptrtoint(smem.sc.ptr)) + wave * 256 + + def lds_ptr(byte_off): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(lds_wave + byte_off))) + + for kb in range_constexpr(KB): + vals = [Vec(raw[kb])[i] for i in range(4)] + sb, scale = _e8m0_bf16(_quad_max(_amax_bf16(vals))) + buffer_ops.buffer_store(_quant_bf16(vals, scale), p_rsrc, out + piece * 8, offset_is_bytes=True, + soffset_bytes=kb * 256, cache_modifier=NT_STORE) # fmt: skip + if piece % 4 == 0: + llvm.StoreOp( + llvm.TruncOp(T.i8, _raw(sb), llvm.IntegerOverflowFlags.none).result, + lds_ptr(half * 128 + kb * 8 + group), + ) + fx.rocdl.s_waitcnt(lgkmcnt=0) + # lane l: scale dword l % 32 of the pair's row l // 32 + s_val = fx.Int32(llvm.load(T.i32, lds_ptr(lane * 4), alignment=4)) + s_row = (bx * waves + wave) * 2 + lane // 32 + buffer_ops.buffer_store(s_val, p_rsrc, s_row * ROW_BYTES + D + (lane % 32) * 4, offset_is_bytes=True) + + @flyc.jit + def launch_mxfp8_pack_q4096(X: fx.Tensor, P: fx.Tensor, i32_m: fx.Int32, grid: fx.Int32, stream: fx.Stream): + kernel(X, P, i32_m).launch(grid=(fx.Int64(grid), 1, 1), block=(64 * waves, 1, 1), stream=stream) + + launch_mxfp8_pack_q4096.kernel_name = name + launch_mxfp8_pack_q4096.rows_per_wg = 2 * waves + return launch_mxfp8_pack_q4096 + + +@functools.cache +def _launcher(waves: int = WAVES): + return compile_mxfp8_pack_q4096(waves) + + +def mxfp8_pack_q4096(x: torch.Tensor, out: torch.Tensor | None = None, waves: int = WAVES) -> torch.Tensor: + """bf16 ``[M, 4096]`` contiguous -> uint8 ``[M, 4224]`` packed rows (module doc; ``reference.pack_reference``). + ``out``: a contiguous ``[M, 4224]`` uint8 tensor to write (a pool slot's view), else a new one. One launch.""" + from amd_kernels.common.tensor_shim import _run_compiled + + m, d = x.shape + assert d == D and x.dtype == torch.bfloat16 and x.is_contiguous(), (x.shape, x.dtype) + if out is None: + out = torch.empty((m, ROW_BYTES), dtype=torch.uint8, device=x.device) + assert out.shape == (m, ROW_BYTES) and out.dtype == torch.uint8 and out.is_contiguous(), (out.shape, out.dtype) + if m: + exe = _launcher(waves) + grid = (m + exe.rows_per_wg - 1) // exe.rows_per_wg + _run_compiled(exe, x.view(torch.uint8).view(-1), out.view(-1), m, grid, torch.cuda.current_stream()) + return out diff --git a/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/reference.py b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/reference.py new file mode 100644 index 0000000..5cf94f7 --- /dev/null +++ b/amd_kernels/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/reference.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The torch contract of the two-rank sequence-parallel hand-offs (root README, rule 3). + +``pack_reference(x)``: the MXFP8 pack of bf16 ``[M, 4096]`` rows into uint8 ``[M, 4224]``: per 32-value block +``e = clamp(ceil(log2(max(amax, tiny) / 448)) + 127, 0, 254)`` (E8M0), ``code = e4m3fn(x * 2^(127 - e))``, round to +nearest even; a row is its 4096 codes, then its 128 scale bytes. One exception, the library quantizer's +(``gemm/mxfp8_common/mxfp8_quant.py``): a block whose maximum is below 2^-119 (byte 0, a denormal scale) is +converted with 2^-126, i.e. its codes are those of ``x * 2^126``. Bit for bit, no tolerance; it equals vLLM's +Triton ``sp_pack.mxfp8_quant_pack`` everywhere but in such blocks (activations never hold a block of nonzero +values below 2^-119; an all-zero block packs to zero codes either way). + +The phases (``handoffs.SeqParallelHandoffs``) launch other entries' kernels on this rank's rows and move their bytes +with ``pull_collectives_ranks2`` (or ``sdma_push_ranks2``): every phase output is bit for bit those kernels' outputs +on the rows the collective gives the rank (``pull_collectives_ranks2/reference.py``: gathers concatenate the ranks' +shards in rank order, the all-to-all unpack is ``unpack_reference`` with o_proj_full's scale shuffle, the +reduce-scatters add in fp32 in rank order from +0). Their numerics are those entries' contracts: ``gemm/mxfp8_common`` +(o_proj_full), ``norm/gemma_add_rmsnorm_h6144`` (the bf16 post-attention norm of a MoE layer), +``moe/mxfp4_h6144_i1536_e128_top4`` (``prepare_mxfp4_moe``; the gathered MoE input is its routed payload, the +routed-only ``MoeInput``; ``shared_expert_full``, the ``shared_mxfp8_ref`` contract), and the fused norm collectives of +``pull_collectives_ranks2``. + +The shared expert of a MoE layer is local to the owner of its rows: rank p computes the full shared expert ``h`` of +its own rows from their prepared MXFP8 operands and adds it into its own rows' slot sum in the MoE tail, in this fixed +order for token row i of rank o(i): on rank p, ``P_p(i) = bf16((((s0 + s1) + s2) + s3) [+ h(i) if p == o(i)])`` in +fp32 over rank p's bf16 slots, then ``y(i) = bf16((+0 + P_0(i)) + P_1(i))`` on o(i) (``slots4_reduce_scatter_reference`` +with the owner-form shared), then the residual add, norm and quant. The result does not depend on when the shared +expert's side stream ran (its join is an event on its launch). +""" + +import torch + +D = 4096 +E4M3_MAX = 448.0 + + +def pack_reference(x: torch.Tensor) -> torch.Tensor: + """bf16 ``[M, 4096]`` -> uint8 ``[M, 4224]`` (module docstring).""" + m, d = x.shape + assert d == D and x.dtype == torch.bfloat16 + blocks = x.float().reshape(m, d // 32, 32) + amax = blocks.abs().amax(dim=-1).clamp_min(torch.finfo(torch.float32).tiny) + e = (amax / E4M3_MAX).log2().ceil().add(127).clamp(0, 254) + # 2^(127 - e), except 2^126 for byte 0 (the kernel's conversion scale) + rescale = torch.exp2(127 - e.clamp_min(1)) + codes = (blocks * rescale.unsqueeze(-1)).reshape(m, d).to(torch.float8_e4m3fn).view(torch.uint8) + return torch.cat([codes, e.to(torch.uint8)], dim=1) diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/README.md b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/README.md new file mode 100644 index 0000000..9f10057 --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/README.md @@ -0,0 +1,46 @@ +# Dense decoder layers around their attention on two TP ranks, MXFP8 linears, hidden 6144, I 6144 per rank + +## What it does + +A dense decoder layer stack without its attention, on two TP ranks: the input fc split by N (5 taps of 6144 -> +6144, 3072 columns per rank), the embedding owner read + Gemma norm + MXFP8 quant, the all-reduce + add + Gemma norm +(+ quant) tails, the MLP (gate_up [12288, 6144] -> SwiGLU-OAI -> down [6144, 6144], TP partial), the stack around the +caller's attention and the context K / V chain (Gemma norm + MXFP8 GEMM [2560, 6144] + multi-layer K norm, RoPE, fp8 +insert into 2-KV-head caches). Callers: the MiniMax-M3.1 DSpark draft at TP2 and the target's dense layers 0-2 +(`norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2`, form `fp32`). Each call picks its consumer GEMM's code +layout (`gemm.mxfp8_common.gemm.prequant_code_tile_rows`) and row-count variant; weights are preshuffled MXFP8. +Contract (`reference.py`): compositions of the per-kernel references, cosine ~1; each call equals its per-kernel +chain bitwise. + +## Entry points + +- `DenseDecoderRanks2`: the layers' communicators, built collectively on both ranks after the weights load, before + any capture; its `fc` / `fc_cat` (eager), `embed_norm_quant`, `ar_norm_quant`, `ar_norm`, `forward` (the stack + around the caller's attention callback, 1-2048 ids), `store_context_kv`, `capture`, `close`. +- `stack_blocker`, `context_kv_blocker`: why a model's layer stack or context K / V chain does not fit (None: fits). +- `mxfp8_swiglu_oai_mlp`: the MLP's down output from the gate_up input codes, 1-2048 rows, `bf16_chain` or `fp32`; + `mlp_supported`: whether the MLP weights fit. +- `gemma_rmsnorm_mxfp8_linear`: Gemma norm -> MXFP8 codes in the GEMM's layout -> the GEMM, any row count and N. + +The MLP and the norm + GEMM are `amd_kernels.warm_up` entries; the row-count dispatch lives in `mlp.py`. + +## Performance + +The whole draft step at TP2 (fc + embedding + 5 layers with the draft attention q32 / kv2 + final norm), 7 rows per +request, MAX over 2 ranks, median us (`benchmark_draft_forward.py`): the block's fc eager + its stack in one graph vs +the TP2 stock path (replicated fc, `VocabParallelEmbedding`, Triton norms and `tl.dot_scaled` linears, AITER custom +all-reduce) in one graph. Roofline: serial sum of max(HBM 6.5 TB/s, MXFP8 5 PF / bf16 2.5 PF) per op + the fc +exchange and ten tails over the one xGMI link at 56 GB/s (its link part in brackets). + +| requests (rows) | 1 (7) | 8 (56) | 16 (112) | 32 (224) | 40 (280) | 48 (336) | 52 (364) | 64 (448) | 128 (896) | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| stock TP2 | 656 | 1014 | 1262 | 1956 | 2232 | 2466 | 2600 | 2965 | 5222 | +| this block | 370 | 574 | 801 | 1149 | 1329 | 1551 | 1624 | 1878 | 3341 | +| roofline (link) | 180 (16) | 282 (102) | 403 (204) | 644 (409) | 765 (511) | 886 (613) | 946 (665) | 1127 (818) | 2223 (1636) | + +## Tests + +```bash +python3 -m pytest tests/decoder/dense_mxfp8_h6144_i6144_ranks2 -q # test_decoder_ranks2.py: 2 GPUs +python3 benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py # 2 GPUs +``` diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py new file mode 100644 index 0000000..32467f7 --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The dense decoder layers around their attention on two TP ranks, MXFP8 linears, hidden 6144, I 6144 per rank: +the split input fc, the embedding + norm + quant, the all-reduce + norm (+ quant) tails, the MLP block, the layer +stack around the caller's attention and the context K / V chain. ``DenseDecoderRanks2`` holds the communicators; the +MLP and the norm + GEMM are plain entries.""" + +from .host import FC_N, MAX_ROWS, WORLD, DenseDecoderRanks2, DenseLayer, context_kv_blocker, stack_blocker +from .mlp import MXFP8_SWIGLU_FORMS, mlp_supported, mxfp8_swiglu_oai_mlp +from .norm_linear import gemma_rmsnorm_mxfp8_linear + +__all__ = ["DenseDecoderRanks2", "DenseLayer", "FC_N", "MAX_ROWS", "MXFP8_SWIGLU_FORMS", "WORLD", "context_kv_blocker", + "gemma_rmsnorm_mxfp8_linear", "mlp_supported", "mxfp8_swiglu_oai_mlp", "stack_blocker"] diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/host.py b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/host.py new file mode 100644 index 0000000..8e6c66b --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/host.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``DenseDecoderRanks2``: the communicators of the dense decoder layers on two TP ranks (built collectively, after +the weights load, before any CUDA graph capture) and the calls that use them. Each call picks the code layout its +consumer GEMM reads (``gemm.mxfp8_common.prequant_code_tile_rows`` of the consumer's (N, K)): the caller passes the +consumer's weight shape, not a tile.""" + +from collections.abc import Callable, Sequence +from dataclasses import dataclass + +import torch + +H = 6144 +FC_K = 5 * H # the input fc reads 5 taps +FC_N = 3072 # the fc's output columns per rank +WORLD = 2 +MAX_ROWS = 2048 # the communicators' row bound (the fc goes in pieces of this many rows) + + +def _tile(m: int, consumer_shape) -> int: + from amd_kernels.gemm.mxfp8_common import gemm as G + + n, k = consumer_shape + return G.prequant_code_tile_rows(m, n, k, torch.bfloat16) + + +@dataclass(frozen=True) +class DenseLayer: + """One layer's weights for ``DenseDecoderRanks2.forward`` (MXFP8 weights preshuffled).""" + + input_norm: torch.Tensor # [6144] bf16 + qkv_shape: tuple[int, int] # (N, K) of the qkv GEMM that reads the input norm's codes + post_norm: torch.Tensor # [6144] bf16 + gate_up_w: torch.Tensor + gate_up_ws: torch.Tensor + down_w: torch.Tensor + down_ws: torch.Tensor + + +def stack_blocker(world: int, max_rows: int, on_library: Sequence[bool], fc_weight: torch.Tensor, + embedding: torch.Tensor, layers: Sequence[DenseLayer]) -> str | None: + """Why ``DenseDecoderRanks2`` cannot run this layer stack (None: it can): ``world`` ranks, up to ``max_rows`` rows + per call, ``on_library`` per linear whether it runs ``gemm.mxfp8_common`` on this block's codes, the replicated + fc weight, this rank's embedding shard and every layer's weights.""" + from .mlp import mlp_supported + + if world != WORLD: + return f"{world} ranks (two-rank communicators)" + if not 1 <= max_rows <= MAX_ROWS: + return f"{max_rows} rows per call (1-{MAX_ROWS})" + if not all(on_library): + return "a linear does not run gemm.mxfp8_common on this block's codes" + if tuple(fc_weight.shape) != (H, FC_K) or fc_weight.dtype != torch.bfloat16: + return f"fc weight {tuple(fc_weight.shape)} {fc_weight.dtype} (bf16 [{H}, {FC_K}])" + e = embedding + if e.dim() != 2 or e.shape[1] != H or e.dtype != torch.bfloat16 or not e.is_contiguous(): + return f"embedding {tuple(e.shape)} {e.dtype} (contiguous bf16 [V / 2, {H}])" + if not layers: + return "no layers" + for i, layer in enumerate(layers): + if not mlp_supported(layer.gate_up_w, layer.down_w): + return f"layer {i}: FFN weights (MXFP8 gate_up [12288, 6144], down [6144, 6144])" + if layer.qkv_shape[1] != H: + return f"layer {i}: qkv K {layer.qkv_shape[1]}" + for w in (layer.input_norm, layer.post_norm): + if tuple(w.shape) != (H,) or w.dtype != torch.bfloat16 or not w.is_contiguous(): + return f"layer {i}: norm weight {tuple(w.shape)} {w.dtype} (contiguous bf16 [{H}])" + return None + + +def context_kv_blocker(proj_on_library: bool, proj_w: torch.Tensor, rotary_dim: int, neox: bool, + cos_sin_cache: torch.Tensor, kv_caches: list[torch.Tensor], k_scales: list[torch.Tensor], + v_scales: list[torch.Tensor]) -> str | None: + """Why ``store_context_kv`` cannot store these layers' context K / V (None: it can): the projection on + ``gemm.mxfp8_common`` with every layer's K / V rows, the RoPE (``rope_blocker``), the caches and scales + (``supports_multilayer``).""" + from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64 import rope_blocker + from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64.kv_norm_rope_insert_multilayer import ( + supports_multilayer, + ) + + if not proj_on_library: + return "the context projection does not run gemm.mxfp8_common" + why = rope_blocker(rotary_dim, neox, cos_sin_cache) + if why is not None: + return why + if not supports_multilayer(kv_caches, k_scales, v_scales): + return "the KV caches (equal e4m3 [pages, kv_heads, page, 256]) or their scales (fp32)" + width = len(kv_caches) * 2 * kv_caches[0].shape[1] * 128 + if tuple(proj_w.shape) != (width, H): + return f"context projection {tuple(proj_w.shape)} (every layer's K / V rows: [{width}, {H}])" + return None + + +class DenseDecoderRanks2: + """The input fc split by N (``bf16_concat_linear_n3072_k30720``'s ``ConcatLinearComm``; eager), the owner read of + the two vocabulary shards (``vocab_gather_h6144``) and the all-reduce + Gemma norm tails + (``allreduce_gemma_bf16_mxfp8_h6144_ranks2``'s ``Bf16ArNormComm``). Built and warmed up collectively: both ranks, + the same order. ``fc_weight`` is the replicated [6144, 30720] weight (this rank keeps its 3072 rows, packed; the + caller may free the replicated one), ``embedding`` this rank's vocabulary shard [V / 2, 6144].""" + + def __init__(self, rank: int, group, device, *, fc_weight: torch.Tensor, embedding: torch.Tensor, + max_rows: int = MAX_ROWS): + from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import warmup_vocab_gather_gemma_mxfp8 + from amd_kernels.embedding.vocab_gather_h6144 import VocabGather + from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import pack_weight + from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.comm import ConcatLinearComm + from amd_kernels.norm.allreduce_gemma_bf16_mxfp8_h6144_ranks2 import Bf16ArNormComm + + assert tuple(fc_weight.shape) == (H, FC_K) and fc_weight.dtype == torch.bfloat16 + assert 1 <= max_rows <= MAX_ROWS + self.rank, self.group, self.device, self.max_rows = rank, group, torch.device(device), max_rows + self.fc_weight = pack_weight(fc_weight[rank * FC_N:(rank + 1) * FC_N]) + self.fc_comm = ConcatLinearComm(self.fc_weight, rank, group, max_rows=max_rows) + self.vocab = VocabGather(embedding, rank, group) + assert self.vocab.ranks == WORLD + self.ar = Bf16ArNormComm(rank, group, self.device) + # collective warm-up (real exchanges on both ranks): the fc tiers, the lookups, the all-reduce + norm row tiers + # in both layouts and residual forms + w = torch.ones(H, dtype=torch.bfloat16, device=self.device) + self.fc_comm.warm_up() + self.vocab.warm_up() + warmup_vocab_gather_gemma_mxfp8(self.vocab, w) + self.ar.warm_up(w) + torch.cuda.synchronize(self.device) + + # ------------------------------------------------------------ the input fc + def fc(self, taps: list[torch.Tensor], num_tokens: int, out: torch.Tensor) -> None: + """out[:num_tokens] = cat(taps)[:num_tokens] @ fc^T (each [>= num_tokens, 6144] bf16, unit column stride, + row stride a multiple of 8, read in place; ``out`` contiguous rows); both ranks the same call, eager, in pieces + of ``max_rows``.""" + assert len(taps) == 5, len(taps) + for t in taps: + assert t.dtype == torch.bfloat16 and t.dim() == 2 and t.shape[1] == H and t.shape[0] >= num_tokens + assert t.stride(1) == 1 and t.stride(0) % 8 == 0, t.stride() + for i in range(0, num_tokens, self.max_rows): + j = min(i + self.max_rows, num_tokens) + self.fc_comm([t[i:j] for t in taps], out[i:j]) + + def fc_cat(self, x: torch.Tensor) -> torch.Tensor: + """fc of the concatenated taps ``x`` [rows, 30720] bf16.""" + m = x.shape[0] + out = torch.empty((m, H), dtype=x.dtype, device=x.device) + self.fc([x[:, i * H:(i + 1) * H] for i in range(5)], m, out) + return out + + # ------------------------------------------------------------ embedding, all-reduce tails + def embed_norm_quant(self, ids: torch.Tensor, norm_weight: torch.Tensor, eps: float, consumer_shape): + """(codes, scales, residual): the embedding rows of ``ids`` (the owner reads, no all-reduce), their Gemma + norm and its MXFP8 codes in the layout of the consumer GEMM (N, K = ``consumer_shape``).""" + from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import vocab_gather_gemma_mxfp8 + + return vocab_gather_gemma_mxfp8(ids, self.vocab, norm_weight, eps, _tile(ids.numel(), consumer_shape)) + + def ar_norm_quant(self, x: torch.Tensor, residual: torch.Tensor, norm_weight: torch.Tensor, eps: float, + consumer_shape): + """(codes, scales, residual) of all-reduce(x) + residual -> Gemma norm -> MXFP8 in the consumer's layout; + the residual stays on each rank's owned rows (pass the previous tail's residual).""" + return self.ar.ar_add_rmsnorm_mxfp8(x, residual, norm_weight, eps, _tile(x.shape[0], consumer_shape), + owned_residual=True) + + def ar_norm(self, x: torch.Tensor, residual: torch.Tensor, norm_weight: torch.Tensor, eps: float): + """(bf16 norm output, residual) of the last tail.""" + return self.ar.ar_add_rmsnorm(x, residual, norm_weight, eps, owned_residual=True) + + def forward(self, ids: torch.Tensor, layers: Sequence[DenseLayer], + attn: Callable[[int, torch.Tensor, torch.Tensor], torch.Tensor], final_norm: torch.Tensor, eps: float, + swiglu: tuple[float, float, float], form: str = "bf16_chain") -> torch.Tensor: + """The layer stack on ``ids`` (1-2048): embedding + norm + quant, then per layer ``attn(i, codes, scales)`` + (the caller's attention + o_proj on the qkv GEMM's codes, its TP partial [m, 6144] bf16), the all-reduce + + norm + quant tail, the MLP block (``swiglu`` = (alpha, beta, limit), ``form``) and the next tail; the last + tail's bf16 norm output [m, 6144] with ``final_norm``.""" + from .mlp import mxfp8_swiglu_oai_mlp + + ids = ids.reshape(-1) + m = ids.numel() + q, s, residual = self.embed_norm_quant(ids, layers[0].input_norm, eps, layers[0].qkv_shape) + for i, layer in enumerate(layers): + o = attn(i, q, s) + q, s, residual = self.ar_norm_quant(o, residual, layer.post_norm, eps, layer.gate_up_w.shape) + d = mxfp8_swiglu_oai_mlp(q, s, m, layer.gate_up_w, layer.gate_up_ws, layer.down_w, layer.down_ws, + *swiglu, form=form) + if i + 1 == len(layers): + return self.ar_norm(d, residual, final_norm, eps)[0] + nxt = layers[i + 1] + q, s, residual = self.ar_norm_quant(d, residual, nxt.input_norm, eps, nxt.qkv_shape) + raise ValueError("no layers") + + @staticmethod + def store_context_kv(states: torch.Tensor, hidden_norm: torch.Tensor, eps: float, proj_w: torch.Tensor, + proj_ws: torch.Tensor, positions: torch.Tensor, k_norm: torch.Tensor, k_eps: float, + cos_sin_cache: torch.Tensor, kv_caches: list[torch.Tensor], slot_mapping: torch.Tensor, + k_scales: list[torch.Tensor], v_scales: list[torch.Tensor]) -> None: + """The context K / V of every layer into its fp8 cache: ``gemma_rmsnorm_mxfp8_linear(states, hidden_norm, + eps, proj_w, proj_ws)`` (all layers' K / V rows), then + ``attention/norm_rope_fp8_insert_d128_rot64``'s ``kv_norm_rope_insert_multilayer`` at ``slot_mapping``.""" + from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64.kv_norm_rope_insert_multilayer import ( + kv_norm_rope_insert_multilayer, + ) + + from .norm_linear import gemma_rmsnorm_mxfp8_linear + + if states.shape[0] == 0: + return + kv = gemma_rmsnorm_mxfp8_linear(states, hidden_norm, eps, proj_w, proj_ws) + kv_norm_rope_insert_multilayer(kv, positions, k_norm, cos_sin_cache, kv_caches, + [slot_mapping] * len(kv_caches), k_scales, v_scales, k_eps) + + def capture(self): + """Around the CUDA graph capture of the calls (both ranks, the same graphs); the two-rank tails stage their + inputs, so this registers nothing (it matches the four-rank API).""" + return self.ar.capture() + + def close(self) -> None: + for c in (self.fc_comm, self.vocab, self.ar): + c.close() diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/mlp.py b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/mlp.py new file mode 100644 index 0000000..be1273f --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/mlp.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The dense MLP block on its gate_up input codes: gate_up GEMM -> SwiGLU-OAI + MXFP8 quant -> down GEMM (TP +partial), or at 1-16 rows the fused gate_up + SwiGLU kernel -> down on its bf16 output. The row-count dispatch and +the code layouts (``gemm.mxfp8_common.prequant_code_tile_rows`` of each GEMM) live here; the GEMM calls are the ones +``gemm.mxfp8_common``'s linear integration makes for codes / bf16 inputs, so the bits equal the per-kernel chain.""" + +import torch + +from amd_kernels.warm_up import entry + +H, I = 6144, 6144 +FUSED_GATEUP_MAX_ROWS = 16 # gemm.mxfp8_gateup_swiglu_n12288_k6144_i6144's rows (one 16-row code block) +MAX_ROWS = 2048 +MXFP8_SWIGLU_FORMS = ("bf16_chain", "fp32") + + +def _gemm_codes(q, s, w, ws, m): + """The MXFP8 GEMM on pre-quantized codes, as the linear integration picks it (decode row on tiled codes).""" + from amd_kernels.gemm.mxfp8_common import gemm as G + + return G.mxfp8_gemm_prequant(q, s, w, ws, torch.bfloat16, m).reshape(m, w.shape[0]) + + +def mlp_supported(gate_up_w: torch.Tensor, down_w: torch.Tensor) -> bool: + """Whether ``mxfp8_swiglu_oai_mlp`` takes these weights: preshuffled MXFP8 gate_up [12288, 6144] (gate | up) and + down [6144, 6144].""" + return ( + tuple(gate_up_w.shape) == (2 * I, H) + and tuple(down_w.shape) == (H, I) + and gate_up_w.dtype == down_w.dtype == torch.float8_e4m3fn + ) + + +def _mlp(q, s, m, gate_up_w, gate_up_ws, down_w, down_ws, alpha, beta, limit, form="bf16_chain"): + """down's output [m, 6144] bf16 (the TP partial: no reduction) of the MLP on gate_up's input ``q`` / ``s`` (the + MXFP8 codes and shuffled scales of m rows in the layout ``prequant_code_tile_rows(m, 12288, 6144)`` names, as + ``allreduce_gemma_bf16_mxfp8_h6144_ranks2`` / ``vocab_gather_gemma_mxfp8_h6144`` write them). Weights: + preshuffled MXFP8 (``gemm.mxfp8_common.preshuffle_mxfp8_weight``), gate_up [12288, 6144] (gate | up), down + [6144, 6144]. ``form``: the SwiGLU numerics (``bf16_chain`` = each op rounded to bf16, ``fp32``). The first call + of each form compiles every row count's variants (1-2048 rows).""" + from amd_kernels.activation.swiglu_oai_mxfp8_i6144 import swiglu_oai_mxfp8 + from amd_kernels.gemm.mxfp8_common import gemm as G + + assert 1 <= m <= MAX_ROWS and form in MXFP8_SWIGLU_FORMS + assert tuple(gate_up_w.shape) == (2 * I, H) and tuple(down_w.shape) == (H, I) + if m <= FUSED_GATEUP_MAX_ROWS and q.shape[0] == 16: + from amd_kernels.gemm.mxfp8_gateup_swiglu_n12288_k6144_i6144 import mxfp8_gateup_swiglu_bf16 + + act = mxfp8_gateup_swiglu_bf16(q, s, gate_up_w, gate_up_ws, m, alpha, beta, limit, form=form) + return G.mxfp8_linear(act.reshape(-1, I), down_w, down_ws, torch.bfloat16).reshape(m, H) + h = _gemm_codes(q, s, gate_up_w, gate_up_ws, m) + tile = G.prequant_code_tile_rows(m, H, I, torch.bfloat16) + qd, sd = swiglu_oai_mxfp8(h, alpha, beta, limit, tile, form=form) + return _gemm_codes(qd, sd, down_w, down_ws, m) + + +def _mode(q, s, m, gate_up_w, gate_up_ws, down_w, down_ws, alpha, beta, limit, form="bf16_chain"): + return form + + +def _inputs(mode, n, q, s, m, gate_up_w, gate_up_ws, down_w, down_ws, alpha, beta, limit, form="bf16_chain"): + """``n`` rows of zero codes in the layout the gate_up GEMM reads at n rows (the caller's weights; nothing the + call writes is the caller's).""" + from amd_kernels.gemm.mxfp8_common import gemm as G + + tile = G.prequant_code_tile_rows(n, 2 * I, H, torch.bfloat16) + rows = -(-n // max(tile, 1)) * max(tile, 1) + qn = torch.zeros((rows, H), dtype=torch.float8_e4m3fn, device=q.device) + sn = torch.full((-(-rows // 32) * 32, H // 32), 127, dtype=torch.uint8, device=q.device) + return (qn, sn, n, gate_up_w, gate_up_ws, down_w, down_ws, alpha, beta, limit), {"form": mode} + + +mxfp8_swiglu_oai_mlp = entry("decoder/dense_mxfp8_h6144_i6144_ranks2:mlp", mode=_mode, + sizes=lambda mode: range(1, MAX_ROWS + 1), inputs=_inputs)(_mlp) diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/norm_linear.py b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/norm_linear.py new file mode 100644 index 0000000..f34ac88 --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/norm_linear.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Plain Gemma RMSNorm -> MXFP8 codes in the consumer GEMM's layout -> the MXFP8 GEMM, one call (the context K/V +projection of a draft: ``hidden_norm`` + the K/V rows of every layer's qkv, N 2560 at TP2).""" + +import torch + +from amd_kernels.warm_up import entry + +from .mlp import _gemm_codes + +H = 6144 +# the norm's tiers (row-major / tiled codes, the four-wave form above 512 rows) and the first row of every window of +# the (2560, 6144) table and its shape host (rect 1537-2048, 192x256 3073-4800); the linear integration's own warm-up +# compiles the GEMM's table up to its token budget +WARMUP_ROWS = tuple(range(1, 257)) + (288, 289, 320, 384, 385, 448, 512, 513, 576, 577, 640, 641, 768, 769, 896, 897, + 1024, 1025, 1152, 1153, 1280, 1281, 1408, 1409, 1536, 1537, 2048, 2049, 3072, + 3073, 4096, 4097, 4800, 4801, 6144, 8192, 10240, 10241, 12288, 16384, 24576, + 32768) + + +def _norm_linear(x: torch.Tensor, norm_weight: torch.Tensor, eps: float, w: torch.Tensor, + ws: torch.Tensor) -> torch.Tensor: + """[m, N] bf16 = MXFP8 GEMM(quant(gemma_rmsnorm(x)), W) for ``x`` [m, 6144] bf16 and a preshuffled MXFP8 weight + [N, 6144]: the norm writes the codes the GEMM's table row reads (tiled or row-major). The quant reads the bf16 + norm output (``gemma_rmsnorm_bf16_mxfp8_h6144``'s rule). The first call compiles the norm's and the GEMM's + variants.""" + from amd_kernels.gemm.mxfp8_common import gemm as G + from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144 import gemma_rmsnorm_mxfp8 + + m = x.shape[0] + n, k = w.shape + assert k == H + if m == 0: + return torch.empty((0, n), dtype=torch.bfloat16, device=x.device) + tile = G.prequant_code_tile_rows(m, n, k, torch.bfloat16) + q, s = gemma_rmsnorm_mxfp8(x, norm_weight, eps, tile) + return _gemm_codes(q, s, w, ws, m) + + +def _inputs(mode, n, x, norm_weight, eps, w, ws): + return (torch.zeros((n, H), dtype=torch.bfloat16, device=x.device), norm_weight, eps, w, ws), {} + + +gemma_rmsnorm_mxfp8_linear = entry("decoder/dense_mxfp8_h6144_i6144_ranks2:norm_linear", + mode=lambda x, norm_weight, eps, w, ws: tuple(w.shape), + sizes=lambda mode: WARMUP_ROWS, inputs=_inputs)(_norm_linear) diff --git a/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/reference.py b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/reference.py new file mode 100644 index 0000000..790576a --- /dev/null +++ b/amd_kernels/decoder/dense_mxfp8_h6144_i6144_ranks2/reference.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The contract of the dense decoder block's calls: compositions of the per-kernel references. + +- ``mxfp8_swiglu_oai_mlp``: ``a`` = the dequantized gate_up input codes; ``h = bf16(a @ Wgu^T)`` (fp32 accumulate); + ``act`` = SwiGLU-OAI of ``h`` in the caller's form (``bf16_chain``: each op rounded to bf16, the uncompiled aten + chain; ``fp32``: one rounding); ``out = bf16(quant(act) @ Wd^T)`` (MXFP8 quant per 32 columns, e8m0 scales). +- ``gemma_rmsnorm_mxfp8_linear``: ``bf16(quant(bf16(gemma_rmsnorm(x))) @ W^T)``. +- ``DenseDecoderRanks2``: its kernels' own contracts (``bf16_concat_linear_n3072_k30720``, ``vocab_gather_h6144`` + + ``vocab_gather_gemma_mxfp8_h6144`` at two ranks, ``allreduce_gemma_bf16_mxfp8_h6144_ranks2``); it only picks each + consumer's code layout and the fc's 2048-row pieces. ``forward`` and ``store_context_kv`` are the sequences of + these calls (and ``norm_rope_fp8_insert_d128_rot64``'s multi-layer insert) they replace, bit for bit. + +Tolerance: cosine ~1 against these references (the GEMMs' fp32 K order; the fused 1-16-row gate_up rounds each dot +product to bf16 like the GEMM); the tests compare every call with the per-kernel chain bit for bit. +""" + +import torch + +from amd_kernels.gemm.mxfp8_common.reference import dequantize_mxfp8, quantize_mxfp8 +from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144.reference import gemma_rmsnorm_reference + + +def _swiglu(h, alpha, beta, limit, form): + i = h.shape[-1] // 2 + if form == "bf16_chain": + gate = torch.clamp(h[..., :i], max=limit) + up = torch.clamp(h[..., i:], min=-limit, max=limit) + return gate * torch.sigmoid(alpha * gate) * (up + beta) + gate = torch.clamp(h[..., :i].float(), max=limit) + up = torch.clamp(h[..., i:].float(), min=-limit, max=limit) + return (gate * torch.sigmoid(alpha * gate) * (up + beta)).bfloat16() + + +def mlp_reference(y, gate_up_codes, gate_up_exp, down_codes, down_exp, alpha, beta, limit, form="bf16_chain"): + """``y`` bf16 [m, 6144] (the gate_up input before its quant), row-major MXFP8 weights -> bf16 [m, 6144].""" + a = dequantize_mxfp8(*quantize_mxfp8(y)) + h = (a @ dequantize_mxfp8(gate_up_codes, gate_up_exp).T).bfloat16() + act = _swiglu(h, alpha, beta, limit, form) + d = dequantize_mxfp8(*quantize_mxfp8(act)) + return (d @ dequantize_mxfp8(down_codes, down_exp).T).bfloat16() + + +def norm_linear_reference(x, norm_weight, eps, codes, exp): + a = dequantize_mxfp8(*quantize_mxfp8(gemma_rmsnorm_reference(x, norm_weight, eps))) + return (a @ dequantize_mxfp8(codes, exp).T).bfloat16() diff --git a/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/README.md b/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/README.md index 623ace2..25db133 100644 --- a/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/README.md +++ b/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/README.md @@ -1,4 +1,4 @@ -# Peer vocabulary lookup + Gemma RMSNorm + MXFP8 quant on four ranks, hidden 6144 +# Peer vocabulary lookup + Gemma RMSNorm + MXFP8 quant on two or four ranks, hidden 6144 ## What it does @@ -6,15 +6,15 @@ One launch for the owner embedding lookup of `embedding/vocab_gather_h6144` foll `norm/gemma_rmsnorm_bf16_mxfp8_h6144`'s plain Gemma norm and MXFP8 quant: it returns the same codes and shuffled scales, and the raw bf16 embedding rows as the residual of the first post-attention tail. It reads the peer weights through a live `VocabGather` mapping (no second weight copy, no communication buffers, no handshake, no output -atomics). Shapes: four equal contiguous immutable bf16 vocabulary shards, hidden 6144; 0-2048 contiguous int32 / -int64 ids; norm weight bf16 `[6144]`, epsilon a runtime fp32. Outputs: e4m3 codes `[ceil(rows / tile) * tile, -6144]` in the 16x16 tiled layout (tile 0: row-major `[rows, 6144]`), E8M0 scales `[ceil(padded_rows / 32) * 32, 192]` -in AITER's shuffled layout, residual `[rows, 6144]` bf16; padding unwritten. Caller: the MiniMax-M3.1 DSpark draft's -layer 0 at TP4 (embedding + no-residual input norm + quant, the codes feed layer 0's QKV GEMM), through -`decoder/dense_mxfp8_h6144_i3072_ranks4` `embed_norm_quant`. Contract (`reference.py`): the lookup's rule (invalid -ids give zero rows; bf16 subnormals and signed zeros become +0), then the plain norm with its 12 wave partials and -FMA / reduction tree, rounded to bf16, then the per-32 E8M0 quantizer: codes and scales equal lookup + plain norm -bit for bit; norm accuracy follows the plain norm's fp64 contract. +atomics). Shapes: two or four equal contiguous immutable bf16 vocabulary shards, hidden 6144; 0-2048 contiguous int32 +/ int64 ids; norm weight bf16 `[6144]`, epsilon a runtime fp32. Outputs: e4m3 codes `[ceil(rows / tile) * tile, 6144]` +in the 16x16 tiled layout (tile 0: row-major `[rows, 6144]`), E8M0 scales `[ceil(padded_rows / 32) * 32, 192]` in +AITER's shuffled layout, residual `[rows, 6144]` bf16; padding unwritten. Caller: the MiniMax-M3.1 DSpark draft's +layer 0 (embedding + no-residual input norm + quant, the codes feed layer 0's QKV GEMM), through `embed_norm_quant` +of `decoder/dense_mxfp8_h6144_i3072_ranks4` (TP4) and `decoder/dense_mxfp8_h6144_i6144_ranks2` (TP2). Contract +(`reference.py`): the lookup's rule (invalid ids give zero rows; bf16 subnormals and signed zeros become +0), then the +plain norm with its 12 wave partials and FMA / reduction tree, rounded to bf16, then the per-32 E8M0 quantizer: codes +and scales equal lookup + plain norm bit for bit; norm accuracy follows the plain norm's fp64 contract. ## Entry points @@ -22,25 +22,29 @@ bit for bit; norm accuracy follows the plain norm's fp64 contract. a live `embedding.vocab_gather_h6144.VocabGather`; the code tile is a multiple of 16 (the consumer GEMM's `prequant_code_tile_rows`) or 0. Keep the mapping and its weight alive and unchanged through every graph replay, and close it collectively after all graphs finish. -- `warmup_vocab_gather_gemma_mxfp8`: compiles all 8 variants (2 wave forms x 2 id widths x 2 code layouts); call it - before serving or graph capture. +- `warmup_vocab_gather_gemma_mxfp8`: compiles all 8 variants of the mapping's world size (2 wave forms x 2 id widths + x 2 code layouts); call it before serving or graph capture. -Dispatch: `host.py` picks the 12-wave or four-wave kernel by row count. +Dispatch: `host.py` picks the 12-wave or four-wave kernel by row count, and the variant by the mapping's world size. ## Performance -MI355X, four ranks; graph replay of 100 changing id batches (7 rows per -request: six mask ids and one random anchor), 51200-row shards, warm weights, max over ranks, median of 5 alternated -rounds x 10 replays, us per call; baseline = `VocabGather` + `gemma_rmsnorm_mxfp8`. +MI355X, graph replay of 100 changing id batches (7 rows per request: six mask ids and one random anchor), warm +weights, max over ranks, median of alternated rounds, us per call. 2 ranks: 100032-row shards (one xGMI link), stock = +`VocabParallelEmbedding` (mask / gather / fill + AITER custom all-reduce) + `gemma_rmsnorm_mxfp8` +(`embedding/vocab_gather_h6144/benchmark_stock.py --norm`); 4 ranks: 51200-row shards, baseline = `VocabGather` + +`gemma_rmsnorm_mxfp8`. | Requests (rows) | 1 (7) | 10 (70) | 16 (112) | 32 (224) | 64 (448) | 128 (896) | 256 (1792) | |---|---|---|---|---|---|---|---| -| lookup + norm / quant | 6.05 | 7.12 | 7.62 | 9.48 | 15.69 | 29.14 | 53.92 | -| fused | 4.27 | 5.20 | 5.60 | 6.85 | 10.39 | 18.44 | 32.64 | +| 2 ranks, stock + norm / quant | 27.56 | 41.57 | 51.54 | 79.74 | 133.30 | 240.20 | 442.99 | +| 2 ranks, fused | 4.48 | 5.88 | 6.65 | 9.36 | 16.38 | 30.91 | 54.57 | +| 4 ranks, lookup + norm / quant | 6.11 | 7.17 | 7.66 | 9.69 | 16.03 | 29.55 | 57.18 | +| 4 ranks, fused | 4.40 | 5.37 | 5.77 | 7.00 | 10.75 | 19.81 | 35.47 | ## Tests ```bash -python3 -m pytest tests/embedding/vocab_gather_gemma_mxfp8_h6144 -q # 4 GPUs -python3 benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py # 4 GPUs +python3 -m pytest tests/embedding/vocab_gather_gemma_mxfp8_h6144 -q # 4 GPUs and 2 GPUs +python3 benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py --world 2 # or 4 ``` diff --git a/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/host.py b/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/host.py index ee0d9e3..eae290e 100644 --- a/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/host.py +++ b/amd_kernels/embedding/vocab_gather_gemma_mxfp8_h6144/host.py @@ -12,10 +12,10 @@ @functools.cache -def get_kernel(tiled, id_bits, fourwave): +def get_kernel(tiled, id_bits, fourwave, ranks=4): compiler = (compile_vocab_gather_gemma_mxfp8_fourwave if fourwave else compile_vocab_gather_gemma_mxfp8) - return compiler(tiled, id_bits) + return compiler(tiled, id_bits, ranks) def vocab_gather_gemma_mxfp8(ids, lookup, weight, eps=1e-6, code_tile_rows=16): @@ -39,7 +39,7 @@ def vocab_gather_gemma_mxfp8(ids, lookup, weight, eps=1e-6, code_tile_rows=16): residual = torch.empty((m, 6144), device=ids.device, dtype=torch.bfloat16) if m: _run_compiled( - get_kernel(code_tile_rows > 0, 32 if ids.dtype == torch.int32 else 64, m > 512), + get_kernel(code_tile_rows > 0, 32 if ids.dtype == torch.int32 else 64, m > 512, lookup.ranks), ids.data_ptr(), lookup.peers.data_ptr(), lookup.shard_vocab, weight.data_ptr(), q.data_ptr(), scales.data_ptr(), residual.data_ptr(), q.numel(), scales.numel(), m, struct.unpack('=0)&(token=0)&(token=0)&(token=0)&(token=4*shard)]=0 + out[(ids<0)|(ids>=vocab)]=0 return out def reference(ids,weights,norm_weight,eps=1e-6): diff --git a/amd_kernels/embedding/vocab_gather_h6144/README.md b/amd_kernels/embedding/vocab_gather_h6144/README.md index 3ea24a8..cb36589 100644 --- a/amd_kernels/embedding/vocab_gather_h6144/README.md +++ b/amd_kernels/embedding/vocab_gather_h6144/README.md @@ -1,44 +1,48 @@ -# Vocabulary embedding by owner reads on four ranks, hidden 6144 +# Vocabulary embedding by owner reads on two or four ranks, hidden 6144 ## What it does -Reads each token's embedding row straight from the rank that owns its vocabulary row: four TP ranks keep their -bf16 vocabulary shards (four equal contiguous `[V, 6144]` shards in rank order, a plain contiguous partition with no +Reads each token's embedding row straight from the rank that owns its vocabulary row: two or four TP ranks keep their +bf16 vocabulary shards (equal contiguous `[V, 6144]` shards in rank order, a plain contiguous partition with no added-vocabulary gaps) and peers map them once at load time over HIP IPC. One stateless kernel in place of the mask / gather / fill + all-reduce of `VocabParallelEmbedding`: no weight copy, no payload sum, no staging buffer and no per-call handshake. Token `t` reads shard `t // V`, row `t % V`; ids are a contiguous 1-D int32 or int64 tensor; the output is a contiguous bf16 `[rows, 6144]` on each rank. Caller: the MiniMax-M3.1 DSpark draft's embedding at -TP4 (V = 50016, vocabulary 200064), through `decoder/dense_mxfp8_h6144_i3072_ranks4`; -`embedding/vocab_gather_gemma_mxfp8_h6144` reuses the mapping. Contract (`reference.py`): finite rows bit for bit; -bf16 subnormals and signed zeros become +0 (AITER's fp32 flush when it adds the one owner to zeros); invalid ids -(< 0 or >= 4V) write +0; replays and eager calls are bitwise equal on every rank. +TP4 (V = 50016) through `decoder/dense_mxfp8_h6144_i3072_ranks4` and at TP2 (V = 100032) through +`decoder/dense_mxfp8_h6144_i6144_ranks2`; `embedding/vocab_gather_gemma_mxfp8_h6144` reuses the mapping. Contract +(`reference.py`): finite rows bit for bit; bf16 subnormals and signed zeros become +0 (AITER's fp32 flush when it adds +the one owner to zeros); invalid ids (< 0 or >= ranks x V) write +0; replays and eager calls are bitwise equal on every +rank. ## Entry points -- `VocabGather`: the collective load-time mapping of the four shards, built on every rank after selecting its GPU - and after this rank's shard reaches its final allocation; it checks equal shard sizes, publishes the GPU-written - weight to peers and exchanges the IPC handles. The weights stay alive and unchanged until `close()`; never move or - overwrite them while a captured graph may read them. +- `VocabGather`: the collective load-time mapping of the shards (world 2 or 4), built on every rank after selecting + its GPU and after this rank's shard reaches its final allocation; it checks equal shard sizes, publishes the + GPU-written weight to peers and exchanges the IPC handles. The weights stay alive and unchanged until `close()`; + never move or overwrite them while a captured graph may read them. - `VocabGather.__call__`: the `[rows, 6144]` bf16 rows of the ids, eager or inside a CUDA graph; no inter-rank synchronization per call. -- `VocabGather.warm_up`: compiles the int32 and int64 variants (one per id dtype, any row count). +- `VocabGather.warm_up`: compiles the int32 and int64 variants (one per id dtype and world size, any row count). - `VocabGather.close`: collective on every rank, after all graph use. -No dispatch: one kernel per id dtype (`host.py`), rows a runtime argument. +No dispatch: one kernel per id dtype and world size (`host.py`), rows a runtime argument. ## Performance -MI355X, four ranks; graph replay of 100 distinct batches (7 rows per -request: one random anchor + six mask ids), warm weights, max over ranks, median of 5 rounds x 10 replays, us per -call. The unfused chain (mask / gather / fill + all-reduce) is timed in the whole draft step, not as a unit. +MI355X, real shard size (200064 / ranks rows); graph replay of 100 distinct batches (7 rows per request: one random +anchor + six mask ids), warm weights, max over ranks, median us per call. Stock = `VocabParallelEmbedding`'s mask / +gather / fill + AITER custom all-reduce (`benchmark_stock.py`). TP2 pulls half the anchors over one xGMI link. | Requests (rows) | 1 (7) | 10 (70) | 16 (112) | 32 (224) | 64 (448) | 128 (896) | 256 (1792) | |---|---|---|---|---|---|---|---| -| `lookup` us | 3.10 | 4.02 | 4.43 | 5.40 | 7.59 | 13.51 | 26.70 | +| 2 ranks, stock | 24.81 | 37.99 | 48.47 | 75.79 | 128.46 | 235.09 | 438.18 | +| 2 ranks, `lookup` | 3.23 | 4.60 | 5.37 | 7.29 | 12.06 | 20.46 | 42.69 | +| 4 ranks, `lookup` | 3.21 | 4.23 | 4.72 | 6.02 | 8.90 | 15.51 | 28.46 | ## Tests ```bash -python3 -m pytest tests/embedding/vocab_gather_h6144 -q # 4 GPUs -python3 benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py # 4 GPUs +python3 -m pytest tests/embedding/vocab_gather_h6144 -q # 4 GPUs and 2 GPUs +python3 benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py --world 2 # or 4 +python3 benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py --world 2 ``` diff --git a/amd_kernels/embedding/vocab_gather_h6144/host.py b/amd_kernels/embedding/vocab_gather_h6144/host.py index 506a2f5..371e04f 100644 --- a/amd_kernels/embedding/vocab_gather_h6144/host.py +++ b/amd_kernels/embedding/vocab_gather_h6144/host.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""A collective load-time mapping of the four vocabulary shards; no weight copy.""" +"""A collective load-time mapping of the two or four vocabulary shards; no weight copy.""" import ctypes import functools import torch @@ -12,19 +12,20 @@ @functools.cache -def _kernel(dtype):return compile_vocab_gather(32 if dtype==torch.int32 else 64) +def _kernel(dtype,ranks):return compile_vocab_gather(32 if dtype==torch.int32 else 64,ranks) class VocabGather: """Keep the checkpoint's bf16 shard on its owner and read it directly. - Construct once on each of four TP ranks after loading weights. Shards have + Construct once on each of the two or four TP ranks after loading weights. Shards have the same vocabulary size, rank order, hidden 6144, and contiguous bf16 data. The weights remain alive and immutable until ``close()``. No GPU copy is retained. Eager and graph calls launch the same stateless FlyDSL kernel. """ def __init__(self,weight,rank,group): - assert dist.get_world_size(group)==4 + self.ranks=dist.get_world_size(group) + assert self.ranks in (2,4),self.ranks assert weight.ndim==2 and weight.shape[1]==H and weight.dtype==torch.bfloat16 and weight.is_contiguous() if torch.cuda.current_device()!=weight.device.index or dist.get_rank(group)!=rank: raise ValueError('select this rank device before mapping weights') @@ -53,7 +54,7 @@ def __call__(self,ids): assert ids.ndim==1 and ids.is_contiguous() and ids.dtype in (torch.int32,torch.int64) and ids.device==self.weight.device m=ids.numel() out=torch.empty((m,H),dtype=torch.bfloat16,device=ids.device) - if m:_run_compiled(_kernel(ids.dtype),ids,self.peers,out,m,self.shard_vocab,torch.cuda.current_stream()) + if m:_run_compiled(_kernel(ids.dtype,self.ranks),ids,self.peers,out,m,self.shard_vocab,torch.cuda.current_stream()) return out def warm_up(self): diff --git a/amd_kernels/embedding/vocab_gather_h6144/ipc.py b/amd_kernels/embedding/vocab_gather_h6144/ipc.py index 0c44190..c29c231 100644 --- a/amd_kernels/embedding/vocab_gather_h6144/ipc.py +++ b/amd_kernels/embedding/vocab_gather_h6144/ipc.py @@ -9,8 +9,6 @@ from amd_kernels.common.hip_runtime import load_hip_runtime -WORLD = 4 - def _run_compiled(exe, *args): """First call compiles and runs (``flyc.compile``); later calls dispatch the @@ -71,9 +69,9 @@ def _open_handle(raw: bytes) -> int: def exchange(obj, rank: int, group) -> list: """Every rank's ``obj`` (broadcasts on the CPU group: all_gather_object fails on gloo under inference mode).""" - out = [[None] for _ in range(WORLD)] - out[rank][0] = obj ranks = sorted(dist.get_process_group_ranks(group=group)) + out = [[None] for _ in ranks] + out[rank][0] = obj for i, r in enumerate(ranks): dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") return [o[0] for o in out] diff --git a/amd_kernels/embedding/vocab_gather_h6144/kernel.py b/amd_kernels/embedding/vocab_gather_h6144/kernel.py index 22fb4bf..2d8b743 100644 --- a/amd_kernels/embedding/vocab_gather_h6144/kernel.py +++ b/amd_kernels/embedding/vocab_gather_h6144/kernel.py @@ -12,15 +12,16 @@ PIECES = 3 # three contiguous 4096-byte pieces of each row -def compile_vocab_gather(id_bits): - name=f"vocab_gather_ranks4_h6144_i{id_bits}" +def compile_vocab_gather(id_bits,ranks=4): + assert ranks in (2,4) + name=f"vocab_gather_ranks{ranks}_h6144_i{id_bits}" @flyc.kernel(name=name,known_block_size=[THREADS,1,1]) def kernel(ids: fx.Tensor, peers: fx.Tensor, out: fx.Tensor, rows: fx.Int32, shard_vocab: fx.Int32): tx, bx = fx.thread_idx.x, fx.block_idx.x row, piece = bx // PIECES, bx % PIECES token = fx.Int64(ids[row]) - valid = (token >= 0) & (token < fx.Int64(shard_vocab * 4)) + valid = (token >= 0) & (token < fx.Int64(shard_vocab * ranks)) safe = valid.select(token,fx.Int64(0)) owner = (safe // fx.Int64(shard_vocab)).to(fx.Int32) local = safe % fx.Int64(shard_vocab) diff --git a/amd_kernels/embedding/vocab_gather_h6144/reference.py b/amd_kernels/embedding/vocab_gather_h6144/reference.py index e599f6c..5a96c71 100644 --- a/amd_kernels/embedding/vocab_gather_h6144/reference.py +++ b/amd_kernels/embedding/vocab_gather_h6144/reference.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Four contiguous vocabulary shards, each [V,6144] bf16, to [M,6144]. +"""Two or four contiguous vocabulary shards, each [V,6144] bf16, to [M,6144]. Read shard token//V, local row token%V. Invalid IDs produce positive zero. Bf16 subnormals and signed zeros become positive zero, as when AITER adds the diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/README.md b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/README.md new file mode 100644 index 0000000..05ebbd4 --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/README.md @@ -0,0 +1,37 @@ +# bf16 concat linear, N 3072 (one rank's shard) x K 30720, + the 2-rank exchange to N 6144 + +## What it does +The MiniMax-M3.1 DSpark draft's input fc at TP2: five bf16 taps [rows, 6144] (target layers 3, 17, 31, 45, 59; K +30720 = their concatenation) times `dspark.fc.weight` bf16 [6144, 30720], bf16 [rows, 6144] out on both ranks. Each +rank keeps its 3072-row weight slice (packed at load), computes its 3072 columns from the taps read in place (no +`torch.cat`), and the communicator exchanges the two shards over the pair's xGMI link into the caller's buffer. Every +output is one fp32 K sum (split-K slabs summed in a fixed order) rounded once to bf16; no quantization. Taps: unit +column stride, row stride % 8 == 0. Contract (`reference.py`): the fp64 product rounded once to bf16; per-row cosine +>= 0.99999, rel. L2 <= 1e-3, bitwise repeats. The slab count sets the fp32 summation order, so bits differ between +row tiers; within a tier they are deterministic. Caller: the TP2 dense decoder block (the draft's split input fc). + +## Entry points +- `pack_weight`: the rank's 3072-row slice as a lossless permutation, at load; free the row-major slice. +- `bf16_concat_linear`: the local bf16 [rows, 3072] shard of 1-2048 rows from the taps in place, graph-safe. +- `warmup_concat_linear`: compiles the row tiers up to a row count. +- `comm.ConcatLinearComm`: the two-rank fc into the caller's contiguous bf16 [rows, 6144] buffer (IPC staging, up to + 2048 rows, the slab reduce fused into the exchange); eager only (its Python slot rotation cannot replay in a graph); + both ranks make the same calls in the same order; its `warm_up` before serving, its `close` on both ranks. + +`host.TIERS` picks the GEMM (kernel, tile, K slabs) by row count; `comm.EXCHANGE` the exchange group shape. + +## Performance +Two ranks, eager fc as the draft calls it, cold rotated weights, MAX over ranks, median us (`benchmark_comm.py`). +Baseline = what vLLM runs at TP2: `torch.cat` + `F.linear` (hipBLASLt) on the replicated [6144, 30720] weight. +Serial roofline = max(weight + taps at 6.5 TB/s, bf16 at 2.5 PF) + the shard exchange at 64 GB/s. + +| rows | 8 | 32 | 64 | 128 | 192 | 256 | 320 | 448 | 512 | 768 | 1024 | 2048 | +|---|---|---|---|---|---|---|---|---|---|---|---|---| +| replicated F.linear | 67.7 | 69.8 | 77.3 | 96.8 | 129.4 | 126.0 | 162.7 | 179.8 | 225.2 | 275.4 | 343.0 | 628.8 | +| ConcatLinearComm | 35.7 | 41.2 | 46.7 | 62.4 | 81.7 | 98.1 | 116.5 | 147.1 | 166.0 | 249.3 | 308.7 | 601.0 | +| serial roofline | 29.9 | 32.4 | 35.8 | 42.5 | 49.3 | 56.0 | 62.8 | 76.8 | 87.8 | 131.7 | 175.6 | 351.2 | + +## Tests +- `python3 -m pytest tests/gemm/bf16_concat_linear_n3072_k30720 -q` (exchange / compile: 2 GPUs) +- `python3 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py [--sweep]` (1 GPU, local) +- `python3 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py [--sweep]` (2 GPUs) diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/__init__.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/__init__.py new file mode 100644 index 0000000..4e572b9 --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +from .host import bf16_concat_linear,pack_weight,warmup_concat_linear + +__all__=['bf16_concat_linear','pack_weight','warmup_concat_linear'] diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/comm.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/comm.py new file mode 100644 index 0000000..8700686 --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/comm.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Eager two-rank N-shard projection with a two-slot shard exchange.""" +import ctypes +import functools +import torch +import torch.distributed as dist +from amd_kernels.common import hip_runtime as ops +from . import host +from .host import run +from .reduce_gather import DATA_OFF,GROUPS,compile_reduce_gather + +WORLD=2 +# (max rows, rows per group, column pieces per row, push (write the peer's slot) or pull (read it)); sweep: +# benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py --sweep +EXCHANGE=((32,1,6,False),(1<<31,1,1,False)) + + +class _DevicePointer: + """A raw device allocation as a tensor (``__cuda_array_interface__``).""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = { + "shape": (nbytes,), + "typestr": "|u1", + "data": (ptr, False), + "version": 3, + } + + +def exchange(obj, rank: int, group) -> list: + """Every rank's ``obj`` (broadcasts on the CPU group: all_gather_object fails + on gloo under inference mode).""" + out = [[None] for _ in range(WORLD)] + out[rank][0] = obj + ranks = sorted(dist.get_process_group_ranks(group=group)) + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +@functools.cache +def reduce_gather_kernel(rpw,cpg,split,push):return compile_reduce_gather(rpw,cpg,split,push) + + +def _fit(rows,rpw,cpg): + while (rows+rpw-1)//rpw*cpg>GROUPS:rpw*=2 + return rpw + + +def exchange_for(rows): + """(rows per group, column pieces, push) of the shard exchange; groups never exceed the flag lines.""" + rpw,cpg,push=next((r,c,p) for top,r,c,p in EXCHANGE if rows<=top) + return _fit(rows,rpw,cpg),cpg,push + + +class ConcatLinearComm: + def __init__(self,packed_weight,rank,group,max_rows=host.MAX_ROWS): + assert dist.get_world_size(group)==WORLD and packed_weight.shape==(host.N,host.K) and 1<=max_rows<=host.MAX_ROWS + self.weight,self.rank,self.group=packed_weight,rank,group + self.max_rows=max_rows + self.slot_bytes=max_rows*host.N*2 + ptr,handle=ops.allocate_shared_buffer_and_handle(DATA_OFF+2*self.slot_bytes) + every=exchange(handle,rank,group) + self.bufs=[ptr if r==rank else ops.open_mem_handle(every[r]) for r in range(WORLD)] + self.counts=torch.zeros(GROUPS,dtype=torch.int32,device=packed_weight.device) + self.call=0 + torch.cuda.synchronize();dist.barrier(group=group) + + def __call__(self,taps,out,config=None,exchange_config=None): + """The fc of the five taps into ``out`` [rows, 6144]; ``config`` / ``exchange_config`` override the tier and + (rows per group, column pieces, push) (sweeps).""" + if torch.cuda.is_current_stream_capturing():raise RuntimeError('full concat-linear communicator is eager only; Python slot rotation cannot replay') + m=taps[0].shape[0] + assert m<=self.max_rows and out.shape==(m,2*host.N) and out.is_contiguous() and out.dtype==torch.bfloat16 + slot=self.call%2;self.call+=1 + cfg=config or host.config_for(m) + partials=host._partials_config(taps,self.weight,*cfg) + rpw,cpg,push=exchange_config or exchange_for(m) + run(reduce_gather_kernel(_fit(m,rpw,cpg),cpg,cfg[3],push),partials,*self.bufs,self.counts.data_ptr(),out,m, + self.rank,slot,self.slot_bytes,torch.cuda.current_stream()) + return out + + def warm_up(self): + """Compile every (tier, exchange) variant a row count up to ``max_rows`` reaches.""" + reps={} + for m in range(1,self.max_rows+1): + reps.setdefault((host.config_for(m),exchange_for(m)),m) + for m in reps.values(): + xs=[torch.zeros((m,host.H),dtype=torch.bfloat16,device=self.weight.device) for _ in range(5)] + self(xs,torch.empty((m,2*host.N),dtype=torch.bfloat16,device=self.weight.device)) + + def close(self): + torch.cuda.synchronize();dist.barrier(group=self.group) + hip=ops._hip() + hip.hipFree.argtypes=[ctypes.c_void_p];hip.hipIpcCloseMemHandle.argtypes=[ctypes.c_void_p] + for r,p in enumerate(self.bufs): + rc=(hip.hipFree if r==self.rank else hip.hipIpcCloseMemHandle)(ctypes.c_void_p(p)) + assert rc==0,rc + self.bufs=[] diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/flags.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/flags.py new file mode 100644 index 0000000..d123fd6 --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/flags.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""System-scope epoch flags, copied from the all-reduce family.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr.typing import T + +def _gptr(addr): + return fx.inttoptr( + fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + fx.Int64(addr), + ) + + +def _flag_store(addr, v): + """An atomic exchange at system scope (write-through).""" + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.xchg, + _gptr(addr).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ) + + +def _flag_read(addr): + """An atomic add of 0 at system scope (a spin on it is never hoisted).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _gptr(addr).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +@flyc.jit +def _spin(a, e): + f = _flag_read(a) + while (f - e) < 0: # signed difference: the counts may wrap + f = _flag_read(a) diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/host.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/host.py new file mode 100644 index 0000000..43ca46d --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/host.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +import functools +import torch +from .kernel import compile_reduce +from .kernel_staged_a_eightwave import compile_concat_linear_staged_a_eightwave +from .kernel_lds import compile_concat_linear_lds + +N, K, H = 3072, 30720, 6144 +MAX_ROWS = 2048 # the tiers' row bound (a caller with more rows splits them) +# (max rows, (kernel, row tiles, column tiles per wave, K slabs)); cold-weight graph sweep: +# benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py --sweep +TIERS = ( + (32, ("lds3", 1, 8, 10)), + (64, ("lds3", 2, 8, 10)), + (96, ("lds3", 3, 8, 10)), + (128, ("lds3", 4, 8, 10)), + (192, ("lds3", 6, 8, 10)), + (256, ("lds3", 8, 8, 10)), + (320, ("lds3", 5, 16, 10)), + (448, ("lds", 7, 16, 10)), + (512, ("lds", 8, 16, 10)), + (768, ("lds", 8, 12, 5)), + (1024, ("lds", 8, 16, 5)), + (MAX_ROWS, ("eightwave", 8, 3, 2)), +) +_COMPILERS = { + "eightwave": compile_concat_linear_staged_a_eightwave, + "lds": lambda mt, nr, split, _: compile_concat_linear_lds(mt, nr, split, 2), + "lds1": lambda mt, nr, split, _: compile_concat_linear_lds(mt, nr, split, 1), + "lds3": lambda mt, nr, split, _: compile_concat_linear_lds(mt, nr, split, 2, 3), +} + + +@functools.cache +def get_kernel(kind, mt, nt, split): + return _COMPILERS[kind](mt, nt, split, 2) + + +@functools.cache +def get_reduce(split):return compile_reduce(split) + + +def run(exe,*args): + import flydsl.compiler as flyc + if not hasattr(exe,'_cf'):exe._cf=flyc.compile(exe,*args) + else:exe._cf(*args) + + +def _partials_config(taps,weight,kind,mt,nt,split): + rows=taps[0].shape[0] + partials=torch.empty((split,rows,N),device=weight.device,dtype=torch.float32) + run(get_kernel(kind,mt,nt,split),*(x.data_ptr() for x in taps),weight.data_ptr(),partials.data_ptr(),rows,*(x.stride(0) for x in taps),torch.cuda.current_stream()) + return partials + + +def _linear_config(taps,weight,kind,mt,nt,split,out=None): + rows=taps[0].shape[0] + partials=_partials_config(taps,weight,kind,mt,nt,split) + out=out if out is not None else torch.empty((rows,N),device=weight.device,dtype=torch.bfloat16) + run(get_reduce(split),partials,out,rows,torch.cuda.current_stream()) + return out + + +def pack_weight(weight): + """Lossless load-time layout; replace and release the row-major weight.""" + return weight.view(N//16,16,K//32,4,8).permute(0,2,3,1,4).contiguous().view(N,K) + + +def config_for(rows): + """The row tier (kernel, row tiles, column tiles per wave, K slabs); the slab count sets the fp32 sum order.""" + assert 0 < rows <= MAX_ROWS, rows + return next(cfg for top, cfg in TIERS if rows <= top) + + +def bf16_concat_linear(taps,weight,out=None,config=None): + """Project five bf16 taps in place through the packed N-shard; ``config`` overrides the tier (sweeps).""" + assert len(taps)==5 and weight.shape==(N,K) and weight.dtype==torch.bfloat16 and weight.is_contiguous() + m=taps[0].shape[0] + assert 0=max_rows:break diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel.py new file mode 100644 index 0000000..2836013 --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The split-K slab reduce of every tier: the fp32 slabs summed in slab order, one bf16 rounding.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +N=3072 + + +def compile_reduce(split): + @flyc.kernel(name=f'bf16_linear_reduce_n3072_sk{split}',known_block_size=[256,1,1]) + def kernel(partials:fx.Tensor,out:fx.Tensor,rows:fx.Int32): + t=fx.block_idx.x*256+fx.thread_idx.x + valid=tLDS DMA (weight in the packed layout, 2 KB per fragment, read back in lane order; +taps row-major with a 16-byte XOR swizzle), ``ring`` blocks in flight. Each slab keeps the ascending-K MFMA order. +Based on gemm/bf16_n50016_k6144/large.py. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ods +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +N, K, H = 3072, 30720, 6144 +NWV = 4 +TAP_BLOCKS = H // 64 +W_CM = 2 # non-temporal weight DMAs: every weight byte is read by one M tile at a time + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _sb(): + fx.rocdl.sched_barrier(0) + + +def _lds_ptr(byte_addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(byte_addr))) + + +def compile_concat_linear_lds(mt, nr, split, mh=2, ring=2): + MT, NR, MH, R = mt, nr, mh, ring + NB = K // 64 // split # 64-K blocks per slab + assert TAP_BLOCKS % NB == 0 and NR % NWV == 0 and (N // 16) % NR == 0, (NB, NR) + NRW = NR // NWV + NWT = NWV * MH + MTT = MT * MH + TM = 16 * MTT + NTILES = N // (16 * NR) + XD = MTT * 2 + XPW = (XD + NWT - 1) // NWT + PER = NRW * 2 // MH + XPW + SLOT = (NR + MTT) * 2048 + LDS = R * SLOT + assert MH in (1, 2) and LDS <= 160 * 1024 and (R - 1) * PER <= 63, (LDS, R, PER) + name = f"bf16_concat_linear_n3072_k30720_lds_mt{MT}_mh{MH}_nr{NR}_r{R}_sk{split}" + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, LDS, 16] + + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel(x0: fx.Int64, x1: fx.Int64, x2: fx.Int64, x3: fx.Int64, x4: fx.Int64, + weight: fx.Int64, partials: fx.Int64, rows: fx.Int32, + sx0: fx.Int32, sx1: fx.Int32, sx2: fx.Int32, sx3: fx.Int32, sx4: fx.Int32): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave_all = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wave = wave_all % NWV + mhalf = wave_all // NWV + # XCD = bid % 8: the M tiles of one (column tile, slab), which share its weight slab, share an XCD when + # split x column tiles is a multiple of 8 + s = bid % split + nt = (bid // split) % NTILES + mb = bid // (split * NTILES) + m0 = mb * TM + kb0 = s * NB + tap = kb0 // TAP_BLOCKS + xbase = (tap == 0).select(x0, (tap == 1).select(x1, (tap == 2).select(x2, (tap == 3).select(x3, x4)))) + sx = (tap == 0).select(sx0, (tap == 1).select(sx1, (tap == 2).select(sx2, (tap == 3).select(sx3, sx4)))) + xsoff = (kb0 % TAP_BLOCKS) * 128 + g = lane // 16 + l16 = lane % 16 + r8 = lane // 8 + + xr = buffer_ops.create_buffer_resource_from_addr(xbase, num_records_bytes=fx.Int64(rows) * fx.Int64(sx * 2)) + wr = buffer_ops.create_buffer_resource_from_addr(weight, num_records_bytes=N * K * 2) + pr = buffer_ops.create_buffer_resource_from_addr(partials, num_records_bytes=fx.Int64(rows) * (N * split * 4)) + lds_base = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + + w_src, w_dst = [], [] + for i in range_constexpr(NRW): + rn = wave + i * NWV + for jq in range_constexpr(2 // MH): + jj = mhalf + jq if MH == 2 else jq + w_src.append((nt * NR + rn) * (K * 32) + (kb0 * 2 + jj) * 1024 + lane * 16) + w_dst.append((rn * 2 + jj) * 1024) + x_src, x_dst = [], [] + for i in range_constexpr(XPW): + d = wave_all + i * NWT + d = (d < XD).select(d, fx.Int32(XD - 1)) # spare DMAs repeat the last one (same bytes, same place) + jx = d % 2 + row = m0 + (d // 2) * 16 + jx * 8 + r8 + row = (row < rows).select(row, rows - 1) # rows past M read the last row; never stored + px = (lane % 8) ^ ((r8 // 2) + jx * 4) + x_src.append(row * (sx * 2) + px * 16) + x_dst.append((NR * 2 + d) * 1024) + aux_w = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), W_CM) + aux_x = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + + def issue(rs, dst, src, b, soff, aux): + d = fx.Int32(fx.rocdl.readfirstlane(T.i32, lds_base + (b % R) * SLOT + dst)) + lp = fx.to_llvm_ptr(fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), fx.Int64(d))) + _rocdl_ods.raw_ptr_buffer_load_async_lds( + rs, lp, fx.Int32(16).ir_value(), fx.Int32(src).ir_value(), + fx.Int32(soff).ir_value(), fx.Int32(0).ir_value(), aux=aux) + + def issue_w(q, b): + issue(wr, w_dst[q], w_src[q], b, b * 2048, aux_w) + + def issue_x(q, b): + issue(xr, x_dst[q], x_src[q], b, xsoff + b * 128, aux_x) + + def dma(b): + for q in range_constexpr(NRW * 2 // MH): + issue_w(q, b) + for q in range_constexpr(XPW): + issue_x(q, b) + + r8r = l16 % 8 + hh = l16 // 8 + x_off = [hh * 1024 + (r8r * 8 + ((kk * 4 + g) ^ ((r8r // 2) + hh * 4))) * 16 for kk in range_constexpr(2)] + + def rd(b, byte_off): + p = _lds_ptr(lds_base + (b % R) * SLOT + byte_off) + return Vec(llvm.load(T.vec(4, T.i32), p, alignment=16)).bitcast(fx.BFloat16) + + acc = [[Vec.filled(4, 0.0, fx.Float32) for _ in range_constexpr(MT)] for _ in range_constexpr(NRW)] + P = (MT + 1) // 2 + + def read_w(b, kk): + return [rd(b, ((wave + i * NWV) * 2 + kk) * 1024 + lane * 16) for i in range_constexpr(NRW)] + + def read_x(b, kk, lo, hi): + return [rd(b, (NR + mhalf * MT + m) * 2048 + x_off[kk]) for m in range_constexpr(lo, hi)] + + def mfmas(wv, xv, lo, hi, prefetch=None, operand=0): + for i in range_constexpr(NRW): + # spread the next DMAs between MFMA groups instead of one burst before the computation + if prefetch is not None: + if operand == 0: + for jq in range_constexpr(2 // MH): + issue_w(i * (2 // MH) + jq, prefetch) + elif i < XPW: + issue_x(i, prefetch) + _sb() + for m in range_constexpr(lo, hi): + acc[i][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), [wv[i], xv[m - lo], acc[i][m], 0, 0, 0])) + if prefetch is not None and operand == 1: + for q in range_constexpr(NRW, XPW): + issue_x(q, prefetch) + + def barrier(): + llvm.inline_asm(None, [], "s_barrier", "~{memory}", has_side_effects=True) + + # half-block software pipeline (kk 0 / 1 of each 64-K block); slot b is refilled with block b + R once every + # wave has read it (lgkmcnt(0) before the barrier) + for b in range_constexpr(min(R, NB)): + dma(b) + _sb() + fx.rocdl.s_waitcnt(vmcnt=(min(R, NB) - 1) * PER) + barrier() + _sb() + w_a = read_w(0, 0) + x_a0 = read_x(0, 0, 0, P) + x_a1 = read_x(0, 0, P, MT) + for b in range_constexpr(NB): + _sb() + w_b = read_w(b, 1) + x_b0 = read_x(b, 1, 0, P) + _sb() + mfmas(w_a, x_a0, 0, P) + _sb() + x_b1 = read_x(b, 1, P, MT) + _sb() + mfmas(w_a, x_a1, P, MT) + _sb() + if const_expr(b + 1 < NB): + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=(min(b + R - 1, NB - 1) - (b + 1)) * PER) + barrier() + _sb() + w_a = read_w(b + 1, 0) + x_a0 = read_x(b + 1, 0, 0, P) + _sb() + mfmas(w_b, x_b0, 0, P, (b + R) if b + R < NB else None, 0) + _sb() + if const_expr(b + 1 < NB): + x_a1 = read_x(b + 1, 0, P, MT) + _sb() + mfmas(w_b, x_b1, P, MT, (b + R) if b + R < NB else None, 1) + + for i in range_constexpr(NRW): + for m in range_constexpr(MT): + r = m0 + (mhalf * MT + m) * 16 + l16 + c = (nt * NR + wave + i * NWV) * 16 + g * 4 + buffer_ops.buffer_store(acc[i][m], pr, (s * rows + r) * N + c, mask=r < rows) + + @flyc.jit + def launch_bf16_concat_linear_lds_n3072_k30720(x0: fx.Int64, x1: fx.Int64, x2: fx.Int64, x3: fx.Int64, + x4: fx.Int64, weight: fx.Int64, partials: fx.Int64, + rows: fx.Int32, sx0: fx.Int32, sx1: fx.Int32, sx2: fx.Int32, + sx3: fx.Int32, sx4: fx.Int32, stream: fx.Stream): + kernel(x0, x1, x2, x3, x4, weight, partials, rows, sx0, sx1, sx2, sx3, sx4).launch( + grid=(fx.Int64(((rows + TM - 1) // TM) * NTILES * split), 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + launch_bf16_concat_linear_lds_n3072_k30720.kernel_name = name + return launch_bf16_concat_linear_lds_n3072_k30720 diff --git a/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel_staged_a_eightwave.py b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel_staged_a_eightwave.py new file mode 100644 index 0000000..df4b16b --- /dev/null +++ b/amd_kernels/gemm/bf16_concat_linear_n3072_k30720/kernel_staged_a_eightwave.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Five bf16 inputs read in place, whole output-column shard, fp32 split-K slabs.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +N,K,H,NW=3072,30720,6144,8 + + +def compile_concat_linear_staged_a_eightwave(mt,nt,split,unroll=2): + assert mt%NW==0 and unroll==2 + assert K % (32*split)==0 and N%(16*nt*NW)==0 + steps=K//32//split + assert steps%unroll==0 and (H//32)%unroll==0 + nblocks=N//(16*nt*NW) + name=f'bf16_concat_linear_n3072_k30720_vec4_staged_a_eightwave_u{unroll}_mt{mt}_nt{nt}_sk{split}' + def mfma(a,b,c): + return Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4,fx.Float32),[a.bitcast(fx.BFloat16),b.bitcast(fx.BFloat16),c,0,0,0])) + + @fx.struct + class Shared: + a:fx.Array[fx.Int32,2*mt*64*4,16] + + @flyc.kernel(name=name,known_block_size=[64*NW,1,1]) + def kernel(x0:fx.Int64,x1:fx.Int64,x2:fx.Int64,x3:fx.Int64,x4:fx.Int64, + weight:fx.Int64,partials:fx.Int64,rows:fx.Int32, + sx0:fx.Int32,sx1:fx.Int32,sx2:fx.Int32,sx3:fx.Int32,sx4:fx.Int32): + tx,bid=fx.thread_idx.x,fx.block_idx.x + sm=fx.SharedAllocator().allocate(Shared).peek() + av16=fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32,sm.a.ptr),fx.make_layout(2*mt*64*4,1)),fx.make_layout(4,1)) + atom=fx.make_copy_atom(fx.UniversalCopy128b(),fx.Int32) + lane=tx%64 + wave=fx.Int32(fx.rocdl.readfirstlane(T.i32,tx//64)) + s=bid%split + nb=(bid//split)%nblocks + mb=bid//(split*nblocks) + wr=buffer_ops.create_buffer_resource_from_addr(weight,num_records_bytes=N*K*2) + pr=buffer_ops.create_buffer_resource_from_addr(partials,num_records_bytes=fx.Int64(rows)*(N*split*4)) + bases=(x0,x1,x2,x3,x4) + strides=(sx0,sx1,sx2,sx3,sx4) + acc=[Vec.filled(4,0.0,fx.Float32) for _ in range_constexpr(mt*nt)] + for tap in range_constexpr(5): + sx=strides[tap] + xr=buffer_ops.create_buffer_resource_from_addr(bases[tap],num_records_bytes=fx.Int64(rows)*fx.Int64(sx*2)) + first=(s*steps>tap*(H//32)).select(s*steps,fx.Int32(tap*(H//32))) + last=((s+1)*steps<(tap+1)*(H//32)).select((s+1)*steps,fx.Int32((tap+1)*(H//32))) + span=(last>first).select(last-first,fx.Int32(0)) + def operands(step,valid): + kk=step*32 + av=[Vec(buffer_ops.buffer_load(wr,((nb*NW*nt+wave*nt+j)*(K//32)+step)*256+lane*4, + vec_width=4,dtype=fx.Int32,mask=valid)) for j in range_constexpr(nt)] + local=(kk-tap*H) + bv=[Vec(buffer_ops.buffer_load(xr,((mb*mt*16)+(tx//4)+i*(NW*16))*(sx//2)+(local+(tx%4)*8)//2, + vec_width=4,dtype=fx.Int32,mask=valid&(((mb*mt*16)+(tx//4)+i*(NW*16))0) + result=init + for it,old in range(fx.Index(0),fx.Index(span//unroll),fx.Index(1),init=[fx.as_ir_value(v) for v in init]): + current=old + for u in range_constexpr(unroll): + step=fx.Int32(it)*unroll+u + safe=(step+1=0.99999, relative L2 <=1e-3 and bitwise repeatability. +""" +import torch + + +def concat_linear_reference(taps,weight): + return (torch.cat(taps,dim=1).double()@weight.double().T).bfloat16() + diff --git a/amd_kernels/gemm/bf16_n100032_k6144/README.md b/amd_kernels/gemm/bf16_n100032_k6144/README.md new file mode 100644 index 0000000..b1d7754 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/README.md @@ -0,0 +1,57 @@ +# bf16 GEMM, N 100032 x K 6144 (the TP2 lm_head vocabulary shard) + +## What it does +`out [M, 100032] = x [M, 6144] @ W^T`, bf16 operands, fp32 sums in ascending K, one bf16 rounding: the TP2 vocabulary +shard of the MiniMax-M3.1-Flash lm_head (`lm_head.weight` bf16 [200064, 6144], rank r holds rows +[100032 r, 100032 (r + 1))). One tensor serves the target lm_head (verify rows) and the DSpark draft lm_head +(requests x 7 rows). W layout: `row` (the checkpoint's shard) or `shuf` (a bit-exact permutation once at load, so one +DMA reads 1 KB in one piece); both give the same bits. No split-K, partials or output atomics; every tile variant +gives the same bits (one wave's K-ascending MFMA chain per output). Contract (`reference.py`): the fp64 product +rounded once to bf16; per-row cosine >= 0.99999, rel. L2 <= 1e-3, deterministic; NaN / inf propagate as in an fp32 +sum. Caller: `sampling/batch_sharded_markov_v200064_h6144_r256_ranks2` (target and draft logits). + +## Entry points +- `shuffle_weight`, `shuffle_weight_` (in place), `unshuffle_weight`: the `shuf` layout; replace the parameter once + before graph capture, keep the target / draft alias, free the row-major copy. +- `bf16_gemm_n100032_k6144`: the logits of any row count (launches of 2048 rows), graph-safe, no host sync; `out` may + be a column slice of a wider row-major tensor (a vocabulary half of [rows, 200064]). With a device row map: the + mapped rows in map order (`rows`), or (with an exchange slot of `collectives/all_to_all_bf16_n100032_ranks2`) the + mapped rows written into the send slot and published, the caller then runs the exchange's pull. An + `amd_kernels.warm_up` entry: the first call of each mode (layout x contiguous / rows / publish) compiles its 15 tile + variants. +- `warmup_bf16_gemm_n100032_k6144`: the same warm-up for one mode, explicitly (publish: a private slot). +- `supports`: whether a weight is the [100032, 6144] bf16 contiguous shard (either layout) and there is no bias. + +`host.tile_config` picks the kernel, tiles, waves and ring by row count. + +## Performance +One MI355X. Graph replay, `shuf`, 50 distinct inputs (the 1.23 GB weight exceeds the MALL), ten replays, five +alternated rounds, median µs. vLLM = `rocm_unquantized_gemm_impl` on the row-major shard (wvSplitK at 1 row, +hipBLASLt / AITER above; VLLM_ROCM_USE_AITER=1). Roofline = max(bytes / 6.5 TB/s, flops / 2.5 PF). `publish` = the +mapped-row kernels with system-scope stores into the exchange slot and the ticket (the `rows` form stores plainly). + +| M | vLLM | contiguous | publish | roofline | contiguous / roofline | +|---:|---:|---:|---:|---:|---:| +| 1 | 182.8 | 176.9 | 178.4 | 189.1 | 1.07 | +| 8 | 199.4 | 177.4 | 179.1 | 189.4 | 1.07 | +| 16 | 207.6 | 177.9 | 180.0 | 189.6 | 1.07 | +| 64 | 254.4 | 184.1 | 188.8 | 191.2 | 1.04 | +| 96 | 268.7 | 195.0 | 198.1 | 192.2 | 0.99 | +| 128 | 278.6 | 205.7 | 211.2 | 193.3 | 0.94 | +| 144 | 292.0 | 241.0 | 243.0 | 193.8 | 0.80 | +| 160 | 297.4 | 255.0 | 262.1 | 194.3 | 0.76 | +| 192 | 304.9 | 280.3 | 289.0 | 195.4 | 0.70 | +| 224 | 315.5 | 308.9 | 310.3 | 196.4 | 0.64 | +| 256 | 325.6 | 332.3 | 335.1 | 197.5 | 0.59 | +| 320 | 345.0 | 448.2 | 459.5 | 199.6 | 0.45 | +| 384 | 391.3 | 505.7 | 516.7 | 201.7 | 0.40 | +| 512 | 570.4 | 592.7 | 596.3 | 251.7 | 0.42 | +| 1024 | 951.2 | 1035.0 | 1044.2 | 503.5 | 0.49 | +| 2048 | 1927.9 | 2042.1 | 2058.4 | 1007.0 | 0.49 | + +It beats vLLM through 224 rows (the owned rows of up to 56 requests in batch-sharded sampling); from 256 rows +hipBLASLt / AITER are faster. + +## Tests +- `python3 -m pytest tests/gemm/bf16_n100032_k6144 -q` (`test_publish.py`: 2 GPUs) +- `python3 benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py [M ...]` diff --git a/amd_kernels/gemm/bf16_n100032_k6144/__init__.py b/amd_kernels/gemm/bf16_n100032_k6144/__init__.py new file mode 100644 index 0000000..332ee03 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +from .host import (bf16_gemm_n100032_k6144, shuffle_weight, shuffle_weight_, supports, + unshuffle_weight, warmup_bf16_gemm_n100032_k6144) + +__all__ = ["bf16_gemm_n100032_k6144", "shuffle_weight", "shuffle_weight_", "supports", + "unshuffle_weight", "warmup_bf16_gemm_n100032_k6144"] diff --git a/amd_kernels/gemm/bf16_n100032_k6144/host.py b/amd_kernels/gemm/bf16_n100032_k6144/host.py new file mode 100644 index 0000000..0e95d0c --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/host.py @@ -0,0 +1,220 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Host side of the lm_head shard GEMM: the load-time weight layout, the tile by M, the launch, the warm-up.""" +import functools + +import torch + +from amd_kernels.warm_up import entry + +from .kernel import compile_bf16_gemm_n100032_k6144 +from .indexed import compile_indexed_bf16_gemm_n100032_k6144 +from .large import compile_large_bf16_gemm_n100032_k6144 +from .large_indexed import compile_large_indexed_bf16_gemm_n100032_k6144 +from .reference import K, N + +MAX_M = 2048 + + +def supports(w: torch.Tensor, bias: torch.Tensor | None = None) -> bool: + """Whether ``w`` is the [100032, 6144] bf16 contiguous shard this GEMM (and ``shuffle_weight_``) takes; the GEMM + has no bias.""" + return bias is None and tuple(w.shape) == (N, K) and w.dtype == torch.bfloat16 and w.is_contiguous() + + +def shuffle_weight(w: torch.Tensor) -> torch.Tensor: + """The load-time layout [6252 groups, 96 K blocks, 16 rows, 64 K] of the [100032, 6144] shard, as [100032, 6144]. + + Bit-exact (a permutation). The caller replaces the parameter with it once, before graph capture, and frees the + row-major tensor (no second copy); every reader of the weight then calls bf16_gemm_n100032_k6144(shuffled=True). + """ + assert w.shape == (N, K) and w.dtype == torch.bfloat16 + return w.view(N // 16, 16, K // 64, 64).permute(0, 2, 1, 3).contiguous().view(N, K) + + +def shuffle_weight_(w: torch.Tensor) -> torch.Tensor: + """shuffle_weight in place: ``w`` keeps its storage and holds the load-time layout (the temporary is freed).""" + ws = shuffle_weight(w) + w.copy_(ws) + del ws + return w + + +def unshuffle_weight(ws: torch.Tensor) -> torch.Tensor: + """The inverse of shuffle_weight (row-major [100032, 6144]).""" + assert ws.shape == (N, K) and ws.dtype == torch.bfloat16 + return ws.view(N // 16, K // 64, 16, 64).permute(0, 2, 1, 3).contiguous().view(N, K) + + +def tile_config(m: int) -> tuple: + """(kernel, 16-row fragments per wave, M halves, vocabulary tiles, ring) for m rows; the bounds and tiles are the + fastest of the graph-replay sweep over 1-2048 rows (README). Above 256 rows the M tiles of one vocabulary tile + run on one XCD (the weight tile is re-read from L2); their fragment counts are balanced.""" + if m <= 96: + return "small", (m + 15) // 16, 1, 256, 0 + if m <= 128: + return "small", (m + 31) // 32, 2, 256, 0 + if m <= 144: + return "small", 9, 1, 256, 2 + mtiles = (m + 255) // 256 + mt = (m + 32 * mtiles - 1) // (32 * mtiles) + if m <= 192 or 256 < m <= 384: + return "small", mt, 2, 512, 3 + return "large", mt, 2, (400 if m <= 224 else 512 if m <= 512 else 448), 0 + + +FORMS = ("contiguous", "rows", "publish") + + +@functools.cache +def _kernel(cfg: tuple, shuf: bool, form: str): + """The tile variant: contiguous rows, mapped rows (``rows``), or mapped rows published into the exchange.""" + kind, mt, mh, tiles, ring = cfg + if form != "contiguous": + if kind == "large": + return compile_large_indexed_bf16_gemm_n100032_k6144(mt, int(shuf), tiles, publish=form == "publish") + return compile_indexed_bf16_gemm_n100032_k6144(mt, int(shuf), mh, tiles, ring, publish=form == "publish") + if kind == "large": + return compile_large_bf16_gemm_n100032_k6144(mt, int(shuf), tiles) + return compile_bf16_gemm_n100032_k6144(mt, int(shuf), mh, tiles, ring) + + +def _tiling(m: int, shuffled: bool, form: str): + cfg = tile_config(m) + rows = 16 * cfg[1] * cfg[2] + return _kernel(cfg, bool(shuffled), form), (m + rows - 1) // rows + + +def _check_out(out: torch.Tensor, m: int): + """A row-major [m, 100032] bf16 view whose rows may be wider (a column slice): row stride in dwords.""" + assert out.shape == (m, N) and out.dtype == torch.bfloat16 and out.stride(1) == 1 + assert out.data_ptr() % 4 == 0 and (m <= 1 or out.stride(0) % 2 == 0) and out.stride(0) >= N + assert m * out.stride(0) * 2 <= 0xFFFFFFFF + return out.stride(0) // 2 if m > 1 else N // 2 + + +def _run(exe, *args): + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +def _bf16_gemm(x: torch.Tensor, w: torch.Tensor, out: torch.Tensor | None = None, *, + shuffled: bool = False, row_indices: torch.Tensor | None = None, + publish: tuple | None = None) -> torch.Tensor | None: + """out [M, 100032] bf16 = x [M, 6144] bf16 @ W^T, fp32 accumulate; W = w (row-major) or shuffle_weight's layout. + + Any M (above 2048 rows: launches of 2048 rows each, the same bits as separate calls). Deterministic, no atomic + output adds. CUDA-graph safe (no host sync, no allocation when out is given). ``out`` may be a column slice of a + wider row-major tensor (unit column stride, even row stride). The first call of each mode (layout, form) + compiles the mode's variants (``amd_kernels.warm_up``). + + row_indices (contiguous device int64 [M], each index in [0, len(x)), duplicates valid, re-read on every graph + replay, 1 <= M <= 2048): the rows x[row_indices], in map order (form ``rows``); the same bits as the contiguous + call on x[row_indices]. + publish = RowAllToAll.slot() of all_to_all_bf16_n100032_ranks2, with row_indices and no out: the mapped rows go + straight into the exchange's send slot and the kernel publishes them (no stage copy); the caller then runs + RowAllToAll.pull(M, counts, out). Returns None. + """ + assert publish is None or row_indices is not None, "publish needs row_indices" + assert x.ndim == 2 and x.shape[1] == K and x.dtype == torch.bfloat16 and x.is_contiguous() + assert w.shape == (N, K) and w.dtype == torch.bfloat16 and w.is_contiguous() and w.device == x.device + stream = torch.cuda.current_stream() + if row_indices is not None: + m = row_indices.numel() + assert row_indices.ndim == 1 and row_indices.dtype == torch.int64 and row_indices.is_contiguous() + assert row_indices.device == x.device and x.numel() * x.element_size() <= 0xFFFFFFFF and 1 <= m <= MAX_M + if publish is not None: + pool, slot_bytes, counter, flag = publish + assert out is None and m * N * 2 <= slot_bytes + assert counter.dtype == torch.int32 and counter.device == x.device + exe, mtiles = _tiling(m, shuffled, "publish") + _run(exe, x.data_ptr(), w.data_ptr(), pool, row_indices, x.shape[0], m, mtiles, counter, slot_bytes, + flag, stream) + return None + if out is None: + out = torch.empty(m, N, dtype=torch.bfloat16, device=x.device) + ldy = _check_out(out, m) + assert out.device == x.device + exe, mtiles = _tiling(m, shuffled, "rows") + _run(exe, x.data_ptr(), w.data_ptr(), out.data_ptr(), row_indices, x.shape[0], m, mtiles, ldy, stream) + return out + m = x.shape[0] + if out is None: + out = torch.empty(m, N, dtype=torch.bfloat16, device=x.device) + if m > MAX_M: + for i in range(0, m, MAX_M): + _bf16_gemm(x[i:i + MAX_M], w, out[i:i + MAX_M], shuffled=shuffled) + return out + ldy = _check_out(out, m) + assert out.device == x.device + if m: + exe, mtiles = _tiling(m, shuffled, "contiguous") + _run(exe, x.data_ptr(), w.data_ptr(), out.data_ptr(), m, mtiles, ldy, stream) + return out + + +def _form(row_indices, publish) -> str: + return "contiguous" if row_indices is None else ("rows" if publish is None else "publish") + + +def _mode(x, w, out=None, *, shuffled=False, row_indices=None, publish=None): + """What a call fixes about its kernels besides M (``amd_kernels.warm_up``): the weight layout and the call form.""" + return bool(shuffled), _form(row_indices, publish) + + +_KEEP: list = [] # the private publish slot of the warm-up call being made + + +def _inputs(mode, n, x, w, out=None, *, shuffled=False, row_indices=None, publish=None): + """One warm-up call at ``n`` rows (zeros, row 0) into a private output. A variant compiled before the warm-up + launches even in compile-only mode, so a publish call gets a private slot pair, counter and flag, never the + caller's exchange.""" + if mode[1] == "contiguous": + return (torch.zeros(n, K, dtype=torch.bfloat16, device=x.device), w), {"shuffled": shuffled} + idx = torch.zeros(n, dtype=torch.int64, device=x.device) + if mode[1] == "rows": + return (x, w), {"shuffled": shuffled, "row_indices": idx} + slot_bytes = n * N * 2 + _KEEP[:] = [torch.empty(2 * slot_bytes, dtype=torch.uint8, device=x.device), + torch.zeros(1, dtype=torch.int32, device=x.device), + torch.zeros(16, dtype=torch.int32, device=x.device)] + publish = (_KEEP[0].data_ptr(), slot_bytes, _KEEP[1], _KEEP[2].data_ptr()) + return (x, w), {"shuffled": shuffled, "row_indices": idx, "publish": publish} + + +# one row count per tile variant (tile_config); every other row count up to MAX_M reuses one of these +WARMUP_ROWS = sorted({tile_config(m): m for m in range(MAX_M, 0, -1)}.values()) +WARMUP_MODES = FORMS + +bf16_gemm_n100032_k6144 = entry("gemm/bf16_n100032_k6144", mode=_mode, sizes=lambda mode: WARMUP_ROWS, + inputs=_inputs)(_bf16_gemm) + + +def warmup_bf16_gemm_n100032_k6144(device: torch.device, shuffled: bool, mode: str = "contiguous") -> None: + """Compile the tile variants of one call mode for the caller's weight layout, before graph capture. + + Each mode is its own kernel, so warm up only the modes the caller runs (once per mode): + contiguous = bf16_gemm_n100032_k6144(x, w), rows = row_indices=..., publish = row_indices=..., publish=... + """ + assert mode in WARMUP_MODES, mode + w = torch.zeros(N, K, dtype=torch.bfloat16, device=device) + x = torch.zeros(max(WARMUP_ROWS), K, dtype=torch.bfloat16, device=device) + indices = torch.arange(x.shape[0], dtype=torch.int64, device=device) + publish = None + if mode == "publish": # a private scratch slot pair, counter and flag (no peer is involved) + slot_bytes = x.shape[0] * N * 2 + pool = torch.empty(2 * slot_bytes, dtype=torch.uint8, device=device) + counter = torch.zeros(1, dtype=torch.int32, device=device) + flag = torch.zeros(16, dtype=torch.int32, device=device) + publish = (pool.data_ptr(), slot_bytes, counter, flag.data_ptr()) + for m in WARMUP_ROWS: + if mode == "contiguous": + bf16_gemm_n100032_k6144(x[:m], w, shuffled=shuffled) + else: + bf16_gemm_n100032_k6144(x, w, shuffled=shuffled, row_indices=indices[:m], publish=publish) + torch.cuda.synchronize(device) diff --git a/amd_kernels/gemm/bf16_n100032_k6144/indexed.py b/amd_kernels/gemm/bf16_n100032_k6144/indexed.py new file mode 100644 index 0000000..2682dbc --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/indexed.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Indexed activation variant of the small-tile BF16 lm_head GEMM (kernel.py's tiles). + +The device's int64 row map selects each activation row before its LDS DMA. +The epilogue writes in map order, so the target hidden gather and the draft +owner-sorting copy disappear. The map is re-read on every graph replay. +Weight layout and fixed K-ascending sums match the contiguous variant. +The output goes into a row-major tensor of any even row stride (`rows`) or into +all_to_all_bf16_n100032_ranks2's send slot (`publish`, publish.py). +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ods +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .publish import SYS_CM, flag_add +from .reference import K, N + +TILES = 512 # vocabulary tiles per M tile: two rounds of the 256 CUs, 12-13 groups each +G16 = N // 16 # 6252 16-row groups +NWV = 4 # waves +NB = K // 64 # 96 K blocks +W_CM = 2 # nt +MT_MAX = 16 + + +def ring_depth(mt: int, wf: int) -> int: + """Ring slots that fit the 160 KB LDS (one slot = (wf + mt) x 2 KB), at most 4.""" + return min(4, 160 // ((wf + mt) * 2)) + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _sb(): + fx.rocdl.sched_barrier(0) + + +def _lds_ptr(byte_addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(byte_addr))) + + +def compile_indexed_bf16_gemm_n100032_k6144(mt: int, shuf: int, mh: int = 1, tiles: int = TILES, ring: int = 0, publish: bool = True): + NRB = G16 // tiles + NBIG = G16 - NRB * tiles # the first NBIG tiles take one more group + NR = NRB + 1 + NRW = (NR + NWV - 1) // NWV # group slots per wave + # MH: M halves; MH 2 = 8 waves (2 per SIMD), wave w takes the groups of w % 4 and the M fragments of half w // 4 + MT, SHUF, MH = mt, shuf, mh + NWT = NWV * MH # waves per workgroup + MTT = MT * MH # activation fragments per workgroup + R = ring or (ring_depth(MTT, NR + 1) if MH == 1 else 2) + TM = 16 * MTT + XD = MTT * 2 # activation DMAs per block (2 per 16-row fragment) + XPW = (XD + NWT - 1) // NWT + PER = NRW * 2 // MH + XPW # DMAs per wave per block (MH 2: each half issues one of a group's two DMAs) + WF = NR + 1 # weight positions per slot: 13 groups + the junk position + SLOT = (WF + MTT) * 2048 + LDS = R * SLOT + assert 1 <= MT <= MT_MAX and LDS <= 160 * 1024 and (R - 1) * PER <= 63, (LDS, R, PER) + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, LDS, 16] + + name = f"bf16_gemm_n100032_k6144_indexed_t{tiles}_tm{TM}_r{R}" + ("_shuf" if SHUF else "_row") + ("_w8" if MH == 2 else "") + name = name + ("_publish" if publish else "_rows") + + def body(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy, y_cm): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave_all = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wave = wave_all % NWV # the weight groups + mhalf = wave_all // NWV # the M half (0 when MH 1) + # workgroup b runs on XCD b % 8; the M tiles of one vocabulary tile follow each other on that XCD + xcd = bid % 8 + j = bid // 8 + t = (j // i32_mtiles) * 8 + xcd + m0 = (j % i32_mtiles) * TM + big = t < NBIG + n0 = big.select(t * (16 * NR), t * (16 * NRB) + NBIG * 16) + nfrag = big.select(fx.Int32(NR), fx.Int32(NRB)) + g = lane // 16 + l16 = lane % 16 + r8 = lane // 8 + + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=i32_xrows * (K * 2)) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K * 2) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y, num_records_bytes=i32_m * i32_ldy * 4) + lds_base = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + + # DMA jj of a fragment: rows 8 jj + r8, 16-byte piece (lane % 8) ^ ((r8 >> 1) | (jj << 2)) of the line + w_src, w_dst = [], [] + for i in range_constexpr(NRW): + rn = wave + i * NWV + ok = rn < nfrag + for jq in range_constexpr(2 // MH): + jj = mhalf + jq if MH == 2 else jq + pc = (lane % 8) ^ ((r8 // 2) + jj * 4) + if const_expr(SHUF): + off = (n0 // 16 + rn) * (K * 32) + (jj * 8 + r8) * 128 + pc * 16 + else: + off = (n0 + rn * 16 + jj * 8 + r8) * (K * 2) + pc * 16 + w_src.append(ok.select(off, fx.Int32(0x70000000))) # past the tile: zeros, no traffic + w_dst.append((ok.select(rn, fx.Int32(NR)) * 2 + jj) * 1024) + x_src, x_dst = [], [] + for i in range_constexpr(XPW): + d = wave_all + i * NWT + d = (d < XD).select(d, fx.Int32(XD - 1)) # spare DMAs repeat the last one (same bytes, same place) + jx = d % 2 + row = m0 + (d // 2) * 16 + jx * 8 + r8 + row = (row < i32_m).select(row, i32_m - 1) # rows past M read the last row; never stored + row = fx.Int32(arg_indices[row]) + px = (lane % 8) ^ ((r8 // 2) + jx * 4) + x_src.append(row * (K * 2) + px * 16) + x_dst.append((WF * 2 + d) * 1024) + aux_w = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), W_CM) + aux_x = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + + def issue(rs, dst, src, b, soff, aux): + d = fx.Int32(fx.rocdl.readfirstlane(T.i32, lds_base + (b % R) * SLOT + dst)) + lp = fx.to_llvm_ptr(fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), fx.Int64(d))) + _rocdl_ods.raw_ptr_buffer_load_async_lds( + rs, lp, fx.Int32(16).ir_value(), fx.Int32(src).ir_value(), + fx.Int32(soff).ir_value(), fx.Int32(0).ir_value(), aux=aux) + + def dma(b): + for q in range_constexpr(NRW * 2 // MH): + issue(wr, w_dst[q], w_src[q], b, b * (2048 if SHUF else 128), aux_w) + for q in range_constexpr(XPW): + issue(xr, x_dst[q], x_src[q], b, b * 128, aux_x) + + wpos = [(wave + i * NWV < NR).select(wave + i * NWV, fx.Int32(NR)) for i in range_constexpr(NRW)] + r8r = l16 % 8 + hh = l16 // 8 + rd_off = [hh * 1024 + (r8r * 8 + ((kk * 4 + g) ^ ((r8r // 2) + hh * 4))) * 16 for kk in range_constexpr(2)] + + def rd(b, pos, kk): + p = _lds_ptr(lds_base + (b % R) * SLOT + pos * 2048 + rd_off[kk]) + return Vec(llvm.load(T.vec(4, T.i32), p, alignment=16)).bitcast(fx.BFloat16) + + acc = [[Vec.filled(4, 0.0, fx.Float32) for _ in range_constexpr(MT)] for _ in range_constexpr(NRW)] + + def read_ops(b, kk): + return ([rd(b, wpos[i], kk) for i in range_constexpr(NRW)], + [rd(b, WF + mhalf * MT + m, kk) for m in range_constexpr(MT)]) + + def mfmas(ops): + wv, xv = ops + for i in range_constexpr(NRW): + for m in range_constexpr(MT): + acc[i][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), [wv[i], xv[m], acc[i][m], 0, 0, 0])) + + def barrier(): + llvm.inline_asm(None, [], "s_barrier", "~{memory}", has_side_effects=True) + + # half-block software pipeline: the LDS reads of (b, kk 1) overlap the MFMAs of (b, kk 0), those of + # (b + 1, kk 0) the MFMAs of (b, kk 1); slot b is refilled (block b + R) once every wave has read it + # (lgkmcnt(0) before the barrier), so R blocks are in flight + for b in range_constexpr(R): + dma(b) + _sb() + fx.rocdl.s_waitcnt(vmcnt=(R - 1) * PER) + barrier() + _sb() + ops_a = read_ops(0, 0) + for b in range_constexpr(NB): + _sb() + ops_b = read_ops(b, 1) + _sb() + mfmas(ops_a) + _sb() + if b + 1 < NB: + # blocks issued so far: 0 .. min(b + R - 1, NB - 1); block b + 1 must have landed + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=(min(b + R - 1, NB - 1) - (b + 1)) * PER) + barrier() + _sb() + ops_a = read_ops(b + 1, 0) + _sb() + if b + R < NB: + dma(b + R) + _sb() + mfmas(ops_b) + + for i in range_constexpr(NRW): + rn = wave + i * NWV + n = n0 + rn * 16 + g * 4 + for m in range_constexpr(MT): + r = m0 + (mhalf * MT + m) * 16 + l16 + pk = acc[i][m].to(fx.BFloat16).bitcast(fx.Int32) + buffer_ops.buffer_store(pk, yr, r * i32_ldy + n // 2, mask=(rn < nfrag) & (r < i32_m), + cache_modifier=y_cm) + + # Publish: the output goes to the exchange's slot (counter[0] & 1) with system-scope stores, and the last + # workgroup publishes it (all_to_all_bf16_n100032_ranks2's stage_publish protocol: the same ticket and flag words). + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel_publish(arg_x: fx.Int64, arg_w: fx.Int64, arg_pool: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, counter: fx.Tensor, + i64_slot_bytes: fx.Int64, arg_flag: fx.Int64): + slot = fx.Int32(counter[0]) & 1 + body(arg_x, arg_w, arg_pool + fx.Int64(slot) * i64_slot_bytes, arg_indices, i32_xrows, i32_m, i32_mtiles, + fx.Int32(N // 2), SYS_CM) + # every workgroup's stores land before its ticket; the last of them publishes (dynamic ifs stay in the + # kernel body: FlyDSL rewrites only this function's control flow) + nwg = tiles * i32_mtiles + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if fx.thread_idx.x == 0: + ticket = flag_add(arg_flag + 4, 1) + if ticket == nwg - 1: + flag_add(arg_flag + 4, -nwg) + seq = fx.Int32(counter[0]) + 1 + counter[0] = seq + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=64) + buffer_ops.buffer_store(seq, fr, 0, cache_modifier=SYS_CM) + + # Rows: plain stores into a row-major output of row stride i32_ldy dwords (a column slice of a wider tensor). + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel_rows(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, arg_indices: fx.Tensor, i32_xrows: fx.Int32, + i32_m: fx.Int32, i32_mtiles: fx.Int32, i32_ldy: fx.Int32): + body(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy, 0) + + @flyc.jit + def launch_indexed_bf16_gemm_n100032_k6144_rows(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, i32_ldy: fx.Int32, + stream: fx.Stream): + kernel_rows(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy).launch( + grid=(tiles * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + @flyc.jit + def launch_indexed_bf16_gemm_n100032_k6144_publish(arg_x: fx.Int64, arg_w: fx.Int64, arg_pool: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, counter: fx.Tensor, + i64_slot_bytes: fx.Int64, arg_flag: fx.Int64, stream: fx.Stream): + kernel_publish(arg_x, arg_w, arg_pool, arg_indices, i32_xrows, i32_m, i32_mtiles, counter, i64_slot_bytes, + arg_flag).launch(grid=(tiles * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + return launch_indexed_bf16_gemm_n100032_k6144_publish if publish else launch_indexed_bf16_gemm_n100032_k6144_rows diff --git a/amd_kernels/gemm/bf16_n100032_k6144/kernel.py b/amd_kernels/gemm/bf16_n100032_k6144/kernel.py new file mode 100644 index 0000000..b837128 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/kernel.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""bf16 [M, 6144] x bf16 [100032, 6144]^T -> bf16 [M, 100032], fp32 accumulate (the TP2 lm_head vocabulary shard). + +Tiles: the 6252 16-row vocabulary groups go to 256 tiles of 24-25 groups (one CU round) or to 512 tiles of 12-13 +(two rounds); the first 108 tiles take one more group; wave w takes the groups w, w + 4, ... of its tile +(`host.tile_config` picks tiles, waves and ring by rows). A workgroup covers all rows of its M tile (16 MT rows per +M half; the M tiles of one vocabulary tile run back to back on one XCD, so the weight tile is re-read from L2). + +Data path: per 64-K block, both operands go HBM / L2 -> LDS by asynchronous `buffer_load ... lds` (no VGPRs spent +on bytes in flight) into a ring of R slots. One slot = the tile's NR weight fragments (+ one junk position that +takes the out-of-bounds zeros of a missing last group) + the MT activation fragments, 2 KB each (16 rows x 128 B). +One DMA moves 8 rows x a 128-byte line; lane l writes bytes [16 l, 16 l + 16) of its 1 KB, and the global piece +it fetches is XOR-swizzled so that the MFMA lane (row l16, K group g) reads its 16 bytes conflict-free. The waves +share the activation slot: before block b + 1 each wave waits for its own DMAs (vmcnt) and its LDS reads +(lgkmcnt 0), one `s_barrier`, then refills slot b (block b + R). Half-block software pipeline: the ds_reads of +(b, K half 1) overlap the MFMAs of (b, half 0), those of (b + 1, half 0) the MFMAs of (b, half 1) +(v_mfma_f32_16x16x32_bf16, K in ascending order). +Weight loads are non-temporal (`nt`): the shard (1229 MB) streams through. + +Weight layouts (one tensor serves every M and both heads): +- ROW: the checkpoint's [100032, 6144] row-major shard; +- SHUF: `shuffle_weight`'s load-time layout [6252 groups, 96 K blocks, 16 rows, 64 K]: every DMA reads 1 KB in one + piece and a group's bytes follow each other block after block (bit-exact transform, one copy). + +Every output element is one wave's fp32 sum in K order (no split-K, no atomics), rounded once to bf16. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ods +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .reference import K, N + +TILES = 512 # vocabulary tiles per M tile: two rounds of the 256 CUs, 12-13 groups each +G16 = N // 16 # 6252 16-row groups +NWV = 4 # waves +NB = K // 64 # 96 K blocks +W_CM = 2 # nt +MT_MAX = 16 + + +def ring_depth(mt: int, wf: int) -> int: + """Ring slots that fit the 160 KB LDS (one slot = (wf + mt) x 2 KB), at most 4.""" + return min(4, 160 // ((wf + mt) * 2)) + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _sb(): + fx.rocdl.sched_barrier(0) + + +def _lds_ptr(byte_addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(byte_addr))) + + +def compile_bf16_gemm_n100032_k6144(mt: int, shuf: int, mh: int = 1, tiles: int = TILES, ring: int = 0): + NRB = G16 // tiles + NBIG = G16 - NRB * tiles # the first NBIG tiles take one more group + NR = NRB + 1 + NRW = (NR + NWV - 1) // NWV # group slots per wave + # MH: M halves; MH 2 = 8 waves (2 per SIMD), wave w takes the groups of w % 4 and the M fragments of half w // 4 + MT, SHUF, MH = mt, shuf, mh + NWT = NWV * MH # waves per workgroup + MTT = MT * MH # activation fragments per workgroup + R = ring or (ring_depth(MTT, NR + 1) if MH == 1 else 2) + TM = 16 * MTT + XD = MTT * 2 # activation DMAs per block (2 per 16-row fragment) + XPW = (XD + NWT - 1) // NWT + PER = NRW * 2 // MH + XPW # DMAs per wave per block (MH 2: each half issues one of a group's two DMAs) + WF = NR + 1 # weight positions per slot: 13 groups + the junk position + SLOT = (WF + MTT) * 2048 + LDS = R * SLOT + assert 1 <= MT <= MT_MAX and LDS <= 160 * 1024 and (R - 1) * PER <= 63, (LDS, R, PER) + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, LDS, 16] + + name = f"bf16_gemm_n100032_k6144_t{tiles}_tm{TM}_r{R}" + ("_shuf" if SHUF else "_row") + ("_w8" if MH == 2 else "") + + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_m: fx.Int32, i32_mtiles: fx.Int32, + i32_ldy: fx.Int32): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave_all = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wave = wave_all % NWV # the weight groups + mhalf = wave_all // NWV # the M half (0 when MH 1) + # workgroup b runs on XCD b % 8; the M tiles of one vocabulary tile follow each other on that XCD + xcd = bid % 8 + j = bid // 8 + t = (j // i32_mtiles) * 8 + xcd + m0 = (j % i32_mtiles) * TM + big = t < NBIG + n0 = big.select(t * (16 * NR), t * (16 * NRB) + NBIG * 16) + nfrag = big.select(fx.Int32(NR), fx.Int32(NRB)) + g = lane // 16 + l16 = lane % 16 + r8 = lane // 8 + + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=i32_m * (K * 2)) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K * 2) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y, num_records_bytes=i32_m * i32_ldy * 4) + lds_base = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + + # DMA jj of a fragment: rows 8 jj + r8, 16-byte piece (lane % 8) ^ ((r8 >> 1) | (jj << 2)) of the line + w_src, w_dst = [], [] + for i in range_constexpr(NRW): + rn = wave + i * NWV + ok = rn < nfrag + for jq in range_constexpr(2 // MH): + jj = mhalf + jq if MH == 2 else jq + pc = (lane % 8) ^ ((r8 // 2) + jj * 4) + if const_expr(SHUF): + off = (n0 // 16 + rn) * (K * 32) + (jj * 8 + r8) * 128 + pc * 16 + else: + off = (n0 + rn * 16 + jj * 8 + r8) * (K * 2) + pc * 16 + w_src.append(ok.select(off, fx.Int32(0x70000000))) # past the tile: zeros, no traffic + w_dst.append((ok.select(rn, fx.Int32(NR)) * 2 + jj) * 1024) + x_src, x_dst = [], [] + for i in range_constexpr(XPW): + d = wave_all + i * NWT + d = (d < XD).select(d, fx.Int32(XD - 1)) # spare DMAs repeat the last one (same bytes, same place) + jx = d % 2 + row = m0 + (d // 2) * 16 + jx * 8 + r8 + row = (row < i32_m).select(row, i32_m - 1) # rows past M read the last row; never stored + px = (lane % 8) ^ ((r8 // 2) + jx * 4) + x_src.append(row * (K * 2) + px * 16) + x_dst.append((WF * 2 + d) * 1024) + aux_w = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), W_CM) + aux_x = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + + def issue(rs, dst, src, b, soff, aux): + d = fx.Int32(fx.rocdl.readfirstlane(T.i32, lds_base + (b % R) * SLOT + dst)) + lp = fx.to_llvm_ptr(fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), fx.Int64(d))) + _rocdl_ods.raw_ptr_buffer_load_async_lds( + rs, lp, fx.Int32(16).ir_value(), fx.Int32(src).ir_value(), + fx.Int32(soff).ir_value(), fx.Int32(0).ir_value(), aux=aux) + + def dma(b): + for q in range_constexpr(NRW * 2 // MH): + issue(wr, w_dst[q], w_src[q], b, b * (2048 if SHUF else 128), aux_w) + for q in range_constexpr(XPW): + issue(xr, x_dst[q], x_src[q], b, b * 128, aux_x) + + wpos = [(wave + i * NWV < NR).select(wave + i * NWV, fx.Int32(NR)) for i in range_constexpr(NRW)] + r8r = l16 % 8 + hh = l16 // 8 + rd_off = [hh * 1024 + (r8r * 8 + ((kk * 4 + g) ^ ((r8r // 2) + hh * 4))) * 16 for kk in range_constexpr(2)] + + def rd(b, pos, kk): + p = _lds_ptr(lds_base + (b % R) * SLOT + pos * 2048 + rd_off[kk]) + return Vec(llvm.load(T.vec(4, T.i32), p, alignment=16)).bitcast(fx.BFloat16) + + acc = [[Vec.filled(4, 0.0, fx.Float32) for _ in range_constexpr(MT)] for _ in range_constexpr(NRW)] + + def read_ops(b, kk): + return ([rd(b, wpos[i], kk) for i in range_constexpr(NRW)], + [rd(b, WF + mhalf * MT + m, kk) for m in range_constexpr(MT)]) + + def mfmas(ops): + wv, xv = ops + for i in range_constexpr(NRW): + for m in range_constexpr(MT): + acc[i][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), [wv[i], xv[m], acc[i][m], 0, 0, 0])) + + def barrier(): + llvm.inline_asm(None, [], "s_barrier", "~{memory}", has_side_effects=True) + + # half-block software pipeline: the LDS reads of (b, kk 1) overlap the MFMAs of (b, kk 0), those of + # (b + 1, kk 0) the MFMAs of (b, kk 1); slot b is refilled (block b + R) once every wave has read it + # (lgkmcnt(0) before the barrier), so R blocks are in flight + for b in range_constexpr(R): + dma(b) + _sb() + fx.rocdl.s_waitcnt(vmcnt=(R - 1) * PER) + barrier() + _sb() + ops_a = read_ops(0, 0) + for b in range_constexpr(NB): + _sb() + ops_b = read_ops(b, 1) + _sb() + mfmas(ops_a) + _sb() + if b + 1 < NB: + # blocks issued so far: 0 .. min(b + R - 1, NB - 1); block b + 1 must have landed + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=(min(b + R - 1, NB - 1) - (b + 1)) * PER) + barrier() + _sb() + ops_a = read_ops(b + 1, 0) + _sb() + if b + R < NB: + dma(b + R) + _sb() + mfmas(ops_b) + + for i in range_constexpr(NRW): + rn = wave + i * NWV + n = n0 + rn * 16 + g * 4 + for m in range_constexpr(MT): + r = m0 + (mhalf * MT + m) * 16 + l16 + pk = acc[i][m].to(fx.BFloat16).bitcast(fx.Int32) + buffer_ops.buffer_store(pk, yr, r * i32_ldy + n // 2, mask=(rn < nfrag) & (r < i32_m)) + + @flyc.jit + def launch_bf16_gemm_n100032_k6144(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_m: fx.Int32, + i32_mtiles: fx.Int32, + i32_ldy: fx.Int32, stream: fx.Stream): + kernel(arg_x, arg_w, arg_y, i32_m, i32_mtiles, i32_ldy).launch( + grid=(tiles * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + return launch_bf16_gemm_n100032_k6144 diff --git a/amd_kernels/gemm/bf16_n100032_k6144/large.py b/amd_kernels/gemm/bf16_n100032_k6144/large.py new file mode 100644 index 0000000..3404000 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/large.py @@ -0,0 +1,245 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Large-M BF16 lm_head GEMM with a wider vocabulary tile and bounded operand liveness. + +400-512 vocabulary tiles cover 12-16 fragments each. Eight waves take two M +halves, sharing activation and weight LDS rings. Splitting each LDS operand +prefetch into two M slices removes the wider tile's spills. The epilogue +reuses the dead LDS ring to write 16-byte contiguous output pieces. +K remains ascending in fp32, with one BF16 rounding and no output atomics. +The load-time weight layout is the same as the small-M kernel's. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ods +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .reference import K, N + +G16 = N // 16 +NWV = 4 +NB = K // 64 +VOCAB_TILES = 400 # 15-16 groups per tile; the epilogue's 256-column row pitch needs <= 16 groups +RING = 2 +M_HALVES = 2 +W_CM = 2 # nt wins the full-shaped sweep; cached reads are slower + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _sb(): + fx.rocdl.sched_barrier(0) + + +def _lds_ptr(byte_addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(byte_addr))) + + +def compile_large_bf16_gemm_n100032_k6144(mt: int, shuf: int, tiles: int = VOCAB_TILES): + TILES, mh, ring = tiles, M_HALVES, RING + NRB = G16 // TILES + NBIG = G16 - NRB * TILES + NR = NRB + 1 + NRW = (NR + NWV - 1) // NWV + # Wave w owns vocabulary fragments w % 4 and M half w // 4. + MT, SHUF, MH = mt, shuf, mh + NWT = NWV * MH # waves per workgroup + MTT = MT * MH # activation fragments per workgroup + R = ring + TM = 16 * MTT + XD = MTT * 2 # activation DMAs per block (2 per 16-row fragment) + XPW = (XD + NWT - 1) // NWT + PER = NRW * 2 // MH + XPW # DMAs per wave per block (MH 2: each half issues one of a group's two DMAs) + WF = NR + 1 # weight positions per slot: 16 groups + one junk position + SLOT = (WF + MTT) * 2048 + LDS = max(R * SLOT, TM * 256 * 2) + assert 5 <= MT <= 8 and NR <= 16 and LDS <= 160 * 1024 and (R - 1) * PER <= 63, (LDS, R, PER) + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, LDS, 16] + + name = f"bf16_gemm_n100032_k6144_large_tiles{TILES}_tm{TM}_r{R}" + ("_shuf" if SHUF else "_row") + ("_w8" if MH == 2 else "") + "_interleaved" + + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_m: fx.Int32, i32_mtiles: fx.Int32, + i32_ldy: fx.Int32): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave_all = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wave = wave_all % NWV # the weight groups + mhalf = wave_all // NWV # the M half (0 when MH 1) + # workgroup b runs on XCD b % 8; the M tiles of one vocabulary tile follow each other on that XCD + xcd = bid % 8 + j = bid // 8 + t = (j // i32_mtiles) * 8 + xcd + m0 = (j % i32_mtiles) * TM + big = t < NBIG + n0 = big.select(t * (16 * NR), t * (16 * NRB) + NBIG * 16) + nfrag = big.select(fx.Int32(NR), fx.Int32(NRB)) + g = lane // 16 + l16 = lane % 16 + r8 = lane // 8 + + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=i32_m * (K * 2)) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K * 2) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y, num_records_bytes=i32_m * i32_ldy * 4) + lds_base = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + + # DMA jj of a fragment: rows 8 jj + r8, 16-byte piece (lane % 8) ^ ((r8 >> 1) | (jj << 2)) of the line + w_src, w_dst = [], [] + for i in range_constexpr(NRW): + rn = wave + i * NWV + ok = rn < nfrag + for jq in range_constexpr(2 // MH): + jj = mhalf + jq if MH == 2 else jq + pc = (lane % 8) ^ ((r8 // 2) + jj * 4) + if const_expr(SHUF): + off = (n0 // 16 + rn) * (K * 32) + (jj * 8 + r8) * 128 + pc * 16 + else: + off = (n0 + rn * 16 + jj * 8 + r8) * (K * 2) + pc * 16 + w_src.append(ok.select(off, fx.Int32(0x70000000))) # past the tile: zeros, no traffic + w_dst.append((ok.select(rn, fx.Int32(NR)) * 2 + jj) * 1024) + x_src, x_dst = [], [] + for i in range_constexpr(XPW): + d = wave_all + i * NWT + d = (d < XD).select(d, fx.Int32(XD - 1)) # spare DMAs repeat the last one (same bytes, same place) + jx = d % 2 + row = m0 + (d // 2) * 16 + jx * 8 + r8 + row = (row < i32_m).select(row, i32_m - 1) # rows past M read the last row; never stored + px = (lane % 8) ^ ((r8 // 2) + jx * 4) + x_src.append(row * (K * 2) + px * 16) + x_dst.append((WF * 2 + d) * 1024) + aux_w = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), W_CM) + aux_x = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + + def issue(rs, dst, src, b, soff, aux): + d = fx.Int32(fx.rocdl.readfirstlane(T.i32, lds_base + (b % R) * SLOT + dst)) + lp = fx.to_llvm_ptr(fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), fx.Int64(d))) + _rocdl_ods.raw_ptr_buffer_load_async_lds( + rs, lp, fx.Int32(16).ir_value(), fx.Int32(src).ir_value(), + fx.Int32(soff).ir_value(), fx.Int32(0).ir_value(), aux=aux) + + def dma(b): + for q in range_constexpr(NRW * 2 // MH): + issue(wr, w_dst[q], w_src[q], b, b * (2048 if SHUF else 128), aux_w) + for q in range_constexpr(XPW): + issue(xr, x_dst[q], x_src[q], b, b * 128, aux_x) + + wpos = [(wave + i * NWV < NR).select(wave + i * NWV, fx.Int32(NR)) for i in range_constexpr(NRW)] + r8r = l16 % 8 + hh = l16 // 8 + rd_off = [hh * 1024 + (r8r * 8 + ((kk * 4 + g) ^ ((r8r // 2) + hh * 4))) * 16 for kk in range_constexpr(2)] + + def rd(b, pos, kk): + p = _lds_ptr(lds_base + (b % R) * SLOT + pos * 2048 + rd_off[kk]) + return Vec(llvm.load(T.vec(4, T.i32), p, alignment=16)).bitcast(fx.BFloat16) + + acc = [[Vec.filled(4, 0.0, fx.Float32) for _ in range_constexpr(MT)] for _ in range_constexpr(NRW)] + + P = (MT + 1) // 2 + def read_w(b, kk): + return [rd(b, wpos[i], kk) for i in range_constexpr(NRW)] + def read_x(b, kk, lo, hi): + return [rd(b, WF + mhalf * MT + m, kk) for m in range_constexpr(lo, hi)] + def mfmas(wv, xv, lo, hi, prefetch=None, operand=0): + for i in range_constexpr(NRW): + # Spread future DMA requests between four-MFMA groups instead + # of issuing an eight-request burst before the computation. + if prefetch is not None: + if operand == 0: + issue(wr, w_dst[i], w_src[i], prefetch, prefetch * (2048 if SHUF else 128), aux_w) + elif i < XPW: + issue(xr, x_dst[i], x_src[i], prefetch, prefetch * 128, aux_x) + _sb() + for m in range_constexpr(lo, hi): + acc[i][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), [wv[i], xv[m - lo], acc[i][m], 0, 0, 0])) + + def barrier(): + llvm.inline_asm(None, [], "s_barrier", "~{memory}", has_side_effects=True) + + # half-block software pipeline: the LDS reads of (b, kk 1) overlap the MFMAs of (b, kk 0), those of + # (b + 1, kk 0) the MFMAs of (b, kk 1); slot b is refilled (block b + R) once every wave has read it + # (lgkmcnt(0) before the barrier), so R blocks are in flight + for b in range_constexpr(R): + dma(b) + _sb() + fx.rocdl.s_waitcnt(vmcnt=(R - 1) * PER) + barrier() + _sb() + w_a = read_w(0, 0) + x_a0 = read_x(0, 0, 0, P) + x_a1 = read_x(0, 0, P, MT) + for b in range_constexpr(NB): + _sb() + w_b = read_w(b, 1) + x_b0 = read_x(b, 1, 0, P) + _sb() + mfmas(w_a, x_a0, 0, P) + _sb() + x_b1 = read_x(b, 1, P, MT) + _sb() + mfmas(w_a, x_a1, P, MT) + _sb() + if const_expr(b + 1 < NB): + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=(min(b + R - 1, NB - 1) - (b + 1)) * PER) + barrier() + _sb() + w_a = read_w(b + 1, 0) + x_a0 = read_x(b + 1, 0, 0, P) + _sb() + _sb() + mfmas(w_b, x_b0, 0, P, (b + R) if b + R < NB else None, 0) + _sb() + if const_expr(b + 1 < NB): + x_a1 = read_x(b + 1, 0, P, MT) + _sb() + mfmas(w_b, x_b1, P, MT, (b + R) if b + R < NB else None, 1) + + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=0) + barrier() + out_view = fx.make_view(fx.recast_iter(fx.Int32, smem.buf.ptr), fx.make_layout(TM * 128, 1)) + out8 = fx.logical_divide(out_view, fx.make_layout(2, 1)) + out16 = fx.logical_divide(out_view, fx.make_layout(4, 1)) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + atom16 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + for i in range_constexpr(NRW): + rn = wave + i * NWV + for m in range_constexpr(MT): + r = (mhalf * MT + m) * 16 + l16 + c = rn * 16 + g * 4 + pk = acc[i][m].to(fx.BFloat16).bitcast(fx.Int32) + phys = (r * 128 + c // 2) ^ ((r % 8) * 4) + rmem = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + rmem.store(pk) + fx.copy(atom8, rmem, fx.slice(out8, (None, phys // 2))) + fx.rocdl.s_waitcnt(lgkmcnt=0) + barrier() + for it in range_constexpr((TM * 256 // 8 + 64 * NWT - 1) // (64 * NWT)): + v = tx + it * (64 * NWT) + r = v // 32 + c = (v % 32) * 8 + phys = (r * 128 + c // 2) ^ ((r % 8) * 4) + rmem = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(out16, (None, phys // 4)), rmem) + val = Vec(rmem.load()) + buffer_ops.buffer_store(val, yr, (m0 + r) * i32_ldy + (n0 + c) // 2, + mask=(r < TM) & (m0 + r < i32_m) & (c < nfrag * 16)) + + @flyc.jit + def launch_large_bf16_gemm_n100032_k6144(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_m: fx.Int32, i32_mtiles: fx.Int32, + i32_ldy: fx.Int32, stream: fx.Stream): + kernel(arg_x, arg_w, arg_y, i32_m, i32_mtiles, i32_ldy).launch( + grid=(TILES * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + return launch_large_bf16_gemm_n100032_k6144 diff --git a/amd_kernels/gemm/bf16_n100032_k6144/large_indexed.py b/amd_kernels/gemm/bf16_n100032_k6144/large_indexed.py new file mode 100644 index 0000000..38cfb45 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/large_indexed.py @@ -0,0 +1,284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Indexed large-M BF16 lm_head GEMM with a wider vocabulary tile and bounded operand liveness: mapped rows into a +row-major tensor of any even row stride (`rows`) or into the row exchange's send slot (`publish`, publish.py). + +400-512 vocabulary tiles cover 12-16 fragments each. Eight waves take two M +halves, sharing activation and weight LDS rings. Splitting each LDS operand +prefetch into two M slices removes the wider tile's spills. The epilogue +reuses the dead LDS ring to write 16-byte contiguous output pieces. +K remains ascending in fp32, with one BF16 rounding and no output atomics. +The load-time weight layout is the same as the small-M kernel's. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ods +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .publish import SYS_CM, flag_add +from .reference import K, N + +G16 = N // 16 +NWV = 4 +NB = K // 64 +VOCAB_TILES = 400 # 15-16 groups per tile; the epilogue's 256-column row pitch needs <= 16 groups +RING = 2 +M_HALVES = 2 +W_CM = 2 # nt wins the full-shaped sweep; cached reads are slower + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _sb(): + fx.rocdl.sched_barrier(0) + + +def _lds_ptr(byte_addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(byte_addr))) + + +def compile_large_indexed_bf16_gemm_n100032_k6144(mt: int, shuf: int, tiles: int = VOCAB_TILES, publish: bool = True): + TILES, mh, ring = tiles, M_HALVES, RING + NRB = G16 // TILES + NBIG = G16 - NRB * TILES + NR = NRB + 1 + NRW = (NR + NWV - 1) // NWV + # Wave w owns vocabulary fragments w % 4 and M half w // 4. + MT, SHUF, MH = mt, shuf, mh + NWT = NWV * MH # waves per workgroup + MTT = MT * MH # activation fragments per workgroup + R = ring + TM = 16 * MTT + XD = MTT * 2 # activation DMAs per block (2 per 16-row fragment) + XPW = (XD + NWT - 1) // NWT + PER = NRW * 2 // MH + XPW # DMAs per wave per block (MH 2: each half issues one of a group's two DMAs) + WF = NR + 1 # weight positions per slot: 16 groups + one junk position + SLOT = (WF + MTT) * 2048 + LDS = max(R * SLOT, TM * 256 * 2) + assert 5 <= MT <= 8 and NR <= 16 and LDS <= 160 * 1024 and (R - 1) * PER <= 63, (LDS, R, PER) + + @fx.struct + class Shared: + buf: fx.Array[fx.Uint8, LDS, 16] + + name = f"bf16_gemm_n100032_k6144_large_indexed_tiles{TILES}_tm{TM}_r{R}" + ("_shuf" if SHUF else "_row") + ("_w8" if MH == 2 else "") + "_interleaved" + name = name + ("_publish" if publish else "_rows") + + def body(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy, y_cm): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave_all = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + wave = wave_all % NWV # the weight groups + mhalf = wave_all // NWV # the M half (0 when MH 1) + # workgroup b runs on XCD b % 8; the M tiles of one vocabulary tile follow each other on that XCD + xcd = bid % 8 + j = bid // 8 + t = (j // i32_mtiles) * 8 + xcd + m0 = (j % i32_mtiles) * TM + big = t < NBIG + n0 = big.select(t * (16 * NR), t * (16 * NRB) + NBIG * 16) + nfrag = big.select(fx.Int32(NR), fx.Int32(NRB)) + g = lane // 16 + l16 = lane % 16 + r8 = lane // 8 + + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=i32_xrows * (K * 2)) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K * 2) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y, num_records_bytes=i32_m * i32_ldy * 4) + lds_base = fx.Int32(fx.ptrtoint(smem.buf.ptr)) + + # DMA jj of a fragment: rows 8 jj + r8, 16-byte piece (lane % 8) ^ ((r8 >> 1) | (jj << 2)) of the line + w_src, w_dst = [], [] + for i in range_constexpr(NRW): + rn = wave + i * NWV + ok = rn < nfrag + for jq in range_constexpr(2 // MH): + jj = mhalf + jq if MH == 2 else jq + pc = (lane % 8) ^ ((r8 // 2) + jj * 4) + if const_expr(SHUF): + off = (n0 // 16 + rn) * (K * 32) + (jj * 8 + r8) * 128 + pc * 16 + else: + off = (n0 + rn * 16 + jj * 8 + r8) * (K * 2) + pc * 16 + w_src.append(ok.select(off, fx.Int32(0x70000000))) # past the tile: zeros, no traffic + w_dst.append((ok.select(rn, fx.Int32(NR)) * 2 + jj) * 1024) + x_src, x_dst = [], [] + for i in range_constexpr(XPW): + d = wave_all + i * NWT + d = (d < XD).select(d, fx.Int32(XD - 1)) # spare DMAs repeat the last one (same bytes, same place) + jx = d % 2 + row = m0 + (d // 2) * 16 + jx * 8 + r8 + row = (row < i32_m).select(row, i32_m - 1) # rows past M read the last row; never stored + row = fx.Int32(arg_indices[row]) + px = (lane % 8) ^ ((r8 // 2) + jx * 4) + x_src.append(row * (K * 2) + px * 16) + x_dst.append((WF * 2 + d) * 1024) + aux_w = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), W_CM) + aux_x = ir.IntegerAttr.get(ir.IntegerType.get_signless(32), 0) + + def issue(rs, dst, src, b, soff, aux): + d = fx.Int32(fx.rocdl.readfirstlane(T.i32, lds_base + (b % R) * SLOT + dst)) + lp = fx.to_llvm_ptr(fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), fx.Int64(d))) + _rocdl_ods.raw_ptr_buffer_load_async_lds( + rs, lp, fx.Int32(16).ir_value(), fx.Int32(src).ir_value(), + fx.Int32(soff).ir_value(), fx.Int32(0).ir_value(), aux=aux) + + def dma(b): + for q in range_constexpr(NRW * 2 // MH): + issue(wr, w_dst[q], w_src[q], b, b * (2048 if SHUF else 128), aux_w) + for q in range_constexpr(XPW): + issue(xr, x_dst[q], x_src[q], b, b * 128, aux_x) + + wpos = [(wave + i * NWV < NR).select(wave + i * NWV, fx.Int32(NR)) for i in range_constexpr(NRW)] + r8r = l16 % 8 + hh = l16 // 8 + rd_off = [hh * 1024 + (r8r * 8 + ((kk * 4 + g) ^ ((r8r // 2) + hh * 4))) * 16 for kk in range_constexpr(2)] + + def rd(b, pos, kk): + p = _lds_ptr(lds_base + (b % R) * SLOT + pos * 2048 + rd_off[kk]) + return Vec(llvm.load(T.vec(4, T.i32), p, alignment=16)).bitcast(fx.BFloat16) + + acc = [[Vec.filled(4, 0.0, fx.Float32) for _ in range_constexpr(MT)] for _ in range_constexpr(NRW)] + + P = (MT + 1) // 2 + def read_w(b, kk): + return [rd(b, wpos[i], kk) for i in range_constexpr(NRW)] + def read_x(b, kk, lo, hi): + return [rd(b, WF + mhalf * MT + m, kk) for m in range_constexpr(lo, hi)] + def mfmas(wv, xv, lo, hi, prefetch=None, operand=0): + for i in range_constexpr(NRW): + # Spread future DMA requests between four-MFMA groups instead + # of issuing an eight-request burst before the computation. + if prefetch is not None: + if operand == 0: + issue(wr, w_dst[i], w_src[i], prefetch, prefetch * (2048 if SHUF else 128), aux_w) + elif i < XPW: + issue(xr, x_dst[i], x_src[i], prefetch, prefetch * 128, aux_x) + _sb() + for m in range_constexpr(lo, hi): + acc[i][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), [wv[i], xv[m - lo], acc[i][m], 0, 0, 0])) + + def barrier(): + llvm.inline_asm(None, [], "s_barrier", "~{memory}", has_side_effects=True) + + # half-block software pipeline: the LDS reads of (b, kk 1) overlap the MFMAs of (b, kk 0), those of + # (b + 1, kk 0) the MFMAs of (b, kk 1); slot b is refilled (block b + R) once every wave has read it + # (lgkmcnt(0) before the barrier), so R blocks are in flight + for b in range_constexpr(R): + dma(b) + _sb() + fx.rocdl.s_waitcnt(vmcnt=(R - 1) * PER) + barrier() + _sb() + w_a = read_w(0, 0) + x_a0 = read_x(0, 0, 0, P) + x_a1 = read_x(0, 0, P, MT) + for b in range_constexpr(NB): + _sb() + w_b = read_w(b, 1) + x_b0 = read_x(b, 1, 0, P) + _sb() + mfmas(w_a, x_a0, 0, P) + _sb() + x_b1 = read_x(b, 1, P, MT) + _sb() + mfmas(w_a, x_a1, P, MT) + _sb() + if const_expr(b + 1 < NB): + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=(min(b + R - 1, NB - 1) - (b + 1)) * PER) + barrier() + _sb() + w_a = read_w(b + 1, 0) + x_a0 = read_x(b + 1, 0, 0, P) + _sb() + _sb() + mfmas(w_b, x_b0, 0, P, (b + R) if b + R < NB else None, 0) + _sb() + if const_expr(b + 1 < NB): + x_a1 = read_x(b + 1, 0, P, MT) + _sb() + mfmas(w_b, x_b1, P, MT, (b + R) if b + R < NB else None, 1) + + fx.rocdl.s_waitcnt(lgkmcnt=0, vmcnt=0) + barrier() + out_view = fx.make_view(fx.recast_iter(fx.Int32, smem.buf.ptr), fx.make_layout(TM * 128, 1)) + out8 = fx.logical_divide(out_view, fx.make_layout(2, 1)) + out16 = fx.logical_divide(out_view, fx.make_layout(4, 1)) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + atom16 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + for i in range_constexpr(NRW): + rn = wave + i * NWV + for m in range_constexpr(MT): + r = (mhalf * MT + m) * 16 + l16 + c = rn * 16 + g * 4 + pk = acc[i][m].to(fx.BFloat16).bitcast(fx.Int32) + phys = (r * 128 + c // 2) ^ ((r % 8) * 4) + rmem = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + rmem.store(pk) + fx.copy(atom8, rmem, fx.slice(out8, (None, phys // 2))) + fx.rocdl.s_waitcnt(lgkmcnt=0) + barrier() + for it in range_constexpr((TM * 256 // 8 + 64 * NWT - 1) // (64 * NWT)): + v = tx + it * (64 * NWT) + r = v // 32 + c = (v % 32) * 8 + phys = (r * 128 + c // 2) ^ ((r % 8) * 4) + rmem = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(out16, (None, phys // 4)), rmem) + val = Vec(rmem.load()) + buffer_ops.buffer_store(val, yr, (m0 + r) * i32_ldy + (n0 + c) // 2, + mask=(r < TM) & (m0 + r < i32_m) & (c < nfrag * 16), cache_modifier=y_cm) + + # Publish: the output goes to the exchange's slot (counter[0] & 1) with system-scope stores, and the last + # workgroup publishes it (all_to_all_bf16_n100032_ranks2's stage_publish protocol: the same ticket and flag words). + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel_publish(arg_x: fx.Int64, arg_w: fx.Int64, arg_pool: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, counter: fx.Tensor, + i64_slot_bytes: fx.Int64, arg_flag: fx.Int64): + slot = fx.Int32(counter[0]) & 1 + body(arg_x, arg_w, arg_pool + fx.Int64(slot) * i64_slot_bytes, arg_indices, i32_xrows, i32_m, i32_mtiles, + fx.Int32(N // 2), SYS_CM) + # every workgroup's stores land before its ticket; the last of them publishes (dynamic ifs stay in the + # kernel body: FlyDSL rewrites only this function's control flow) + nwg = TILES * i32_mtiles + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + if fx.thread_idx.x == 0: + ticket = flag_add(arg_flag + 4, 1) + if ticket == nwg - 1: + flag_add(arg_flag + 4, -nwg) + seq = fx.Int32(counter[0]) + 1 + counter[0] = seq + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=64) + buffer_ops.buffer_store(seq, fr, 0, cache_modifier=SYS_CM) + + # Rows: plain stores into a row-major output of row stride i32_ldy dwords (a column slice of a wider tensor). + @flyc.kernel(name=name, known_block_size=[64 * NWT, 1, 1]) + def kernel_rows(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, arg_indices: fx.Tensor, i32_xrows: fx.Int32, + i32_m: fx.Int32, i32_mtiles: fx.Int32, i32_ldy: fx.Int32): + body(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy, 0) + + @flyc.jit + def launch_large_indexed_bf16_gemm_n100032_k6144_rows(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, i32_ldy: fx.Int32, + stream: fx.Stream): + kernel_rows(arg_x, arg_w, arg_y, arg_indices, i32_xrows, i32_m, i32_mtiles, i32_ldy).launch( + grid=(TILES * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + @flyc.jit + def launch_large_indexed_bf16_gemm_n100032_k6144_publish(arg_x: fx.Int64, arg_w: fx.Int64, arg_pool: fx.Int64, arg_indices: fx.Tensor, + i32_xrows: fx.Int32, i32_m: fx.Int32, i32_mtiles: fx.Int32, counter: fx.Tensor, + i64_slot_bytes: fx.Int64, arg_flag: fx.Int64, stream: fx.Stream): + kernel_publish(arg_x, arg_w, arg_pool, arg_indices, i32_xrows, i32_m, i32_mtiles, counter, i64_slot_bytes, + arg_flag).launch(grid=(TILES * i32_mtiles, 1, 1), block=(64 * NWT, 1, 1), stream=stream) + + return launch_large_indexed_bf16_gemm_n100032_k6144_publish if publish else launch_large_indexed_bf16_gemm_n100032_k6144_rows diff --git a/amd_kernels/gemm/bf16_n100032_k6144/publish.py b/amd_kernels/gemm/bf16_n100032_k6144/publish.py new file mode 100644 index 0000000..0cfed72 --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/publish.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The publish epilogue: the GEMM output lands in an IPC slot of all_to_all_bf16_n100032_ranks2 and the last +workgroup publishes it, so the exchange runs its pull only (no stage copy). + +The same protocol and words as that entry's stage_publish kernel: the slot is counter[0] & 1, the payload goes out +with system-scope stores (no dirty XCD L2 lines for the peers' system loads), every workgroup waits for its stores +before lane 0 takes a ticket on flag word 1, and the last arriver resets the ticket, increments the counter and +stores it to flag word 0 with a system-scope store. Tickets and the counter are counters, not output adds. +The tail itself sits in each kernel_publish body (FlyDSL rewrites dynamic control flow only there). +""" +import flydsl.expr as fx +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr.typing import T + +SYS_CM = 0x11 # sc0 sc1: system scope + + +def _global_i32_ptr(addr_i64): + ptr_ty = fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4) + return fx.inttoptr(ptr_ty, fx.Int64(addr_i64)) + + +def flag_add(addr_i64, v): + """Atomic add at system scope on an uncached word; returns the old value.""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _global_i32_ptr(addr_i64).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) diff --git a/amd_kernels/gemm/bf16_n100032_k6144/reference.py b/amd_kernels/gemm/bf16_n100032_k6144/reference.py new file mode 100644 index 0000000..92f06be --- /dev/null +++ b/amd_kernels/gemm/bf16_n100032_k6144/reference.py @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The contract: y = x @ w^T for bf16 x [M, 6144] and the bf16 lm_head shard w [100032, 6144]. + +The product of the bf16 operands in fp64, rounded once to bf16. The kernel accumulates in fp32 in +a fixed order, so it differs from the reference (and from hipBLASLt) only in the sum order: per row +cosine >= 0.99999 and relative L2 <= 1e-3, deterministic bits. NaN / inf propagate as in an fp32 +accumulation (a NaN in a row of x makes that output row NaN). +""" +import torch + +N, K = 100032, 6144 + + +def bf16_gemm_n100032_k6144_reference(x: torch.Tensor, w: torch.Tensor, + row_indices: torch.Tensor | None = None) -> torch.Tensor: + if row_indices is not None: + x = x[row_indices] + return (x.double() @ w.double().t()).bfloat16() diff --git a/amd_kernels/gemm/bf16_n200064_k256/README.md b/amd_kernels/gemm/bf16_n200064_k256/README.md index b1b66be..529e02a 100644 --- a/amd_kernels/gemm/bf16_n200064_k256/README.md +++ b/amd_kernels/gemm/bf16_n200064_k256/README.md @@ -1,23 +1,23 @@ # bf16 projection, N 200064 x K 256, packed selected rows ## What it does -`out [n, 200064] = x[rows] @ W^T` for 0-64 selected rows, bf16 in, fp32 accumulate, one bf16 rounding: the DSpark -Markov projection (W = `markov_w2` [200064, 256]) of only the owner rank's requests in the MiniMax-M3.1-Flash TP4 -batch-sharded Markov draft loop. W stays row-major (no copy, no transform). The activation is packed bit for bit into -the MFMA layout [pad / 16, 8, 4, 16, 8] (= [m16, k32, k8, row16, feature8]) viewed as bf16 [pad, 256], pad = 16 / 32 / -64: `pack_bf16_rows` packs the first step's rows of a table (`markov_w1`), and -`sampling/gumbel_combine_markov_embed_v200064_r256` writes the next steps' rows in this layout. Padding rows never -reach an output. Contract (`reference.py`): the fp64 product rounded once to bf16; per-row cosine >= 0.99999, rel. L2 -<= 1e-3, deterministic. The sum order moves a few bf16 elements against hipBLASLt (<= 0.01%, up to 192 ulp); sampled -tokens and accepted lengths stay equal over 4.19M request-steps per class. Caller: -`sampling/batch_sharded_markov_v200064_h6144_r256_ranks4`. +`out [n, 200064] = x[rows] @ W^T` for 0-128 selected rows, bf16 in, fp32 accumulate, one bf16 rounding: the DSpark +Markov projection (W = `markov_w2` [200064, 256]) of only the owner rank's requests in the MiniMax-M3.1-Flash +batch-sharded Markov draft loop (up to 64 rows per rank at TP4, 128 at TP2). W stays row-major (no copy, no +transform). The activation is packed bit for bit into the MFMA layout [pad / 16, 8, 4, 16, 8] (= [m16, k32, k8, row16, +feature8]) viewed as bf16 [pad, 256], pad = 16 / 32 / 64 / 128: `pack_bf16_rows` packs the first step's rows of a +table (`markov_w1`), and `sampling/gumbel_combine_markov_embed_v200064_r256` writes the next steps' rows in this layout. +Padding rows never reach an output. Contract (`reference.py`): the fp64 product rounded once to bf16; per-row cosine +>= 0.99999, rel. L2 <= 1e-3, deterministic. The sum order moves a few bf16 elements against hipBLASLt (<= 0.01%, up +to 192 ulp); sampled tokens and accepted lengths stay equal over 4.19M request-steps per class. Callers: +`sampling/batch_sharded_markov_v200064_h6144_r256_ranks4` and `..._ranks2`. ## Entry points -- `padded_rows`: the packed row count (16, 32 or 64) for a number of selected rows. +- `padded_rows`: the packed row count (16, 32, 64 or 128) for a number of selected rows. - `pack_bf16_rows`: packs the selected rows of a table into the caller's packed buffer; indices are not checked (no host sync), graph-safe. - `bf16_gemm_packed_rows`: the projection of the packed rows into a contiguous bf16 output; zero rows launch nothing. -- `warmup_bf16_gemm_packed_rows`: compiles the four GEMMs and three packs; call it before graph capture. +- `warmup_bf16_gemm_packed_rows`: compiles the five GEMMs and four packs; call it before graph capture. `host.py` picks the kernel and the pad by row count. No output atomics. @@ -25,12 +25,14 @@ tokens and accepted lengths stay equal over 4.19M request-steps per class. Calle One MI355X. The whole seven-step Markov chain (projection, Gumbel partials, tail): four input sets, three rotated `markov_w2` copies, twelve chains per graph, five alternated rounds, median µs. Baseline = projecting every request with the fused row-major tail. Projection alone: 100 packed inputs, three rotated -weights. +weights. The 65-128-row tier (TP2) is timed as the projection alone: at 128 rows two 64-row calls take 51.3 µs, the +HBM roofline (6.5 TB/s) is 23.6 µs. | Requests | Owned rows | Chain, baseline µs | Chain µs | Projection alone µs | |---|---|---|---|---| -| 10 / 16 / 32 | 3 / 4 / 8 | 209.8 / 216.8 / 277.8 | 199.3 / 201.7 / 210.4 | 17.07 / 17.17 / 17.09 | -| 64 / 128 / 256 | 16 / 32 / 64 | 302.8 / 432.9 / 665.5 | 244.8 / 312.1 / 443.8 | 17.23 / 20.74 / 26.02 | +| 10 / 16 / 32 (TP4) | 3 / 4 / 8 | 209.8 / 216.8 / 277.8 | 199.3 / 201.7 / 210.4 | 17.07 / 17.17 / 17.09 | +| 64 / 128 / 256 (TP4) | 16 / 32 / 64 | 302.8 / 432.9 / 665.5 | 244.8 / 312.1 / 443.8 | 17.23 / 20.74 / 26.02 | +| 192 / 256 (TP2) | 96 / 128 | | | 33.66 / 35.01 | ## Tests - `python3 -m pytest tests/gemm/bf16_n200064_k256 tests/sampling/gumbel_combine_markov_embed_v200064_r256 -q` diff --git a/amd_kernels/gemm/bf16_n200064_k256/host.py b/amd_kernels/gemm/bf16_n200064_k256/host.py index a78030f..17c16b8 100644 --- a/amd_kernels/gemm/bf16_n200064_k256/host.py +++ b/amd_kernels/gemm/bf16_n200064_k256/host.py @@ -8,6 +8,7 @@ from .kernel import compile_bf16_gemm_packed_rows_shuffled from .kernel_direct import compile_bf16_gemm_packed_rows +from .kernel_m128 import compile_bf16_gemm_packed_rows_m128 from .pack import compile_pack_bf16_rows from .reference import K, N @@ -17,12 +18,14 @@ def padded_rows(n: int) -> int: - assert 0 <= n <= 64 - return 16 if n <= 16 else (32 if n <= 32 else 64) + assert 0 <= n <= 128 + return 16 if n <= 16 else (32 if n <= 32 else (64 if n <= 64 else 128)) @functools.cache def _gemm(mt: int, shuffled: bool): + if mt == 8: + return compile_bf16_gemm_packed_rows_m128() if shuffled: return compile_bf16_gemm_packed_rows_shuffled(mt) return compile_bf16_gemm_packed_rows(mt) @@ -65,7 +68,7 @@ def pack_bf16_rows(x: torch.Tensor, row_indices: torch.Tensor, packed: torch.Ten def bf16_gemm_packed_rows( packed: torch.Tensor, weight: torch.Tensor, n: int, out: torch.Tensor, ) -> torch.Tensor: - """FP32-accumulated bf16 projection of 0-64 packed rows; original weight layout.""" + """FP32-accumulated bf16 projection of 0-128 packed rows; original weight layout.""" pad = padded_rows(n) assert packed.shape == (pad, K) and packed.dtype == torch.bfloat16 and packed.is_contiguous() assert weight.shape == (N, K) and weight.dtype == torch.bfloat16 and weight.is_contiguous() @@ -80,10 +83,10 @@ def bf16_gemm_packed_rows( def warmup_bf16_gemm_packed_rows(device: torch.device) -> None: - """Compile the four GEMMs and three pack variants before graph capture.""" + """Compile the five GEMMs and four pack variants before graph capture.""" w = torch.zeros(N, K, dtype=torch.bfloat16, device=device) - x = torch.zeros(64, K, dtype=torch.bfloat16, device=device) - for n in (1, 8, 17, 33): + x = torch.zeros(128, K, dtype=torch.bfloat16, device=device) + for n in (1, 8, 17, 33, 65): p = torch.empty(padded_rows(n), K, dtype=torch.bfloat16, device=device) indices = torch.arange(n, device=device) pack_bf16_rows(x, indices, p) diff --git a/amd_kernels/gemm/bf16_n200064_k256/kernel_m128.py b/amd_kernels/gemm/bf16_n200064_k256/kernel_m128.py new file mode 100644 index 0000000..b7f5d9d --- /dev/null +++ b/amd_kernels/gemm/bf16_n200064_k256/kernel_m128.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""65-128 packed rows: the 64-row kernel's tile (four waves, 128 vocabulary columns per block) run over two 64-row +halves with the weight registers loaded once. Holding all 128 rows' activations would take 256 VGPRs per lane, so +each half loads its activations, runs its MFMAs and writes through the LDS transpose before the next half.""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T, Vector as Vec +from .reference import N as V + +TN = 128 +HALVES = 2 +MH = 4 # 16-row fragments per half + + +def compile_bf16_gemm_packed_rows_m128(): + NR = TN // 64 + + @fx.struct + class Shared: + out: fx.Array[fx.Int32, MH * 16 * TN // 2, 16] + + @flyc.kernel(name=f'bf16_gemm_packed_rows_n200064_k256_mt8_tn{TN}', known_block_size=[256, 1, 1]) + def kernel(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_n: fx.Int32): + tx, bid = fx.thread_idx.x, fx.block_idx.x + smem = fx.SharedAllocator().allocate(Shared).peek() + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + n0 = bid * TN + wave * (TN // 4) + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=HALVES * MH * 8192) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=V * 256 * 2) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y, num_records_bytes=i32_n * V * 2) + wv = [[Vec(buffer_ops.buffer_load(wr, (n0 + rn * 16 + lane % 16) * 128 + k * 16 + (lane // 16) * 4, + vec_width=4, dtype=fx.Int32)) for k in range_constexpr(8)] + for rn in range_constexpr(NR)] + lds16 = fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32, smem.out.ptr), + fx.make_layout(MH * 16 * TN // 2, 1)), fx.make_layout(4, 1)) + atom = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + for h in range_constexpr(HALVES): + xv = [[Vec(buffer_ops.buffer_load(xr, (h * MH + m) * 2048 + k * 256 + lane * 4, vec_width=4, + dtype=fx.Int32, mask=(lane % 16 + (h * MH + m) * 16) < i32_n)) + for k in range_constexpr(8)] for m in range_constexpr(MH)] + acc = [[Vec.filled(4, 0.0, fx.Float32) for m in range_constexpr(MH)] for rn in range_constexpr(NR)] + for k in range_constexpr(8): + for rn in range_constexpr(NR): + for m in range_constexpr(MH): + acc[rn][m] = Vec(fx.rocdl.mfma_f32_16x16x32_bf16( + Vec.make_type(4, fx.Float32), + [wv[rn][k].bitcast(fx.BFloat16), xv[m][k].bitcast(fx.BFloat16), acc[rn][m], 0, 0, 0])) + if h: + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() # every wave has read the first half's tile + for rn in range_constexpr(NR): + for m in range_constexpr(MH): + r = lane % 16 + m * 16 + for e in range_constexpr(2): + pk = fx.Int32(Vec.from_elements([fx.Float32(acc[rn][m][2 * e]), + fx.Float32(acc[rn][m][2 * e + 1])], + fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)[0]) + c = wave * (TN // 4) + rn * 16 + (lane // 16) * 4 + e * 2 + phys = (r * (TN // 2) + c // 2) ^ ((r % 8) * 4) + smem.out.ptr[phys] = pk + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + for it in range_constexpr((MH * 16 * TN // 8 + 255) // 256): + v = tx + it * 256 + row = v // (TN // 8) + col = (v % (TN // 8)) * 8 + if v < MH * 16 * TN // 8: + phys = (row * (TN // 2) + col // 2) ^ ((row % 8) * 4) + rm = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom, fx.slice(lds16, (None, phys // 4)), rm) + val = Vec(fx.memref_load_vec(rm)) + c = bid * TN + col + rr = row + h * MH * 16 + buffer_ops.buffer_store(val, yr, rr * (V // 2) + c // 2, mask=(rr < i32_n) & (c < V)) + + @flyc.jit + def launch_bf16_gemm_packed_rows_m128(arg_x: fx.Int64, arg_w: fx.Int64, arg_y: fx.Int64, i32_n: fx.Int32, + stream: fx.Stream): + kernel(arg_x, arg_w, arg_y, i32_n).launch(grid=((V + TN - 1) // TN, 1, 1), block=(256, 1, 1), stream=stream) + return launch_bf16_gemm_packed_rows_m128 diff --git a/amd_kernels/gemm/bf16_n200064_k256/reference.py b/amd_kernels/gemm/bf16_n200064_k256/reference.py index 45d3f03..b8fa23f 100644 --- a/amd_kernels/gemm/bf16_n200064_k256/reference.py +++ b/amd_kernels/gemm/bf16_n200064_k256/reference.py @@ -3,7 +3,7 @@ """bf16 GEMM with packed activation rows, original bf16 weight, fp32 accumulate. Activation layout: [m16, k32, k8, row16, feature8], flattened and viewed as -[padded_rows,256]; rows are padded to 16/32/64. Weight remains [200064,256], +[padded_rows,256]; rows are padded to 16/32/64/128. Weight remains [200064,256], row major, shared with every existing reader. Output is row-major bf16. Reference uses fp64 accumulation, rounded once to bf16. Gate: row cosine >=0.99999 and relative L2 <=1e-3, deterministic. Accumulation order can move @@ -14,7 +14,7 @@ def pack_rows_reference(x,indices): - n=indices.numel();pad=16 if n<=16 else (32 if n<=32 else 64) + n=indices.numel();pad=16 if n<=16 else (32 if n<=32 else (64 if n<=64 else 128)) selected=torch.zeros(pad,K,dtype=torch.bfloat16,device=x.device) selected[:n]=x.index_select(0,indices) return selected.view(-1,16,8,4,8).permute(0,2,3,1,4).contiguous().view(pad,K) diff --git a/amd_kernels/gemm/mxfp8_common/README.md b/amd_kernels/gemm/mxfp8_common/README.md index 658895b..6ca7ef8 100644 --- a/amd_kernels/gemm/mxfp8_common/README.md +++ b/amd_kernels/gemm/mxfp8_common/README.md @@ -1,13 +1,13 @@ # MXFP8 GEMM: the dispatcher and the shared kernels (gfx950) ## What it does -`out [M, N] = x [M, K] @ W^T` for the MXFP8 dense linears of MiniMax-M3.1-Flash at TP4. W: E4M3 codes [N, K] + E8M0 -per-32 scales [N, K / 32], preshuffled at load ((16, 16) codes, AITER `shuffle_scale_a16w4` scales). x: bf16 / fp16 -[M, K], K % 256 == 0, quantized per 1 x 32 block. Shapes (N x K): 2560 x 6144 QKV, 6144 x 2048 o_proj, 6144 x 8192 -`o_proj_full`, 6144 x 6144 dense gate_up, 6144 x 3072 dense down, 1536 x 6144 / 6144 x 768 shared-expert gate_up / -down, 2304 x 6144 draft QKV, 1280 x 6144 draft context K / V. Callers: vLLM's MXFP8 linears, the norm + quant -producers, the QKV chain, SP hand-offs. `reference.py`: exact dequant, fp32 sums, one rounding; cos >= 0.99999, -L2 <= 1e-3, bitwise repeats. +`out [M, N] = x [M, K] @ W^T` for the MXFP8 dense linears of MiniMax-M3.1-Flash. W: E4M3 codes [N, K] + E8M0 per-32 +scales [N, K / 32], preshuffled at load ((16, 16) codes, AITER `shuffle_scale_a16w4` scales). x: bf16 / fp16 [M, K], +K % 256 == 0, quantized per 1 x 32 block. Shapes (N x K) of the TP4 / TP2 shards: QKV 2560 / 4992 x 6144, o_proj +6144 x 2048 / 4096, `o_proj_full` 6144 x 8192, dense gate_up 6144 / 12288 x 6144, dense down 6144 x 3072 / 6144, +shared-expert gate_up 1536 / 3072 x 6144 and down 6144 x 768 / 1536, draft QKV 2304 / 4608 x 6144, draft context K / V +1280 / 2560 x 6144. Callers: vLLM's MXFP8 linears, the norm + quant producers, the QKV chain, SP hand-offs. +`reference.py`: exact dequant, fp32 sums, one rounding; cos >= 0.99999, L2 <= 1e-3, bitwise repeats. ## Entry points - `gemm.preshuffle_mxfp8_weight`, `gemm.preshuffle_mxfp8_weight_`: the load-time weight preshuffle; the in-place form @@ -28,19 +28,25 @@ L2 <= 1e-3, bitwise repeats. (16, 16) codes. Dispatch: `gemm._select_config` reads the tuned table `mxfp8_gemm_tuned.py`; a shape folder's `host.select` -(`gemm._SHAPE_HOSTS`) answers first. +(`gemm._SHAPE_HOSTS`) answers first; the `decode_splitk` rows run `mxfp8_gemm_decode_splitk.py`. `out=` (a caller's +arena) is accepted for o_proj 6144 x 2048 / 4096. ## Performance -Graph replay, real TP4 rank 0 weights in cold rotated copies (> 400 MB), µs per call, median. +Graph replay, cold rotated weight copies (> 400 MB), bf16 entry (quantizer + GEMM), µs per call, median; baseline +vLLM's Triton `tl.dot_scaled` linear (`rocm_native._mxfp8_dot_scaled_linear`, its `_M31_TILES`), the same bf16 input. -| N x K | M | µs | Baseline µs, or our delta | Baseline | -|---|---|---|---|---| -| 1536 x 6144 | 1-256 | 4.28-13.13 | -19..-49% / -18..-53% | AITER `any` row / vLLM nightly `dot_scaled` | -| 6144 x 768 | 1-256 | 2.79-7.56 | -20..-62% / -19..-47% | AITER `any` row / vLLM nightly `dot_scaled` | -| shared expert, c16 decode mix | per MoE layer | 11.96 | 19.46 / 18.80 µs | AITER rows / vLLM nightly | -| 2304 x 6144 | 112 / 896 | 8.61 / 27.48 | 10.06 / 35.26 µs | non-temporal W / AITER 128x128 | -| 1280 x 6144 | 256 / 512 / 2048 | 11.75 / 13.75 / 41.09 | 15.53 / 51.29 / 68.37 µs | generic AITER fallback | +| N x K | M 1 | M 8 | M 32 | M 128 | M 256 | M 416 | M 1024 | M 8192 | vs Triton | +|---|---|---|---|---|---|---|---|---|---| +| 3072 x 6144 | 5.18 | 6.53 | 8.98 | 12.22 | 16.03 | 21.15 | 36.69 | 140.2 | -30..-49% | +| 6144 x 1536 | 3.51 | 3.79 | 5.82 | 8.09 | 9.90 | 12.39 | 19.19 | 76.7 | -15..-46% | +| 4608 x 6144 | 6.57 | 8.02 | 10.93 | 14.24 | 19.18 | 26.13 | 40.08 | 194.2 | -38..-58% | +| 12288 x 6144 | 12.96 | 14.65 | 17.62 | 26.20 | 32.77 | 49.40 | 75.99 | 456.9 | -28..-59% | +| 6144 x 6144 | 7.55 | 8.65 | 12.03 | 15.52 | 21.32 | 28.74 | 47.34 | 232.9 | -17..-53% | +| 2560 x 6144 | 4.56 | 6.34 | 9.72 | 13.83 | 16.59 | 20.05 | 27.33 | 126.1 | +1..-59% | ## Tests - `python3 -m pytest tests/gemm/mxfp8_common -q` - `python3 benchmarks/gemm/mxfp8_common/benchmark_gemm.py --variant ` +- `python3 benchmarks/gemm/mxfp8_common/benchmark_table.py --shape N,K` (vLLM on the path for the baseline); + `sweep_rows.py --shape N,K --ms ... --family all` re-tunes a shape's rows; `benchmark_decode_splitk.py --shape N,K + --ms ... --cands ...` A/Bs the table's row against explicit decode / split-K decode rows. diff --git a/amd_kernels/gemm/mxfp8_common/gemm.py b/amd_kernels/gemm/mxfp8_common/gemm.py index 4038dbf..fae8c1c 100644 --- a/amd_kernels/gemm/mxfp8_common/gemm.py +++ b/amd_kernels/gemm/mxfp8_common/gemm.py @@ -11,8 +11,7 @@ served default; ``DBO`` / ``set_dbo_query`` stand in for vLLM's ``dbo_enabled()``; and ``mxfp8_linear`` / ``mxfp8_gemm`` take an optional explicit ``config`` (tests and benchmarks). ``_select_config`` picks the table row. The kernels and rows of -one shape live in that shape's folder (``_SHAPE_HOSTS``: gemm/mxfp8_n2560_k6144, -mxfp8_n6144_k2048, mxfp8_n6144_k8192). +one shape live in that shape's folder (``_SHAPE_HOSTS``: gemm/mxfp8_n_k). """ import csv @@ -20,6 +19,7 @@ import functools import importlib import io +import threading from dataclasses import dataclass import torch @@ -57,12 +57,17 @@ class _Config: # pre-quantized activation (mxfp8_gemm) runs such a row as a plain decode row _DECODE_FQ_KERNEL = "decode_fq" _DECODE_KERNELS = (_DECODE_KERNEL, _DECODE_FQ_KERNEL) +# The split-K decode kernel (mxfp8_gemm_decode_splitk.py) rows: tile_m = 16 * row tiles of an +# m-block, tile_n = 16 * column tiles per MFMA wave * waves_m, waves_m = MFMA waves, waves_n = +# loader waves, split_k = K splits (summed in the kernel), la_w = W steps ahead, la_a = A ring +# slots (0: as many as the LDS holds); same (16, 16) activation as the decode rows. +_DECODE_SPLITK_KERNEL = "decode_splitk" # the LDS-staged kernel (mxfp8_gemm_staged.py) _STAGED_KERNEL = "staged_ab" # rows whose activation is the weight's (16, 16) code layout (the decode-tier rows): only a # producer that writes that layout (``prequant_code_tile_rows``) or a caller that quantizes # itself (allow_decode) may select them -_SHUFFLED_A_KERNELS = _DECODE_KERNELS +_SHUFFLED_A_KERNELS = (*_DECODE_KERNELS, _DECODE_SPLITK_KERNEL) # the persistent 4-wave kernel (mxfp8_gemm_4wave.py): a 256x256 tile per workgroup of 4 # waves, min(tiles, 256) workgroups; `_nt` = non-temporal output stores (the output # larger than the 256 MB infinity cache); row-major codes, preshuffled scales, N % 256 == 0 @@ -75,7 +80,9 @@ class _Config: # host on first use. _SHAPE_HOSTS = { (2560, 6144): "amd_kernels.gemm.mxfp8_n2560_k6144.host", + (4992, 6144): "amd_kernels.gemm.mxfp8_n4992_k6144.host", (6144, 2048): "amd_kernels.gemm.mxfp8_n6144_k2048.host", + (6144, 4096): "amd_kernels.gemm.mxfp8_n6144_k4096.host", (6144, 8192): "amd_kernels.gemm.mxfp8_n6144_k8192.host", } @@ -143,7 +150,7 @@ def _load_tuned_configs() -> dict[ if config.kernel not in ( "aiter", _STAGED_KERNEL, - *_DECODE_KERNELS, + *_SHUFFLED_A_KERNELS, *_FOURWAVE_KERNELS, ): raise ValueError(f"Unknown MXFP8 GEMM kernel: {config.kernel}") @@ -188,6 +195,8 @@ def _load_tuned_configs() -> dict[ ) ): raise ValueError(f"Unsupported decode MXFP8 GEMM config: {row}") + if config.kernel == _DECODE_SPLITK_KERNEL and not _splitk_config_ok(*shape[:2], config): + raise ValueError(f"Unsupported split-K decode MXFP8 GEMM config: {row}") if config.kernel == _STAGED_KERNEL: tiles = (64, 128) if config.waves_n == 2 else (128, 256) if not ( @@ -210,6 +219,41 @@ def _load_tuned_configs() -> dict[ } +def _splitk_config_ok(n: int, k: int, config: _Config) -> bool: + steps = k // 128 + return ( + config.tile_m % 16 == 0 + and config.waves_m >= 1 + and config.tile_n % (16 * config.waves_m) == 0 + and n % config.tile_n == 0 + and 1 <= config.waves_n + and config.waves_m + config.waves_n <= 8 + and 2 <= config.split_k <= 8 + and k % 256 == 0 + and steps % config.split_k == 0 + and (steps // config.split_k) % 2 == 0 + and config.la_w >= 1 + and config.la_a >= 0 + ) + + +def splitk_rows( + first: int, last: int, m_blocks: int, tile_n: int, waves: int, loaders: int, split_k: int, lookahead: int, + ring: int = 0, +) -> tuple[tuple[int, _Config], ...]: + """(last row, config) of the split-K decode rows over ``first``-``last`` rows: ``m_blocks`` m-blocks of + 16 * ceil(ceil(M / m_blocks) / 16) rows, one config per m-block size.""" + rows = [] + m = first + while m <= last: + tile_m = 16 * -(-(-(-m // m_blocks)) // 16) + top = min(last, tile_m * m_blocks) + rows.append((top, _Config(tile_m, tile_n, 128, 0, 0, split_k, _DECODE_SPLITK_KERNEL, waves, loaders, + lookahead, ring))) + m = top + 1 + return tuple(rows) + + def tuned_shapes(out_dtype: torch.dtype = torch.bfloat16) -> frozenset[tuple[int, int]]: """The (N, K) weight shapes whose table has rows tuned for ``out_dtype`` (bf16 or fp16 output). A caller that switches its linears to this entry by shape switches @@ -332,8 +376,8 @@ def _fourwave_launcher( # Every 4-wave launch asks vLLM's ``vllm.v1.worker.ubatching.dbo_enabled()`` (true -# while a step runs as two microbatches; the TP4 deployments split prefill steps of >= 8192 -# tokens, so a microbatch has >= 4096 rows). A server passes that query with +# while a step runs as two microbatches, from the server's DBO prefill threshold on, e.g. 8192 +# tokens: microbatches of >= 4096 rows). A server passes that query with # ``set_dbo_query``; without one, ``DBO`` answers (tests and benchmarks set it). DBO = False _dbo_query = None @@ -448,6 +492,77 @@ def _reduce_launcher(_constexprs): return exe +@functools.cache +def _splitk_launcher(n, k, config: _Config): + from .mxfp8_gemm_decode_splitk import compile_mxfp8_gemm_decode_splitk + + return compile_mxfp8_gemm_decode_splitk( + N=n, K=k, RT=config.tile_m // 16, NIW=config.tile_n // (16 * config.waves_m), NWM=config.waves_m, + NL=config.waves_n, KS=config.split_k, LW=config.la_w, R=config.la_a, + ) + + +# The split-K decode rows' epoch counters: one zeroed set per (device, stream), the stream a call is +# launched or captured on. Calls on one stream never overlap, so they share a set; a graph uses its +# capture stream's set and must not replay concurrently with another graph captured on that stream +# (vLLM captures and replays on its one compute stream). +_SPLITK_COUNTER_SETS = 64 +_SPLITK_COUNTER_LOCK = threading.Lock() +_SPLITK_POOL: dict = {} +_SPLITK_COUNTERS: dict = {} + + +def _splitk_counters(device: torch.device, dwords: int) -> torch.Tensor: + from .mxfp8_gemm_decode_splitk import CNT_STRIDE, GRID_MAX + + size = GRID_MAX // 2 * 8 * CNT_STRIDE # >= 2 splits, <= 8 waves per workgroup + if dwords > size: + raise ValueError(f"split-K decode row needs {dwords} counter dwords (> {size})") + key = (device, torch.cuda.current_stream(device).cuda_stream) + cnt = _SPLITK_COUNTERS.get(key) + if cnt is not None: + return cnt + with _SPLITK_COUNTER_LOCK: + cnt = _SPLITK_COUNTERS.get(key) + if cnt is not None: + return cnt + pool = _SPLITK_POOL.get(device) + if pool is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("split-K decode MXFP8 GEMM: first call inside CUDA graph capture; run it once " + "eagerly on the device first (warmup_mxfp8_gemm_weight does)") + pool = torch.zeros(_SPLITK_COUNTER_SETS * size, dtype=torch.int32, device=device) + torch.cuda.current_stream(device).synchronize() + _SPLITK_POOL[device] = pool + used = sum(1 for d, _ in _SPLITK_COUNTERS if d == device) + if used >= _SPLITK_COUNTER_SETS: + raise RuntimeError(f"split-K decode MXFP8 GEMM: more than {_SPLITK_COUNTER_SETS} streams on {device}") + cnt = _SPLITK_COUNTERS[key] = pool[used * size:(used + 1) * size] + return cnt + + +def _launch_splitk(config, activation, activation_scale, weight, weight_scale, out): + """One launch of the split-K decode kernel over the whole [M, N] bf16 output (its own reduce).""" + from amd_kernels.common.tensor_shim import _run_compiled + + from .mxfp8_gemm_decode_splitk import GRID_MAX, geometry + + m, n = out.shape + k = weight.shape[1] + grid, n_mb, counters, slab_bytes = geometry( + n, m, config.tile_m // 16, config.tile_n // (16 * config.waves_m), config.waves_m, config.split_k + ) + if grid > GRID_MAX: + raise ValueError(f"split-K decode row {config} at M {m} needs {grid} > {GRID_MAX} workgroups") + u8 = lambda t: t.view(torch.uint8).view(-1) # noqa: E731 + slab = torch.empty(slab_bytes, dtype=torch.uint8, device=out.device) + _run_compiled( + _splitk_launcher(n, k, config), u8(activation), u8(weight), u8(out), u8(activation_scale), + u8(weight_scale), slab, u8(_splitk_counters(out.device, counters)), m, + activation_scale.numel() * activation_scale.element_size(), n_mb, grid, torch.cuda.current_stream(), + ) # fmt: skip + + class SkPartials: """Split-K partials consumed by a fused epilogue or standalone reduce.""" @@ -568,10 +683,14 @@ def _preshuffle_activation_scale(scale: torch.Tensor) -> torch.Tensor: return shuffle_scale_a16w4(scale, 1, False) +# the o_proj (N, K) whose callers may pass their own output arena (``out=``) +_OPROJ_OUT_SHAPES = frozenset({(6144, 2048), (6144, 4096)}) + + def _validate_oproj_output(out, activation, weight, weight_scale, dtype, m, n, k, activation_scale=None): """Validate a caller-owned contiguous BF16 o_proj arena before any kernel writes it.""" - if (n, k) != (6144, 2048) or dtype != torch.bfloat16: - raise ValueError("out= supports BF16 o_proj N6144 K2048") + if (n, k) not in _OPROJ_OUT_SHAPES or dtype != torch.bfloat16: + raise ValueError("out= supports BF16 o_proj N6144 K2048 / K4096") if out.shape != (m, n) or out.dtype != dtype or out.device != activation.device or not out.is_contiguous(): raise ValueError("out= must be contiguous [M, 6144] BF16 on the activation device") inputs = (activation, weight, weight_scale) + (() if activation_scale is None else (activation_scale,)) @@ -592,7 +711,7 @@ def mxfp8_linear( activation quantizer is the one the table's kernel wants (the decode kernel reads the codes in the weight's (16, 16) layout, the others row-major). ``config``: run this configuration instead of the table's (tests, benchmarks). ``out`` may supply a - contiguous BF16 o_proj arena (N6144 K2048); + contiguous BF16 o_proj arena (N6144, K 2048 or 4096); the returned tensor is that same object. No output copy is made. """ m, k = x.shape @@ -620,12 +739,15 @@ def mxfp8_linear( True, ) return out - if config.kernel == _DECODE_KERNEL: + if config.kernel in (_DECODE_KERNEL, _DECODE_SPLITK_KERNEL): if out is not None: _validate_oproj_output(out, x, weight, weight_scale, out_dtype, m, n, k) activation, activation_scale = mxfp8_e4m3_quantize_shuffled(x, config.tile_m) out = torch.empty((m, n), dtype=out_dtype, device=x.device) if out is None else out - _launch_decode(config, activation, activation_scale, weight, weight_scale, out) + if config.kernel == _DECODE_SPLITK_KERNEL: + _launch_splitk(config, activation, activation_scale, weight, weight_scale, out) + else: + _launch_decode(config, activation, activation_scale, weight, weight_scale, out) return out activation, activation_scale = mxfp8_e4m3_quantize_aiter(x) return mxfp8_gemm( @@ -648,8 +770,8 @@ def mxfp8_gemm( """``A @ W^T`` for a row-major MXFP8 activation (codes [M, K], scales [M, K/32] or preshuffled) and a preshuffled weight. Row-major codes never take a decode row (those read the (16, 16) layout: ``mxfp8_gemm_shuffled``, or ``mxfp8_linear`` from bf16). - ``out`` may supply a contiguous BF16 o_proj arena (N6144 K2048); the returned tensor is - that same object. ``config``: run this configuration instead of the table's (tests, + ``out`` may supply a contiguous BF16 o_proj arena (N6144, K 2048 or 4096); the returned + tensor is that same object. ``config``: run this configuration instead of the table's (tests, benchmarks).""" m, k = activation.shape n = weight.shape[0] @@ -749,7 +871,8 @@ def mxfp8_gemm_shuffled( ([ceil(m / tile) * tile, K], the tile from ``prequant_code_tile_rows``) and scales in AITER's shuffled layout ([ceil(rows / 32) * 32, K / 32]). With ``defer_reduce``, TP4 QKV also tunes the split for a fused norm/RoPE/insert consumer: return - ``SkPartials`` where measured faster, keeping the producer's code layout. + ``SkPartials`` where measured faster, keeping the producer's code layout; a split-K + decode row sums its splits in the kernel and returns bf16 either way. An explicit ``config`` (tests/benchmarks) overrides both tables.""" n, k = weight.shape[0], activation.shape[1] held = activation.shape[0] @@ -776,6 +899,9 @@ def mxfp8_gemm_shuffled( f"{tuple(activation_scale.shape)} do not match M {m}, tile {config.tile_m}" ) out = torch.empty((m, n), dtype=out_dtype, device=activation.device) + if config.kernel == _DECODE_SPLITK_KERNEL: + _launch_splitk(config, activation, activation_scale, weight, weight_scale, out) + return out result = _launch_decode( config, activation, diff --git a/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode.py b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode.py index 1179618..e7cd419 100644 --- a/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode.py +++ b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode.py @@ -314,6 +314,7 @@ def compile_mxfp8_gemm_decode( OWNER_READ: bool | None = None, OUT_CM: int | None = None, INTERLEAVE_PRIME: bool | None = None, + GROUPED: bool | None = None, ): """Kernel for one (N, K) weight, ``RT`` 16-row tiles per workgroup (M <= 16*RT per m-block), ``NI`` 16-column tiles per workgroup, ``KW`` K-waves. Grid = @@ -342,6 +343,9 @@ def compile_mxfp8_gemm_decode( source issues them (off, the scheduler hoists them: a little faster while the registers last). The tuned rows carry the values the sweep found clean and fastest. + ``GROUPED`` forces the grouped XCD order of the m-blocks on or off (a shape folder's rows, + sweeps); None: the built-in rows below. + ``INTERLEAVE_PRIME=False`` runs the preceding W-first burst prologue for A/B tests. Measured rows otherwise alternate one W step and its A fragments (A first for o_proj). @@ -416,7 +420,7 @@ def compile_mxfp8_gemm_decode( or (N == 6144 and K == 2048 and NI == 6 and RT in (7, 8) and KW == 2 and w_cm == W_CACHE_MOD) ) - ) + ) if GROUPED is None else (bool(GROUPED) and KS == 1 and not A_BF16) @fx.struct class Shared: diff --git a/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode_splitk.py b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode_splitk.py new file mode 100644 index 0000000..5a8ce17 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_decode_splitk.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""gfx950 MXFP8 decode GEMM with K split over workgroups and an in-kernel fixed-order reduce: + + out[m, n] = sum_k A[m, k] * W[n, k] (E4M3 codes, per-32 E8M0 scales, bf16 out) + +Inputs are the decode kernel's (``mxfp8_gemm_decode``): A codes in the (16, 16) layout, rows padded to +``n_mb * 16 * RT``, A scales in AITER's shuffled layout (rows padded to 32), the preshuffled W and W scales. + +Workgroup = one (m-block, column block, K split): BM = 16 * RT rows, BN = 16 * NIW * NWM columns, K steps +[kz * SWG, (kz + 1) * SWG) of 128. + - NWM MFMA waves: wave w owns the NIW 16-column tiles [w * NIW, (w + 1) * NIW) x all RT row tiles (accumulators in + AGPRs); it streams its own W into VGPRs (non-temporal, LW steps ahead), every W scale dword of the split first: + a scale issued behind later codes would make the in-order vmcnt wait drain the W lookahead. + - NL loader waves stream the workgroup's A slice into an LDS ring of R step slots (``buffer_load ... lds``, 1 KB + code blocks and 256 B scale blocks, exact vmcnt waits); the MFMA waves read A with ds_read_b128. + - One ``s_barrier`` per K step: loaders pass it once step s has landed, MFMA waves once their reads of step s - 1 + are issued; after it the loaders refill step s - 1's slot with step s + R - 1. +An m-block starting on an odd 16-row tile re-pairs its A scale bytes (the decode kernel's repair; RT = 1 shifts). + +Hand-off (README rule 7: no atomic accumulation): wave w of split kz cuts its RT * NIW tiles into KS groups, stores +every group but group kz as fp32 partials (1 KB per 16x16 tile, the MFMA lane layout) into the slab, drains them +(vmcnt(0)) and arrives on the (tile, wave) counter. Once the epoch is complete it sums group kz over the splits in +split order ((P0 + P1) + P2) + ... (its own partial from AGPRs), rounds once to bf16 and stores. Every output is +the same KS sequential MFMA chains over equal K ranges summed in split order, whichever split reduces it: the decode +kernel's bytes where its K-waves x splits partition K the same way (KS 3 = the kw3 rows). + +Epoch counters: each call adds ``EPOCH`` to every counter it uses (arrival weights EPOCH // KS, plus one for the +first EPOCH % KS splits), so they stay a multiple of EPOCH between calls and are never reset; calls sharing a +counter set must not overlap. + +Placement: workgroup b = (g * KS + kz) * 8 + x with g = (nb // 8) * n_mb + mb, x = nb % 8, so the KS splits of a +tile (and the m-blocks of a column block) share b % 8. The partials use the default cache policy and are read from +the writer's L2: this needs the workgroups of one b % 8 on one XCD, which the gfx950 round-robin dispatch gives +(tests/gemm/mxfp8_common/test_decode_splitk.py checks it next to a concurrent kernel). Coherent (sc1) partials +cost 0.1-0.6 us per call. The wait needs the KS splits of a tile co-resident: grid <= 256 workgroups. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .mxfp8_gemm_4wave import _asm_void, _uniform_i32 +from .mxfp8_gemm_decode import PRELOAD_HINTS, W_CACHE_MOD, _addr, _mfma, _pack8, _pin_accumulators, _raw + +LDS_MAX = 160 * 1024 +CNT_STRIDE = 32 # dwords: one counter per 128-byte line +EPOCH = 8 # what one call adds to each counter it uses: a power of 2 >= KS +GRID_MAX = 256 # every workgroup resident at once (the epoch wait) +VMCNT_MAX = 63 +# 8 waves put two on each SIMD: 256 VGPR + AGPR per wave. Past the budget the allocator parks MFMA operands in AGPRs +# and the copy back races the inline-asm MFMA (wrong sums; ``mxfp8_gemm_decode.VGPR_BUDGET``). +REG_BUDGET = 256 + + +def geometry(N, M, RT, NIW, NWM, KS): + """(grid, m-blocks, counter dwords, slab bytes) of one call.""" + nb = N // (16 * NIW * NWM) + n_mb = -(-M // (16 * RT)) + grid = -(-nb // 8) * 8 * n_mb * KS + return grid, n_mb, (grid // KS) * NWM * CNT_STRIDE, grid * NWM * RT * NIW * 1024 + + +@launcher +def compile_mxfp8_gemm_decode_splitk(*, N, K, RT, NIW, NWM, NL, KS, LW, R=0): + """Launcher for one (N, K) and tiling: ``R`` ring slots (0: as many as the LDS holds).""" + KSTEPS = K // 128 + assert K % 256 == 0 and 2 <= KS <= EPOCH and KSTEPS % KS == 0 and (KSTEPS // KS) % 2 == 0, (K, KS) + SWG = KSTEPS // KS + BN = 16 * NIW * NWM + assert N % BN == 0, (N, BN) + NB = N // BN + BM = 16 * RT + SCB = K // 256 + RP = (RT + 1) // 2 # 32-row scale groups + CP = NIW if NIW % 2 else NIW // 2 + NT = RT * NIW # 16x16 tiles per MFMA wave + SLOT = RT * 2048 + RP * 256 + R = min(R or (LDS_MAX // SLOT), SWG) + LW = min(LW, SWG) + NWAVE = NWM + NL + assert 2 <= R and R * SLOT <= LDS_MAX and NWAVE <= 8, (R, SLOT, NWAVE) + assert NWAVE <= 4 or NT * 4 + 8 * NIW * (LW + 1) + 40 <= REG_BUDGET, ("registers", NT, NIW, LW) + + def step_blocks(s): + bl = [("a", rt, h) for rt in range(RT) for h in range(2)] + if s % 2 == 0: + bl += [("s", rp, 0) for rp in range(RP)] + return bl + + def loader_blocks(l, s): + return [b for j, b in enumerate(step_blocks(s)) if j % NL == l] + + for l in range(NL): + assert sum(len(loader_blocks(l, s)) for s in range(R - 1)) <= VMCNT_MAX, ("vmcnt", l) + groups = [list(range(g * NT // KS, (g + 1) * NT // KS)) for g in range(KS)] + + @fx.struct + class Shared: + ring: fx.Array[fx.Uint8, R * SLOT, 16] + + name = f"mxfp8_gemm_decode_splitk_n{N}_k{K}_rt{RT}_niw{NIW}_nwm{NWM}_nl{NL}_ks{KS}_lw{LW}_r{R}" + + @flyc.kernel(name=name, known_block_size=[64 * NWAVE, 1, 1]) + def kernel( + i32_m: fx.Int32, + a_scale_bytes: fx.Int32, + n_mb: fx.Int32, + arg_w: fx.Int64, + arg_ws: fx.Int64, + arg_as: fx.Int64, + arg_a: fx.Int64, + arg_out: fx.Int64, + arg_slab: fx.Int64, + arg_cnt: fx.Int64, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + l16, q16 = lane % 16, lane // 16 + x = bx % 8 + kz = (bx // 8) % KS + g = bx // (8 * KS) + tile = g * 8 + x + mb = g % n_mb + nb = (g // n_mb) * 8 + x + lds_base = fx.Int32(fx.ptrtoint(smem.ring.ptr)) + kstep0 = kz * SWG + + if nb < NB: + if wave >= NWM: + a_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_a, num_records_bytes=n_mb * (BM * K)) + as_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_as, num_records_bytes=a_scale_bytes) + a_voff = [(mb * RT + rt) * (16 * K) + lane * 16 for rt in range_constexpr(RT)] + s_voff = [(((mb * RT) // 2 + rp) * SCB * 64 + lane) * 4 for rp in range_constexpr(RP)] + ring_s = _uniform_i32(lds_base) + + def issue(s, blk): + kind, i, h = blk + slot = s % R + if const_expr(kind == "a"): + m0 = ring_s + slot * SLOT + i * 2048 + h * 1024 + soff = _uniform_i32((kstep0 + s) * 2048 + h * 1024) + _asm_void([fx.as_ir_value(fx.Int32(m0)), fx.as_ir_value(fx.Int32(a_voff[i])), + fx.as_ir_value(a_rsrc), fx.as_ir_value(fx.Int32(soff))], + "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds", "s,v,s,s", "~{m0}") + else: + m0 = ring_s + slot * SLOT + RT * 2048 + i * 256 + soff = _uniform_i32(((kstep0 + s) // 2) * 256) + _asm_void([fx.as_ir_value(fx.Int32(m0)), fx.as_ir_value(fx.Int32(s_voff[i])), + fx.as_ir_value(as_rsrc), fx.as_ir_value(fx.Int32(soff))], + "s_mov_b32 m0, $0\nbuffer_load_dword $1, $2, $3 offen lds", "s,v,s,s", "~{m0}") + + for l in range_constexpr(NL): + if wave == NWM + l: + for s in range_constexpr(R - 1): + for blk in loader_blocks(l, s): + issue(s, blk) + for s in range_constexpr(SWG): + younger = sum(len(loader_blocks(l, t)) for t in range(s + 1, min(s + R - 1, SWG))) + _asm_void([], f"s_waitcnt vmcnt({younger})", "") + _asm_void([], "s_barrier", "") + if const_expr(s + R - 1 < SWG): + for blk in loader_blocks(l, s + R - 1): + issue(s + R - 1, blk) + else: + w_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=N * K) + ws_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_ws, num_records_bytes=(N // 32) * SCB * 256) + n16 = [nb * (NIW * NWM) + wave * NIW + ni for ni in range_constexpr(NIW)] + w_dw = [(n16[ni] * (16 * K) + kstep0 * 2048 + q16 * 256 + l16 * 16) // 4 for ni in range_constexpr(NIW)] + sc_k = (kstep0 // 2) * 64 + ws_dw = [((n16[cp if NIW % 2 else 2 * cp] // 2) * SCB) * 64 + sc_k + lane for cp in range_constexpr(CP)] + w_shift = [(n16[ni] % 2) * 8 for ni in range_constexpr(NIW)] if NIW % 2 else None + + def load_pair(dw, s): + kw = dict(vec_width=4, dtype=fx.Int32, cache_modifier=W_CACHE_MOD) + return _pack8(buffer_ops.buffer_load(w_rsrc, dw, soffset_bytes=s * 2048, **kw), + buffer_ops.buffer_load(w_rsrc, dw, soffset_bytes=s * 2048 + 1024, **kw)) + + def load_w_scales(p): + sc = [fx.Int32(buffer_ops.buffer_load(ws_rsrc, d, vec_width=1, dtype=fx.Int32, + soffset_bytes=p * 256)) for d in ws_dw] + if const_expr(w_shift is not None): + sc = [v.shrui(sh) for v, sh in zip(sc, w_shift)] + return sc + + ring_v = lds_base + lane * 16 + sring_v = lds_base + lane * 4 + + def lds_ptr(addr): + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int32(addr))) + + def read_a(s, rt): + base = ring_v + (s % R) * SLOT + rt * 2048 + return _pack8(Vec(llvm.load(T.vec(4, T.i32), lds_ptr(base), alignment=16)), + Vec(llvm.load(T.vec(4, T.i32), lds_ptr(base + 1024), alignment=16))) + + def read_a_scale(s, rp): + return fx.Int32(llvm.load(T.i32, lds_ptr(sring_v + (s % R) * SLOT + RT * 2048 + rp * 256), + alignment=4)) + + acc = [[None] * NIW for _ in range(RT)] + wq = [None] * SWG + wsc = [load_w_scales(p) for p in range_constexpr(SWG // 2)] + fx.rocdl.sched_barrier(0) + for s in range_constexpr(LW): + wq[s] = [load_pair(w_dw[ni], s) for ni in range_constexpr(NIW)] + fx.rocdl.sched_barrier(0) + sw = sa = None + for s in range_constexpr(SWG): + if const_expr(s + LW < SWG): + wq[s + LW] = [load_pair(w_dw[ni], s + LW) for ni in range_constexpr(NIW)] + fx.rocdl.sched_barrier(0) + _asm_void([], "s_barrier", "") + fx.rocdl.sched_barrier(0) + bb = wq[s] + if const_expr(s % 2 == 0): + sw = wsc[s // 2] + sa = [read_a_scale(s, rp) for rp in range_constexpr(RP)] + if const_expr(RT == 1): + sa = [sa[0].shrui(((mb % 2) * 8))] + if const_expr(RT > 1 and RT % 2 == 1): + # an m-block starting on an odd 16-row tile: re-pair the scale bytes to the even layout + # the MFMA immediates assume (the decode kernel's repair) + odd = (mb * RT) % 2 == 1 + raw = sa + sa = [odd.select((raw[p].shrui(fx.Int32(8)) & fx.Int32(0x00FF00FF)) + | ((raw[min(p + 1, RP - 1)] & fx.Int32(0x00FF00FF)) << 8), raw[p]) + for p in range_constexpr(RP)] + aq = [None] * RT + for rt in range_constexpr(min(2, RT)): + aq[rt] = read_a(s, rt) + for rt in range_constexpr(RT): + if const_expr(rt + 2 < RT): + aq[rt + 2] = read_a(s, rt + 2) + for ni in range_constexpr(NIW): + acc[rt][ni] = _mfma(bb[ni], aq[rt], acc[rt][ni], sw[ni if NIW % 2 else ni // 2], + sa[rt // 2], 0 if NIW % 2 else ni % 2, rt % 2, s % 2) + aq[rt] = None + fx.rocdl.sched_barrier(0) + wq[s] = None + + flat = _pin_accumulators([acc[rt][ni] for rt in range(RT) for ni in range(NIW)]) + slab_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_slab, num_records_bytes=0x7FFFFFFF) + out_rsrc = buffer_ops.create_buffer_resource_from_addr(arg_out, num_records_bytes=i32_m * (N * 2)) + + def slab_off(k): + return ((tile * KS + k) * NWM + wave) * (NT * 1024) + lane * 16 + + def store_out(i, v): + row = mb * BM + (i // NIW) * 16 + l16 + col = n16[i % NIW] * 16 + q16 * 4 + buffer_ops.buffer_store(v.to(fx.BFloat16).bitcast(fx.Int32), out_rsrc, (row * N + col) * 2, + offset_is_bytes=True, mask=row < i32_m) + + cnt_addr = arg_cnt + fx.Int64((tile * NWM + wave) * (CNT_STRIDE * 4)) + inc = (kz < EPOCH % KS).select(fx.Int32(EPOCH // KS + 1), fx.Int32(EPOCH // KS)) + + def atomic_add(value): + ptr = fx.inttoptr(fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + cnt_addr) + return fx.Int32(llvm.AtomicRMWOp(llvm.AtomicBinOp.add, ptr.llvm_ptr, fx.Int32(value).ir_value(), + llvm.AtomicOrdering.monotonic, syncscope="agent", + alignment=4).result) + + for r in range_constexpr(KS): + if kz == r: + mine = groups[r] + for i in [i for gi in range(KS) if gi != r for i in groups[gi]]: + buffer_ops.buffer_store(Vec(flat[i]).bitcast(fx.Int32), slab_rsrc, slab_off(r) + i * 1024, + offset_is_bytes=True) + fx.rocdl.s_waitcnt(vmcnt=0) + if lane == 0: + old = atomic_add(inc) + base = old - (old & (EPOCH - 1)) # i32 wrap included + arrived = old + inc - base + while arrived < EPOCH: + arrived = atomic_add(0) - base + parts = [[None if k == r else Vec(buffer_ops.buffer_load( + slab_rsrc, (slab_off(k) + i * 1024) // 4, vec_width=4, dtype=fx.Int32)).bitcast(fx.Float32) + for k in range_constexpr(KS)] for i in mine] + for j, i in enumerate(mine): + v = None + for k in range_constexpr(KS): + p = Vec(flat[i]) if k == r else parts[j][k] + v = p if v is None else v + p + store_out(i, v) + + @flyc.jit + def launch(A: fx.Tensor, W: fx.Tensor, OUT: fx.Tensor, A_scale: fx.Tensor, W_scale: fx.Tensor, + SLAB: fx.Tensor, CNT: fx.Tensor, i32_m: fx.Int32, a_scale_bytes: fx.Int32, n_mb: fx.Int32, + grid: fx.Int32, stream: fx.Stream): + kernel(i32_m, a_scale_bytes, n_mb, _addr(W), _addr(W_scale), _addr(A_scale), _addr(A), _addr(OUT), + _addr(SLAB), _addr(CNT)).launch(grid=(fx.Int64(grid), 1, 1), block=(64 * NWAVE, 1, 1), stream=stream) + + launch.kernel_name = name + launch.compile_hints = PRELOAD_HINTS + return launch diff --git a/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_tuned.py b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_tuned.py index 3d6bdc8..09ebfb1 100644 --- a/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_tuned.py +++ b/amd_kernels/gemm/mxfp8_common/mxfp8_gemm_tuned.py @@ -3,17 +3,23 @@ """The tuned table ``gemm._select_config`` reads (``gemm._load_tuned_configs``), as CSV text in a Python string, so the package carries no data file. -Measured TP4 rows: QKV -2560 x 6144 at 385-1536 (except 513-576), 1537-2048 and 3073-4096, o_proj 6144 x 2048 at 33-48, 65-80, 129-160, -161-192, 257-320, 385-448 and 1281-3328, o_proj 6144 x 8192 at 417-4095; the bf16 rows of -the TP4 shared expert, gate_up 1536 x 6144 and down 6144 x 768 at 1-256. +Measured rows: QKV +2560 x 6144 at 385-1536 (except 513-576), 1537-4096, gate_up / down 6144 x 6144 at 257-320, +385-448 and 641-1280, o_proj 6144 x 2048 at 33-48, 65-80, 129-160, +161-192, 257-320, 385-448 and 1281-3328, o_proj 6144 x 8192 at 417-4095; the bf16 rows of the shared expert +gate_up 1536 x 6144 and down 6144 x 768 at 1-256; every bf16 row of 3072 x 6144, 6144 x 1536, 4608 x 6144 and +12288 x 6144 (sweep: benchmarks/gemm/mxfp8_common/sweep_rows.py). The split-K decode rows (6144 x 6144 at 33-256, +3072 x 6144 and 4608 x 6144 at 33-256, 12288 x 6144 at 49-224, 2-4% slower at 225-256) beat the rows they replaced +at every multiple of 8 and window start, 6144 x 6144 at every row count; its 225-256 row looks 2 W steps ahead (3 +spills a VGPR, 1% slower) (benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py). The 4992 x 6144 and 6144 x 4096 rows (``min_m`` +65536) only list those shapes in ``gemm.tuned_shapes``; their folders' ``host.select`` picks the rows. Graph-replay tuned on one MI355X with pointer-distinct rotating weights above the LLC. One row per (M upper bound, N, K, output dtype): ``M`` is an inclusive upper bound, the rows of a shape are scanned in M order and the first bound >= M decides; ``out_dtype`` ``any`` rows are the AITER defaults, ``bf16`` / ``fp16`` rows override them within their measured range (``min_m`` narrows a row's range from below). Columns: ``kernel`` the variant (``aiter``, ``staged_ab``, ``decode``, ``decode_fq``, -``4wave``, ``4wave_nt``), then the ``gemm._Config`` fields. +``decode_splitk``, ``4wave``, ``4wave_nt``), then the ``gemm._Config`` fields. """ TUNED_CSV = """\ @@ -234,7 +240,7 @@ 1408,2560,6144,bf16,decode,176,80,128,3,0,0,0,1,,,,0 1536,2560,6144,bf16,decode,192,80,128,3,0,0,0,1,,,,0 2048,2560,6144,bf16,4wave,128,256,128,2,2,1,1,1,, -3072,2560,6144,bf16,4wave_nt,128,256,128,2,2,1,1,1,, +3072,2560,6144,bf16,4wave_nt,256,128,128,2,2,1,1,1,, 4096,2560,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, 10240,2560,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, 65536,2560,6144,bf16,4wave,256,256,128,2,2,1,1,1,, @@ -257,18 +263,20 @@ 8,6144,6144,bf16,decode_fq,8,32,128,2,0,0,0,1,6,3 16,6144,6144,bf16,decode,16,32,128,1,0,0,0,1,7,7 32,6144,6144,bf16,decode,32,32,128,1,0,0,0,1,, -64,6144,6144,bf16,decode,64,32,128,3,0,0,0,1,, -80,6144,6144,bf16,decode,48,48,128,3,0,0,0,1,, -96,6144,6144,bf16,decode,96,32,128,3,0,0,0,1,, -128,6144,6144,bf16,decode,64,48,128,3,0,0,0,1,, -192,6144,6144,bf16,decode,96,48,128,4,0,0,0,1,, -224,6144,6144,bf16,decode,112,48,128,3,0,0,0,1,, -256,6144,6144,bf16,decode,128,48,128,3,0,0,0,1,, +64,6144,6144,bf16,decode_splitk,32,96,128,3,3,0,0,2,3,0,, +96,6144,6144,bf16,decode_splitk,48,96,128,3,3,0,0,2,3,0,, +128,6144,6144,bf16,decode_splitk,64,96,128,3,3,0,0,2,3,0,, +160,6144,6144,bf16,decode_splitk,80,96,128,3,3,0,0,2,3,0,, +192,6144,6144,bf16,decode_splitk,96,96,128,3,3,0,0,2,3,0,, +224,6144,6144,bf16,decode_splitk,112,96,128,3,3,0,0,2,3,0,, +256,6144,6144,bf16,decode_splitk,128,96,128,3,3,0,0,2,2,0,, +320,6144,6144,bf16,decode,48,96,128,2,0,0,0,1,,,,0 384,6144,6144,bf16,decode,96,96,128,3,0,0,0,1,, +448,6144,6144,bf16,decode,64,96,128,2,0,0,0,1,,,,0 512,6144,6144,bf16,decode,128,96,128,4,0,0,0,1,, 640,6144,6144,bf16,decode,160,96,128,3,0,0,0,1,, -768,6144,6144,bf16,decode,96,96,128,1,0,0,0,1,, -1280,6144,6144,bf16,4wave,128,256,128,2,2,1,1,1,, +1024,6144,6144,bf16,decode,128,96,128,2,0,0,0,1,,,,0 +1280,6144,6144,bf16,4wave_nt,256,128,128,2,2,1,1,1,, 65536,6144,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, 8,6144,2048,bf16,decode_fq,8,32,128,2,0,0,0,1,, 16,6144,2048,bf16,decode_fq,16,32,128,2,0,0,0,1,, @@ -332,4 +340,104 @@ 160,6144,768,bf16,decode_fq,32,64,128,3,0,0,0,1,, 192,6144,768,bf16,decode,48,64,128,1,0,0,0,1,,,,0 256,6144,768,bf16,decode_fq,64,96,128,3,0,0,0,1,, +4,3072,6144,bf16,decode_fq,4,16,128,2,0,0,0,1,, +8,3072,6144,bf16,decode_fq,8,16,128,3,0,0,0,1,, +16,3072,6144,bf16,decode_fq,16,48,128,1,0,0,0,8,, +32,3072,6144,bf16,decode_fq,32,48,128,2,0,0,0,4,, +48,3072,6144,bf16,decode_splitk,48,48,128,3,3,0,0,4,3,0,, +64,3072,6144,bf16,decode_splitk,64,48,128,3,3,0,0,4,3,0,, +80,3072,6144,bf16,decode_splitk,80,48,128,3,3,0,0,4,3,0,, +96,3072,6144,bf16,decode_splitk,96,48,128,3,3,0,0,4,3,0,, +112,3072,6144,bf16,decode_splitk,112,48,128,3,3,0,0,4,3,0,, +128,3072,6144,bf16,decode_splitk,128,48,128,3,3,0,0,4,3,0,, +144,3072,6144,bf16,decode_splitk,144,48,128,3,3,0,0,4,3,0,, +160,3072,6144,bf16,decode_splitk,160,48,128,3,3,0,0,4,3,0,, +176,3072,6144,bf16,decode_splitk,176,48,128,3,3,0,0,4,3,0,, +192,3072,6144,bf16,decode_splitk,192,48,128,3,3,0,0,4,3,0,, +208,3072,6144,bf16,decode_splitk,208,48,128,3,3,0,0,4,3,0,, +224,3072,6144,bf16,decode_splitk,112,48,128,3,3,0,0,2,4,0,, +256,3072,6144,bf16,decode_splitk,128,48,128,3,3,0,0,2,4,0,, +320,3072,6144,bf16,decode,80,48,128,3,0,0,0,1,, +384,3072,6144,bf16,decode,96,48,128,3,0,0,0,1,, +448,3072,6144,bf16,decode,112,48,128,3,0,0,0,1,,,,0 +512,3072,6144,bf16,decode,128,48,128,3,0,0,0,1,,,,0 +640,3072,6144,bf16,decode,160,48,128,3,0,0,0,1,,,,0 +768,3072,6144,bf16,decode,192,48,128,3,0,0,0,1,,,,0 +896,3072,6144,bf16,aiter,128,64,256,0,0,2,4,1,, +1024,3072,6144,bf16,staged_ab,128,256,128,2,4,2,4,2,, +1280,3072,6144,bf16,staged_ab,128,128,128,2,4,2,4,1,, +2560,3072,6144,bf16,4wave_nt,256,128,128,2,2,1,1,1,, +65536,3072,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, +8,6144,1536,bf16,decode_fq,8,32,128,3,0,0,0,1,, +16,6144,1536,bf16,decode_fq,16,32,128,3,0,0,0,1,, +32,6144,1536,bf16,decode_fq,32,32,128,3,0,0,0,1,, +48,6144,1536,bf16,decode,48,32,128,1,0,0,0,1,, +64,6144,1536,bf16,decode,32,48,128,1,0,0,0,1,, +96,6144,1536,bf16,decode,48,48,128,3,0,0,0,1,, +128,6144,1536,bf16,decode,32,64,128,1,0,0,0,1,,,,0 +160,6144,1536,bf16,decode,80,48,128,3,0,0,0,1,, +192,6144,1536,bf16,decode,48,64,128,1,0,0,0,1,,,,0 +256,6144,1536,bf16,decode,64,64,128,1,0,0,0,1,,,,0 +288,6144,1536,bf16,decode,144,48,128,3,0,0,0,1,, +320,6144,1536,bf16,decode,80,96,128,2,0,0,0,1,, +384,6144,1536,bf16,decode,96,96,128,3,0,0,0,1,, +448,6144,1536,bf16,decode,64,96,128,2,0,0,0,1,,,,0 +512,6144,1536,bf16,decode,128,96,128,2,0,0,0,1,, +640,6144,1536,bf16,decode,160,96,128,2,0,0,0,1,, +768,6144,1536,bf16,decode,112,96,128,2,0,0,0,1,,,,0 +896,6144,1536,bf16,decode,112,96,128,2,0,0,0,1,,,,0 +1024,6144,1536,bf16,4wave_nt,256,128,128,2,2,1,1,1,, +1280,6144,1536,bf16,4wave_nt,128,256,128,2,2,1,1,1,, +65536,6144,1536,bf16,4wave_nt,256,256,128,2,2,1,1,1,, +4,12288,6144,bf16,decode_fq,8,32,128,1,0,0,0,1,, +8,12288,6144,bf16,decode_fq,8,48,128,4,0,0,0,1,, +16,12288,6144,bf16,decode_fq,16,48,128,3,0,0,0,1,, +32,12288,6144,bf16,decode,32,48,128,1,0,0,0,1,, +48,12288,6144,bf16,decode,48,48,128,1,0,0,0,1,, +64,12288,6144,bf16,decode_splitk,64,96,128,6,2,0,0,2,3,4,, +80,12288,6144,bf16,decode_splitk,80,96,128,6,2,0,0,2,3,4,, +96,12288,6144,bf16,decode_splitk,96,96,128,6,2,0,0,2,3,4,, +112,12288,6144,bf16,decode_splitk,112,96,128,6,2,0,0,2,3,4,, +128,12288,6144,bf16,decode_splitk,128,96,128,6,2,0,0,2,3,4,, +144,12288,6144,bf16,decode_splitk,144,96,128,6,2,0,0,2,3,4,, +160,12288,6144,bf16,decode_splitk,160,96,128,6,2,0,0,2,3,4,, +176,12288,6144,bf16,decode_splitk,176,96,128,6,2,0,0,2,3,4,, +192,12288,6144,bf16,decode_splitk,192,96,128,6,2,0,0,2,3,4,, +208,12288,6144,bf16,decode_splitk,208,96,128,6,2,0,0,2,3,4,, +224,12288,6144,bf16,decode_splitk,224,96,128,6,2,0,0,2,3,4,, +256,12288,6144,bf16,decode,128,96,128,2,0,0,0,1,, +288,12288,6144,bf16,decode,96,96,128,2,0,0,0,1,,,,0 +320,12288,6144,bf16,decode,112,96,128,2,0,0,0,1,,,,0 +384,12288,6144,bf16,decode,128,96,128,2,0,0,0,1,,,,0 +448,12288,6144,bf16,decode,160,96,128,2,0,0,0,1,,,,0 +512,12288,6144,bf16,4wave_nt,256,128,128,2,2,1,1,1,, +640,12288,6144,bf16,decode,160,96,128,2,0,0,0,1,,,,0 +768,12288,6144,bf16,decode,128,96,128,1,0,0,0,1,,,,0 +65536,12288,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, +4,4608,6144,bf16,decode_fq,4,32,128,2,0,0,0,1,, +8,4608,6144,bf16,decode_fq,8,32,128,2,0,0,0,1,, +16,4608,6144,bf16,decode_fq,16,64,128,2,0,0,0,3,, +32,4608,6144,bf16,decode,16,32,128,1,0,0,0,1,,,,0 +48,4608,6144,bf16,decode_splitk,48,64,128,4,4,0,0,3,2,0,, +64,4608,6144,bf16,decode_splitk,64,64,128,4,4,0,0,3,2,0,, +80,4608,6144,bf16,decode_splitk,80,64,128,4,4,0,0,3,2,0,, +96,4608,6144,bf16,decode_splitk,96,64,128,4,4,0,0,3,2,0,, +112,4608,6144,bf16,decode_splitk,112,64,128,4,4,0,0,3,2,0,, +128,4608,6144,bf16,decode_splitk,64,128,128,4,4,0,0,3,2,0,, +160,4608,6144,bf16,decode_splitk,80,128,128,4,4,0,0,3,2,0,, +192,4608,6144,bf16,decode_splitk,96,128,128,4,4,0,0,3,2,0,, +224,4608,6144,bf16,decode_splitk,112,128,128,4,4,0,0,3,2,0,, +256,4608,6144,bf16,decode_splitk,128,128,128,4,4,0,0,3,2,0,, +288,4608,6144,bf16,decode,48,64,128,1,0,0,0,1,,,,0 +384,4608,6144,bf16,decode,64,64,128,1,0,0,0,1,,,,0 +448,4608,6144,bf16,decode,80,64,128,2,0,0,0,1,,,,0 +512,4608,6144,bf16,decode,64,96,128,2,0,0,0,1,,,,0 +640,4608,6144,bf16,decode,128,96,128,2,0,0,0,1,,,,0 +768,4608,6144,bf16,decode,128,64,128,2,0,0,0,1,,,,0 +896,4608,6144,bf16,decode,112,96,128,2,0,0,0,1,,,,0 +1024,4608,6144,bf16,decode,128,96,128,2,0,0,0,1,,,,0 +1536,4608,6144,bf16,4wave,256,128,128,2,2,1,1,1,, +65536,4608,6144,bf16,4wave_nt,256,256,128,2,2,1,1,1,, +65536,4992,6144,bf16,aiter,128,128,128,0,0,2,4,1,,,65536 +65536,6144,4096,bf16,4wave_nt,256,256,128,2,2,1,1,1,,,65536 """ diff --git a/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/README.md b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/README.md new file mode 100644 index 0000000..d44723f --- /dev/null +++ b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/README.md @@ -0,0 +1,39 @@ +# MXFP8 gate_up + SwiGLU-OAI, N 12288 x K 6144, I 6144 (dense MLP, TP2, 1-16 rows) + +## What it does +`act [m, 6144] = SwiGLU-OAI(gate, up)` with `gate | up = x @ W^T` in one launch, for 1-16 rows (one 16-row code block, +the same cost at every row count) of the MiniMax-M3.1-Flash dense MLP at TP2. W is the per-rank dense gate_up weight +as the common dispatcher holds it: MXFP8 [12288, 6144], gate rows 0-6143 and up rows 6144-12287, (16, 16) codes and +AITER-layout scales (`preshuffle_mxfp8_weight`), no copy. x: E4M3 codes in the (16, 16) layout padded to 16 rows ([16, +6144]), E8M0 scales in AITER's layout ([>= 32, 192]). Each dot product rounds to bf16 before the activation; the bf16 +output feeds the down linear (N 6144, K 6144) and its own quantizer. Callers: the TP2 dense decoder block (the +target's dense layers with form `fp32`, the DSpark draft's MLP with `bf16_chain`); above the fused row bound they run +gate_up `mxfp8_linear` + `activation/swiglu_oai_mxfp8_i6144` + down. Contract: `mxfp8_common/reference.py` +`mxfp8_gateup_swiglu_bf16_reference` (fp64 dots rounded to bf16, the caller's form): per-row cosine >= 0.9999, and the +bf16 bits of the decode GEMM's activation with the same K-wave count (`host.CONFIG`: 1 for `fp32`, 3 for +`bf16_chain`). + +## Entry points +- `host.mxfp8_gateup_swiglu_bf16`: the bf16 [m, 6144] activation of 1-16 rows in one launch; the caller picks the + numerics form (`fp32` or `bf16_chain`), never M. +- `host.warmup_mxfp8_gateup_swiglu_bf16`: compiles the forms; call it before graph capture. + +`host.CONFIG` picks the kernel (K-waves, lookahead) by form. + +## Performance +One MI355X. Graph replay, 6 rotated weight copies (> 900 MB), 100 inputs, both sides alternated, median us. Baseline: +what vLLM runs at TP2 without it, Triton `tl.dot_scaled` gate_up + `swiglu_oai_split` (fp32) or the aten chain +(bf16_chain) + the same down linear. Stream roofline (6.5 TB/s): fused kernel 78.0 MB = 12.0 us, FFN 116.9 MB = 18.0 us. + +| M | fp32 FFN | baseline | bf16_chain FFN | baseline | fused kernel fp32 / bf16_chain | +|---|---|---|---|---|---| +| 1 | 22.49 | 29.39 | 22.91 | 41.23 | 14.82 / 15.29 | +| 4 | 22.80 | 29.67 | 23.22 | 49.35 | 14.84 / 15.30 | +| 7 | 23.36 | 30.14 | 23.78 | 49.76 | 14.86 / 15.32 | +| 8 | 23.55 | 30.44 | 23.98 | 50.58 | 14.86 / 15.31 | +| 14 | 25.30 | 33.28 | 25.78 | 53.44 | 14.88 / 15.34 | +| 16 | 25.52 | 33.71 | 26.00 | 53.26 | 14.88 / 15.36 | + +## Tests +- `python3 -m pytest tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144 -q` +- `python3 benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py [--sweep]` diff --git a/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py new file mode 100644 index 0000000..b0eeb90 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Paired gate/up MXFP8 GEMM + SwiGLU-OAI at 1-16 rows (dense MLP, TP2: N 12288, K 6144, I 6144).""" + +from .host import mxfp8_gateup_swiglu_bf16, warmup_mxfp8_gateup_swiglu_bf16 + +__all__ = ["mxfp8_gateup_swiglu_bf16", "warmup_mxfp8_gateup_swiglu_bf16"] diff --git a/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/gateup_swiglu_bf16.py b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/gateup_swiglu_bf16.py new file mode 100644 index 0000000..3ba35d8 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/gateup_swiglu_bf16.py @@ -0,0 +1,605 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Scaled-MFMA gate/up with paired weight tiles and BF16 SwiGLU output. + +The existing packed gate|up weight is read without another layout or copy. +Every GEMM dot rounds to BF16 before the caller-selected activation; the +BF16 activation feeds the unchanged down projection quantizer. +``KW`` K-waves sum through LDS in the order of the decode GEMM with waves_m = KW, so ``bf16_chain`` gives that +materialized GEMM's bf16 bits; its SwiGLU rounds to bf16 at every op, as the draft's aten chain. +Based on mxfp8_gemm_decode.py. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +W_CACHE_MOD = 2 # non-temporal W loads: a decode GEMM reads every weight byte once +# VGPRs the kernel may hold at once. An MFMA operand is 8 contiguous VGPRs, so the register +# allocator runs out well before all 256 are live (measured: clean at 199 live, parking +# operands in AGPRs at 202). A parked operand is copied back with v_accvgpr_read right +# before its MFMA and races the AGPR write: wrong sums, some of them only now and then. +VGPR_BUDGET = 192 +# kernel arguments preloaded into SGPRs instead of s_load'ed before the first loads +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} + + +def _addr(t): + """A tensor argument's device address (in a launch body).""" + return fx.Int64(fx.ptrtoint(fx.get_iter(t))) + + +def _lookahead(SW, RT, NI, RP, CP, vgpr_budget=None, a_vgprs=8, reserve=0, fold=1): + """(W steps, A blocks) issued ahead of the MFMAs: the longest common horizon whose + registers fit. Held at once: the operands in flight plus the one in use of each + queue (8 VGPRs per 128-K operand), a scale dword per 32-row / 32-column group for + every other W step, and the per-lane offset vector of every row / column tile. + An A block is one (step, row tile) pair, or ``fold`` steps of the folded bf16 mode.""" + budget = (vgpr_budget or VGPR_BUDGET) - (RT + NI) - reserve + best = None + for la_a in range(1, SW * RT // fold + 1): + la_w = min(SW, -(-la_a * fold // RT)) + regs = a_vgprs * (la_a + 1) + 8 * NI * (la_w + 1) + (CP + RP) * (la_w // 2 + 2) + if regs > budget: + break + best = (la_w, la_a) + assert best is not None, f"RT {RT} x NI {NI} leaves no VGPRs for the operands" + return best + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _pack8(h0, h1): + """two 4 x i32 halves -> one 8 x i32 MFMA operand (VGPRs 0-3 = K [16*klane, +16), + VGPRs 4-7 = K [64 + 16*klane, +16)).""" + a, b = Vec(h0), Vec(h1) + return Vec.from_elements( + [a[k] for k in range(4)] + [b[k] for k in range(4)], fx.Int32 + ) + + +def _mfma(w_op, a_op, acc, sw, sa, jw, ia, k2): + """One scaled 16x16x128 fp8 MFMA, accumulator pinned in AGPR (``=a`` / tied ``0``). + Operand order (W, A): lane (q, r) accumulates out[row r, columns 4q .. 4q+3]. + Scale byte of a dword = op_sel + 2 * op_sel_hi (16-row / 16-col half, 128-K half).""" + src2 = "$0" if acc is not None else "0" + asm = ( + f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {src2}, $3, $4 " + f"op_sel:[{jw},{ia},0] op_sel_hi:[{k2},{k2},0] cbsz:0 blgp:0" + ) + ops = [ + fx.as_ir_value(w_op), + fx.as_ir_value(a_op), + fx.as_ir_value(sw), + fx.as_ir_value(sa), + ] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(fx.as_ir_value(acc)) + cons += ",0" + return llvm.inline_asm( + Vec.make_type(4, fx.Float32), ops, asm, cons, has_side_effects=True + ) + + +def _pk_max_u16(a, b): + """per-16-bit-half unsigned max (v_pk_max_u16): on sign-cleared bf16 patterns that is + the value order.""" + va = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(a))).result + vb = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(b))).result + return fx.Int32(llvm.BitcastOp(T.i32, llvm.intr_umax(va, vb)).result) + + +def _amax_bf16(dwords): + """max |x| over the 2 * len(dwords) bf16 values, as a 16-bit bf16 pattern in an i32.""" + m = None + for d in dwords: + a = fx.Int32(d) & fx.Int32(0x7FFF7FFF) + m = a if m is None else _pk_max_u16(m, a) + lo, hi = m & fx.Int32(0xFFFF), m >> 16 + return (lo > hi).select(lo, hi) + + +def _i32_max(a, b): + return (a > b).select(a, b) + + +def _swap16(v): + """(even-row value, odd-row value) of this lane's 16-lane row pair (klane q, q ^ 1).""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane16_swap(pair_ty, raw, raw, False, False) + return fx.Int32(llvm.extractvalue(T.i32, res, [0])), fx.Int32( + llvm.extractvalue(T.i32, res, [1]) + ) + + +def _swap32(v): + """(low-half value, high-half value): the values of klanes 0/1 and 2/3.""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane32_swap(pair_ty, raw, raw, False, False) + return fx.Int32(llvm.extractvalue(T.i32, res, [0])), fx.Int32( + llvm.extractvalue(T.i32, res, [1]) + ) + + +def _e8m0_bf16(amax_bits): + """(scale byte, conversion scale) of a block from the bf16 pattern of its maximum. The + byte is the Triton MXFP8 quantizer's, clamp(ceil(log2(max(amax, tiny) / 448)) + 127, 0, + 254): with amax = 2^e * m, ceil(log2(amax / 448)) = e - 8 + [m > 1.75], so it is the + exponent field - 8 + [mantissa > 0x60], which (a + 0x1F) >> 7 gives in one carry; + zero, denormal and tiny maxima land on 0 as in that quantizer. The conversion divides by + 2^(byte - 127); byte 0 would make that a denormal, so such a block (maximum below + 2^-119) is converted with 2^-126 instead.""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(8)) - fx.Int32(8) + scale = ((_i32_max(t, fx.Int32(9)) - fx.Int32(8)) << 23).bitcast(fx.Float32) + return sb, scale + + +def _e8m0_bf16_pp2(amax_bits): + """NVIDIA-consistent variant of ``_e8m0_bf16``: the stored + byte IS the conversion exponent (``max(t, 9) - 8``, never 0), so the stored + scale always equals the 2^(byte - 127) its consumer decodes -- ``_e8m0_bf16`` + stores 0 for a block maximum below ~1.75*2^-119 while converting with 2^-126 + (a 2x error under an OCP consumer, a zeroed block under a ``(byte << 23)`` + one). Identical output for every bf16 block maximum above 1.75*2^-119, which + includes every nonzero normal activation.""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(9)) - fx.Int32(8) + scale = (sb << 23).bitcast(fx.Float32) + return sb, scale + + +def _row_shl(vals, n): + """Each lane takes the value of lane + n of its 16-lane row (v_mov_b32 dpp row_shl; + the last n lanes of a row get 0).""" + zero = _raw(fx.Int32(0)) + return [ + fx.Int32( + fx.rocdl.update_dpp( + T.i32, zero, _raw(fx.Int32(v)), 0x100 + n, 0xF, 0xF, True + ) + ) + for v in vals + ] + + +def _quant_bf16(dwords, scale): + """dwords of bf16 pairs -> half as many dwords of fp8 codes: x / scale, round to nearest + even (v_cvt_scalef32_pk_fp8_bf16, the scale is 2^(sb - 127)).""" + out = [] + for i in range(0, len(dwords), 2): + pair0 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i]))).result + pair1 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i + 1]))).result + # the first conversion's other half is overwritten by the second + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16( + T.i16x2, llvm.mlir_undef(T.i16x2), pair0, _raw(scale), False + ) + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16(T.i16x2, w, pair1, _raw(scale), True) + out.append(fx.Int32(llvm.BitcastOp(T.i32, w).result)) + return Vec.from_elements(out, fx.Int32) + + +def _pin_accumulators(values): + """Route the accumulators through an inline asm that outputs them again (tied + ``=a`` / ``0``) with the XDL wait states inside: the reads that follow depend on + this fence and cannot be hoisted right behind the last inline-asm MFMA (the + compiler sees no hazard through the asm; without it the last MFMA of a wave is + missing from half of each accumulator).""" + one = len(values) == 1 # an inline asm with one output returns it bare + ty = ir.Type.parse( + "vector<4xf32>" + if one + else "!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(values)) + ")>" + ) + constraints = ",".join(["=a"] * len(values) + [str(i) for i in range(len(values))]) + r = llvm.inline_asm( + ty, + [_raw(v) for v in values], + "s_nop 15\ns_nop 15", + constraints, + has_side_effects=True, + ) + if one: + return [r] + return [llvm.extractvalue(T.vec(4, T.f32), r, [i]) for i in range(len(values))] + + +def compile_mxfp8_gateup_swiglu_bf16(form='fp32', KW=2, LA=(7, 7)): + """Fixed 1-16-row, paired 12288x6144 gate/up epilogue into 6144 BF16 activations. + + ``KW`` K-waves sum through LDS in the materialized decode GEMM's order (waves_m = KW). + Scalar activation parameters stay runtime arguments so constant folding cannot change their rounding. + """ + N,K,RT,NI,KS=12288,6144,1,2,1 + vgpr_budget=None + sched_barriers=False + A_BF16=False + FOLD=1 + W_CM=None + assert form in ('fp32','bf16_chain') + w_cm = W_CACHE_MOD if W_CM is None else W_CM + assert w_cm in (0, 1, 2), w_cm + assert K % 256 == 0 and N % (16 * NI) == 0 + KSTEPS = K // 128 # 128-K steps + assert KSTEPS % (KS * KW) == 0 and (KSTEPS // (KS * KW)) % 2 == 0, (K, KS, KW) + SW = KSTEPS // (KS * KW) # steps per K-wave (even: starts on a 256-K scale block) + # Odd RT is legal only on the quantized-A path, where the A-scale dwords are + # re-paired to the m-block's parity at load time (module docstring). A_BF16 builds + # each scale dword itself with the byte at (rt % 2) and stays even-RT. + assert RT == 1 or RT % 2 == 0 or not A_BF16 + BM = 16 * RT // FOLD # rows per m-block + NB = N // (16 * NI) + SCB = K // 256 # scale blocks (64 dwords) per 32-row / 32-column group + RP = max(1, RT // 2) # 32-row scale groups per workgroup + if RT > 1 and RT % 2 == 1: + RP = (RT + 1) // 2 # the parity-straddling block needs one group more + # W scale dwords per step: one per 32-column group, or one per 16-column tile when + # NI is odd (a group can then straddle two workgroups: each tile shifts its half down) + CP = NI + LA_W, LA_A = LA or _lookahead( + SW, RT, NI, RP, CP, vgpr_budget, a_vgprs=16 if A_BF16 else 8, + reserve=(24 + (8 if FOLD > 1 else 0)) if A_BF16 else 0, fold=FOLD, + ) # fmt: skip + assert FOLD in (1, 2, 4) and ( + FOLD == 1 or (A_BF16 and RT == 1 and SW % FOLD == 0) + ), (FOLD, RT, SW) + RPT = 16 // FOLD # rows per 16-row tile + NA = SW * RT // FOLD # A blocks per K-wave + assert 1 <= LA_W <= SW and 1 <= LA_A <= NA, (LA_W, LA_A, SW, RT, FOLD) + RED_SLOT = RT * NI * 64 * 16 # bytes parked per K-wave + LDS_BYTES = max(16, KW * RED_SLOT) + assert LDS_BYTES <= 160 * 1024, "K-reduce scratch exceeds the LDS" + assert N * K <= 0xFFFFFFFF + + @fx.struct + class Shared: + red: fx.Array[fx.Uint8, LDS_BYTES, 16] + + name = ( + f"mxfp8_gateup_swiglu_bf16_{form}_i{N // 2}_" + + f"flydsl_mxfp8_dec_n{N}_k{K}_rt{RT}_ni{NI}_kw{KW}_ks{KS}_la{LA_W}x{LA_A}" + + ("sb" if sched_barriers else "") + + (("_fq" + (str(FOLD) if FOLD > 1 else "")) if A_BF16 else "") + + (f"_wcm{w_cm}" if w_cm != W_CACHE_MOD else "") + ) + + # addresses, not tensors (a tensor argument ends the preload): every argument + # arrives preloaded in SGPRs + @flyc.kernel(name=name, known_block_size=[64 * KW, 1, 1]) + def kernel( + i32_m: fx.Int32, + a_scale_bytes: fx.Int32, + n_mb: fx.Int32, + arg_w: fx.Int64, + arg_ws: fx.Int64, + arg_as: fx.Int64, + arg_a: fx.Int64, + arg_out: fx.Int64, + i32_alpha: fx.Int32, i32_beta: fx.Int32, i32_limit: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + l16, q16 = lane % 16, lane // 16 + if const_expr(KS > 1): + kz, wg = bx // (n_mb * NB), bx % (n_mb * NB) + kwave = kz * KW + wave + else: + kz, wg, kwave = 0, bx, wave + mb, nb = wg // NB, wg % NB + lane_off = q16 * 256 + l16 * 16 # a lane's 16 B inside a 1 KB block + k_base = kwave * (SW * 2048) # this K-wave's first block pair (bytes) + + if const_expr(A_BF16): + a_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_a, num_records_bytes=i32_m * (2 * K) + ) + else: + a_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_a, num_records_bytes=n_mb * (BM * K) + ) + w_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=N * K + ) + as_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_as, num_records_bytes=a_scale_bytes + ) + ws_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_ws, num_records_bytes=(N // 32) * SCB * 256 + ) + out_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_out, num_records_bytes=i32_m * ((N//2) * 2) + ) + + n16 = [nb + ni*(N//32) for ni in range_constexpr(NI)] + r16 = [mb * RT + rt for rt in range_constexpr(RT)] + w_dw = [ + (n16[ni] * (16 * K) + k_base + lane_off) // 4 for ni in range_constexpr(NI) + ] + if const_expr(A_BF16): + # row-major bf16: this lane's row, its 32-byte K segment, this K-wave's first + # step (folded: the lane's row r and step j of each block) + a_row, a_fold = l16 % RPT, l16 // RPT + a_dw = [ + ( + (r16[rt] * RPT + a_row) * (2 * K) + + kwave * (SW * 256) + + a_fold * 256 + + q16 * 32 + ) + // 4 + for rt in range_constexpr(RT) + ] + sa_shift = ( + ((a_fold % 2) * 16) if FOLD > 1 else None + ) # the block step's parity + else: + a_dw = [ + (r16[rt] * (16 * K) + k_base + lane_off) // 4 + for rt in range_constexpr(RT) + ] + sc_lane = q16 * 16 + l16 + sc_k = kwave * (SW // 2 * 64) # this K-wave's first 256-K scale block (dwords) + ws_dw = [ + ((n16[cp] // 2) * SCB) * 64 + sc_k + sc_lane + for cp in range_constexpr(CP) + ] + if const_expr(RT > 1 and RT % 2 == 1): + # odd RT: address the 32-row groups from the m-block's first tile rounded + # down to even (r16[0] // 2 floors); the parity mix below re-pairs them + # (const_expr: a bare if is rewritten to a dynamic scf.if whose branch + # assignments never bind in this scope -> as_dw NameError) + as_dw = [ + ((r16[0] // 2 + rp) * SCB) * 64 + sc_k + sc_lane + for rp in range_constexpr(RP) + ] + else: + as_dw = [ + ((r16[2 * rp] // 2) * SCB) * 64 + sc_k + sc_lane + for rp in range_constexpr(RP) + ] + a_par = r16[0] % 2 if RT > 1 and RT % 2 == 1 else None # m-block parity + # a 16-column (row) tile of an odd count sits in either half of its 32 group: + # move its bytes (half, half + 2) down to (0, 2) and select half 0 + w_shift = [(n16[ni] % 2) * 8 for ni in range_constexpr(NI)] + a_shift = (r16[0] % 2) * 8 if RT == 1 else None + + def load_pair(rsrc, dw, s, nt): + kw = dict(vec_width=4, dtype=fx.Int32) + if const_expr(nt): + kw["cache_modifier"] = w_cm + return _pack8( + buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048, **kw), + buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048 + 1024, **kw), + ) + + def load_scales(rsrc, dws, s, shift): + vals = [ + fx.Int32( + buffer_ops.buffer_load( + rsrc, + dw, + vec_width=1, + dtype=fx.Int32, + soffset_bytes=(s // 2) * 256, + ) + ) + for dw in dws + ] + if const_expr(shift is None): + return vals + if const_expr(isinstance(shift, list)): + return [v.shrui(sh) for v, sh in zip(vals, shift)] + return [v.shrui(shift) for v in vals] + + def repair_a_scale(vals, p): + """Odd RT: re-pair scale dword p to the even-parity layout the MFMA immediates + assume -- dword p must carry block tiles 2p (bytes 0/2) and 2p+1 (bytes 1/3). + An odd-parity block shifts every atom by one half: bytes 0/2 come from half 1 + of group p, bytes 1/3 from half 0 of group p+1 (the last pair's byte 1/3 name a + tile past the block and are never consumed). Called at the dword's first MFMA, + not at its load, so the load keeps its full look-ahead (VMLEAD).""" + nxt = vals[p + 1] if p + 1 < RP else vals[p] + return (a_par == 1).select( + (vals[p].shrui(fx.Int32(8)) & fx.Int32(0x00FF00FF)) + | ((nxt & fx.Int32(0x00FF00FF)) << 8), + vals[p], + ) + + def load_w_step(s): + bb = [load_pair(w_rsrc, w_dw[ni], s, True) for ni in range_constexpr(NI)] + if const_expr(s % 2 == 0): + return bb, ( + load_scales(ws_rsrc, ws_dw, s, w_shift), + None if A_BF16 else load_scales(as_rsrc, as_dw, s, a_shift), + ) + return bb, None + + def load_a(t): + rt, s = t % RT, (t // RT) * FOLD # block t: row tile, first step + if const_expr(A_BF16): + # the two 16-K segments (K 16q and 64 + 16q of the step), 32 B each, bf16 + return [ + buffer_ops.buffer_load( + a_rsrc, + a_dw[rt], + vec_width=4, + dtype=fx.Int32, + soffset_bytes=s * 256 + off, + ) + for off in (0, 16, 128, 144) + ] + return load_pair(a_rsrc, a_dw[rt], s, False) + + def quantize(raw, rt, s): + """raw bf16 of (row tile rt, step s) -> (fp8 operand, this MFMA's scale dword).""" + seg0 = [Vec(raw[0])[i] for i in range(4)] + [ + Vec(raw[1])[i] for i in range(4) + ] + seg1 = [Vec(raw[2])[i] for i in range(4)] + [ + Vec(raw[3])[i] for i in range(4) + ] + # per-32 groups of this row: pair (q, q ^ 1) holds group q // 2 in segment 0 and + # group 2 + q // 2 in segment 1; the other pair's values come with the 32-swap + e0, o0 = _swap16(_amax_bf16(seg0)) + e1, o1 = _swap16(_amax_bf16(seg1)) + own0, own1 = _i32_max(e0, o0), _i32_max(e1, o1) + (sb0, sc0), (sb1, sc1) = _e8m0_bf16(own0), _e8m0_bf16(own1) + # the scale byte the MFMA reads from this lane is group q's + lo0, hi0 = _swap32(sb0) + lo1, hi1 = _swap32(sb1) + sbm = (q16 == 0).select( + sb0, (q16 == 1).select(hi0, (q16 == 2).select(lo1, sb1)) + ) + a_op = _pack8(_quant_bf16(seg0, sc0), _quant_bf16(seg1, sc1)) + if const_expr(FOLD > 1): + return a_op, sbm << sa_shift + return a_op, sbm << (8 * (2 * (s % 2) + (rt % 2))) + + # Both operands run ahead of the MFMAs: W by LA_W steps, A by LA_A (step, row + # tile) blocks in the order the MFMAs consume them; with everything in flight + # from the start a wave ends when its last block lands, not after one memory + # latency per step. + acc = [[None] * NI for _ in range(RT)] + w_q = [None] * SW + a_q = [None] * NA + for s in range_constexpr(LA_W): + w_q[s] = load_w_step(s) + for t in range_constexpr(LA_A): + a_q[t] = load_a(t) + if const_expr(sched_barriers): + fx.rocdl.sched_barrier(0) + sw = sa = None + held = None # folded mode: the block's quantized operand and scale dword + for s in range_constexpr(SW): + if const_expr(s + LA_W < SW): + w_q[s + LA_W] = load_w_step(s + LA_W) + bb, sc = w_q[s] + if const_expr(s % 2 == 0): + sw, sa = sc + if const_expr(RT > 1 and RT % 2 == 1): + sa_raw, sa = sa, [None] * RP # repaired at first use (both steps) + for rt in range_constexpr(RT): + if const_expr(FOLD == 1): + t = s * RT + rt + if const_expr(t + LA_A < NA): + a_q[t + LA_A] = load_a(t + LA_A) + if const_expr(A_BF16): + a_op, sa_v = quantize(a_q[t], rt, s) + else: + if const_expr(RT > 1 and RT % 2 == 1 and sa[rt // 2] is None): + sa[rt // 2] = repair_a_scale(sa_raw, rt // 2) + a_op, sa_v = a_q[t], sa[rt // 2] + a_q[t] = None + else: + b, j = s // FOLD, s % FOLD + if const_expr(j == 0): + if const_expr(b + LA_A < NA): + a_q[b + LA_A] = load_a(b + LA_A) + held = quantize(a_q[b], rt, s) + a_q[b] = None + if const_expr(j == 0): + a_op, sa_v = held + else: + moved = _row_shl( + [Vec(held[0])[i] for i in range(8)] + [held[1]], RPT * j + ) + a_op, sa_v = Vec.from_elements(moved[:8], fx.Int32), moved[8] + for ni in range_constexpr(NI): + acc[rt][ni] = _mfma( + bb[ni], + a_op, + acc[rt][ni], + sw[ni], + sa_v, + 0, + rt % 2, + s % 2, + ) + # the loads depend on nothing, so the scheduler would pull later steps' + # loads up to here: more operands alive than _lookahead budgeted + if const_expr(sched_barriers): + fx.rocdl.sched_barrier(0) + w_q[s] = None + + # Keep GEMM's BF16 materialization before the model's activation. + flat = _pin_accumulators([acc[rt][ni] for rt in range(RT) for ni in range(NI)]) + lds_base = fx.Int32(fx.ptrtoint(smem.red.ptr)) + def lds_ptr(i,w): + off=w*RED_SLOT+i*1024 + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"),_raw(fx.Int64(lds_base+off+lane*16))) + for i in range_constexpr(RT*NI): + llvm.StoreOp(_raw(Vec(flat[i]).bitcast(fx.Int32)),lds_ptr(i,wave),alignment=16) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if wave==0: + vals=[] + for i in range_constexpr(2): + v=Vec(flat[i]) + for w in range_constexpr(1,KW): + v=v+Vec(llvm.load(T.vec(4,T.i32),lds_ptr(i,w),alignment=16)).bitcast(fx.Float32) + vals.append(v.to(fx.BFloat16).to(fx.Float32)) + gate,up=vals + alpha,beta,limit=i32_alpha.bitcast(fx.Float32),i32_beta.bitcast(fx.Float32),i32_limit.bitcast(fx.Float32) + def bf(v):return v.to(fx.BFloat16).to(fx.Float32) + ys=[] + for e in range_constexpr(4): + gv=fx.Float32(gate[e]).minimumf(limit) + uv=fx.Float32(up[e]).maximumf(-limit).minimumf(limit) + if const_expr(form=="bf16_chain"): + ag=bf(alpha*gv) + sig=bf(fx.Float32(1.0)/(fx.Float32(1.0)+fx.math.exp(-ag))) + y=bf(bf(gv*sig)*bf(uv+beta)) + else: + sig=fx.Float32(1.0)/(fx.Float32(1.0)+fx.math.exp(-(alpha*gv))) + y=(gv*sig)*(uv+beta) + ys.append(y) + row=mb*16+l16 + col=nb*16+q16*4 + packed=Vec.from_elements(ys,fx.Float32).to(fx.BFloat16).bitcast(fx.Int32) + buffer_ops.buffer_store(packed,out_rsrc,(row*(N//2)+col)//2,mask=row budget: + break + best = (la_w, la_a) + assert best is not None, f"RT {RT} x NI {NI} leaves no VGPRs for the operands" + return best + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _pack8(h0, h1): + """two 4 x i32 halves -> one 8 x i32 MFMA operand (VGPRs 0-3 = K [16*klane, +16), + VGPRs 4-7 = K [64 + 16*klane, +16)).""" + a, b = Vec(h0), Vec(h1) + return Vec.from_elements( + [a[k] for k in range(4)] + [b[k] for k in range(4)], fx.Int32 + ) + + +def _mfma(w_op, a_op, acc, sw, sa, jw, ia, k2): + """One scaled 16x16x128 fp8 MFMA, accumulator pinned in AGPR (``=a`` / tied ``0``). + Operand order (W, A): lane (q, r) accumulates out[row r, columns 4q .. 4q+3]. + Scale byte of a dword = op_sel + 2 * op_sel_hi (16-row / 16-col half, 128-K half).""" + src2 = "$0" if acc is not None else "0" + asm = ( + f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {src2}, $3, $4 " + f"op_sel:[{jw},{ia},0] op_sel_hi:[{k2},{k2},0] cbsz:0 blgp:0" + ) + ops = [ + fx.as_ir_value(w_op), + fx.as_ir_value(a_op), + fx.as_ir_value(sw), + fx.as_ir_value(sa), + ] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(fx.as_ir_value(acc)) + cons += ",0" + return llvm.inline_asm( + Vec.make_type(4, fx.Float32), ops, asm, cons, has_side_effects=True + ) + + +def _pk_max_u16(a, b): + """per-16-bit-half unsigned max (v_pk_max_u16): on sign-cleared bf16 patterns that is + the value order.""" + va = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(a))).result + vb = llvm.BitcastOp(T.i16x2, _raw(fx.Int32(b))).result + return fx.Int32(llvm.BitcastOp(T.i32, llvm.intr_umax(va, vb)).result) + + +def _amax_bf16(dwords): + """max |x| over the 2 * len(dwords) bf16 values, as a 16-bit bf16 pattern in an i32.""" + m = None + for d in dwords: + a = fx.Int32(d) & fx.Int32(0x7FFF7FFF) + m = a if m is None else _pk_max_u16(m, a) + lo, hi = m & fx.Int32(0xFFFF), m >> 16 + return (lo > hi).select(lo, hi) + + +def _i32_max(a, b): + return (a > b).select(a, b) + + +def _swap16(v): + """(even-row value, odd-row value) of this lane's 16-lane row pair (klane q, q ^ 1).""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane16_swap(pair_ty, raw, raw, False, False) + return fx.Int32(llvm.extractvalue(T.i32, res, [0])), fx.Int32( + llvm.extractvalue(T.i32, res, [1]) + ) + + +def _swap32(v): + """(low-half value, high-half value): the values of klanes 0/1 and 2/3.""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane32_swap(pair_ty, raw, raw, False, False) + return fx.Int32(llvm.extractvalue(T.i32, res, [0])), fx.Int32( + llvm.extractvalue(T.i32, res, [1]) + ) + + +def _e8m0_bf16(amax_bits): + """(scale byte, conversion scale) of a block from the bf16 pattern of its maximum. The + byte is the Triton MXFP8 quantizer's, clamp(ceil(log2(max(amax, tiny) / 448)) + 127, 0, + 254): with amax = 2^e * m, ceil(log2(amax / 448)) = e - 8 + [m > 1.75], so it is the + exponent field - 8 + [mantissa > 0x60], which (a + 0x1F) >> 7 gives in one carry; + zero, denormal and tiny maxima land on 0 as in that quantizer. The conversion divides by + 2^(byte - 127); byte 0 would make that a denormal, so such a block (maximum below + 2^-119) is converted with 2^-126 instead.""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(8)) - fx.Int32(8) + scale = ((_i32_max(t, fx.Int32(9)) - fx.Int32(8)) << 23).bitcast(fx.Float32) + return sb, scale + + +def _e8m0_bf16_pp2(amax_bits): + """NVIDIA-consistent variant of ``_e8m0_bf16``: the stored + byte IS the conversion exponent (``max(t, 9) - 8``, never 0), so the stored + scale always equals the 2^(byte - 127) its consumer decodes -- ``_e8m0_bf16`` + stores 0 for a block maximum below ~1.75*2^-119 while converting with 2^-126 + (a 2x error under an OCP consumer, a zeroed block under a ``(byte << 23)`` + one). Identical output for every bf16 block maximum above 1.75*2^-119, which + includes every nonzero normal activation.""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(9)) - fx.Int32(8) + scale = (sb << 23).bitcast(fx.Float32) + return sb, scale + + +def _row_shl(vals, n): + """Each lane takes the value of lane + n of its 16-lane row (v_mov_b32 dpp row_shl; + the last n lanes of a row get 0).""" + zero = _raw(fx.Int32(0)) + return [ + fx.Int32( + fx.rocdl.update_dpp( + T.i32, zero, _raw(fx.Int32(v)), 0x100 + n, 0xF, 0xF, True + ) + ) + for v in vals + ] + + +def _quant_bf16(dwords, scale): + """dwords of bf16 pairs -> half as many dwords of fp8 codes: x / scale, round to nearest + even (v_cvt_scalef32_pk_fp8_bf16, the scale is 2^(sb - 127)).""" + out = [] + for i in range(0, len(dwords), 2): + pair0 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i]))).result + pair1 = llvm.BitcastOp(T.bf16x2, _raw(fx.Int32(dwords[i + 1]))).result + # the first conversion's other half is overwritten by the second + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16( + T.i16x2, llvm.mlir_undef(T.i16x2), pair0, _raw(scale), False + ) + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16(T.i16x2, w, pair1, _raw(scale), True) + out.append(fx.Int32(llvm.BitcastOp(T.i32, w).result)) + return Vec.from_elements(out, fx.Int32) + + +def _pin_accumulators(values): + """Route the accumulators through an inline asm that outputs them again (tied + ``=a`` / ``0``) with the XDL wait states inside: the reads that follow depend on + this fence and cannot be hoisted right behind the last inline-asm MFMA (the + compiler sees no hazard through the asm; without it the last MFMA of a wave is + missing from half of each accumulator).""" + one = len(values) == 1 # an inline asm with one output returns it bare + ty = ir.Type.parse( + "vector<4xf32>" + if one + else "!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(values)) + ")>" + ) + constraints = ",".join(["=a"] * len(values) + [str(i) for i in range(len(values))]) + r = llvm.inline_asm( + ty, + [_raw(v) for v in values], + "s_nop 15\ns_nop 15", + constraints, + has_side_effects=True, + ) + if one: + return [r] + return [llvm.extractvalue(T.vec(4, T.f32), r, [i]) for i in range(len(values))] + + +def compile_mxfp8_gateup_swiglu_bf16_onewave(form='fp32', LA=(7, 7)): + """Fixed 1-16-row, paired 12288x6144 gate/up epilogue into 6144 BF16 activations, target fp32 form. + + One wave owns the whole K sum (the materialized decode GEMM with waves_m = 1). Every dot still rounds to BF16 + before activation; scalar activation parameters stay runtime arguments so constant folding cannot change their + rounding. + """ + N,K,RT,NI,KW,KS=12288,6144,1,2,1,1 + vgpr_budget=None + sched_barriers=False + A_BF16=False + FOLD=1 + W_CM=None + assert form == 'fp32' + w_cm = W_CACHE_MOD if W_CM is None else W_CM + assert w_cm in (0, 1, 2), w_cm + assert K % 256 == 0 and N % (16 * NI) == 0 + KSTEPS = K // 128 # 128-K steps + assert KSTEPS % (KS * KW) == 0 and (KSTEPS // (KS * KW)) % 2 == 0, (K, KS, KW) + SW = KSTEPS // (KS * KW) # steps per K-wave (even: starts on a 256-K scale block) + # Odd RT is legal only on the quantized-A path, where the A-scale dwords are + # re-paired to the m-block's parity at load time (module docstring). A_BF16 builds + # each scale dword itself with the byte at (rt % 2) and stays even-RT. + assert RT == 1 or RT % 2 == 0 or not A_BF16 + BM = 16 * RT // FOLD # rows per m-block + NB = N // (16 * NI) + SCB = K // 256 # scale blocks (64 dwords) per 32-row / 32-column group + RP = max(1, RT // 2) # 32-row scale groups per workgroup + if RT > 1 and RT % 2 == 1: + RP = (RT + 1) // 2 # the parity-straddling block needs one group more + # W scale dwords per step: one per 32-column group, or one per 16-column tile when + # NI is odd (a group can then straddle two workgroups: each tile shifts its half down) + CP = NI + LA_W, LA_A = LA or _lookahead( + SW, RT, NI, RP, CP, vgpr_budget, a_vgprs=16 if A_BF16 else 8, + reserve=(24 + (8 if FOLD > 1 else 0)) if A_BF16 else 0, fold=FOLD, + ) # fmt: skip + assert FOLD in (1, 2, 4) and ( + FOLD == 1 or (A_BF16 and RT == 1 and SW % FOLD == 0) + ), (FOLD, RT, SW) + RPT = 16 // FOLD # rows per 16-row tile + NA = SW * RT // FOLD # A blocks per K-wave + assert 1 <= LA_W <= SW and 1 <= LA_A <= NA, (LA_W, LA_A, SW, RT, FOLD) + assert N * K <= 0xFFFFFFFF + + name = ( + f"mxfp8_gateup_swiglu_bf16_onewave_{form}_i{N // 2}_direct_" + + f"flydsl_mxfp8_dec_n{N}_k{K}_rt{RT}_ni{NI}_kw{KW}_ks{KS}_la{LA_W}x{LA_A}" + + ("sb" if sched_barriers else "") + + (("_fq" + (str(FOLD) if FOLD > 1 else "")) if A_BF16 else "") + + (f"_wcm{w_cm}" if w_cm != W_CACHE_MOD else "") + ) + + # addresses, not tensors (a tensor argument ends the preload): every argument + # arrives preloaded in SGPRs + @flyc.kernel(name=name, known_block_size=[64 * KW, 1, 1]) + def kernel( + i32_m: fx.Int32, + a_scale_bytes: fx.Int32, + n_mb: fx.Int32, + arg_w: fx.Int64, + arg_ws: fx.Int64, + arg_as: fx.Int64, + arg_a: fx.Int64, + arg_out: fx.Int64, + i32_alpha: fx.Int32, i32_beta: fx.Int32, i32_limit: fx.Int32, + ): + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + l16, q16 = lane % 16, lane // 16 + if const_expr(KS > 1): + kz, wg = bx // (n_mb * NB), bx % (n_mb * NB) + kwave = kz * KW + wave + else: + kz, wg, kwave = 0, bx, wave + mb, nb = wg // NB, wg % NB + lane_off = q16 * 256 + l16 * 16 # a lane's 16 B inside a 1 KB block + k_base = kwave * (SW * 2048) # this K-wave's first block pair (bytes) + + if const_expr(A_BF16): + a_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_a, num_records_bytes=i32_m * (2 * K) + ) + else: + a_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_a, num_records_bytes=n_mb * (BM * K) + ) + w_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=N * K + ) + as_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_as, num_records_bytes=a_scale_bytes + ) + ws_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_ws, num_records_bytes=(N // 32) * SCB * 256 + ) + out_rsrc = buffer_ops.create_buffer_resource_from_addr( + arg_out, num_records_bytes=i32_m * ((N//2) * 2) + ) + + n16 = [nb + ni*(N//32) for ni in range_constexpr(NI)] + r16 = [mb * RT + rt for rt in range_constexpr(RT)] + w_dw = [ + (n16[ni] * (16 * K) + k_base + lane_off) // 4 for ni in range_constexpr(NI) + ] + if const_expr(A_BF16): + # row-major bf16: this lane's row, its 32-byte K segment, this K-wave's first + # step (folded: the lane's row r and step j of each block) + a_row, a_fold = l16 % RPT, l16 // RPT + a_dw = [ + ( + (r16[rt] * RPT + a_row) * (2 * K) + + kwave * (SW * 256) + + a_fold * 256 + + q16 * 32 + ) + // 4 + for rt in range_constexpr(RT) + ] + sa_shift = ( + ((a_fold % 2) * 16) if FOLD > 1 else None + ) # the block step's parity + else: + a_dw = [ + (r16[rt] * (16 * K) + k_base + lane_off) // 4 + for rt in range_constexpr(RT) + ] + sc_lane = q16 * 16 + l16 + sc_k = kwave * (SW // 2 * 64) # this K-wave's first 256-K scale block (dwords) + ws_dw = [ + ((n16[cp] // 2) * SCB) * 64 + sc_k + sc_lane + for cp in range_constexpr(CP) + ] + if const_expr(RT > 1 and RT % 2 == 1): + # odd RT: address the 32-row groups from the m-block's first tile rounded + # down to even (r16[0] // 2 floors); the parity mix below re-pairs them + # (const_expr: a bare if is rewritten to a dynamic scf.if whose branch + # assignments never bind in this scope -> as_dw NameError) + as_dw = [ + ((r16[0] // 2 + rp) * SCB) * 64 + sc_k + sc_lane + for rp in range_constexpr(RP) + ] + else: + as_dw = [ + ((r16[2 * rp] // 2) * SCB) * 64 + sc_k + sc_lane + for rp in range_constexpr(RP) + ] + a_par = r16[0] % 2 if RT > 1 and RT % 2 == 1 else None # m-block parity + # a 16-column (row) tile of an odd count sits in either half of its 32 group: + # move its bytes (half, half + 2) down to (0, 2) and select half 0 + w_shift = [(n16[ni] % 2) * 8 for ni in range_constexpr(NI)] + a_shift = (r16[0] % 2) * 8 if RT == 1 else None + + def load_pair(rsrc, dw, s, nt): + kw = dict(vec_width=4, dtype=fx.Int32) + if const_expr(nt): + kw["cache_modifier"] = w_cm + return _pack8( + buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048, **kw), + buffer_ops.buffer_load(rsrc, dw, soffset_bytes=s * 2048 + 1024, **kw), + ) + + def load_scales(rsrc, dws, s, shift): + vals = [ + fx.Int32( + buffer_ops.buffer_load( + rsrc, + dw, + vec_width=1, + dtype=fx.Int32, + soffset_bytes=(s // 2) * 256, + ) + ) + for dw in dws + ] + if const_expr(shift is None): + return vals + if const_expr(isinstance(shift, list)): + return [v.shrui(sh) for v, sh in zip(vals, shift)] + return [v.shrui(shift) for v in vals] + + def repair_a_scale(vals, p): + """Odd RT: re-pair scale dword p to the even-parity layout the MFMA immediates + assume -- dword p must carry block tiles 2p (bytes 0/2) and 2p+1 (bytes 1/3). + An odd-parity block shifts every atom by one half: bytes 0/2 come from half 1 + of group p, bytes 1/3 from half 0 of group p+1 (the last pair's byte 1/3 name a + tile past the block and are never consumed). Called at the dword's first MFMA, + not at its load, so the load keeps its full look-ahead (VMLEAD).""" + nxt = vals[p + 1] if p + 1 < RP else vals[p] + return (a_par == 1).select( + (vals[p].shrui(fx.Int32(8)) & fx.Int32(0x00FF00FF)) + | ((nxt & fx.Int32(0x00FF00FF)) << 8), + vals[p], + ) + + def load_w_step(s): + bb = [load_pair(w_rsrc, w_dw[ni], s, True) for ni in range_constexpr(NI)] + if const_expr(s % 2 == 0): + return bb, ( + load_scales(ws_rsrc, ws_dw, s, w_shift), + None if A_BF16 else load_scales(as_rsrc, as_dw, s, a_shift), + ) + return bb, None + + def load_a(t): + rt, s = t % RT, (t // RT) * FOLD # block t: row tile, first step + if const_expr(A_BF16): + # the two 16-K segments (K 16q and 64 + 16q of the step), 32 B each, bf16 + return [ + buffer_ops.buffer_load( + a_rsrc, + a_dw[rt], + vec_width=4, + dtype=fx.Int32, + soffset_bytes=s * 256 + off, + ) + for off in (0, 16, 128, 144) + ] + return load_pair(a_rsrc, a_dw[rt], s, False) + + def quantize(raw, rt, s): + """raw bf16 of (row tile rt, step s) -> (fp8 operand, this MFMA's scale dword).""" + seg0 = [Vec(raw[0])[i] for i in range(4)] + [ + Vec(raw[1])[i] for i in range(4) + ] + seg1 = [Vec(raw[2])[i] for i in range(4)] + [ + Vec(raw[3])[i] for i in range(4) + ] + # per-32 groups of this row: pair (q, q ^ 1) holds group q // 2 in segment 0 and + # group 2 + q // 2 in segment 1; the other pair's values come with the 32-swap + e0, o0 = _swap16(_amax_bf16(seg0)) + e1, o1 = _swap16(_amax_bf16(seg1)) + own0, own1 = _i32_max(e0, o0), _i32_max(e1, o1) + (sb0, sc0), (sb1, sc1) = _e8m0_bf16(own0), _e8m0_bf16(own1) + # the scale byte the MFMA reads from this lane is group q's + lo0, hi0 = _swap32(sb0) + lo1, hi1 = _swap32(sb1) + sbm = (q16 == 0).select( + sb0, (q16 == 1).select(hi0, (q16 == 2).select(lo1, sb1)) + ) + a_op = _pack8(_quant_bf16(seg0, sc0), _quant_bf16(seg1, sc1)) + if const_expr(FOLD > 1): + return a_op, sbm << sa_shift + return a_op, sbm << (8 * (2 * (s % 2) + (rt % 2))) + + # Both operands run ahead of the MFMAs: W by LA_W steps, A by LA_A (step, row + # tile) blocks in the order the MFMAs consume them; with everything in flight + # from the start a wave ends when its last block lands, not after one memory + # latency per step. + acc = [[None] * NI for _ in range(RT)] + w_q = [None] * SW + a_q = [None] * NA + for s in range_constexpr(LA_W): + w_q[s] = load_w_step(s) + for t in range_constexpr(LA_A): + a_q[t] = load_a(t) + if const_expr(sched_barriers): + fx.rocdl.sched_barrier(0) + sw = sa = None + held = None # folded mode: the block's quantized operand and scale dword + for s in range_constexpr(SW): + if const_expr(s + LA_W < SW): + w_q[s + LA_W] = load_w_step(s + LA_W) + bb, sc = w_q[s] + if const_expr(s % 2 == 0): + sw, sa = sc + if const_expr(RT > 1 and RT % 2 == 1): + sa_raw, sa = sa, [None] * RP # repaired at first use (both steps) + for rt in range_constexpr(RT): + if const_expr(FOLD == 1): + t = s * RT + rt + if const_expr(t + LA_A < NA): + a_q[t + LA_A] = load_a(t + LA_A) + if const_expr(A_BF16): + a_op, sa_v = quantize(a_q[t], rt, s) + else: + if const_expr(RT > 1 and RT % 2 == 1 and sa[rt // 2] is None): + sa[rt // 2] = repair_a_scale(sa_raw, rt // 2) + a_op, sa_v = a_q[t], sa[rt // 2] + a_q[t] = None + else: + b, j = s // FOLD, s % FOLD + if const_expr(j == 0): + if const_expr(b + LA_A < NA): + a_q[b + LA_A] = load_a(b + LA_A) + held = quantize(a_q[b], rt, s) + a_q[b] = None + if const_expr(j == 0): + a_op, sa_v = held + else: + moved = _row_shl( + [Vec(held[0])[i] for i in range(8)] + [held[1]], RPT * j + ) + a_op, sa_v = Vec.from_elements(moved[:8], fx.Int32), moved[8] + for ni in range_constexpr(NI): + acc[rt][ni] = _mfma( + bb[ni], + a_op, + acc[rt][ni], + sw[ni], + sa_v, + 0, + rt % 2, + s % 2, + ) + # the loads depend on nothing, so the scheduler would pull later steps' + # loads up to here: more operands alive than _lookahead budgeted + if const_expr(sched_barriers): + fx.rocdl.sched_barrier(0) + w_q[s] = None + + # Keep GEMM's BF16 materialization before the model's activation. + flat = _pin_accumulators([acc[rt][ni] for rt in range(RT) for ni in range(NI)]) + # One wave owns each complete dot: no LDS exchange or workgroup barrier. + vals=[Vec(flat[i]).to(fx.BFloat16).to(fx.Float32) for i in range_constexpr(2)] + gate,up=vals + alpha,beta,limit=i32_alpha.bitcast(fx.Float32),i32_beta.bitcast(fx.Float32),i32_limit.bitcast(fx.Float32) + ys=[] + for e in range_constexpr(4): + gv=fx.Float32(gate[e]).minimumf(limit) + uv=fx.Float32(up[e]).maximumf(-limit).minimumf(limit) + sig=fx.Float32(1.0)/(fx.Float32(1.0)+fx.math.exp(-(alpha*gv))) + y=(gv*sig)*(uv+beta) + ys.append(y) + row=mb*16+l16 + col=nb*16+q16*4 + packed=Vec.from_elements(ys,fx.Float32).to(fx.BFloat16).bitcast(fx.Int32) + buffer_ops.buffer_store(packed,out_rsrc,(row*(N//2)+col)//2,mask=row (K-waves, W / A lookahead); K-waves 1 runs the one-wave kernel. 14.9 / 15.3 us at 8 rows, no AGPR parking +# (benchmark_gateup_swiglu_bf16.py --sweep) +CONFIG = {"fp32": (1, (7, 8)), "bf16_chain": (3, (4, 8))} + + +@functools.cache +def _launcher(form, kw, la): + if kw == 1: + from .gateup_swiglu_bf16_onewave import compile_mxfp8_gateup_swiglu_bf16_onewave + return compile_mxfp8_gateup_swiglu_bf16_onewave(form, LA=la) + from .gateup_swiglu_bf16 import compile_mxfp8_gateup_swiglu_bf16 + return compile_mxfp8_gateup_swiglu_bf16(form, KW=kw, LA=la) + + +def _bits(v): + return struct.unpack('= 32 and activation_scale.shape[1] == K // 32 + exe = _launcher(form, *(config or CONFIG[form])) + out = torch.empty((m, I), device=activation.device, dtype=torch.bfloat16) + u8 = lambda t: t.view(torch.uint8).view(-1) # noqa: E731 + _run_compiled(exe, u8(activation), u8(weight), u8(out), u8(activation_scale), u8(weight_scale), + m, activation_scale.numel(), 1, exe.n_blocks, _bits(alpha), _bits(beta), _bits(limit), + torch.cuda.current_stream()) + return out + + +def warmup_mxfp8_gateup_swiglu_bf16(activation, activation_scale, weight, weight_scale, forms=('fp32', 'bf16_chain')): + """Compile the reached activation forms before graphs or serving.""" + for form in forms: + mxfp8_gateup_swiglu_bf16(activation, activation_scale, weight, weight_scale, 8, form=form) diff --git a/amd_kernels/gemm/mxfp8_n4992_k6144/README.md b/amd_kernels/gemm/mxfp8_n4992_k6144/README.md new file mode 100644 index 0000000..d44b8bf --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n4992_k6144/README.md @@ -0,0 +1,52 @@ +# MXFP8 GEMM, N 4992 x K 6144 (QKV projection, TP2) + +## What it does +The QKV projection of the MiniMax-M3.1-Flash sparse attention layer at TP2: N 4992 = q (32 heads) | k (2) | v (2) | +index-q (2) | index-k (1, replicated) x 128, K 6144, bf16 out. Every bf16 row of the shape and its persistent kernel, +registered with the dispatcher in [`../mxfp8_common`](../mxfp8_common/README.md) (`gemm._SHAPE_HOSTS`); layouts and +the numerics contract are that folder's. Callers: the fused norm + quant producer writes (16, 16) codes in the tile +`prequant_code_tile_rows` names (decode rows) or row-major codes (persistent rows); the deferred partials feed the +split-K reduce + norm / RoPE / NVFP4 insert of `attention/norm_rope_nvfp4_insert_d128_rot64_p128` (32 / 2 / 2 heads). + +## Entry points +- The common entries (`mxfp8_linear`, `mxfp8_gemm`, `mxfp8_gemm_prequant`, `prequant_code_tile_rows` + + `mxfp8_gemm_shuffled`, which returns `SkPartials` on a deferred row); `warmup_mxfp8_gemm_weight` compiles every row. +- `host.select`: this shape's config for a row count (decode and split-K decode rows to 768, then the persistent + tile of the fewest rounds). +- `host.select_deferred`: the split-K partial row that the fused norm / RoPE / insert chain reduces. +- `host.persistent_tile` / `host.persistent_plan`: the persistent kernel's tile for a row count, and its launches + (one, or a row split into whole rounds of one tile plus the rest on another). +- `host.decode_launcher`: the decode kernel in the grouped XCD order for the rows of three or more m-blocks. +- `host.LAUNCHERS`: the launchers of the persistent kernel (one launch, row split). + +The `host.py` selectors are the dispatch; `mxfp8_common`'s `gemm.py` calls them. + +## Performance +One MI355X. Graph replay, layer 39 TP2 rank 0 weights in 14 cold copies (> 400 MB), up to 100 seeded inputs, +3 x 5 alternated rounds, median µs. GEMM: pre-quantized codes in (the producer's layout) -> bf16; chain: + the +TP2 norm / RoPE / NVFP4 insert (`mxfp8_gemm_qkv_norm_rope_nvfp4_insert`). Baseline: vLLM's TP2 fallback, Triton +`tl.dot_scaled` (`_M31_TILES[(4992, 6144)]`) on row-major codes (+ the insert). Roofline: 6.5 TB/s, 5 PF. + +| M | Row | GEMM µs | Baseline µs | Chain µs | Baseline µs | % of GEMM roofline | +|---|---|---|---|---|---|---| +| 8 | decode 16x32 (chain: split-2 partials) | 7.58 | 10.58 | 10.27 | 13.38 | 64% | +| 32 | decode 32x32 (chain: split-2 partials) | 8.83 | 11.67 | 11.07 | 14.69 | 56% | +| 64 | split-K decode 64x64, K split 3 | 9.56 | 14.25 | 12.90 | 17.86 | 53% | +| 96 | split-K decode 96x64, K split 3 | 10.57 | 19.71 | 14.15 | 23.05 | 48% | +| 128 | split-K decode 2 x 64x128, K split 3 | 11.30 | 21.12 | 14.91 | 24.59 | 46% | +| 176 | split-K decode 2 x 96x128, K split 3 | 13.12 | 23.04 | 17.77 | 26.91 | 40% | +| 256 | split-K decode 2 x 128x128, K split 3 | 15.71 | 36.37 | 20.89 | 40.69 | 35% | +| 384 | decode 192x48 | 20.99 | 35.36 | 26.56 | 40.24 | 28% | +| 512 | decode 128x96, grouped | 24.54 | 64.21 | 30.57 | 70.21 | 26% | +| 768 | decode 256x64, grouped | 30.31 | 65.81 | 36.71 | 72.33 | 31% | +| 1024 | persistent 192x128 | 39.61 | 94.58 | 47.25 | 102.01 | 32% | +| 2048 | persistent 256x192 | 56.50 | 114.10 | 67.30 | 124.54 | 44% | +| 4096 | persistent 256x192 | 101.48 | 184.16 | 120.16 | 201.22 | 50% | +| 8192 | persistent 256x192 + 256x128 split | 192.12 | 379.86 | 230.31 | 408.90 | 52% | +| 16384 | persistent 256x192 | 372.23 | 815.05 | 442.13 | 869.98 | 54% | +| 32768 | persistent 256x192 | 758.59 | 2068.21 | 888.06 | 2175.64 | 53% | + +## Tests +- `python3 -m pytest tests/gemm/mxfp8_n4992_k6144 -q` +- `python3 benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py --checkpoint ` +- `python3 benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py --decode 16,256 --chain 64 --prefill 4096` diff --git a/amd_kernels/gemm/mxfp8_n4992_k6144/__init__.py b/amd_kernels/gemm/mxfp8_n4992_k6144/__init__.py new file mode 100644 index 0000000..2711202 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n4992_k6144/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP8 GEMM rows and kernels of the QKV projection N 4992 x K 6144 (TP2). Called through ``gemm.mxfp8_common``.""" diff --git a/amd_kernels/gemm/mxfp8_n4992_k6144/host.py b/amd_kernels/gemm/mxfp8_n4992_k6144/host.py new file mode 100644 index 0000000..d03ca40 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n4992_k6144/host.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MXFP8 GEMM rows and kernels of the QKV projection N 4992 x K 6144 (TP2), registered with the common +dispatcher (``gemm.mxfp8_common.gemm._SHAPE_HOSTS``): every bf16 row of the shape (the tuned table only lists the shape), the +deferred (split-K partial) rows of the fused norm / RoPE / insert chain, and the launches of the persistent kernel +(``persistent``), one or two (a row split) per call.""" + +import functools + +import torch + +from amd_kernels.gemm.mxfp8_common.gemm import ( + _DECODE_KERNEL, + _PLAIN_GRID_MAX, + _Config, + _fourwave_grid_max, + splitk_rows, +) + +N, K = 4992, 6144 +_PERSISTENT_KERNEL = "persistent_n4992_k6144" +_PERSISTENT_SPLIT_KERNEL = "persistent_split_n4992_k6144" + + +def _decode(tile_m, tile_n, k_waves, split_k=1, lookahead=(0, 0)): + return _Config(tile_m, tile_n, 128, 0, 0, split_k, _DECODE_KERNEL, k_waves, 0, *lookahead) + + +def _persistent(tile_m, tile_n, kernel=_PERSISTENT_KERNEL): + return _Config(tile_m, tile_n, 128, 1, 1, 1, kernel, 2, 2) + + +# Decode rows on (16, 16) codes: (largest M, row); sweep: benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py. +# An explicit lookahead (W steps, A blocks) where the deepest one parks operands in AGPRs (tests/.../test_isa.py) +# or a shallower one measured faster. 49-256: the split-K decode kernel, K split 3 (the kw3 rows' K order, bitwise), +# one m-block to 112 rows, two above: -9..-25% at every multiple of 8 and window start +# (benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py); with the insert -6..-8% at 49-64 against the deferred +# split-2 chain, +-2% at 33-48. +DECODE_ROWS = ( + (16, _decode(16, 32, 1)), + (32, _decode(32, 32, 1)), + (48, _decode(48, 32, 1, lookahead=(4, 9))), + *splitk_rows(49, 112, 1, 64, 4, 4, 3, 2), + *splitk_rows(113, 256, 2, 128, 4, 4, 3, 2), + (320, _decode(160, 48, 4)), + (384, _decode(192, 48, 3, lookahead=(1, 8))), + (448, _decode(224, 48, 3, lookahead=(1, 10))), + (512, _decode(128, 96, 3)), + (576, _decode(192, 64, 3, lookahead=(1, 8))), + (624, _decode(208, 64, 3, lookahead=(1, 6))), + (672, _decode(224, 64, 3)), + (720, _decode(240, 64, 3)), + (768, _decode(256, 64, 3, lookahead=(1, 8))), +) +DECODE_MAX = DECODE_ROWS[-1][0] +# Rows from 385 take the grouped XCD order of the decode kernel (an XCD's contiguous eighth of the column-major +# four-m-block groups: W and A shared in its L2): -4.6..-15% at 448-720 rows against the plain order; flat at two +# m-blocks below. With them the decode rows reach 768 (256x64: 30.2 us against the persistent tile's 34.1). +GROUPED_ROWS = frozenset(config for top, config in DECODE_ROWS if top > 384) +# row-major codes at up to this many rows keep the common fallback (AITER split-K) row: 12.6..24.7 us against the +# persistent 128x128 tile's ~33 us (same sweep) +ROW_MAJOR_FALLBACK_MAX = 192 + +# Deferred rows of the chain (partials into the split-K reduce + norm / RoPE / insert), where they beat the decode +# row + insert; same sweep, --chain. +DEFERRED_ROWS = ( + (16, _decode(16, 48, 1, 2)), + (32, _decode(32, 48, 1, 2)), + (48, _decode(48, 64, 1, 3)), +) + +# Persistent tiles: microseconds per round of 256 workgroups (K 6144, graph replay, cold weights; same sweep, +# --prefill). A row count takes the tile of the fewest round-microseconds, the larger tile on a tie. +PERSISTENT_ROUND_US = {(256, 192): 50.0, (256, 128): 39.5, (192, 192): 42.0, (192, 128): 34.5, (128, 192): 37.0, + (128, 128): 33.0} + + +def _rounds(m, tile): + return -(-(-(-m // tile[0]) * (N // tile[1])) // 256) + + +@functools.cache +def persistent_tile(m: int) -> tuple[int, int]: + """The persistent kernel's tile for ``m`` rows.""" + return min(PERSISTENT_ROUND_US, key=lambda t: (_rounds(m, t) * PERSISTENT_ROUND_US[t], -t[0] * t[1])) + + +# A row split (a first launch of whole rounds, the rest with another tile) is taken where the round model predicts +# 5% or more: measured -0.4..-8.7% at 5632-20480 rows, +0.6% (noise) at 2560 (same sweep, --split; every tile sums +# K in the same order, so the split's bytes are the one launch's). +SPLIT_MIN_GAIN = 0.05 + + +@functools.cache +def persistent_plan(m: int) -> tuple[tuple[int, tuple[int, int]], ...]: + """The persistent launches for ``m`` rows: (first row, tile) each, in launch order.""" + tile = persistent_tile(m) + best, plan = _rounds(m, tile) * PERSISTENT_ROUND_US[tile] * (1 - SPLIT_MIN_GAIN), ((0, tile),) + for first in PERSISTENT_ROUND_US: + for rounds in range(1, _rounds(m, first)): + rows = 256 * rounds // (N // first[1]) * first[0] + if not 0 < rows < m: + continue + for rest in PERSISTENT_ROUND_US: + us = rounds * PERSISTENT_ROUND_US[first] + _rounds(m - rows, rest) * PERSISTENT_ROUND_US[rest] + if us < best: + best, plan = us, ((0, first), (rows, rest)) + return plan + + +def decode_launcher(config: _Config, a_bf16: bool): + """The decode kernel of a ``GROUPED_ROWS`` row, or None (the common decode kernel).""" + if a_bf16 or config not in GROUPED_ROWS: + return None + from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import compile_mxfp8_gemm_decode + + return compile_mxfp8_gemm_decode( + N=N, K=K, RT=config.tile_m // 16, NI=config.tile_n // 16, KW=config.waves_m, KS=config.split_k, + LA=(config.la_w, config.la_a) if config.la_w else None, sched_barriers=bool(config.waves_n), GROUPED=True, + ) + + +def _decode_row(m): + for top, config in DECODE_ROWS: + if m <= top: + return config + return None + + +def select(m: int, out_dtype: torch.dtype, allow_decode: bool) -> _Config | None: + """The shape's row for ``m`` rows (``_select_config``); None keeps the common fallback (fp16 output, row-major + codes at up to ``ROW_MAJOR_FALLBACK_MAX`` rows).""" + if out_dtype != torch.bfloat16: + return None + if allow_decode and m <= DECODE_MAX: + return _decode_row(m) + if m <= ROW_MAJOR_FALLBACK_MAX: + return None + plan = persistent_plan(m) + return _persistent(*plan[0][1], _PERSISTENT_SPLIT_KERNEL if len(plan) > 1 else _PERSISTENT_KERNEL) + + +def select_deferred(m: int, config: _Config) -> _Config: + """The chain's split-K row for ``m`` rows of producer codes in ``config``'s tile, else ``config``. A deferred + row reads a prefix of the producer's codes: ``ceil(m / tile) * tile`` never exceeds the producer's rows.""" + if config.kernel != _DECODE_KERNEL: + return config + for top, row in DEFERRED_ROWS: + if m <= top: + held = -(-m // config.tile_m) * config.tile_m + return row if -(-m // row.tile_m) * row.tile_m <= held else config + return config + + +@functools.cache +def _persistent_launcher(tiles: tuple, grid_max: int): + from .persistent import compile_mxfp8_gemm_n4992_k6144 + + return compile_mxfp8_gemm_n4992_k6144(tiles, True, grid_max) + + +def _launch_tile(tile, grid_max, activation, activation_scale, weight, weight_scale, out): + from amd_kernels.common.tensor_shim import _run_compiled + + m, n = out.shape + _run_compiled( + _persistent_launcher((tile[0] // 64, tile[1] // 64), grid_max), + activation.view(torch.int8).view(-1), weight.view(torch.int8).view(-1), out.view(-1), + activation_scale.view(torch.uint8).view(-1), weight_scale.view(torch.uint8).view(-1), + m, n, -(-m // tile[0]) * (n // tile[1]), torch.cuda.current_stream(), + ) + + +_compiled_all = [] + + +def _launch_persistent(tile, activation, activation_scale, weight, weight_scale, out): + m, n = out.shape + if activation.shape[1] != K or n != N or n % tile[1]: + raise ValueError(f"persistent N4992 K6144 GEMM got K {activation.shape[1]}, N {n}, tile {tile}") + # Inside a DBO microbatch more tiles than CUs take the plain grid. Every tile and grid form writes the same + # bytes; the first launch outside capture runs (compiles) them all, so no later row count or DBO step compiles. + if not _compiled_all and not torch.cuda.is_current_stream_capturing(): + for other in PERSISTENT_ROUND_US: + for grid_max in (256, _PLAIN_GRID_MAX): + _launch_tile(other, grid_max, activation, activation_scale, weight, weight_scale, out) + _compiled_all.append(True) + grid_max = _fourwave_grid_max() if -(-m // tile[0]) * (n // tile[1]) > 256 else 256 + _launch_tile(tile, grid_max, activation, activation_scale, weight, weight_scale, out) + + +def _launch_split(activation, activation_scale, weight, weight_scale, out): + """``persistent_plan``'s launches over row ranges (row-major codes, scale rows in 32-row groups).""" + m = out.shape[0] + plan = persistent_plan(m) + for i, (first, tile) in enumerate(plan): + last = plan[i + 1][0] if i + 1 < len(plan) else m + _launch_persistent(tile, activation[first:last], activation_scale[first:], weight, weight_scale, + out[first:last]) + + +LAUNCHERS = { + _PERSISTENT_KERNEL: lambda config, a, sa, w, sw, out: _launch_persistent((config.tile_m, config.tile_n), a, sa, + w, sw, out), + _PERSISTENT_SPLIT_KERNEL: lambda config, a, sa, w, sw, out: _launch_split(a, sa, w, sw, out), +} diff --git a/amd_kernels/gemm/mxfp8_n4992_k6144/persistent.py b/amd_kernels/gemm/mxfp8_n4992_k6144/persistent.py new file mode 100644 index 0000000..c54b1b6 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n4992_k6144/persistent.py @@ -0,0 +1,740 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Persistent 4-wave MXFP8 GEMM for N4992 K6144, row-major codes and shuffled scales. + +The pipeline of ``mxfp8_common``'s 4-wave kernel with a tile that divides N 4992 +(64 n_a rows x 64 n_b columns, n_b 2 or 3: 39 or 26 column tiles) and its own tile +grid, so no work item reaches a column past N (the common kernel's 256-column parents +would wrap the last 128 columns into the next row). Quadrants of an odd tile count +re-pair E8M0 words across their 32-row scale groups; scale DMA copies contiguous +128-row packets. A 48-column quadrant row (n_b 3) leaves the output through an +unswizzled LDS transpose of 16-byte pieces, two stores per 16-row block. Every +accumulator visits K in increasing order, one 16x16x128 MFMA per step, so every tile +(and a launch over a row range) writes the same bytes: the host picks tiles per row +count and splits a call's rows over two tiles on that. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import Vector as Vec +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_4wave import ( + BLOCK_K, + G2SLoaderAsm, + MfmaRowStationary, + S2RLoader, + StoreCLds as StoreCLdsBase, + _Buf, + _LDS_BUF_BYTES, + _N_WAVES, + _PACK, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOTS, + _SCALE_SLOT_BYTES, + _ceildiv, + _g2s_thunks, + _global_swizzle, + _lds_ptr_t, + _min, + _riffle, + _s2r_thunks, + _uniform_i32, + _xcd_swizzle, + wait_barrier, + ScaleGatherLDS as ScaleGatherBase, +) + + +class ScaleGatherLDS(ScaleGatherBase): + def tile_group(self, a_base_tile, b_base_tile): + """This wave's first 32-row group of a tile (wave-uniform).""" + base_tile = fx.arith.select( + self._is_a, + a_base_tile + self._q * fx.Int32(128), + b_base_tile + self._q * fx.Int32(128), + ) + return _uniform_i32(base_tile // fx.Int32(32)) + + def set_tile_voff(self, G): + """This tile's per-lane gather offset with the K-block folded out (it rides + the uniform soffset instead): computed once per tile, so the K loop carries + no per-gather voffset math.""" + grp = fx.Int32(G) + self._blk + self._voff_g = fx.as_ir_value( + (grp * fx.Int32(self.row_i32) + self._in16 * fx.Int32(4)) * fx.Int32(4) + ) + +class ScaleLoaderLDS: + """The wave's scale dwords of a block: lane (klane, r) reads the dword of group + (half, gi) at its (klane, mlane) position -- the layout the MFMA wants.""" + + def __init__(self, n_tiles, lane_id, quarter, lds_base_ptr, region_off): + self.n_groups = (n_tiles + 1) // _PACK + self.n_tiles = n_tiles + self.quarter = quarter + self.lane_id = lane_id + self._region_base = ( + fx.Int32(fx.ptrtoint(lds_base_ptr)) + + fx.Int32(region_off) + ) + + def read_half(self, slot, half): + L = self.lane_id + base = ( + self._region_base + + fx.Int32(slot) * fx.Int32(_SCALE_SLOT_BYTES) + + fx.Int32((L // 4) * 16 + (L % 4) * 4) + ) + out = [] + for gi in range_constexpr(self.n_groups): + row = self.quarter * (16 * self.n_tiles) + half * (32 * self.n_tiles) + gi * 32 + group = row // 32 + ptr = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(base + group * 256)) + lo = fx.Int32(_llvm.LoadOp(fx.Int32.ir_type, ptr, alignment=4).result) + if const_expr(self.n_tiles % 2 == 1): + ptr2 = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(base + (group + 1) * 256)) + hi = fx.Int32(_llvm.LoadOp(fx.Int32.ir_type, ptr2, alignment=4).result) + odd = (row % 32) != 0 + packed = (lo.shrui(fx.Int32(8)) & 0x00ff00ff) | ((hi & 0x00ff00ff) << 8) + lo = odd.select(packed, lo) + out.append(lo) + + return out + + def read(self, slot): + return self.read_half(slot, 0), self.read_half(slot, 1) + +class StoreCLds(StoreCLdsBase): + """``StoreCLds`` for any quadrant width: 16-byte pieces of a 16-row block, lane-linear + over ceil(16 * 2 n_b / 64) reads; lanes past the block repeat a piece (the same bytes + to the same address).""" + + def __init__(self, C, c_rows, c_cols, c_idx_fn, lds_base_ptr, wave_id, n_a, n_b, nt=False): + self.c_cols = c_cols + self.c_rows = c_rows + self.c_idx_fn = c_idx_fn + self.n_a = n_a + self.n_b = n_b + self.pieces = 2 * n_b + per_block = 16 * self.pieces + self.slabs = (per_block + 63) // 64 + self.buf_bytes = 16 * 32 * n_b + self.cpol = 2 if nt else 0 + self.rsrc = buffer_ops.create_buffer_resource( + C, max_size=False, num_records_bytes=c_rows * c_cols * 2 + ) + lane = fx.thread_idx.x % 64 + g = lane // 16 + r = lane % 16 + row_b = 32 * n_b + base = fx.Int32(fx.ptrtoint(lds_base_ptr)) + wave_id * fx.Int32(2 * self.buf_bytes) + self.w_off = [base + r * fx.Int32(row_b) + fx.Int32(32 * tj) + g * fx.Int32(8) + for tj in range_constexpr(n_b)] + self.r_off, self.st_offs = [], [] + for h in range_constexpr(self.slabs): + left = per_block - 64 * h + lin = fx.Int32(64 * h) + (lane if left >= 64 else lane % left) + row, q = lin // self.pieces, lin % self.pieces + self.r_off.append(base + row * fx.Int32(row_b) + q * fx.Int32(16)) + self.st_offs.append((row * c_cols) * 2 + q * 16) + + def _read_store(self, base_row, base_col, ti, h, buf): + ptr = _llvm.inttoptr( + _lds_ptr_t(), fx.as_ir_value(self.r_off[h] + fx.Int32(buf * self.buf_bytes)) + ) + vec4 = _ir.VectorType.get([4], fx.Int32.ir_type) + data = Vec(_llvm.LoadOp(vec4, ptr, alignment=16).result) + off = ((base_row + ti * 16) * self.c_cols + base_col) * 2 + self.st_offs[h] + buffer_ops.buffer_store( + data, self.rsrc, off, offset_is_bytes=True, cache_modifier=self.cpol + ) + + +def compile_mxfp8_gemm_n4992_k6144(tiles=(4, 2), store_nt=False, grid_max=256): + """The persistent kernel with ``tiles`` = (n_a, n_b) 16-row / 16-column MFMA tiles per + wave quadrant; ``grid_max`` above the work-item count launches a plain grid.""" + K, N = 6144, 4992 + N_TILES_A, N_TILES_B = tiles + assert N_TILES_A in (2, 3, 4) and N_TILES_B in (2, 3) + BLOCK_M = 64 * N_TILES_A + BLOCK_N = 64 * N_TILES_B + assert N % BLOCK_N == 0 + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + assert K % 256 == 0 + K_ITERS = K // BLOCK_K # even + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + assert ( + LDS_BLOCK_M * BLOCK_K <= _LDS_BUF_BYTES + and LDS_BLOCK_N * BLOCK_K <= _LDS_BUF_BYTES + ) + # grid_max: workgroups of the launch (256 = one per CU, persistent); a larger value gives + # every work item its own workgroup (a plain grid), which no co-resident kernel holds back + GRID_MAX = grid_max + name = (f"flydsl_mxfp8_n4992_k6144_{BLOCK_M}x{BLOCK_N}" + ("_nt" if store_nt else "") + + ("" if grid_max == 256 else f"_g{grid_max}")) + SCALE_SLOTS = _SCALE_SLOTS + OUT_LDS_BYTES = _N_WAVES * 2 * 16 * 32 * N_TILES_B + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, 8 * _LDS_BUF_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_SLOTS * _SCALE_SLOT_BYTES, 16] + out_lds: fx.Array[fx.Int8, OUT_LDS_BYTES, 16] + + @flyc.kernel(name=name) + def kernel_gemm( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + n_work: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + _base_ptr = lds.all_lds.ptr + + def _buf(idx): + return _Buf(_base_ptr, idx * _LDS_BUF_BYTES) + + a_cur0, a_cur1, a_next0, a_next1 = _buf(0), _buf(1), _buf(2), _buf(3) + b_cur0, b_cur1, b_next0, b_next1 = _buf(4), _buf(5), _buf(6), _buf(7) + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + n_pid_m = _ceildiv(c_m, BLOCK_M) + n_blocks = c_n // BLOCK_N + grid = _min(fx.Int32(n_work), fx.Int32(GRID_MAX)) + wg = fx.Int32(fx.block_idx.x) + my_tiles = (fx.Int32(n_work) - wg + grid - fx.Int32(1)) // grid + wave_i = wave_id // 2 + wave_j = wave_id % 2 + A_K_STEP = BLOCK_K + B_K_STEP = 2 * 1024 # preshuffled (16, 16): a 16-row block spans 2 KB per 128 K + + def tile_coords(t): + """work item -> (tile_i, tile_j) in the XCD order of ``_xcd_swizzle``; a negative / + out-of-range index lands beyond M so its stores are dropped by the buffer bounds.""" + return _xcd_swizzle(n_pid_m, n_blocks, t) + + def tile_bases(ti, tj): + """the g2s bases of a tile (row-major A, preshuffled W) and its scale groups""" + return ( + (ti * BLOCK_M) * K, + (ti * BLOCK_M + LDS_BLOCK_M) * K, + (tj * BLOCK_N) * K, + (tj * BLOCK_N + LDS_BLOCK_N) * K, + ) + + mfma = MfmaRowStationary(N_TILES_A, N_TILES_B) + _scale_base_ptr = lds.scale_lds.ptr + scale_gather = ScaleGatherLDS( + A_scale, + B_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + c_m, + c_n, + N_TILES_A, + N_TILES_B, + ) + scale_gather.set_wave_base() + # the two scale-slot LDS addresses (slot 0 / 1) precomputed once; the + # gather's K-block offset rides the uniform soffset inside the K loop + _gm0_s = ( + scale_gather._wave_base_s, + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + fx.Int32(_SCALE_SLOT_BYTES) + ), + ) + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(b): + return fx.Int32(b) % fx.Int32(SCALE_SLOTS) + + gl_off_a = _global_swizzle(lane_id, wave_id, K, N_TILES_A, False) + gl_off_b = _global_swizzle(lane_id, wave_id, K, N_TILES_B, True) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=c_m * K + ) + b_rsrc = buffer_ops.create_buffer_resource( + B_T, max_size=False, num_records_bytes=c_n * K + ) + a_g2s = G2SLoaderAsm(a_rsrc, gl_off_a, N_TILES_A, wave_id) + b_g2s = G2SLoaderAsm(b_rsrc, gl_off_b, N_TILES_B, wave_id) + a_g2s.set_wave_base(_base_ptr) + b_g2s.set_wave_base(_base_ptr) + a_s2r = S2RLoader(wave_i, N_TILES_A) + b_s2r = S2RLoader(wave_j, N_TILES_B) + store_c = (StoreCLds if N_TILES_B % 2 else StoreCLdsBase)( + C, + c_m, + c_n, + mfma.idx, + lds.out_lds.ptr, + wave_id, + N_TILES_A, + N_TILES_B, + nt=store_nt, + ) + + # a step issues [ac0 (n_a) | bc0 (n_b) | bc1 (n_b) | ac1 (n_a)] loads per wave: at a + # step's top the previous step's may all be in flight; mid-step the previous step's + # ac1 and this step's first two batches + _MAIN_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(block, holder): + # the caller passes the LDS slot itself (the block's K offset rides the + # uniform soffset, so no block -> slot math is left) + s = block + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _g2s_bases(bases, step): + a0, a1, b0, b1 = bases + st = fx.Int32(step) + return ( + fx.Int32(a0) + st * fx.Int32(A_K_STEP), + fx.Int32(a1) + st * fx.Int32(A_K_STEP), + fx.Int32(b0) + st * fx.Int32(B_K_STEP), + fx.Int32(b1) + st * fx.Int32(B_K_STEP), + ) + + def _one_step( + parity, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + load, + gather, + read_block, + zero_acc=False, + extra=None, + ): + """One K-step: this step's MFMAs with, interleaved, the LDS reads for the next + step, the global loads ``load`` of the step after (the resolved (a0, a1, b0, + b1) soffsets), at even steps the gather ``gather`` of the next scale block (the + precomputed (m0, soff)), at odd steps the read of scale block ``read_block`` + (the LDS slot itself) into the VGPR carry; + ``extra(g, c00f, c01f)`` (called before group g = 0..3 with the accumulators + finished so far, None before) returns more thunks for that group: the stores + of this tile's c00 / c01 in the last step, of the previous tile's c10 / c11 + in the first.""" + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + a0_off, a1_off, b0_off, b1_off = load + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(read_block, _scn) if parity == 1 else [] + _sc_gather = ( + [] + if parity == 1 + else [lambda: scale_gather.gather_soff(gather[0], gather[1])] + ) + ex = extra if extra is not None else (lambda g, c00, c01: ()) + + wait_barrier(_MAIN_VMCNT) + il = _riffle( + _g2s_thunks(a_g2s, ac0, a0_off, N_TILES_A), + _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False), + ) + list(ex(0, None, None)) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + il = _riffle( + _g2s_thunks(b_g2s, bc0, b0_off, N_TILES_B), list(ex(1, c00f, None)) + ) + c01f = mfma.call( + a0f, b1f_in, c01f, saR0, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + a1f = _a1 + + wait_barrier(_SEG2_VMCNT) + il = (_riffle(_g2s_thunks(b_g2s, bc1, b1_off, N_TILES_B), _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False)) + + _sc_gather + _rd_scn[:2] + list(ex(2, c00f, c01f))) # fmt: skip + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + a0nf = _a0n + il = ( + _riffle( + _g2s_thunks(a_g2s, ac1, a1_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + + _rd_scn[2:] + + list(ex(3, c00f, c01f)) + ) + c11f = mfma.call( + a1f, b1f_in, c11f, saR1, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + b0nf, b1nf = _b0n, _b1n + + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) if parity == 1 else sc + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = N_TILES_A + n_b = N_TILES_B + n_ga = (N_TILES_A + 1) // _PACK + n_gb = (N_TILES_B + 1) // _PACK + n_sc = 2 * n_ga + 2 * n_gb + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC0, sbC1 = sc + return ( + [_R(v) for v in saR0] + + [_R(v) for v in saR1] + + [_R(v) for v in sbC0] + + [_R(v) for v in sbC1] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC0 = list(flat[o : o + n_gb]) + o += n_gb + sbC1 = list(flat[o : o + n_gb]) + return (saR0, saR1, sbC0, sbC1) + + # ---- prologue: the first tile's steps 0 / 1 and its scale blocks 0 / 1 ---- + t0 = wg + ti0, tj0 = tile_coords(t0) + bases0 = tile_bases(ti0, tj0) + G0 = scale_gather.tile_group(ti0 * BLOCK_M, tj0 * BLOCK_N) + scale_gather.set_tile_voff(G0) + scale_gather.gather_soff(_gm0_s[0], 0) + scale_gather.gather_soff(_gm0_s[1], 256) + s0 = _g2s_bases(bases0, 0) + s1 = _g2s_bases(bases0, 1) + a_g2s.load(a_cur0, s0[0]) + b_g2s.load(b_cur0, s0[2]) + b_g2s.load(b_cur1, s0[3]) + a_g2s.load(a_cur1, s0[1]) + a_g2s.load(a_next0, s1[0]) + b_g2s.load(b_next0, s1[2]) + b_g2s.load(b_next1, s1[3]) + a_g2s.load(a_next1, s1[1]) + # 2 gathers + 32 loads in flight: a_cur0 landed at vmcnt(28), b_cur0 / b_cur1 at 20 + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + zero4 = Vec.filled(4, 0.0, fx.Float32) + c10_prev = [zero4] * N_ACCUMS + c11_prev = [zero4] * N_ACCUMS + + # ---- the tile loop: state = fragments of the tile's step 0, its scale block 0, the + # previous tile's c10 / c11 (stored during this tile's first step), the tile index + tile_state = ( + [_R(x) for x in a0_frag] + [_R(x) for x in b0_frag] + [_R(x) for x in b1_frag] + _flat_sc(sc0) + + [_R(x) for x in c10_prev] + [_R(x) for x in c11_prev] + [_R(fx.Int32(t0)), _R(fx.Int32(-1))] + ) # fmt: skip + for jt, tstate in range(0, my_tiles, 1, init=tile_state): + off = 0 + a0f = list(tstate[off : off + n_a]) + off += n_a + b0f = list(tstate[off : off + n_b]) + off += n_b + b1f = list(tstate[off : off + n_b]) + off += n_b + sc = _unflat_sc(tstate[off : off + n_sc]) + off += n_sc + c10p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_cur = fx.Int32(tstate[off]) + t_prev = fx.Int32(tstate[off + 1]) + t_next = t_cur + grid + ti, tj = tile_coords(t_cur) + bases = tile_bases(ti, tj) + G = scale_gather.tile_group(ti * BLOCK_M, tj * BLOCK_N) + scale_gather.set_tile_voff(G) + # the next tile (the last tile loads a tile beyond the grid: rows beyond M read + # zero through the buffer bounds and nothing of it is stored) + tin, tjn = tile_coords(t_next) + bases_n = tile_bases(tin, tjn) + Gn = scale_gather.tile_group(tin * BLOCK_M, tjn * BLOCK_N) + # the previous tile's c10 / c11 store bases (row beyond M when there is none) + tip, tjp = tile_coords(t_prev) + has_prev = t_prev >= fx.Int32(0) + prev_row1 = fx.arith.select( + has_prev, + tip * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M, + fx.Int32(c_m), + ) + prev_col0 = fx.arith.select( + has_prev, tjp * BLOCK_N + wave_j * (N_TILES_B * 16), fx.Int32(0) + ) + prev_col1 = fx.arith.select(has_prev, prev_col0 + LDS_BLOCK_N, fx.Int32(0)) + base_row = ti * BLOCK_M + wave_i * (N_TILES_A * 16) + base_col = tj * BLOCK_N + wave_j * (N_TILES_B * 16) + + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # step 0: zero accumulators, the previous tile's c10 / c11 stores spread over the + # groups before the new c10 (born in group 2) / c11 (group 3), gather block 1 + prev_thunks = store_c.thunks(c10p, prev_row1, prev_col0) + store_c.thunks( + c11p, prev_row1, prev_col1 + ) + n_prev3 = ( + len(prev_thunks) + 2 + ) // 3 # over the three groups before the new c11 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, _g2s_bases(bases, 2), (_gm0_s[1], 256), None, zero_acc=True, + extra=lambda g, c00, c01: prev_thunks[n_prev3 * g : n_prev3 * (g + 1)], + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, _g2s_bases(bases, 3), None, 1 + ) + # the four g2s streams advance by uniform strides and the gather + # block by 256 B; carry them as uniform accumulators (incremented + # per step) instead of recomputing base | (step << s) every K-step + ka0 = fx.Int32(bases[0]) + fx.Int32(4 * A_K_STEP) + ka1 = fx.Int32(bases[1]) + fx.Int32(4 * A_K_STEP) + kb0 = fx.Int32(bases[2]) + fx.Int32(4 * B_K_STEP) + kb1 = fx.Int32(bases[3]) + fx.Int32(4 * B_K_STEP) + kg = fx.Int32(512) # gather block soff at kk=2: block 2 << 8 + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg)] + ) # fmt: skip + for kk, state in range(2, K_ITERS - 2, UNROLL, init=init_state): + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + bufs = bufs0 + off += N_ACCUMS # past c11f: the uniform address accumulators + ka0 = fx.Int32(state[off]) + ka1 = fx.Int32(state[off + 1]) + kb0 = fx.Int32(state[off + 2]) + kb1 = fx.Int32(state[off + 3]) + kg = fx.Int32(state[off + 4]) + for u in range_constexpr(UNROLL): + if const_expr(UNROLL == 4): + # kk == 2 (mod 4): u=0 gathers an even block (slot 0), u=2 an + # odd one (slot 1) -- gather m0 slots folded; read slots stay + # runtime (see below). + lds_off = (ka0, ka1, kb0, kb1) + gath = None + if const_expr(u % 2 == 0): + gath = ( + _gm0_s[u // 2], + kg if u == 0 else kg + fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + # keep the read slot a runtime value: folding it to a + # constant made LLVM un-pair the scale ds_read2st64 + # into 2 ds_read_b32 (+2 LDS/step at the ISA gate) + rslot = ( + fx.Int32(kk) // fx.Int32(2) + fx.Int32(1 + u // 2) + ) % fx.Int32(2) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + fx.Int32(1) + lds_off = _g2s_bases(bases, kc + fx.Int32(2)) + gath = None + if const_expr(u % 2 == 0): + gath = ( + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + + (block % fx.Int32(2)) + * fx.Int32(_SCALE_SLOT_BYTES) + ), + block * fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + rslot = block % fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + lds_off, + gath, + rslot, + ) + ka0 = ka0 + fx.Int32(A_K_STEP) + ka1 = ka1 + fx.Int32(A_K_STEP) + kb0 = kb0 + fx.Int32(B_K_STEP) + kb1 = kb1 + fx.Int32(B_K_STEP) + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg + fx.Int32(UNROLL * 128))] + ) # fmt: skip + state = yield new_state + + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + # step K_ITERS - 2 (even): loads of the next tile's step 0, gather of its block 0 + scale_gather.set_tile_voff(Gn) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, + a0f, + b0f, + b1f, + sc, + accs, + bufs0, + _g2s_bases(bases_n, 0), + (_gm0_s[0], 0), + None, + ) + # step K_ITERS - 1 (odd): loads of the next tile's step 1, read of its block 0; + # this tile's c00 / c01 stored over the groups after their last MFMAs (c00: 1..3, + # c01: 2..3; the two quadrants' thunks stay concatenated, they share the LDS buffer) + n_q3 = ( + 2 * store_c.n_thunks + 2 + ) // 3 # thunks per group: two quadrants over 3 groups + + def _last_extra(g, c00, c01): + # (compile-time python only: no mutable state -- the DSL rewriter turns + # body-level literals into None -- and every `if` under const_expr) + if const_expr(g == 0): + return () + t00 = store_c.thunks(c00, base_row, base_col) + if const_expr(g == 1): + return t00[:n_q3] + t01 = store_c.thunks(c01, base_row, base_col + LDS_BLOCK_N) + if const_expr(g == 2): + return t00[n_q3:] + t01[: 2 * n_q3 - len(t00)] + return t01[2 * n_q3 - len(t00) :] + + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + _g2s_bases(bases_n, 1), + None, + 0, + extra=_last_extra, + ) + new_tile_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + [_R(t_next), _R(t_cur)] + ) # fmt: skip + tstate = yield new_tile_state + + # ---- epilogue: the last tile's c10 / c11 ---- + off = n_a + 2 * n_b + n_sc + c10_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_last = fx.Int32(tstate[off + 1]) + til, tjl = tile_coords(t_last) + last_row1 = til * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M + last_col0 = tjl * BLOCK_N + wave_j * (N_TILES_B * 16) + store_c.store(c10_last, last_row1, last_col0) + store_c.store(c11_last, last_row1, last_col0 + LDS_BLOCK_N) + + @flyc.jit + def launch_gemm_n4992_k6144( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + n_work: fx.Int32, + stream: fx.Stream, + ): + grid_x = _min(n_work, fx.Int32(GRID_MAX)) + kernel_gemm( + A, B_T, C, A_scale, B_scale, c_m, c_n, n_work, + value_attrs={"rocdl.waves_per_eu": 1, "rocdl.flat_work_group_size": "256,256"}, + ).launch(grid=(grid_x, 1, 1), block=(256, 1, 1), stream=stream) # fmt: skip + + launch_gemm_n4992_k6144.kernel_name = name + return launch_gemm_n4992_k6144 diff --git a/amd_kernels/gemm/mxfp8_n6144_k4096/README.md b/amd_kernels/gemm/mxfp8_n6144_k4096/README.md new file mode 100644 index 0000000..9b4f6eb --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n6144_k4096/README.md @@ -0,0 +1,41 @@ +# MXFP8 GEMM, N 6144 x K 4096 (o_proj, TP2) + +## What it does +The attention output projection of MiniMax-M3.1-Flash at TP2: K 4096 = 32 heads x 128, N 6144, bf16 out; the target's +attention layers and the DSpark draft's attention (64 heads / 2 ranks). TP2 has no sequence-parallel `o_proj_full`, so +this shape serves every prefill chunk too (to 32768 rows). The rows and kernels of this shape, registered with the +dispatcher in [`../mxfp8_common`](../mxfp8_common/README.md) (`gemm._SHAPE_HOSTS`); layouts and the numerics contract +are that folder's. A producer of o_proj codes writes them in the tile `prequant_code_tile_rows` names (decode rows to +768, row-major above). + +## Entry points +- The common entries (`mxfp8_linear`, `mxfp8_gemm`, `mxfp8_gemm_shuffled`, `mxfp8_gemm_prequant`, + `warmup_mxfp8_gemm_weight`). +- `host.select`: this shape's config for a row count, for every row count (bf16 out; split-K decode rows at + 49-224); it is the dispatch. +- `host.select_prequant`: the decode row for producer codes. +- `host.decode_launcher`: the owner-read decode kernel of the 96-column two-K-wave rows (None: the common one). +- `host.LAUNCHERS`: the launcher of each kernel `select` names (the 128x192 and one-tile 256x192 copies at K 4096, + the K-generic persistent 256x192 of `gemm/mxfp8_n6144_k8192`). + +## Performance +One MI355X. Graph replay, layer 39 o_proj TP2 rank 0 weights (`[:, :4096]`) in 16 cold copies (> 400 MB), seeded +inputs, 15 alternated rounds, median us per GEMM. Baseline: vLLM's Triton `tl.dot_scaled` linear, the TP2 fallback +(`_mxfp8_dot_scaled_gemm` on codes, `_mxfp8_dot_scaled_linear` on bf16). Codes: the producer's MXFP8 codes +(`mxfp8_gemm_prequant`); bf16: `mxfp8_linear` (quantizer included). Roofline: max(bytes / 6.5 TB/s, flops / 5 PF). + +| M | codes us | baseline us | bf16 us | baseline us | roofline us | +|---|---|---|---|---|---| +| 1 / 8 | 5.84 / 5.90 | 8.92 / 9.02 | 5.77 / 6.33 | 11.12 / 11.23 | 4.00 | +| 32 / 64 | 6.77 / 8.09 | 14.66 / 22.30 | 8.99 / 10.56 | 17.02 / 24.80 | 4.07 / 4.16 | +| 128 / 256 | 9.54 / 13.11 | 18.14 / 31.90 | 12.19 / 16.63 | 19.04 / 33.23 | 4.32 / 4.64 | +| 320 / 384 | 14.44 / 15.62 | 39.43 / 44.88 | 18.03 / 19.36 | 43.24 / 47.61 | 4.81 / 4.97 | +| 512 / 768 | 18.38 / 25.75 | 46.54 / 65.12 | 22.73 / 30.48 | 46.26 / 68.32 | 5.29 / 7.73 | +| 1024 / 2048 | | | 32.24 / 47.24 | 67.80 / 84.81 | 10.31 / 20.62 | +| 4096 / 8192 | | | 91.34 / 160.39 | 167.74 / 348.64 | 41.23 / 82.46 | +| 16384 / 32768 | | | 316.21 / 621.37 | 735.65 / 1811.77 | 164.9 / 329.9 | + +## Tests +- `python3 -m pytest tests/gemm/mxfp8_n6144_k4096 -q` +- `python3 benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py --checkpoint --form codes` +- `python3 benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py --checkpoint --form bf16` diff --git a/amd_kernels/gemm/mxfp8_n6144_k4096/__init__.py b/amd_kernels/gemm/mxfp8_n6144_k4096/__init__.py new file mode 100644 index 0000000..b14fa99 --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n6144_k4096/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP8 GEMM rows and kernels of the o_proj N 6144 x K 4096 (TP2). Called through ``gemm.mxfp8_common``.""" diff --git a/amd_kernels/gemm/mxfp8_n6144_k4096/host.py b/amd_kernels/gemm/mxfp8_n6144_k4096/host.py new file mode 100644 index 0000000..b7da84b --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n6144_k4096/host.py @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MXFP8 GEMM rows and kernels of the o_proj N 6144 x K 4096 (TP2), registered with the common dispatcher +(``gemm.mxfp8_common.gemm._SHAPE_HOSTS``).""" + +import functools + +import torch + +from amd_kernels.gemm.mxfp8_common.gemm import ( + _DECODE_FQ_KERNEL, + _DECODE_KERNEL, + _FOURWAVE_NT_KERNEL, + _STAGED_KERNEL, + _Config, + _fourwave_grid_max, + splitk_rows, +) +from amd_kernels.gemm.mxfp8_n6144_k8192.host import _prefill_n6144_k8192_192_launcher + +_N, _K = 6144, 4096 +_TILE_128_KERNEL = "n6144_k4096_128x192" +_ONE_TILE_KERNEL = "n6144_k4096_256x192_one_tile" +_PERSISTENT_KERNEL = "n6144_k4096_256x192" + + +def _decode(tile_m, tile_n, k_waves, kernel=_DECODE_KERNEL, sched_barriers=0): + return _Config(tile_m, tile_n, 128, 0, 0, 1, kernel, k_waves, sched_barriers) + + +def _tile(tile_m, tile_n, kernel): + return _Config(tile_m, tile_n, 128, 1, 1, 1, kernel, 2, 2) + + +# (last row, config): the decode rows on (16, 16) codes (a producer's, or the bf16 activation quantized for them); +# swept over tiles 16..160 x 16..96, K-waves 1/2/4/8, split-K 1/2 (benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py). +# 49-224: the split-K decode kernel (two m-blocks, K split 2), -3..-13% at every multiple of 8 and window start; +2..+4% +# at 33-48, +0.5..+1% at 240-256 on the layer-39 weights (benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py, +# benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py). +_DECODE_ROWS = ( + (16, _decode(16, 32, 1)), + (32, _decode(32, 32, 1, sched_barriers=1)), + (48, _decode(48, 32, 1)), + *splitk_rows(49, 224, 2, 96, 3, 3, 2, 3), + (256, _decode(64, 96, 2, sched_barriers=1)), + (320, _decode(80, 96, 2, sched_barriers=1)), + (384, _decode(96, 96, 2, sched_barriers=1)), + (448, _decode(112, 96, 2, sched_barriers=1)), + (512, _decode(128, 96, 2)), + (576, _decode(144, 96, 2, sched_barriers=1)), + (640, _decode(160, 96, 2, sched_barriers=1)), + (768, _decode(96, 96, 2, sched_barriers=1)), +) +# The two-K-wave 96-column rows with scheduling barriers read owned accumulators in the owner's branch +# (OWNER_READ): -0.5..-3.5 % at 225-448 and 513-768 rows, bitwise equal (sweep_rows.py --family decode). +_OWNER_READ = frozenset({(64, 96), (80, 96), (96, 96), (112, 96), (144, 96), (160, 96)}) +# a bf16 activation at few rows: the decode kernel quantizes it in registers (no quantizer launch) +_DECODE_FQ_ROWS = ( + (4, _decode(4, 32, 2, _DECODE_FQ_KERNEL)), + (8, _decode(8, 32, 2, _DECODE_FQ_KERNEL)), + (16, _decode(16, 32, 2, _DECODE_FQ_KERNEL)), + (32, _decode(16, 48, 4, _DECODE_FQ_KERNEL)), +) +# row-major codes (above 768 rows also the bf16 activation's rows, after the quantizer). The 4-wave 256x256 row +# and the persistent 256x192 tile alternate with their wave quantization: the 256x192 tile -1.0..-3.5 % at 3329-4096 +# and 5121-6144 rows, +4..18 % elsewhere above 2048 (bf16 in; sweep_rows.py --family prefill). +_PREFILL_ROWS = ( + (32, _Config(32, 32, 256, 0, 0, 4)), + (128, _Config(64, 64, 256, 0, 0, 2)), + (384, _Config(128, 64, 256, 2, 4, 1)), + (640, _Config(128, 128, 128, 2, 4, 1, _STAGED_KERNEL, 2, 4)), + (1024, _tile(128, 192, _TILE_128_KERNEL)), + (1280, _tile(256, 128, _FOURWAVE_NT_KERNEL)), + (2048, _tile(256, 192, _ONE_TILE_KERNEL)), + (3328, _tile(256, 256, _FOURWAVE_NT_KERNEL)), + (4096, _tile(256, 192, _PERSISTENT_KERNEL)), + (5120, _tile(256, 256, _FOURWAVE_NT_KERNEL)), + (6144, _tile(256, 192, _PERSISTENT_KERNEL)), +) +_LARGE = _tile(256, 256, _FOURWAVE_NT_KERNEL) + + +def _row(rows, m): + for last, config in rows: + if m <= last: + return config + return None + + +def select(m: int, out_dtype: torch.dtype, allow_decode: bool) -> _Config | None: + """This shape's config for ``m`` rows ahead of the tuned table (``_select_config``), or None.""" + if out_dtype != torch.bfloat16: + return None + if allow_decode: + config = _row(_DECODE_FQ_ROWS, m) or _row(_DECODE_ROWS, m) + if config is not None: + return config + return _row(_PREFILL_ROWS, m) or _LARGE + + +def select_prequant(m: int, out_dtype: torch.dtype) -> _Config | None: + """The config for producer codes (``_select_prequant_config``): the decode rows without the in-kernel quantizer.""" + if out_dtype != torch.bfloat16: + return None + return _row(_DECODE_ROWS, m) + + +def decode_launcher(config: _Config, a_bf16: bool): + """The decode kernel of an owner-read row (``_OWNER_READ``), or None (the common decode kernel).""" + if ( + a_bf16 or config.kernel != _DECODE_KERNEL or (config.tile_m, config.tile_n) not in _OWNER_READ + or (config.waves_m, config.split_k, config.waves_n) != (2, 1, 1) + or config.la_w or config.la_a or config.w_cm is not None + ): + return None + from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import compile_mxfp8_gemm_decode + + return compile_mxfp8_gemm_decode( + N=_N, K=_K, RT=config.tile_m // 16, NI=config.tile_n // 16, KW=2, KS=1, + OWNER_READ=True, + ) + + +def _args(activation, activation_scale, weight, weight_scale, out): + assert activation.shape[1] == _K and out.shape[1] == _N and out.dtype == torch.bfloat16 + return ( + activation.view(torch.int8).view(-1), weight.view(torch.int8).view(-1), out.view(-1), + activation_scale.view(torch.uint8).view(-1), weight_scale.view(torch.uint8).view(-1), + ) + + +@functools.cache +def _tile_128_launcher(): + from .tile_128x192 import compile_mxfp8_gemm_n6144_k4096_128x192 + return compile_mxfp8_gemm_n6144_k4096_128x192() + + +def _launch_128(activation, activation_scale, weight, weight_scale, out): + from amd_kernels.common.tensor_shim import _run_compiled + _run_compiled( + _tile_128_launcher(), *_args(activation, activation_scale, weight, weight_scale, out), + activation.shape[0], _N, torch.cuda.current_stream(), + ) + + +@functools.cache +def _one_tile_launcher(): + from .prefill_256x192_one_tile import compile_mxfp8_gemm_n6144_k4096_256x192_one_tile + return compile_mxfp8_gemm_n6144_k4096_256x192_one_tile() + + +def _launch_256(activation, activation_scale, weight, weight_scale, out, *, one_tile): + from amd_kernels.common.tensor_shim import _run_compiled + m = out.shape[0] + n_work = ((m + 255) // 256) * (_N // 192) + if one_tile: + if n_work > 256: + raise ValueError("the one-tile 256x192 GEMM needs at most 256 tiles") + exe = _one_tile_launcher() + else: + # more tiles than CUs: inside a DBO microbatch the plain grid, as the 4-wave rows + exe = _prefill_n6144_k8192_192_launcher(_K, _fourwave_grid_max() if n_work > 256 else 256) + _run_compiled( + exe, *_args(activation, activation_scale, weight, weight_scale, out), + m, _N, 0, n_work, torch.cuda.current_stream(), + ) + + +LAUNCHERS = { + _TILE_128_KERNEL: lambda config, a, sa, w, sw, out: _launch_128(a, sa, w, sw, out), + _ONE_TILE_KERNEL: lambda config, a, sa, w, sw, out: _launch_256(a, sa, w, sw, out, one_tile=True), + _PERSISTENT_KERNEL: lambda config, a, sa, w, sw, out: _launch_256(a, sa, w, sw, out, one_tile=False), +} diff --git a/amd_kernels/gemm/mxfp8_n6144_k4096/prefill_256x192_one_tile.py b/amd_kernels/gemm/mxfp8_n6144_k4096/prefill_256x192_one_tile.py new file mode 100644 index 0000000..457b14e --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n6144_k4096/prefill_256x192_one_tile.py @@ -0,0 +1,842 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One output tile per workgroup, MXFP8 GEMM n6144_k4096_256x192 (gemm/mxfp8_n6144_k8192's one-tile kernel at K 4096). + +The table selects this form only when the entire grid fits one CU round. +There is no prior tile to store and no persistent-tile state to carry. +The final two K steps issue no next-tile DMAs or scale gathers; the last +step does not read next-tile fragments. At each tail step the first +vmcnt(0) + barrier completes every operand DMA on every wave. These LDS +slots then stay immutable, so the second wait covers only this wave's +LDS reads. Output transpose buffers are wave-private and separate. +""" +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T, Vector as Vec +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_4wave import ( + BLOCK_K, + G2SLoaderAsm, + MfmaRowStationary, + PARENT_M, + S2RLoader, + _Buf, + _LDS_BUF_BYTES, + _N_WAVES, + _PACK, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOTS, + _SCALE_SLOT_BYTES, + _acc_read, + _ceildiv, + _cvt_pk_bf16, + _divmod_nonneg, + _g2s_thunks, + _global_swizzle, + _lds_ptr_t, + _min, + _riffle, + _s2r_thunks, + _uniform_i32, + _xcd_swizzle, + wait_barrier, + ScaleGatherLDS as ScaleGatherBase, +) +PARENT_N = 192 + +class ScaleGatherLDS(ScaleGatherBase): + def tile_group(self, a_base_tile, b_base_tile): + """This wave's first 32-row group of a tile (wave-uniform).""" + base_tile = fx.arith.select( + self._is_a, + a_base_tile + self._q * fx.Int32(128), + b_base_tile + self._q * fx.Int32(128), + ) + return _uniform_i32(base_tile // fx.Int32(32)) + + def set_tile_voff(self, G): + """This tile's per-lane gather offset with the K-block folded out (it rides + the uniform soffset instead): computed once per tile, so the K loop carries + no per-gather voffset math.""" + grp = fx.Int32(G) + self._blk + self._voff_g = fx.as_ir_value( + (grp * fx.Int32(self.row_i32) + self._in16 * fx.Int32(4)) * fx.Int32(4) + ) + +class ScaleLoaderLDS: + """The wave's scale dwords of a block: lane (klane, r) reads the dword of group + (half, gi) at its (klane, mlane) position -- the layout the MFMA wants.""" + + def __init__(self, n_tiles, lane_id, quarter, lds_base_ptr, region_off): + self.n_groups = (n_tiles + 1) // _PACK + self.n_tiles = n_tiles + self.quarter = quarter + self.lane_id = lane_id + self._region_base = ( + fx.Int32(fx.ptrtoint(lds_base_ptr)) + + fx.Int32(region_off) + ) + + def read_half(self, slot, half): + L = self.lane_id + base = ( + self._region_base + + fx.Int32(slot) * fx.Int32(_SCALE_SLOT_BYTES) + + fx.Int32((L // 4) * 16 + (L % 4) * 4) + ) + out = [] + for gi in range_constexpr(self.n_groups): + row = self.quarter * (16 * self.n_tiles) + half * (32 * self.n_tiles) + gi * 32 + group = row // 32 + ptr = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(base + group * 256)) + lo = fx.Int32(_llvm.LoadOp(fx.Int32.ir_type, ptr, alignment=4).result) + if const_expr(self.n_tiles % 2 == 1): + ptr2 = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(base + (group + 1) * 256)) + hi = fx.Int32(_llvm.LoadOp(fx.Int32.ir_type, ptr2, alignment=4).result) + odd = (row % 32) != 0 + packed = (lo.shrui(fx.Int32(8)) & 0x00ff00ff) | ((hi & 0x00ff00ff) << 8) + lo = odd.select(packed, lo) + out.append(lo) + + return out + + def read(self, slot): + return self.read_half(slot, 0), self.read_half(slot, 1) + +class StoreCLds: + """BF16 transpose for 48-column quadrants, padded to 64 columns in LDS. + + Each MFMA row writes three 16-column fragments into a 64-column block. + A warp reads eight whole rows; the final two 8-column pieces are masked. + Power-of-two LDS rows retain the original XOR bank-conflict swizzle. + """ + + def __init__( + self, + C, + c_rows, + c_cols, + c_idx_fn, + lds_base_ptr, + wave_id, + n_a, + n_b, + nt=False, + ): + self.c_cols = c_cols + self.c_rows = c_rows + self.c_idx_fn = c_idx_fn + self.n_a = n_a + self.n_b = n_b + n_b = 4 # 48 useful columns, 64-column LDS rows + self.pieces = 2 * n_b # 16-byte pieces per block row (32 n_b bytes) + self.rows_per_read = 64 // self.pieces # rows one 64-lane read / store covers + self.slabs = 16 // self.rows_per_read # reads per block + self.buf_bytes = 16 * 32 * n_b + self.cpol = ( + 2 if nt else 0 + ) # gfx950 buffer_store bits: 0x2 = nt (streaming, write-once output) + self.rsrc = buffer_ops.create_buffer_resource( + C, max_size=False, num_records_bytes=c_rows * c_cols * 2 + ) + lane = fx.thread_idx.x % 64 + g = lane // 16 + r = lane % 16 + row_b = 32 * n_b + base = fx.Int32(fx.ptrtoint(lds_base_ptr)) + wave_id * fx.Int32( + 2 * self.buf_bytes + ) + # write side: lane (g, r) holds columns 4 g .. 4 g + 3 of row r of each 16x16 tile + self.w_off = [ + base + + r * fx.Int32(row_b) + + (((fx.Int32(2 * tj) + g // 2) ^ (r % self.pieces)) * fx.Int32(16)) + + (g % 2) * fx.Int32(8) + for tj in range_constexpr(n_b) + ] + # read side: lane (r2 = lane // pieces, q = lane % pieces) takes piece q of row r2 of each slab + q = lane % self.pieces + r2 = lane // self.pieces + self.r_off = [ + base + + (r2 + h * self.rows_per_read) * fx.Int32(row_b) + + ((q ^ (r2 % self.pieces)) * fx.Int32(16)) + for h in range_constexpr(self.slabs) + ] + self.st_off = ( + r2 * c_cols + q * 8 + ) * 2 # the per-lane part of the store byte offset + + @property + def n_thunks(self): + return self.n_a * (1 + self.slabs) + + def _write(self, c_frag, ti, tj, buf): + v = c_frag[self.c_idx_fn(ti, tj)] + rd = _acc_read + d0 = _cvt_pk_bf16(rd(v, 0), rd(v, 1)) + d1 = _cvt_pk_bf16(rd(v, 2), rd(v, 3)) + vec = Vec.from_elements([fx.Int32(d0), fx.Int32(d1)], fx.Int32) + ptr = _llvm.inttoptr( + _lds_ptr_t(), + fx.as_ir_value(self.w_off[tj] + fx.Int32(buf * self.buf_bytes)), + ) + _llvm.StoreOp(fx.as_ir_value(vec), ptr, alignment=8) + + def _read_store(self, base_row, base_col, ti, h, buf): + ptr = _llvm.inttoptr( + _lds_ptr_t(), fx.as_ir_value(self.r_off[h] + fx.Int32(buf * self.buf_bytes)) + ) + vec4 = _ir.VectorType.get([4], fx.Int32.ir_type) + data = Vec(_llvm.LoadOp(vec4, ptr, alignment=16).result) + off = ( + (base_row + ti * 16 + h * self.rows_per_read) * self.c_cols + base_col + ) * 2 + self.st_off + buffer_ops.buffer_store( + data, self.rsrc, off, offset_is_bytes=True, cache_modifier=self.cpol, mask=(fx.thread_idx.x % 8 < 6) + ) + + def thunks(self, c_frag, base_row, base_col): + """n_a (1 + slabs) thunks per quadrant: the blocks through the two buffers -- W0 W1 + R0 W2 R1 ... (a wave's LDS ops run in order, so a buffer's re-write after its reads + needs no wait); quadrants sharing the buffers must be concatenated, not + interleaved.""" + order = [("w", 0)] + for b in range(1, self.n_a): + order += [("w", b), ("r", b - 1)] + order.append(("r", self.n_a - 1)) + ts = [] + for kind, ti in order: + if kind == "w": + ts.append( + lambda ti=ti: [ + self._write(c_frag, ti, tj, ti % 2) for tj in range(self.n_b) + ] + ) + else: + for h in range(self.slabs): + ts.append( + lambda ti=ti, h=h: self._read_store( + base_row, base_col, ti, h, ti % 2 + ) + ) + return ts + + def store(self, c_frag, base_row, base_col): + for t in self.thunks(c_frag, base_row, base_col): + t() + +def compile_mxfp8_gemm_n6144_k4096_256x192_one_tile(): + """Compile the row-major-code 256x192 one-tile kernel at K 4096 (at most 256 tiles).""" + K, MN, store_nt, tiles = 4096, None, True, (4, 3) + grid_max, use_xcd_remap = 256, True + N_TILES_A, N_TILES_B = tiles + assert N_TILES_A == 4 and N_TILES_B == 3 + BLOCK_M = 64 * N_TILES_A + BLOCK_N = 64 * N_TILES_B + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + SUB_M = PARENT_M // BLOCK_M # sub-tiles per parent tile + SUB_N = PARENT_N // BLOCK_N + SUB_SPLIT = SUB_M * SUB_N + assert K % 256 == 0 + K_ITERS = K // BLOCK_K # even + KB = K // 256 # scale blocks + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + assert ( + LDS_BLOCK_M * BLOCK_K <= _LDS_BUF_BYTES + and LDS_BLOCK_N * BLOCK_K <= _LDS_BUF_BYTES + ) + # grid_max: workgroups of the launch (256 = one per CU, persistent); a larger value gives + # every work item its own workgroup (a plain grid), which no co-resident kernel holds back + GRID_MAX = grid_max + name = "flydsl_mxfp8_256x192_n6144_k4096_nt_one_tile" + SCALE_SLOTS = _SCALE_SLOTS + OUT_LDS_BYTES = _N_WAVES * 2 * 16 * 32 * 4 + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, 8 * _LDS_BUF_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_SLOTS * _SCALE_SLOT_BYTES, 16] + out_lds: fx.Array[fx.Int8, OUT_LDS_BYTES, 16] + + @flyc.kernel(name=name) + def kernel_gemm( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + _base_ptr = lds.all_lds.ptr + + def _buf(idx): + return _Buf(_base_ptr, idx * _LDS_BUF_BYTES) + + a_cur0, a_cur1, a_next0, a_next1 = _buf(0), _buf(1), _buf(2), _buf(3) + b_cur0, b_cur1, b_next0, b_next1 = _buf(4), _buf(5), _buf(6), _buf(7) + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + if const_expr(MN is not None): + c_m, c_n = const_expr(MN[0]), const_expr(MN[1]) + n_pid_m = _ceildiv(c_m, PARENT_M) # the parent tiles' grid + n_blocks = _ceildiv(c_n, PARENT_N) + grid = _min(fx.Int32(n_work), fx.Int32(GRID_MAX)) + wg = fx.Int32(fx.block_idx.x) + wave_i = wave_id // 2 + wave_j = wave_id % 2 + A_K_STEP = BLOCK_K + B_K_STEP = 2 * 1024 # preshuffled (16, 16): a 16-row block spans 2 KB per 128 K + + def tile_coords(t): + """work item -> (tile_i, tile_j) in units of this kernel's tile. The parent tiles + are indexed in the XCD order of ``_xcd_swizzle``; a sub-tile launch keeps a + workgroup on its XCD's parents: item ``t = 8 q + x`` is sub-tile ``q % SUB_SPLIT`` + of parent ``tile_off + 8 (q / SUB_SPLIT) + x`` (``tile_off`` a multiple of 8). A + negative / out-of-range index lands beyond M so its stores are dropped by the + buffer bounds.""" + if const_expr(use_xcd_remap and SUB_SPLIT > 1): + q, x = _divmod_nonneg(t, 8) + p_intra, sub = _divmod_nonneg(q, SUB_SPLIT) + parent = fx.Int32(tile_off) + p_intra * 8 + x + else: + parent, sub = _divmod_nonneg(t, SUB_SPLIT) + parent = fx.Int32(tile_off) + parent + if const_expr(use_xcd_remap): + pi, pj = _xcd_swizzle(n_pid_m, n_blocks, parent) + else: + pi, pj = parent // n_blocks, parent % n_blocks + si, sj = _divmod_nonneg(sub, SUB_N) + return pi * SUB_M + si, pj * SUB_N + sj + + def tile_bases(ti, tj): + """the g2s bases of a tile (row-major A, preshuffled W) and its scale groups""" + return ( + (ti * BLOCK_M) * K, + (ti * BLOCK_M + LDS_BLOCK_M) * K, + (tj * BLOCK_N) * K, + (tj * BLOCK_N + LDS_BLOCK_N) * K, + ) + + mfma = MfmaRowStationary(N_TILES_A, N_TILES_B) + _scale_base_ptr = lds.scale_lds.ptr + scale_gather = ScaleGatherLDS( + A_scale, + B_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + c_m, + c_n, + N_TILES_A, + N_TILES_B, + ) + scale_gather.set_wave_base() + # the two scale-slot LDS addresses (slot 0 / 1) precomputed once; the + # gather's K-block offset rides the uniform soffset inside the K loop + _gm0_s = ( + scale_gather._wave_base_s, + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + fx.Int32(_SCALE_SLOT_BYTES) + ), + ) + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(b): + return fx.Int32(b) % fx.Int32(SCALE_SLOTS) + + gl_off_a = _global_swizzle(lane_id, wave_id, K, N_TILES_A, False) + gl_off_b = _global_swizzle(lane_id, wave_id, K, N_TILES_B, True) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=c_m * K + ) + b_rsrc = buffer_ops.create_buffer_resource( + B_T, max_size=False, num_records_bytes=c_n * K + ) + a_g2s = G2SLoaderAsm(a_rsrc, gl_off_a, N_TILES_A, wave_id) + b_g2s = G2SLoaderAsm(b_rsrc, gl_off_b, N_TILES_B, wave_id) + a_g2s.set_wave_base(_base_ptr) + b_g2s.set_wave_base(_base_ptr) + a_s2r = S2RLoader(wave_i, N_TILES_A) + b_s2r = S2RLoader(wave_j, N_TILES_B) + store_c = StoreCLds( + C, + c_m, + c_n, + mfma.idx, + lds.out_lds.ptr, + wave_id, + N_TILES_A, + N_TILES_B, + nt=store_nt, + ) + + # a step issues [ac0 (n_a) | bc0 (n_b) | bc1 (n_b) | ac1 (n_a)] loads per wave: at a + # step's top the previous step's may all be in flight; mid-step the previous step's + # ac1 and this step's first two batches + _MAIN_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(block, holder): + # the caller passes the LDS slot itself (the block's K offset rides the + # uniform soffset, so no block -> slot math is left) + s = block + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _g2s_bases(bases, step): + a0, a1, b0, b1 = bases + st = fx.Int32(step) + return ( + fx.Int32(a0) + st * fx.Int32(A_K_STEP), + fx.Int32(a1) + st * fx.Int32(A_K_STEP), + fx.Int32(b0) + st * fx.Int32(B_K_STEP), + fx.Int32(b1) + st * fx.Int32(B_K_STEP), + ) + + def _one_step( + parity, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + load, + gather, + read_block, + zero_acc=False, + extra=None, + load_next=True, + read_next=True, + ): + """One K-step: this step's 64 MFMAs with, interleaved, the LDS reads for the next + step, the global loads ``load`` of the step after (the resolved (a0, a1, b0, + b1) soffsets), at even steps the gather ``gather`` of the next scale block (the + precomputed (m0, soff)), at odd steps the read of scale block ``read_block`` + (the LDS slot itself) into the VGPR carry; + ``extra(g, c00f, c01f)`` (called before group g = 0..3 with the accumulators + finished so far, None before) returns more thunks for that group: the stores + of this tile's c00 / c01 in the last step, of the previous tile's c10 / c11 + in the first.""" + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + a0_off, a1_off, b0_off, b1_off = load + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(read_block, _scn) if parity == 1 and read_next else [] + _sc_gather = ( + [] + if parity == 1 or not load_next + else [lambda: scale_gather.gather_soff(gather[0], gather[1])] + ) + ex = extra if extra is not None else (lambda g, c00, c01: ()) + + wait_barrier(_MAIN_VMCNT if load_next else 0) + il = _riffle( + (_g2s_thunks(a_g2s, ac0, a0_off, N_TILES_A) if load_next else []), + _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False), + ) + list(ex(0, None, None)) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + il = _riffle( + (_g2s_thunks(b_g2s, bc0, b0_off, N_TILES_B) if load_next else []), list(ex(1, c00f, None)) + ) + c01f = mfma.call( + a0f, b1f_in, c01f, saR0, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + a1f = _a1 + + if const_expr(load_next): + wait_barrier(_SEG2_VMCNT) + else: + # Every operand DMA completed at this tail step's first barrier. + # No future writer reuses these immutable LDS slots. + _rocdl.s_waitcnt(lgkmcnt=0) + il = (_riffle((_g2s_thunks(b_g2s, bc1, b1_off, N_TILES_B) if load_next else []), (_s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False) if read_next else [])) + + _sc_gather + _rd_scn[:2] + list(ex(2, c00f, c01f))) # fmt: skip + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + a0nf = _a0n if read_next else a0f + il = ( + _riffle( + (_g2s_thunks(a_g2s, ac1, a1_off, N_TILES_A) if load_next else []), + (_s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) if read_next else []) + + (_s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True) if read_next else []), + ) + + _rd_scn[2:] + + list(ex(3, c00f, c01f)) + ) + c11f = mfma.call( + a1f, b1f_in, c11f, saR1, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + b0nf, b1nf = (_b0n, _b1n) if read_next else (b0f, b1f_in) + + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) if parity == 1 and read_next else sc + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = N_TILES_A + n_b = N_TILES_B + n_ga = (N_TILES_A + 1) // _PACK + n_gb = (N_TILES_B + 1) // _PACK + n_sc = 2 * n_ga + 2 * n_gb + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC0, sbC1 = sc + return ( + [_R(v) for v in saR0] + + [_R(v) for v in saR1] + + [_R(v) for v in sbC0] + + [_R(v) for v in sbC1] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC0 = list(flat[o : o + n_gb]) + o += n_gb + sbC1 = list(flat[o : o + n_gb]) + return (saR0, saR1, sbC0, sbC1) + + # ---- prologue: the first tile's steps 0 / 1 and its scale blocks 0 / 1 ---- + t0 = wg + ti0, tj0 = tile_coords(t0) + bases0 = tile_bases(ti0, tj0) + G0 = scale_gather.tile_group(ti0 * BLOCK_M, tj0 * BLOCK_N) + scale_gather.set_tile_voff(G0) + scale_gather.gather_soff(_gm0_s[0], 0) + scale_gather.gather_soff(_gm0_s[1], 256) + s0 = _g2s_bases(bases0, 0) + s1 = _g2s_bases(bases0, 1) + a_g2s.load(a_cur0, s0[0]) + b_g2s.load(b_cur0, s0[2]) + b_g2s.load(b_cur1, s0[3]) + a_g2s.load(a_cur1, s0[1]) + a_g2s.load(a_next0, s1[0]) + b_g2s.load(b_next0, s1[2]) + b_g2s.load(b_next1, s1[3]) + a_g2s.load(a_next1, s1[1]) + # 2 gathers + 32 loads in flight: a_cur0 landed at vmcnt(28), b_cur0 / b_cur1 at 20 + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + zero4 = Vec.filled(4, 0.0, fx.Float32) + c10_prev = [zero4] * N_ACCUMS + c11_prev = [zero4] * N_ACCUMS + + # ---- the tile loop: state = fragments of the tile's step 0, its scale block 0, the + # previous tile's c10 / c11 (stored during this tile's first step), the tile index + tile_state = ( + [_R(x) for x in a0_frag] + [_R(x) for x in b0_frag] + [_R(x) for x in b1_frag] + _flat_sc(sc0) + + [_R(x) for x in c10_prev] + [_R(x) for x in c11_prev] + [_R(fx.Int32(t0)), _R(fx.Int32(-1))] + ) # fmt: skip + tstate = tile_state + off = 0 + a0f = list(tstate[off : off + n_a]) + off += n_a + b0f = list(tstate[off : off + n_b]) + off += n_b + b1f = list(tstate[off : off + n_b]) + off += n_b + sc = _unflat_sc(tstate[off : off + n_sc]) + off += n_sc + c10p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_cur = fx.Int32(tstate[off]) + t_prev = fx.Int32(tstate[off + 1]) + t_next = t_cur + grid + ti, tj = tile_coords(t_cur) + bases = tile_bases(ti, tj) + G = scale_gather.tile_group(ti * BLOCK_M, tj * BLOCK_N) + scale_gather.set_tile_voff(G) + # the next tile (the last tile loads a tile beyond the grid: rows beyond M read + # zero through the buffer bounds and nothing of it is stored) + tin, tjn = tile_coords(t_next) + bases_n = tile_bases(tin, tjn) + Gn = scale_gather.tile_group(tin * BLOCK_M, tjn * BLOCK_N) + # the previous tile's c10 / c11 store bases (row beyond M when there is none) + tip, tjp = tile_coords(t_prev) + has_prev = t_prev >= fx.Int32(0) + prev_row1 = fx.arith.select( + has_prev, + tip * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M, + fx.Int32(c_m), + ) + prev_col0 = fx.arith.select( + has_prev, tjp * BLOCK_N + wave_j * (N_TILES_B * 16), fx.Int32(0) + ) + prev_col1 = fx.arith.select(has_prev, prev_col0 + LDS_BLOCK_N, fx.Int32(0)) + base_row = ti * BLOCK_M + wave_i * (N_TILES_A * 16) + base_col = tj * BLOCK_N + wave_j * (N_TILES_B * 16) + + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # step 0: zero accumulators, the previous tile's c10 / c11 stores spread over the + # groups before the new c10 (born in group 2) / c11 (group 3), gather block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, _g2s_bases(bases, 2), (_gm0_s[1], 256), None, zero_acc=True, + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, _g2s_bases(bases, 3), None, 1 + ) + # the four g2s streams advance by uniform strides and the gather + # block by 256 B; carry them as uniform accumulators (incremented + # per step) instead of recomputing base | (step << s) every K-step + ka0 = fx.Int32(bases[0]) + fx.Int32(4 * A_K_STEP) + ka1 = fx.Int32(bases[1]) + fx.Int32(4 * A_K_STEP) + kb0 = fx.Int32(bases[2]) + fx.Int32(4 * B_K_STEP) + kb1 = fx.Int32(bases[3]) + fx.Int32(4 * B_K_STEP) + kg = fx.Int32(512) # gather block soff at kk=2: block 2 << 8 + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg)] + ) # fmt: skip + for kk, state in range(2, K_ITERS - 2, UNROLL, init=init_state): + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + bufs = bufs0 + off += N_ACCUMS # past c11f: the uniform address accumulators + ka0 = fx.Int32(state[off]) + ka1 = fx.Int32(state[off + 1]) + kb0 = fx.Int32(state[off + 2]) + kb1 = fx.Int32(state[off + 3]) + kg = fx.Int32(state[off + 4]) + for u in range_constexpr(UNROLL): + if const_expr(UNROLL == 4): + # kk == 2 (mod 4): u=0 gathers an even block (slot 0), u=2 an + # odd one (slot 1) -- gather m0 slots folded; read slots stay + # runtime (see below). + lds_off = (ka0, ka1, kb0, kb1) + gath = None + if const_expr(u % 2 == 0): + gath = ( + _gm0_s[u // 2], + kg if u == 0 else kg + fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + # keep the read slot a runtime value: folding it to a + # constant made LLVM un-pair the scale ds_read2st64 + # into 2 ds_read_b32 (+2 LDS/step at the ISA gate) + rslot = ( + fx.Int32(kk) // fx.Int32(2) + fx.Int32(1 + u // 2) + ) % fx.Int32(2) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + fx.Int32(1) + lds_off = _g2s_bases(bases, kc + fx.Int32(2)) + gath = None + if const_expr(u % 2 == 0): + gath = ( + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + + (block % fx.Int32(2)) + * fx.Int32(_SCALE_SLOT_BYTES) + ), + block * fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + rslot = block % fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + lds_off, + gath, + rslot, + ) + ka0 = ka0 + fx.Int32(A_K_STEP) + ka1 = ka1 + fx.Int32(A_K_STEP) + kb0 = kb0 + fx.Int32(B_K_STEP) + kb1 = kb1 + fx.Int32(B_K_STEP) + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg + fx.Int32(UNROLL * 128))] + ) # fmt: skip + state = yield new_state + + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + # step K_ITERS - 2: finish this tile; there is no next tile to prefetch. + scale_gather.set_tile_voff(Gn) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, + a0f, + b0f, + b1f, + sc, + accs, + bufs0, + _g2s_bases(bases_n, 0), + (_gm0_s[0], 0), + None, load_next=False, + ) + # step K_ITERS - 1 (odd): loads of the next tile's step 1, read of its block 0; + # this tile's c00 / c01 stored over the groups after their last MFMAs (c00: 1..3, + # c01: 2..3; the two quadrants' thunks stay concatenated, they share the LDS buffer) + n_q3 = ( + 2 * store_c.n_thunks + 2 + ) // 3 # thunks per group: two quadrants over 3 groups + + def _last_extra(g, c00, c01): + # (compile-time python only: no mutable state -- the DSL rewriter turns + # body-level literals into None -- and every `if` under const_expr) + if const_expr(g == 0): + return () + t00 = store_c.thunks(c00, base_row, base_col) + if const_expr(g == 1): + return t00[:n_q3] + t01 = store_c.thunks(c01, base_row, base_col + LDS_BLOCK_N) + if const_expr(g == 2): + return t00[n_q3:] + t01[: 2 * n_q3 - len(t00)] + return t01[2 * n_q3 - len(t00) :] + + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + _g2s_bases(bases_n, 1), + None, + 0, + extra=_last_extra, load_next=False, read_next=False, + ) + new_tile_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + [_R(t_next), _R(t_cur)] + ) # fmt: skip + tstate = new_tile_state + + # ---- epilogue: the last tile's c10 / c11 ---- + off = n_a + 2 * n_b + n_sc + c10_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_last = fx.Int32(tstate[off + 1]) + til, tjl = tile_coords(t_last) + last_row1 = til * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M + last_col0 = tjl * BLOCK_N + wave_j * (N_TILES_B * 16) + store_c.store(c10_last, last_row1, last_col0) + store_c.store(c11_last, last_row1, last_col0 + LDS_BLOCK_N) + + @flyc.jit + def launch_one_tile_gemm_256x192( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + stream: fx.Stream, + ): + """work items [tile_off, tile_off + n_work / SUB_SPLIT) of the parent grid + ceil(M / 256) x ceil(N / 256); the whole GEMM is (0, parents * SUB_SPLIT)""" + grid_x = _min(n_work, fx.Int32(GRID_MAX)) + kernel_gemm( + A, B_T, C, A_scale, B_scale, c_m, c_n, tile_off, n_work, + value_attrs={"rocdl.waves_per_eu": 1, "rocdl.flat_work_group_size": "256,256"}, + ).launch(grid=(grid_x, 1, 1), block=(256, 1, 1), stream=stream) # fmt: skip + + launch_one_tile_gemm_256x192.kernel_name = name + launch_one_tile_gemm_256x192.sub_split = SUB_SPLIT + return launch_one_tile_gemm_256x192 diff --git a/amd_kernels/gemm/mxfp8_n6144_k4096/tile_128x192.py b/amd_kernels/gemm/mxfp8_n6144_k4096/tile_128x192.py new file mode 100644 index 0000000..6239c9d --- /dev/null +++ b/amd_kernels/gemm/mxfp8_n6144_k4096/tile_128x192.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""N6144 K4096 MXFP8 GEMM, 128x192 tiles with a terminal K pipeline. + +Adapted from gemm/mxfp8_n6144_k2048/tile_128x192.py (the same tile at K 2048). Four +waves keep the three-A / four-W operand ring and increasing-K accumulation order; the +ring slots repeat every 12 K steps, so 24 looped and 4 peeled steady steps precede the +terminal four (on ring phase 4..7). The final four K128 steps omit wraparound A/W prefetches, +unused scale gathers, and the final fragment reads. The first terminal step +retains the steady wait; the second permits NA+NB pending requests, and the +last two drain VMEM. Every step retains its all-wave barrier. Output remains +the same coalesced BF16 transpose with streaming stores. The caller provides +row-major activation codes and preshuffled scales. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr, rocdl +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_4wave import ( + G2SLoaderAsm, + MfmaRowStationary, + S2RLoader, + _asm_void, + _Buf, + _global_swizzle, + _s2r_thunks, + _uniform_i32, +) + +BLOCK_M, BLOCK_N = 128, 192 +K_UNROLL = 12 + + +def compile_mxfp8_gemm_n6144_k4096_128x192(): + K = 4096 + BM, BN = BLOCK_M, BLOCK_N + KSTEPS = K // 128 + NA, NB = BM // 32, BN // 32 # MFMA tiles per wave; also DMA rounds per panel + AB, BB = BM * 128, BN * 128 + # gfx950 buffer_store bits: 0x2 = nt. The output is written once; NT stores keep the + # operands in L2 / MALL (QKV -2.9..-3.1%, o_proj K2048 -6.8..-7.1%, K8192 -1.7..-2.1%). + CPOL = 2 + name = "flydsl_mxfp8_128x192_n6144_k4096_tail_pipe3a4w_xor_sb_nt" + + @fx.struct + class Shared: + data: fx.Array[fx.Int8, 3 * AB + 4 * BB, 16] + scales: fx.Array[fx.Int8, 8192, 16] + + @flyc.kernel(name=name, known_block_size=(256, 1, 1)) + def kernel( + A: fx.Tensor, B: fx.Tensor, C: fx.Tensor, AS: fx.Tensor, BS: fx.Tensor, + m: fx.Int32, n: fx.Int32, + ): + tid = fx.Int32(fx.gpu.thread_id("x")) + wid = fx.Int32(_uniform_i32(tid // 64)) + lane = tid % 64 + wi, wj = wid // 2, wid % 2 + wg = fx.Int32(fx.gpu.block_id("x")) + nm, nn = (m + BM - 1) // BM, n // BN + # Workgroup w runs on XCD w%8. Deal contiguous column strips to it, + # so its CUs share W through L2 rather than reading every W column. + total = nm * nn + x, q = wg % 8, wg // 8 + base, extra = total // 8, total % 8 + idx = x * base + (x < extra).select(x, extra) + q + pi, pj = idx % nm, idx // nm + ar, br = pi * BM, pj * BN + lds = fx.SharedAllocator().allocate(Shared).peek() + a0, a1 = _Buf(lds.data.ptr, 0), _Buf(lds.data.ptr, AB) + b0, b1 = _Buf(lds.data.ptr, 3 * AB), _Buf(lds.data.ptr, 3 * AB + BB) + ag = G2SLoaderAsm( + buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * K), + _global_swizzle(lane, wid, K, NA, False), NA, wid, + ) + bg = G2SLoaderAsm( + buffer_ops.create_buffer_resource(B, max_size=False, num_records_bytes=n * K), + _global_swizzle(lane, wid, K, NB, True), NB, wid, + ) + ag.set_wave_base(lds.data.ptr) + bg.set_wave_base(lds.data.ptr) + al, bl = S2RLoader(wi, NA), S2RLoader(wj, NB) + sar = buffer_ops.create_buffer_resource( + AS, max_size=False, num_records_bytes=((m + 31) // 32 * 32) * (K // 32), + ) + sbr = buffer_ops.create_buffer_resource(BS, max_size=False, num_records_bytes=n * (K // 32)) + sp = fx.recast_iter(fx.Int32, lds.scales.ptr) + is_a = wid == 0 + sr = fx.arith.select(is_a, sar, sbr) + grp = is_a.select(ar // 32, br // 32 + (wid - 1) * 4) + lane // 16 + svoff = grp * (K // 256) * 256 + (lane % 16) * 16 + swb = _uniform_i32(fx.Int32(fx.ptrtoint(lds.scales.ptr)) + wid * 1024) + + def gather(blk, slot): + # Async G2S asm avoids LLVM alias waits before unrelated operand LDS reads. + _asm_void( + [fx.as_ir_value(fx.Int32(swb) + slot * 4096), fx.as_ir_value(svoff), + fx.as_ir_value(sr), _uniform_i32(blk * 256)], + "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds", + "s,v,s,s", "~{m0}", + ) + + def scales(kt): + sa, sb = [], [] + slot = (kt // 2) % 2 + sl = (lane // 16) * 16 + lane % 16 + for i in range_constexpr(NA): + r = wi * (NA * 16) + i * 16 + off = slot * 1024 + (r // 32) * 64 + sl + sa.append(fx.Int32(sp[off]).shrui((r % 32) // 16 * 8)) + for j in range_constexpr(NB): + r = wj * (NB * 16) + j * 16 + off = slot * 1024 + 256 + (r // 32) * 64 + sl + sb.append(fx.Int32(sp[off]).shrui((r % 32) // 16 * 8)) + return sa, sb + + mf = MfmaRowStationary(NA, NB) + ag.load(a0, ar * K) + bg.load(b0, br * K) + ag.load(a1, ar * K + 128) + bg.load(b1, br * K + 2048) + bg.load(_Buf(lds.data.ptr, 3 * AB + 2 * BB), br * K + 4096) + gather(fx.Int32(0), fx.Int32(0)) + gather(fx.Int32(1), fx.Int32(1)) + rocdl.s_waitcnt(vmcnt=0, lgkmcnt=0) + rocdl.s_barrier() + zero = Vec.filled(4, 0.0, fx.Float32) + init = ( + [fx.as_ir_value(zero)] * (NA * NB) + + [fx.as_ir_value(v) for v in al.load(a0)] + + [fx.as_ir_value(v) for v in bl.load(b0, True)] + ) + rocdl.s_waitcnt(lgkmcnt=0) + rocdl.s_barrier() + def one_step(it, u, acc, av, bv, final=False, t=0): + kt = fx.Int32(it) + u + fut, nr = fx.Int32((u + 3) % 4), fx.Int32((u + 1) % 4) + an = _Buf(lds.data.ptr, fx.Int32((u + 2) % 3) * AB) + bn = _Buf(lds.data.ptr, 3 * AB + fut * BB) + na_buf = _Buf(fx.add_offset(lds.data.ptr, fx.Int32((u + 1) % 3) * AB), 0) + nb_buf = _Buf(fx.add_offset(lds.data.ptr, 3 * AB + nr * BB), 0) + sa, sb = scales(kt) + nxt = (kt + 3) % (K // 128) + if const_expr(not final or t < 2): + ag.load(an, ar * K + ((kt + 2) % (K // 128)) * 128) + if const_expr(not final or t == 0): + bg.load(bn, br * K + nxt * 2048) + # One future A panel and two W panels (NA+2*NB DMAs per wave) may stay + # in flight, and after an odd step also its post-barrier scale gather. + # The following panel is complete on every wave before LDS reads; at an + # odd step the gather issued three steps ago (the next packet) is too. + # Terminal t=1 issues only the last A: the last W + A may remain. From t=2, + # no future DMA remains, so drain before the next fragment reads. + wait_loads = ((NA + 2 * NB + 1) if t == 0 else (NA + NB if t == 1 else 0)) if final else (NA + 2 * NB + (1 if u % 2 == 0 else 0)) + rocdl.s_waitcnt(vmcnt=wait_loads, lgkmcnt=0) + rocdl.s_barrier() + if const_expr(u % 2 == 1 and not final): + # Waves 0-2 gather the packets every wave reads: overwrite the slot of + # this step's packet only after the barrier, when every wave's reads of + # it (lgkmcnt(0) above) are complete. + gather((kt // 2 + 2) % (K // 256), (kt // 2 + 2) % 2) + anext, bnext = [None] * NA, [None] * NB + rd = (_s2r_thunks(al, na_buf, anext, NA, False) + _s2r_thunks(bl, nb_buf, bnext, NB, True)) if not final or t < 3 else [] + for z, (i, j) in enumerate(mf._order()): + if const_expr(z < len(rd)): + rd[z]() + # Inline-asm MFMA pins the accumulator in AGPRs. + acc[i * NB + j] = mf._mfma(av[i], bv[j], acc[i * NB + j], sa[i], sb[j], u % 2, 0, 0) + av, bv = (anext, bnext) if not final or t < 3 else (av, bv) + return acc, av, bv + + # The ring slots repeat every 12 steps: 24 looped steps, 4 peeled steady steps, then the + # final four on phase 4..7 (t = terminal index). + for it, state in range(fx.Index(0), fx.Index(KSTEPS - 8), fx.Index(K_UNROLL), init=init): + acc = list(state[:NA * NB]) + av = list(state[NA * NB:NA * NB + NA]) + bv = list(state[NA * NB + NA:]) + for u in range_constexpr(K_UNROLL): + acc, av, bv = one_step(it, u, acc, av, bv) + state = yield acc + [fx.as_ir_value(v) for v in av] + [fx.as_ir_value(v) for v in bv] + acc = list(state[:NA * NB]) + av = list(state[NA * NB:NA * NB + NA]) + bv = list(state[NA * NB + NA:]) + for u in range_constexpr(4): + acc, av, bv = one_step(fx.Int32(KSTEPS - 8), u, acc, av, bv) + for u in range_constexpr(4, 8): + acc, av, bv = one_step(fx.Int32(KSTEPS - 8), u, acc, av, bv, final=True, t=u - 4) + results = acc + [fx.as_ir_value(v) for v in av] + [fx.as_ir_value(v) for v in bv] + + + # All DMAs and fragment reads finish before the operand ring becomes output LDS. + rocdl.s_waitcnt(vmcnt=0, lgkmcnt=0) + rocdl.s_barrier() + p16 = fx.recast_iter(fx.Int16, lds.data.ptr) + for i in range_constexpr(NA): + for j in range_constexpr(NB): + a = Vec(results[i * NB + j]).to(fx.BFloat16).bitcast(fx.Int16) + for z in range_constexpr(4): + # MFMA is W*A^T: lane%16 is row, 4*(lane//16)+z is column. + rr = wi * (NA * 16) + i * 16 + lane % 16 + cc = wj * (NB * 16) + j * 16 + 4 * (lane // 16) + z + p16[rr * BN + (cc ^ ((rr % 8) * 8))] = a[z] + rocdl.s_waitcnt(lgkmcnt=0) + rocdl.s_barrier() + out = buffer_ops.create_buffer_resource(C, max_size=False, num_records_bytes=m * n * 2) + p32 = fx.recast_iter(fx.Int32, lds.data.ptr) + for z in range_constexpr(BM * BN // 2048): + lin = (tid + z * 256) * 8 + rr, cc = lin // BN, lin % BN + off = ((ar + rr) * n + br + cc) // 2 + read = (rr * BN + (cc ^ ((rr % 8) * 8))) // 2 + v = Vec.from_elements([p32[read + i] for i in range(4)], fx.Int32) + buffer_ops.buffer_store(v, out, off, cache_modifier=CPOL) + + @flyc.jit + def launch_gemm_n6144_k4096_128x192( + A: fx.Tensor, B: fx.Tensor, C: fx.Tensor, AS: fx.Tensor, BS: fx.Tensor, + m: fx.Int32, n: fx.Int32, stream: fx.Stream, + ): + kernel(A, B, C, AS, BS, m, n, value_attrs={"rocdl.waves_per_eu": 1}).launch( + grid=(((m + BM - 1) // BM) * (n // BN), 1, 1), block=(256, 1, 1), stream=stream, + ) + + launch_gemm_n6144_k4096_128x192.kernel_name = name + return launch_gemm_n6144_k4096_128x192 diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/README.md b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/README.md index 6190964..e9ad350 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/README.md +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/README.md @@ -5,37 +5,32 @@ The MiniMax-M3.1 sparse-attention indexer: per (index head, query row, 128-token row's visible tokens, then each row's top-16 blocks (score desc, ties to the larger id, -1 padded) as ids or with the attend's page-16 table and context bound. Query e4m3 `[rows, heads, 128]`; NVFP4 index pages `[blocks,128,72]` uint8 (per token 64 B e2m1 codes, 8 B e4m3 scales) scored as `e4m3_rn_satfinite(e2m1 x scale)` on fp8 MFMA; sentinels 1e30 -(init) / 1e29 (local blocks); <= 8192 blocks (1M tokens) per row. One GPU at 1 / 2 / 4 heads, or four TP ranks at -decode (a head each) on a replicated or rank-owned cache (rank r holds blocks b % 4 == r, page p at slot p // 4). +(init) / 1e29 (local blocks); <= 8192 blocks (1M tokens) per row. One GPU at 1 / 2 / 4 index-q heads (TP4 1, TP2 2), +or at decode four TP4 ranks (a head each; replicated or rank-owned cache: rank r blocks b % 4 == r, slot p // 4) or two +TP2 ranks (all four heads over blocks b % 2 == r, partial top-16 exchanged and merged; replicated cache). Callers: the model's sparse attention layers, `attention/nvfp4_sparse_decode_gqa16_d128_p128_top16`. Contract (`reference.py`): scores within 2e-4 of `max(|ref|, 1)` of fp64; sentinels, ids, tables, bounds exact (AITER's `pa_sparse_block_topk`); routes bitwise equal. ## Entry points -- `decode_top16.IndexDecodeTop16`: a step's decode top-16 ids into `[heads, rows, 16]` int32; built with the four - ranks' gloo group it is a collective and takes the four-GPU route by request count, else one GPU. A call can stay - on one GPU (DBO micro-batches). +- `decode_top16.IndexDecodeTop16`: a step's decode top-16 ids `[heads, rows, 16]` int32; built with the four (TP4) or + two (TP2) ranks' gloo group a collective that takes the multi-GPU route by request count (one GPU per call: DBO). - `prefill_top16.index_prefill_top16`: the top-16 ids of every ragged prefill row in one call (one-GPU scorer, then the - ids-only top-k); `index_prefill_top16_blocker`: why a model's indexer does not fit (None: it fits); `warm_up`: one - small call that compiles every prefill variant. + ids-only top-k); `index_prefill_top16_blocker`: why a model's indexer does not fit; `warm_up`: compiles it all. - `score_decode.nvfp4_index_score_top16`: one-GPU decode scores + top-16, two launches; `nvfp4_index_score_rows`: the scores alone, for a consumer that selects in its prologue (`topk.top16_indices_p128_plain_wave4`); - `nvfp4_index_top16_from_scores`: the selection alone; `nvfp4_index_score_decode`: scores at any head and row count - (heads x rows <= 32). + `nvfp4_index_top16_from_scores`: the selection alone; `nvfp4_index_score_decode`: any heads x rows <= 32. - `score_prefill.nvfp4_index_score_prefill`: fp32 scores of ragged prefill rows (`host.nvfp4_e4m3_lut`: its key table); `supports_nvfp4_index_score_prefill`: its shape gate. -- `topk.index_topk_decode_indices`, `topk.index_topk_prefill_indices`: top-16 ids of score rows; - `topk.index_topk_prefill`: ids + page table + context bounds, AITER's `pa_sparse_block_topk` outputs; - `topk.supports_index_topk`: their shape gate. -- `split4.host.init_rank_owned`: the four-GPU buffers, one set per micro-batch, a collective before graph capture; - `split4.host.get_rank_owned`: a micro-batch's set, whose `score_topk` gives the four-GPU scores + top-16 and - `score_push` the scores alone in rank-owned order. +- `topk.index_topk_decode_indices`, `topk.index_topk_prefill_indices`: top-16 ids of score rows; `index_topk_prefill`: + ids + page table + context bounds (AITER's `pa_sparse_block_topk` outputs); `supports_index_topk`: the shape gate. +- `split4.host.init_rank_owned` / `get_rank_owned`: the four-GPU buffers per micro-batch (`score_topk`, `score_push`: + the scores alone in rank-owned order); `split2.host.IndexSplit2`: the two-GPU buffers and `score_topk`. -Decode calls read lengths on the device (graph-safe at max_model_len). `nvfp4_index_score_top16`, `_rows`, -`score_topk` and `index_prefill_top16` are `amd_kernels.warm_up` entries (the prefill entry warms one request per -top-k window tile); the scorers and top-k called alone are not. Dispatch: `decode_top16.py` picks the route, -`score_decode/host.py` the one-head scorer, `split4/host.py` the four-GPU kernels by rows, `score_prefill/host.py` -the tile by work, `topk/host.py` the window tile by `max_seq_len`. +Decode calls read lengths on the device (graph-safe at max_model_len). `nvfp4_index_score_top16`, `_rows`, both +`score_topk` and `index_prefill_top16` are `amd_kernels.warm_up` entries. Dispatch: `decode_top16.py` the route, +`score_decode/host.py` the scorer and selector by heads and requests, `split4/host.py` / `split2/host.py` the multi-GPU +kernels, `score_prefill/host.py` the tile by heads and work, `topk/host.py` the window tile by `max_seq_len`. ## Performance Graph replay, host 1M, cold pages, us per call, max over ranks; AgentX = request i of N at the (i + 0.5) / N quantile @@ -46,15 +41,19 @@ of AgentX c16 + c32 inputs; AgentX decode = the decode-time contexts (Tests) of | `nvfp4_index_score_top16` | 8 x 1 / 8 x 4 / 4 x 1 | 32K, 192K | 5.83 / 7.43 / 5.72, 7.87 / 14.75 / 7.78 | `nvfp4_index_score_decode` + `index_topk_decode_indices` 8.23 / 9.38 / 8.06, 10.49 / 20.16 / 10.25 | | `index_prefill_top16`, 1 head, four GPUs at once | M 4096 / 8192 / 16384 x 1 | prefix 0 / 8K / 32K / 128K / 256K | 11.2 / 17.6 / 35.5 / 74.6 / 126.7, 22.0 / 29.3 / 50.2 / 123.2 / 226.5, 44.7 / 57.7 / 88.8 / 241.3 / 435.3 | vLLM's in-tree scorer + top-k: 11.1 / 17.7 / 42.9 / 91.3 / 146.2, 21.8 / 29.4 / 58.9 / 140.6 / 244.4, 44.7 / 61.3 / 98.2 / 261.1 / 453.8 | | `IndexDecodeTop16` four GPUs | 4 x 6 / 12 / 24 / 32, 8 x 5 / 12 / 16 | AgentX decode | 13.57 / 20.67 / 26.70 / 28.50, 16.20 / 23.06 / 23.60 | a four-rank flag wait, then the selector: 14.99 / 21.54 / 27.59 / 28.92, 17.53 / 23.79 / 23.99 | +| `nvfp4_index_score_top16`, 2 heads (TP2) | 8 x 1 / 4 / 16 / 32 / 48 / 64, 4 x 32 / 64 / 128 | 32K-192K log-uniform | 6.75 / 10.52 / 23.25 / 40.98 / 58.58 / 78.65, 39.97 / 74.46 / 141.49 | `nvfp4_index_score_decode` + `index_topk_decode_indices`: 8.44 / 14.38 / 32.23 / 61.50 / 95.21 / 120.05, 49.96 / 104.43 / 213.64 | +| `IndexDecodeTop16` two GPUs (TP2) + the decode attention (the decode block) | 8 x 8 / 12 / 16 / 24 / 32 / 48 | 128K-224K | 34.54 / 42.18 / 48.00 / 66.23 / 80.65 / 111.40 | one GPU (scorer + the attention's top-16): 36.34 / 47.99 / 57.52 / 82.16 / 105.16 / 147.74 | +| `index_prefill_top16`, 2 heads (TP2), scorer + top-k | M 4096 / 8192 / 16384 x 1 | prefix 0 / 56K / 112K | 14.3 / 110.6 / 386.0 | the scorer plan at the one-head thresholds: 15.7 / 109.7 / 386.4 | ## Tests -A decode-time contexts file (`--lens-json` of the block benchmark) holds, per request count N, the (i + 0.5) / N -quantiles of the contexts of the requests decoding at sampled times of a workload's client records. +A decode-time contexts file (`--lens-json`): per request count N, the (i + 0.5) / N quantiles of decoding contexts. ```bash -HIP_VISIBLE_DEVICES=0,1,2,3 python3 -m pytest tests/indexer/indexer_nvfp4_d128_p128_top16 -q # four-GPU tests skip on fewer cards -python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py [--prefill] +HIP_VISIBLE_DEVICES=0,1,2,3 python3 -m pytest tests/indexer/indexer_nvfp4_d128_p128_top16 -q # *_ranks4 / *_ranks2 skip on fewer cards +python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py [--prefill] [--heads 2] +python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py --ref base= --ql 8 # TP2 python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_rank_owned.py --ql 4 --decode '' --lens-json --reqs 8,32,64 # 4 GPUs HIP_VISIBLE_DEVICES=0,1,2,3 python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks4.py --ql 4 --lens-json --reqs 4,6,8,12 # the decode block on each route (SPLIT_MIN_REQS) +HIP_VISIBLE_DEVICES=0,1 python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py --ql 8 --lens-json --reqs 8,10,12,16 # TP2 (SPLIT2_MIN_REQS) python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_decode_topk.py --selector rank_owned --ref today= --lens --reqs 32,48 --ql 4 ``` diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/decode_top16.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/decode_top16.py index 04e374c..db76723 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/decode_top16.py +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/decode_top16.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """The decode top-16 of a step in one call: the four-GPU scorer on the rank-owned or replicated index cache -(``split4.host.IndexSplit4RankOwned.score_topk``) or the one-GPU scorer + top-16 -(``score_decode.nvfp4_index_score_top16``), chosen by request count; the ids are the same on both routes -(``reference.topk_reference``). +(``split4.host.IndexSplit4RankOwned.score_topk``, TP4), the two-GPU one on the replicated cache +(``split2.host.IndexSplit2.score_topk``, TP2) or the one-GPU scorer + top-16 (``score_decode.nvfp4_index_score_top16``), +chosen by request count; the ids are the same on every route (``reference.topk_reference``). """ import torch @@ -16,6 +16,8 @@ # 24.30 / 27.92 vs four 25.46 / 26.64; 4 rows x 7 / 8 23.45 / 28.36 vs 24.59 / 24.75 SPLIT_MIN_REQS = {8: 5, 4: 8} WARM_CONTEXT = 65536 # context of the four-GPU route's warm-up calls +# the two-GPU route from this many requests, by rows per request (the decode block, benchmark_index_decode_ranks2.py) +SPLIT2_MIN_REQS = {8: 8, 4: 6} def split_min_reqs(query_len: int) -> int: @@ -40,13 +42,26 @@ def use_split( ) +def use_split2(num_reqs: int, query_len: int, num_index_heads: int, *, split_query_len: int, + split_max_rows: int) -> bool: + """Whether a step takes the two-GPU route built for ``split_query_len`` rows per request and ``split_max_rows`` + rows.""" + return ( + num_index_heads == 2 + and query_len == split_query_len + and num_reqs * query_len <= split_max_rows + and num_reqs >= SPLIT2_MIN_REQS.get(query_len, 1) + ) + + class IndexDecodeTop16: """The decode top-16 of every (index head, row) on this GPU; built once per process, at the same point on every rank when ``group`` is given (a collective: IPC buffers, then the warm-up's launches). Args: - group: the four ranks' gloo group for the four-GPU route, or None (one GPU only). The route needs four - ranks and 1-``decode_score.MAX_QL`` rows per request; otherwise ``split_off`` says why it is off. + group: the ranks' gloo group for the multi-GPU route, or None (one GPU only). Four ranks: 1 index head and + 1-``decode_score.MAX_QL`` rows per request; two ranks: 2 index heads and 4 or 8 rows per request + (``split2``); otherwise ``split_off`` says why it is off. sharded: the index cache holds only this rank's pages (False: the replicated cache). warm_up: compile every variant now. """ @@ -68,6 +83,7 @@ def __init__( self.query_len, self.max_reqs, self.max_seq_len = query_len, max_reqs, max_seq_len self.num_index_heads, self.sharded = num_index_heads, sharded self.split = None + self.split2 = None self.split_off = None if group is not None else "no process group" if group is not None: import torch.distributed as dist @@ -76,8 +92,19 @@ def __init__( from .split4.host import WORLD, IndexSplit4RankOwned world = dist.get_world_size(group) - if world != WORLD: - self.split_off = f"needs {WORLD} ranks ({world})" + if world == 2: + from .split2 import kernel as split2_kernel + + reqs = min(split2_kernel.MAX_REQS, max_reqs) + if num_index_heads != 2 or query_len not in (4, 8): + self.split_off = f"two ranks need 2 index heads x 4 or 8 rows ({num_index_heads} x {query_len})" + else: + from .split2.host import IndexSplit2 + + self.split2 = IndexSplit2(rank, group, self.device, reqs * query_len, score_width(max_seq_len), + query_len) + elif world != WORLD: + self.split_off = f"needs 2 or {WORLD} ranks ({world})" elif not 1 <= query_len <= decode_score.MAX_QL: self.split_off = f"{query_len} decode rows per request (1-{decode_score.MAX_QL})" else: @@ -89,8 +116,16 @@ def __init__( self.warm_up() def route(self, num_reqs: int, query_len: int, num_index_heads: int, split: bool = True) -> str: - """``"split4"`` or ``"one_gpu"``; ``split`` False keeps one GPU (DBO micro-batches, where no collective - orders the four-GPU push).""" + """``"split4"``, ``"split2"`` or ``"one_gpu"``; ``split`` False keeps one GPU (DBO micro-batches, where no + collective orders the multi-GPU exchange).""" + cp2 = self.split2 + if ( + split + and cp2 is not None + and use_split2(num_reqs, query_len, num_index_heads, split_query_len=cp2.query_len, + split_max_rows=cp2.max_rows) + ): + return "split2" cp = self.split if ( split @@ -119,7 +154,12 @@ def __call__( ) -> None: """The top-16 block ids of every decode row; graph-safe (lengths are read on the device).""" heads = iq.shape[1] - if self.route(seq_lens.shape[0], query_len, heads, split) == "split4": + route = self.route(seq_lens.shape[0], query_len, heads, split) + if route == "split2": + self.split2.score_topk(iq, index_cache, block_table, seq_lens, init_blocks, local_blocks, topk_idx, + max_seq_len) + return + if route == "split4": self.split.score_topk( iq[:, 0], index_cache, @@ -168,6 +208,18 @@ def warm_up(self) -> None: score = torch.empty((heads, rows, score_width(ctx)), dtype=torch.float32, device=dev) q_idx, cache, table, seq, init, local = args nvfp4_index_score_rows(q_idx, cache, score, table, seq, init, local, query_len=ql, max_seq_len=ctx) + if self.split2 is not None: + # a real call on both ranks (the entry's first call compiles every size first) + self.split2.score_topk( + torch.zeros((ql, heads, 128), dtype=torch.float8_e4m3fn, device=dev), + ix, + torch.zeros((1, -(-WARM_CONTEXT // BLK)), dtype=torch.int32, device=dev), + torch.full((1,), WARM_CONTEXT, dtype=torch.int32, device=dev), + 0, + 1, + torch.empty((heads, ql, 16), dtype=torch.int32, device=dev), + self.max_seq_len, + ) if self.split is None or heads != 1: return from .split4 import decode_score diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/prefill_top16.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/prefill_top16.py index 589afcf..f39c195 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/prefill_top16.py +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/prefill_top16.py @@ -23,6 +23,8 @@ # scorer's 128-row tiles), at MAX_SEQ_LEN the 512-row tiles. WARM_ROWS = 512 WARM_CONTEXTS = tuple(s * 64 * BLK for s in PREFILL_STRIPS) + (MAX_SEQ_LEN,) +# two heads: also 64 rows (the 128-row tiles; 512 rows take the 256-row tiles there) +WARM_TINY = 64 def index_prefill_top16_blocker( @@ -90,7 +92,7 @@ def _mode(iq, *args, **kwargs): def _sizes(mode): - return WARM_CONTEXTS + return ((WARM_TINY,) if mode[0] == 2 else ()) + WARM_CONTEXTS def _call(heads: int, n: int, device): @@ -122,7 +124,7 @@ def _inputs(mode, n, iq, *args, **kwargs): def warm_up(device, num_index_heads: int) -> None: """Before serving: one small call, whose first-call warm-up compiles every variant of ``num_index_heads`` heads - on ``device`` (``WARM_CONTEXTS``); nothing at a head count the entry does not serve.""" + on ``device`` (``_sizes``); nothing at a head count the entry does not serve.""" if num_index_heads not in INDEX_HEADS: return with torch.inference_mode(): diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/columns.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/columns.py new file mode 100644 index 0000000..ee55fb5 --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/columns.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MFMA column layout of the two-head decode scorers: column c holds query row c // 2 of index head c % 2 (Q contiguous +``[rows, 2, 128]``: column c at c * 128 B from the request's first row), scores ``[2, reqs * QL, S]`` contiguous over +(head, row).""" + + +def columns(ql: int): + """(columns per request, query row of a column, score index of (column, request, requests, row stride, block)).""" + return ( + 2 * ql, + lambda c: c >> 1, + lambda c, req, reqs, stride, b: (c & 1) * (reqs * ql * stride) + (req * ql + (c >> 1)) * stride + b, + ) diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/host.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/host.py index c1d2a2b..46c39d5 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/host.py +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/host.py @@ -153,37 +153,102 @@ def nvfp4_index_score_decode( ) +# Tests / benchmarks: pin the scorer ("page", "first_page", "first_page_xcd", "sector", "scan"; two heads: "page", +# "first_page_xcd", "runs"), or "chunked" for ``nvfp4_index_score_decode``. +PIN_SCORER: str | None = None +# two heads: runs from 13 requests (32K-192K, 12: first page 18.7 vs runs 20.2 us, 16: 24.3 vs 21.1; sweep_decode_h2.py) +RUNS_MIN_REQS = 13 + + +def _fast_scorer(reqs: int, query_len: int, heads: int) -> str | None: + """The scorer of ``reqs`` x ``query_len`` rows at ``heads`` index heads, or None (chunked). One head through 16 + requests: two waves per page at 1 / 3 requests, a scalar first table entry at 2 (XCD-local score lines at 8 rows), + sectors at 4 x 8 rows, independent scans otherwise. Two heads: two waves per page at one request, the XCD-local + first-page scans to RUNS_MIN_REQS, then 16-block runs balanced over the workgroups.""" + if PIN_SCORER is not None: + return None if PIN_SCORER == "chunked" else PIN_SCORER + if query_len not in (4, 8) or reqs < 1: + return None + if heads == 2: + from .score_runs_d128_p128_h2 import MAX_REQS + + return "page" if reqs == 1 else "first_page_xcd" if reqs < RUNS_MIN_REQS else "runs" if reqs <= MAX_REQS \ + else None + if heads != 1 or reqs > 16: + return None + if reqs in (1, 3): + return "page" + if reqs == 2: + return "first_page_xcd" if query_len == 8 else "first_page" + if reqs == 4 and query_len == 8: + return "sector" + return "scan" + + +# Tests / benchmarks: pin the run scorer's grid and unit bound. +PIN_RUNS_GRID: int | None = None +PIN_RUNS_UNITS: int | None = None +# two workgroups per CU, 4-block units below 64 runs: within 2% of the best of grids 256-2048 (sweep_decode_h2.py) +RUNS_GRID = 512 +RUNS_UNITS = 256 + + +def _score_runs(q_idx, index_cache, score, block_table, seq_lens, init_blocks, local_blocks, query_len): + """Two heads, runs of 16 blocks balanced over the workgroups (``score_runs_d128_p128_h2``).""" + from .score_runs_d128_p128_h2 import compile_score_runs + + _run_compiled( + compile_score_runs(query_len), q_idx.data_ptr(), index_cache.data_ptr(), + nvfp4_e4m3_lut(q_idx.device).data_ptr(), score.data_ptr(), block_table.data_ptr(), seq_lens.data_ptr(), + int(index_cache.stride(0)), int(score.stride(0)), int(score.stride(1)), _end_bytes(score), + int(block_table.stride(0)), _end_bytes(block_table), q_idx.shape[0] // query_len, int(init_blocks), + int(local_blocks), PIN_RUNS_GRID or RUNS_GRID, PIN_RUNS_UNITS or RUNS_UNITS, _current_stream(), + ) + + +def _columns_ok(q_idx, score) -> bool: + """Two heads: Q rows ``[rows, 2, 128]`` and the score heads contiguous, 16-B aligned score rows + (``columns.columns``).""" + rows, heads, _ = q_idx.shape + return heads == 1 or (q_idx.stride(1) == 128 and q_idx.stride(0) == heads * 128 + and score.stride(0) == rows * score.stride(1) and score.stride(1) % 4 == 0) + + def _score_rows(q_idx, index_cache, score, block_table, seq_lens, init_blocks, local_blocks, query_len, max_seq_len): - """The scores of ``nvfp4_index_score_top16``: one index head x 4 / 8 rows through 16 requests on the one-head - scorers (two waves per page at 1 / 3 requests, a scalar first table entry at 2, sectors at 4 x 8 rows, independent - scans otherwise), else ``nvfp4_index_score_decode``.""" + """The scores of ``nvfp4_index_score_top16``: the scorer of ``_fast_scorer``, else ``nvfp4_index_score_decode``.""" rows, heads, _ = q_idx.shape - plain_head = heads == 1 and query_len in (4, 8) and rows > 0 - if plain_head and rows // query_len <= 16: - from .score_d128_p128_h1 import compile_score + reqs = rows // query_len + kind = _fast_scorer(reqs, query_len, heads) if rows > 0 and _columns_ok(q_idx, score) else None + if kind == "runs": + _score_runs(q_idx, index_cache, score, block_table, seq_lens, init_blocks, local_blocks, query_len) + elif kind is not None: max_blocks = max(1, _cdiv(max_seq_len, 128)) - reqs = rows // query_len - page_coop = reqs in (1, 3) - waves = 2 if page_coop else 1 + waves = 2 if kind == "page" else 1 splits = min(max(1, TARGET_WAVES // (waves * reqs)), max_blocks) # A lane holds one strided page-table entry: a 1M context needs at most eleven per wave. assert _cdiv(max_blocks, splits) <= 64 - from .score_page_d128_p128_h1 import compile_score_page - if reqs == 2: - # XCD-local 128-B score lines at 8 rows only - from .score_firstpage_d128_p128_h1 import compile_score_first_page - exe = compile_score_first_page(query_len, query_len == 8) - elif reqs == 4 and query_len == 8: - from .score_sector_d128_p128_h1 import compile_score_sector - exe = compile_score_sector(query_len) + if heads == 1: + from . import score_d128_p128_h1 as scan, score_firstpage_d128_p128_h1 as first + from . import score_page_d128_p128_h1 as page, score_sector_d128_p128_h1 as sector + + if kind == "page": + exe = page.compile_score_page(query_len) + elif kind in ("first_page", "first_page_xcd"): + exe = first.compile_score_first_page(query_len, kind == "first_page_xcd") + elif kind == "sector": + exe = sector.compile_score_sector(query_len) + else: + exe = scan.compile_score(query_len, waves) else: - exe = compile_score_page(query_len) if page_coop else compile_score(query_len, waves) + from . import score_firstpage_d128_p128_h2 as first, score_page_d128_p128_h2 as page + + exe = page.compile_score_page(query_len) if kind == "page" else first.compile_score_first_page(query_len) _run_compiled( exe, q_idx.data_ptr(), index_cache.data_ptr(), nvfp4_e4m3_lut(q_idx.device).data_ptr(), score.data_ptr(), block_table.data_ptr(), seq_lens.data_ptr(), int(index_cache.stride(0)), - int(q_idx.stride(0)), int(score.stride(1)), _end_bytes(score), + int(q_idx.stride(0) if heads == 1 else q_idx.stride(1)), int(score.stride(1)), _end_bytes(score), _end_bytes(block_table), max_blocks, splits, reqs, int(block_table.stride(0)), int(init_blocks), int(local_blocks), _current_stream(), ) @@ -247,8 +312,7 @@ def nvfp4_index_top16_from_scores( from ..topk import index_topk_decode_indices heads, rows = int(topk_idx.shape[0]), int(rows) - plain_head = heads == 1 and query_len in (4, 8) and rows > 0 - if plain_head and rows <= 128: + if _plain_selector(rows, query_len, heads): _run_compiled( _get_top16_plain_wave4(), score.data_ptr(), int(score.stride(0)), int(score.stride(1)), score_width(max_seq_len), seq_lens.data_ptr(), @@ -261,6 +325,18 @@ def nvfp4_index_top16_from_scores( ) +# Tests / benchmarks: pin the selector (True: the plain four-wave one). +PIN_PLAIN: bool | None = None + + +def _plain_selector(rows: int, query_len: int, heads: int) -> bool: + """The plain-order four-wave selector through 128 rows at one head, 256 at two (x 4 / 8 rows per request), else + ``topk.index_topk_decode_indices`` (two heads, sweep_decode_h2.py: 256 rows 3.33 vs 3.62 us, 512 4.12 vs 3.87).""" + if PIN_PLAIN is not None: + return PIN_PLAIN + return query_len in (4, 8) and 0 < rows <= {1: 128, 2: 256}.get(heads, 0) + + @functools.cache @launcher def _get_top16_plain_wave4(): diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_firstpage_d128_p128_h2.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_firstpage_d128_p128_h2.py new file mode 100644 index 0000000..7456dcc --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_firstpage_d128_p128_h2.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Independent fp8-MFMA page scans at two index heads (``columns.columns``): ``score_firstpage_d128_p128_h1`` with +its XCD-local unit-to-block order and 2 x QL columns. + +Read the first page-table entry as a scalar. Prefetch later entries after LUT +staging, bounded by device lengths. This avoids unused table reads at short +contexts while keeping the later pages in registers for long contexts. +The LUT, MFMA and fixed maximum order match the general one-head scorer. +""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr, const_expr +from amd_kernels.warm_up import launcher +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.common import buffer_ops +from .columns import columns +from .kernel import (_xlane_max4, _maxf_nn, _dequant8, _perm, PERM_EVEN, + PERM_ODD, _mfma_fp8_16x16x128, PAGE_BYTES, PAGE_CODES, NT) + + +@functools.cache +@launcher +def compile_score_first_page(ql: int): + assert ql in (4, 8) + NW = 1 + QL = ql + QC, row_of, score_index = columns(QL) + name = f"nvfp4_fp8mfma_index_score_d128_p128_r{QL}h2_w{NW}_firstpage_xcd" + + @fx.struct + class Shared: + lut: fx.Array[fx.Int32, 512, 16] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel(q: fx.Int64, kv: fx.Int64, lut: fx.Int64, score: fx.Int64, + bt: fx.Int64, seqs: fx.Int64, page_stride: fx.Int64, + q_stride: fx.Int32, score_stride: fx.Int32, score_bytes: fx.Int64, + table_bytes: fx.Int64, max_blocks: fx.Int32, + splits: fx.Int32, reqs: fx.Int32, bt_stride: fx.Int32, init: fx.Int32, local: fx.Int32): + sm = fx.SharedAllocator().allocate(Shared).peek() + tx = fx.thread_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + col, kl = lane % 16, lane // 16 + req = fx.block_idx.y + b0 = (fx.block_idx.x) * NW + wave + # NW == 1. Units below splits rounded down to 256 score block + # 256 (y // 32) + 32 (x % 8) + y % 32 (y = x // 8): a bijection + # there, so the 32 blocks of one 128-B score line come from + # workgroups of one XCD (equal x % 8 share one); the rest keep x. + yg = b0 >> 3 + remap = ((yg >> 5) << 8) + ((b0 & 7) << 5) + (yg & 31) + b0 = (b0 < ((splits >> 8) << 8)).select(remap, b0) + step = splits * NW + def resource(addr, nbytes): + return buffer_ops.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes) + # The first scalar table entry and Q do not depend on the length. + tab = fx.Int32(buffer_ops.buffer_load(resource(bt, table_bytes), req * bt_stride + b0, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + seq = fx.Int32(buffer_ops.buffer_load(resource(seqs + req.to(fx.Int64) * 4, 4), fx.Int32(0), + vec_width=1, dtype=fx.Int32, is_scalar=True)) + seq = (seq < 0).select(fx.Int32(0), seq) + qr = resource(q + req.to(fx.Int64) * QC * q_stride, QC * q_stride) + qs = [] + for h in range_constexpr(2): + d = Vec(buffer_ops.buffer_load(qr, ((col * q_stride + kl * 32) >> 2) + h * 4, + vec_width=4, dtype=fx.Int32, mask=col < QC)) + qs += [_perm(fx.Int32(d[1]), fx.Int32(d[0]), PERM_EVEN), + _perm(fx.Int32(d[1]), fx.Int32(d[0]), PERM_ODD), + _perm(fx.Int32(d[3]), fx.Int32(d[2]), PERM_EVEN), + _perm(fx.Int32(d[3]), fx.Int32(d[2]), PERM_ODD)] + qf = Vec.from_elements(qs, fx.Int32) + lr = resource(lut, 2048) + lut_quads = fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32, sm.lut.ptr), + fx.make_layout(512, 1)), fx.make_layout(4, 1)) + atom128 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + for k in range_constexpr(512 // (64 * NW * 4)): + lv = Vec(buffer_ops.buffer_load(lr, tx * 4 + k * 64 * NW * 4, vec_width=4, dtype=fx.Int32)) + r = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r.store(lv) + fx.copy(atom128, r, fx.slice(lut_quads, (None, tx + k * 64 * NW))) + fx.gpu.barrier() + lut_view = fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32, sm.lut.ptr), + fx.make_layout(512, 1)), fx.make_layout(2, 1)) + atom = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + def lut_load(s): + r = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + fx.copy(atom, fx.slice(lut_view, (None, s)), r) + v = Vec(r.load()) + return fx.Int32(v[0]), fx.Int32(v[1]) + def tile(kr, t): + c = Vec(buffer_ops.buffer_load(kr, col * 16 + kl * 4, + vec_width=4, dtype=fx.Int32, soffset_bytes=t * 1024, cache_modifier=NT)) + s = fx.Int32(fx.Int16(buffer_ops.buffer_load(kr, col * 4 + kl, + vec_width=1, dtype=fx.Int16, soffset_bytes=PAGE_CODES + t * 128, + cache_modifier=NT)).to(fx.Int32)) + return c, s + sr = resource(score, score_bytes) + nblk = ((seq + 127) >> 7) + # Later entries overlap the first page; one lane holds each entry. + future = fx.Int32(buffer_ops.buffer_load(resource(bt, table_bytes), req * bt_stride + b0 + (lane + 1) * step, + vec_width=1, dtype=fx.Int32, mask=b0 + (lane + 1) * step < ((seq + 127) >> 7))) + neg_inf = fx.Float32(float('-inf')) + # Peeling removes the first/future selector from the page loop. + @flyc.jit + def page_value(b: fx.Int32, page: fx.Int32): + kr = resource(kv + page.to(fx.Int64) * page_stride, PAGE_BYTES) + cs, ss = [], [] + for t in range_constexpr(8): + c, s = tile(kr, t) + cs.append(c) + ss.append(s) + fx.rocdl.sched_barrier(0) + frags = [] + for t in range_constexpr(8): + t0, t1 = lut_load(ss[t] & 255), lut_load((ss[t] >> 8) & 255) + fs = [] + for e in range_constexpr(4): + lo, hi = _dequant8(fx.Int32(cs[t][e]), *(t0 if e < 2 else t1)) + fs += [lo, hi] + frags.append(Vec.from_elements(fs, fx.Int32)) + fx.rocdl.sched_barrier(0) + # The dots do not depend on the causal mask. Issue them once, + # then keep the original token order in either maximum reduction. + accs = [_mfma_fp8_16x16x128(frags[t], qf, Vec.filled(4, 0.0, fx.Float32)) + for t in range_constexpr(8)] + def reduce_page(masked): + run = neg_inf + lim = seq - QL + row_of(col) - b * 128 - kl * 4 + for t in range_constexpr(8): + for e in range_constexpr(4): + x = accs[t][e] + if const_expr(masked): + x = (lim >= t * 16 + e).select(x, neg_inf) + run = _maxf_nn(run, x) + return _xlane_max4(run) + mx = neg_inf + if b * 128 + 127 > seq - QL: + mx = reduce_page(True) + else: + mx = reduce_page(False) + row_len = seq - QL + row_of(col) + 1 + rb = (((row_len < 0).select(fx.Int32(0), row_len) + 127) >> 7) + l0 = rb - local + l0 = (l0 < 0).select(fx.Int32(0), l0) + vis = b < rb + v = (vis & (b < init)).select(fx.Float32(1e30), mx) + v = (vis & (b >= l0)).select(fx.Float32(1e29), v) + return v + b = b0 + if b < nblk: + v = page_value(b, tab) + # Complete the future table prefetch before the first score store. + # The later loop then reads ready lanes without waiting for that store. + fx.rocdl.s_waitcnt(vmcnt=0) + buffer_ops.buffer_store(v, sr, score_index(col, req, reqs, score_stride, b), + mask=(col < QC) & (kl == 0)) + b = b + step + j = fx.Int32(0) + while b < nblk: + page = fx.Int32(fx.rocdl.readlane(T.i32, future, j)) + v = page_value(b, page) + buffer_ops.buffer_store(v, sr, score_index(col, req, reqs, score_stride, b), + mask=(col < QC) & (kl == 0)) + b = b + step + j = j + 1 + + @flyc.jit + def launch_first_page(q: fx.Int64, kv: fx.Int64, lut: fx.Int64, score: fx.Int64, + bt: fx.Int64, seqs: fx.Int64, page_stride: fx.Int64, + q_stride: fx.Int32, score_stride: fx.Int32, score_bytes: fx.Int64, + table_bytes: fx.Int64, max_blocks: fx.Int32, + splits: fx.Int32, reqs: fx.Int32, bt_stride: fx.Int32, init: fx.Int32, local: fx.Int32, stream: fx.Stream): + kernel(q, kv, lut, score, bt, seqs, page_stride, q_stride, score_stride, + score_bytes, table_bytes, max_blocks, splits, reqs, bt_stride, init, local).launch( + grid=(fx.Int64(splits), fx.Int64(reqs), 1), block=(64 * NW, 1, 1), stream=stream) + launch_first_page.kernel_name = name + return launch_first_page diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_page_d128_p128_h2.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_page_d128_p128_h2.py new file mode 100644 index 0000000..e7b35a8 --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_page_d128_p128_h2.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two waves per page at two index heads (``columns.columns``: ``score_page_d128_p128_h1`` with 2 x QL columns), fp8 +MFMA, register Q and scalar page-table loads. + +Each wave scores half the page. LDS combines their maxima in a fixed order; +only wave zero writes the score. The host uses this at one request. +Units past the request's length leave right after the length load. +The same LUT and one fp8 MFMA per token dot preserve the ordinary scorer bits.""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr, const_expr +from amd_kernels.warm_up import launcher +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.common import buffer_ops +from .columns import columns +from .kernel import (_xlane_max4, _maxf_nn, _dequant8, _perm, PERM_EVEN, + PERM_ODD, _mfma_fp8_16x16x128, PAGE_BYTES, PAGE_CODES, NT) + + +@functools.cache +@launcher +def compile_score_page(ql: int): + assert ql in (4, 8) + NW = 2 + QL = ql + QC, row_of, score_index = columns(QL) + TILES = 8 // NW + name = f"nvfp4_fp8mfma_index_score_page_d128_p128_r{QL}h2_w{NW}_scale_first" + + @fx.struct + class Shared: + lut: fx.Array[fx.Int32, 512, 16] + partial: fx.Array[fx.Int32, NW * 16, 16] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel(q: fx.Int64, kv: fx.Int64, lut: fx.Int64, score: fx.Int64, + bt: fx.Int64, seqs: fx.Int64, page_stride: fx.Int64, + q_stride: fx.Int32, score_stride: fx.Int32, score_bytes: fx.Int64, + table_bytes: fx.Int64, max_blocks: fx.Int32, + splits: fx.Int32, reqs: fx.Int32, bt_stride: fx.Int32, init: fx.Int32, local: fx.Int32): + sm = fx.SharedAllocator().allocate(Shared).peek() + tx = fx.thread_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + col, kl = lane % 16, lane // 16 + req = fx.block_idx.y + b0 = fx.block_idx.x + step = splits + def resource(addr, nbytes): + return buffer_ops.create_buffer_resource_from_addr(addr, num_records_bytes=nbytes) + # The first scalar table load is independent of the length. Later + # pages reload their scalar entry: no unused strided prefetch or readlane. + tab = fx.Int32(buffer_ops.buffer_load(resource(bt, table_bytes), req * bt_stride + b0, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + seq = fx.Int32(buffer_ops.buffer_load(resource(seqs + req.to(fx.Int64) * 4, 4), fx.Int32(0), + vec_width=1, dtype=fx.Int32, is_scalar=True)) + seq = (seq < 0).select(fx.Int32(0), seq) + nblk = ((seq + 127) >> 7) + # A unit past the request's blocks (most of the 1M plan's 1536 at short + # contexts) stops before the Q and LUT loads: M 8 / 32K -8%. + if b0 < nblk: + qr = resource(q + req.to(fx.Int64) * QC * q_stride, QC * q_stride) + qs = [] + for h in range_constexpr(2): + d = Vec(buffer_ops.buffer_load(qr, ((col * q_stride + kl * 32) >> 2) + h * 4, + vec_width=4, dtype=fx.Int32, mask=col < QC)) + qs += [_perm(fx.Int32(d[1]), fx.Int32(d[0]), PERM_EVEN), + _perm(fx.Int32(d[1]), fx.Int32(d[0]), PERM_ODD), + _perm(fx.Int32(d[3]), fx.Int32(d[2]), PERM_EVEN), + _perm(fx.Int32(d[3]), fx.Int32(d[2]), PERM_ODD)] + qf = Vec.from_elements(qs, fx.Int32) + lr = resource(lut, 2048) + # Stage the unmodified LUT with 16-byte loads and LDS writes. + lut_quads = fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32, sm.lut.ptr), + fx.make_layout(512, 1)), fx.make_layout(4, 1)) + atom128 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + for k in range_constexpr(512 // (64 * NW * 4)): + lv = Vec(buffer_ops.buffer_load(lr, tx * 4 + k * 64 * NW * 4, vec_width=4, dtype=fx.Int32)) + r = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r.store(lv) + fx.copy(atom128, r, fx.slice(lut_quads, (None, tx + k * 64 * NW))) + fx.gpu.barrier() + lut_view = fx.logical_divide(fx.make_view(fx.recast_iter(fx.Int32, sm.lut.ptr), + fx.make_layout(512, 1)), fx.make_layout(2, 1)) + atom = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + def lut_load(s): + r = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + fx.copy(atom, fx.slice(lut_view, (None, s)), r) + v = Vec(r.load()) + return fx.Int32(v[0]), fx.Int32(v[1]) + def codes(kr,t): + return Vec(buffer_ops.buffer_load(kr, col * 16 + kl * 4, + vec_width=4, dtype=fx.Int32, soffset_bytes=t * 1024, cache_modifier=NT)) + def scale(kr,t): + return fx.Int32(fx.Int16(buffer_ops.buffer_load(kr, col * 4 + kl, + vec_width=1, dtype=fx.Int16, soffset_bytes=PAGE_CODES + t * 128, + cache_modifier=NT)).to(fx.Int32)) + + sr = resource(score, score_bytes) + j = fx.Int32(0) + b = b0 + neg_inf = fx.Float32(float('-inf')) + # The LUT staging has already drained the prologue VMEM. Keep that + # fact explicit so the loop does not wait for prior score stores. + fx.rocdl.s_waitcnt(vmcnt=0) + while b < nblk: + page = tab + if j != 0: + page = fx.Int32(buffer_ops.buffer_load(resource(bt, table_bytes), req * bt_stride + b, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + kr = resource(kv + page.to(fx.Int64) * page_stride, PAGE_BYTES) + # Issue scales first: their LUT reads can overlap the later + # code loads. The dequantization and maximum order are unchanged. + ss = [scale(kr,t + wave * TILES) for t in range_constexpr(TILES)] + cs = [codes(kr,t + wave * TILES) for t in range_constexpr(TILES)] + fx.rocdl.sched_barrier(0) + frags = [] + for t in range_constexpr(TILES): + t0, t1 = lut_load(ss[t] & 255), lut_load((ss[t] >> 8) & 255) + fs = [] + for e in range_constexpr(4): + lo, hi = _dequant8(fx.Int32(cs[t][e]), *(t0 if e < 2 else t1)) + fs += [lo, hi] + frags.append(Vec.from_elements(fs, fx.Int32)) + fx.rocdl.sched_barrier(0) + def reduce_page(masked): + run = neg_inf + lim = seq - QL + row_of(col) - b * 128 - kl * 4 - wave * TILES * 16 + accs = [] + for t in range_constexpr(TILES): + af = frags[t] + acc = _mfma_fp8_16x16x128(af, qf, Vec.filled(4, 0.0, fx.Float32)) + accs.append(acc) + if const_expr(t >= 1): + for e in range_constexpr(4): + x = accs[t - 1][e] + if const_expr(masked): + x = (lim >= (t - 1) * 16 + e).select(x, neg_inf) + run = _maxf_nn(run, x) + for e in range_constexpr(4): + x = accs[TILES - 1][e] + if const_expr(masked): + x = (lim >= (TILES - 1) * 16 + e).select(x, neg_inf) + run = _maxf_nn(run, x) + return _xlane_max4(run) + mx = neg_inf + if b * 128 + 127 > seq - QL: + mx = reduce_page(True) + else: + mx = reduce_page(False) + if kl == 0: + sm.partial.ptr[wave * 16 + col] = mx.bitcast(fx.Int32) + fx.gpu.barrier() + mx = fx.Int32(sm.partial.ptr[col]).bitcast(fx.Float32) + for w in range_constexpr(1, NW): + mx = _maxf_nn(mx, fx.Int32(sm.partial.ptr[w * 16 + col]).bitcast(fx.Float32)) + fx.gpu.barrier() + row_len = seq - QL + row_of(col) + 1 + rb = (((row_len < 0).select(fx.Int32(0), row_len) + 127) >> 7) + l0 = rb - local + l0 = (l0 < 0).select(fx.Int32(0), l0) + vis = b < rb + v = (vis & (b < init)).select(fx.Float32(1e30), mx) + v = (vis & (b >= l0)).select(fx.Float32(1e29), v) + buffer_ops.buffer_store(v, sr, score_index(col, req, reqs, score_stride, b), + mask=(col < QC) & (kl == 0) & (wave == 0)) + b = b + step + j = j + 1 + + @flyc.jit + def launch_score_page_scale_first(q: fx.Int64, kv: fx.Int64, lut: fx.Int64, score: fx.Int64, + bt: fx.Int64, seqs: fx.Int64, page_stride: fx.Int64, + q_stride: fx.Int32, score_stride: fx.Int32, score_bytes: fx.Int64, + table_bytes: fx.Int64, max_blocks: fx.Int32, + splits: fx.Int32, reqs: fx.Int32, bt_stride: fx.Int32, init: fx.Int32, local: fx.Int32, stream: fx.Stream): + kernel(q, kv, lut, score, bt, seqs, page_stride, q_stride, score_stride, + score_bytes, table_bytes, max_blocks, splits, reqs, bt_stride, init, local).launch( + grid=(fx.Int64(splits), fx.Int64(reqs), 1), block=(64 * NW, 1, 1), stream=stream) + launch_score_page_scale_first.kernel_name = name + # The 16-word preload covers the pointers and page/Q/score strides (chain at 1 / 3 requests: -1-2%). + launch_score_page_scale_first.compile_hints = {"llvm_options": { + "amdgpu-kernarg-preload": True, "amdgpu-kernarg-preload-count": 16}} + return launch_score_page_scale_first diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_runs_d128_p128_h2.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_runs_d128_p128_h2.py new file mode 100644 index 0000000..dc78d7d --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_decode/score_runs_d128_p128_h2.py @@ -0,0 +1,439 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-head decode index scorer over runs of 16 blocks of every request, balanced over the workgroups (fp8 MFMA). + +Column c of the one 16-column MFMA group is query row c // 2 of index head c % 2 (``columns.columns``). Work: the +requests' runs (16 consecutive blocks), prefix-summed in LDS; workgroup g takes runs g, g + grid, ..., wave w quarter w +(4 blocks) of each, the next page loading tile by tile under the current one, and stores the quarter's 4 scores of a +(head, row) as 16 B. A call of few runs (device-side lengths, the host's bound) splits each run into four 4-block units +instead, wave w its block w. A wave keeps its request's Q columns in registers and 64 pages' table entries in one +VGPR (readlane per page). The page math and maximum order are the one-head scorers' (same bits). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..score_prefill.kernel import ( + PERM_EVEN, + PERM_ODD, + _bits, + _dequant8, + _global_i32_ptr, + _maxf_nn, + _mfma_fp8_16x16x128, + _pack8, + _perm, +) +from .columns import columns +from .kernel import BLK, MT, NT, PAGE_BYTES, PAGE_CODES, SENTINEL_INIT, SENTINEL_LOCAL, D, _xlane_max4 + +NW = 4 # waves per workgroup: quarter w of every run +RUN = 16 # blocks per run +QD = RUN // NW # blocks per quarter +WIN = 64 # pages per block-table window (one VGPR) +MAX_REQS = 64 * NW # one request per thread in the prefix sum +NR_LOG = MAX_REQS.bit_length() - 1 + + +def _uni(x): + return fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.Int32(x))) + + +def _min(a, b): + return (a < b).select(a, b) + + +def _max(a, b): + return (a > b).select(a, b) + + +def _wave_scan(x, lane, steps): + """Inclusive prefix sum through the first 2**steps lanes.""" + for d in range_constexpr(steps): + y = fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane - (1 << d)) * 4, x)) + x = x + (lane >= (1 << d)).select(y, fx.Int32(0)) + return x + + +@functools.cache +@launcher +def compile_score_runs(ql: int): + """The scorer of ``ql`` (4 or 8) rows per request at two index heads.""" + assert ql in (4, 8) + QL = ql + QC, row_of, _ = columns(QL) + name = f"nvfp4_fp8mfma_index_score_runs_d128_p128_r{QL}h2" + + @fx.struct + class Shared: + lut: fx.Array[fx.Int32, 512, 16] # per scale byte: keys of magnitudes 0..3, 4..7 + incl: fx.Array[fx.Int32, MAX_REQS] # inclusive prefix of the run counts + seq: fx.Array[fx.Int32, MAX_REQS] + wsum: fx.Array[fx.Int32, NW] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_q: fx.Int64, # [num_reqs * QL, 2, 128] e4m3 + arg_kv: fx.Int64, # NVFP4 index pages [num_blocks, 128, 72] u8 + arg_lut: fx.Int64, # nvfp4_e4m3_lut [256] i64 + arg_score: fx.Int64, # [2, num_reqs * QL, S] fp32 + arg_bt: fx.Int64, # block table [num_reqs, >= max_blocks] i32 + arg_seq: fx.Int64, # seq_lens [num_reqs] i32 + i64_page_stride: fx.Int64, # bytes between pages + i32_s_head: fx.Int32, # score stride(0), stride(1) (elements) + i32_s_row: fx.Int32, + i64_s_bytes: fx.Int64, + i32_bt_stride: fx.Int32, + i64_bt_bytes: fx.Int64, + i32_num_reqs: fx.Int32, + i32_init: fx.Int32, # leading blocks forced in (sentinel 1e30) + i32_local: fx.Int32, # trailing blocks forced in (sentinel 1e29) + i32_nwg: fx.Int32, # workgroups (the grid) + i32_unit_max: fx.Int32, # units of 4 blocks (one page per wave) up to this many, else runs + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = _uni(tx // 64) + col, kl = lane % 16, lane // 16 + q_row = row_of(col) + head = col & 1 + incl_l, seq_l = smem.incl.ptr, smem.seq.ptr + + btr = buffer_ops.create_buffer_resource_from_addr(arg_bt, num_records_bytes=i64_bt_bytes) + qr = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=fx.Int64(i32_num_reqs) * (QC * D) + ) + sr = buffer_ops.create_buffer_resource_from_addr(arg_score, num_records_bytes=i64_s_bytes) + + # e4m3 table; per request its runs, prefix-summed + lut_g = _global_i32_ptr(arg_lut) + lut_l = smem.lut.ptr + for i in range_constexpr(2): + lut_l[tx * 2 + i] = fx.Int32(lut_g[tx * 2 + i]) + r_ok = tx < i32_num_reqs + seq_t = fx.Int32(_global_i32_ptr(arg_seq)[r_ok.select(tx, fx.Int32(0))]) + seq_t = r_ok.select(seq_t, fx.Int32(0)) + seq_t = (seq_t < 0).select(fx.Int32(0), seq_t) + nb_t = (seq_t + (BLK - 1)) // BLK + runs_t = (nb_t + (RUN - 1)) // RUN + x = _wave_scan(runs_t, lane, 6) + if lane == 63: + smem.wsum.ptr[wave] = x + seq_l[tx] = seq_t + fx.gpu.barrier() + for w in range_constexpr(NW - 1): + x = x + (wave > w).select(fx.Int32(smem.wsum.ptr[w]), fx.Int32(0)) + incl_l[tx] = x + fx.gpu.barrier() + + total = _uni(incl_l[MAX_REQS - 1]) + unit = total * NW <= i32_unit_max + + lut8 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.lut.ptr), fx.make_layout(512, 1)), + fx.make_layout(2, 1), + ) + lut_atom = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + + def lut_load(sbyte): + r = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + fx.copy(lut_atom, fx.slice(lut8, (None, sbyte)), r) + v = Vec(r.load()) + return fx.Int32(v[0]), fx.Int32(v[1]) + + code_off = (col * 64 + kl * 16) // 4 # dwords + # token col's group scales 2 kl, 2 kl + 1: half kl % 2 of a dword + scale_off = col * 2 + kl // 2 # dwords + # v_perm selectors: scale byte 2 (kl % 2) (+ 1) of the dword, zeros above + scale_sel = (kl % 2) * 2 + 0x0C0C0C00 + + def page_rsrc(page): + return buffer_ops.create_buffer_resource_from_addr( + arg_kv + fx.Int64(page) * i64_page_stride, num_records_bytes=PAGE_BYTES + ) + + def load_tile(kr, ok, t): + """Tile t of a page: this lane's 16 B of codes (token 16 t + col, dims 32 kl ..) and its group scales; + ``ok`` false: no memory access.""" + c = Vec( + buffer_ops.buffer_load( + kr, code_off, vec_width=4, dtype=fx.Int32, mask=ok, soffset_bytes=t * 1024, cache_modifier=NT + ) + ) + sc = fx.Int32( + buffer_ops.buffer_load( + kr, scale_off, vec_width=1, dtype=fx.Int32, mask=ok, soffset_bytes=PAGE_CODES + t * 128, + cache_modifier=NT, + ) + ) + return c, sc + + def dequant_tile(c, sc): + """The A fragment of a tile (e4m3 keys, even dims then odd dims per 8).""" + t0 = lut_load(_perm(sc, sc, scale_sel)) + t1 = lut_load(_perm(sc, sc, scale_sel + 1)) + f = [] + for e in range_constexpr(4): + lo, hi = _dequant8(fx.Int32(Vec(c)[e]), *(t0 if e < 2 else t1)) + f += [lo, hi] + return Vec.from_elements(f, fx.Int32) + + def q_load(r, ok): + """This lane's 32 B of Q (column col of request r, dims 32 kl ..), raw.""" + off = ((r * QC + col) * D + kl * 32) // 4 + okc = ok & (col < QC) + return [ + Vec(buffer_ops.buffer_load(qr, off + hf * 4, vec_width=4, dtype=fx.Int32, mask=okc)) + for hf in range_constexpr(2) + ] + + def q_frag(a, b): + """B operand: the 32 B in the key's dim order (even dims, odd dims per 8).""" + halves = [] + for v in (a, b): + d = [fx.Int32(Vec(v)[e]) for e in range_constexpr(4)] + halves.append( + Vec.from_elements( + [ + _perm(d[1], d[0], PERM_EVEN), + _perm(d[1], d[0], PERM_ODD), + _perm(d[3], d[2], PERM_EVEN), + _perm(d[3], d[2], PERM_ODD), + ], + fx.Int32, + ) + ) + return _pack8(halves[0], halves[1]) + + zero4 = fx.Vector.filled(4, 0.0, fx.Float32) + neg_inf = fx.Float32(float("-inf")) + sent_init = fx.Float32(SENTINEL_INIT) + sent_local = fx.Float32(SENTINEL_LOCAL) + + def reduce(v, t, run, masked, lim): + for e in range_constexpr(4): + x = v[e] + if const_expr(masked): + x = (lim >= t * 16 + e).select(x, neg_inf) + run = _maxf_nn(run, x) + return run + + def page_pass(cs, ss, qf, krn, okn, masked, lim): + """The max over the page's tokens of this lane's column; tile t of the next page loads into tile t's freed + registers (the barriers keep LLVM from hoisting the loads). Only the reduction depends on the causal + mask.""" + accs = [] + nxt = [] + for t in range_constexpr(MT): + f = dequant_tile(cs[t], ss[t]) + nxt.append(load_tile(krn, okn, t)) + accs.append(_mfma_fp8_16x16x128(f, qf, zero4)) + fx.rocdl.sched_barrier(0) + mx = neg_inf + if masked: + run = neg_inf + for t in range_constexpr(MT): + run = reduce(accs[t], t, run, True, lim) + mx = _xlane_max4(run) + else: + run = neg_inf + for t in range_constexpr(MT): + run = reduce(accs[t], t, run, False, None) + mx = _xlane_max4(run) + return mx, nxt + + def row_value(mx, seq, q_off, b): + """AITER's sentinels on block b of the row at q_off (a block both init and local is local).""" + v1 = seq - QL + q_off + 1 # the row's visible tokens + rb = ((v1 < 0).select(fx.Int32(0), v1) + (BLK - 1)) // BLK + l0 = rb - i32_local + local0 = (l0 < 0).select(fx.Int32(0), l0) + vis = b < rb + v = (vis & (b < i32_init)).select(sent_init, mx) + return (vis & (b >= local0)).select(sent_local, v) + + def scan(UNIT): + """The pages of this workgroup's items: units (``UNIT``, a quarter of a run, wave w its block w) or runs + (wave w its quarter w).""" + qdv = 1 if UNIT else QD # pages per item and wave + items = total * NW if UNIT else total + nitems = (pid < items).select((items - pid + i32_nwg - 1) // i32_nwg, 0) + nslots = nitems * qdv # pages of this wave + w0 = fx.Int32(0) + while w0 < nslots: + # the window's pages, one per lane: page s is item pid + (s / qdv) * grid's block (run k = item / 4 + # for units) of request rj (binary search in the prefix) + s = w0 + lane + it = pid + (s if UNIT else s >> 2) * i32_nwg + k = (it >> 2) if UNIT else it + rj = fx.Int32(0) + for bs in range_constexpr(NR_LOG): + step = MAX_REQS >> (bs + 1) + rj = (fx.Int32(incl_l[rj + step - 1]) <= k).select(rj + step, rj) + valid = s < nslots + rj = valid.select(rj, fx.Int32(0)) + excl = (rj > 0).select(fx.Int32(incl_l[_max(rj - 1, 0)]), fx.Int32(0)) + seqj = fx.Int32(seq_l[rj]) + nblkj = (seqj + (BLK - 1)) // BLK + in_run = ((it & 3) * NW + wave) if UNIT else (wave * QD + (s & 3)) + bj = (k - excl) * RUN + in_run + frow = seqj - (QL - 1) + rbf = ((frow < 0).select(fx.Int32(0), frow) + (BLK - 1)) // BLK + ll = nblkj - i32_local + loc_last = (ll < 0).select(fx.Int32(0), ll) + # blocks every row sees whole: init / local ones only need their sentinel + skip = (bj < rbf) & ((bj < i32_init) | (bj >= loc_last)) + need = valid & (bj < nblkj) & ~skip + pgj = fx.Int32( + buffer_ops.buffer_load( + btr, need.select(rj * i32_bt_stride + bj, fx.Int32(0)), vec_width=1, dtype=fx.Int32 + ) + ) + pgj = need.select(pgj, fx.Int32(-1)) + n = _min(nslots - w0, WIN) + + p_0 = fx.Int32(fx.rocdl.readlane(T.i32, pgj, 0)) + r_0 = fx.Int32(fx.rocdl.readlane(T.i32, rj, 0)) + ok0 = p_0 >= 0 + kr0 = page_rsrc(ok0.select(p_0, fx.Int32(0))) + fx.rocdl.sched_barrier(0) + c0, s0 = load_tile(kr0, ok0, 0) + fx.rocdl.sched_barrier(0) + c1, s1 = load_tile(kr0, ok0, 1) + fx.rocdl.sched_barrier(0) + c2, s2 = load_tile(kr0, ok0, 2) + fx.rocdl.sched_barrier(0) + c3, s3 = load_tile(kr0, ok0, 3) + fx.rocdl.sched_barrier(0) + c4, s4 = load_tile(kr0, ok0, 4) + fx.rocdl.sched_barrier(0) + c5, s5 = load_tile(kr0, ok0, 5) + fx.rocdl.sched_barrier(0) + c6, s6 = load_tile(kr0, ok0, 6) + fx.rocdl.sched_barrier(0) + c7, s7 = load_tile(kr0, ok0, 7) + fx.rocdl.sched_barrier(0) + qa, qb = q_load(r_0, n > 0) + # the loop head merges this path with the back edge: nothing pending + fx.rocdl.s_waitcnt(vmcnt=0) + qf = q_frag(qa, qb) + qna, qnb = qa, qb + sw = n < 0 # false: Q of the next page already loaded + v0 = neg_inf + v1 = neg_inf + v2 = neg_inf + v3 = neg_inf + j = fx.Int32(0) + while j < n: + r = fx.Int32(fx.rocdl.readlane(T.i32, rj, j)) + b = fx.Int32(fx.rocdl.readlane(T.i32, bj, j)) + seq = fx.Int32(fx.rocdl.readlane(T.i32, seqj, j)) + if sw: + qf = q_frag(qna, qnb) + jn = j + 1 + okj = jn < n + jc = okj.select(jn, fx.Int32(0)) + pgn = fx.Int32(fx.rocdl.readlane(T.i32, pgj, jc)) + rn = fx.Int32(fx.rocdl.readlane(T.i32, rj, jc)) + okn = okj & (pgn >= 0) + krn = page_rsrc(okn.select(pgn, fx.Int32(0))) + sw = okj & (rn != r) + qna, qnb = q_load(rn, sw) + fx.rocdl.sched_barrier(0) + mx, nxt = page_pass( + (c0, c1, c2, c3, c4, c5, c6, c7), + (s0, s1, s2, s3, s4, s5, s6, s7), + qf, + krn, + okn, + b * BLK + (BLK - 1) > seq - QL, + seq - QL + q_row - b * BLK - kl * 4, + ) + c0, s0 = nxt[0] + c1, s1 = nxt[1] + c2, s2 = nxt[2] + c3, s3 = nxt[3] + c4, s4 = nxt[4] + c5, s5 = nxt[5] + c6, s6 = nxt[6] + c7, s7 = nxt[7] + ov = row_value(mx, seq, q_row, b) + off = head * i32_s_head + (r * QL + q_row) * i32_s_row + b + nb = (seq + (BLK - 1)) // BLK + if const_expr(UNIT): + buffer_ops.buffer_store(ov, sr, off, mask=(kl == 0) & (col < QC) & (b < nb)) + else: + e = (w0 + j) & 3 + v0 = (e == 0).select(ov, v0) + v1 = (e == 1).select(ov, v1) + v2 = (e == 2).select(ov, v2) + v3 = (e == 3).select(ov, v3) + if e == QD - 1: + # the quarter's 4 blocks of this (head, row) as 16 B (positions past the row's blocks: + # unspecified) + buffer_ops.buffer_store( + Vec.from_elements([_bits(v0), _bits(v1), _bits(v2), _bits(v3)], fx.Int32), + sr, + off - 3, + mask=(kl == 0) & (col < QC) & (b - 3 < nb), + ) + j = jn + w0 = w0 + WIN + + # one code path per split (selects in the page loop cost run calls ~1%) + if unit: + scan(True) + else: + scan(False) + + @flyc.jit + def launch( + arg_q: fx.Int64, + arg_kv: fx.Int64, + arg_lut: fx.Int64, + arg_score: fx.Int64, + arg_bt: fx.Int64, + arg_seq: fx.Int64, + i64_page_stride: fx.Int64, + i32_s_head: fx.Int32, + i32_s_row: fx.Int32, + i64_s_bytes: fx.Int64, + i32_bt_stride: fx.Int32, + i64_bt_bytes: fx.Int64, + i32_num_reqs: fx.Int32, + i32_init: fx.Int32, + i32_local: fx.Int32, + i32_nwg: fx.Int32, + i32_unit_max: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_q, + arg_kv, + arg_lut, + arg_score, + arg_bt, + arg_seq, + i64_page_stride, + i32_s_head, + i32_s_row, + i64_s_bytes, + i32_bt_stride, + i64_bt_bytes, + i32_num_reqs, + i32_init, + i32_local, + i32_nwg, + i32_unit_max, + ).launch(grid=(fx.Int64(i32_nwg), 1, 1), block=(64 * NW, 1, 1), stream=stream) + + launch.kernel_name = name + return launch diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_prefill/host.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_prefill/host.py index d308b9a..3b61b34 100644 --- a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_prefill/host.py +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/score_prefill/host.py @@ -99,6 +99,8 @@ def _get_kernel_shared(heads: int, mq: int, fp8: bool): SMALL_WORK = 4096 MID_WORK = 16384 SMALL_TARGET = 512 +# two heads: kernel_shared through MID_WORK, 32-row wave tiles below this (2048 + 32K: 31.5 -> 23.9; sweep_prefill_plan) +H2_MQ1_WORK = 256 # Tests / benchmarks: pin the kernel (True: kernel_shared), its 32-row tiles per wave, or the workgroup target. PIN_SHARED: bool | None = None PIN_MQ: int | None = None @@ -191,9 +193,10 @@ def _score_plan(heads, batch, max_query_len, max_seq_len, device, exact=False, """The scorer, segments, query tiles, grid and route of a call; shared by the standalone and gathered chains.""" max_blocks = _cdiv(max_seq_len, _kernel.BLK) work = max_blocks * _cdiv(max_query_len, 256) * batch * heads - small = max_query_len * heads <= 128 or work < SMALL_WORK + tiny = max_query_len * heads <= 128 + small = tiny or work < (MID_WORK if heads == 2 else SMALL_WORK) shared = exact or (small if PIN_SHARED is None else PIN_SHARED) - mq = PIN_MQ or (1 if small else 2) + mq = PIN_MQ or (1 if small and (heads != 2 or tiny or work < H2_MQ1_WORK) else 2) target = PIN_TARGET or ( SMALL_TARGET if shared and work < MID_WORK diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/__init__.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/__init__.py new file mode 100644 index 0000000..a23264a --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The decode indexer split over two GPUs (TP2, gfx950) on the replicated index cache: rank r scores all four heads +over its blocks b % 2 == r, the ranks exchange each head's partial top 16 and merge (``host.IndexSplit2``).""" diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/decode_score.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/decode_score.py new file mode 100644 index 0000000..d2b9fa8 --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/decode_score.py @@ -0,0 +1,689 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Decode index scorer of one of two ranks: all four index heads over the rank's blocks b % 2 == me, the index-q +exchange fused (FlyDSL, gfx950). + +Columns: MFMA group g (16 columns) column c is head c % 4, row 4 g + c // 4 of the request (QL = 4: one group, 8: two). +Heads 2 me, 2 me + 1 read this rank's index-q, the other two the peer's rows in this rank's receive buffer. Scores +go to a local buffer [4, rows, S] in half order: block 2 j + me at position j. + +Index-q: workgroup g sends this rank's rows of requests g, g + grid, ... to the peer's receive buffer, then a flag per +request; a wave issues its first page's loads before it waits for its window's flags. Workgroup 0 ends with the peer's +flag of request 0, so no rank starts the next call's merge pushes before the peer has finished this call's merge. + +Work, by the device-side lengths: up to ``pages_runs`` runs (16 of the rank's blocks of a request), each wave takes a +contiguous range of the rank's pages of all requests in order (balanced to one page; a 4-byte store per page and +(head, row)); above, ``score_decode.score_runs_d128_p128_h2``'s runs: workgroup g takes runs g, g + grid, ..., wave w +quarter w of each, a 16-byte store per quarter and (head, row). Request prefixes of both counts live in LDS; the next +page loads tile by tile under the current one, a request's Q only on a request change. The page pass and maximum +order are ``split4.decode_score``'s (the one-head scorers' bits). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..score_decode.kernel import BLK, MT, NT, PAGE_BYTES, PAGE_CODES, SENTINEL_INIT, SENTINEL_LOCAL, D, _xlane_max4 +from ..score_prefill.kernel import ( + PERM_EVEN, + PERM_ODD, + _bits, + _dequant8, + _global_i32_ptr, + _maxf_nn, + _mfma_fp8_16x16x128, + _pack8, + _perm, + _xlane_max4_pair, +) +from .kernel import CNT_MERGE, CNT_SCORE, CNT_WORDS, F_IQ, FLAG_BYTES, HEADS, MAX_REQS, SC1, SYS, WORLD, _flag_read + +NW = 4 # waves per workgroup: quarter w of every run +RUN = 16 # this rank's blocks per run +QD = RUN // NW # blocks per quarter +WIN = 64 # pages per block-table window (one VGPR) +NR_LOG = MAX_REQS.bit_length() - 1 +MAX_QL = 8 + + +def _uni(x): + return fx.Int32(fx.rocdl.readfirstlane(T.i32, fx.Int32(x))) + + +def _min(a, b): + return (a < b).select(a, b) + + +def _max(a, b): + return (a > b).select(a, b) + + +def _wave_scan(x, lane, steps): + """Inclusive prefix sum through the first 2**steps lanes.""" + for d in range_constexpr(steps): + y = fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane - (1 << d)) * 4, x)) + x = x + (lane >= (1 << d)).select(y, fx.Int32(0)) + return x + + +@functools.cache +@launcher +def compile_score_half(ql: int): + """The scorer of ``ql`` (4 or 8) rows per request.""" + assert ql in (4, 8) + QL = ql + NG = HEADS * QL // 16 # groups of four rows: every column is a real (head, row) + QB = QL * 2 * D # bytes of a request's index-q rows on one rank + name = f"nvfp4_fp8mfma_index_score_half_ranks2_d128_p128_r{QL}h4" + + @fx.struct + class Shared: + lut: fx.Array[fx.Int32, 512, 16] # per scale byte: keys of magnitudes 0..3, 4..7 + incl: fx.Array[fx.Int32, MAX_REQS] # inclusive prefix of the run counts + inclp: fx.Array[fx.Int32, MAX_REQS] # inclusive prefix of the page counts + seq: fx.Array[fx.Int32, MAX_REQS] + wsum: fx.Array[fx.Int32, 2 * NW] + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_iq: fx.Int64, # this rank's index-q [num_reqs * QL, 2, 128] e4m3 + arg_qrcv: fx.Int64, # this rank's receive buffer of the peer's rows [rmax, 2, 128] + arg_qpeer: fx.Int64, # the peer's receive buffer + arg_kv: fx.Int64, # NVFP4 index pages [num_blocks, 128, 72] u8 + arg_lut: fx.Int64, # nvfp4_e4m3_lut [256] i64 + arg_score: fx.Int64, # [4, num_reqs * QL, S] fp32, half order + arg_bt: fx.Int64, # block table [num_reqs, >= max_blocks] i32 + arg_seq: fx.Int64, # seq_lens [num_reqs] i32 + arg_cnt: fx.Int64, # device counters (kernel.CNT_*) + arg_fmine: fx.Int64, # this rank's flag words + arg_fpeer: fx.Int64, # the peer's + i32_rmax: fx.Int32, + i64_page_stride: fx.Int64, + i32_s_head: fx.Int32, # score stride(0), stride(1) (elements) + i32_s_row: fx.Int32, + i64_s_bytes: fx.Int64, + i32_bt_stride: fx.Int32, + i64_bt_bytes: fx.Int64, + i32_num_reqs: fx.Int32, + i32_init: fx.Int32, # leading blocks forced in (sentinel 1e30) + i32_local: fx.Int32, # trailing blocks forced in (sentinel 1e29) + i32_me: fx.Int32, + i32_nwg: fx.Int32, # workgroups (the grid) + i32_pages_runs: fx.Int32, # balanced page ranges up to this many runs, else runs + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = _uni(tx // 64) + col, kl = lane % 16, lane // 16 + st_qoff = col // HEADS + st_head = col % HEADS + own = (st_head >> 1) == i32_me + hl = st_head & 1 + incl_l, seq_l = smem.incl.ptr, smem.seq.ptr + + cr = buffer_ops.create_buffer_resource_from_addr(arg_cnt, num_records_bytes=CNT_WORDS * 4) + calls = _uni(buffer_ops.buffer_load(cr, CNT_SCORE, vec_width=1, dtype=fx.Int32, cache_modifier=SC1)) + 1 + if (pid == 0) & (tx == 0): + buffer_ops.buffer_store(calls, cr, CNT_MERGE, cache_modifier=SYS) + btr = buffer_ops.create_buffer_resource_from_addr(arg_bt, num_records_bytes=i64_bt_bytes) + qbytes = fx.Int64(i32_rmax) * (2 * D) + ir = buffer_ops.create_buffer_resource_from_addr(arg_iq, num_records_bytes=fx.Int64(i32_num_reqs) * QB) + qr = buffer_ops.create_buffer_resource_from_addr(arg_qrcv, num_records_bytes=qbytes) + qp = buffer_ops.create_buffer_resource_from_addr(arg_qpeer, num_records_bytes=qbytes) + sr = buffer_ops.create_buffer_resource_from_addr(arg_score, num_records_bytes=i64_s_bytes) + + # this rank's index-q rows of requests pid, pid + nwg, ... to the peer + rs = pid + while (wave == 0) & (rs < i32_num_reqs): + for c in range_constexpr(QB // 1024): + v = buffer_ops.buffer_load(ir, (rs * QB + c * 1024 + lane * 16) // 4, vec_width=4, dtype=fx.Int32) + buffer_ops.buffer_store(v, qp, (rs * QB + c * 1024 + lane * 16) // 4, cache_modifier=SYS) + rs = rs + i32_nwg + + # e4m3 table; per request its runs of this rank's blocks, prefix-summed + lut_g = _global_i32_ptr(arg_lut) + lut_l = smem.lut.ptr + for i in range_constexpr(2): + lut_l[tx * 2 + i] = fx.Int32(lut_g[tx * 2 + i]) + r_ok = tx < i32_num_reqs + seq_t = fx.Int32(_global_i32_ptr(arg_seq)[r_ok.select(tx, fx.Int32(0))]) + seq_t = r_ok.select(seq_t, fx.Int32(0)) + seq_t = (seq_t < 0).select(fx.Int32(0), seq_t) + nb_t = (seq_t + (BLK - 1)) // BLK + mine_t = (nb_t > i32_me).select((nb_t - i32_me + 1) >> 1, fx.Int32(0)) + runs_t = (mine_t + (RUN - 1)) // RUN + x = _wave_scan(runs_t, lane, 6) + xp = _wave_scan(mine_t, lane, 6) + if lane == 63: + smem.wsum.ptr[wave] = x + smem.wsum.ptr[NW + wave] = xp + seq_l[tx] = seq_t + fx.gpu.barrier() + for w in range_constexpr(NW - 1): + x = x + (wave > w).select(fx.Int32(smem.wsum.ptr[w]), fx.Int32(0)) + xp = xp + (wave > w).select(fx.Int32(smem.wsum.ptr[NW + w]), fx.Int32(0)) + incl_l[tx] = x + smem.inclp.ptr[tx] = xp + fx.gpu.barrier() + + total = _uni(incl_l[MAX_REQS - 1]) + pages_mode = total <= i32_pages_runs + + lut8 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.lut.ptr), fx.make_layout(512, 1)), + fx.make_layout(2, 1), + ) + lut_atom = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + + def lut_load(sbyte): + r = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + fx.copy(lut_atom, fx.slice(lut8, (None, sbyte)), r) + v = Vec(r.load()) + return fx.Int32(v[0]), fx.Int32(v[1]) + + code_off = (col * 64 + kl * 16) // 4 # dwords + scale_off = col * 2 + kl // 2 # dwords + scale_sel = (kl % 2) * 2 + 0x0C0C0C00 + + def page_rsrc(page): + return buffer_ops.create_buffer_resource_from_addr( + arg_kv + fx.Int64(page) * i64_page_stride, num_records_bytes=PAGE_BYTES + ) + + def load_tile(kr, ok, t): + """Tile t of a page: this lane's 16 B of codes (token 16 t + col, dims 32 kl ..) and its group scales.""" + c = Vec( + buffer_ops.buffer_load( + kr, code_off, vec_width=4, dtype=fx.Int32, mask=ok, soffset_bytes=t * 1024, cache_modifier=NT + ) + ) + sc = fx.Int32( + buffer_ops.buffer_load( + kr, scale_off, vec_width=1, dtype=fx.Int32, mask=ok, soffset_bytes=PAGE_CODES + t * 128, + cache_modifier=NT, + ) + ) + return c, sc + + def dequant_tile(c, sc): + t0 = lut_load(_perm(sc, sc, scale_sel)) + t1 = lut_load(_perm(sc, sc, scale_sel + 1)) + f = [] + for e in range_constexpr(4): + lo, hi = _dequant8(fx.Int32(Vec(c)[e]), *(t0 if e < 2 else t1)) + f += [lo, hi] + return Vec.from_elements(f, fx.Int32) + + def q_or(xs, ys): + return [ + Vec.from_elements([fx.Int32(Vec(x)[e]) | fx.Int32(Vec(y)[e]) for e in range_constexpr(4)], fx.Int32) + for x, y in zip(xs, ys) + ] + + def q_part(r, ok, peer): + """Per group this lane's 32 B of Q (row r * QL + 4 g + st_qoff, head st_head, dims 32 kl ..), raw, from + this rank's rows (``peer`` False) or the peer's; the other heads' columns read zeros (masked loads).""" + out = [] + src, m = (qr, ok & ~own) if peer else (ir, ok & own) + for g in range_constexpr(NG): + off = (((r * QL + 4 * g + st_qoff) * 2 + hl) * D + kl * 32) // 4 + for hf in range_constexpr(2): + out.append(Vec(buffer_ops.buffer_load(src, off + hf * 4, vec_width=4, dtype=fx.Int32, mask=m))) + return out + + def q_load(r, ok): + return q_or(q_part(r, ok, False), q_part(r, ok, True)) + + def q_frag(a, b): + halves = [] + for v in (a, b): + d = [fx.Int32(Vec(v)[e]) for e in range_constexpr(4)] + halves.append( + Vec.from_elements( + [ + _perm(d[1], d[0], PERM_EVEN), + _perm(d[1], d[0], PERM_ODD), + _perm(d[3], d[2], PERM_EVEN), + _perm(d[3], d[2], PERM_ODD), + ], + fx.Int32, + ) + ) + return _pack8(halves[0], halves[1]) + + zero4 = fx.Vector.filled(4, 0.0, fx.Float32) + neg_inf = fx.Float32(float("-inf")) + sent_init = fx.Float32(SENTINEL_INIT) + sent_local = fx.Float32(SENTINEL_LOCAL) + + def reduce(v, t, run, masked, lim): + for e in range_constexpr(4): + x = v[e] + if const_expr(masked): + x = (lim >= t * 16 + e).select(x, neg_inf) + run = _maxf_nn(run, x) + return run + + def page_pass(cs, ss, qf0, qf1, krn, okn, masked, lim): + """Per group the max over the page's tokens of this lane's column; tile t of the next page loads into tile + t's freed registers. Only the reduction depends on the causal mask.""" + accs = [] + nxt = [] + for t in range_constexpr(MT): + f = dequant_tile(cs[t], ss[t]) + nxt.append(load_tile(krn, okn, t)) + if const_expr(NG == 2): + accs.append([_mfma_fp8_16x16x128(f, qf0, zero4), _mfma_fp8_16x16x128(f, qf1, zero4)]) + else: + accs.append([_mfma_fp8_16x16x128(f, qf0, zero4)]) + fx.rocdl.sched_barrier(0) + mx0 = neg_inf + mx1 = neg_inf + if masked: + run0 = neg_inf + run1 = neg_inf + for t in range_constexpr(MT): + run0 = reduce(accs[t][0], t, run0, True, lim) + if const_expr(NG == 2): + run1 = reduce(accs[t][1], t, run1, True, lim + 4) + if const_expr(NG == 2): + mx0, mx1 = _xlane_max4_pair(run0, run1) + else: + mx0 = _xlane_max4(run0) + else: + run0 = neg_inf + run1 = neg_inf + for t in range_constexpr(MT): + run0 = reduce(accs[t][0], t, run0, False, None) + if const_expr(NG == 2): + run1 = reduce(accs[t][1], t, run1, False, None) + if const_expr(NG == 2): + mx0, mx1 = _xlane_max4_pair(run0, run1) + else: + mx0 = _xlane_max4(run0) + return mx0, mx1, nxt + + def row_value(mx, seq, q_off, b): + """AITER's sentinels on block b of the row at q_off (a block both init and local is local).""" + v1 = seq - QL + q_off + 1 + rb = ((v1 < 0).select(fx.Int32(0), v1) + (BLK - 1)) // BLK + l0 = rb - i32_local + local0 = (l0 < 0).select(fx.Int32(0), l0) + vis = b < rb + v = (vis & (b < i32_init)).select(sent_init, mx) + return (vis & (b >= local0)).select(sent_local, v) + + # the index-q flags follow the pushed rows + fx.rocdl.s_waitcnt(vmcnt=0) + fpr = buffer_ops.create_buffer_resource_from_addr(arg_fpeer, num_records_bytes=FLAG_BYTES) + rs = pid + while (wave == 0) & (rs < i32_num_reqs): + buffer_ops.buffer_store(calls, fpr, F_IQ + rs, mask=lane == 0, cache_modifier=SYS) + rs = rs + i32_nwg + + def scan(): + """Runs mode: the pages of this workgroup's runs, wave w its quarter w.""" + items = total + nitems = (pid < items).select((items - pid + i32_nwg - 1) // i32_nwg, 0) + nslots = nitems * QD + w0 = fx.Int32(0) + while w0 < nslots: + s = w0 + lane + k = pid + (s >> 2) * i32_nwg + rj = fx.Int32(0) + for bs in range_constexpr(NR_LOG): + step = MAX_REQS >> (bs + 1) + rj = (fx.Int32(incl_l[rj + step - 1]) <= k).select(rj + step, rj) + valid = s < nslots + rj = valid.select(rj, fx.Int32(0)) + excl = (rj > 0).select(fx.Int32(incl_l[_max(rj - 1, 0)]), fx.Int32(0)) + seqj = fx.Int32(seq_l[rj]) + nblkj = (seqj + (BLK - 1)) // BLK + in_run = wave * QD + (s & 3) + jj = (k - excl) * RUN + in_run # half-order position + bj = jj * WORLD + i32_me + frow = seqj - (QL - 1) + rbf = ((frow < 0).select(fx.Int32(0), frow) + (BLK - 1)) // BLK + ll = nblkj - i32_local + loc_last = (ll < 0).select(fx.Int32(0), ll) + # blocks every row sees whole: init / local ones only need their sentinel + skip = (bj < rbf) & ((bj < i32_init) | (bj >= loc_last)) + need = valid & (bj < nblkj) & ~skip + pgj = fx.Int32( + buffer_ops.buffer_load( + btr, need.select(rj * i32_bt_stride + bj, fx.Int32(0)), vec_width=1, dtype=fx.Int32 + ) + ) + pgj = need.select(pgj, fx.Int32(-1)) + n = _min(nslots - w0, WIN) + + p_0 = fx.Int32(fx.rocdl.readlane(T.i32, pgj, 0)) + r_0 = fx.Int32(fx.rocdl.readlane(T.i32, rj, 0)) + ok0 = p_0 >= 0 + kr0 = page_rsrc(ok0.select(p_0, fx.Int32(0))) + fx.rocdl.sched_barrier(0) + c0, s0 = load_tile(kr0, ok0, 0) + fx.rocdl.sched_barrier(0) + c1, s1 = load_tile(kr0, ok0, 1) + fx.rocdl.sched_barrier(0) + c2, s2 = load_tile(kr0, ok0, 2) + fx.rocdl.sched_barrier(0) + c3, s3 = load_tile(kr0, ok0, 3) + fx.rocdl.sched_barrier(0) + c4, s4 = load_tile(kr0, ok0, 4) + fx.rocdl.sched_barrier(0) + c5, s5 = load_tile(kr0, ok0, 5) + fx.rocdl.sched_barrier(0) + c6, s6 = load_tile(kr0, ok0, 6) + fx.rocdl.sched_barrier(0) + c7, s7 = load_tile(kr0, ok0, 7) + fx.rocdl.sched_barrier(0) + # this rank's Q of the first page, then the window's requests' index-q rows from the peer: lane i + # polls its page's request + q_own = q_part(r_0, n > 0, False) + want = lane < n + word = arg_fmine + fx.Int64((F_IQ + rj) * 4) + f = calls + if want: + f = _flag_read(word) + while fx.Int64(fx.rocdl.ballot(T.i64, want & (f < calls))) != 0: + if want & (f < calls): + f = _flag_read(word) + qr0 = q_or(q_own, q_part(r_0, n > 0, True)) + fx.rocdl.s_waitcnt(vmcnt=0) + qa0, qb0 = qr0[0], qr0[1] + qa1, qb1 = qr0[-2], qr0[-1] + qf0 = q_frag(qa0, qb0) + qf1 = qf0 + if const_expr(NG == 2): + qf1 = q_frag(qa1, qb1) + qna0, qnb0, qna1, qnb1 = qa0, qb0, qa1, qb1 + sw = n < 0 # false: Q of the next page already loaded + v00 = neg_inf + v01 = neg_inf + v02 = neg_inf + v03 = neg_inf + v10 = neg_inf + v11 = neg_inf + v12 = neg_inf + v13 = neg_inf + j = fx.Int32(0) + while j < n: + r = fx.Int32(fx.rocdl.readlane(T.i32, rj, j)) + b = fx.Int32(fx.rocdl.readlane(T.i32, bj, j)) + seq = fx.Int32(fx.rocdl.readlane(T.i32, seqj, j)) + if sw: + qf0 = q_frag(qna0, qnb0) + if const_expr(NG == 2): + qf1 = q_frag(qna1, qnb1) + jn = j + 1 + okj = jn < n + jc = okj.select(jn, fx.Int32(0)) + pgn = fx.Int32(fx.rocdl.readlane(T.i32, pgj, jc)) + rn = fx.Int32(fx.rocdl.readlane(T.i32, rj, jc)) + okn = okj & (pgn >= 0) + krn = page_rsrc(okn.select(pgn, fx.Int32(0))) + sw = okj & (rn != r) + if sw: + qrn = q_load(rn, sw) + qna0, qnb0 = qrn[0], qrn[1] + if const_expr(NG == 2): + qna1, qnb1 = qrn[2], qrn[3] + fx.rocdl.sched_barrier(0) + res = page_pass( + (c0, c1, c2, c3, c4, c5, c6, c7), + (s0, s1, s2, s3, s4, s5, s6, s7), + qf0, + qf1, + krn, + okn, + b * BLK + (BLK - 1) > seq - QL, + seq - QL + st_qoff - b * BLK - kl * 4, + ) + c0, s0 = res[2][0] + c1, s1 = res[2][1] + c2, s2 = res[2][2] + c3, s3 = res[2][3] + c4, s4 = res[2][4] + c5, s5 = res[2][5] + c6, s6 = res[2][6] + c7, s7 = res[2][7] + ov0 = row_value(res[0], seq, st_qoff, b) + ov1 = ov0 + if const_expr(NG == 2): + ov1 = row_value(res[1], seq, st_qoff + 4, b) + pos = b >> 1 + nb = (seq + (BLK - 1)) // BLK + offs = [ + st_head * i32_s_head + (r * QL + 4 * g + st_qoff) * i32_s_row + pos for g in range_constexpr(NG) + ] + e = (w0 + j) & 3 + v00 = (e == 0).select(ov0, v00) + v01 = (e == 1).select(ov0, v01) + v02 = (e == 2).select(ov0, v02) + v03 = (e == 3).select(ov0, v03) + if const_expr(NG == 2): + v10 = (e == 0).select(ov1, v10) + v11 = (e == 1).select(ov1, v11) + v12 = (e == 2).select(ov1, v12) + v13 = (e == 3).select(ov1, v13) + if e == QD - 1: + # the quarter's 4 positions of a (head, row) as 16 B (past the row's blocks: unspecified) + vsets = [[v00, v01, v02, v03], [v10, v11, v12, v13]] + for g in range_constexpr(NG): + buffer_ops.buffer_store( + Vec.from_elements([_bits(vv) for vv in vsets[g]], fx.Int32), + sr, + offs[g] - 3, + mask=(kl == 0) & (b - 3 * WORLD < nb), + ) + j = jn + w0 = w0 + WIN + + def scan_pages(): + """Pages mode: the rank's pages of all requests in order, wave gw = pid * NW + wave a contiguous range of + them (balanced to one page).""" + totp = _uni(smem.inclp.ptr[MAX_REQS - 1]) + nwv = i32_nwg * NW + gw = pid * NW + wave + qd = totp // nwv + rem = totp - qd * nwv + start = gw * qd + _min(gw, rem) + cnt = qd + (gw < rem).select(fx.Int32(1), fx.Int32(0)) + w0 = fx.Int32(0) + while w0 < cnt: + s = w0 + lane + gpg = start + s + rj = fx.Int32(0) + for bs in range_constexpr(NR_LOG): + step = MAX_REQS >> (bs + 1) + rj = (fx.Int32(smem.inclp.ptr[rj + step - 1]) <= gpg).select(rj + step, rj) + valid = s < cnt + rj = valid.select(rj, fx.Int32(0)) + excl = (rj > 0).select(fx.Int32(smem.inclp.ptr[_max(rj - 1, 0)]), fx.Int32(0)) + seqj = fx.Int32(seq_l[rj]) + nblkj = (seqj + (BLK - 1)) // BLK + jj = gpg - excl + bj = jj * WORLD + i32_me + frow = seqj - (QL - 1) + rbf = ((frow < 0).select(fx.Int32(0), frow) + (BLK - 1)) // BLK + ll = nblkj - i32_local + loc_last = (ll < 0).select(fx.Int32(0), ll) + skip = (bj < rbf) & ((bj < i32_init) | (bj >= loc_last)) + need = valid & (bj < nblkj) & ~skip + pgj = fx.Int32( + buffer_ops.buffer_load( + btr, need.select(rj * i32_bt_stride + bj, fx.Int32(0)), vec_width=1, dtype=fx.Int32 + ) + ) + pgj = need.select(pgj, fx.Int32(-1)) + n = _min(cnt - w0, WIN) + + p_0 = fx.Int32(fx.rocdl.readlane(T.i32, pgj, 0)) + r_0 = fx.Int32(fx.rocdl.readlane(T.i32, rj, 0)) + ok0 = p_0 >= 0 + kr0 = page_rsrc(ok0.select(p_0, fx.Int32(0))) + fx.rocdl.sched_barrier(0) + c0, s0 = load_tile(kr0, ok0, 0) + fx.rocdl.sched_barrier(0) + c1, s1 = load_tile(kr0, ok0, 1) + fx.rocdl.sched_barrier(0) + c2, s2 = load_tile(kr0, ok0, 2) + fx.rocdl.sched_barrier(0) + c3, s3 = load_tile(kr0, ok0, 3) + fx.rocdl.sched_barrier(0) + c4, s4 = load_tile(kr0, ok0, 4) + fx.rocdl.sched_barrier(0) + c5, s5 = load_tile(kr0, ok0, 5) + fx.rocdl.sched_barrier(0) + c6, s6 = load_tile(kr0, ok0, 6) + fx.rocdl.sched_barrier(0) + c7, s7 = load_tile(kr0, ok0, 7) + fx.rocdl.sched_barrier(0) + q_own = q_part(r_0, n > 0, False) + want = lane < n + word = arg_fmine + fx.Int64((F_IQ + rj) * 4) + f = calls + if want: + f = _flag_read(word) + while fx.Int64(fx.rocdl.ballot(T.i64, want & (f < calls))) != 0: + if want & (f < calls): + f = _flag_read(word) + qr0 = q_or(q_own, q_part(r_0, n > 0, True)) + fx.rocdl.s_waitcnt(vmcnt=0) + qa0, qb0 = qr0[0], qr0[1] + qa1, qb1 = qr0[-2], qr0[-1] + qf0 = q_frag(qa0, qb0) + qf1 = qf0 + if const_expr(NG == 2): + qf1 = q_frag(qa1, qb1) + qna0, qnb0, qna1, qnb1 = qa0, qb0, qa1, qb1 + sw = n < 0 + j = fx.Int32(0) + while j < n: + r = fx.Int32(fx.rocdl.readlane(T.i32, rj, j)) + b = fx.Int32(fx.rocdl.readlane(T.i32, bj, j)) + seq = fx.Int32(fx.rocdl.readlane(T.i32, seqj, j)) + if sw: + qf0 = q_frag(qna0, qnb0) + if const_expr(NG == 2): + qf1 = q_frag(qna1, qnb1) + jn = j + 1 + okj = jn < n + jc = okj.select(jn, fx.Int32(0)) + pgn = fx.Int32(fx.rocdl.readlane(T.i32, pgj, jc)) + rn = fx.Int32(fx.rocdl.readlane(T.i32, rj, jc)) + okn = okj & (pgn >= 0) + krn = page_rsrc(okn.select(pgn, fx.Int32(0))) + sw = okj & (rn != r) + if sw: + qrn = q_load(rn, sw) + qna0, qnb0 = qrn[0], qrn[1] + if const_expr(NG == 2): + qna1, qnb1 = qrn[2], qrn[3] + fx.rocdl.sched_barrier(0) + res = page_pass( + (c0, c1, c2, c3, c4, c5, c6, c7), + (s0, s1, s2, s3, s4, s5, s6, s7), + qf0, + qf1, + krn, + okn, + b * BLK + (BLK - 1) > seq - QL, + seq - QL + st_qoff - b * BLK - kl * 4, + ) + c0, s0 = res[2][0] + c1, s1 = res[2][1] + c2, s2 = res[2][2] + c3, s3 = res[2][3] + c4, s4 = res[2][4] + c5, s5 = res[2][5] + c6, s6 = res[2][6] + c7, s7 = res[2][7] + ov0 = row_value(res[0], seq, st_qoff, b) + ov1 = ov0 + if const_expr(NG == 2): + ov1 = row_value(res[1], seq, st_qoff + 4, b) + ovs = [ov0, ov1] + pos = b >> 1 + for g in range_constexpr(NG): + off = st_head * i32_s_head + (r * QL + 4 * g + st_qoff) * i32_s_row + pos + buffer_ops.buffer_store(ovs[g], sr, off, mask=kl == 0) + j = jn + w0 = w0 + WIN + + if pages_mode: + scan_pages() + else: + scan() + + if (pid == 0) & (tx == 0): + word0 = arg_fmine + fx.Int64(F_IQ * 4) + f0 = _flag_read(word0) + while f0 < calls: + f0 = _flag_read(word0) + + @flyc.jit + def launch( + arg_iq: fx.Int64, + arg_qrcv: fx.Int64, + arg_qpeer: fx.Int64, + arg_kv: fx.Int64, + arg_lut: fx.Int64, + arg_score: fx.Int64, + arg_bt: fx.Int64, + arg_seq: fx.Int64, + arg_cnt: fx.Int64, + arg_fmine: fx.Int64, + arg_fpeer: fx.Int64, + i32_rmax: fx.Int32, + i64_page_stride: fx.Int64, + i32_s_head: fx.Int32, + i32_s_row: fx.Int32, + i64_s_bytes: fx.Int64, + i32_bt_stride: fx.Int32, + i64_bt_bytes: fx.Int64, + i32_num_reqs: fx.Int32, + i32_init: fx.Int32, + i32_local: fx.Int32, + i32_me: fx.Int32, + i32_nwg: fx.Int32, + i32_pages_runs: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_iq, + arg_qrcv, + arg_qpeer, + arg_kv, + arg_lut, + arg_score, + arg_bt, + arg_seq, + arg_cnt, + arg_fmine, + arg_fpeer, + i32_rmax, + i64_page_stride, + i32_s_head, + i32_s_row, + i64_s_bytes, + i32_bt_stride, + i64_bt_bytes, + i32_num_reqs, + i32_init, + i32_local, + i32_me, + i32_nwg, + i32_pages_runs, + ).launch(grid=(fx.Int64(i32_nwg), 1, 1), block=(64 * NW, 1, 1), stream=stream) + + launch.kernel_name = name + return launch diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/host.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/host.py new file mode 100644 index 0000000..2508990 --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/host.py @@ -0,0 +1,157 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Buffers and launches of the decode indexer split over two GPUs (TP2, gfx950). + +``IndexSplit2`` is made by a collective (both ranks at the same point, outside graph capture) and owns, in uncached +memory the peer maps: the index-q receive buffer [max_rows, 2, 128] e4m3, the partial top-16 receive buffer +[2, max_rows, 16] x (key, count, block, count) int32 and the flag words; plus device call counters (graph-safe) and +the half's ids scratch. A call is two launches: ``decode_score.py`` (scores of all four heads over this rank's blocks, +index-q exchanged inside), then ``select_merge.py`` (partial top 16, exchange, merge into this rank's two heads). +""" + +import torch +import torch.distributed as dist +from amd_kernels.warm_up import entry + +from .. import hip_ipc +from ..host_stream import current_stream as _current_stream +from ..reference import score_width +from . import kernel as _kernel + +WORLD = _kernel.WORLD +# two workgroups per CU; page ranges to 1.25 runs per workgroup (180K: -4..-11% vs runs at 12 / 14 requests, 48 runs) +SCORE_GRID = 512 +PAGES_RUNS = 640 +# Tests / benchmarks: pin the scorer's grid and page-range bound (0: runs always). +PIN_GRID: int | None = None +PIN_PAGES_RUNS: int | None = None + + +def _run_compiled(exe, *args): + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +def _end_bytes(t: torch.Tensor) -> int: + return (sum((n - 1) * s for n, s in zip(t.shape, t.stride())) + 1) * t.element_size() + + +class _DevicePointer: + """A raw device allocation as a tensor (``__cuda_array_interface__``).""" + + def __init__(self, ptr: int, nbytes: int): + self.__cuda_array_interface__ = {"shape": (nbytes,), "typestr": "|u1", "data": (ptr, False), "version": 3} + + +def _exchange(obj, rank: int, group) -> list: + out = [[None] for _ in range(WORLD)] + out[rank][0] = obj + for i, r in enumerate(sorted(dist.get_process_group_ranks(group=group))): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +class IndexSplit2: + """This rank's half of the two-GPU decode indexer for up to ``max_rows`` rows (``query_len`` per request, 4 or 8) + and ``max_width`` score positions (``reference.score_width`` of the longest context).""" + + def __init__(self, rank: int, group, device: torch.device, max_rows: int, max_width: int, query_len: int): + assert dist.get_world_size(group) == WORLD + assert query_len in (4, 8), query_len + assert max_rows <= _kernel.MAX_ROWS and max_rows // query_len <= _kernel.MAX_REQS, max_rows + self.rank, self.device = rank, torch.device(device) + self.max_rows, self.max_width, self.query_len = max_rows, max_width, query_len + qbytes = max_rows * 2 * _kernel.D + lbytes = 2 * max_rows * 16 * 16 + mine = [hip_ipc.allocate_shared_buffer_and_handle(n) for n in (qbytes, lbytes, _kernel.FLAG_BYTES)] + peers = _exchange([h for _, h in mine], rank, group) + other = 1 - rank + self.q_ptrs = [mine[0][0], hip_ipc.open_mem_handle(peers[other][0])] + self.l_ptrs = [mine[1][0], hip_ipc.open_mem_handle(peers[other][1])] + self.f_ptrs = [mine[2][0], hip_ipc.open_mem_handle(peers[other][2])] + torch.as_tensor(_DevicePointer(self.f_ptrs[0], _kernel.FLAG_BYTES), device=self.device).zero_() + self.counts = torch.zeros(_kernel.CNT_WORDS, dtype=torch.int32, device=self.device) + self.ids = torch.empty(_kernel.HEADS, max_rows, 16, dtype=torch.int32, device=self.device) + torch.accelerator.synchronize(self.device) + dist.barrier(group=group) + + def score_topk( + self, + iq: torch.Tensor, # [rows, 2, 128] e4m3, this rank's index heads + index_cache: torch.Tensor, # NVFP4 index pages [num_blocks, 128, 72] uint8 (replicated) + block_table: torch.Tensor, # [num_reqs, >= max_blocks] int32 + seq_lens: torch.Tensor, # [num_reqs] int32 + init_blocks: int, + local_blocks: int, + topk_idx: torch.Tensor, # [2, rows, 16] int32, written + max_seq_len: int, + ) -> None: + """This rank's two heads' decode top-16 block ids, bit for bit ``score_decode.nvfp4_index_score_top16`` on + one GPU; both ranks call it with the same lengths. Graph-safe (lengths are read on the device).""" + from ..score_prefill.host import nvfp4_e4m3_lut + from .decode_score import compile_score_half + from .select_merge import compile_select_merge + + rows = iq.shape[0] + ql = self.query_len + assert rows % ql == 0 and rows <= self.max_rows, (rows, ql, self.max_rows) + assert iq.dtype == torch.float8_e4m3fn and iq.shape[1:] == (2, _kernel.D) and iq.is_contiguous() + assert topk_idx.dtype == torch.int32 and topk_idx.shape[0] == 2 and topk_idx.shape[1] >= rows + assert topk_idx.stride(2) == 1 + width = score_width(max_seq_len) + assert width <= self.max_width, (width, self.max_width) + num_reqs = rows // ql + if num_reqs == 0: + return + half = max(64, width // 2) + score = torch.empty(_kernel.HEADS, rows, half, dtype=torch.float32, device=iq.device) + stream = _current_stream() + _run_compiled( + compile_score_half(ql), iq.data_ptr(), self.q_ptrs[0], self.q_ptrs[1], index_cache.data_ptr(), + nvfp4_e4m3_lut(iq.device).data_ptr(), score.data_ptr(), block_table.data_ptr(), seq_lens.data_ptr(), + self.counts.data_ptr(), self.f_ptrs[0], self.f_ptrs[1], self.max_rows, int(index_cache.stride(0)), + int(score.stride(0)), int(score.stride(1)), _end_bytes(score), int(block_table.stride(0)), + _end_bytes(block_table), num_reqs, int(init_blocks), int(local_blocks), self.rank, + PIN_GRID or SCORE_GRID, PAGES_RUNS if PIN_PAGES_RUNS is None else PIN_PAGES_RUNS, stream, + ) + _run_compiled( + compile_select_merge(), score.data_ptr(), int(score.stride(0)), int(score.stride(1)), half, + seq_lens.data_ptr(), ql, self.ids.data_ptr(), self.l_ptrs[0], self.l_ptrs[1], self.counts.data_ptr(), + topk_idx.data_ptr(), int(topk_idx.stride(0)), int(topk_idx.stride(1)), + self.max_rows, self.rank, rows, stream, + ) + + +def _mode(cp, iq, *args, **kwargs): + return cp.query_len, cp.max_rows // cp.query_len, cp.max_width, iq.device.index + + +def _sizes(mode): + _, max_reqs, _, _ = mode + return sorted({min(n, max_reqs) for n in (1, 64, max_reqs)}) + + +def _inputs(mode, n, cp, iq, index_cache, block_table, seq_lens, init_blocks, local_blocks, topk_idx, max_seq_len): + ql, _, width, _ = mode + rows, dev = n * ql, iq.device + args = ( + cp, + torch.zeros(rows, 2, 128, dtype=torch.float8_e4m3fn, device=dev), + index_cache, + torch.zeros(n, width, dtype=torch.int32, device=dev), + torch.full((n,), ql, dtype=torch.int32, device=dev), + init_blocks, local_blocks, + torch.empty(2, rows, 16, dtype=torch.int32, device=dev), + width * 128, + ) + return args, {} + + +IndexSplit2.score_topk = entry( + "indexer/nvfp4_index_score_top16_ranks2_d128_p128", mode=_mode, sizes=_sizes, inputs=_inputs, +)(IndexSplit2.score_topk) diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/kernel.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/kernel.py new file mode 100644 index 0000000..cad679b --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/kernel.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Flag words and device counters of the decode indexer split over two GPUs (``decode_score.py``, +``select_merge.py``; TP2, gfx950, FlyDSL). + +Flags carry the call count (graph-safe, never reset): a sender drains its sc0 sc1 stores, then stores the count into +the receiver's flag word; the receiver spins until the word reaches it (the merge's entries carry the count +themselves). The count lives on the device in two words: +the scorer reads ``CNT_SCORE`` and its workgroup 0 writes the count to ``CNT_MERGE``; the merge reads ``CNT_MERGE`` +and its workgroup 0 writes it back to ``CNT_SCORE``, so no launch rewrites a word its own workgroups read. +""" + +import flydsl.expr as fx +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr.typing import T + +WORLD = 2 +HEADS = 4 # index heads over both ranks; rank r owns heads 2 r, 2 r + 1 +SYS = 0x11 # sc0 | sc1: system-coherent +SC1 = 0x10 # load: coherent across the XCDs' L2s +D = 128 +MAX_REQS = 256 # one request per thread of the scorer's prefix sum +MAX_ROWS = MAX_REQS * 8 +# flag words per rank (uncached int32): [F_IQ + r] the peer's index-q rows of request r +F_IQ = 0 +FLAG_WORDS = MAX_REQS +FLAG_BYTES = FLAG_WORDS * 4 +CNT_SCORE, CNT_MERGE = 0, 1 +CNT_WORDS = 64 + + +def _gptr(addr): + return fx.inttoptr( + fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + fx.Int64(addr), + ) + + +def _flag_read(addr): + """An atomic add of 0 at system scope (never hoisted out of a spin loop).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _gptr(addr).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) diff --git a/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/select_merge.py b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/select_merge.py new file mode 100644 index 0000000..740c9e3 --- /dev/null +++ b/amd_kernels/indexer/indexer_nvfp4_d128_p128_top16/split2/select_merge.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Top-16 of the decode indexer split over two GPUs, after ``decode_score.py``: per (row, head) the top 16 of the +rank's half, exchanged as (key, block) pairs and merged (FlyDSL, gfx950). + +Grid (rows, 4): y 0-1 select the peer's heads 2 (1 - me) + y and store the 16 entries into the peer's receive buffer; +y 2-3 select this rank's heads 2 me + y - 2, spin on the peer's entries of the row and merge. An entry is (key, call +count, block, call count), 16 B from one lane: each 8-B half lands whole, so both counts equal to this call's say the +entry is complete (no flag word, no drain before it). The storing workgroups are dispatched first, so the spinning +ones never hold the CUs they need. Selection over the half is the plain four-wave selector +(``topk.top16_indices_p128_plain_wave4``) on the half order (block 2 j + me at j): its ids j go to blocks 2 j + me, +the order preserved. The key is the score's in AITER's order (NaN as -inf); the merged top 16 of the 32 by (key, +block) descending, -1 after the live count, is the one-GPU top 16 bit for bit (the halves are disjoint). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..topk.kernel import _comp, _key, _one +from ..topk.prefill_kernel import INT_MIN, KEY_NEG_INF, KEY_POS_INF, SEL_DWORDS, _rsrc +from ..topk.top16_indices_p128_plain_wave4 import CAND_DWORDS, first_window, row_top16_w4_ids +from .kernel import CNT_MERGE, CNT_SCORE, CNT_WORDS, SC1, SYS + + +@functools.cache +@launcher +def compile_select_merge(): + name = "index_top16_select_merge_ranks2_p128_w28w64_plain_wave4" + + @fx.struct + class Shared: + cand: fx.Array[fx.Int32, CAND_DWORDS, 16] + sel: fx.Array[fx.Int32, SEL_DWORDS, 16] + tmp: fx.Array[fx.Int32, 260, 16] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel( + arg_score: fx.Int64, # [4, rows, S] fp32, half order + s_head: fx.Int64, + s_row: fx.Int64, + width: fx.Int32, # positions per score row the selection reads + arg_seq: fx.Int64, + ql: fx.Int32, + arg_ids: fx.Int64, # scratch [4, rmax, 16] i32: the half's ids + arg_lrcv: fx.Int64, # this rank's receive buffer [2, rmax, 16] x (key, count, block, count) + arg_lpeer: fx.Int64, # the peer's + arg_cnt: fx.Int64, + arg_topk: fx.Int64, # this rank's heads [2, rows, 16] i32 + t_head: fx.Int64, + t_row: fx.Int64, + i32_rmax: fx.Int32, + i32_me: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + n, y, tx = fx.block_idx.x, fx.block_idx.y, fx.thread_idx.x + lane = tx % 64 + cr = buffer_ops.create_buffer_resource_from_addr(arg_cnt, num_records_bytes=CNT_WORDS * 4) + calls = fx.Int32( + fx.rocdl.readfirstlane( + T.i32, fx.Int32(buffer_ops.buffer_load(cr, CNT_MERGE, vec_width=1, dtype=fx.Int32, cache_modifier=SC1)) + ) + ) + if (n == 0) & (y == 0) & (tx == 0): + buffer_ops.buffer_store(calls, cr, CNT_SCORE, cache_modifier=SYS) + push = y < 2 + hl = y & 1 + head = push.select(fx.Int32(1) - i32_me, i32_me) * 2 + hl + row = arg_score + (head.to(fx.Int64) * s_head + n.to(fx.Int64) * s_row) * 4 + pre = first_window(tx, row, width) + seq = fx.Int32(buffer_ops.buffer_load( + _rsrc(arg_seq + (n // ql).to(fx.Int64) * 4, 4), fx.Int32(0), vec_width=1, is_scalar=True, + )) + kv0 = seq - ql + n % ql + 1 + nblk = ((kv0 > 0).select(kv0, fx.Int32(0)) + 127) // 128 + mine = (nblk > i32_me).select((nblk - i32_me + 1) >> 1, fx.Int32(0)) + ids_row = arg_ids + (head.to(fx.Int64) * i32_rmax.to(fx.Int64) + n.to(fx.Int64)) * 64 + fast = row_top16_w4_ids(tx, row, width, mine, smem.cand.ptr, smem.sel.ptr, smem.tmp.ptr, ids_row, pre) + j = fx.Int32(smem.tmp.ptr[lane & 15]) + if fast == 0: + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + j = fx.Int32(buffer_ops.buffer_load(_rsrc(ids_row, 64), lane & 15, vec_width=1, dtype=fx.Int32, + cache_modifier=SYS)) + + # lanes 0-15: this half's pairs, 16-31: the peer's (own heads); an entry is (key, calls, block, calls), each + # 8-B half written whole, so the count in both says the entry is this call's + lr = buffer_ops.create_buffer_resource_from_addr(arg_lrcv, num_records_bytes=fx.Int64(i32_rmax) * 512) + slot = (hl * i32_rmax + n) * 16 + (lane & 15) + ok = (lane < 16) & (j >= 0) + raw = fx.Int32(buffer_ops.buffer_load(_rsrc(row, fx.Int64(width) * 4), j, vec_width=1, dtype=fx.Int32, + mask=ok)) + k = _key(raw) + k = ((k > KEY_POS_INF) | (k < KEY_NEG_INF)).select(fx.Int32(KEY_NEG_INF), k) + key = ok.select(k, fx.Int32(INT_MIN)) + blk = ok.select(j * 2 + i32_me, fx.Int32(-1)) + if push: + if tx < 16: + lp = buffer_ops.create_buffer_resource_from_addr(arg_lpeer, num_records_bytes=fx.Int64(i32_rmax) * 512) + buffer_ops.buffer_store(Vec.from_elements([key, calls, blk, calls], fx.Int32), lp, slot * 4, + cache_modifier=SYS) + else: + if tx < 64: + peer = (lane >= 16) & (lane < 32) + pv = Vec(buffer_ops.buffer_load(lr, slot * 4, vec_width=4, dtype=fx.Int32, mask=peer, + cache_modifier=SYS)) + while fx.Int64(fx.rocdl.ballot( + T.i64, peer & ((fx.Int32(pv[1]) != calls) | (fx.Int32(pv[3]) != calls)))) != 0: + # the fence keeps the load in the loop + _llvm.FenceOp(_llvm.AtomicOrdering.acq_rel, syncscope="workgroup") + pv = Vec(buffer_ops.buffer_load(lr, slot * 4, vec_width=4, dtype=fx.Int32, mask=peer, + cache_modifier=SYS)) + key = peer.select(fx.Int32(pv[0]), key) + blk = peer.select(fx.Int32(pv[2]), blk) + live = blk >= 0 + me_c = _comp(key, blk) + rank = fx.Int32(0) + for m in range_constexpr(32): + km = fx.Int32(fx.rocdl.readlane(T.i32, key, m)) + bm = fx.Int32(fx.rocdl.readlane(T.i32, blk, m)) + rank = rank + _one(_comp(km, bm) > me_c) + smem.tmp.ptr[lane] = fx.Int32(-1) + fx.rocdl.s_waitcnt(lgkmcnt=0) + if live & (lane < 32) & (rank < 16): + smem.tmp.ptr[rank] = blk + fx.rocdl.s_waitcnt(lgkmcnt=0) + out = arg_topk + (hl.to(fx.Int64) * t_head + n.to(fx.Int64) * t_row) * 4 + if lane < 16: + buffer_ops.buffer_store(fx.Int32(smem.tmp.ptr[lane]), _rsrc(out, 64), lane) + + @flyc.jit + def launch( + arg_score: fx.Int64, + s_head: fx.Int64, + s_row: fx.Int64, + width: fx.Int32, + arg_seq: fx.Int64, + ql: fx.Int32, + arg_ids: fx.Int64, + arg_lrcv: fx.Int64, + arg_lpeer: fx.Int64, + arg_cnt: fx.Int64, + arg_topk: fx.Int64, + t_head: fx.Int64, + t_row: fx.Int64, + i32_rmax: fx.Int32, + i32_me: fx.Int32, + rows: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_score, s_head, s_row, width, arg_seq, ql, arg_ids, arg_lrcv, arg_lpeer, arg_cnt, arg_topk, t_head, t_row, i32_rmax, i32_me, + ).launch(grid=(fx.Int64(rows), 4, 1), block=(256, 1, 1), stream=stream) + + launch.kernel_name = name + launch.compile_hints = {"llvm_options": {"amdgpu-kernarg-preload": True, "amdgpu-kernarg-preload-count": 16}} + return launch diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/README.md b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/README.md new file mode 100644 index 0000000..e4182e9 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/README.md @@ -0,0 +1,60 @@ +# MXFP4 MoE with MXFP8 shared expert, H6144 I1536 E128 top4 (gfx950) + +## What it does + +One TP2 rank's MoE block of MiniMax-M3.1-Flash, FlyDSL only: bf16x3 split-K router + biased top-4 of 128 experts +(weights include the routed scale 2.0), MXFP4 x MXFP4 routed experts (I 1536 per rank, SwiGLU-OAI: alpha 1.702, +limit 7, up + 1), MXFP8 shared expert (the rank's I 1536, from a shard copy or read in place from the full I 3072 +copy). Caller: vLLM's MoE layer (decode graphs to 1024 rows, eager prefill). +Input: normed BF16 `[M, 6144]`, contiguous, M 1-87381. One quant rule at every M: MXFP4 with Quark-even e8m0 per 32 +for the routed input and GEMM1 output, E4M3 with e8m0 `2^ceil(log2(amax / 448))` per 32 for the shared input and FP32 +SwiGLU output; M picks only tiles, launches and sum order. Slot k = `topk_weights[:, k] * expert(x)` over the full +I 1536, rounded once to BF16; no output atomics, bitwise repeatable, graph replay = eager. Contract (`reference.py`): +`mxfp4_quant_ref`, `shared_mxfp8_quant_ref` bit for bit, `w4a4_moe_ref` within rel. L2 4e-3, `shared_mxfp8_ref`. +Router partials: K split and sum order follow M and the router kernel (weight err <= 5.1e-7). The one-launch GEMMs +share one per-device ready-counter arena, so two calls must not run concurrently on one device. + +## Entry points + +- `MoeWeights.from_row_major`: the router and routed experts' checkpoint tensors (and the `SharedWeights`, which + `mxfp4_moe` needs) in the kernel layouts, plus a 3 MiB line-layout gate copy; once, before graph capture. +- `SharedWeights.from_row_major`: the shared expert's MXFP8 weights and scales as a bit-exact permutation (AITER's + preshuffle): a TP2 shard, or the full expert with the TP rank whose shard the BF16 entry computes (`SHARED_FULL`). +- `unsupported_reason`: why a checkpoint's MoE tensors or activation do not fit (None: they fit). +- `mxfp4_moe`: the MoE block of a normed BF16 input or a prepared input: reduced, BF16 `[M, 6144]` (slots 0..3, then + shared, summed in FP32), or the partials (slots `[M, 4, 6144]`, shared `[M, 6144]`) for a fused slots collective. + An `amd_kernels.warm_up` entry: the first call of a mode compiles its variants at every `chain.WARM_UP_TOKENS` count. +- `warm_up_mxfp4_moe`: at start-up, loads every variant of a mode (`sp`: the SP modes too); one layer serves all. +- `prepare_mxfp4_moe`: routes and quantizes the local SP shard into a `MoeInput`; `MoeInputCompact` (7328 B/token) + replaces the FP4 codes by a 1-bit mask; `MoeInput.routed` (3296 B/token) gives the routed slots only. +- `gather_mxfp4_moe`: packs and gathers the compact (or with `routed_only` the routed) payload by a warmed + `AllGatherComm` of `world` ranks (default 2; at most 32768 gathered rows: 16384 per rank at two ranks). +- `shared_expert_full`: the full shared expert (I 3072) of prepared MXFP8 rows, on a side stream if asked (SP: the + rows' owner adds it in its reduce-scatter). +- `set_dbo_query`: the server's query whether a call runs inside a DBO micro-batch (vLLM: + `vllm.v1.worker.ubatching.dbo_enabled`); inside one the shared expert stays on the caller's stream. + +Dispatch is by M only, also under graph capture: `chain.py` picks the router, sort and expert host +(`experts_a4w4_small.py`, `experts_a4w4_bm128.py`, `experts_a4w4.py`), each host its kernels. + +## Performance + +One MI355X, synthetic weights (`benchmark_shared_moe.py --baseline`), shared expert on both sides: reduced entry vs +the block vLLM runs at TP2 without it (router GEMM + AITER biased top-k + AITER W4A4 `fused_moe` + Triton MXFP8 shared +MLP on a side stream), same inputs, same process; graph replay of 100 inputs (16 above 2048 rows), median of 7 x 10 +replays, us. Last row: the partials mode. Repeated A/Bs at 384-640 rows: -1.3..-10.5%. + +| M | 8 | 32 | 64 | 128 | 256 | 320 | 384 | 448 | 512 | 640 | 1024 | 2048 | 4096 | 8192 | 32768 | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| baseline | 97.3 | 215.8 | 294.5 | 327.1 | 351.0 | 357.6 | 364.2 | 377.3 | 373.6 | 401.2 | 437.4 | 537.9 | 821.6 | 1366.7 | 5362.5 | +| this entry | 79.9 | 189.9 | 253.8 | 301.1 | 324.6 | 342.7 | 353.9 | 364.4 | 372.7 | 395.2 | 407.7 | 451.7 | 656.6 | 1005.7 | 3256.0 | +| change | -17.9% | -12.0% | -13.8% | -8.0% | -7.5% | -4.2% | -2.8% | -3.4% | -0.2% | -1.5% | -6.8% | -16.0% | -20.1% | -26.4% | -39.3% | +| partials | 78.4 | | 254.6 | | 317.3 | | 343.0 | | 359.9 | | 402.1 | | 625.4 | | 2981.8 | + +## Tests + +```bash +python3 -m pytest tests/moe/mxfp4_h6144_i1536_e128_top4 -q # test_gather_ranks2.py: 2 GPUs +python3 benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_moe.py --baseline # entry vs the vLLM TP2 block +# also: benchmark_prepared.py, benchmark_sidecar.py, benchmark_baseline_moe.py (the baseline alone) +``` diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/__init__.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/__init__.py new file mode 100644 index 0000000..2fb8123 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/__init__.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP4 routed MoE H6144 I1536 E128 top4 with an optional MXFP8 shared expert, dispatched by token count (README).""" + +import torch + +from .prepared_input import MoeInput, MoeInputCompact, prepare_mxfp4_moe, gather_mxfp4_moe +from .shared_expert import SharedWeights, set_dbo_query, shared_expert_full +from .chain import MoeWeights, mxfp4_moe, warm_up_mxfp4_moe + +# the public entry; private names (DISPATCH, mxfp4_moe_experts, ...) stay in .chain +__all__ = ["MoeWeights", "SharedWeights", "mxfp4_moe", "warm_up_mxfp4_moe", "MoeInput", "MoeInputCompact", + "prepare_mxfp4_moe", "gather_mxfp4_moe", "set_dbo_query", "shared_expert_full", "unsupported_reason", + "SHARED_FULL"] + +H, I, E, TOPK = 6144, 1536, 128, 4 # noqa: E741 +SWIGLU_OAI = (1.702, 7.0, 1.0) # alpha, limit, beta (up + beta) +# SharedWeights takes the full shared expert (I 2 x 1536) and computes the TP2 shard by view; shared_expert_full +SHARED_FULL = True + + +def unsupported_reason(w13, w13_scale, w2, w2_scale, shared_w13, shared_s13, shared_w2, shared_s2, + swiglu=SWIGLU_OAI) -> str | None: + """None when ``MoeWeights.from_row_major`` (with ``SharedWeights``) takes these checkpoint-layout tensors: + MXFP4 routed experts [E, 2I, H/2] / [E, H, I/2] + e8m0 scales, MXFP8 shared expert [2I, H] / [H, I] + scales (the + TP2 shard) or [4I, H] / [H, 2I] + scales (the full expert), SwiGLU-OAI (alpha, limit, beta) as the kernels compute + it.""" + if any(abs(a - b) > 1e-6 for a, b in zip(swiglu, SWIGLU_OAI)): + return f"needs SwiGLU-OAI (alpha, limit, beta) {SWIGLU_OAI}, got {tuple(swiglu)}" + si = 2 * I if shared_w13 is not None and tuple(shared_w13.shape) == (4 * I, H) else I + for name, t, shape, dtype in ( + ("w13", w13, (E, 2 * I, H // 2), torch.uint8), ("w13_scale", w13_scale, (E, 2 * I, H // 32), torch.uint8), + ("w2", w2, (E, H, I // 2), torch.uint8), ("w2_scale", w2_scale, (E, H, I // 32), torch.uint8), + ("shared w13", shared_w13, (2 * si, H), torch.float8_e4m3fn), + ("shared w13 scale", shared_s13, (2 * si, H // 32), None), + ("shared w2", shared_w2, (H, si), torch.float8_e4m3fn), ("shared w2 scale", shared_s2, (H, si // 32), None), + ): + if t is None or tuple(t.shape) != shape or (dtype is not None and t.dtype != dtype): + got = None if t is None else (tuple(t.shape), t.dtype) + return f"{name}: needs {shape} {dtype or 'e8m0'}, got {got}" + return None diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/a16w4_utils.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/a16w4_utils.py new file mode 100644 index 0000000..271bb60 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/a16w4_utils.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# Copyright (C) 2025-2026 FlyDSL Project Contributors +"""Helpers of the MXFP4 MoE GEMMs: raw pointers, the agent-scope ready counters that order a fused GEMM1 -> GEMM2 +launch, and the inline-sort routing table.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import arith, gpu, range_constexpr, rocdl +from flydsl.expr.typing import T + +from .common.utils import _global_i32_at, _global_i32_ptr, _raw + +LOG2E = 1.4426950408889634 + + +# cache policy bit SC1 (device scope: write-through stores) and the bytes between two m-blocks' counters (ready +# count, then consumer count): each block in its own 256 B line, so the polls of waiting tiles spread over channels +SC1 = 16 +READY_STRIDE = 256 + + +def _global_atomic_agent(op, addr_i64, value): + ptr = _global_i32_ptr(addr_i64) + return fx.Int32( + llvm.AtomicRMWOp( + op, + ptr.llvm_ptr, + fx.Int32(value).ir_value(), + llvm.AtomicOrdering.monotonic, + syncscope="agent", + alignment=4, + ).result + ) + + +def global_atomic_add_agent(addr_i64, value): + """``old = *addr; *addr += value`` on a global i32, device (agent) scope, relaxed.""" + return _global_atomic_agent(llvm.AtomicBinOp.add, addr_i64, value) + + +@flyc.jit +def wait_ready(addr, need, consumers): + """A fused launch's GEMM2 tile: thread 0 spins until the m-block's ready count at ``addr`` reaches ``need`` (every + GEMM1 tile of the block stored its rows), then counts itself at ``addr + 4``; the block's last consumer resets + both words to 0 for the next call (every consumer has passed its wait by then, and no GEMM1 tile of the block is + left). The workgroup waits at a barrier. No acquire fence: the rows were stored write-through and no load of this + launch touched their lines before the wait; an agent acquire (``buffer_inv sc1``) per tile cost +13 us at 8 tokens + and +76 us at 32 (it invalidates the XCD's L2 each time).""" + if fx.thread_idx.x == 0: + got = global_atomic_add_agent(addr, 0) + while got < need: + rocdl.s_sleep(2) + got = global_atomic_add_agent(addr, 0) + old = global_atomic_add_agent(addr + fx.Int64(4), 1) + if old == consumers - 1: + _global_atomic_agent(llvm.AtomicBinOp.xchg, addr, 0) + _global_atomic_agent(llvm.AtomicBinOp.xchg, addr + fx.Int64(4), 0) + gpu.barrier() + + +def poll_ready(addr, cpol=SC1): + """Thread 0's first read of a ready counter (device scope; the other lanes' out-of-range offset drops theirs), + issued early so its latency overlaps the work item's setup; ``wait_ready_from`` consumes the value.""" + rsrc = buffer_ops.create_buffer_resource_from_addr(addr, num_records_bytes=8) + off = (fx.thread_idx.x == 0).select(fx.Int32(0), fx.Int32(0x7FFFFFF0)) + return fx.Int32( + llvm.call_intrinsic( + T.i32, + "llvm.amdgcn.raw.ptr.buffer.atomic.add.i32", + [_raw(fx.Int32(0)), _raw(rsrc), _raw(off), _raw(fx.Int32(0)), _raw(fx.Int32(cpol))], + [], + [], + ) + ) + + +@flyc.jit +def wait_ready_from(addr, need, consumers, got): + """``wait_ready`` whose first poll ``got`` (``poll_ready``) was issued earlier.""" + if fx.thread_idx.x == 0: + while got < need: + rocdl.s_sleep(2) + got = global_atomic_add_agent(addr, 0) + old = global_atomic_add_agent(addr + fx.Int64(4), 1) + if old == consumers - 1: + _global_atomic_agent(llvm.AtomicBinOp.xchg, addr, 0) + _global_atomic_agent(llvm.AtomicBinOp.xchg, addr + fx.Int64(4), 0) + gpu.barrier() + + +def inline_sort_max_pairs(n_tokens, topk, bm): + """Routing pairs the inline-sort table scans (64 per wave pass): 64 when n_tokens * topk <= 64, else bm * topk + while n_tokens <= bm, else every pair rounded up to whole passes; a trace-time int (part of the kernel name).""" + n, k = int(n_tokens), int(topk) + if n * k <= 64: + return 64 + if n <= int(bm): + return int(bm) * k + return (n * k + 63) // 64 * 64 + + +def inline_sort_multi_block(n_tokens, bm): + """More tokens than block rows: an expert's rows may span several m-blocks.""" + return int(n_tokens) > int(bm) + + +def inline_sort_table(arg_topk, i32_ntok, TOPK, p_i32, lane, tab, max_pairs=64, bm=16, + multi_block=False): + """Inline sort: each block builds its own sorted-ids table from ``topk_ids`` with a wave ballot, no sort kernel. + + Routing pair q = token * TOPK + slot (row-major topk_ids). Block p owns expert e = topk_ids[p] iff p is the first + pair with that expert; its rows are all pairs with expert e, in pair order (n_tokens <= bm: one m-block per + expert). ``tab`` (i32 LDS, ``bm + 1`` entries) holds token | slot << 24 per row, token = n_tokens for padding rows + (the sort's rows for this block); non-matching pairs are parked in slot ``bm``. Returns (expert id, owner, row + count, build_table); non-owner blocks (duplicate experts) must exit. Pairs are scanned 64 per wave pass; + ``max_pairs`` (``inline_sort_max_pairs``) sets the number of passes. + + ``multi_block`` (n_tokens > bm, trace-time): an expert can have more than ``bm`` rows. Block p owns iff the rank r + of pair p among its expert's pairs is a multiple of ``bm``, and its rows are that expert's ranks r .. r + bm - 1. + """ + n_chunks = (int(max_pairs) + 63) // 64 + n_pairs = i32_ntok * fx.Int32(TOPK) + last = n_pairs - fx.Int32(1) + p_lane = p_i32 % fx.Int32(64) + p_chunk = p_i32 // fx.Int32(64) + qs, pvs = [], [] + for c in range_constexpr(n_chunks): + q = lane + fx.Int32(c * 64) + idx = fx.Int32(arith.minsi(_raw(q), _raw(last))) + v = fx.Int32(_global_i32_at(arg_topk, idx)) + qs.append(q) + pvs.append((q < n_pairs).select(v, fx.Int32(-1))) + # my expert = value of pair p (uniform) + e = fx.Int32(rocdl.readlane(T.i32, _raw(pvs[0]), _raw(p_lane))) + for c in range_constexpr(1, n_chunks): + e_c = fx.Int32(rocdl.readlane(T.i32, _raw(pvs[c]), _raw(p_lane))) + e = (p_chunk == fx.Int32(c)).select(e_c, e) + base = fx.Int32(0) + rank_p = fx.Int32(0) + slots, fuseds, ranks, matches = [], [], [], [] + for c in range_constexpr(n_chunks): + is_match = pvs[c] == e + mask = rocdl.ballot(T.i64, _raw(is_match)) + mask_lo = arith.trunci(T.i32, mask) + mask_hi = arith.trunci(T.i32, arith.shrui(mask, arith.constant(32, type=T.i64))) + below = fx.Int32( + rocdl.mbcnt_hi( + T.i32, mask_hi, rocdl.mbcnt_lo(T.i32, mask_lo, _raw(fx.Int32(0))) + ) + ) + rank = base + below # row of pair q among this expert's pairs (pair order) + fuseds.append( + (qs[c] // fx.Int32(TOPK)) | ((qs[c] % fx.Int32(TOPK)) << fx.Int32(24)) + ) + if multi_block: + ranks.append(rank) + matches.append(is_match) + else: + slots.append(is_match.select(rank, fx.Int32(bm))) # non-matching -> slot bm + r_p = fx.Int32(rocdl.readlane(T.i32, _raw(rank), _raw(p_lane))) + rank_p = (p_chunk == fx.Int32(c)).select(r_p, rank_p) + # matches so far = rank at lane 63 + lane 63's own match bit + tot = rank + is_match.select(fx.Int32(1), fx.Int32(0)) + base = fx.Int32(rocdl.readlane(T.i32, _raw(tot), _raw(fx.Int32(63)))) + if multi_block: + # block p: ranks rank_p .. rank_p+bm-1 of its expert; other pairs -> slot bm + owner = (rank_p % fx.Int32(bm)) == fx.Int32(0) + for c in range_constexpr(n_chunks): + rel = ranks[c] - rank_p + mine = matches[c] & (rel >= fx.Int32(0)) & (rel < fx.Int32(bm)) + slots.append(mine.select(rel, fx.Int32(bm))) + else: + owner = rank_p == fx.Int32(0) + + def build_table(): + # padding sentinel in every row slot first (0..31 covers bm <= 32), then the rows + tab[lane % 32] = i32_ntok + gpu.barrier() + for c in range_constexpr(n_chunks): + tab[slots[c]] = fuseds[c] + gpu.barrier() + + # the kernel calls build_table() under `if owner:` (a uniform scf.if), so duplicate-expert blocks leave after the + # loads and ballots without the two barriers + return e, owner, base, build_table diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/chain.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/chain.py new file mode 100644 index 0000000..ca48eb1 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/chain.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Routed MXFP4 H6144 I1536 E128 top4: one entry, shape-based dispatch.""" +import os +from dataclasses import dataclass + +import torch + +from amd_kernels.warm_up import entry + +H, I, E, TOPK = 6144, 1536, 128, 4 +# upper token bound -> expert GEMM variant; one Quark-even MXFP4 activation quant at every M (small-tile host to 640 +# rows: -1.7..-2.6% at 576-640 vs the BM128 launch with the shared tiles, +13% at 768) +DISPATCH = ( + (640, "a4w4_small"), + (2047, "a4w4_bm128"), + (4096, "a4w4_bm128_router_quant"), + (87381, "a4w4_router_quant"), +) +# fused router + quant from here, every call site: routing prep -3.7..-8.6 us at 1025-2047, +1.0..+4.5 at 513-1024 +ROUTER_QUANT_FROM = 1025 + + +@dataclass +class MoeWeights: + """The router (FP32 [E, H] row-major and its 3 MiB line layout), the bias FP32 [E] and the experts in the kernel + layouts (gate / up interleaved MXFP4 values and e8m0 scales); built once by ``from_row_major`` before graph capture. + """ + router_weight: torch.Tensor + correction_bias: torch.Tensor + w13: torch.Tensor + w13_scale: torch.Tensor + w2: torch.Tensor + w2_scale: torch.Tensor + a4w4_scales: object + routed_scaling_factor: float = 2.0 + # the entry needs both (``from_row_major`` builds the gate's line layout; ``shared`` is the caller's) + router_weight_lines: torch.Tensor | None = None + shared_weights: object | None = None + + @classmethod + def from_row_major(cls, router_weight, correction_bias, w13, w13_scale, w2, w2_scale, *, shared=None, + routed_scaling_factor=2.0): + """The checkpoint layout (row-major uint8 MXFP4 values / e8m0 scales, gate rows then up rows; a + ``SharedWeights``, which ``mxfp4_moe`` needs) -> the entry's layouts, built once before graph capture.""" + from .layout import w13_layout, w2_layout, router_line_layout + from .prefill_h6144_i1536 import Mxfp4PrefillScales + scales = Mxfp4PrefillScales.from_row_major(w13_scale, w2_scale) + return cls(router_weight, correction_bias, w13_layout(w13), scales.w13, + w2_layout(w2), scales.w2, scales, float(routed_scaling_factor), + router_weight_lines=router_line_layout(router_weight), shared_weights=shared) + + +def variant_for(n_tokens): + for bound, name in DISPATCH: + if 0 < n_tokens <= bound: + return name + raise ValueError(f"unsupported token count {n_tokens}") + + +def _check_weights(w): + if w.shared_weights is None or w.router_weight_lines is None: + raise ValueError("needs MoeWeights with the shared expert and the line-layout gate: " + "MoeWeights.from_row_major(..., shared=SharedWeights...)") + + +def _route_parts(parts, w): + from .router_topk import splitk_biased_topk + m = parts.shape[0] + ids = torch.empty((m, TOPK), dtype=torch.int32, device=parts.device) + wt = torch.empty((m, TOPK), dtype=torch.float32, device=parts.device) + splitk_biased_topk(parts, w.correction_bias, wt, ids, w.routed_scaling_factor) + return wt, ids + + +def _route_quant_small(x, w, *, s8_tiles=False): + """Below ``ROUTER_QUANT_FROM`` rows: the split-K router, then the top-4 with the MXFP4 + MXFP8 input quant in one + launch (at ``router_dual_quant_for`` rows the quant workgroups join the router launch) -> ((q4, s4), (q8, s8), + topk weights, topk ids); ``s8_tiles``: the FP8 scales in the shared GEMMs' tiled layout.""" + from .router_gemm import router_gemm_partials + from .router_gemm_topk_mxfp4_mxfp8_quant import router_dual_quant_for, router_gemm_topk_mxfp4_mxfp8_quant + from .router_topk_mxfp4_mxfp8_quant import splitk_biased_topk_mxfp4_mxfp8_quant + n = x.shape[0] + if router_dual_quant_for(n): + q4, q8, wt, ids, _ = router_gemm_topk_mxfp4_mxfp8_quant( + x, w.router_weight, w.correction_bias, w.routed_scaling_factor) + return q4, q8, wt, ids + parts = router_gemm_partials(x, w.router_weight, w.router_weight_lines) + if parts is None: + raise ValueError("dual input quant requires the split-K router range") + ids = torch.empty((n, TOPK), dtype=torch.int32, device=x.device) + wt = torch.empty((n, TOPK), dtype=torch.float32, device=x.device) + q4, q8 = splitk_biased_topk_mxfp4_mxfp8_quant(parts, w.correction_bias, wt, ids, w.routed_scaling_factor, x, + s8_tiles=s8_tiles) + return q4, q8, wt, ids + + +def _prepare(x, w): + """From ``ROUTER_QUANT_FROM`` rows, where the shared input quant runs apart: the fused router + MXFP4 input quant on + the line-layout gate, then the split-K top-4 -> ((q4, s4), topk weights, topk ids).""" + from .router_mxfp4_quant_line_layout import router_mxfp4_quant_line_layout + q4, s4, parts = router_mxfp4_quant_line_layout(x, w.router_weight_lines) + wt, ids = _route_parts(parts, w) + return (q4, s4), wt, ids + + +def route(x, w): + """The BF16 entry's routing (topk weights, topk ids), exposed for numerical checks.""" + if x.shape[0] < ROUTER_QUANT_FROM: + return _route_quant_small(x, w)[2:] + return _prepare(x, w)[1:] + + +def mxfp4_moe_experts(x_quant, topk_weights, topk_ids, w): + """The routed experts on the MXFP4 input ``x_quant`` = (codes, scales), already routed (``topk_weights`` include + the routed scale) -> the BF16 [M, 4, H] slots: to 512 rows the small-tile host's routed launches alone, from 513 + the BM128 (to 4096) and BM256 hosts (the entry takes the small-tile host to 640 rows).""" + n = x_quant[0].shape[0] + variant_for(n) + if n <= 512: + return _small_experts(x_quant, topk_weights, topk_ids, w) + if n <= 4096: + from .experts_a4w4_bm128 import a4w4_experts + else: + from .experts_a4w4 import a4w4_experts + return a4w4_experts(x_quant, w.w13, w.w2, w.a4w4_scales, topk_weights, topk_ids) + + +def _small_experts(x_quant, topk_weights, topk_ids, w): + from .experts_a4w4_small import a4w4_experts + return a4w4_experts(x_quant, w.w13, w.w2, w.a4w4_scales, topk_weights, topk_ids, shared=None)[0] + + +def mxfp4_moe(x, w, *, return_partials=False): + """Normed BF16 [M, 6144], or a ``MoeInput`` / ``MoeInputCompact`` (routed and quantized before a sequence-parallel + gather) -> the routed experts plus the MXFP8 shared expert (with a full ``SharedWeights``: the TP rank's shard). + + Partials mode returns the slots BF16 [M, 4, H] and the shared output BF16 [M, H]; reduced mode sums slots 0..3, + then the shared output, in FP32. A routed-only ``MoeInput`` (``MoeInput.routed``) takes partials mode only and + returns (slots, None). + """ + from .prepared_input import MoeInput, MoeInputCompact, _mxfp4_moe_prepared + _check_weights(w) + if isinstance(x, MoeInputCompact): + x = x.expand() + if isinstance(x, MoeInput): + return _mxfp4_moe_prepared(x, w, return_partials=return_partials) + variant_for(x.shape[0]) + if x.ndim != 2 or x.shape[1] != H or x.dtype != torch.bfloat16 or not x.is_contiguous(): + raise ValueError("expected contiguous BF16 [M,6144]") + from .shared_expert import (shared_expert, shared_expert_parallel, shared_parallel_for, shared_dual_quant_for, + shared_folded_for, shared_in_persistent_for, SHARED_QUANT_REDUCED_ONLY) + from .experts_a4w4_small import id_rows_for + n = x.shape[0] + sw = w.shared_weights + current = torch.cuda.current_stream(x.device) + shared_persistent = shared_in_persistent_for(n) + dual = shared_dual_quant_for(n) or shared_persistent + # the shared tiles join the routed 16-row launches; otherwise their own launches + folded = shared_folded_for(n) and dual and variant_for(n) == "a4w4_small" and not shared_persistent + parallel = shared_parallel_for(n) and not folded and not shared_persistent + if return_partials and not shared_persistent and SHARED_QUANT_REDUCED_ONLY[0] <= n <= SHARED_QUANT_REDUCED_ONLY[1]: + dual = False + tiled_shared = n >= ROUTER_QUANT_FROM + if dual: + if tiled_shared: + from .router_mxfp4_mxfp8_quant_line_layout import router_mxfp4_mxfp8_quant_line_layout + prepared, xq8, parts_router = router_mxfp4_mxfp8_quant_line_layout(x, w.router_weight_lines) + if n > 8192: + wt, ids = _route_parts(parts_router, w) + else: + # the combined launch (641-1024) reads the shared scales tiled + prepared, xq8, wt, ids = _route_quant_small(x, w, s8_tiles=shared_persistent) + if folded or shared_persistent: + shared = None + elif parallel: + shared = shared_expert_parallel(x, sw, current, x_quant=xq8, scales_tiled=tiled_shared) + else: + shared = shared_expert(x, sw, x_quant=xq8, scales_tiled=tiled_shared) + elif parallel: + shared = shared_expert_parallel(x, sw, current) + else: + shared = shared_expert(x, sw) + if not dual: + prepared, wt, ids = _prepare(x, w) + elif tiled_shared and n <= 8192: + # to 8192 rows the shared stream forks before the top-k, above after it (measured overlap) + wt, ids = _route_parts(parts_router, w) + if folded: + from .experts_a4w4_small import a4w4_experts + # 16-96 rows: GEMM1 builds its rows from topk_ids + parts, shared = a4w4_experts(prepared, w.w13, w.w2, w.a4w4_scales, wt, ids, shared=(*xq8, sw), + id_rows=id_rows_for(n)) + elif shared_persistent: + from .experts_a4w4_bm128 import a4w4_experts + parts, shared = a4w4_experts(prepared, w.w13, w.w2, w.a4w4_scales, wt, ids, shared=(*xq8, sw)) + elif variant_for(n) == "a4w4_small": + parts = _small_experts(prepared, wt, ids, w) + else: + parts = mxfp4_moe_experts(prepared, wt, ids, w) + if parallel: + current.wait_stream(sw.stream) + shared.record_stream(current) + if return_partials: + return parts, shared + from .common.reduce_slots_shared import reduce_slots_shared + return reduce_slots_shared(parts, shared) + + +# One shape per router wave/split tier, sort tier and expert GEMM variant (33: the router without +# quant tasks at 33-57; 64: the router with appended quant tasks at 58-64; +# the id-rows GEMMs of the BF16 entry with shared weights at 16-95: 16 / 17 / 33 their 64 / 128 / +# 256 pair slots, 65 their 512 slots; 16 and 17 the two shared down-tile widths; 96: the routed launches alone +# and the side-stream shared expert; 321: their 32-row blocks). +WARM_UP_TOKENS = (2, 8, 9, 12, 15, 16, 17, 33, 64, 65, 96, 128, 129, 192, 193, 256, 257, 321, 385, 512, + 513, 640, 641, 769, 1408, 1536, 1537, 2048, 2049, 3584, 3841, 4096, 4097, 5377, 7681, 8192, 8193, 10881, 16384, 16385, + 32768) + + +def shared_kind(sw): + """The shared weights' part of a warm-up mode: False (none), True (a shard copy), "full" (the full copy: the + BF16 entry's view variants).""" + return False if sw is None else "full" if getattr(sw, "full", False) else True + + +def _mode(x, w, *, return_partials=False): + """What a call fixes about its kernels besides the token count (``amd_kernels.warm_up``).""" + from .prepared_input import MoeInput, MoeInputCompact + kind = ("compact" if isinstance(x, MoeInputCompact) else + ("routed" if x.routed_only else "prepared") if isinstance(x, MoeInput) else "bf16") + return kind, shared_kind(w.shared_weights), w.router_weight_lines is not None, bool(return_partials) + + +def _inputs(mode, n, x, w, *, return_partials=False): + """One warm-up call of the mode at ``n`` tokens (zeros: the dispatch depends on shapes only).""" + from .prepared_input import prepare_mxfp4_moe + xn = torch.zeros((n, H), device=w.router_weight.device, dtype=torch.bfloat16) + if mode[0] != "bf16": + xn = prepare_mxfp4_moe(xn, w) + if mode[0] == "compact": + xn = xn.compact() + elif mode[0] == "routed": + xn = xn.routed() + return (xn, w), {"return_partials": return_partials} + + +mxfp4_moe = entry("moe/mxfp4_h6144_i1536_e128_top4", mode=_mode, sizes=lambda mode: WARM_UP_TOKENS, + inputs=_inputs)(mxfp4_moe) + + +_loaded: set = set() + + +def _node_workers() -> int: + # every GPU of a node warms up at once: an eighth of this process's CPUs each + return max(1, min(16, len(os.sched_getaffinity(0)) // 8)) + + +def warm_up_mxfp4_moe(w, *, partials=False, sp=False, workers=None): + """Loads every variant of the BF16-input reduced mode, and of the partials mode with ``partials``, into this + process before serving, once per mode and device: the first call compiles them all (``amd_kernels.warm_up``, + ``workers`` processes, default ``_node_workers()``; the count stays for the process), then one call per + ``WARM_UP_TOKENS`` size; one layer's weights serve every layer. ``sp``: also the sequence-parallel modes, the + preparation, the routed-only partials and, with a full ``SharedWeights``, ``shared_expert_full``.""" + from amd_kernels import warm_up + from .prepared_input import prepare_mxfp4_moe + from .shared_expert import FULL_WARM_UP_ROWS, shared_expert_full + + dev = w.router_weight.device + todo = [(dev, _mode(None, w, return_partials=p)) for p in ((False, True) if partials else (False,))] + full = sp and shared_kind(w.shared_weights) == "full" + if sp: + todo += [(dev, ("routed", *_mode(None, w, return_partials=True)[1:]))] + if full: + todo += [(dev, "shared_full")] + todo = [key for key in todo if key not in _loaded] + if not todo: + return + warm_up.set_workers(_node_workers() if workers is None else workers) + + def zeros(m, cols=H, dtype=torch.bfloat16): + return torch.zeros((m, cols), device=dev, dtype=dtype) + + with torch.inference_mode(): + for key in todo: + if key[1] == "shared_full": + for m in FULL_WARM_UP_ROWS: + shared_expert_full(zeros(m, H, torch.uint8), zeros(m, H // 32, torch.uint8), w.shared_weights) + elif key[1][0] == "routed": + for m in WARM_UP_TOKENS: + mxfp4_moe(prepare_mxfp4_moe(zeros(m), w).routed(), w, return_partials=True) + else: + for m in WARM_UP_TOKENS: + mxfp4_moe(zeros(m), w, return_partials=key[1][3]) + _loaded.add(key) + torch.cuda.synchronize(dev) + torch.cuda.empty_cache() # the warm-up's buffers back to the device before the caller sizes its caches diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/__init__.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/__init__.py new file mode 100644 index 0000000..ab89d28 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Sort, tile-map, reduce, loader and launch helpers shared by this entry's MoE kernels.""" diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/epilogue.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/epilogue.py new file mode 100644 index 0000000..2e85688 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/epilogue.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""LDS vector access, bf16 / e4m3 packing and the nnan max of the MoE GEMM epilogues.""" + +import flydsl.expr as fx +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import arith as _arith +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from .loaders import _lds_ptr_t + +_NUM_XCDS = 8 + + +def _lds_ptr(addr_i32): + return _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(addr_i32)) + + +def _lds_load_vec(addr_i32, n): + ty = _ir.VectorType.get([n], _T.i32) + return _llvm.LoadOp(ty, _lds_ptr(addr_i32), alignment=4 * n).result + + +def _lds_store_vec(vec, addr_i32, n): + _llvm.StoreOp(vec, _lds_ptr(addr_i32), alignment=4 * n) + + +def _bf16x2(a, b): + """Two f32 -> one i32 of two bf16 (RNE, v_cvt_pk_bf16_f32: the same rounding as ``arith.truncf``).""" + return Vec.from_elements([a, b], fx.Float32).to(fx.BFloat16).bitcast(fx.Int32)[0] + + +def _cvt_pk_fp8(old, a, b, scale_f32, hi: bool): + """v_cvt_scalef32_pk_fp8_f32: (a, b) / scale -> two e4m3 into the low (``hi`` False) or high 16 bits of ``old`` + (the intrinsic works on <2 x i16>; i32 in and out here).""" + v2i16 = _ir.VectorType.get([2], _ir.IntegerType.get_signless(16)) + old_v = _llvm.bitcast(v2i16, fx.as_ir_value(old)) + res = _llvm.call_intrinsic( + v2i16, + "llvm.amdgcn.cvt.scalef32.pk.fp8.f32", + [ + old_v, + fx.as_ir_value(a), + fx.as_ir_value(b), + fx.as_ir_value(scale_f32), + fx.Boolean(hi).ir_value(), + ], + [], + [], + ) + return fx.Int32(_llvm.bitcast(_T.i32, res)) + + +def _maxf_nn(a, b): + """v_max_f32 with nnan: without the flag LLVM canonicalizes both inputs first (``v_max x, x, x``, maxnum must + quiet sNaNs), 2 extra VALU per max.""" + fm = _ir.Attribute.parse("#arith.fastmath") + return fx.Float32( + _arith.MaxNumFOp(fx.as_ir_value(a), fx.as_ir_value(b), fastmath=fm).result + ) + + +def _v2i32(a, b): + return Vec.from_elements([a, b], fx.Int32).ir_value() + + +_STORE_CPOL = 0x2 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/launch.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/launch.py new file mode 100644 index 0000000..ef60a99 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/launch.py @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The compile-or-dispatch call of this entry's hosts.""" + +import flydsl.compiler as flyc + + +def _run_compiled(exe, *args): + """The first call compiles and runs (``flyc.compile``); later calls dispatch the cached CompiledFunction.""" + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/loaders.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/loaders.py new file mode 100644 index 0000000..b8dffcd --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/loaders.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Global -> LDS and LDS -> register loaders, the scale gathers and the SwiGLU of the MXFP4 MoE GEMMs (gfx950).""" + +import flydsl.expr as fx +from amd_kernels.common import ( + buffer_ops, +) +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +_N_WAVES = 4 + + +def divmod(a, b): + """Divmod for DSL values (the builtin rejects them).""" + return (a // b, a % b) + + +def swizzle_128(row, col): + """The dense kernel's 128-B-row XOR swizzle: (row, col) -> (row', col').""" + offset = row * 128 + col + swizzle = ((offset % (16 * 128)) >> 8) << 4 + swizzled_offset = offset ^ swizzle + return swizzled_offset // 128, swizzled_offset % 128 + + +class _Buf: + def __init__(self, base_ptr, byte_off): + self.base_ptr = base_ptr + self.byte_off = byte_off + + @property + def ptr(self): + return fx.add_offset(self.base_ptr, self.byte_off) + + +_gep = buffer_ops.get_element_ptr + + +def _lds_ptr_t(): + return _ir.Type.parse("!llvm.ptr<3>") + + +def _asm_void(operands, asm_string, constraints, clobbers=""): + """Side-effecting void inline asm (LLVM sees no memory op -> no waitcnt added).""" + if clobbers: + constraints = f"{constraints},{clobbers}" + _llvm.inline_asm(None, operands, asm_string, constraints, has_side_effects=True) + + +def wait_barrier(count): + """``s_waitcnt vmcnt(count) lgkmcnt(0)`` + ``s_barrier``""" + _rocdl.s_waitcnt(vmcnt=count, lgkmcnt=0) + _rocdl.s_barrier() + + +def _uniform_i32(value): + """Cast to i32 and force a wave-uniform SGPR value for scalar inline-asm operands.""" + raw = fx.as_ir_value(value) if not isinstance(value, _ir.Value) else value + if raw.type != _T.i32: + raw = fx.as_ir_value(fx.Int32(raw)) + return _rocdl.readfirstlane(_T.i32, raw) + + +def _swizzled_col(row, col): + """The swizzled 128-B-row byte column ``swizzle_128`` lands (row, col) on.""" + r, c = swizzle_128(row, col) + return c + + +class G2SLoaderAsm: + """global -> LDS, 16 B per lane per step; ``gl_offsets[step]`` is the loop-invariant per-lane byte offset, the + K-step goes in soffset. With ``step_bytes`` the steps share ``gl_offsets[0]`` and advance by that uniform stride in + soffset (one VGPR per loader instead of one per step); ``soff_bytes`` is a uniform constant added to every soffset + (e.g. the second half's rows).""" + + def __init__( + self, rsrc, gl_offsets, n_load_steps, wave_id, step_bytes=None, soff_bytes=0 + ): + self.rsrc = fx.as_ir_value(rsrc) + self.gl_offsets = gl_offsets + self.n_load_steps = n_load_steps + self.wave_id = wave_id + self.step_bytes = step_bytes + self.soff_bytes = soff_bytes + + @property + def _step_stride(self): + # m0 (LDS byte) advance per step: 4 waves x 64 lanes x 16 B. + return _N_WAVES * 1024 + + def set_wave_base(self, base_ptr): + # the wave-uniform LDS base, readfirstlane'd into an SGPR once + wb = fx.Int32(fx.ptrtoint(base_ptr)) + fx.Int32(self.wave_id * 1024) + self._wave_base_s = _rocdl.readfirstlane(_T.i32, fx.as_ir_value(wb)) + + def _lds_base_sgpr(self, lds_dst): + m0 = fx.Int32(self._wave_base_s) + fx.Int32(lds_dst.byte_off) + return fx.as_ir_value(m0) + + def _voffset(self, step): + if self.step_bytes is not None: + return fx.as_ir_value(fx.Int32(self.gl_offsets[0])) + return fx.as_ir_value(fx.Int32(self.gl_offsets[step])) + + def _emit(self, lds_dst, k_offset, step): + # set m0 at step 0, s_add it for the rest + voff = self._voffset(step) + extra = self.soff_bytes + (step * self.step_bytes if self.step_bytes else 0) + if extra: + k_offset = fx.Int32(k_offset) + fx.Int32(extra) + soff = _uniform_i32(k_offset) # scalar soffset (K-step, + the uniform part) + stride = self._step_stride + # s_add_u32 writes SCC: declare it, or the compiler may keep a live SCC (a loop-exit compare) across this asm + if step == 0: + m0 = self._lds_base_sgpr(lds_dst) + asm = "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds" + _asm_void([m0, voff, self.rsrc, soff], asm, "s,v,s,s", "~{scc}") + else: + asm = ( + f"s_add_u32 m0, {stride}, m0\nbuffer_load_dwordx4 $0, $1, $2 offen lds" + ) + _asm_void([voff, self.rsrc, soff], asm, "v,s,s", "~{scc}") + + def load(self, lds_dst, k_offset): + for step in range_constexpr(self.n_load_steps): + self._emit(lds_dst, k_offset, step) + + def load_one(self, lds_dst, k_offset, step): + self._emit(lds_dst, k_offset, step) + + +class S2RLoader128B: + """LDS -> registers: two 16-byte halves of a swizzled 128-byte row. + + ``fp6`` (paired-step layout): a lane group's operand is 24 bytes read in two pieces (``load_piece``), so it is 6 + consecutive VGPRs with no VALU moves. Even step: c1 (16 B at unit g) + the low 8 B of c2 (unit 4 + g), read with + c2's high 8 B for the pair's odd step; odd step: those 8 B + c3 (16 B at unit g of the odd line, 4 + g for the + second pair's). + + ``fp6_headtail`` (head / tail layout, gemm2): ``load`` / ``load_one`` give the lane group's 16-B head (unit g) and + its 8-B tail (step byte 64 + 8g).""" + + def __init__(self, wave_idx, n_tiles, fp6=False, fp6_headtail=False): + self.lane_id = fx.thread_idx.x % 64 + self.wave_idx = wave_idx + self.n_tiles = n_tiles + self.fp6 = fp6 + self.headtail = fp6_headtail + + def _vec_load_16xf8(self, lds_src, dyn_offset, const_offset, n=4, align=16): + total_off = lds_src.byte_off + const_offset + window_base = (total_off // 0x10000) * 0x10000 + imm = total_off - window_base + assert 0 <= imm <= 0xFFFF + vaddr = fx.Int32(fx.ptrtoint(lds_src.base_ptr)) + fx.Int32( + window_base + dyn_offset + ) + lds_ptr = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(vaddr)) + if imm != 0: + lds_ptr = _gep(lds_ptr, static_byte_offset=imm) + vec_i32 = _ir.VectorType.get([n], fx.Int32.ir_type) + # volatile 8-B pieces: otherwise two tiles' pieces merge into one ds_read2st64_b64 and v_movs build the operands + load = _llvm.LoadOp( + vec_i32, lds_ptr, alignment=min(align, 4 * n), volatile_=n == 2 or None + ) + return Vec(load.result) + + def _offset(self, unit, half, preshuffled): + row = self.wave_idx * (self.n_tiles * 16) + self.lane_id % 16 + if const_expr(preshuffled): + return (row // 8) * 1024 + (row % 8) * 16 + unit * 128 + half + row_swz, col_swz = swizzle_128(row, unit * 16) + return row_swz * 128 + col_swz + half + + def _dyn_offset(self, step, preshuffled): + if const_expr(self.headtail and step == 1): # the tail: 8 B at 64 + 8g + g = self.lane_id // 16 + return self._offset(4 + g // 2, (g % 2) * 8, preshuffled) + return self._offset(self.lane_id // 16 + step * 4, 0, preshuffled) + + def _n(self, step): + return 2 if self.headtail and step == 1 else 4 + + def load(self, lds_src, preshuffled=False): + if const_expr(self.fp6): + return self.load_even(lds_src, preshuffled) + frag = [] + for i in range_constexpr(self.n_tiles): + halves = [] + for step in range_constexpr(2): + dyn = self._dyn_offset(step, preshuffled) + n = self._n(step) + v = self._vec_load_16xf8(lds_src, dyn, i * 2048, n, 4 * n) + halves.append(v.bitcast(fx.Int32)) + frag.append(halves) + return frag + + def load_one(self, lds_src, i, ksub, preshuffled=False): + dyn = self._dyn_offset(ksub, preshuffled) + n = self._n(ksub) + return self._vec_load_16xf8(lds_src, dyn, i * 2048, n, 4 * n).bitcast(fx.Int32) + + def load_piece(self, lds_src, i, unit_hi, half, n, preshuffled=False): + """fp6: ``n`` dwords at byte ``8 * half`` of unit g (+ 4 if ``unit_hi``).""" + unit = self.lane_id // 16 + 4 * unit_hi + dyn = self._offset(unit, 8 * half, preshuffled) + align = 16 if half == 0 else 8 + return self._vec_load_16xf8(lds_src, dyn, i * 2048, n, align).bitcast(fx.Int32) + + def load_even(self, lds_src, preshuffled=False, with_hi=True): + """fp6 even step: per tile [c1, c2 low, c2 high] (``with_hi``) or [c1, c2 low].""" + pieces = _FP6_EVEN_PIECES if with_hi else _FP6_EVEN_PIECES[:2] + return [ + [ + self.load_piece(lds_src, i, uh, hf, nn, preshuffled) + for uh, hf, nn in pieces + ] + for i in range_constexpr(self.n_tiles) + ] + + +_FP6_EVEN_PIECES = ((0, 0, 4), (1, 0, 2), (1, 1, 2)) # c1, c2 low, c2 high + + +def _flat_frag(frag): + return [fx.as_ir_value(x) for t in frag for x in t] + + +def _unflat_frag(flat, n_tiles, arity=2): + return [list(flat[arity * i : arity * i + arity]) for i in range(n_tiles)] + + +def _g2s_thunks(g2s, dst, gl_off, n_steps): + return [lambda s=s: g2s.load_one(dst, gl_off, s) for s in range(n_steps)] + + +def _riffle(glb, lds): + """Interleave the global and LDS thunk lists proportionally.""" + if not glb or not lds: + return list(glb) + list(lds) + out = [] + step = len(lds) / len(glb) + li = 0 + for gi, t in enumerate(glb): + out.append(t) + upto = int(round((gi + 1) * step)) + out += lds[li:upto] + li = upto + return out + lds[li:] + + +def _s2r_thunks(s2r, src, holder, n, pre): + ts = [] + for i in range(n): + for ks in range(_FP4_PACK): + + def f(i=i, ks=ks): + if holder[i] is None: + holder[i] = [None, None] + holder[i][ks] = s2r.load_one(src, i, ks, preshuffled=pre) + + ts.append(f) + return ts + + +def _min(a, b): + return (a < b).select(a, b) + + +def _divmod_nonneg(a, b): + if const_expr(isinstance(b, int) and b > 0 and (b & (b - 1)) == 0): + sh = b.bit_length() - 1 + return (a >> sh, a & (b - 1)) if const_expr(sh > 0) else (a, 0) + return divmod(a, b) + + +_FP4_PACK = 2 # pack_M = pack_N = pack_K = 2 + + +_SCALE_QUARTER_BYTES = 1024 # one gather = 4 blocks = one wave's share of one operand + + +_SCALE_REGION_BYTES = 2 * _SCALE_QUARTER_BYTES # all of A (or all of B) for one K-step + + +_SCALE_SLOT_BYTES = _N_WAVES * _SCALE_QUARTER_BYTES # 4096 + + +_SCALE_SLOTS = 4 + + +_SCALE_LDS_BYTES = _SCALE_SLOTS * _SCALE_SLOT_BYTES # 16 KB + + +_SCALE_A_REGION = 0 + + +_SCALE_B_REGION = _SCALE_REGION_BYTES + + +class ScaleGatherMoE: + """One ``buffer_load_dwordx4 ... lds`` per wave per K-step: waves 0 / 1 fetch the A-scale blocks (rows of the + sorted scale), waves 2 / 3 the B-scale blocks (gate slab + up slab of the expert). A 256-B block = 32 rows x 8 + K-blocks; lane block ``blk = lane // 16`` fetches row group ``G_wave + (blk // 2) * half_groups + blk % 2``: blocks + 0 / 1 belong to LDS half 0 (rows R0 / gate), blocks 2 / 3 to half 1 (rows R1 / up).""" + + def __init__( + self, + a_scale, + b_scale, + K, + lane_id, + wave_id, + lds_base_ptr, + a_scale_bytes, + b_scale_bytes, + a_wave_groups, + a_half_groups, + b_wave_groups, + b_half_groups, + ): + self.row_i32 = (K // 256) * 64 # i32 per 32-row group (32 rows x K/32 bytes) + self.wave_id = wave_id + self.a_rsrc = fx.as_ir_value( + buffer_ops.create_buffer_resource( + a_scale, max_size=False, num_records_bytes=a_scale_bytes + ) + ) + self.b_rsrc = fx.as_ir_value( + buffer_ops.create_buffer_resource( + b_scale, max_size=False, num_records_bytes=b_scale_bytes + ) + ) + self._blk = lane_id // 16 + self._in16 = lane_id % 16 + self._lds_base = fx.Int32(fx.ptrtoint(lds_base_ptr)) + self._a_wave_groups = a_wave_groups + self._a_half_groups = a_half_groups + self._b_wave_groups = b_wave_groups + self._b_half_groups = b_half_groups + + def set_wave_base(self, a_base_row, b_base_row): + """``a_base_row``: first sorted row of the m-tile; ``b_base_row``: first W13 row (gate) of the n-tile, both + multiples of 32.""" + wid = fx.Int32(_rocdl.readfirstlane(_T.i32, fx.as_ir_value(self.wave_id))) + self._wave_base_s = fx.as_ir_value( + self._lds_base + wid * fx.Int32(_SCALE_QUARTER_BYTES) + ) + is_a = wid < fx.Int32(2) + q = wid % fx.Int32(2) + g_a = a_base_row // fx.Int32(32) + q * fx.Int32(self._a_wave_groups) + g_b = b_base_row // fx.Int32(32) + q * fx.Int32(self._b_wave_groups) + self._G = _uniform_i32(is_a.select(g_a, g_b)) + self._HS = _uniform_i32( + is_a.select(fx.Int32(self._a_half_groups), fx.Int32(self._b_half_groups)) + ) + self._rsrc = fx.arith.select(is_a, self.a_rsrc, self.b_rsrc) + self._soff0 = _uniform_i32(fx.Int32(0)) + + def gather(self, kstep, slot): + grp = ( + fx.Int32(self._G) + (self._blk // 2) * fx.Int32(self._HS) + (self._blk % 2) + ) + i32_off = ( + grp * fx.Int32(self.row_i32) + + fx.Int32(kstep) * fx.Int32(64) + + self._in16 * fx.Int32(4) + ) + voff = fx.as_ir_value(i32_off * fx.Int32(4)) # bytes + addr = fx.Int32(self._wave_base_s) + fx.Int32(slot) * fx.Int32( + _SCALE_SLOT_BYTES + ) + asm = "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds" + _asm_void( + [fx.as_ir_value(addr), voff, self._rsrc, self._soff0], + asm, + "s,v,s,s", + "~{m0}", + ) + + +class ScaleLoaderLDS: + def __init__(self, n_tiles, lane_id, quarter, lds_base_ptr, region_off): + assert n_tiles % _FP4_PACK == 0 + self.n_groups = n_tiles // _FP4_PACK + self.lane_id = lane_id + self._region_base = ( + fx.Int32(fx.ptrtoint(lds_base_ptr)) + + fx.Int32(region_off) + + quarter * fx.Int32(_SCALE_QUARTER_BYTES) + ) + + def _slot_wave_byte(self, slot): + return self._region_base + fx.Int32(slot) * fx.Int32(_SCALE_SLOT_BYTES) + + def read_half(self, slot, half): + L = self.lane_id + base = self._slot_wave_byte(slot) + fx.Int32((L // 4) * 16 + (L % 4) * 4) + grp_list = [] + for gi in range_constexpr(self.n_groups): + blk = half * 2 + gi + vaddr = base + fx.Int32(blk * 256) + lds_ptr = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(vaddr)) + load = _llvm.LoadOp(fx.Int32.ir_type, lds_ptr, alignment=4) + grp_list.append(fx.Int32(load.result)) + return grp_list + + def read(self, slot): + return self.read_half(slot, 0), self.read_half(slot, 1) + + +def _bits(f): + return fx.Float32(f).bitcast(fx.Int32) + + +def _as_f32(i): + return fx.Int32(i).bitcast(fx.Float32) + + +def _permlane16_swap(d_a, d_b): + pair_ty = _ir.Type.parse("!llvm.struct<(i32, i32)>") + res = _rocdl.permlane16_swap( + pair_ty, fx.as_ir_value(d_a), fx.as_ir_value(d_b), False, False + ) + return fx.Int32(_llvm.extractvalue(_T.i32, res, [0])), fx.Int32( + _llvm.extractvalue(_T.i32, res, [1]) + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/mfma_order.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/mfma_order.py new file mode 100644 index 0000000..c860dc8 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/mfma_order.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MFMA issue orders shared by the gfx950 MXFP8 FlyDSL GEMMs (pure Python).""" + + +def row_stationary_order(n_a: int, n_b: int) -> list[tuple[int, int]]: + """(A tile, B tile) issue order of one quadrant, row by row: A tile i stays for ``n_b`` MFMAs while the B tiles + serpentine, so adjacent MFMAs switch one operand; each accumulator keeps its K order (bitwise the zig-zag's).""" + cols = list(range(n_b)) + return [(i, j) for i in range(n_a) for j in (cols[::-1] if i % 2 else cols)] diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/reduce_slots_shared.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/reduce_slots_shared.py new file mode 100644 index 0000000..cc714d6 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/reduce_slots_shared.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The reduced output: routed slots 0..3, then the shared expert, summed in fp32 to bf16 (H 6144).""" +import functools +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + + +@functools.cache +@launcher +def compile_reduce(): + @flyc.kernel(name='moe_top4_shared_sum_h6144',known_block_size=[256,1,1]) + def kernel(P:fx.Tensor,S:fx.Tensor,O:fx.Tensor,m:fx.Int32): + tx=fx.thread_idx.x + row,chunk=fx.block_idx.x//3,fx.block_idx.x%3 + col=chunk*2048+tx*8 + pr=buffer_ops.create_buffer_resource(P,max_size=False,num_records_bytes=m*4*6144*2) + sr=buffer_ops.create_buffer_resource(S,max_size=False,num_records_bytes=m*6144*2) + out=buffer_ops.create_buffer_resource(O,max_size=False,num_records_bytes=m*6144*2) + acc=Vec.filled(8,0.0,fx.Float32) + for k in range_constexpr(4): + v=Vec(buffer_ops.buffer_load(pr,((row*4+k)*6144+col)//2,vec_width=4,dtype=fx.Int32)).bitcast(fx.BFloat16).to(fx.Float32) + acc=acc+v + sh=Vec(buffer_ops.buffer_load(sr,(row*6144+col)//2,vec_width=4,dtype=fx.Int32)).bitcast(fx.BFloat16).to(fx.Float32) + buffer_ops.buffer_store((acc+sh).to(fx.BFloat16).bitcast(fx.Int32),out,(row*6144+col)//2) + @flyc.jit + def launch_moe_slots_shared_reduce(P:fx.Tensor,S:fx.Tensor,O:fx.Tensor,m:fx.Int32,stream:fx.Stream): + kernel(P,S,O,m).launch(grid=(m*3,1,1),block=(256,1,1),stream=stream) + return launch_moe_slots_shared_reduce + + +def reduce_slots_shared(parts,shared): + from .launch import _run_compiled + out=torch.empty_like(shared) + _run_compiled(compile_reduce(),parts.view(-1),shared.view(-1),out.view(-1),shared.shape[0],torch.cuda.current_stream()) + return out diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/scale_sort_tile_map.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/scale_sort_tile_map.py new file mode 100644 index 0000000..87516af --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/scale_sort_tile_map.py @@ -0,0 +1,281 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""GEMM1's sorted + swizzled e8m0 activation scales (H 6144: 192 bytes per token row, the bytes of aiter's +``e8m0_shuffle`` of the gathered rows) and, with ``with_tile_map``, ``tile_map.py``'s table from the first ``TM_CTAS`` +CTAs, in one launch (gfx950). + +Per 32-row tile, each row gathers its token's 192 scale bytes (8 threads per row, three 8-B masked loads; invalid and +padding rows stage 0), then the tile's byte (xl, y) lands at tile * 6144 + (y // 8) * 256 + ((y % 8) % 4) * 64 + +(xl % 16) * 4, packed little-endian as [xl][y], [xl + 16][y], [xl][y + 4], [xl + 16][y + 4]. Tiles past +num_valid_ids[0] are skipped (the GEMM never reads them). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .epilogue import _lds_ptr +from .loaders import _min +from amd_kernels.warm_up import launcher + +SCN = 192 # H / 32 scale bytes per token row +LDS_STRIDE = SCN + 4 # +4B pad: kills the 32-bank aliasing of the xl/xl+16 reads +TILE_B = 32 * SCN # 6144 B per swizzled tile + +TM_CTAS = 16 # the tile_map duty lives on CTAs [0, TM_CTAS) +_MAX_BLOCKS = 4096 # tile_map.py: >= max_sorted / BM at every covered shape +_SEARCH_STEPS = 13 # 2^13 > _MAX_BLOCKS + + +def _lds_u8(base, off): + v = _llvm.LoadOp( + _ir.IntegerType.get_signless(8), _lds_ptr(base + off), alignment=1 + ).result + return fx.Int32(_llvm.ZExtOp(T.i32, v).result) + + +def _lds_store_dw(val, base, off): + _llvm.StoreOp(fx.Int32(val).ir_value(), _lds_ptr(base + off), alignment=4) + + +@launcher +def compile_scale_sort_tile_map(*, block_m: int, a_group_bj: int = 0, with_tile_map: bool = True): + """The scale sort for sort blocks of ``block_m`` rows, with the tile map on the first ``TM_CTAS`` CTAs. + + Args: + a_group_bj: ``tile_map.py``'s grouped tile order (a trace-time int; 0: n-major). + with_tile_map: False drops the tile-map CTAs (the routing sort writes the map; their two 13-step binary + searches over global memory were this kernel's longest path). + """ + bm_shift = block_m.bit_length() - 1 + name = f"scale_sort_tile_map_h6144_i1536_bm{block_m}" + if not with_tile_map: + name = f"scale_sort_h6144_i1536_bm{block_m}" + if a_group_bj: + name += f"_agrp{a_group_bj}" + TM = TM_CTAS if with_tile_map else 0 + NB_N = 12 # I // 128 with I = 1536 + N_BITS = (NB_N - 1).bit_length() + TAIL_ITERS = (_MAX_BLOCKS * NB_N + 8) // (256 * TM_CTAS) + 1 + assert a_group_bj == 0 or NB_N % a_group_bj == 0 + + @fx.struct + class Shared: + stage: fx.Array[fx.Int8, 32 * LDS_STRIDE] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel( + scale: fx.Tensor, # [n_tok, SCN] u8, token order + sorted_ids: fx.Tensor, # [capacity] i32, token | slot<<24 + sorted_expert_ids: fx.Tensor, # [capacity // block_m] i32 + num_valid: fx.Tensor, # [2] i32 + out: fx.Tensor, # [capacity32, SCN] u8, swizzled tiles + tile_map: fx.Tensor, # [grid_entries + 1] i32 + n_tok: fx.Int32, + grid_entries: fx.Int32, + i32_sid_dw: fx.Int32, # sorted_ids capacity in dwords (bounds for masks) + ): + tx, bx = fx.thread_idx.x, fx.block_idx.x + nv = fx.Int32(num_valid[0]) + if bx < TM: + # tile_map.py's kernel: thread gtid handles sorted block gtid + gtid = bx * 256 + tx + vb = nv >> bm_shift # valid blocks (num_valid is BM-padded) + + def _eid(i): + return fx.Int32(sorted_expert_ids[i]) + + def _bound(e, upper): + """First index in [0, vb) whose expert id is > e / >= e.""" + lo = fx.Int32(0) + n = vb + for _ in range_constexpr(_SEARCH_STEPS): + half = n // 2 + mid = lo + half + probe = _eid(_min(mid, vb - 1)) + go_right = (n > 0) & ((probe <= e) if upper else (probe < e)) + lo = go_right.select(mid + 1, lo) + n = go_right.select(n - half - 1, half) + return lo + + m = gtid + if m < vb: + e = _eid(m) + lo = _bound(e, False) + hi = _bound(e, True) + cnt = hi - lo + # tile_map.py's _pos: n-major, or 8-m-block x a_group_bj-slab groups, m fastest + for n in range_constexpr(NB_N): + if const_expr(a_group_bj > 0): + mg = (m - lo) // 8 + mi = (m - lo) - mg * 8 + gs = _min(cnt - mg * 8, fx.Int32(8)) + pos = ( + lo * NB_N + + mg * (8 * NB_N) + + (n // a_group_bj) * (gs * a_group_bj) + + (n % a_group_bj) * gs + + mi + ) + else: + pos = lo * NB_N + cnt * n + (m - lo) + tile_map[pos] = (m << N_BITS) | n + tail0 = vb * NB_N + for it in range_constexpr(TAIL_ITERS): + i = tail0 + gtid + it * (256 * TM_CTAS) + if i < grid_entries: + tile_map[i] = fx.Int32(-1) + if gtid == 0: + tile_map[grid_entries] = tail0 + else: + tile = bx - TM + if tile * 32 < nv: + lds = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(lds.stage.ptr)) + # load phase: thread handles row i = tx//8, bytes [kk, kk+24) + i = tx // 8 + kk4 = (tx % 8) * 6 # 24 bytes = 6 dwords; kk4 = kk / 4 + row = tile * 32 + i + # masked: rows >= num_valid (past capacity too) load 0, which the row < nv term excludes again + idr = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=i32_sid_dw * 4 + ) + in_nv = row < nv + sid = fx.Int32( + buffer_ops.buffer_load( + idr, row, vec_width=1, dtype=fx.Int32, mask=in_nv + ) + ) + tok = sid & 0x00FFFFFF + ok = in_nv & (tok < n_tok) + sr = buffer_ops.create_buffer_resource( + scale, max_size=False, num_records_bytes=n_tok * SCN + ) + row_dw = tok * (SCN // 4) + kk4 + w = [ + Vec( + buffer_ops.buffer_load( + sr, + row_dw + 2 * j, + vec_width=2, + dtype=fx.Int32, + mask=ok, + ) + ) + for j in range_constexpr(3) + ] + for j in range_constexpr(3): + for d in range_constexpr(2): + _lds_store_dw( + fx.Int32(w[j][d]), + base, + i * LDS_STRIDE + (kk4 + 2 * j + d) * 4, + ) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + # store phase: 1536 dwords; thread tx takes d = tx + 256*j + or_ = buffer_ops.create_buffer_resource(out, max_size=True) + tile_dw = tile * (TILE_B // 4) + for j in range_constexpr(6): + d = tx + 256 * j + y_blk = d >> 6 + t63 = d & 63 + xl = t63 & 15 + c = t63 >> 4 + col_a = y_blk * 8 + c + b0 = _lds_u8(base, xl * LDS_STRIDE + col_a) + b1 = _lds_u8(base, (xl + 16) * LDS_STRIDE + col_a) + b2 = _lds_u8(base, xl * LDS_STRIDE + col_a + 4) + b3 = _lds_u8(base, (xl + 16) * LDS_STRIDE + col_a + 4) + packed = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) + buffer_ops.buffer_store( + packed, or_, tile_dw + y_blk * 64 + c * 16 + xl + ) + + @flyc.jit + def launch_scale_sort_tile_map_mxfp4_i1536( + scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + num_valid: fx.Tensor, + out: fx.Tensor, + tile_map: fx.Tensor, + n_tok: fx.Int32, + grid_entries: fx.Int32, + n_tiles: fx.Int32, + i32_sid_dw: fx.Int32, + stream: fx.Stream, + ): + kernel( + scale, + sorted_ids, + sorted_expert_ids, + num_valid, + out, + tile_map, + n_tok, + grid_entries, + i32_sid_dw, + ).launch( + grid=(fx.Int64(n_tiles + TM), 1, 1), + block=(256, 1, 1), + stream=stream, + ) + + launch_scale_sort_tile_map_mxfp4_i1536.kernel_name = name + return launch_scale_sort_tile_map_mxfp4_i1536 + + +def _run_compiled(exe, *args): + """The first call compiles and runs (``flyc.compile``); later calls dispatch the cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +@functools.cache +def _get_scale_sort_tile_map(block_m: int, a_group_bj: int = 0, with_tile_map: bool = True): + return compile_scale_sort_tile_map(block_m=block_m, a_group_bj=a_group_bj, with_tile_map=with_tile_map) + + +def scale_sort_tile_map( + scale: torch.Tensor, # [n_tokens, 192] u8 / fp8_e8m0, token order + sorted_ids: torch.Tensor, # [capacity] int32 + sorted_expert_ids: torch.Tensor, # [capacity // block_m] int32 + num_valid: torch.Tensor, # [2] int32 + out: torch.Tensor, # [capacity32, 192] u8 / fp8_e8m0 (swizzled out) + tile_map: torch.Tensor, # [grid_entries + 1] int32 + grid_entries: int, + n_tokens: int, + block_m: int, + a_group_bj: int = 0, + with_tile_map: bool = True, +) -> None: + """GEMM1's sorted + swizzled activation scales and, with ``with_tile_map``, the tile map, in one launch.""" + n_tiles = (sorted_ids.numel() + 31) // 32 + _run_compiled( + _get_scale_sort_tile_map(block_m, a_group_bj, with_tile_map), + scale.view(torch.uint8).view(-1), + sorted_ids.view(-1), + sorted_expert_ids.view(-1), + num_valid.view(-1), + out.view(torch.uint8).view(-1), + tile_map.view(-1), + n_tokens, + grid_entries, + n_tiles, + sorted_ids.numel(), + torch.cuda.current_stream(), + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort.py new file mode 100644 index 0000000..31cb47c --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The routing sort's buffers and bounds (aiter's ``moe_3stage_sort.cuh`` contract), written by ``sort_count_all``, +``sort_scales_all`` and ``sort_tile_map``. + +Outputs: ``sorted_ids[row] = token | slot << 24`` (padding rows: ``n_tokens``), ``sorted_weights`` (padding 0), +``sorted_expert_ids`` per block, ``num_valid_ids`` = [padded total, the last expert's first row]. Rows inside an expert +come out in atomic arrival order; consumers do not rely on it. +""" + +from dataclasses import dataclass + +import torch + +SORT_CTAS = 128 # histogram / placement blocks +THREADS = 1024 # >= E and >= block_m + + +def max_sorted_rows(n_tokens: int, E: int, topk: int, block_m: int) -> int: + """Aiter's bound: every active expert pads by < block_m rows.""" + active = min(E, n_tokens * topk) + cumsum_max = n_tokens * topk + active * (block_m - 1) + return ((cumsum_max + block_m - 1) // block_m) * block_m + + +@dataclass +class SortBuffers: + sorted_ids: torch.Tensor + sorted_expert_ids: torch.Tensor + num_valid_ids: ( + torch.Tensor + ) # [2] i32: padded row count, the last expert's first row + sorted_weights: torch.Tensor + block_offsets: torch.Tensor # workspace [E * SORT_CTAS]: per-CTA counts + max_sorted: int + block_m: int + + @staticmethod + def allocate(n_tokens, E, topk, block_m, device): + ms = max_sorted_rows(n_tokens, E, topk, block_m) + i32 = torch.int32 + return SortBuffers( + sorted_ids=torch.empty(ms, dtype=i32, device=device), + sorted_expert_ids=torch.empty(ms // block_m, dtype=i32, device=device), + num_valid_ids=torch.empty( + 2, dtype=i32, device=device + ), # written by sort_place_pad + sorted_weights=torch.empty(ms, dtype=torch.float32, device=device), + block_offsets=torch.empty(E * SORT_CTAS, dtype=i32, device=device), + max_sorted=ms, + block_m=block_m, + ) + + def launch_args(self, topk_ids, topk_weights, n_tokens): + return ( + topk_ids.contiguous().int().view(-1), + topk_weights.contiguous().float().view(-1), + self.sorted_ids, + self.sorted_expert_ids, + self.num_valid_ids, + self.sorted_weights, + self.block_offsets, + int(n_tokens), + torch.cuda.current_stream(), + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_count_all.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_count_all.py new file mode 100644 index 0000000..b66c025 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_count_all.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``sort.py``'s routing sort in one launch, plus GEMM1's tile map: every CTA histograms all (token, slot) pairs in +LDS itself (those before its range and the rest: the totals and per-CTA prefix ``sort_place_pad`` reads from the +workspace), then pads, scans, places and pads as ``sort_place_pad``. Each CTA reads all of ``topk_ids`` (L2 resident), +so the host takes it while that costs less than the second launch. + +Block b's expert comes from a search over the start rows (blocks spread over all CTAs, so skewed routing does not +serialize them); the same thread writes block b's ``I // 128`` entries of ``tile_map.py``'s table. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +from .sort import SORT_CTAS, THREADS +from .tile_map import _MAX_BLOCKS +from .utils import _lds_atomic_add_i32, wave_incl_scan_i32 +from amd_kernels.warm_up import launcher + + +@functools.cache +@launcher +def compile_moe_sort_count_all(*, E: int, topk: int, block_m: int, I: int): # noqa: E741 + assert (block_m & (block_m - 1)) == 0 and max(E, block_m) <= THREADS + assert topk == 4 and 2 * E <= THREADS + NB_N = I // 128 + N_BITS = (NB_N - 1).bit_length() + assert _MAX_BLOCKS * NB_N + 16 <= SORT_CTAS * THREADS # the tail in one pass + experts_per_cta = (E + SORT_CTAS - 1) // SORT_CTAS + bm_shift = block_m.bit_length() - 1 + tag = f"E{E}_K{topk}_BM{block_m}" + SCAN_W = THREADS + E_WAVES = (E + 63) // 64 # waves holding an expert's padded total + search_steps = E.bit_length() + 1 + # SUB counters per histogram bin, picked by lane: lanes of a wave that count the same expert (skewed routing) hit + # SUB addresses, not one (an LDS atomic to one address serializes the wave's lanes) + SUB = 8 + assert (2 * E * SUB) % THREADS == 0 and 2 * E <= THREADS + + @fx.struct + class Shared: + cursor: fx.Array[fx.Int32, E] + total: fx.Array[fx.Int32, E] + starts: fx.Array[fx.Int32, E + 1] # expert start rows, [E] = padded total + hist: fx.Array[fx.Int32, 2 * E * SUB] # pairs before this CTA's range | the rest, SUB counters each + scan: fx.Array[fx.Int32, 2 * SCAN_W] + + def pair_range(n_tok, bx): + """This CTA's [start, end) of the n_tok * topk routing pairs (as sort.py).""" + total = n_tok * topk + per_cta = (total + (SORT_CTAS - 1)) // SORT_CTAS + start = bx * per_cta + end = start + per_cta + return start, (end < total).select(end, total) + + @flyc.kernel(name=f"sort_all_place_pad_tile_map_{tag}_i{I}", known_block_size=[THREADS, 1, 1]) + def sort_all_place_pad( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + num_valid: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_w: fx.Tensor, + tile_map: fx.Tensor, # [grid_entries + 1] i32 + n_tok: fx.Int32, + grid_entries: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + cursor, total_c, starts, hist = smem.cursor.ptr, smem.total.ptr, smem.starts.ptr, smem.hist.ptr + scan = [smem.scan.ptr, smem.scan.ptr + SCAN_W] + tx, bx = fx.thread_idx.x, fx.block_idx.x + start, end = pair_range(n_tok, bx) + # 1. histogram of all pairs: [0, E) those before this CTA's range, [E, 2E) the rest, SUB counters per bin + for z in range_constexpr(2 * E * SUB // THREADS): + hist[tx + z * THREADS] = 0 + fx.gpu.barrier() + sub = tx % SUB + ids_r = buffer_ops.create_buffer_resource( + topk_ids, max_size=False, num_records_bytes=n_tok * (topk * 4) + ) + for ct in range(tx, n_tok, THREADS): + cv = Vec(buffer_ops.buffer_load(ids_r, ct * topk, vec_width=4, dtype=fx.Int32)) + for ck in range_constexpr(topk): + cp = ct * topk + ck + half = (cp >= start).select(fx.Int32(E), fx.Int32(0)) + _lds_atomic_add_i32(hist + (half + fx.Int32(cv[ck])) * SUB + sub, 1) + fx.gpu.barrier() + if tx < 2 * E: + hs = fx.Int32(0) + for j in range_constexpr(SUB): + hs = hs + fx.Int32(hist[tx * SUB + j]) + scan[1][tx] = hs + fx.gpu.barrier() + # 2. inclusive scan of the padded totals (0 beyond E) -> start rows + in_e = tx < E + ec = in_e.select(tx, fx.Int32(0)) + pre = fx.Int32(scan[1][ec]) + tot = pre + fx.Int32(scan[1][E + ec]) + padded = in_e.select((tot + (block_m - 1)) & ~(block_m - 1), fx.Int32(0)) + if in_e: + total_c[tx] = tot + # per wave by DPP, the totals of the waves holding experts through LDS (one barrier, not one per round) + incl = wave_incl_scan_i32(padded) + lane, wave = tx % 64, tx // 64 + scan[0][(lane == 63).select(wave, fx.Int32(E_WAVES))] = incl + fx.gpu.barrier() + for w in range_constexpr(E_WAVES - 1): + incl = incl + (fx.Int32(w) < wave).select(fx.Int32(scan[0][w]), fx.Int32(0)) + if in_e: + st = incl - padded + starts[tx] = st + cursor[tx] = st + pre + if tx == E - 1: + starts[E] = incl + fx.gpu.barrier() + nv = fx.Int32(starts[E]) + if bx == 0: + if tx == 0: + num_valid[0] = nv + num_valid[1] = fx.Int32(starts[E - 1]) # the last expert's first row + # 3. block b = gtid: its expert = the first e with starts[e + 1] > b * block_m, its tile-map entries; then the + # tile map's -1 tail and count + gtid = bx * THREADS + tx + vb = nv >> bm_shift + if gtid < vb: + r0 = gtid << bm_shift + lo = fx.Int32(0) + n = fx.Int32(E) + for _ in range_constexpr(search_steps): + hn = n // 2 + mid = lo + hn + probe = fx.Int32(starts[(mid < E).select(mid + 1, fx.Int32(E))]) + go_right = (n > 0) & (probe <= r0) + lo = go_right.select(mid + 1, lo) + n = go_right.select(n - hn - 1, hn) + sorted_expert_ids[gtid] = lo + blo = fx.Int32(starts[lo]) >> bm_shift + cnt = (fx.Int32(starts[lo + 1]) >> bm_shift) - blo + for nn in range_constexpr(NB_N): + tile_map[blo * NB_N + cnt * nn + (gtid - blo)] = (gtid << N_BITS) | nn + tail = vb * NB_N + gtid + if tail < grid_entries: + tile_map[tail] = fx.Int32(-1) + if gtid == 0: + tile_map[grid_entries] = vb * NB_N + # 4. place this CTA's pairs + for pi in range(start + tx, end, THREADS): + pp = fx.Int32(pi) + prow = _lds_atomic_add_i32(cursor + fx.Int32(topk_ids[pp]), 1) + sorted_ids[prow] = ((pp // topk) & 0x00FFFFFF) | ((pp % topk) << 24) + sorted_w[prow] = fx.Float32(topk_w[pp]) + # 5. padding: every expert pads by < block_m rows, lanes [0, block_m) do it + if tx < block_m: + for ee in range_constexpr(experts_per_cta): + pe = bx * experts_per_cta + ee + if pe < E: + zlo = fx.Int32(starts[pe]) + fx.Int32(total_c[pe]) + tx + for zr in range(zlo, fx.Int32(starts[pe + 1]), THREADS): + sorted_ids[fx.Int32(zr)] = n_tok # token n_tokens, slot 0 + sorted_w[fx.Int32(zr)] = fx.Float32(0.0) + + @flyc.jit + def launch_sort_all_tile_map_mxfp4_i1536( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + num_valid: fx.Tensor, + sorted_w: fx.Tensor, + tile_map: fx.Tensor, + n_tokens: fx.Int32, + grid_entries: fx.Int32, + stream: fx.Stream, + ): + sort_all_place_pad( + topk_ids, topk_w, num_valid, sorted_expert_ids, sorted_ids, sorted_w, tile_map, + n_tokens, grid_entries, + ).launch(grid=(SORT_CTAS, 1, 1), block=(THREADS, 1, 1), stream=stream) # fmt: skip + + return launch_sort_all_tile_map_mxfp4_i1536 + + +def moe_sort_count_all(bufs, topk_ids, topk_weights, tile_map, grid_entries, n_tokens, *, E, I): # noqa: E741 + """The routing sort into ``bufs`` (a ``SortBuffers``) and GEMM1's tile map, one launch.""" + import torch + + from .launch import _run_compiled + + # the cached CompiledFunction: a direct jit call re-resolves its cache key on every call (~100+ us of host time) + _run_compiled( + compile_moe_sort_count_all(E=E, topk=topk_ids.shape[1], block_m=bufs.block_m, I=I), + *bufs.launch_args(topk_ids, topk_weights, n_tokens)[:6], + tile_map.view(-1), + int(n_tokens), + int(grid_entries), + torch.cuda.current_stream(), + ) # fmt: skip diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_scales_all.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_scales_all.py new file mode 100644 index 0000000..1497cfe --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_scales_all.py @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``sort_count_all.py``'s routing sort and tile map plus ``scale_sort_tile_map.py``'s scale tiles, in one launch. + +The scale tiles need every row's token, so placement is owned by row blocks, not pair ranges: CTA c owns the sorted +blocks ``[c * bpc, (c + 1) * bpc)`` (skewed routing spreads a hot expert's blocks over many CTAs), and a block scan of +the threads' match counts of the block's expert ranks its pairs in pair order. Rows inside an expert thus come out in +pair order, not atomic order; a GEMM output element depends on its own row only, so the entry's outputs are bit-equal. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +from .scale_sort_tile_map import LDS_STRIDE, SCN, TILE_B, _lds_store_dw, _lds_u8 +from .sort import SORT_CTAS, THREADS, max_sorted_rows +from .tile_map import _MAX_BLOCKS +from .utils import _lds_atomic_add_i32, wave_incl_scan_i32 +from amd_kernels.warm_up import launcher + +# tokens per thread of the pair scan, by token count (one compile each) +TPT_TIERS = ((1024, 1), (2048, 2), (4096, 4), (8192, 8)) +MAX_TOKENS = TPT_TIERS[-1][0] + + +def tpt_for(n_tokens: int) -> int: + for bound, tpt in TPT_TIERS: + if n_tokens <= bound: + return tpt + raise ValueError(f"sort_scales_all covers up to {MAX_TOKENS} tokens, got {n_tokens}") + + +@functools.cache +@launcher +def compile_moe_sort_scales_all(*, E: int, topk: int, block_m: int, I: int, tpt: int): # noqa: E741 + assert (block_m & (block_m - 1)) == 0 and 128 <= block_m <= THREADS + assert topk == 4 and 2 * E <= THREADS + NB_N = I // 128 + N_BITS = (NB_N - 1).bit_length() + assert _MAX_BLOCKS * NB_N + 16 <= SORT_CTAS * THREADS # the tail in one pass + bm_shift = block_m.bit_length() - 1 + tag = f"E{E}_K{topk}_BM{block_m}" + NWAVE = THREADS // 64 + search_steps = E.bit_length() + 1 + SUB = 8 # sub-counters per histogram bin, picked by lane (skewed routing) + assert (E * SUB) % THREADS == 0 + TILES = block_m // 32 # 32-row scale tiles per block + TPP = THREADS // 256 # tiles staged per pass (256 threads per tile) + PASSES = TILES // TPP + assert TILES % TPP == 0 + max_blocks = max_sorted_rows(THREADS * tpt, E, topk, block_m) // block_m + BPC_MAX = (max_blocks + SORT_CTAS - 1) // SORT_CTAS + + @fx.struct + class Shared: + total: fx.Array[fx.Int32, E] + starts: fx.Array[fx.Int32, E + 1] # expert start rows, [E] = padded total + hist: fx.Array[fx.Int32, E * SUB] + wsum: fx.Array[fx.Int32, NWAVE + 1] # wave totals of a block scan + blk_e: fx.Array[fx.Int32, 1] # the current block's expert + rows: fx.Array[fx.Int32, block_m + 64] # the block's sorted ids, then a dump slot per lane + stage: fx.Array[fx.Int8, TPP * 32 * LDS_STRIDE] + + @flyc.kernel( + name=f"sort_all_place_pad_tile_map_scales_{tag}_i{I}_t{tpt}", + known_block_size=[THREADS, 1, 1], + ) + def sort_scales_all( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + num_valid: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_w: fx.Tensor, + tile_map: fx.Tensor, # [grid_entries + 1] i32 + scale: fx.Tensor, # [n_tok, SCN] u8, token order + out: fx.Tensor, # [capacity32, SCN] u8, swizzled tiles + n_tok: fx.Int32, + grid_entries: fx.Int32, + i32_ctas: fx.Int32, # grid size (>= SORT_CTAS) + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + total_c, starts, hist, rows = smem.total.ptr, smem.starts.ptr, smem.hist.ptr, smem.rows.ptr + stage_base = fx.Int32(fx.ptrtoint(smem.stage.ptr)) + wsum, blk_e = smem.wsum.ptr, smem.blk_e.ptr + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = tx // 64 + + def block_incl_scan(v): + """Inclusive scan of v over the block: per wave by DPP, then the wave totals through LDS and one barrier; + the caller barriers before the next scan.""" + x = wave_incl_scan_i32(v) + wsum[(lane == 63).select(wave, fx.Int32(NWAVE))] = x # [NWAVE] = dump slot + fx.gpu.barrier() + off = fx.Int32(0) + for w in range_constexpr(NWAVE): + off = off + (fx.Int32(w) < wave).select(fx.Int32(wsum[w]), fx.Int32(0)) + return x + off + + # 1. this thread's tokens' ids; histogram of all pairs + for z in range_constexpr(E * SUB // THREADS): + hist[tx + z * THREADS] = 0 + ids_r = buffer_ops.create_buffer_resource( + topk_ids, max_size=False, num_records_bytes=n_tok * (topk * 4) + ) + toks, ok, cid = [], [], [] + for j in range_constexpr(tpt): + t = tx + j * THREADS # coalesced; the rank order is (thread, j, slot) + v = t < n_tok + toks.append(t) + ok.append(v) + cid.append(Vec(buffer_ops.buffer_load(ids_r, t * topk, vec_width=4, dtype=fx.Int32, mask=v))) + fx.gpu.barrier() + sub = tx % SUB + for j in range_constexpr(tpt): + if ok[j]: + for k in range_constexpr(topk): + _lds_atomic_add_i32(hist + fx.Int32(cid[j][k]) * SUB + sub, 1) + fx.gpu.barrier() + # 2. totals, inclusive scan of the padded totals -> start rows + in_e = tx < E + ec = in_e.select(tx, fx.Int32(0)) + tot = fx.Int32(0) + for s in range_constexpr(SUB): + tot = tot + fx.Int32(hist[ec * SUB + s]) + padded = in_e.select((tot + (block_m - 1)) & ~(block_m - 1), fx.Int32(0)) + if in_e: + total_c[tx] = tot + incl = block_incl_scan(padded) + if in_e: + starts[tx] = incl - padded + if tx == E - 1: + starts[E] = incl + fx.gpu.barrier() + nv = fx.Int32(starts[E]) + if bx == 0: + if tx == 0: + num_valid[0] = nv + num_valid[1] = fx.Int32(starts[E - 1]) # the last expert's first row + + def expert_of_row(r0): + """The first e with starts[e + 1] > r0.""" + lo = fx.Int32(0) + n = fx.Int32(E) + for _ in range_constexpr(search_steps): + hn = n // 2 + mid = lo + hn + probe = fx.Int32(starts[(mid < E).select(mid + 1, fx.Int32(E))]) + go_right = (n > 0) & (probe <= r0) + lo = go_right.select(mid + 1, lo) + n = go_right.select(n - hn - 1, hn) + return lo + + # 3. block b = gtid: its expert and tile-map entries; the tile map's tail, count + gtid = bx * THREADS + tx + vb = nv >> bm_shift + if gtid < vb: + lo = expert_of_row(gtid << bm_shift) + sorted_expert_ids[gtid] = lo + blo = fx.Int32(starts[lo]) >> bm_shift + cnt = (fx.Int32(starts[lo + 1]) >> bm_shift) - blo + for nn in range_constexpr(NB_N): + tile_map[blo * NB_N + cnt * nn + (gtid - blo)] = (gtid << N_BITS) | nn + tail = vb * NB_N + gtid + if tail < grid_entries: + tile_map[tail] = fx.Int32(-1) + if gtid == 0: + tile_map[grid_entries] = vb * NB_N + # 4. this CTA's blocks: placement in pair order, padding, scale tiles (the host launches a CTA per possible + # block, so bpc is 1; the loop keeps any grid valid) + bpc = (vb + (i32_ctas - 1)) // i32_ctas + b0 = bx * bpc + b1 = b0 + bpc + b1 = (b1 < vb).select(b1, vb) + sr = buffer_ops.create_buffer_resource( + scale, max_size=False, num_records_bytes=n_tok * SCN + ) + or_ = buffer_ops.create_buffer_resource(out, max_size=True) + w_r = buffer_ops.create_buffer_resource( + topk_w, max_size=False, num_records_bytes=n_tok * (topk * 4) + ) + for blk in range(b0, b1): + row0 = fx.Int32(blk) << bm_shift + # the block's expert: the one whose padded rows hold row0 (one compare per thread, no dependent search) + if in_e: + if (fx.Int32(starts[tx]) <= row0) & (row0 < fx.Int32(starts[tx + 1])): + blk_e[0] = tx + fx.gpu.barrier() + e = fx.Int32(blk_e[0]) + e_start = fx.Int32(starts[e]) + e_tot = fx.Int32(total_c[e]) + r0 = row0 - e_start # the block's first rank of e + # exclusive scan of the threads' match counts of e + mc = fx.Int32(0) + for j in range_constexpr(tpt): + for k in range_constexpr(topk): + hit = ok[j] & (fx.Int32(cid[j][k]) == e) + mc = mc + hit.select(fx.Int32(1), fx.Int32(0)) + rank = block_incl_scan(mc) - mc - r0 + # the pairs ranked in this block put their id in its LDS row (no branch: the others write their lane's dump + # slot) + for j in range_constexpr(tpt): + for k in range_constexpr(topk): + hit = ok[j] & (fx.Int32(cid[j][k]) == e) + put = hit & (rank >= 0) & (rank < block_m) + rows[put.select(rank, block_m + lane)] = (toks[j] & 0x00FFFFFF) | (k << 24) + rank = rank + hit.select(fx.Int32(1), fx.Int32(0)) + fx.gpu.barrier() + # one thread per row: sorted id and weight (stored after the first scale loads are out); rows past e's total + # are padding + in_b = tx < block_m + pad = r0 + tx >= e_tot + val = (pad | ~in_b).select(n_tok, fx.Int32(rows[in_b.select(tx, fx.Int32(0))])) + pair = (val & 0x00FFFFFF) * topk + (val >> 24) + wgt = fx.Float32(buffer_ops.buffer_load(w_r, pair, vec_width=1, dtype=fx.Float32, mask=~pad & in_b)) + # scale tiles: 256 threads per tile, TPP tiles per pass + lt = tx % 256 + ti = tx // 256 + i = lt // 8 # row of the tile + kk4 = (lt % 8) * 6 # 24 bytes = 6 dwords of the row + st = stage_base + ti * (32 * LDS_STRIDE) + for p in range_constexpr(PASSES): + tile_r = (p * TPP + ti) * 32 # first row of the tile in the block + okr = r0 + tile_r + i < e_tot # padding rows stage zeros + tok = fx.Int32(rows[okr.select(tile_r + i, fx.Int32(0))]) & 0x00FFFFFF + row_dw = tok * (SCN // 4) + kk4 + wv = [ + Vec(buffer_ops.buffer_load(sr, row_dw + 2 * jj, vec_width=2, dtype=fx.Int32, mask=okr)) + for jj in range_constexpr(3) + ] + if p == 0: + if in_b: + sorted_ids[row0 + tx] = val # padding: token n_tokens, slot 0 + sorted_w[row0 + tx] = wgt # padding: 0 (masked load) + for jj in range_constexpr(3): + for dd in range_constexpr(2): + _lds_store_dw(fx.Int32(wv[jj][dd]), st, i * LDS_STRIDE + (kk4 + 2 * jj + dd) * 4) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + tile_dw = ((row0 + tile_r) >> 5) * (TILE_B // 4) + for jj in range_constexpr(6): + dw = lt + 256 * jj + y_blk = dw >> 6 + t63 = dw & 63 + xl = t63 & 15 + c = t63 >> 4 + col_a = y_blk * 8 + c + q0 = _lds_u8(st, xl * LDS_STRIDE + col_a) + q1 = _lds_u8(st, (xl + 16) * LDS_STRIDE + col_a) + q2 = _lds_u8(st, xl * LDS_STRIDE + col_a + 4) + q3 = _lds_u8(st, (xl + 16) * LDS_STRIDE + col_a + 4) + packed = q0 | (q1 << 8) | (q2 << 16) | (q3 << 24) + buffer_ops.buffer_store(packed, or_, tile_dw + y_blk * 64 + c * 16 + xl) + fx.gpu.barrier() + + @flyc.jit + def launch_sort_scales_all_mxfp4_i1536( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + num_valid: fx.Tensor, + sorted_w: fx.Tensor, + tile_map: fx.Tensor, + scale: fx.Tensor, + out: fx.Tensor, + n_tokens: fx.Int32, + grid_entries: fx.Int32, + i32_ctas: fx.Int32, + stream: fx.Stream, + ): + sort_scales_all( + topk_ids, topk_w, num_valid, sorted_expert_ids, sorted_ids, sorted_w, tile_map, + scale, out, n_tokens, grid_entries, i32_ctas, + ).launch(grid=(fx.Int64(i32_ctas), 1, 1), block=(THREADS, 1, 1), stream=stream) # fmt: skip + + assert BPC_MAX >= 1 + return launch_sort_scales_all_mxfp4_i1536 + + +def moe_sort_scales_all(bufs, topk_ids, topk_weights, scale, a_s, tile_map, grid_entries, n_tokens, *, E, I): # noqa: E741 + """The routing sort into ``bufs`` (a ``SortBuffers``), GEMM1's tile map and the sorted scale tiles ``a_s`` (rows < + num_valid) in one launch, up to ``MAX_TOKENS``.""" + import torch + from .launch import _run_compiled + + # the cached CompiledFunction, not a direct jit call (host cost) + _run_compiled( + compile_moe_sort_scales_all( + E=E, topk=topk_ids.shape[1], block_m=bufs.block_m, I=I, tpt=tpt_for(n_tokens) + ), + *bufs.launch_args(topk_ids, topk_weights, n_tokens)[:6], + tile_map.view(-1), + scale.view(torch.uint8).view(-1), + a_s.view(torch.uint8).view(-1), + int(n_tokens), + int(grid_entries), + max(SORT_CTAS, bufs.max_sorted // bufs.block_m), # a CTA per possible block + torch.cuda.current_stream(), + ) # fmt: skip diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_tile_map.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_tile_map.py new file mode 100644 index 0000000..5547078 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/sort_tile_map.py @@ -0,0 +1,221 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``sort.py``'s two-launch routing sort with a CTA-major count workspace, plus GEMM1's n-major tile map. + +The count launch stores [SORT_CTAS, E] expert histograms, so placement loads contiguous experts per wave and sums eight +partial sums through LDS. CTA e writes expert e's tile-map run (``tile_map.py``'s layout) from the start rows it +already has, without the tile map's binary searches. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr + +from .utils import _lds_atomic_add_i32, wave_incl_scan_i32 + +from .sort import SORT_CTAS, THREADS +from amd_kernels.warm_up import launcher + +# eight threads per expert: fewer requests than four, coalesced across experts +COUNT_PARTS = 8 +NB_N = 12 # I / 128 for I1536 +N_BITS = 4 + + +@functools.cache +@launcher +def compile_moe_sort_tile_map(*, E: int, topk: int, block_m: int): + assert E <= SORT_CTAS and topk == 4 + assert (block_m & (block_m - 1)) == 0 and max(E, block_m) <= THREADS + experts_per_cta = (E + SORT_CTAS - 1) // SORT_CTAS + bm_shift = block_m.bit_length() - 1 + tag = f"E{E}_K{topk}_BM{block_m}" + + SCAN_W = THREADS # the padded-count scan runs on the whole block (no branch) + assert E <= SCAN_W and (SCAN_W & (SCAN_W - 1)) == 0 + PART = COUNT_PARTS # threads per expert summing the per-CTA counts in place_pad + assert PART * E <= THREADS and SORT_CTAS % PART == 0 and PART * E <= 2 * SCAN_W + E_WAVES = (E + 63) // 64 # waves holding an expert's padded count + + @fx.struct + class Shared: + count: fx.Array[fx.Int32, E] # per-CTA count / cursor + total: fx.Array[fx.Int32, E] # expert total + padded: fx.Array[fx.Int32, E] + prefix: fx.Array[ + fx.Int32, E + ] # rows of expert e placed by the CTAs before this one + starts: fx.Array[fx.Int32, E + 1] # expert start rows, [E] = padded total + scan: fx.Array[ + fx.Int32, 2 * SCAN_W + ] # ping-pong scan arrays; first the PART sums + + def pair_range(n_tok, bx): + """This CTA's [start, end) of the n_tok * topk routing pairs.""" + total = n_tok * topk + per_cta = (total + (SORT_CTAS - 1)) // SORT_CTAS + start = bx * per_cta + end = start + per_cta + return start, (end < total).select(end, total) + + @flyc.kernel(name=f"sort_count_cta_major_{tag}", known_block_size=[THREADS, 1, 1]) + def sort_count(topk_ids: fx.Tensor, block_offsets: fx.Tensor, n_tok: fx.Int32): + count = fx.SharedAllocator().allocate(Shared).peek().count.ptr + tx, bx = fx.thread_idx.x, fx.block_idx.x + start, end = pair_range(n_tok, bx) + if tx < E: + count[tx] = 0 + fx.gpu.barrier() + for i in range(start + tx, end, THREADS): + _lds_atomic_add_i32(count + fx.Int32(topk_ids[fx.Int32(i)]), 1) + fx.gpu.barrier() + if tx < E: + block_offsets[bx * E + tx] = fx.Int32(count[tx]) + + @flyc.kernel(name=f"sort_place_pad_tile_map_{tag}", known_block_size=[THREADS, 1, 1]) + def sort_place_pad( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + block_offsets: fx.Tensor, + num_valid: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_w: fx.Tensor, + n_tok: fx.Int32, + tile_map: fx.Tensor, + grid_entries: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + cursor, total_c, padded_c = smem.count.ptr, smem.total.ptr, smem.padded.ptr + prefix_c, starts = smem.prefix.ptr, smem.starts.ptr + scan = [smem.scan.ptr, smem.scan.ptr + SCAN_W] + tx, bx = fx.thread_idx.x, fx.block_idx.x + # 1. per expert: the total over the SORT_CTAS counts and the rows the CTAs + # before this one place; PART threads per expert, SORT_CTAS / PART + # counts each (independent loads), partial sums through LDS + e, k = tx % E, tx // E + if tx < PART * E: + tot = fx.Int32(0) + pre = fx.Int32(0) + for i in range_constexpr(SORT_CTAS // PART): + c = k * (SORT_CTAS // PART) + i + v = fx.Int32(block_offsets[c * E + e]) + tot = tot + v + pre = pre + (c < bx).select(v, 0) + scan[0][tx] = tot + scan[1][tx] = pre + fx.gpu.barrier() + if tx < E: + tot = fx.Int32(0) + pre = fx.Int32(0) + for i in range_constexpr(PART): + tot = tot + fx.Int32(scan[0][i * E + tx]) + pre = pre + fx.Int32(scan[1][i * E + tx]) + total_c[tx] = tot + padded_c[tx] = (tot + (block_m - 1)) & ~(block_m - 1) + prefix_c[tx] = pre + fx.gpu.barrier() + # 2. inclusive scan of the padded counts (0 beyond E) -> expert start rows + # (every CTA computes them): per wave by DPP, the totals of the waves + # holding experts through LDS, one barrier + in_e = tx < E + pd = in_e.select(fx.Int32(padded_c[in_e.select(tx, 0)]), 0) + incl = wave_incl_scan_i32(pd) + lane, wave = tx % 64, tx // 64 + scan[0][(lane == 63).select(wave, fx.Int32(E_WAVES))] = incl + fx.gpu.barrier() + for w in range_constexpr(E_WAVES - 1): + incl = incl + (fx.Int32(w) < wave).select(fx.Int32(scan[0][w]), fx.Int32(0)) + if tx < E: + start = incl - pd + starts[tx] = start + cursor[tx] = start + fx.Int32(prefix_c[tx]) + if tx == E - 1: + starts[E] = incl + fx.gpu.barrier() + if bx == 0: + if tx == 0: + num_valid[0] = fx.Int32(starts[E]) + num_valid[1] = fx.Int32(starts[E - 1]) # the last expert's first row + # every CTA knows every expert start: CTA e writes expert e's map + if bx < E: + blo = fx.Int32(starts[bx]) >> bm_shift + cnt = (fx.Int32(starts[bx + 1]) >> bm_shift) - blo + if tx // 64 < NB_N: + nn = tx // 64 + for j in range(tx % 64, cnt, 64): + local = fx.Int32(j) + m = blo + local + tile_map[blo * NB_N + cnt * nn + local] = (m << N_BITS) | nn + for j in range(tx, cnt, THREADS): + sorted_expert_ids[blo + fx.Int32(j)] = bx + vb = fx.Int32(starts[E]) >> bm_shift + tail = vb * NB_N + bx * THREADS + tx + if tail < grid_entries: + tile_map[tail] = fx.Int32(-1) + if bx == 0: + if tx == 0: + tile_map[grid_entries] = vb * NB_N + # 3. place this CTA's pairs + start, end = pair_range(n_tok, bx) + for i in range(start + tx, end, THREADS): + p = fx.Int32(i) + row = _lds_atomic_add_i32(cursor + fx.Int32(topk_ids[p]), 1) + sorted_ids[row] = ((p // topk) & 0x00FFFFFF) | ((p % topk) << 24) + sorted_w[row] = fx.Float32(topk_w[p]) + fx.gpu.barrier() + # 4. padding: every expert pads by < block_m rows, lanes [0, block_m) do it + if tx < block_m: + for ee in range_constexpr(experts_per_cta): + e = bx * experts_per_cta + ee + if e < E: + lo = fx.Int32(starts[e]) + fx.Int32(total_c[e]) + tx + for r in range(lo, fx.Int32(starts[e + 1]), THREADS): + sorted_ids[fx.Int32(r)] = n_tok # token n_tokens, slot 0 + sorted_w[fx.Int32(r)] = fx.Float32(0.0) + + @flyc.jit + def launch_sort_tile_map_mxfp4_i1536( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + num_valid: fx.Tensor, + sorted_w: fx.Tensor, + block_offsets: fx.Tensor, + n_tokens: fx.Int32, + tile_map: fx.Tensor, + grid_entries: fx.Int32, + stream: fx.Stream, + ): + sort_count(topk_ids, block_offsets, n_tokens).launch( + grid=(SORT_CTAS, 1, 1), block=(THREADS, 1, 1), stream=stream + ) + sort_place_pad( + topk_ids, + topk_w, + block_offsets, + num_valid, + sorted_expert_ids, + sorted_ids, + sorted_w, + n_tokens, + tile_map, + grid_entries, + ).launch(grid=(SORT_CTAS, 1, 1), block=(THREADS, 1, 1), stream=stream) + + return launch_sort_tile_map_mxfp4_i1536 + + +def moe_sort_tile_map(bufs, topk_ids, topk_weights, tile_map, grid_entries, n_tokens, *, E): + """The two-launch sort and the I1536 n-major tile map into the caller's buffers.""" + from .launch import _run_compiled + + args = bufs.launch_args(topk_ids, topk_weights, n_tokens) + # the cached CompiledFunction, not a direct jit call (host cost) + _run_compiled( + compile_moe_sort_tile_map(E=E, topk=topk_ids.shape[1], block_m=bufs.block_m), + *args[:-1], tile_map, int(grid_entries), args[-1], + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/tile_map.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/tile_map.py new file mode 100644 index 0000000..62990f8 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/tile_map.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project + +"""GEMM1's expert-major tile order. Sorted rows come in expert order, so an expert's valid m-blocks are one run +``[lo, hi)`` of ``sorted_expert_ids``; its (m, n) tiles are listed together, n-major, so consecutive blocks share the +W13 slab:: + + tile_map[NB_N*lo + n*(hi - lo) + (m - lo)] = m << N_BITS | n (n < NB_N) + tile_map[valid_blocks*NB_N .. grid) = -1 (idle blocks) + tile_map[grid] = valid_blocks*NB_N + +``tile_map_grid`` keeps the table's last ``CLAIM_COUNTERS`` entries past every valid one, so they always come out -1: +the persistent GEMM1s use them as per-XCD claim counters (the others see idle blocks). The routing sorts +(``sort_count_all``, ``sort_tile_map``, ``sort_scales_all``) and ``scale_sort_tile_map`` write it. +""" + +_MAX_BLOCKS = ( + 4096 # >= max_sorted / BM: 1409 at 32768 tokens (BM 128) and at 65536 (BM 256) +) +CLAIM_COUNTERS = 8 # one per XCD (gemm1_prefill_h6144_i3072_persistent) + + +def tile_map_grid(num_m_blocks: int, I: int) -> int: # noqa: E741 + """GEMM1's grid for the table: every (m, n) tile of the allocation rounded up to 8 XCDs, plus the + ``CLAIM_COUNTERS`` spare entries.""" + return (num_m_blocks * (I // 128) + 7) // 8 * 8 + CLAIM_COUNTERS diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/utils.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/utils.py new file mode 100644 index 0000000..b5444be --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/common/utils.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# Copyright (C) 2025-2026 FlyDSL Project Contributors +"""Kernarg preload hints, raw-pointer and LDS-atomic helpers and the DPP wave scan.""" + +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr.typing import T + +LOG2E = 1.4426950408889634 +# launch.compile_hints: the kernel arguments (first 56 bytes) preloaded into SGPRs instead of s_load'ed +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} + + +def _raw(v): + if not isinstance(v, ir.Value) and hasattr(v, "ir_value"): + return v.ir_value() + return v + + +def _tensor_addr(t): + """A launch's tensor argument as a device address (a tensor kernel argument ends the kernarg preload).""" + return fx.Int64(fx.ptrtoint(fx.get_iter(t))) + + +def _global_i32_ptr(addr_i64): + """Typed global i32 pointer at a raw device address (``p[i]`` loads / stores).""" + ptr_ty = fx.PointerType.get( + T.i32, address_space=fx.AddressSpace.Global, alignment=4 + ) + return fx.inttoptr(ptr_ty, fx.Int64(addr_i64)) + + +def _lds_atomic_add_i32(ptr, value): + """``old = *ptr; *ptr += value`` on an LDS i32 pointer (workgroup scope), returns ``old``; fx has no atomic + wrapper, so this is a raw LLVM op.""" + return fx.Int32( + llvm.AtomicRMWOp( + llvm.AtomicBinOp.add, + ptr.llvm_ptr, + fx.Int32(value).ir_value(), + llvm.AtomicOrdering.monotonic, + syncscope="workgroup", + alignment=4, + ).result + ) + + +def _global_i32_at(addr_i64, idx): + return _global_i32_ptr(addr_i64)[idx] + + +_WAVE_SCAN_DPP = ( + (0x111, 0xF, True), + (0x112, 0xF, True), + (0x114, 0xF, True), + (0x118, 0xF, True), + (0x142, 0xA, False), + (0x143, 0xC, False), +) + + +def wave_incl_scan_i32(x): + """Inclusive prefix sum of an i32 over the 64 lanes of a wave (DPP, no LDS).""" + from flydsl._mlir.dialects import rocdl as _rocdl_ops + + zero = _raw(fx.Int32(0)) + for ctrl, row_mask, bound in _WAVE_SCAN_DPP: + t = _rocdl_ops.update_dpp(T.i32, zero, _raw(x), ctrl, row_mask, 0xF, bound) + x = x + fx.Int32(t) + return x diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4.py new file mode 100644 index 0000000..76a9f12 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The W4A4 expert host on 256-row blocks (from 4097 tokens): the routing sort writes GEMM1's tile map +(``common/sort_count_all.py`` to ``SORT_COUNT_ALL_MAX_TOKENS``, ``common/sort_tile_map.py`` above), the scale sort runs +without tile-map CTAs, then the persistent GEMM1 + GEMM2 run in one launch (``gemm12_a4w4``) or two. +""" + +import functools + +import torch + +from .prefill_h6144_i1536 import GEMM2_N_SPLIT, Mxfp4PrefillScales, gemm2_grid + +H, I, BM = 6144, 1536, 256 # noqa: E741 + +# one-launch sort to here: sort + scale sort -1.7 us at 8192, +0.9 at 16384, +9.1 at 32768 vs the two-launch sort +SORT_COUNT_ALL_MAX_TOKENS = 8192 + + +@functools.cache +def _gemm1(num_experts): + from .gemm1_prefill_h6144_i1536 import compile_moe_gemm1_mxfp4_h6144_i1536 + return compile_moe_gemm1_mxfp4_h6144_i1536(E=num_experts, BLOCK_M=BM) + + +@functools.cache +def _gemm2(num_experts, topk, n_split): + from .gemm2_a4w4_bm256_half_rows import compile_moe_gemm2_mxfp4_h6144_i1536 + return compile_moe_gemm2_mxfp4_h6144_i1536( + E=num_experts, topk=topk, n_split=n_split, sort_block_m=BM + ) + + +# one launch (both GEMMs skip a 256-row tile's upper half at <= 128 routes) wins at 4097-8192, two above (README) +GEMM12_TOKENS = (4097, 8192) +# its GEMM2 skips each wave's wholly padded 16-row epilogue groups to here: faster at 4097-7168, +0.4..0.6 us at 8192 +GEMM12_EPILOGUE_GROUPS_MAX_TOKENS = 7424 + + +def gemm12_for(n_tokens: int) -> bool: + return GEMM12_TOKENS[0] <= n_tokens <= GEMM12_TOKENS[1] + + +@functools.cache +def _gemm12(num_experts, topk, n_split, epilogue_groups=False): + from .gemm12_a4w4 import compile_moe_gemm12_mxfp4_h6144_i1536 + return compile_moe_gemm12_mxfp4_h6144_i1536( + E=num_experts, topk=topk, n_split=n_split, bm=256, epilogue_groups=epilogue_groups + ) + + +def a4w4_experts( + x_quant: tuple[torch.Tensor, torch.Tensor], + w13: torch.Tensor, + w2: torch.Tensor, + scales: Mxfp4PrefillScales, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, +) -> torch.Tensor: + """The slots ``topk_weights[:, k] * expert_{topk_ids[:, k]}(x)``, bf16 [n, topk, H], for the MXFP4 input ``x_quant`` + = (e2m1 codes [n, H/2], e8m0 [n, H/32], row-major); deterministic (no atomics into outputs).""" + from .common.launch import ( + _run_compiled, + ) + from .common.sort import SortBuffers + from .common.sort_count_all import moe_sort_count_all + from .common.sort_tile_map import moe_sort_tile_map + from .common.scale_sort_tile_map import scale_sort_tile_map + from .common.tile_map import ( + tile_map_grid, + ) + + xq, xs = x_quant + n = xq.shape[0] + num_experts = w13.shape[0] + topk = topk_ids.shape[1] + dev = xq.device + stream = torch.cuda.current_stream() + n_split = GEMM2_N_SPLIT + b = SortBuffers.allocate(n, num_experts, topk, BM, dev) + ids32 = topk_ids.to(torch.int32).contiguous() + w32 = topk_weights.to(torch.float32).contiguous() + nmb = b.max_sorted // BM + rows = nmb * BM + grid1 = tile_map_grid(nmb, I) + tile_map = torch.empty((grid1 + 1,), dtype=torch.int32, device=dev) + one_launch = n <= SORT_COUNT_ALL_MAX_TOKENS + if one_launch: + moe_sort_count_all(b, ids32, w32, tile_map, grid1, n, E=num_experts, I=I) + else: + moe_sort_tile_map(b, ids32, w32, tile_map, grid1, n, E=num_experts) + a_s = torch.empty( + ((b.sorted_ids.shape[0] + 31) // 32 * 32, H // 32), + dtype=torch.uint8, + device=dev, + ) + scale_sort_tile_map( + xs, b.sorted_ids, b.sorted_expert_ids, b.num_valid_ids, + a_s, tile_map, grid1, n, BM, with_tile_map=False, + ) + hq = torch.empty((rows, I // 2), dtype=torch.uint8, device=dev) + hs = torch.empty((rows * (I // 32),), dtype=torch.uint8, device=dev) + partials = torch.empty((n * topk, H), dtype=torch.bfloat16, device=dev) + if gemm12_for(n): + from .experts_a4w4_bm128 import _GEMM12_MAX_BLOCKS, gemm12_ready_counters + assert nmb <= _GEMM12_MAX_BLOCKS + _run_compiled( + _gemm12(num_experts, topk, n_split, n <= GEMM12_EPILOGUE_GROUPS_MAX_TOKENS), + xq.view(-1), + w13.view(torch.uint8).view(torch.int64).view(-1), + hq.view(-1), + a_s.view(-1), + scales.w13.view(-1), + hs, + b.sorted_ids, + b.sorted_expert_ids, + n, + nmb, + int(a_s.numel()), + tile_map, + grid1, + w2.view(torch.uint8).view(torch.int64).view(-1), + partials.view(-1), + scales.w2.view(-1), + b.sorted_weights, + b.num_valid_ids, + gemm12_ready_counters(dev).data_ptr(), + gemm2_grid(nmb, n_split), + stream, + ) + partials = partials.view(n, topk, H) + return partials + _run_compiled( + _gemm1(num_experts), + xq.view(-1), + w13.view(torch.uint8).view(torch.int64).view(-1), + hq.view(-1), + a_s.view(-1), + scales.w13.view(-1), + hs, + b.sorted_ids, + b.sorted_expert_ids, + n, + nmb, + int(a_s.numel()), + tile_map, + grid1, + stream, + ) + _run_compiled( + _gemm2(num_experts, topk, n_split), + hq.view(-1), + w2.view(torch.uint8).view(torch.int64).view(-1), + partials.view(-1), + hs, + scales.w2.view(-1), + b.sorted_ids, + b.sorted_expert_ids, + b.sorted_weights, + b.num_valid_ids, + n, + nmb, + gemm2_grid(nmb, n_split), + stream, + ) + partials = partials.view(n, topk, H) + return partials diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_bm128.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_bm128.py new file mode 100644 index 0000000..1cfeee5 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_bm128.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The W4A4 expert host on 128-row blocks (641-4096 tokens; 641-3583 with the shared expert): one routing-sort launch +writes the tile map and, to ``SORT_SCALES_MAX_TOKENS``, the sorted scale tiles (``common/sort_scales_all.py``), then +GEMM1 + GEMM2 run in one launch (``gemm12_a4w4``; with the shared expert ``gemm12_a4w4_shared_a8w8``). +""" + +import functools + +import torch + +from .prefill_h6144_i1536 import GEMM2_N_SPLIT, Mxfp4PrefillScales, gemm2_grid + +H, I, BM = 6144, 1536, 128 # noqa: E741 + + +# the sort writes the scale tiles too (one launch): -1.1..-1.6 us at 513-2048, -0.7 at 4096; +1.4 at 8192 (BM256 host) +SORT_SCALES_MAX_TOKENS = 4096 + + +def sort_writes_scales(n_tokens: int) -> bool: + return n_tokens <= SORT_SCALES_MAX_TOKENS + + +# one launch at every row count of this host (README); not below 513: gemm12_a4w4 is not deterministic at 1-17 tokens +GEMM12_TOKENS = (513, 4096) +# its GEMM2 with adaptive chunks at 513-704 and 3584-4096, fixed chunks between +GEMM12_ADAPTIVE_MAX, GEMM12_ADAPTIVE_FROM = 704, 3584 + + +def gemm12_for(n_tokens: int) -> bool: + return GEMM12_TOKENS[0] <= n_tokens <= GEMM12_TOKENS[1] + + +def gemm12_adaptive(n_tokens: int) -> bool: + return n_tokens <= GEMM12_ADAPTIVE_MAX or n_tokens >= GEMM12_ADAPTIVE_FROM + + +@functools.cache +def _gemm12(num_experts, topk, n_split, n_tokens=2049, *, adaptive=False): + from .gemm12_a4w4 import compile_moe_gemm12_mxfp4_h6144_i1536 + return compile_moe_gemm12_mxfp4_h6144_i1536( + E=num_experts, topk=topk, n_tokens=n_tokens, n_split=n_split, adaptive=adaptive, + ) + + +@functools.cache +def _gemm12_shared(n_tokens,adaptive,nt,view=False): + from .gemm12_a4w4_shared_a8w8 import compile_moe_gemm12_mxfp4_shared_mxfp8_h6144_i1536 + return compile_moe_gemm12_mxfp4_shared_mxfp8_h6144_i1536(n_tokens=n_tokens,adaptive=adaptive,shared_nt=nt, + view=view) + + +# m-block counters of gemm12_a4w4 (ready + consumer count per block, 256 B apart), zeroed once per device; every call +# leaves them at zero +_GEMM12_MAX_BLOCKS = 2048 # 1494 BM256 blocks at 87381 tokens +_ready: dict = {} + + +def gemm12_ready_counters(device): + from .a16w4_utils import READY_STRIDE + key = torch.device(device).index + if key not in _ready: + _ready[key] = torch.zeros( + (_GEMM12_MAX_BLOCKS * READY_STRIDE // 4,), dtype=torch.int32, device=device + ) + return _ready[key] + + +def a4w4_experts( + x_quant: tuple[torch.Tensor, torch.Tensor], + w13: torch.Tensor, + w2: torch.Tensor, + scales: Mxfp4PrefillScales, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + *, + shared=None, +) -> torch.Tensor: + """The slots ``topk_weights[:, k] * expert_{topk_ids[:, k]}(x)``, bf16 [n, topk, H], for the MXFP4 input ``x_quant`` + = (e2m1 codes [n, H/2], e8m0 [n, H/32], row-major); deterministic (no atomics into outputs). + + With ``shared=(FP8 codes, tiled scales, SharedWeights)`` (641-3583 rows) it returns ``(partials, shared)`` from the + combined persistent launch, which borrows the otherwise unused last ready-arena slot: the routed blocks stay below + that slot and every call restores both shared metadata words to zero. + """ + from .common.launch import _run_compiled + from .common.sort import SortBuffers + from .common.sort_count_all import moe_sort_count_all + from .common.sort_scales_all import moe_sort_scales_all + from .common.scale_sort_tile_map import scale_sort_tile_map + from .common.tile_map import ( + tile_map_grid, + ) + + xq, xs = x_quant + n = xq.shape[0] + num_experts = w13.shape[0] + topk = topk_ids.shape[1] + dev = xq.device + stream = torch.cuda.current_stream() + n_split = GEMM2_N_SPLIT + # canonical rows dedupe the probe and padding-epilogue variants + probe_rows = 513 if n <= 1536 else (1537 if n <= 2048 else 2049) + b = SortBuffers.allocate(n, num_experts, topk, BM, dev) + nmb = b.max_sorted // BM + rows = nmb * BM + grid1 = tile_map_grid(nmb, I) + tile_map = torch.empty((grid1 + 1,), dtype=torch.int32, device=dev) + ids32 = topk_ids.to(torch.int32).contiguous() + w32 = topk_weights.to(torch.float32).contiguous() + a_s = torch.empty( + ((b.sorted_ids.shape[0] + 31) // 32 * 32, H // 32), + dtype=torch.uint8, + device=dev, + ) + if sort_writes_scales(n): + moe_sort_scales_all(b, ids32, w32, xs, a_s, tile_map, grid1, n, E=num_experts, I=I) + else: + moe_sort_count_all(b, ids32, w32, tile_map, grid1, n, E=num_experts, I=I) + scale_sort_tile_map( + xs, b.sorted_ids, b.sorted_expert_ids, b.num_valid_ids, + a_s, tile_map, grid1, n, BM, with_tile_map=False, + ) # fmt: skip + hq = torch.empty((rows, I // 2), dtype=torch.uint8, device=dev) + hs = torch.empty((rows * (I // 32),), dtype=torch.uint8, device=dev) + partials = torch.empty((n * topk, H), dtype=torch.bfloat16, device=dev) + shared_args=() + if shared is not None: + assert 385<=n<=4096 and nmb<_GEMM12_MAX_BLOCKS # bit-equal from 385 rows + q8,s8,sw=shared + sw13,ss13,sw2,ss2=sw.tp_operands() + hq8=torch.empty((n,I),device=dev,dtype=torch.uint8) + hs8=torch.empty(((n+31)//32*32,I//32),device=dev,dtype=torch.uint8) + shared_out=torch.empty((n,6144),device=dev,dtype=torch.bfloat16) + p1=((n+255)//256)*(I//128) + if n<=1536:p1=((p1+7)//8*8)*2 + shared_args=(q8.view(torch.int8).view(-1),s8.view(-1),sw13.view(torch.int8),ss13, + hq8.view(-1),hs8.view(-1),sw2.view(torch.int8),ss2,shared_out.view(-1), + gemm12_ready_counters(dev).data_ptr()+(_GEMM12_MAX_BLOCKS-1)*256, + (p1+7)//8*8,min(((n+255)//256)*24,256)) + assert (gemm12_for(n) or shared is not None) and nmb <= _GEMM12_MAX_BLOCKS + _run_compiled( + (_gemm12(num_experts, topk, n_split, probe_rows, adaptive=gemm12_adaptive(n)) + if shared is None else _gemm12_shared(probe_rows,gemm12_adaptive(n),n>=2048,shared[2].full)), + xq.view(-1), + w13.view(torch.uint8).view(torch.int64).view(-1), + hq.view(-1), + a_s.view(-1), + scales.w13.view(-1), + hs, + b.sorted_ids, + b.sorted_expert_ids, + n, + nmb, + int(a_s.numel()), + tile_map, + grid1, + w2.view(torch.uint8).view(torch.int64).view(-1), + partials.view(-1), + scales.w2.view(-1), + b.sorted_weights, + b.num_valid_ids, + gemm12_ready_counters(dev).data_ptr(), + *shared_args, + gemm2_grid(nmb, n_split), + stream, + ) + partials = partials.view(n, topk, H) + if shared is not None:return partials,shared_out + return partials diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_small.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_small.py new file mode 100644 index 0000000..68af91d --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/experts_a4w4_small.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The W4A4 expert host on 16 / 32-row blocks (to 640 tokens): below 96 rows the shared expert's tiles lead both routed +launches, rows routed inline (``*_inline_sort``), built by GEMM1 from ``topk_ids`` (``*_id_rows``) or sorted by +``sort_a16w4``; from 96 rows the routed launches alone on sorted rows (the shared expert on the side stream).""" +import torch +from .sort_a16w4 import moe_sort_a16w4 +from .common.launch import _run_compiled + +H, I = 6144, 1536 + +# inline routing to here (from 17 an expert may need two 16-row blocks): -0.27..-0.38 us at 16, -1.0..-2.4 at 12-15 +INLINE_SORT_MAX = 16 + +# BF16 entry with the shared expert: GEMM1 builds its 16-row blocks from topk_ids (``*_id_rows``) at these rows +ID_ROWS_FROM, ID_ROWS_MAX = 16, 96 + + +def id_rows_for(n): + return ID_ROWS_FROM <= n <= ID_ROWS_MAX + + +def id_pairs_for(n): + """Routing pair slots GEMM1 scans in the id-rows tier (a trace-time int, >= 4 * n).""" + return 64 if n <= 16 else 128 if n <= 32 else 256 if n <= 64 else 512 + + +def a4w4_experts(x_quant, w13, w2, scales, topk_weights, topk_ids, *, shared, kw=None, pf=3, tn=256, bm=None, + inline=None, id_rows=False): + """``shared`` = (MXFP8 codes, scales, SharedWeights): the shared gate / up tiles run first in the routed GEMM1 + launch, the shared down tiles first in the routed GEMM2 launch (tile rows and K-waves by ``shared_fold_tile_for``); + returns (slots, shared). ``shared`` None (sorted rows only): the routed launches alone, returns (slots, None).""" + xq, xs = x_quant + n = xq.shape[0] + selected_kw, selected_bm = small_config_for(n) + if kw is None: + kw = selected_kw + if bm is None: + bm = selected_bm + if inline is None: + inline = n <= INLINE_SORT_MAX and not id_rows and shared is not None + if inline: + assert n <= 16 and bm == 16 + from .gemm1_a4w4_inline_sort import compile_gemm1_shared_first + from .gemm2_a4w4_inline_sort import compile_gemm2_shared_first as get_gemm2_sf + ids, wt, eids, nv = topk_ids.data_ptr(), topk_weights.view(-1), topk_ids.view(-1), topk_ids.view(-1) + rows = n * 4 * 16 + elif id_rows: + assert n <= ID_ROWS_MAX and kw == 4 and bm == 16 + from .gemm1_a4w4_id_rows import compile_gemm1_shared_first + from .gemm2_a4w4_id_rows import compile_gemm2_shared_first as get_gemm2_sf + from .sort_a16w4 import max_sorted_rows + rows = max_sorted_rows(n, 128, 4, bm) + # GEMM1 reads topk_ids, writes the block -> expert map (+ block count) and the rows + eids = torch.empty(rows // 16 + 1, device=xq.device, dtype=torch.int32) + nv = torch.empty(rows, device=xq.device, dtype=torch.int32) + ids, wt = topk_ids.view(-1), topk_weights.view(-1) + else: + from .gemm1_a4w4_small import compile_gemm1_shared_first + from .gemm2_a4w4_small import compile_gemm2_shared_first as get_gemm2_sf + ids, wt, eids, nv = moe_sort_a16w4(topk_ids, topk_weights, 128, bm) + rows = ids.numel() + q = torch.empty((rows, I // 2), device=xq.device, dtype=torch.uint8) + s = torch.empty((rows, I // 32), device=xq.device, dtype=torch.uint8) + out = torch.empty((n, 4, 6144), device=xq.device, dtype=torch.bfloat16) + routing_args = (ids,) if inline else (ids, eids, nv) + ids2 = nv if id_rows else ids # GEMM2's rows: GEMM1's published table in the id-rows tier + stream = torch.cuda.current_stream() + if shared is None: + assert kw == 4 and not inline and not id_rows + unused = (q.view(-1),) * 6 + _run_compiled(compile_gemm1_shared_first(kw=kw, pf=pf, bm=bm, sh_bm=0), xq.view(-1), + xs.view(torch.uint8).view(-1), w13.view(-1), scales.w13.view(-1), q.view(-1), s.view(-1), + *routing_args, *unused, n, rows, 0, stream) + _run_compiled(get_gemm2_sf(tn=tn, bm=bm, sh_bm=0), q.view(-1), s.view(-1), w2.view(-1), + scales.w2.view(-1), out.view(-1), ids2, wt, eids, nv, *unused[:5], n, rows, 0, stream) + return out, None + sh_bm, sh_kw, sh_pf = shared_fold_tile_for(n) + assert kw == 4 and (bm == 16 or not inline) and ((sh_bm, sh_kw, sh_pf) == (16, 4, 2) or not inline) + q8, s8, sw = shared + view = sw.full + sw13, ss13, sw2, ss2 = sw.tp_operands() + hq = torch.empty((n, I), device=xq.device, dtype=torch.uint8) + hs = torch.empty((n, I // 32), device=xq.device, dtype=torch.uint8) + nsh = (n + sh_bm - 1) // sh_bm * (I // 32 // (4 // sh_kw)) + g1_kw = {"view": view} if inline else {"sh_bm": sh_bm, "sh_kw": sh_kw, "sh_pf": sh_pf, "view": view} + if id_rows: + g1_kw["pairs"] = id_pairs_for(n) + _run_compiled(compile_gemm1_shared_first(kw=kw, pf=pf, bm=bm, **g1_kw), xq.view(-1), + xs.view(torch.uint8).view(-1), w13.view(-1), scales.w13.view(-1), q.view(-1), s.view(-1), + *routing_args, q8.view(-1), s8.view(-1), sw13, ss13, + hq.view(-1), hs.view(-1), n, rows, nsh, stream) + sh_tn = shared_g2_columns_for(n) + sh_out = torch.empty((n, 6144), device=xq.device, dtype=torch.bfloat16) + g2_bm = shared_fold_down_rows_for(sh_bm, bm) + g2_kw = {"view": view} if inline else {"sh_bm": g2_bm, "view": view} + _run_compiled(get_gemm2_sf(tn=tn, bm=bm, sh_tn=sh_tn, **g2_kw), q.view(-1), s.view(-1), w2.view(-1), + scales.w2.view(-1), out.view(-1), ids2, wt, *((eids, nv) if not inline else ()), + hq.view(-1), hs.view(-1), sw2, ss2, sh_out.view(-1), + n, rows, (n + g2_bm - 1) // g2_bm * (6144 // sh_tn), stream) + return out, sh_out + + +def shared_g2_columns_for(n): + """Columns per shared down tile inside the routed GEMM2 launch.""" + return 64 if n <= 16 else 256 + + +def shared_fold_tile_for(n): + """(rows, GEMM1 K-waves, GEMM1 prefetch depth) of the shared tiles that lead the routed launches: fewer K-waves + give a shared workgroup more columns, so the activations are read fewer times; 64-row tiles read the gate / up + weight a quarter as often as 16-row ones, but need prefetch depth 1 to keep the launch at two waves per SIMD, and + their idle rows cost more at <= 128 rows.""" + if n <= 96: + return 16, 4, 2 + if n <= 128: + return 16, 2, 2 + return (64, 4, 1) if n <= 384 else (64, 2, 1) + + +def shared_fold_down_rows_for(sh_bm, bm): + """Rows per folded shared down tile: at most the routed tile's rows, so the launch keeps the routed GEMM2's LDS + (rows x 1 KB) and registers (16-row down tiles next to 16-row routed tiles: -0.4..-1.0% at 129-256 rows; next to + 32-row routed tiles: +0.9..+1.1%).""" + return min(sh_bm, bm) + + +def small_config_for(n): + """(GEMM1 K-waves, block rows): 32-row blocks from 321 rows, where more experts take more than 16 rows (routed + launches alone: -0.4% at 320, +1.0% at 352 with 16-row blocks).""" + return 4, (16 if n <= 320 else 32) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4.py new file mode 100644 index 0000000..fa97c6f --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The W4A4 GEMM1 and GEMM2 in one launch (128- or 256-row tiles): workgroups [0, N_CTAS) run GEMM1's persistent tile +loop (one per CU, per-XCD claims), the rest GEMM2's (m-tile, n-chunk) work items, each computing what it computes in +the BM256 host's two launches (bit-equal slots). GEMM2 tiles start on the CUs GEMM1 leaves while its last tiles run: no +partial last GEMM1 round, no GEMM2 ramp, no launch gap between the W13 and the W2 stream. + +A GEMM1 wave stores its rows write-through and counts the tile in its m-block's ready counter at the next tile's tail +wait, which retires the workgroup's stores (24 units per block, 48 with split-column tasks); a GEMM2 work item waits +for its block, and the block's last consumer resets the counters (``a16w4_utils.wait_ready``). Counters only order +work: every output element has one writer. With six fixed chunks GEMM2 work item w runs on XCD w % 8 and follows +GEMM1's claim order, earliest first. Deadlock-free: every GEMM1 workgroup has a lower id than every GEMM2 one and never +waits, and each XCD dispatches its workgroups in id order. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import const_expr + +from amd_kernels.warm_up import launcher + + +@launcher +def compile_moe_gemm12_mxfp4_h6144_i1536( + *, E: int, topk: int, n_split: int = 6, bm: int = 128, n_tokens: int = 2049, + adaptive: bool = False, epilogue_groups: bool = False, +): + """The fused launch on ``bm``-row tiles: GEMM1 skips the upper half's math on tiles with at most ``bm`` / 2 valid + rows (``gemm1_a4w4_bm128_half_rows`` / ``gemm1_a4w4_bm256_half_rows``); at 256 GEMM2 skips it too. + + Args: + n_tokens: picks GEMM2's row-probe form (``bm`` 128). + adaptive: GEMM2's adaptive chunks (``bm`` 128); the device-selected split also gives the number of + counter-reset consumers. + epilogue_groups: GEMM2 also skips each wave's wholly padded 16-row epilogue groups (``bm`` 256, + ``gemm2_a4w4_bm256_epilogue_groups``). + """ + if bm == 128: + from .gemm1_a4w4_bm128_half_rows import compile_moe_gemm1_mxfp4_h6144_i1536 as _compile_g1 + from .gemm2_a4w4_bm128 import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + k1 = _compile_g1(E=E, BLOCK_M=128, signal=True) + g2_kw = dict(n_tokens=n_tokens, n_split=n_split) + if adaptive: + from .gemm2_a4w4_bm128_adaptive import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + g2_kw = dict(n_tokens=n_tokens, pad_skip=True) + elif n_tokens <= 1536: + # the fixed-chunk host takes this GEMM2 at 705-1536 rows + from .gemm2_a4w4_bm128_epilogue_groups import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + stem = "kernel_gemm2_mxfp4_h6144_i1536_bm128" + else: + assert bm == 256 + from .gemm1_a4w4_bm256_half_rows import compile_moe_gemm1_mxfp4_h6144_i1536 as _compile_g1 + from .gemm2_a4w4_bm256_half_rows import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + if epilogue_groups: + from .gemm2_a4w4_bm256_epilogue_groups import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + k1 = _compile_g1(E=E, BLOCK_M=256, signal=True) + g2_kw = dict(sort_block_m=256, n_split=n_split) + stem = "kernel_gemm2_mxfp4_h6144_i1536" + n_ready = getattr(k1, "ready_per_block", k1.n_tiles_n * 4) + k2 = _compile_g2(E=E, topk=topk, wait_for=n_ready, **g2_kw) + N_CTAS = k1.n_ctas + t1, s1 = k1.lds_bytes + t2, s2, stg, n_sids = k2.lds_bytes + name = f"kernel_gemm12_mxfp4_h6144_i1536_bm{bm}" + k2.kernel_name[len(stem):] + f"_ns{n_split}_g1_half_rows" + + if bm == 256: + name = (f"kernel_gemm12_mxfp4_h6144_i1536_bm256_half_rows" + + ("_epilogue_groups" if epilogue_groups else "") + f"_ns{n_split}") + + # GEMM1 = tiles + scale slots + claim words, GEMM2 = tiles + scale slots + staging + ids. If both side by side + # exceed the CU's 160 KB (BM256: 161 KB), GEMM1's scale slots share GEMM2's staging bytes, not GEMM2's scale slots. + flat = max(t1, t2) + max(s1, s2) + stg + 4 * n_sids <= 160 * 1024 + a_bytes = max(s1, s2) if flat else max(s1, stg) + b_bytes = stg if flat else s2 + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, max(t1, t2), 16] + scale_lds: fx.Array[fx.Int8, a_bytes, 16] # scales (GEMM1; flat: GEMM2 too) + stage_lds: fx.Array[fx.Int8, b_bytes, 16] # flat: GEMM2's staging, else its scales + sids: fx.Array[fx.Int32, max(n_sids, 4), 16] # GEMM2's ids / GEMM1's claim words + + @flyc.kernel(name=name) + def kernel_gemm12( + A: fx.Tensor, + W13: fx.Tensor, + HQ: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + HS: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid1: fx.Int32, + W2: fx.Tensor, + OUT: fx.Tensor, + W2_scale: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + ready: fx.Int64, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + bx = fx.block_idx.x + if bx < N_CTAS: + k1.tile( + lds.all_lds.ptr, lds.scale_lds.ptr, lds.sids.ptr, bx, + A, W13, HQ, A_scale, W13_scale, HS, sorted_ids, sorted_expert_ids, + n_tokens, num_m_blocks, a_scale_bytes, tile_map_t, grid1, ready, + ) # fmt: skip + else: + if const_expr(flat): + k2.tile( + lds.all_lds.ptr, lds.scale_lds.ptr, lds.stage_lds.ptr, lds.sids.ptr, + bx - N_CTAS, HQ, W2, OUT, HS, W2_scale, sorted_ids, sorted_expert_ids, + sorted_weights, num_valid_ids, n_tokens, num_m_blocks, ready, + ) # fmt: skip + else: + k2.tile( + lds.all_lds.ptr, lds.stage_lds.ptr, lds.scale_lds.ptr, lds.sids.ptr, + bx - N_CTAS, HQ, W2, OUT, HS, W2_scale, sorted_ids, sorted_expert_ids, + sorted_weights, num_valid_ids, n_tokens, num_m_blocks, ready, + ) # fmt: skip + + @flyc.jit + def launch_gemm12_mxfp4_h6144_i1536( + A: fx.Tensor, + W13: fx.Tensor, + HQ: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + HS: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid1: fx.Int32, + W2: fx.Tensor, + OUT: fx.Tensor, + W2_scale: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + ready: fx.Int64, + grid2: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm12( + A, W13, HQ, A_scale, W13_scale, HS, sorted_ids, sorted_expert_ids, + n_tokens, num_m_blocks, a_scale_bytes, tile_map_t, grid1, + W2, OUT, W2_scale, sorted_weights, num_valid_ids, ready, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid2 + N_CTAS, 1, 1), block=(256, 1, 1), stream=stream) # fmt: skip + + launch_gemm12_mxfp4_h6144_i1536.kernel_name = name + return launch_gemm12_mxfp4_h6144_i1536 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4_shared_a8w8.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4_shared_a8w8.py new file mode 100644 index 0000000..0817a90 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm12_a4w4_shared_a8w8.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Routed MXFP4 and shared MXFP8 GEMM1 / GEMM2 in one persistent launch (BF16-entry rows 641-3583). + +Both formats run their own tile bodies and quantization; producers precede consumers and never wait on them, the +workgroup classes alias whole LDS arenas, and no output uses an atomic add. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import const_expr + +from amd_kernels.warm_up import launcher + + +@launcher +def compile_moe_gemm12_mxfp4_shared_mxfp8_h6144_i1536(*,n_tokens,adaptive,shared_nt,view=False): + """Four workgroup classes in launch order: shared GEMM1, routed GEMM1, shared GEMM2, routed GEMM2. + + The producers finish without waiting; a shared GEMM1 workgroup retires every wave's stores before its leader + bumps the ready counter, and the last shared consumer restores the two ready words. + """ + E,topk,bm,n_split=128,4,128,6 + half_rows=True + from .gemm1_a4w4_bm128_half_rows import compile_moe_gemm1_mxfp4_h6144_i1536 as _compile_g1 + if adaptive: + from .gemm2_a4w4_bm128_adaptive import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + g2_kw=dict(n_tokens=n_tokens,pad_skip=True) + elif n_tokens<=1536: + from .gemm2_a4w4_bm128_epilogue_groups import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + g2_kw=dict(n_tokens=n_tokens,n_split=6) + else: + from .gemm2_a4w4_bm128 import compile_moe_gemm2_mxfp4_h6144_i1536 as _compile_g2 + g2_kw=dict(n_tokens=n_tokens,n_split=6) + k1=_compile_g1(E=128,BLOCK_M=128,signal=True) + stem="kernel_gemm2_mxfp4_h6144_i1536_bm128" + n_ready = getattr(k1, "ready_per_block", k1.n_tiles_n * 4) + k2 = _compile_g2(E=E, topk=topk, wait_for=n_ready, **g2_kw) + N_CTAS = k1.n_ctas + t1, s1 = k1.lds_bytes + t2, s2, stg, n_sids = k2.lds_bytes + from .shared_gemm1_a8w8_write_through import compile_shared_gemm1_a8w8_write_through + from .shared_dense import _gemm2 + from .a16w4_utils import global_atomic_add_agent,wait_ready + # 128-row shared tiles through 1536 rows, 256-row tiles above + shared_tiles=(2,4) if n_tokens<=1536 else (4,4) + from .shared_gemm1_a8w8 import VIEW_UP_ROWS + sh1=compile_shared_gemm1_a8w8_write_through(tiles=shared_tiles,up_rows=VIEW_UP_ROWS if view else None) + sh2=_gemm2((4,4),nt=shared_nt,view=view) + SHARED_SUB_SPLIT=sh1.sub_split + name=(f"kernel_gemm12_a4w4_shared_a8w8_h6144_i1536"+k2.kernel_name[len(stem):] + +f"_st{shared_tiles[0]}x{shared_tiles[1]}"+("_nt" if shared_nt else "")+("_view" if view else "")) + + # GEMM1 = tiles + scale slots + claim words, GEMM2 = tiles + scale slots + staging + ids. If both side by side + # exceed the CU's 160 KB (BM256: 161 KB), GEMM1's scale slots share GEMM2's staging bytes instead of its slots. + flat = max(t1, t2) + max(s1, s2) + stg + 4 * n_sids <= 160 * 1024 + a_bytes = max(s1, s2) if flat else max(s1, stg) + b_bytes = stg if flat else s2 + + raw_bytes=max(max(t1,t2)+a_bytes+b_bytes+4*max(n_sids,4),sum(sh2.lds_bytes)) + assert raw_bytes<=160*1024,(raw_bytes,bm,t1,s1,t2,s2,stg,n_sids) + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, raw_bytes, 16] + + @flyc.kernel(name=name) + def kernel_gemm12( + A: fx.Tensor, + W13: fx.Tensor, + HQ: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + HS: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid1: fx.Int32, + W2: fx.Tensor, + OUT: fx.Tensor, + W2_scale: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + ready: fx.Int64, + A8: fx.Tensor, AS8: fx.Tensor, W138: fx.Tensor, WS138: fx.Tensor, + Q8: fx.Tensor, S8: fx.Tensor, W28: fx.Tensor, WS28: fx.Tensor, OUT8: fx.Tensor, + shared_ready: fx.Int64, nsh1: fx.Int32, nsh2: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + bx = fx.block_idx.x + p1=((n_tokens+255)//256)*12 + if const_expr(SHARED_SUB_SPLIT>1): + p1=((p1+7)//8*8)*SHARED_SUB_SPLIT + needed=p1 + if bx < nsh1: + if bx= 4 and (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + K_BYTES = K // 2 + N_BITS = (I // 128 - 1).bit_length() + N0 = I // 16 # 16-column (gate + up) groups per expert + + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES + A_BUFS = 4 * a_lds_size + LDS_TILES_BYTES = A_BUFS + 4 * b_lds_size + + A_WAVE_GROUPS = N_TILES_A // 2 # groups per wave per LDS half + A_HALF_GROUPS = LDS_BLOCK_M // 32 + B_WAVE_GROUPS = 4 # consecutive groups per wave quarter + B_HALF_GROUPS = 2 # quarter block b -> group G + b (via (b//2)*2 + b%2) + SCALE_COLS_OUT = I // 32 # e8m0 per output row + OUT_SC_BLOCKS_PER_ROW32 = SCALE_COLS_OUT // 8 + W_BYTES = (2 * I) * K_BYTES + assert W_BYTES <= 0xFFFFFFFF, "buffer resources address 4 GB" + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, _SCALE_LDS_BYTES, 16] + claim: fx.Array[fx.Int32, 4, 16] + + STORE_KW = {"cache_modifier": SC1} if signal else {} + + @flyc.jit + def tile( + _base_ptr, + _scale_lds_ptr, + claim_slot, + bx, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + ready, + ): + """Workgroup ``bx``'s persistent tile loop on its LDS tiles (``_base_ptr``), scale slots (``_scale_lds_ptr``) + and claim words (``claim_slot``); with ``signal``, ``ready`` holds ``gemm12_a4w4``'s m-block counters.""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + b_cur0 = _Buf(_base_ptr, A_BUFS + 0 * b_lds_size) + b_cur1 = _Buf(_base_ptr, A_BUFS + 1 * b_lds_size) + b_next0 = _Buf(_base_ptr, A_BUFS + 2 * b_lds_size) + b_next1 = _Buf(_base_ptr, A_BUFS + 3 * b_lds_size) + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + tid = fx.thread_idx.x + lane_id = tid % 64 + wave_id = tid // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * 4) + ) + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + # tile_map[grid_size] = valid entries; XCD x (workgroup w runs on XCD w % 8) claims its run [x * per_xcd, + # min((x + 1) * per_xcd, valid)) through tile_map[grid_size - CLAIM_COUNTERS + x] (-1 from the tile map) + tm_rsrc = buffer_ops.create_buffer_resource( + tile_map_t, max_size=False, num_records_bytes=(grid_size + 1) * 4 + ) + xcd = bx % N_XCDS + n_valid = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, grid_size, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + per_xcd = (n_valid + fx.Int32(N_XCDS - 1)) // fx.Int32(N_XCDS) + run_lo = xcd * per_xcd + run_n = _min(per_xcd, n_valid - run_lo) + # only thread 0 is in range: one claim per workgroup + claim_off = (tid == 0).select( + (grid_size - CLAIM_COUNTERS + xcd) * fx.Int32(4), fx.Int32(0x7FFFFFF0) + ) + + def _entry_of(c): + """Tile-map entry of a claimed counter value c (uniform; run index c + 1), -1 past the run.""" + t = c + fx.Int32(1) + ok = t < run_n + e = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, + ok.select(run_lo + t, fx.Int32(0)), + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + return ok.select(e, fx.Int32(-1)) + + def _expert_of(entry): + tile_i = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) + return fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, tile_i, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + + def _a_offsets(entry): + """Per-lane gathered A offsets of both LDS halves (sorted ids loads).""" + m_base = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) * BLOCK_M + offs = [] + for half in range_constexpr(2): + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + half * LDS_BLOCK_M + row, + vec_width=1, + dtype=fx.Int32, + ) + ) + # padded rows: tok == n_tokens -> OOB -> zeros + tok = sid & fx.Int32(0x00FFFFFF) + offs.append(tok * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + class _Tile: + """Loaders and addresses of one tile (entry, its expert, its A offsets).""" + + def __init__(self, entry, expert, a_offs, live=True): + """``live`` False: every load of the tile is out of range (no memory + traffic; the loop exits before anything reads the LDS).""" + self.entry = entry + tile_i = entry >> N_BITS + self.tile_j = entry & ((1 << N_BITS) - 1) + self.m_base = tile_i * BLOCK_M + weight_group0 = self.tile_j * fx.Int32(LDS_BLOCK_N // 16) + group0 = expert * fx.Int32(N0) + weight_group0 + + def _b_offsets(gu): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + # a16w4 order: gate block g at 2g, its up block at 2g + 1 + blk = (weight_group0 + fx.Int32(row // 16)) * fx.Int32( + 2 + ) + fx.Int32(gu) + offs.append( + blk * fx.Int32(K_BYTES * 16) + + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + ) + return offs + + self.scale_gather = ScaleGatherMoE( + A_scale, + W13_scale, + K, + lane_id, + wave_id, + _scale_lds_ptr, + a_scale_bytes * live, + E * 2 * I * (K // 32) * live, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + self.scale_gather.set_wave_base(self.m_base, group0 * fx.Int32(32)) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=n_tokens * K_BYTES * live + ) + b_rsrc = buffer_ops.create_buffer_resource( + W13, + max_size=False, + num_records_bytes=fx.Int32(W_BYTES) * live, + base_byte_offset=fx.Int64(expert) * W_BYTES, + ) + self.a0 = G2SLoaderAsm(a_rsrc, a_offs[:N_TILES_A], N_TILES_A, wave_id) + self.a1 = G2SLoaderAsm(a_rsrc, a_offs[N_TILES_A:], N_TILES_A, wave_id) + self.b0 = _G2SCM(b_rsrc, _b_offsets(0), N_TILES_B, wave_id, cm=" nt") + self.b1 = _G2SCM(b_rsrc, _b_offsets(1), N_TILES_B, wave_id, cm=" nt") + for ld in (self.a0, self.a1, self.b0, self.b1): + ld.set_wave_base(_base_ptr) + + def prologue_groups(self): + """Scale blocks 0-2 and K steps 0, 1 into the cur / next buffers as groups of thunks, in the issue order + the loop-top waits count (a loader's steps stay together: they chain m0).""" + sg = self.scale_gather + return [ + [ + lambda: sg.gather(0, _slot(0)), + lambda: sg.gather(1, _slot(1)), + lambda: sg.gather(2, _slot(2)), + ], + [lambda: self.a0.load(a_cur0, fx.Int32(0 * A_K_STEP))], + [lambda: self.b0.load(b_cur0, fx.Int32(0 * B_K_STEP))], + [lambda: self.b1.load(b_cur1, fx.Int32(0 * B_K_STEP))], + [lambda: self.a1.load(a_cur1, fx.Int32(0 * A_K_STEP))], + [lambda: self.a0.load(a_next0, fx.Int32(1 * A_K_STEP))], + [lambda: self.b0.load(b_next0, fx.Int32(1 * B_K_STEP))], + [lambda: self.b1.load(b_next1, fx.Int32(1 * B_K_STEP))], + [lambda: self.a1.load(a_next1, fx.Int32(1 * A_K_STEP))], + ] + + def prologue(self): + for grp in self.prologue_groups(): + for th in grp: + th() + + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = 2 * 1024 # 128 B of K = two 1 KB blocks + + def _slot(g): + return fx.Int32(g) % fx.Int32(_SCALE_SLOTS) + + _scale_base_ptr = _scale_lds_ptr + + def _publish(i, v): + """LDS claim word i <- v (thread 0 hands a claimed index to the waves).""" + claim_slot[i] = v + + out_rsrc = buffer_ops.create_buffer_resource( + OUT_Q, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * (I // 2)) + ) + osc_rsrc = buffer_ops.create_buffer_resource( + OUT_scale, + max_size=False, + num_records_bytes=num_m_blocks * (BLOCK_M * SCALE_COLS_OUT), + ) + g = lane_id // 16 + r16 = lane_id % 16 + + # ---- first claim (exposed once per workgroup) ---- + first = _buffer_atomic_add(tm_rsrc, claim_off, 1) + if tid == 0: + _publish(0, first) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + entry = _entry_of( + fx.Int32( + _rocdl.readfirstlane(_T.i32, fx.as_ir_value(fx.Int32(claim_slot[0]))) + ) + ) + expert = _expert_of(entry) + a_offs = _a_offsets(entry) + if entry >= 0: + _Tile(entry, expert, a_offs).prologue() + + # ``half`` is a Python constant: the runtime dispatch below emits two bodies with the prologue's G2S load counts + def _run_tile(entry, expert, a_offs, prev_mb=None, half=False): + """One tile (its prologue already issued); claims, prepares and issues the next tile's prologue and returns + its (entry, expert, A offsets). The tile's objects live here: pre-loop objects whose methods the loop + body called would become loop-carried state of the scf.while.""" + tile = _Tile(entry, expert, a_offs) + a0_g2s, a1_g2s, b0_g2s, b1_g2s = tile.a0, tile.a1, tile.b0, tile.b1 + scale_gather = tile.scale_gather + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + mfma = MfmaRowStationaryFp4(N_TILES_A, N_TILES_B) + + # gathers + a_cur0 landed: b0/b1/a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + # b_cur0 AND b_cur1 landed: a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0 + sc0_sbC1) + + def _top_vmcnt(k2): + return 2 * N_TILES_A + 2 * N_TILES_B + 1 # fp4: a gather every step + + def _seg2_vmcnt(k2): + return 2 * N_TILES_A + N_TILES_B + 1 + + _STEP0_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _STEP0_SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(kc_idx, holder): + s = _slot(fx.Int32(kc_idx)) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step( + kc, + k2, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + zero_acc=False, + top_vmcnt=None, + seg2_vmcnt=None, + ): + kc_i = fx.Int32(kc) + top_vmcnt = _top_vmcnt(k2) if top_vmcnt is None else top_vmcnt + seg2_vmcnt = _seg2_vmcnt(k2) if seg2_vmcnt is None else seg2_vmcnt + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC = sc + c00f, c01f, c10f, c11f = accs + + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + a_off = (kc_i + fx.Int32(2)) * fx.Int32(A_K_STEP) + b_off = (kc_i + fx.Int32(2)) * fx.Int32(B_K_STEP) + + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(kc_i + fx.Int32(1), _scn) + _gk = _min(kc_i + fx.Int32(3), fx.Int32(K_ITERS - 1)) + _sc_gather = [lambda: scale_gather.gather(_gk, _slot(_gk))] + + wait_barrier(top_vmcnt) + il = ( + _riffle( + _g2s_thunks(a0_g2s, ac0, a_off, N_TILES_A), + ([] if half else _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False)), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + + il = _riffle(_g2s_thunks(b0_g2s, bc0, b_off, N_TILES_B), _rd_scn[2:]) + # the second half's wait also under c01f's MFMAs (the barrier keeps its own): it overlaps them + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=seg2_vmcnt, lgkmcnt=0)] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + a1f = _a1 + + wait_barrier(seg2_vmcnt) + il = ( + _riffle( + _g2s_thunks(b1_g2s, bc1, b_off, N_TILES_B), + _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False), + ) + + _sc_gather + ) + if const_expr(not half): + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + else: + for th in il: + th() + c10f = c00f + a0nf = _a0n + + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + # likewise the next step's top wait under c11f's MFMAs + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=_top_vmcnt(1 - k2), lgkmcnt=0)] + if const_expr(not half): + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + else: + for th in il: + th() + c11f = c01f + b0nf = _b0n + b1nf = _b1n + + sc_next = (_scn[0], _scn[1], _scn[2] + _scn[3]) + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = 2 * N_TILES_A + n_b = 2 * N_TILES_B + n_ga = N_TILES_A // 2 + n_sc = 2 * n_ga + N_TILES_B + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC = sc + return ( + [_R(v) for v in saR0] + [_R(v) for v in saR1] + [_R(v) for v in sbC] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC = list(flat[o : o + N_TILES_B]) + return (saR0, saR1, sbC) + + _accs0 = tuple([None] * N_ACCUMS for _ in range(4)) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 0, + 0, + a0_frag, + b0_frag, + b1_frag, + sc0, + _accs0, + bufs0, + zero_acc=True, + top_vmcnt=_STEP0_VMCNT, + seg2_vmcnt=_STEP0_SEG2_VMCNT, + ) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 1, 1, a0f, b0f, b1f, sc, accs, _swap_bufs(bufs0) + ) + + init_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + for kk, state in range( # type: ignore[call-overload] + 2, K_ITERS - 2, UNROLL, init=init_state + ): + off = 0 + a0f = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + accs = (c00f, c01f, c10f, c11f) + + bufs = bufs0 + for u in range_constexpr(UNROLL): + a0f, b0f, b1f, sc, accs, bufs = _one_step( + kk + u, u % 2, a0f, b0f, b1f, sc, accs, bufs + ) + + new_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + state = yield new_state + + off = 0 + a0_frag = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + + # Tail step K_ITERS - 2 (no gather; reads the last scale block) + saR0, saR1, sbC = sc + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(fx.Int32(K_ITERS - 1), _scn) + _a1 = [None] * N_TILES_A + wait_barrier((2 * N_TILES_A) + (2 * N_TILES_B)) + # claim the next tile now (thread 0; younger than every load the + # remaining waits count, so they can only over-wait on it) + nxt = _buffer_atomic_add(tm_rsrc, claim_off, 1) + il = ([] if half else _s2r_thunks(a_s2r, a_cur1, _a1, N_TILES_A, False)) + _rd_scn + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 0, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 0) + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + wait_barrier(1 * N_TILES_A) + il = ( + _s2r_thunks(a_s2r, a_next0, _a0n, N_TILES_A, False) + + _s2r_thunks(b_s2r, b_next0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, b_next1, _b1n, N_TILES_B, True) + ) + if const_expr(not half): + c10_frag = mfma.call( + a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 0, interleave=il + ) + else: + for th in il: + th() + c10_frag = c00_frag + if const_expr(not half): + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 0) + else: + c11_frag = c01_frag + a0_frag = _a0n + b0_frag = _b0n + b1_frag = _b1n + + # Tail step K_ITERS - 1 (its A half 1 sits in a_next1) + _a1 = [None] * N_TILES_A + wait_barrier(0) + if tid == 0: + _publish(1, nxt) + if const_expr(signal): + # every wave's loads and stores so far have completed, the previous + # tile's stores included: count its block (4 waves) at no extra wait + if tid == 0: + if prev_mb >= 0: + global_atomic_add_agent( + ready + fx.Int64(prev_mb * READY_STRIDE), 4 + ) + saR0, saR1, sbC = (_scn[0], _scn[1], _scn[2] + _scn[3]) + il = ([] if half else _s2r_thunks(a_s2r, a_next1, _a1, N_TILES_A, False)) + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 1, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 1) + c00_frag = mfma.hold(c00_frag) + if const_expr(not half): + c10_frag = mfma.call(a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 1) + else: + c10_frag = c00_frag + c01_frag = mfma.hold(c01_frag) + if const_expr(not half): + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 1) + else: + c11_frag = c01_frag + c10_frag = mfma.hold(c10_frag) + c11_frag = mfma.hold(c11_frag, wait=_MFMA_RESULT_WAIT) + + # every wave is done with the tile's LDS; the claim is visible + wait_barrier(0) + entry_n = _entry_of( + fx.Int32( + _rocdl.readfirstlane( + _T.i32, fx.as_ir_value(fx.Int32(claim_slot[1])) + ) + ) + ) + expert_n = _expert_of(entry_n) + a_offs_n = _a_offsets(entry_n) + + # ---- epilogue: swiglu-OAI + MXFP4 quant, sorted-row output ---- + col_base = tile.tile_j * LDS_BLOCK_N + wave_j * (N_TILES_B * 16) + colgrp_base = col_base // 32 + + def _epilogue(c_gate, c_up, base_row, inter=(), defer=False): + """One quadrant pair (rows base_row + ti*16 + r16, the wave's 64 gate columns), row tiles in pairs + (2 tp, 2 tp + 1) sharing the e8m0 exchange. ``inter[k]``: thunks run after the k-th (p, ti) block's + SwiGLU. ``defer``: return the stores as thunks and fence each block (the inter thunks carry the next + tile's prologue loads; unfenced, the scheduler bunches them and they stall at issue).""" + stores = [] + k_it = 0 + for p in range_constexpr(N_TILES_B // 2): # 32-col group p + colgrp = colgrp_base + p + sc_in_block = (colgrp % 4) * 64 + r16 * 4 + ((colgrp % 8) // 4) * 2 + for tp in range_constexpr(N_TILES_A // 2): + hs = [] + amaxs = [] + for ti in (2 * tp, 2 * tp + 1): + gv = Vec(c_gate[mfma.idx(ti, 2 * p)]) + gw = Vec(c_gate[mfma.idx(ti, 2 * p + 1)]) + uv = Vec(c_up[mfma.idx(ti, 2 * p)]) + uw = Vec(c_up[mfma.idx(ti, 2 * p + 1)]) + h = [] + for cg, cu in ((gv, uv), (gw, uw)): + h += _swiglu_oai_x4( + [fx.Float32(cg[v]) for v in range_constexpr(4)], + [fx.Float32(cu[v]) for v in range_constexpr(4)], + ) + a01 = _fmax3_abs(h[0], h[1], h[2]) + a34 = _fmax3_abs(h[3], h[4], h[5]) + a67 = _fmax3_abs(h[6], h[7], a01) + amaxs.append(_maxf_nn(a67, a34)) + hs.append(h) + if const_expr(k_it < len(inter)): + for th in inter[k_it]: + th() + k_it += 1 + if const_expr(defer): + fx.rocdl.sched_barrier(0) + e8m0s = _e8m0_pair(amaxs[0], amaxs[1]) + for k in range_constexpr(2): + h = hs[k] + scale_f = _as_f32(e8m0s[k] << 23) + pa = _cvt_pk_fp4(fx.Int32(0), h[0], h[1], scale_f, 0) + pa = _cvt_pk_fp4(pa, h[2], h[3], scale_f, 1) + pb = _cvt_pk_fp4(fx.Int32(0), h[4], h[5], scale_f, 0) + pb = _cvt_pk_fp4(pb, h[6], h[7], scale_f, 1) + # after the swap, lane group g holds tile 2p + g%2, columns (g//2)*8 .. +8: one dword + pa, pb = _permlane16_swap(pa, pb) + row = base_row + (2 * tp + k) * 16 + r16 + col = col_base + (2 * p + (g % 2)) * 16 + (g // 2) * 8 + stores.append( + functools.partial( + buffer_ops.buffer_store, + pa | (pb << 16), + out_rsrc, + row * (I // 2) + col // 2, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + row32 = (base_row // 32) + tp + blk = row32 * OUT_SC_BLOCKS_PER_ROW32 + colgrp // 8 + pair = e8m0s[0] | (e8m0s[1] << 8) + pair16 = fx.Int32(pair).to(fx.Int16) + stores.append( + functools.partial( + buffer_ops.buffer_store, + pair16, + osc_rsrc, + blk * 256 + sc_in_block, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + return stores + + row_r0 = tile.m_base + wave_i * (N_TILES_A * 16) + row_r1 = row_r0 + LDS_BLOCK_M + # the next tile's prologue (past the run: tile 0 with every load out of range, drained before exit) goes out + # under the first quadrant pair's blocks after block 0, which covers the sorted-ids latency; that pair's + # stores wait until those loads are issued (the compiler's wait for the sorted ids would otherwise drain + # them), then spread over the second pair's blocks + live_n = entry_n >= 0 + nt = _Tile( + live_n.select(entry_n, fx.Int32(0)), + expert_n, + a_offs_n, + live=live_n.select(fx.Int32(1), fx.Int32(0)), + ) + grp = nt.prologue_groups() + inter0 = [ + [], + grp[0] + grp[1] + grp[2], + grp[3] + grp[4] + grp[5], + grp[6] + grp[7] + grp[8], + ] + stores0 = _epilogue(c00_frag, c01_frag, row_r0, inter=inter0, defer=True) + n_it = N_TILES_A * (N_TILES_B // 2) + inter1 = [stores0[k::n_it] for k in range(n_it)] + if const_expr(half): + for store in stores0: + store() + else: + _epilogue(c10_frag, c11_frag, row_r1, inter=inter1) + + if const_expr(signal): + return entry_n, expert_n, a_offs_n, entry >> N_BITS + return entry_n, expert_n, a_offs_n + + if const_expr(signal): + prev_mb = fx.Int32(-1) + while entry >= 0: + sid128 = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, (entry >> N_BITS) * BLOCK_M + BLOCK_M // 2, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + if (sid128 & 0x00FFFFFF) < n_tokens: + entry, expert, a_offs, prev_mb = _run_tile(entry, expert, a_offs, prev_mb, half=False) + else: + entry, expert, a_offs, prev_mb = _run_tile(entry, expert, a_offs, prev_mb, half=True) + # the last tile's stores (and sanitized prologue loads) have completed + fx.rocdl.s_waitcnt(vmcnt=0) + if lane_id == 0: + if prev_mb >= 0: + global_atomic_add_agent(ready + fx.Int64(prev_mb * READY_STRIDE), 1) + else: + while entry >= 0: + sid128 = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, (entry >> N_BITS) * BLOCK_M + BLOCK_M // 2, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + if (sid128 & 0x00FFFFFF) < n_tokens: + entry, expert, a_offs = _run_tile(entry, expert, a_offs, half=False) + else: + entry, expert, a_offs = _run_tile(entry, expert, a_offs, half=True) + + # the last tile's (sanitized) prologue loads land before the LDS goes away + fx.rocdl.s_waitcnt(vmcnt=0) + + @flyc.kernel(name="kernel_gemm1_mxfp4_h6144_i1536_bm128_half_rows") + def kernel_gemm1_mxfp4_h6144_i1536_bm128_half_rows( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.claim.ptr, + fx.block_idx.x, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm1_mxfp4_h6144_i1536_bm128_half_rows( + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(N_CTAS, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows.tile = tile + launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows.n_ctas = N_CTAS + launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows.n_tiles_n = I // 128 + launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows.lds_bytes = (LDS_TILES_BYTES, _SCALE_LDS_BYTES) + return launch_gemm1_mxfp4_h6144_i1536_bm128_half_rows diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_bm256_half_rows.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_bm256_half_rows.py new file mode 100644 index 0000000..19706a4 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_bm256_half_rows.py @@ -0,0 +1,931 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm1_prefill_h6144_i1536`` with a second, 128-row body for tiles whose sorted id at row 128 is padding: no +upper-half LDS reads, MFMAs, SwiGLU or quant stores (GEMM2 masks those rows). Both bodies issue the same G2S loads +(padding rows read out of range), so the vmcnt waits and the next tile's prologue under the epilogue still hold; +valid rows are bit for bit the full body's, and ``signal`` counts the same ready units. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from .common.epilogue import _maxf_nn +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_LDS_BYTES, + _SCALE_SLOTS, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _as_f32, + _Buf, + _flat_frag, + _g2s_thunks, + _min, + _permlane16_swap, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .a16w4_utils import READY_STRIDE, SC1, global_atomic_add_agent +from .common.tile_map import CLAIM_COUNTERS +from .i3072.gemm1_prefill_h6144_i3072_persistent import ( + N_XCDS, + MfmaRowStationaryFp4, + _buffer_atomic_add, + _cu_count, + _cvt_pk_fp4, + _e8m0_pair, + _swiglu_oai_x4, +) +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + _G2SCM, + _MFMA_RESULT_WAIT, + BLOCK_K, + _fmax3_abs, +) +from amd_kernels.warm_up import launcher + + +@launcher +def compile_moe_gemm1_mxfp4_h6144_i1536(*, E: int, BLOCK_M: int = 256, signal: bool = False): + """The persistent launcher: one workgroup per CU, ``grid_size`` = the tile map's size. + + Args: + signal: build the body for ``gemm12_a4w4`` (``launch.tile``): write-through stores, and each tile counted in + its m-block's ready counter at the next tile's tail wait (the last tile after the loop's final wait). + """ + H, I = 6144, 1536 # noqa: E741 + assert BLOCK_M == 256 + N_CTAS = _cu_count() + assert N_CTAS % N_XCDS == 0, N_CTAS + K = H + BLOCK_K_BYTES = BLOCK_K + BLOCK_N = 256 # 128 gate cols + the matching 128 up cols + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 16-row tiles per wave per LDS half + N_TILES_B = LDS_BLOCK_N // 2 // 16 # = 4 + + assert K % 256 == 0 and I % LDS_BLOCK_N == 0 + K_ITERS = (K // 2) // BLOCK_K # fp4: 256 K per 128-B step + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert K_ITERS >= 4 and (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + K_BYTES = K // 2 + N_BITS = (I // 128 - 1).bit_length() + N0 = I // 16 # 16-column (gate + up) groups per expert + + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES + A_BUFS = 4 * a_lds_size + LDS_TILES_BYTES = A_BUFS + 4 * b_lds_size + + A_WAVE_GROUPS = N_TILES_A // 2 # groups per wave per LDS half + A_HALF_GROUPS = LDS_BLOCK_M // 32 + B_WAVE_GROUPS = 4 # consecutive groups per wave quarter + B_HALF_GROUPS = 2 # quarter block b -> group G + b (via (b//2)*2 + b%2) + SCALE_COLS_OUT = I // 32 # e8m0 per output row + OUT_SC_BLOCKS_PER_ROW32 = SCALE_COLS_OUT // 8 + W_BYTES = (2 * I) * K_BYTES + assert W_BYTES <= 0xFFFFFFFF, "buffer resources address 4 GB" + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, _SCALE_LDS_BYTES, 16] + claim: fx.Array[fx.Int32, 4, 16] + + STORE_KW = {"cache_modifier": SC1} if signal else {} + + @flyc.jit + def tile( + _base_ptr, + _scale_lds_ptr, + claim_slot, + bx, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + ready, + ): + """Workgroup ``bx``'s persistent tile loop on its LDS tiles (``_base_ptr``), scale slots (``_scale_lds_ptr``) + and claim words (``claim_slot``); with ``signal``, ``ready`` holds ``gemm12_a4w4``'s m-block counters.""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + b_cur0 = _Buf(_base_ptr, A_BUFS + 0 * b_lds_size) + b_cur1 = _Buf(_base_ptr, A_BUFS + 1 * b_lds_size) + b_next0 = _Buf(_base_ptr, A_BUFS + 2 * b_lds_size) + b_next1 = _Buf(_base_ptr, A_BUFS + 3 * b_lds_size) + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + tid = fx.thread_idx.x + lane_id = tid % 64 + wave_id = tid // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * 4) + ) + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + # tile_map[grid_size] = valid entries; XCD x (workgroup w runs on XCD w % 8) claims its run [x * per_xcd, + # min((x + 1) * per_xcd, valid)) through tile_map[grid_size - CLAIM_COUNTERS + x] (-1 from the tile map) + tm_rsrc = buffer_ops.create_buffer_resource( + tile_map_t, max_size=False, num_records_bytes=(grid_size + 1) * 4 + ) + xcd = bx % N_XCDS + n_valid = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, grid_size, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + per_xcd = (n_valid + fx.Int32(N_XCDS - 1)) // fx.Int32(N_XCDS) + run_lo = xcd * per_xcd + run_n = _min(per_xcd, n_valid - run_lo) + # only thread 0 is in range: one claim per workgroup + claim_off = (tid == 0).select( + (grid_size - CLAIM_COUNTERS + xcd) * fx.Int32(4), fx.Int32(0x7FFFFFF0) + ) + + def _entry_of(c): + """Tile-map entry of a claimed counter value c (uniform; run index c + 1), -1 past the run.""" + t = c + fx.Int32(1) + ok = t < run_n + e = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, + ok.select(run_lo + t, fx.Int32(0)), + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + return ok.select(e, fx.Int32(-1)) + + def _expert_of(entry): + tile_i = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) + return fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, tile_i, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + + def _a_offsets(entry): + """Per-lane gathered A offsets of both LDS halves (sorted ids loads).""" + m_base = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) * BLOCK_M + offs = [] + for half in range_constexpr(2): + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + half * LDS_BLOCK_M + row, + vec_width=1, + dtype=fx.Int32, + ) + ) + # padded rows: tok == n_tokens -> OOB -> zeros + tok = sid & fx.Int32(0x00FFFFFF) + offs.append(tok * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + class _Tile: + """Loaders and addresses of one tile (entry, its expert, its A offsets).""" + + def __init__(self, entry, expert, a_offs, live=True): + """``live`` False: every load of the tile is out of range (no memory + traffic; the loop exits before anything reads the LDS).""" + self.entry = entry + tile_i = entry >> N_BITS + self.tile_j = entry & ((1 << N_BITS) - 1) + self.m_base = tile_i * BLOCK_M + weight_group0 = self.tile_j * fx.Int32(LDS_BLOCK_N // 16) + group0 = expert * fx.Int32(N0) + weight_group0 + + def _b_offsets(gu): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + # a16w4 order: gate block g at 2g, its up block at 2g + 1 + blk = (weight_group0 + fx.Int32(row // 16)) * fx.Int32( + 2 + ) + fx.Int32(gu) + offs.append( + blk * fx.Int32(K_BYTES * 16) + + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + ) + return offs + + self.scale_gather = ScaleGatherMoE( + A_scale, + W13_scale, + K, + lane_id, + wave_id, + _scale_lds_ptr, + a_scale_bytes * live, + E * 2 * I * (K // 32) * live, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + self.scale_gather.set_wave_base(self.m_base, group0 * fx.Int32(32)) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=n_tokens * K_BYTES * live + ) + b_rsrc = buffer_ops.create_buffer_resource( + W13, + max_size=False, + num_records_bytes=fx.Int32(W_BYTES) * live, + base_byte_offset=fx.Int64(expert) * W_BYTES, + ) + self.a0 = G2SLoaderAsm(a_rsrc, a_offs[:N_TILES_A], N_TILES_A, wave_id) + self.a1 = G2SLoaderAsm(a_rsrc, a_offs[N_TILES_A:], N_TILES_A, wave_id) + self.b0 = _G2SCM(b_rsrc, _b_offsets(0), N_TILES_B, wave_id, cm=" nt") + self.b1 = _G2SCM(b_rsrc, _b_offsets(1), N_TILES_B, wave_id, cm=" nt") + for ld in (self.a0, self.a1, self.b0, self.b1): + ld.set_wave_base(_base_ptr) + + def prologue_groups(self): + """Scale blocks 0-2 and K steps 0, 1 into the cur / next buffers as groups of thunks, in the issue order + the loop-top waits count (a loader's steps stay together: they chain m0).""" + sg = self.scale_gather + return [ + [ + lambda: sg.gather(0, _slot(0)), + lambda: sg.gather(1, _slot(1)), + lambda: sg.gather(2, _slot(2)), + ], + [lambda: self.a0.load(a_cur0, fx.Int32(0 * A_K_STEP))], + [lambda: self.b0.load(b_cur0, fx.Int32(0 * B_K_STEP))], + [lambda: self.b1.load(b_cur1, fx.Int32(0 * B_K_STEP))], + [lambda: self.a1.load(a_cur1, fx.Int32(0 * A_K_STEP))], + [lambda: self.a0.load(a_next0, fx.Int32(1 * A_K_STEP))], + [lambda: self.b0.load(b_next0, fx.Int32(1 * B_K_STEP))], + [lambda: self.b1.load(b_next1, fx.Int32(1 * B_K_STEP))], + [lambda: self.a1.load(a_next1, fx.Int32(1 * A_K_STEP))], + ] + + def prologue(self): + for grp in self.prologue_groups(): + for th in grp: + th() + + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = 2 * 1024 # 128 B of K = two 1 KB blocks + + def _slot(g): + return fx.Int32(g) % fx.Int32(_SCALE_SLOTS) + + _scale_base_ptr = _scale_lds_ptr + + def _publish(i, v): + """LDS claim word i <- v (thread 0 hands a claimed index to the waves).""" + claim_slot[i] = v + + out_rsrc = buffer_ops.create_buffer_resource( + OUT_Q, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * (I // 2)) + ) + osc_rsrc = buffer_ops.create_buffer_resource( + OUT_scale, + max_size=False, + num_records_bytes=num_m_blocks * (BLOCK_M * SCALE_COLS_OUT), + ) + g = lane_id // 16 + r16 = lane_id % 16 + + # ---- first claim (exposed once per workgroup) ---- + first = _buffer_atomic_add(tm_rsrc, claim_off, 1) + if tid == 0: + _publish(0, first) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + entry = _entry_of( + fx.Int32( + _rocdl.readfirstlane(_T.i32, fx.as_ir_value(fx.Int32(claim_slot[0]))) + ) + ) + expert = _expert_of(entry) + a_offs = _a_offsets(entry) + if entry >= 0: + _Tile(entry, expert, a_offs).prologue() + + # ``half`` is a Python constant: the runtime dispatch below emits two bodies with the prologue's G2S load counts + def _run_tile(entry, expert, a_offs, prev_mb=None, half=False): + """One tile (its prologue already issued); claims, prepares and issues the next tile's prologue and returns + its (entry, expert, A offsets). The tile's objects live here: pre-loop objects whose methods the loop + body called would become loop-carried state of the scf.while.""" + tile = _Tile(entry, expert, a_offs) + a0_g2s, a1_g2s, b0_g2s, b1_g2s = tile.a0, tile.a1, tile.b0, tile.b1 + scale_gather = tile.scale_gather + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + mfma = MfmaRowStationaryFp4(N_TILES_A, N_TILES_B) + + # gathers + a_cur0 landed: b0/b1/a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + # b_cur0 AND b_cur1 landed: a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0 + sc0_sbC1) + + def _top_vmcnt(k2): + return 2 * N_TILES_A + 2 * N_TILES_B + 1 # fp4: a gather every step + + def _seg2_vmcnt(k2): + return 2 * N_TILES_A + N_TILES_B + 1 + + _STEP0_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _STEP0_SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(kc_idx, holder): + s = _slot(fx.Int32(kc_idx)) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step( + kc, + k2, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + zero_acc=False, + top_vmcnt=None, + seg2_vmcnt=None, + ): + kc_i = fx.Int32(kc) + top_vmcnt = _top_vmcnt(k2) if top_vmcnt is None else top_vmcnt + seg2_vmcnt = _seg2_vmcnt(k2) if seg2_vmcnt is None else seg2_vmcnt + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC = sc + c00f, c01f, c10f, c11f = accs + + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + a_off = (kc_i + fx.Int32(2)) * fx.Int32(A_K_STEP) + b_off = (kc_i + fx.Int32(2)) * fx.Int32(B_K_STEP) + + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(kc_i + fx.Int32(1), _scn) + _gk = _min(kc_i + fx.Int32(3), fx.Int32(K_ITERS - 1)) + _sc_gather = [lambda: scale_gather.gather(_gk, _slot(_gk))] + + wait_barrier(top_vmcnt) + il = ( + _riffle( + _g2s_thunks(a0_g2s, ac0, a_off, N_TILES_A), + ([] if half else _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False)), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + + il = _riffle(_g2s_thunks(b0_g2s, bc0, b_off, N_TILES_B), _rd_scn[2:]) + # the second half's wait also under c01f's MFMAs (the barrier keeps its own): it overlaps them + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=seg2_vmcnt, lgkmcnt=0)] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + a1f = _a1 + + wait_barrier(seg2_vmcnt) + il = ( + _riffle( + _g2s_thunks(b1_g2s, bc1, b_off, N_TILES_B), + _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False), + ) + + _sc_gather + ) + if const_expr(not half): + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + else: + for th in il: + th() + c10f = c00f + a0nf = _a0n + + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + # likewise the next step's top wait under c11f's MFMAs + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=_top_vmcnt(1 - k2), lgkmcnt=0)] + if const_expr(not half): + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + else: + for th in il: + th() + c11f = c01f + b0nf = _b0n + b1nf = _b1n + + sc_next = (_scn[0], _scn[1], _scn[2] + _scn[3]) + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = 2 * N_TILES_A + n_b = 2 * N_TILES_B + n_ga = N_TILES_A // 2 + n_sc = 2 * n_ga + N_TILES_B + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC = sc + return ( + [_R(v) for v in saR0] + [_R(v) for v in saR1] + [_R(v) for v in sbC] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC = list(flat[o : o + N_TILES_B]) + return (saR0, saR1, sbC) + + _accs0 = tuple([None] * N_ACCUMS for _ in range(4)) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 0, + 0, + a0_frag, + b0_frag, + b1_frag, + sc0, + _accs0, + bufs0, + zero_acc=True, + top_vmcnt=_STEP0_VMCNT, + seg2_vmcnt=_STEP0_SEG2_VMCNT, + ) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 1, 1, a0f, b0f, b1f, sc, accs, _swap_bufs(bufs0) + ) + + init_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + for kk, state in range( # type: ignore[call-overload] + 2, K_ITERS - 2, UNROLL, init=init_state + ): + off = 0 + a0f = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + accs = (c00f, c01f, c10f, c11f) + + bufs = bufs0 + for u in range_constexpr(UNROLL): + a0f, b0f, b1f, sc, accs, bufs = _one_step( + kk + u, u % 2, a0f, b0f, b1f, sc, accs, bufs + ) + + new_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + state = yield new_state + + off = 0 + a0_frag = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + + # Tail step K_ITERS - 2 (no gather; reads the last scale block) + saR0, saR1, sbC = sc + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(fx.Int32(K_ITERS - 1), _scn) + _a1 = [None] * N_TILES_A + wait_barrier((2 * N_TILES_A) + (2 * N_TILES_B)) + # claim the next tile now (thread 0; younger than every load the + # remaining waits count, so they can only over-wait on it) + nxt = _buffer_atomic_add(tm_rsrc, claim_off, 1) + il = ([] if half else _s2r_thunks(a_s2r, a_cur1, _a1, N_TILES_A, False)) + _rd_scn + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 0, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 0) + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + wait_barrier(1 * N_TILES_A) + il = ( + _s2r_thunks(a_s2r, a_next0, _a0n, N_TILES_A, False) + + _s2r_thunks(b_s2r, b_next0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, b_next1, _b1n, N_TILES_B, True) + ) + if const_expr(not half): + c10_frag = mfma.call( + a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 0, interleave=il + ) + else: + for th in il: + th() + c10_frag = c00_frag + if const_expr(not half): + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 0) + else: + c11_frag = c01_frag + a0_frag = _a0n + b0_frag = _b0n + b1_frag = _b1n + + # Tail step K_ITERS - 1 (its A half 1 sits in a_next1) + _a1 = [None] * N_TILES_A + wait_barrier(0) + if tid == 0: + _publish(1, nxt) + if const_expr(signal): + # every wave's loads and stores so far have completed, the previous + # tile's stores included: count its block (4 waves) at no extra wait + if tid == 0: + if prev_mb >= 0: + global_atomic_add_agent( + ready + fx.Int64(prev_mb * READY_STRIDE), 4 + ) + saR0, saR1, sbC = (_scn[0], _scn[1], _scn[2] + _scn[3]) + il = ([] if half else _s2r_thunks(a_s2r, a_next1, _a1, N_TILES_A, False)) + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 1, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 1) + c00_frag = mfma.hold(c00_frag) + if const_expr(not half): + c10_frag = mfma.call(a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 1) + else: + c10_frag = c00_frag + c01_frag = mfma.hold(c01_frag) + if const_expr(not half): + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 1) + else: + c11_frag = c01_frag + c10_frag = mfma.hold(c10_frag) + c11_frag = mfma.hold(c11_frag, wait=_MFMA_RESULT_WAIT) + + # every wave is done with the tile's LDS; the claim is visible + wait_barrier(0) + entry_n = _entry_of( + fx.Int32( + _rocdl.readfirstlane( + _T.i32, fx.as_ir_value(fx.Int32(claim_slot[1])) + ) + ) + ) + expert_n = _expert_of(entry_n) + a_offs_n = _a_offsets(entry_n) + + # ---- epilogue: swiglu-OAI + MXFP4 quant, sorted-row output ---- + col_base = tile.tile_j * LDS_BLOCK_N + wave_j * (N_TILES_B * 16) + colgrp_base = col_base // 32 + + def _epilogue(c_gate, c_up, base_row, inter=(), defer=False): + """One quadrant pair (rows base_row + ti*16 + r16, the wave's 64 gate columns), row tiles in pairs + (2 tp, 2 tp + 1) sharing the e8m0 exchange. ``inter[k]``: thunks run after the k-th (p, ti) block's + SwiGLU. ``defer``: return the stores as thunks and fence each block (the inter thunks carry the next + tile's prologue loads; unfenced, the scheduler bunches them and they stall at issue).""" + stores = [] + k_it = 0 + for p in range_constexpr(N_TILES_B // 2): # 32-col group p + colgrp = colgrp_base + p + sc_in_block = (colgrp % 4) * 64 + r16 * 4 + ((colgrp % 8) // 4) * 2 + for tp in range_constexpr(N_TILES_A // 2): + hs = [] + amaxs = [] + for ti in (2 * tp, 2 * tp + 1): + gv = Vec(c_gate[mfma.idx(ti, 2 * p)]) + gw = Vec(c_gate[mfma.idx(ti, 2 * p + 1)]) + uv = Vec(c_up[mfma.idx(ti, 2 * p)]) + uw = Vec(c_up[mfma.idx(ti, 2 * p + 1)]) + h = [] + for cg, cu in ((gv, uv), (gw, uw)): + h += _swiglu_oai_x4( + [fx.Float32(cg[v]) for v in range_constexpr(4)], + [fx.Float32(cu[v]) for v in range_constexpr(4)], + ) + a01 = _fmax3_abs(h[0], h[1], h[2]) + a34 = _fmax3_abs(h[3], h[4], h[5]) + a67 = _fmax3_abs(h[6], h[7], a01) + amaxs.append(_maxf_nn(a67, a34)) + hs.append(h) + if const_expr(k_it < len(inter)): + for th in inter[k_it]: + th() + k_it += 1 + if const_expr(defer): + fx.rocdl.sched_barrier(0) + e8m0s = _e8m0_pair(amaxs[0], amaxs[1]) + for k in range_constexpr(2): + h = hs[k] + scale_f = _as_f32(e8m0s[k] << 23) + pa = _cvt_pk_fp4(fx.Int32(0), h[0], h[1], scale_f, 0) + pa = _cvt_pk_fp4(pa, h[2], h[3], scale_f, 1) + pb = _cvt_pk_fp4(fx.Int32(0), h[4], h[5], scale_f, 0) + pb = _cvt_pk_fp4(pb, h[6], h[7], scale_f, 1) + # after the swap, lane group g holds tile 2p + g%2, columns (g//2)*8 .. +8: one dword + pa, pb = _permlane16_swap(pa, pb) + row = base_row + (2 * tp + k) * 16 + r16 + col = col_base + (2 * p + (g % 2)) * 16 + (g // 2) * 8 + stores.append( + functools.partial( + buffer_ops.buffer_store, + pa | (pb << 16), + out_rsrc, + row * (I // 2) + col // 2, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + row32 = (base_row // 32) + tp + blk = row32 * OUT_SC_BLOCKS_PER_ROW32 + colgrp // 8 + pair = e8m0s[0] | (e8m0s[1] << 8) + pair16 = fx.Int32(pair).to(fx.Int16) + stores.append( + functools.partial( + buffer_ops.buffer_store, + pair16, + osc_rsrc, + blk * 256 + sc_in_block, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + return stores + + row_r0 = tile.m_base + wave_i * (N_TILES_A * 16) + row_r1 = row_r0 + LDS_BLOCK_M + # the next tile's prologue (past the run: tile 0 with every load out of range, drained before exit) goes out + # under the first quadrant pair's blocks after block 0, which covers the sorted-ids latency; that pair's + # stores wait until those loads are issued (the compiler's wait for the sorted ids would otherwise drain + # them), then spread over the second pair's blocks + live_n = entry_n >= 0 + nt = _Tile( + live_n.select(entry_n, fx.Int32(0)), + expert_n, + a_offs_n, + live=live_n.select(fx.Int32(1), fx.Int32(0)), + ) + grp = nt.prologue_groups() + inter0 = [ + [], + grp[0] + grp[1], + grp[2], + grp[3], + grp[4], + grp[5], + grp[6], + grp[7] + grp[8], + ] + stores0 = _epilogue(c00_frag, c01_frag, row_r0, inter=inter0, defer=True) + n_it = N_TILES_A * (N_TILES_B // 2) + inter1 = [stores0[k::n_it] for k in range(n_it)] + if const_expr(half): + for store in stores0: + store() + else: + _epilogue(c10_frag, c11_frag, row_r1, inter=inter1) + + if const_expr(signal): + return entry_n, expert_n, a_offs_n, entry >> N_BITS + return entry_n, expert_n, a_offs_n + + if const_expr(signal): + prev_mb = fx.Int32(-1) + while entry >= 0: + sid128 = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, (entry >> N_BITS) * BLOCK_M + BLOCK_M // 2, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + if (sid128 & 0x00FFFFFF) < n_tokens: + entry, expert, a_offs, prev_mb = _run_tile(entry, expert, a_offs, prev_mb, half=False) + else: + entry, expert, a_offs, prev_mb = _run_tile(entry, expert, a_offs, prev_mb, half=True) + # the last tile's stores (and sanitized prologue loads) have completed + fx.rocdl.s_waitcnt(vmcnt=0) + if lane_id == 0: + if prev_mb >= 0: + global_atomic_add_agent(ready + fx.Int64(prev_mb * READY_STRIDE), 1) + else: + while entry >= 0: + sid128 = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, (entry >> N_BITS) * BLOCK_M + BLOCK_M // 2, + vec_width=1, dtype=fx.Int32, is_scalar=True)) + if (sid128 & 0x00FFFFFF) < n_tokens: + entry, expert, a_offs = _run_tile(entry, expert, a_offs, half=False) + else: + entry, expert, a_offs = _run_tile(entry, expert, a_offs, half=True) + + # the last tile's (sanitized) prologue loads land before the LDS goes away + fx.rocdl.s_waitcnt(vmcnt=0) + + @flyc.kernel(name="kernel_gemm1_mxfp4_h6144_i1536_bm256_half_rows") + def kernel_gemm1_mxfp4_h6144_i1536_bm256_half_rows( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.claim.ptr, + fx.block_idx.x, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm1_mxfp4_h6144_i1536_bm256_half_rows( + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(N_CTAS, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows.tile = tile + launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows.n_ctas = N_CTAS + launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows.n_tiles_n = I // 128 + launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows.lds_bytes = (LDS_TILES_BYTES, _SCALE_LDS_BYTES) + return launch_gemm1_mxfp4_h6144_i1536_bm256_half_rows diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_id_rows.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_id_rows.py new file mode 100644 index 0000000..b160d64 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_id_rows.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm1_a4w4_small.py`` whose blocks build their rows from ``topk_ids`` (``routing_ids``: no sort launch, no atomic +counters): every wave histograms the M*4 routing pairs in its own LDS slice, scans the 128 counts into the canonical +expert blocks (experts ascending, ceil(count / 16) blocks each) and finds its block's rows by ballot in pair order. +Column group 0 publishes the block's expert and its 16 packed rows for GEMM2. +""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher +from .routing_ids import wave_expert, wave_block_rows, WAVE_BYTES + +H, I, E = 6144, 1536, 128 + + +def _raw(x): + """Unwrap only at LLVM intrinsic and inline-assembly boundaries.""" + return x.ir_value() if hasattr(x, "ir_value") else x + + +def _mfma(a, b, acc, sa, sb, b_byte, transpose=False): + """Native gfx950 FP4 scaled MFMA into AGPRs; transpose selects GEMM1 C layout.""" + ty = T.vec(4, T.f32) + a_byte = 0 + if transpose: + a, b, sa, sb = b, a, sb, sa + a_byte, b_byte = b_byte, 0 + asm = (f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {'$0' if acc is not None else '0'}, $3, $4 " + f"op_sel:[{a_byte % 2},{b_byte % 2},0] op_sel_hi:[{a_byte // 2},{b_byte // 2},0] cbsz:4 blgp:4") + ops = [_raw(a), _raw(b), _raw(sa), _raw(sb)] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(_raw(acc)) + cons += ",0" + return llvm.inline_asm(ty, ops, asm, cons, has_side_effects=True) + + +def _hold(xs): + """Keep the required inline-MFMA latency before any accumulator reads.""" + ty = ir.Type.parse("!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(xs)) + ")>") + cons = ",".join(["=a"] * len(xs) + [str(i) for i in range(len(xs))]) + v = llvm.inline_asm(ty, [_raw(x) for x in xs], "s_nop 7\ns_nop 7\ns_nop 7", cons, has_side_effects=True) + return [llvm.extractvalue(T.vec(4, T.f32), v, [i]) for i in range(len(xs))] + + +def _lds_ptr(base, off): + """Byte-address the aligned LDS exchange and transpose regions.""" + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(base + off))) + + +def _fp4(old, a, b, scale, sel): + """FlyDSL lacks a wrapper for this FP32-to-FP4 intrinsic.""" + return fx.Int32(llvm.call_intrinsic(T.i32, "llvm.amdgcn.cvt.scalef32.pk.fp4.f32", + [_raw(old), _raw(a), _raw(b), _raw(scale), _raw(fx.Int32(sel))], [], [])) + + +def _swiglu(g, u): + g = (g < fx.Float32(7)).select(g, fx.Float32(7)) + u = (u < fx.Float32(7)).select(u, fx.Float32(7)) + u = (u > fx.Float32(-7)).select(u, fx.Float32(-7)) + t = (g * fx.Float32(1.702)) * fx.Float32(-1.4426950408889634) + e = fx.Float32(fx.rocdl.exp2(T.f32, _raw(t))) + r = fx.Float32(fx.rocdl.rcp(T.f32, _raw(fx.Float32(1) + e))) + return (g * r) * (u + fx.Float32(1)) + + +@functools.cache +def compile_gemm1(*, kw=1, pf=3, bm=16, pairs=256): + assert kw in (1, 2, 4) and bm in (16, 32) and pairs in (64, 128, 256, 512) + nw = 4 // kw + tn, kt, mr = nw * 32, 48 // kw, bm // 16 + assert bm == 16 + ex = 4 * mr * 4 * 64 * 16 + route = ex if kw > 1 else 16 # the waves' routing slices follow the K-wave exchange + + @flyc.jit + def routed_tile(base, rb, A, AS, W, WS, Q, S, IDS, EIDS, NV, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + nk, kk = wave % nw, wave // nw + mb, nb = rb // (I // tn), rb % (I // tn) + hb = base + route + wave * WAVE_BYTES + expert, first, blocks, pair_ids = wave_expert(IDS, m, mb, lane, hb, pairs) + if (tx == 0) & (nb == 0): + if mb == 0: + EIDS[rows // 16] = blocks + if mb < blocks: + EIDS[mb] = expert + if mb < blocks: + col = nb * tn + nk * 32 + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * (H // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=m * (H // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=I * H, + base_byte_offset=fx.Int64(expert) * (I * H)) + ws = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * I * (H // 16)) + qr = buffer_ops.create_buffer_resource(Q, max_size=False, num_records_bytes=rows * (I // 2)) + ss = buffer_ops.create_buffer_resource(S, max_size=False, num_records_bytes=rows * (I // 32)) + wd = [[(((col // 16 + ni) * 2 + gu) * 48 * 1024 + q16 * 256 + l16 * 16) // 4 + kk * kt * 256 + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + swd = [((expert * (I // 16) + col // 16 + ni) * 24) * 64 + q16 * 16 + l16 + kk * (kt // 2) * 64 + for ni in range_constexpr(2)] + + bpre = [[buffer_ops.buffer_load(wr, wd[ni][gu], vec_width=4, dtype=fx.Int32, cache_modifier=2) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + spre = [buffer_ops.buffer_load(ws, swd[ni], vec_width=1, dtype=fx.Int32) for ni in range_constexpr(2)] + # the rows after the first weight loads are in flight + packed = wave_block_rows(pair_ids, expert, first, lane, hb) + if (tx < 16) & (nb == 0): + NV[mb * 16 + tx] = packed + tok = [packed & 0xFFFFFF for mi in range_constexpr(mr)] + ad = [(tok[mi] * (H // 2) + kk * (kt * 64) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [tok[mi] * (H // 32) // 4 + kk * kt for mi in range_constexpr(mr)] + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) + for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, + soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = bpre if k == 0 else [[buffer_ops.buffer_load(wr, wd[ni][gu], vec_width=4, dtype=fx.Int32, + soffset_bytes=k * 1024, cache_modifier=2) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + sb = spre if k == 0 else [buffer_ops.buffer_load(ws, swd[ni], vec_width=1, dtype=fx.Int32, + soffset_bytes=(k // 2) * 256) for ni in range_constexpr(2)] + return aa, sa, bb, sb + + acc = [[[None] * 2 for _ in range(2)] for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(kt): + if const_expr(k + pf < kt): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for ni in range_constexpr(2): + for gu in range_constexpr(2): + acc[mi][ni][gu] = _mfma(aa[mi], bb[ni][gu], acc[mi][ni][gu], + sa[mi], sb[ni], (k % 2) * 2 + gu, True) + flat = _hold([acc[mi][ni][gu] for mi in range(mr) for ni in range(2) for gu in range(2)]) + if const_expr(kw > 1): + for j in range_constexpr(mr * 4): + llvm.StoreOp(_raw(flat[j]), _lds_ptr(base, ((wave * mr * 4 + j) * 64 + lane) * 16), alignment=16) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if kk == 0: + vals = [Vec(v) for v in flat] + for k in range_constexpr(1, kw): + for j in range_constexpr(mr * 4): + other = Vec(llvm.load(T.vec(4, T.f32), _lds_ptr(base, (((nk + k * nw) * mr * 4 + j) * 64 + lane) * 16), alignment=16)) + vals[j] = vals[j] + other + for mi in range_constexpr(mr): + y = [[_swiglu(fx.Float32(vals[mi * 4 + ni * 2][v]), fx.Float32(vals[mi * 4 + ni * 2 + 1][v])) + for v in range_constexpr(4)] for ni in range_constexpr(2)] + amax = fx.math.absf(y[0][0]) + for ni in range_constexpr(2): + for v in range_constexpr(4): + z = fx.math.absf(y[ni][v]) + amax = (amax > z).select(amax, z) + for sh in range_constexpr(2): + z = amax.shuffle_xor(16 << sh, 64) + amax = (amax > z).select(amax, z) + e = (amax.bitcast(fx.Int32) - fx.Int32(0xE00000)) >> 23 + e = (e > 1).select(e, fx.Int32(1)) + scale = (e << 23).bitcast(fx.Float32) + row = mb * bm + mi * 16 + l16 + for ni in range_constexpr(2): + pk = _fp4(fx.Int32(0), y[ni][0], y[ni][1], scale, 0) + pk = _fp4(pk, y[ni][2], y[ni][3], scale, 1) + off = row * (I // 2) + col // 2 + ni * 8 + q16 * 2 + buffer_ops.buffer_store(pk.to(fx.Int16), qr, off, offset_is_bytes=True, mask=tok[mi] < m) + buffer_ops.buffer_store(e.to(fx.Int8), ss, row * (I // 32) + col // 32, + offset_is_bytes=True, mask=(tok[mi] < m) & (q16 == 0)) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm1_shared_first(*, kw=4, pf=3, bm=16, sh_bm=16, sh_kw=4, sh_pf=2, pairs=256, view=False): + """``compile_gemm1`` whose first ``nsh`` workgroups run the shared expert's unsplit MXFP8 + gate/up tiles (``shared_gemm1_a8w8.compile_gemm1_tile``, ``sh_bm`` rows), so the shared + GEMM1 streams next to the routed tiles in one launch; the routed tiles compute exactly as alone.""" + from .shared_gemm1_a8w8 import compile_gemm1_tile + assert kw == 4 and bm in (16, 32) and sh_bm in (16, 32, 64) and sh_kw in (1, 2, 4) + tn = 32 + # shared workgroups use their exchange region; routed ones their exchange + routing slices + sh_ex = 4 * (max(bm, sh_bm if sh_kw > 1 else 16) // 16) * 4 * 64 * 16 + lds = max(sh_ex, 4 * (bm // 16) * 4 * 64 * 16 + 4 * WAVE_BYTES) + sh_name = (("" if sh_bm == 16 else f"_bm{sh_bm}") + ("" if sh_kw == 4 else f"_kw{sh_kw}") + + ("" if sh_pf == 2 else f"_pf{sh_pf}") + ("_view" if view else "")) + name = f"moe_gemm1_id_rows_a4w4_swiglu_quant_h6144_i1536_bm{bm}_tn{tn}_kw{kw}_pf{pf}_p{pairs}_shared_a8w8{sh_name}_first" + sh_tile = compile_gemm1_tile(kw=sh_kw, pf=sh_pf, bm=sh_bm, view=view) + routed_tile = compile_gemm1(kw=kw, pf=pf, bm=bm, pairs=pairs) + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, Q8, S8, m, m) + if bid >= nsh: + routed_tile(base, bid - nsh, A, AS, W, WS, Q, S, IDS, EIDS, NV, m, rows) + + @flyc.jit + def launch_id_rows_gemm1_a4w4_small_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Tensor, EIDS: fx.Tensor, + NV: fx.Tensor, A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, + WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, stream: fx.Stream): + kernel(A, AS, W, WS, Q, S, IDS, EIDS, NV, A8, AS8, W8, WS8, Q8, S8, m, rows, nsh).launch( + grid=(rows // bm * (I // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_id_rows_gemm1_a4w4_small_shared_first.kernel_name = name + return launch_id_rows_gemm1_a4w4_small_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_inline_sort.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_inline_sort.py new file mode 100644 index 0000000..331ff85 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_inline_sort.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm1_a4w4_small.py`` whose blocks build their rows from ``topk_ids`` (``a16w4_utils.inline_sort_table``) +instead of a sort launch; the first weight step is loaded before the table's barriers. +""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .a16w4_utils import inline_sort_table +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 128 + + +def _raw(x): + """Unwrap only at LLVM intrinsic and inline-assembly boundaries.""" + return x.ir_value() if hasattr(x, "ir_value") else x + + +def _mfma(a, b, acc, sa, sb, b_byte, transpose=False): + """Native gfx950 FP4 scaled MFMA into AGPRs; transpose selects GEMM1 C layout.""" + ty = T.vec(4, T.f32) + a_byte = 0 + if transpose: + a, b, sa, sb = b, a, sb, sa + a_byte, b_byte = b_byte, 0 + asm = (f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {'$0' if acc is not None else '0'}, $3, $4 " + f"op_sel:[{a_byte % 2},{b_byte % 2},0] op_sel_hi:[{a_byte // 2},{b_byte // 2},0] cbsz:4 blgp:4") + ops = [_raw(a), _raw(b), _raw(sa), _raw(sb)] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(_raw(acc)) + cons += ",0" + return llvm.inline_asm(ty, ops, asm, cons, has_side_effects=True) + + +def _hold(xs): + """Keep the required inline-MFMA latency before any accumulator reads.""" + ty = ir.Type.parse("!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(xs)) + ")>") + cons = ",".join(["=a"] * len(xs) + [str(i) for i in range(len(xs))]) + v = llvm.inline_asm(ty, [_raw(x) for x in xs], "s_nop 7\ns_nop 7\ns_nop 7", cons, has_side_effects=True) + return [llvm.extractvalue(T.vec(4, T.f32), v, [i]) for i in range(len(xs))] + + +def _lds_ptr(base, off): + """Byte-address the aligned LDS exchange and transpose regions.""" + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(base + off))) + + +def _fp4(old, a, b, scale, sel): + """FlyDSL lacks a wrapper for this FP32-to-FP4 intrinsic.""" + return fx.Int32(llvm.call_intrinsic(T.i32, "llvm.amdgcn.cvt.scalef32.pk.fp4.f32", + [_raw(old), _raw(a), _raw(b), _raw(scale), _raw(fx.Int32(sel))], [], [])) + + +def _swiglu(g, u): + g = (g < fx.Float32(7)).select(g, fx.Float32(7)) + u = (u < fx.Float32(7)).select(u, fx.Float32(7)) + u = (u > fx.Float32(-7)).select(u, fx.Float32(-7)) + t = (g * fx.Float32(1.702)) * fx.Float32(-1.4426950408889634) + e = fx.Float32(fx.rocdl.exp2(T.f32, _raw(t))) + r = fx.Float32(fx.rocdl.rcp(T.f32, _raw(fx.Float32(1) + e))) + return (g * r) * (u + fx.Float32(1)) + + +@functools.cache +def compile_gemm1(*, kw=1, pf=3, bm=16): + assert kw in (1, 2, 4) and bm == 16 + nw = 4 // kw + tn, kt, mr = nw * 32, 48 // kw, bm // 16 + + @flyc.jit + def routed_tile(base, tab, rb, A, AS, W, WS, Q, S, IDS, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + nk, kk = wave % nw, wave // nw + mb, nb = rb // (I // tn), rb % (I // tn) + expert, owner, count, build_tab = inline_sort_table( + IDS, m, 4, mb, lane, tab, max_pairs=64, bm=16, multi_block=True) + if owner: + col = nb * tn + nk * 32 + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * (H // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=m * (H // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=I * H, + base_byte_offset=fx.Int64(expert) * (I * H)) + ws = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * I * (H // 16)) + qr = buffer_ops.create_buffer_resource(Q, max_size=False, num_records_bytes=rows * (I // 2)) + ss = buffer_ops.create_buffer_resource(S, max_size=False, num_records_bytes=rows * (I // 32)) + wd = [[(((col // 16 + ni) * 2 + gu) * 48 * 1024 + q16 * 256 + l16 * 16) // 4 + kk * kt * 256 + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + swd = [((expert * (I // 16) + col // 16 + ni) * 24) * 64 + q16 * 16 + l16 + kk * (kt // 2) * 64 + for ni in range_constexpr(2)] + + # the expert is known: the first weight step goes out before the table's two barriers, the activation + # addresses wait for it + bpre = [[buffer_ops.buffer_load(wr, wd[ni][gu], vec_width=4, dtype=fx.Int32, + cache_modifier=2) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + spre = [buffer_ops.buffer_load(ws, swd[ni], vec_width=1, dtype=fx.Int32) + for ni in range_constexpr(2)] + build_tab() + tok = [fx.Int32(tab[mi * 16 + l16]) & 0xFFFFFF for mi in range_constexpr(mr)] + ad = [(tok[mi] * (H // 2) + kk * (kt * 64) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [tok[mi] * (H // 32) // 4 + kk * kt for mi in range_constexpr(mr)] + + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) + for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, + soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = bpre if k == 0 else [[buffer_ops.buffer_load(wr, wd[ni][gu], vec_width=4, dtype=fx.Int32, + soffset_bytes=k * 1024, cache_modifier=2) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + sb = spre if k == 0 else [buffer_ops.buffer_load(ws, swd[ni], vec_width=1, dtype=fx.Int32, + soffset_bytes=(k // 2) * 256) for ni in range_constexpr(2)] + return aa, sa, bb, sb + + acc = [[[None] * 2 for _ in range(2)] for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(kt): + if const_expr(k + pf < kt): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for ni in range_constexpr(2): + for gu in range_constexpr(2): + acc[mi][ni][gu] = _mfma(aa[mi], bb[ni][gu], acc[mi][ni][gu], + sa[mi], sb[ni], (k % 2) * 2 + gu, True) + flat = _hold([acc[mi][ni][gu] for mi in range(mr) for ni in range(2) for gu in range(2)]) + if const_expr(kw > 1): + for j in range_constexpr(mr * 4): + llvm.StoreOp(_raw(flat[j]), _lds_ptr(base, ((wave * mr * 4 + j) * 64 + lane) * 16), alignment=16) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if kk == 0: + vals = [Vec(v) for v in flat] + for k in range_constexpr(1, kw): + for j in range_constexpr(mr * 4): + other = Vec(llvm.load(T.vec(4, T.f32), _lds_ptr(base, (((nk + k * nw) * mr * 4 + j) * 64 + lane) * 16), alignment=16)) + vals[j] = vals[j] + other + for mi in range_constexpr(mr): + y = [[_swiglu(fx.Float32(vals[mi * 4 + ni * 2][v]), fx.Float32(vals[mi * 4 + ni * 2 + 1][v])) + for v in range_constexpr(4)] for ni in range_constexpr(2)] + amax = fx.math.absf(y[0][0]) + for ni in range_constexpr(2): + for v in range_constexpr(4): + z = fx.math.absf(y[ni][v]) + amax = (amax > z).select(amax, z) + for sh in range_constexpr(2): + z = amax.shuffle_xor(16 << sh, 64) + amax = (amax > z).select(amax, z) + e = (amax.bitcast(fx.Int32) - fx.Int32(0xE00000)) >> 23 + e = (e > 1).select(e, fx.Int32(1)) + scale = (e << 23).bitcast(fx.Float32) + row = mb * bm + mi * 16 + l16 + for ni in range_constexpr(2): + pk = _fp4(fx.Int32(0), y[ni][0], y[ni][1], scale, 0) + pk = _fp4(pk, y[ni][2], y[ni][3], scale, 1) + off = row * (I // 2) + col // 2 + ni * 8 + q16 * 2 + buffer_ops.buffer_store(pk.to(fx.Int16), qr, off, offset_is_bytes=True, mask=tok[mi] < m) + buffer_ops.buffer_store(e.to(fx.Int8), ss, row * (I // 32) + col // 32, + offset_is_bytes=True, mask=(tok[mi] < m) & (q16 == 0)) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm1_shared_first(*, kw=4, pf=3, bm=16, view=False): + """``compile_gemm1`` whose first ``nsh`` workgroups run the shared expert's unsplit MXFP8 + gate/up tiles (``shared_gemm1_a8w8.compile_gemm1_tile``); the routed tiles compute as alone.""" + from .shared_gemm1_a8w8 import compile_gemm1_tile + assert kw == 4 and bm == 16 + tn, mr = 32, 1 + lds = 4 * mr * 4 * 64 * 16 + name = (f"moe_gemm1_inline_sort_a4w4_swiglu_quant_h6144_i1536_bm{bm}_tn{tn}_kw{kw}_pf{pf}_weight_prefetch_shared_a8w8_first" + + ("_view" if view else "")) + sh_tile = compile_gemm1_tile(kw=4, pf=2, bm=16, view=view) + routed_tile = compile_gemm1(kw=kw, pf=pf, bm=bm) + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + tab: fx.Array[fx.Int32, 33] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Int64, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, Q8, S8, m, m) + if bid >= nsh: + routed_tile(base, smem.tab.ptr, bid - nsh, A, AS, W, WS, Q, S, IDS, m, rows) + + @flyc.jit + def launch_gemm1_a4w4_inline_sort_weight_prefetch_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Int64, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, + WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, + stream: fx.Stream): + kernel(A, AS, W, WS, Q, S, IDS, A8, AS8, W8, WS8, Q8, S8, m, rows, nsh).launch( + grid=(rows // bm * (I // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_gemm1_a4w4_inline_sort_weight_prefetch_shared_first.kernel_name = name + return launch_gemm1_a4w4_inline_sort_weight_prefetch_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_small.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_small.py new file mode 100644 index 0000000..dc99a5f --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_a4w4_small.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""16-row MXFP4 gate/up tiles, direct activation loads, FP32 SwiGLU and FP4 quant. + +One wave owns 32 intermediate columns, including their complete quant group. +K-waves sum FP32 gate/up partials through LDS in rank order before SwiGLU. +Weights and scales use the entry's sole prepared layout. +""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 128 + + +def _raw(x): + """Unwrap only at LLVM intrinsic and inline-assembly boundaries.""" + return x.ir_value() if hasattr(x, "ir_value") else x + + +def _mfma(a, b, acc, sa, sb, b_byte, transpose=False): + """Native gfx950 FP4 scaled MFMA into AGPRs; transpose selects GEMM1 C layout.""" + ty = T.vec(4, T.f32) + a_byte = 0 + if transpose: + a, b, sa, sb = b, a, sb, sa + a_byte, b_byte = b_byte, 0 + asm = (f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {'$0' if acc is not None else '0'}, $3, $4 " + f"op_sel:[{a_byte % 2},{b_byte % 2},0] op_sel_hi:[{a_byte // 2},{b_byte // 2},0] cbsz:4 blgp:4") + ops = [_raw(a), _raw(b), _raw(sa), _raw(sb)] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(_raw(acc)) + cons += ",0" + return llvm.inline_asm(ty, ops, asm, cons, has_side_effects=True) + + +def _hold(xs): + """Keep the required inline-MFMA latency before any accumulator reads.""" + ty = ir.Type.parse("!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(xs)) + ")>") + cons = ",".join(["=a"] * len(xs) + [str(i) for i in range(len(xs))]) + v = llvm.inline_asm(ty, [_raw(x) for x in xs], "s_nop 7\ns_nop 7\ns_nop 7", cons, has_side_effects=True) + return [llvm.extractvalue(T.vec(4, T.f32), v, [i]) for i in range(len(xs))] + + +def _lds_ptr(base, off): + """Byte-address the aligned LDS exchange and transpose regions.""" + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(base + off))) + + +def _fp4(old, a, b, scale, sel): + """FlyDSL lacks a wrapper for this FP32-to-FP4 intrinsic.""" + return fx.Int32(llvm.call_intrinsic(T.i32, "llvm.amdgcn.cvt.scalef32.pk.fp4.f32", + [_raw(old), _raw(a), _raw(b), _raw(scale), _raw(fx.Int32(sel))], [], [])) + + +def _swiglu(g, u): + g = (g < fx.Float32(7)).select(g, fx.Float32(7)) + u = (u < fx.Float32(7)).select(u, fx.Float32(7)) + u = (u > fx.Float32(-7)).select(u, fx.Float32(-7)) + t = (g * fx.Float32(1.702)) * fx.Float32(-1.4426950408889634) + e = fx.Float32(fx.rocdl.exp2(T.f32, _raw(t))) + r = fx.Float32(fx.rocdl.rcp(T.f32, _raw(fx.Float32(1) + e))) + return (g * r) * (u + fx.Float32(1)) + + +@functools.cache +def compile_gemm1(*, kw=1, pf=3, bm=16): + assert kw in (1, 2, 4) and bm in (16, 32) + nw = 4 // kw + tn, kt, mr = nw * 32, 48 // kw, bm // 16 + + @flyc.jit + def routed_tile(base, rb, A, AS, W, WS, Q, S, IDS, EIDS, NV, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + nk, kk = wave % nw, wave // nw + mb, nb = rb // (I // tn), rb % (I // tn) + nr = fx.Int32(NV[0]) + if mb * bm < nr: + expert = fx.Int32(EIDS[mb]) + col = nb * tn + nk * 32 + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * (H // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=m * (H // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=I * H, + base_byte_offset=fx.Int64(expert) * (I * H)) + ws = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * I * (H // 16)) + qr = buffer_ops.create_buffer_resource(Q, max_size=False, num_records_bytes=rows * (I // 2)) + ss = buffer_ops.create_buffer_resource(S, max_size=False, num_records_bytes=rows * (I // 32)) + tok = [fx.Int32(IDS[mb * bm + mi * 16 + l16]) & 0xFFFFFF for mi in range_constexpr(mr)] + ad = [(tok[mi] * (H // 2) + kk * (kt * 64) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [tok[mi] * (H // 32) // 4 + kk * kt for mi in range_constexpr(mr)] + wd = [[(((col // 16 + ni) * 2 + gu) * 48 * 1024 + q16 * 256 + l16 * 16) // 4 + kk * kt * 256 + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + swd = [((expert * (I // 16) + col // 16 + ni) * 24) * 64 + q16 * 16 + l16 + kk * (kt // 2) * 64 + for ni in range_constexpr(2)] + + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) + for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, + soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = [[buffer_ops.buffer_load(wr, wd[ni][gu], vec_width=4, dtype=fx.Int32, + soffset_bytes=k * 1024, cache_modifier=2) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + sb = [buffer_ops.buffer_load(ws, swd[ni], vec_width=1, dtype=fx.Int32, + soffset_bytes=(k // 2) * 256) for ni in range_constexpr(2)] + return aa, sa, bb, sb + + acc = [[[None] * 2 for _ in range(2)] for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(kt): + if const_expr(k + pf < kt): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for ni in range_constexpr(2): + for gu in range_constexpr(2): + acc[mi][ni][gu] = _mfma(aa[mi], bb[ni][gu], acc[mi][ni][gu], + sa[mi], sb[ni], (k % 2) * 2 + gu, True) + flat = _hold([acc[mi][ni][gu] for mi in range(mr) for ni in range(2) for gu in range(2)]) + if const_expr(kw > 1): + for j in range_constexpr(mr * 4): + llvm.StoreOp(_raw(flat[j]), _lds_ptr(base, ((wave * mr * 4 + j) * 64 + lane) * 16), alignment=16) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if kk == 0: + vals = [Vec(v) for v in flat] + for k in range_constexpr(1, kw): + for j in range_constexpr(mr * 4): + other = Vec(llvm.load(T.vec(4, T.f32), _lds_ptr(base, (((nk + k * nw) * mr * 4 + j) * 64 + lane) * 16), alignment=16)) + vals[j] = vals[j] + other + for mi in range_constexpr(mr): + y = [[_swiglu(fx.Float32(vals[mi * 4 + ni * 2][v]), fx.Float32(vals[mi * 4 + ni * 2 + 1][v])) + for v in range_constexpr(4)] for ni in range_constexpr(2)] + amax = fx.math.absf(y[0][0]) + for ni in range_constexpr(2): + for v in range_constexpr(4): + z = fx.math.absf(y[ni][v]) + amax = (amax > z).select(amax, z) + for sh in range_constexpr(2): + z = amax.shuffle_xor(16 << sh, 64) + amax = (amax > z).select(amax, z) + e = (amax.bitcast(fx.Int32) - fx.Int32(0xE00000)) >> 23 + e = (e > 1).select(e, fx.Int32(1)) + scale = (e << 23).bitcast(fx.Float32) + row = mb * bm + mi * 16 + l16 + for ni in range_constexpr(2): + pk = _fp4(fx.Int32(0), y[ni][0], y[ni][1], scale, 0) + pk = _fp4(pk, y[ni][2], y[ni][3], scale, 1) + off = row * (I // 2) + col // 2 + ni * 8 + q16 * 2 + buffer_ops.buffer_store(pk.to(fx.Int16), qr, off, offset_is_bytes=True, mask=tok[mi] < m) + buffer_ops.buffer_store(e.to(fx.Int8), ss, row * (I // 32) + col // 32, + offset_is_bytes=True, mask=(tok[mi] < m) & (q16 == 0)) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm1_shared_first(*, kw=4, pf=3, bm=16, sh_bm=16, sh_kw=4, sh_pf=2, view=False): + """``compile_gemm1`` whose first ``nsh`` workgroups run the shared expert's unsplit MXFP8 + gate/up tiles (``shared_gemm1_a8w8.compile_gemm1_tile``, ``sh_bm`` rows), so the shared + GEMM1 streams next to the routed tiles in one launch; the routed tiles compute exactly as alone. ``sh_bm`` 0: the + routed tiles only (the shared arguments are not read), at the routed tiles' registers and LDS. ``view``: the shared + weights are the rank's view of the full copy (``shared_gemm1_a8w8.compile_gemm1``).""" + from .shared_gemm1_a8w8 import compile_gemm1_tile + assert kw == 4 and bm in (16, 32) and sh_bm in (0, 16, 32, 64) and sh_kw in (1, 2, 4) + tn = 32 + lds = 4 * (max(bm, sh_bm if sh_kw > 1 else 16) // 16) * 4 * 64 * 16 + sh_name = (("" if sh_bm == 16 else f"_bm{sh_bm}") + ("" if sh_kw == 4 else f"_kw{sh_kw}") + + ("" if sh_pf == 2 else f"_pf{sh_pf}") + ("_view" if view else "")) + name = f"moe_gemm1_a4w4_swiglu_quant_h6144_i1536_bm{bm}_tn{tn}_kw{kw}_pf{pf}_shared_a8w8{sh_name}_first" + if sh_bm == 0: + lds = 4 * (bm // 16) * 4 * 64 * 16 + name = f"moe_gemm1_a4w4_swiglu_quant_h6144_i1536_bm{bm}_tn{tn}_kw{kw}_pf{pf}" + sh_tile = compile_gemm1_tile(kw=sh_kw, pf=sh_pf, bm=sh_bm, view=view) if sh_bm else None + routed_tile = compile_gemm1(kw=kw, pf=pf, bm=bm) + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if const_expr(sh_bm == 0): + routed_tile(base, bid, A, AS, W, WS, Q, S, IDS, EIDS, NV, m, rows) + else: + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, Q8, S8, m, m) + if bid >= nsh: + routed_tile(base, bid - nsh, A, AS, W, WS, Q, S, IDS, EIDS, NV, m, rows) + + @flyc.jit + def launch_gemm1_a4w4_small_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + Q: fx.Tensor, S: fx.Tensor, IDS: fx.Tensor, EIDS: fx.Tensor, + NV: fx.Tensor, A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, + WS8: fx.Tensor, Q8: fx.Tensor, S8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, stream: fx.Stream): + kernel(A, AS, W, WS, Q, S, IDS, EIDS, NV, A8, AS8, W8, WS8, Q8, S8, m, rows, nsh).launch( + grid=(rows // bm * (I // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_gemm1_a4w4_small_shared_first.kernel_name = name + return launch_gemm1_a4w4_small_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_prefill_h6144_i1536.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_prefill_h6144_i1536.py new file mode 100644 index 0000000..df1b1c1 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm1_prefill_h6144_i1536.py @@ -0,0 +1,898 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MoE GEMM1 for H 6144, I 1536 (gfx950) on 256-row sort blocks, persistent: MXFP4 x MXFP4 gate / up in f32, +SwiGLU-OAI, then MXFP4 of h (Quark-even e8m0, RNE e2m1, from f32: no bf16 rounding). The I 3072 kernel's body +(``i3072.gemm1_prefill_h6144_i3072_persistent``) on W13 in the a16w4 order, so a mixed checkpoint keeps one copy of +its values. Run-to-run bitwise. + +Layouts (bytes): + A [n_tokens, H/2] per-token e2m1 + A_scale [pad32(max_sorted), H/32] sorted rows, e8m0-shuffled + W13 [E, I/16, 2 (gate, up), 16 rows, H/2] ``layout.w13_layout`` (shuffle_weight (16, 16) blocks) + W13_sc [E*2I, H/32] e8m0-shuffled, 32-row groups of 16 gate + 16 up rows + OUT_Q [num_m_blocks*BLOCK_M, I/2] sorted rows, e2m1 + OUT_sc [pad32(num_m_blocks*BLOCK_M), I/32] sorted rows, e8m0-shuffled +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from .common.epilogue import _maxf_nn +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_LDS_BYTES, + _SCALE_SLOTS, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _as_f32, + _Buf, + _flat_frag, + _g2s_thunks, + _min, + _permlane16_swap, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .a16w4_utils import READY_STRIDE, SC1, global_atomic_add_agent +from .common.tile_map import CLAIM_COUNTERS +from .i3072.gemm1_prefill_h6144_i3072_persistent import ( + N_XCDS, + MfmaRowStationaryFp4, + _buffer_atomic_add, + _cu_count, + _cvt_pk_fp4, + _e8m0_pair, + _swiglu_oai_x4, +) +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + _G2SCM, + _MFMA_RESULT_WAIT, + BLOCK_K, + _fmax3_abs, +) +from amd_kernels.warm_up import launcher + + +@launcher +def compile_moe_gemm1_mxfp4_h6144_i1536(*, E: int, BLOCK_M: int = 256, signal: bool = False): + """The persistent launcher: one workgroup per CU, ``grid_size`` = the tile map's size. + + Args: + signal: build the body for ``gemm12_a4w4`` (``launch.tile``): write-through stores, and each tile counted in + its m-block's ready counter at the next tile's tail wait (the last tile after the loop's final wait). + """ + H, I = 6144, 1536 # noqa: E741 + assert BLOCK_M == 256 + N_CTAS = _cu_count() + assert N_CTAS % N_XCDS == 0, N_CTAS + K = H + BLOCK_K_BYTES = BLOCK_K + BLOCK_N = 256 # 128 gate cols + the matching 128 up cols + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 16-row tiles per wave per LDS half + N_TILES_B = LDS_BLOCK_N // 2 // 16 # = 4 + + assert K % 256 == 0 and I % LDS_BLOCK_N == 0 + K_ITERS = (K // 2) // BLOCK_K # fp4: 256 K per 128-B step + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert K_ITERS >= 4 and (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + K_BYTES = K // 2 + N_BITS = (I // 128 - 1).bit_length() + N0 = I // 16 # 16-column (gate + up) groups per expert + + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES + A_BUFS = 4 * a_lds_size + LDS_TILES_BYTES = A_BUFS + 4 * b_lds_size + + A_WAVE_GROUPS = N_TILES_A // 2 # groups per wave per LDS half + A_HALF_GROUPS = LDS_BLOCK_M // 32 + B_WAVE_GROUPS = 4 # consecutive groups per wave quarter + B_HALF_GROUPS = 2 # quarter block b -> group G + b (via (b//2)*2 + b%2) + SCALE_COLS_OUT = I // 32 # e8m0 per output row + OUT_SC_BLOCKS_PER_ROW32 = SCALE_COLS_OUT // 8 + W_BYTES = (2 * I) * K_BYTES + assert W_BYTES <= 0xFFFFFFFF, "buffer resources address 4 GB" + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, _SCALE_LDS_BYTES, 16] + claim: fx.Array[fx.Int32, 4, 16] + + STORE_KW = {"cache_modifier": SC1} if signal else {} + + @flyc.jit + def tile( + _base_ptr, + _scale_lds_ptr, + claim_slot, + bx, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + ready, + ): + """Workgroup ``bx``'s persistent tile loop on its LDS tiles (``_base_ptr``), scale slots (``_scale_lds_ptr``) + and claim words (``claim_slot``); with ``signal``, ``ready`` holds ``gemm12_a4w4``'s m-block counters.""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + b_cur0 = _Buf(_base_ptr, A_BUFS + 0 * b_lds_size) + b_cur1 = _Buf(_base_ptr, A_BUFS + 1 * b_lds_size) + b_next0 = _Buf(_base_ptr, A_BUFS + 2 * b_lds_size) + b_next1 = _Buf(_base_ptr, A_BUFS + 3 * b_lds_size) + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + tid = fx.thread_idx.x + lane_id = tid % 64 + wave_id = tid // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * 4) + ) + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + # tile_map[grid_size] = valid entries; XCD x (workgroup w runs on XCD w % 8) claims its run [x * per_xcd, + # min((x + 1) * per_xcd, valid)) through tile_map[grid_size - CLAIM_COUNTERS + x] (-1 from the tile map) + tm_rsrc = buffer_ops.create_buffer_resource( + tile_map_t, max_size=False, num_records_bytes=(grid_size + 1) * 4 + ) + xcd = bx % N_XCDS + n_valid = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, grid_size, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + per_xcd = (n_valid + fx.Int32(N_XCDS - 1)) // fx.Int32(N_XCDS) + run_lo = xcd * per_xcd + run_n = _min(per_xcd, n_valid - run_lo) + # only thread 0 is in range: one claim per workgroup + claim_off = (tid == 0).select( + (grid_size - CLAIM_COUNTERS + xcd) * fx.Int32(4), fx.Int32(0x7FFFFFF0) + ) + + def _entry_of(c): + """Tile-map entry of a claimed counter value c (uniform; run index c + 1), -1 past the run.""" + t = c + fx.Int32(1) + ok = t < run_n + e = fx.Int32( + buffer_ops.buffer_load( + tm_rsrc, + ok.select(run_lo + t, fx.Int32(0)), + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + return ok.select(e, fx.Int32(-1)) + + def _expert_of(entry): + tile_i = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) + return fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, tile_i, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + + def _a_offsets(entry): + """Per-lane gathered A offsets of both LDS halves (sorted ids loads).""" + m_base = (entry >= 0).select(entry >> N_BITS, fx.Int32(0)) * BLOCK_M + offs = [] + for half in range_constexpr(2): + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + half * LDS_BLOCK_M + row, + vec_width=1, + dtype=fx.Int32, + ) + ) + # padded rows: tok == n_tokens -> OOB -> zeros + tok = sid & fx.Int32(0x00FFFFFF) + offs.append(tok * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + class _Tile: + """Loaders and addresses of one tile (entry, its expert, its A offsets).""" + + def __init__(self, entry, expert, a_offs, live=True): + """``live`` False: every load of the tile is out of range (no memory + traffic; the loop exits before anything reads the LDS).""" + self.entry = entry + tile_i = entry >> N_BITS + self.tile_j = entry & ((1 << N_BITS) - 1) + self.m_base = tile_i * BLOCK_M + weight_group0 = self.tile_j * fx.Int32(LDS_BLOCK_N // 16) + group0 = expert * fx.Int32(N0) + weight_group0 + + def _b_offsets(gu): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + # a16w4 order: gate block g at 2g, its up block at 2g + 1 + blk = (weight_group0 + fx.Int32(row // 16)) * fx.Int32( + 2 + ) + fx.Int32(gu) + offs.append( + blk * fx.Int32(K_BYTES * 16) + + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + ) + return offs + + self.scale_gather = ScaleGatherMoE( + A_scale, + W13_scale, + K, + lane_id, + wave_id, + _scale_lds_ptr, + a_scale_bytes * live, + E * 2 * I * (K // 32) * live, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + self.scale_gather.set_wave_base(self.m_base, group0 * fx.Int32(32)) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=n_tokens * K_BYTES * live + ) + b_rsrc = buffer_ops.create_buffer_resource( + W13, + max_size=False, + num_records_bytes=fx.Int32(W_BYTES) * live, + base_byte_offset=fx.Int64(expert) * W_BYTES, + ) + self.a0 = G2SLoaderAsm(a_rsrc, a_offs[:N_TILES_A], N_TILES_A, wave_id) + self.a1 = G2SLoaderAsm(a_rsrc, a_offs[N_TILES_A:], N_TILES_A, wave_id) + self.b0 = _G2SCM(b_rsrc, _b_offsets(0), N_TILES_B, wave_id, cm=" nt") + self.b1 = _G2SCM(b_rsrc, _b_offsets(1), N_TILES_B, wave_id, cm=" nt") + for ld in (self.a0, self.a1, self.b0, self.b1): + ld.set_wave_base(_base_ptr) + + def prologue_groups(self): + """Scale blocks 0-2 and K steps 0, 1 into the cur / next buffers as groups of thunks, in the issue order + the loop-top waits count (a loader's steps stay together: they chain m0).""" + sg = self.scale_gather + return [ + [ + lambda: sg.gather(0, _slot(0)), + lambda: sg.gather(1, _slot(1)), + lambda: sg.gather(2, _slot(2)), + ], + [lambda: self.a0.load(a_cur0, fx.Int32(0 * A_K_STEP))], + [lambda: self.b0.load(b_cur0, fx.Int32(0 * B_K_STEP))], + [lambda: self.b1.load(b_cur1, fx.Int32(0 * B_K_STEP))], + [lambda: self.a1.load(a_cur1, fx.Int32(0 * A_K_STEP))], + [lambda: self.a0.load(a_next0, fx.Int32(1 * A_K_STEP))], + [lambda: self.b0.load(b_next0, fx.Int32(1 * B_K_STEP))], + [lambda: self.b1.load(b_next1, fx.Int32(1 * B_K_STEP))], + [lambda: self.a1.load(a_next1, fx.Int32(1 * A_K_STEP))], + ] + + def prologue(self): + for grp in self.prologue_groups(): + for th in grp: + th() + + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = 2 * 1024 # 128 B of K = two 1 KB blocks + + def _slot(g): + return fx.Int32(g) % fx.Int32(_SCALE_SLOTS) + + _scale_base_ptr = _scale_lds_ptr + + def _publish(i, v): + """LDS claim word i <- v (thread 0 hands a claimed index to the waves).""" + claim_slot[i] = v + + out_rsrc = buffer_ops.create_buffer_resource( + OUT_Q, max_size=False, num_records_bytes=num_m_blocks * (BLOCK_M * (I // 2)) + ) + osc_rsrc = buffer_ops.create_buffer_resource( + OUT_scale, + max_size=False, + num_records_bytes=num_m_blocks * (BLOCK_M * SCALE_COLS_OUT), + ) + g = lane_id // 16 + r16 = lane_id % 16 + + # ---- first claim (exposed once per workgroup) ---- + first = _buffer_atomic_add(tm_rsrc, claim_off, 1) + if tid == 0: + _publish(0, first) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + entry = _entry_of( + fx.Int32( + _rocdl.readfirstlane(_T.i32, fx.as_ir_value(fx.Int32(claim_slot[0]))) + ) + ) + expert = _expert_of(entry) + a_offs = _a_offsets(entry) + if entry >= 0: + _Tile(entry, expert, a_offs).prologue() + + def _run_tile(entry, expert, a_offs, prev_mb=None): + """One tile (its prologue already issued); claims, prepares and issues the next tile's prologue and returns + its (entry, expert, A offsets). The tile's objects live here: pre-loop objects whose methods the loop + body called would become loop-carried state of the scf.while.""" + tile = _Tile(entry, expert, a_offs) + a0_g2s, a1_g2s, b0_g2s, b1_g2s = tile.a0, tile.a1, tile.b0, tile.b1 + scale_gather = tile.scale_gather + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + mfma = MfmaRowStationaryFp4(N_TILES_A, N_TILES_B) + + # gathers + a_cur0 landed: b0/b1/a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + # b_cur0 AND b_cur1 landed: a1 + the 4 next batches may fly + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0 + sc0_sbC1) + + def _top_vmcnt(k2): + return 2 * N_TILES_A + 2 * N_TILES_B + 1 # fp4: a gather every step + + def _seg2_vmcnt(k2): + return 2 * N_TILES_A + N_TILES_B + 1 + + _STEP0_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _STEP0_SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(kc_idx, holder): + s = _slot(fx.Int32(kc_idx)) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step( + kc, + k2, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + zero_acc=False, + top_vmcnt=None, + seg2_vmcnt=None, + ): + kc_i = fx.Int32(kc) + top_vmcnt = _top_vmcnt(k2) if top_vmcnt is None else top_vmcnt + seg2_vmcnt = _seg2_vmcnt(k2) if seg2_vmcnt is None else seg2_vmcnt + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC = sc + c00f, c01f, c10f, c11f = accs + + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + a_off = (kc_i + fx.Int32(2)) * fx.Int32(A_K_STEP) + b_off = (kc_i + fx.Int32(2)) * fx.Int32(B_K_STEP) + + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(kc_i + fx.Int32(1), _scn) + _gk = _min(kc_i + fx.Int32(3), fx.Int32(K_ITERS - 1)) + _sc_gather = [lambda: scale_gather.gather(_gk, _slot(_gk))] + + wait_barrier(top_vmcnt) + il = ( + _riffle( + _g2s_thunks(a0_g2s, ac0, a_off, N_TILES_A), + _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + + il = _riffle(_g2s_thunks(b0_g2s, bc0, b_off, N_TILES_B), _rd_scn[2:]) + # the second half's wait also under c01f's MFMAs (the barrier keeps its own): it overlaps them + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=seg2_vmcnt, lgkmcnt=0)] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + a1f = _a1 + + wait_barrier(seg2_vmcnt) + il = ( + _riffle( + _g2s_thunks(b1_g2s, bc1, b_off, N_TILES_B), + _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False), + ) + + _sc_gather + ) + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC, 0, k2, interleave=il, zero_acc=zero_acc + ) + a0nf = _a0n + + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + # likewise the next step's top wait under c11f's MFMAs + il += [lambda: fx.rocdl.s_waitcnt(vmcnt=_top_vmcnt(1 - k2), lgkmcnt=0)] + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + ) + b0nf = _b0n + b1nf = _b1n + + sc_next = (_scn[0], _scn[1], _scn[2] + _scn[3]) + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = 2 * N_TILES_A + n_b = 2 * N_TILES_B + n_ga = N_TILES_A // 2 + n_sc = 2 * n_ga + N_TILES_B + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC = sc + return ( + [_R(v) for v in saR0] + [_R(v) for v in saR1] + [_R(v) for v in sbC] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC = list(flat[o : o + N_TILES_B]) + return (saR0, saR1, sbC) + + _accs0 = tuple([None] * N_ACCUMS for _ in range(4)) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 0, + 0, + a0_frag, + b0_frag, + b1_frag, + sc0, + _accs0, + bufs0, + zero_acc=True, + top_vmcnt=_STEP0_VMCNT, + seg2_vmcnt=_STEP0_SEG2_VMCNT, + ) + a0f, b0f, b1f, sc, accs, _ = _one_step( + 1, 1, a0f, b0f, b1f, sc, accs, _swap_bufs(bufs0) + ) + + init_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + for kk, state in range( # type: ignore[call-overload] + 2, K_ITERS - 2, UNROLL, init=init_state + ): + off = 0 + a0f = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + accs = (c00f, c01f, c10f, c11f) + + bufs = bufs0 + for u in range_constexpr(UNROLL): + a0f, b0f, b1f, sc, accs, bufs = _one_step( + kk + u, u % 2, a0f, b0f, b1f, sc, accs, bufs + ) + + new_state = ( + _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + + [_R(x) for x in accs[0]] + + [_R(x) for x in accs[1]] + + [_R(x) for x in accs[2]] + + [_R(x) for x in accs[3]] + ) + state = yield new_state + + off = 0 + a0_frag = _unflat_frag(state[off : off + n_a], N_TILES_A) + off += n_a + b0_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + b1_frag = _unflat_frag(state[off : off + n_b], N_TILES_B) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_frag = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + + # Tail step K_ITERS - 2 (no gather; reads the last scale block) + saR0, saR1, sbC = sc + _scn = [None, None, None, None] # type: list + _rd_scn = _read_scale_thunks(fx.Int32(K_ITERS - 1), _scn) + _a1 = [None] * N_TILES_A + wait_barrier((2 * N_TILES_A) + (2 * N_TILES_B)) + # claim the next tile now (thread 0; younger than every load the + # remaining waits count, so they can only over-wait on it) + nxt = _buffer_atomic_add(tm_rsrc, claim_off, 1) + il = _s2r_thunks(a_s2r, a_cur1, _a1, N_TILES_A, False) + _rd_scn + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 0, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 0) + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + wait_barrier(1 * N_TILES_A) + il = ( + _s2r_thunks(a_s2r, a_next0, _a0n, N_TILES_A, False) + + _s2r_thunks(b_s2r, b_next0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, b_next1, _b1n, N_TILES_B, True) + ) + c10_frag = mfma.call( + a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 0, interleave=il + ) + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 0) + a0_frag = _a0n + b0_frag = _b0n + b1_frag = _b1n + + # Tail step K_ITERS - 1 (its A half 1 sits in a_next1) + _a1 = [None] * N_TILES_A + wait_barrier(0) + if tid == 0: + _publish(1, nxt) + if const_expr(signal): + # every wave's loads and stores so far have completed, the previous + # tile's stores included: count its block (4 waves) at no extra wait + if tid == 0: + if prev_mb >= 0: + global_atomic_add_agent( + ready + fx.Int64(prev_mb * READY_STRIDE), 4 + ) + saR0, saR1, sbC = (_scn[0], _scn[1], _scn[2] + _scn[3]) + il = _s2r_thunks(a_s2r, a_next1, _a1, N_TILES_A, False) + c00_frag = mfma.call( + a0_frag, b0_frag, c00_frag, saR0, sbC, 0, 1, interleave=il + ) + a1_frag = _a1 + c01_frag = mfma.call(a0_frag, b1_frag, c01_frag, saR0, sbC, 1, 1) + c00_frag = mfma.hold(c00_frag) + c10_frag = mfma.call(a1_frag, b0_frag, c10_frag, saR1, sbC, 0, 1) + c01_frag = mfma.hold(c01_frag) + c11_frag = mfma.call(a1_frag, b1_frag, c11_frag, saR1, sbC, 1, 1) + c10_frag = mfma.hold(c10_frag) + c11_frag = mfma.hold(c11_frag, wait=_MFMA_RESULT_WAIT) + + # every wave is done with the tile's LDS; the claim is visible + wait_barrier(0) + entry_n = _entry_of( + fx.Int32( + _rocdl.readfirstlane( + _T.i32, fx.as_ir_value(fx.Int32(claim_slot[1])) + ) + ) + ) + expert_n = _expert_of(entry_n) + a_offs_n = _a_offsets(entry_n) + + # ---- epilogue: swiglu-OAI + MXFP4 quant, sorted-row output ---- + col_base = tile.tile_j * LDS_BLOCK_N + wave_j * (N_TILES_B * 16) + colgrp_base = col_base // 32 + + def _epilogue(c_gate, c_up, base_row, inter=(), defer=False): + """One quadrant pair (rows base_row + ti*16 + r16, the wave's 64 gate columns), row tiles in pairs + (2 tp, 2 tp + 1) sharing the e8m0 exchange. ``inter[k]``: thunks run after the k-th (p, ti) block's + SwiGLU. ``defer``: return the stores as thunks and fence each block (the inter thunks carry the next + tile's prologue loads; unfenced, the scheduler bunches them and they stall at issue).""" + stores = [] + k_it = 0 + for p in range_constexpr(N_TILES_B // 2): # 32-col group p + colgrp = colgrp_base + p + sc_in_block = (colgrp % 4) * 64 + r16 * 4 + ((colgrp % 8) // 4) * 2 + for tp in range_constexpr(N_TILES_A // 2): + hs = [] + amaxs = [] + for ti in (2 * tp, 2 * tp + 1): + gv = Vec(c_gate[mfma.idx(ti, 2 * p)]) + gw = Vec(c_gate[mfma.idx(ti, 2 * p + 1)]) + uv = Vec(c_up[mfma.idx(ti, 2 * p)]) + uw = Vec(c_up[mfma.idx(ti, 2 * p + 1)]) + h = [] + for cg, cu in ((gv, uv), (gw, uw)): + h += _swiglu_oai_x4( + [fx.Float32(cg[v]) for v in range_constexpr(4)], + [fx.Float32(cu[v]) for v in range_constexpr(4)], + ) + a01 = _fmax3_abs(h[0], h[1], h[2]) + a34 = _fmax3_abs(h[3], h[4], h[5]) + a67 = _fmax3_abs(h[6], h[7], a01) + amaxs.append(_maxf_nn(a67, a34)) + hs.append(h) + if const_expr(k_it < len(inter)): + for th in inter[k_it]: + th() + k_it += 1 + if const_expr(defer): + fx.rocdl.sched_barrier(0) + e8m0s = _e8m0_pair(amaxs[0], amaxs[1]) + for k in range_constexpr(2): + h = hs[k] + scale_f = _as_f32(e8m0s[k] << 23) + pa = _cvt_pk_fp4(fx.Int32(0), h[0], h[1], scale_f, 0) + pa = _cvt_pk_fp4(pa, h[2], h[3], scale_f, 1) + pb = _cvt_pk_fp4(fx.Int32(0), h[4], h[5], scale_f, 0) + pb = _cvt_pk_fp4(pb, h[6], h[7], scale_f, 1) + # after the swap, lane group g holds tile 2p + g%2, columns (g//2)*8 .. +8: one dword + pa, pb = _permlane16_swap(pa, pb) + row = base_row + (2 * tp + k) * 16 + r16 + col = col_base + (2 * p + (g % 2)) * 16 + (g // 2) * 8 + stores.append( + functools.partial( + buffer_ops.buffer_store, + pa | (pb << 16), + out_rsrc, + row * (I // 2) + col // 2, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + row32 = (base_row // 32) + tp + blk = row32 * OUT_SC_BLOCKS_PER_ROW32 + colgrp // 8 + pair = e8m0s[0] | (e8m0s[1] << 8) + pair16 = fx.Int32(pair).to(fx.Int16) + stores.append( + functools.partial( + buffer_ops.buffer_store, + pair16, + osc_rsrc, + blk * 256 + sc_in_block, + offset_is_bytes=True, + **STORE_KW, + ) + ) + if const_expr(not defer): + stores.pop()() + return stores + + row_r0 = tile.m_base + wave_i * (N_TILES_A * 16) + row_r1 = row_r0 + LDS_BLOCK_M + # the next tile's prologue (past the run: tile 0 with every load out of range, drained before exit) goes out + # under the first quadrant pair's blocks after block 0, which covers the sorted-ids latency; that pair's + # stores wait until those loads are issued (the compiler's wait for the sorted ids would otherwise drain + # them), then spread over the second pair's blocks + live_n = entry_n >= 0 + nt = _Tile( + live_n.select(entry_n, fx.Int32(0)), + expert_n, + a_offs_n, + live=live_n.select(fx.Int32(1), fx.Int32(0)), + ) + grp = nt.prologue_groups() + inter0 = [ + [], + grp[0] + grp[1], + grp[2], + grp[3], + grp[4], + grp[5], + grp[6], + grp[7] + grp[8], + ] + stores0 = _epilogue(c00_frag, c01_frag, row_r0, inter=inter0, defer=True) + n_it = N_TILES_A * (N_TILES_B // 2) + inter1 = [stores0[k::n_it] for k in range(n_it)] + _epilogue(c10_frag, c11_frag, row_r1, inter=inter1) + + if const_expr(signal): + return entry_n, expert_n, a_offs_n, entry >> N_BITS + return entry_n, expert_n, a_offs_n + + if const_expr(signal): + prev_mb = fx.Int32(-1) + while entry >= 0: + entry, expert, a_offs, prev_mb = _run_tile(entry, expert, a_offs, prev_mb) + # the last tile's stores (and sanitized prologue loads) have completed + fx.rocdl.s_waitcnt(vmcnt=0) + if lane_id == 0: + if prev_mb >= 0: + global_atomic_add_agent(ready + fx.Int64(prev_mb * READY_STRIDE), 1) + else: + while entry >= 0: + entry, expert, a_offs = _run_tile(entry, expert, a_offs) + + # the last tile's (sanitized) prologue loads land before the LDS goes away + fx.rocdl.s_waitcnt(vmcnt=0) + + @flyc.kernel(name="kernel_gemm1_mxfp4_h6144_i1536") + def kernel_gemm1_mxfp4_h6144_i1536( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.claim.ptr, + fx.block_idx.x, + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm1_mxfp4_h6144_i1536( + A: fx.Tensor, + W13: fx.Tensor, + OUT_Q: fx.Tensor, + A_scale: fx.Tensor, + W13_scale: fx.Tensor, + OUT_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + a_scale_bytes: fx.Int32, + tile_map_t: fx.Tensor, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm1_mxfp4_h6144_i1536( + A, + W13, + OUT_Q, + A_scale, + W13_scale, + OUT_scale, + sorted_ids, + sorted_expert_ids, + n_tokens, + num_m_blocks, + a_scale_bytes, + tile_map_t, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(N_CTAS, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm1_mxfp4_h6144_i1536.tile = tile + launch_gemm1_mxfp4_h6144_i1536.n_ctas = N_CTAS + launch_gemm1_mxfp4_h6144_i1536.n_tiles_n = I // 128 + launch_gemm1_mxfp4_h6144_i1536.lds_bytes = (LDS_TILES_BYTES, _SCALE_LDS_BYTES) + return launch_gemm1_mxfp4_h6144_i1536 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128.py new file mode 100644 index 0000000..7f5d780 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128.py @@ -0,0 +1,795 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MoE down projection (GEMM2) for H 6144, I 1536, MXFP4 x MXFP4 -> routing-weighted BF16 slot rows, on 128-row +m-tiles of a 128- or 256-row sort: ``gemm2_a4w4_bm256_half_rows``' one-n-tile, six-step K pipeline and 256 output +columns with half its row MFMAs and A / scale staging, the m-tile's sorted ids in LDS. One writer per output element. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .common.epilogue import ( + _NUM_XCDS, + _STORE_CPOL, + _bf16x2, + _lds_load_vec, + _lds_store_vec, + _v2i32, +) +from .a16w4_utils import READY_STRIDE, poll_ready, wait_ready_from +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOT_BYTES, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _asm_void, + _Buf, + _divmod_nonneg, + _flat_frag, + _g2s_thunks, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .i3072.gemm1_prefill_h6144_i3072_persistent import Mfma16x16x128Fp4 +from .mxfp8.gemm1_prefill_h6144_i3072 import BLOCK_K +from amd_kernels.warm_up import launcher + + +_SCALE_SLOTS2 = 4 + + +@launcher +def compile_moe_gemm2_mxfp4_h6144_i1536( + *, + E: int, + topk: int, + n_tokens: int = 513, + n_split: int = 6, + sort_block_m: int = 128, + wait_for: int = 0, +): + """The launcher; ``launch.tile`` is the workgroup body of the one-launch GEMM1 + GEMM2 kernels. + + Args: + n_tokens: picks the row-probe and padded-half epilogue forms; the launch takes the runtime M. + wait_for: > 0: before loading the intermediate, the body waits until its m-block's ready count reaches it. + """ + # one wave ballots the eight row-group leaders from 1537 tokens; at <= 1536 the SMEM probes measured no slower + wave_probes = n_tokens >= 1537 + # skip row half 1's epilogue when all its rows are padding: a gain at 1537-2048 tokens, noise at 3072 with GEMM1 + pad_skip = 1537 <= n_tokens <= 2048 and sort_block_m == 128 + probe_suffix = ("_wave_probes" if wave_probes else "") + ("_pad_skip" if pad_skip else "") + H, I = 6144, 1536 # noqa: E741 + BM = 128 + BN = 256 + K = I + K_BYTES = K // 2 # A row bytes (packed e2m1) + KW_BYTES = K // 2 # W2 row bytes + BLOCK_K_BYTES = BLOCK_K + EID_SHIFT = (sort_block_m // BM).bit_length() - 1 # 0 or 1 + assert sort_block_m in (BM, 2 * BM) + N_TILES_ALL = H // BN + assert H % BN == 0 and N_TILES_ALL % n_split == 0 + NT = N_TILES_ALL // n_split # n-tiles per CTA + K_ITERS = K_BYTES // BLOCK_K + assert K_ITERS == 6 and K % 256 == 0 + FLAT = K_ITERS # K-steps per iteration: one n-tile, an even count that brings the A / B buffers back + SCALE_BLOCKS = K_ITERS # one 256-B scale block (8 K-groups = 256 K) per step + LDS_BLOCK_M = BM // 2 + LDS_BLOCK_N = BN // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 4 + N_TILES_B = LDS_BLOCK_N // 2 // 16 # 4 + N_ACCUMS = N_TILES_A * N_TILES_B + NB = N_TILES_B + NA = N_TILES_A + OUT_ROW_BYTES = H * 2 + W2_BYTES = H * KW_BYTES + assert W2_BYTES <= 0xFFFFFFFF + B_TILE_BYTES = BN * KW_BYTES # one n-tile of W2 (256 rows) + + B_STAGES = 2 + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES # 16 KB + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES # 16 KB + A_BUFS = 4 * a_lds_size # 64 KB: 2 stages x 2 halves + LDS_TILES_BYTES = A_BUFS + 2 * B_STAGES * b_lds_size # 128 KB: 64 A + 64 W2 + SCALE_LDS_BYTES2 = _SCALE_SLOTS2 * _SCALE_SLOT_BYTES # 16 KB + # epilogue staging, wave-private: 16 rows x (2 halves x 64 cols) bf16, 256 B rows + STG_ROW = 2 * N_TILES_B * 16 * 2 # 256 B: the wave's two 128-B line segments + STG_CH = STG_ROW // 16 # 16 chunks of 16 B per staged row + STG_ROWS = 16 # one (row half, a-tile) at a time + STG_WAVE = STG_ROWS * STG_ROW # 4 KB + STG_LDS_BYTES = _N_WAVES * STG_WAVE # 16 KB + N_FLUSH = (STG_ROWS * 2) // 8 # dwordx4 stores per round per lane: 32 lines, 8 each + SID_LDS_BYTES = BM * 4 # the m-tile's sorted ids, read by every n-tile's epilogue + assert LDS_TILES_BYTES + SCALE_LDS_BYTES2 + STG_LDS_BYTES + SID_LDS_BYTES <= 160 * 1024 + + A_WAVE_GROUPS = N_TILES_A // 2 # 2 + A_HALF_GROUPS = LDS_BLOCK_M // 32 # 4 + B_WAVE_GROUPS = 2 # 32-row groups per wave (64 W2 rows) + B_HALF_GROUPS = LDS_BLOCK_N // 32 # 4: half 1 = 128 rows further + SC_COLS = K // 32 + W2_SC_BYTES = E * H * SC_COLS + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_LDS_BYTES2, 16] + stage_lds: fx.Array[fx.Int8, STG_LDS_BYTES, 16] + sids: fx.Array[fx.Int32, BM, 16] + + @flyc.jit + def tile( + _base_ptr, + _scale_base_ptr, + _stg_ptr, + sid_slot, + bx, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + ready, + ): + """Workgroup ``bx``'s (m-tile, n-chunk) on the given LDS regions; ``ready``: the one-launch kernel's m-block + counters (``wait_for`` > 0).""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + # B: 2 stages x 2 halves + b_st = [ + [_Buf(_base_ptr, A_BUFS + (2 * st + hf) * b_lds_size) for hf in range(2)] + for st in range(B_STAGES) + ] + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + g4 = lane_id // 16 + r16 = lane_id % 16 + + # ---- work item: (m-tile, n-chunk); consecutive m-tiles share an XCD ---- + nv_rsrc = buffer_ops.create_buffer_resource( + num_valid_ids, max_size=False, num_records_bytes=4 + ) + num_valid = fx.Int32( + buffer_ops.buffer_load( + nv_rsrc, fx.Int32(0), vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + n_work = (num_valid // fx.Int32(BM)) * fx.Int32(n_split) + per_xcd = (n_work + fx.Int32(_NUM_XCDS - 1)) // fx.Int32(_NUM_XCDS) + intra, xcd = _divmod_nonneg(bx, _NUM_XCDS) + work = xcd * per_xcd + intra + block_valid = (intra < per_xcd) & (work < n_work) + work_safe = block_valid.select(work, fx.Int32(0)) + tile_i, chunk = _divmod_nonneg(work_safe, n_split) + m_base = tile_i * BM + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + expert = fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, + tile_i >> EID_SHIFT, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BM * 4) + ) + if const_expr(EID_SHIFT > 0): + # a 2*BM-row sort pads each expert to 2*BM rows: a tile led by the sentinel (tok == n_tokens) is empty + first_sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + block_valid = block_valid & ((first_sid & fx.Int32(0x00FFFFFF)) < n_tokens) + chunk_n0 = chunk * NT # first n-tile (global index) of this CTA + # no n-tile rotation: the physical tile is the logical one + nt_rot = fx.Int32(0) + + def _pn(nt): + """CTA-local logical n-tile (sweep order) -> physical n-tile of the chunk""" + x = nt + nt_rot + return (x >= fx.Int32(NT)).select(x - fx.Int32(NT), x) + + def _n_glob(nt): + """Global n-tile of ``nt``, clamped: the redundant loads after the last tile re-fetch the last one.""" + ntc = (nt >= fx.Int32(NT)).select(fx.Int32(NT - 1), nt) + return chunk_n0 + _pn(ntc) + + if block_valid: + if const_expr(wait_for > 0): + # thread 0's first poll overlaps the setup's loads + ready_addr = ready + fx.Int64(tile_i * READY_STRIDE) + got0 = poll_ready(ready_addr) + if const_expr(wave_probes): + # padding is a row suffix: lanes 0-7 probe the eight 16-row group leaders and the low 8 ballot bits + # count the valid prefix in 16-row groups, in every wave the same count as the SMEM probes + probe_sid = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, m_base + (lane_id % 8) * 16, + vec_width=1, dtype=fx.Int32)) + probe_mask = fx.Int64(fx.rocdl.ballot( + T.i64, (probe_sid & 0x00FFFFFF) < n_tokens)) + m_active = fx.math.ctpop(probe_mask.to(fx.Int32) & 255).to(fx.Int32) * 16 + else: + # the sort ends each expert's rows with the sentinel, so padding is a row suffix of the m-tile: one + # scalar load (lgkmcnt, not vmcnt) per 16-row group leader counts the valid rows in 16-row groups + m_active = fx.Int32(0) + for g in range_constexpr(BM // 16): + sid_g = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + g * 16, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + m_active = m_active + ( + (sid_g & fx.Int32(0x00FFFFFF)) < n_tokens + ).select(fx.Int32(16), fx.Int32(0)) + + sw_rsrc = buffer_ops.create_buffer_resource( + sorted_weights, + max_size=False, + num_records_bytes=num_m_blocks * (BM * 4), + ) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=num_m_blocks * (BM * K_BYTES) + ) + b_rsrc = buffer_ops.create_buffer_resource( + W2, + max_size=False, + num_records_bytes=W2_BYTES, + base_byte_offset=fx.Int64(expert) * W2_BYTES, + ) + # padded sorted rows (tok == n_tokens) fall outside the resource: dropped + out_rows = n_tokens * fx.Int32(topk) + out_rsrc = buffer_ops.create_buffer_resource( + OUT, max_size=False, num_records_bytes=out_rows * OUT_ROW_BYTES + ) + + # ---- A rows (sorted, contiguous): row * K_BYTES + swizzled col ---- + def _a_offsets(half): + offs = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + grow = m_base + half * LDS_BLOCK_M + row + offs.append(grow * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + # padding rows (row_in_tile >= m_active) get an OOB offset: no memory access, zeros, still counted in vmcnt + a_oob = num_m_blocks * fx.Int32(BM * K_BYTES) # == a_rsrc records + + def _oob_wrap(offs, half): + out = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + row_in_tile = fx.Int32(half * LDS_BLOCK_M) + row + out.append((row_in_tile >= m_active).select(a_oob, offs[rnd])) + return out + + gl_off_a0 = _oob_wrap(_a_offsets(0), 0) + gl_off_a1 = _oob_wrap(_a_offsets(1), 1) + + # ---- W2: expert row n*256 + hf*128 + r, 16-row blocks of KW_BYTES*16 B (a8 preshuffle inside); + # the n-tile goes in soffset (B_TILE_BYTES per tile) ---- + e_row0 = expert * fx.Int32(H) + + def _b_offsets(hf): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + wrow = fx.Int32(hf * LDS_BLOCK_N + row) + off = (wrow // fx.Int32(16)) * fx.Int32(KW_BYTES * 16) + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + offs.append(off) + return offs + + gl_off_b0 = _b_offsets(0) + gl_off_b1 = _b_offsets(1) + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = BLOCK_K_BYTES * 16 + + mfma = Mfma16x16x128Fp4(N_TILES_A, N_TILES_B) + sb_index = lambda j, hf: (j // 2, j % 2) # noqa: E731 (32-row groups) + + scale_gather = ScaleGatherMoE( + A_scale, + W2_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + num_m_blocks * (BM * SC_COLS), + W2_SC_BYTES, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + + def _set_scale_tile(nt): + scale_gather.set_wave_base(m_base, e_row0 + _n_glob(nt) * fx.Int32(BN)) + + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(nt, g): + return (fx.Int32(nt) * fx.Int32(SCALE_BLOCKS) + fx.Int32(g)) % fx.Int32( + _SCALE_SLOTS2 + ) + + a0_g2s = G2SLoaderAsm(a_rsrc, gl_off_a0, N_TILES_A, wave_id) + a1_g2s = G2SLoaderAsm(a_rsrc, gl_off_a1, N_TILES_A, wave_id) + b0_g2s = G2SLoaderAsm(b_rsrc, gl_off_b0, N_TILES_B, wave_id) + b1_g2s = G2SLoaderAsm(b_rsrc, gl_off_b1, N_TILES_B, wave_id) + for ld in (a0_g2s, a1_g2s, b0_g2s, b1_g2s): + ld.set_wave_base(_base_ptr) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + + # ---- output rows / weights of the 4 rows this lane writes ---- + def _orow(sid): + tok = sid & fx.Int32(0x00FFFFFF) # padded: tok == n_tokens -> OOB + slot = (sid >> 24) & fx.Int32(0xFF) + return tok * fx.Int32(topk) + slot + + # routing weights of the lane's rows (row half h, tile ti, r16). The 32 output-row offsets are not held + # across the main loop: they spilled, and each scratch reload before a store forced vmcnt(0) (stores count + # in vmcnt). + wave_row0 = m_base + wave_i * (N_TILES_A * 16) + row_w = [] + for h in range_constexpr(2): + for ti in range_constexpr(N_TILES_A): + row = wave_row0 + fx.Int32(h * LDS_BLOCK_M + ti * 16) + r16 + row_w.append( + fx.Float32( + buffer_ops.buffer_load( + sw_rsrc, row, vec_width=1, dtype=fx.Float32 + ) + ) + ) + # the m-tile's sorted ids go to LDS before the prologue: a global load in each epilogue would wait (in-order + # vmcnt) behind every inline-asm A / W2 prefetch in flight. Two threads write each id (the same value); the + # prologue's wait_barrier publishes them. + sid_t = fx.thread_idx.x % BM + sid_slot[sid_t] = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base + sid_t, vec_width=1, dtype=fx.Int32 + ) + ) + sid_lds = fx.Int32(fx.ptrtoint(sid_slot)) + stg_base = fx.Int32(fx.ptrtoint(_stg_ptr)) + wave_id * fx.Int32(STG_WAVE) + + def _stg_addr(row, chunk, half): + """Staging address of (row 0..15, 16-B chunk, 8-B half); chunks are XOR-swizzled by the row so the + tile-wise writes and the line-wise reads spread over the banks.""" + return ( + stg_base + + row * fx.Int32(STG_ROW) + + ((chunk ^ (row % STG_CH)) * 16 + half * 8) + ) + + if const_expr(wait_for > 0): + # until GEMM1 has stored all the m-tile's intermediate rows + wait_ready_from(ready_addr, wait_for, n_split, got0) + + # ---- prologue: steps -2 and -1 of the steady-state order (B and A two steps ahead) ---- + # g0 B(0) a0(0) a1(0) g1 B(1) a0(1) g2 a1(1); scale block b is read at step b - 1 + _set_scale_tile(fx.Int32(0)) + scale_gather.gather(0, _slot(0, 0)) + b_soff0 = _n_glob(fx.Int32(0)) * fx.Int32(B_TILE_BYTES) + b0_g2s.load(b_st[0][0], b_soff0 + fx.Int32(0 * B_K_STEP)) + b1_g2s.load(b_st[0][1], b_soff0 + fx.Int32(0 * B_K_STEP)) + a0_g2s.load(a_cur0, fx.Int32(0 * A_K_STEP)) + a1_g2s.load(a_cur1, fx.Int32(0 * A_K_STEP)) + scale_gather.gather(1, _slot(0, 1)) + b0_g2s.load(b_st[1][0], b_soff0 + fx.Int32(1 * B_K_STEP)) + b1_g2s.load(b_st[1][1], b_soff0 + fx.Int32(1 * B_K_STEP)) + a0_g2s.load(a_next0, fx.Int32(1 * A_K_STEP)) + scale_gather.gather(2, _slot(0, 2)) + a1_g2s.load(a_next1, fx.Int32(1 * A_K_STEP)) + + # g0, B(0) and a0(0) landed: a1(0), g1, B(1), a0(1), g2, a1(1) may fly + wait_barrier(NA + 1 + 2 * NB + NA + 1 + NA) + a0_frag = a_s2r.load(a_cur0) + b0_frag = b_s2r.load(b_st[0][0], preshuffled=True) + b1_frag = b_s2r.load(b_st[0][1], preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0, 0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0, 0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + + # Step kc issues B(kc+2) (b0, b1), then after the MID wait a0(kc+2), the scale gather and a1(kc+2), into + # stage (step % 2). The top wait lands a1(kc), the MID wait a0(kc+1). The counts take loads only: stores + # retire out of order with loads, so counting the epilogue's stores races. + def _top_vmcnt(kc): + return 2 * NB + 2 * NA + 1 # one gather per step + + def _mid_vmcnt(kc): + # exact: step kc-1's gather and a1(kc+1), then B(kc+2), are the loads younger than a0(kc+1); a larger + # count would release with a0(kc+1) in flight and c10f's s2r would read stale LDS + return 2 * NB + NA + 1 + + def _read_scale_thunks(nt, kc_next, holder): + # the next step's scales: block kc_next of this tile or block 0 of the next (Python ints only: an ``if`` + # here would trace to an scf.if and lose the binding) + s = _slot(nt + fx.Int32(kc_next // K_ITERS), kc_next % K_ITERS) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step(nt, kc, f, a0f, b0f, b1f_in, sc, accs, abufs): + """One K-step: constexpr ``kc`` of n-tile ``nt``; the parity of ``f``, the iteration's constexpr flat + step, picks the B stage.""" + k2 = f % 2 + ac0, ac1, an0, an1 = abufs + bst = b_st[f % B_STAGES] # consumed this step, refilled for step f + 2 + bnx = b_st[(f + 1) % B_STAGES] # step f + 1's halves, read this step + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + zero_acc = kc == 0 + + _a1 = [None] * NA + _a0n = [None] * NA + _b0n = [None] * NB + _b1n = [None] * NB + # A and B of step kc + 2 (B of tile nt + 1 from kc = K_ITERS - 2 on) + ka = (kc + 2) % K_ITERS + kb = (kc + B_STAGES) % K_ITERS + nt_b = nt + fx.Int32((kc + B_STAGES) // K_ITERS) + a_off = fx.Int32(ka * A_K_STEP) + b_off = _n_glob(nt_b) * fx.Int32(B_TILE_BYTES) + fx.Int32(kb * B_K_STEP) + + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(nt, kc + 1, _scn) + # Gather scales three steps ahead, crossing n-tiles as needed. + future_block = kc + 3 + _g_blk = future_block % SCALE_BLOCKS + _g_slot = _slot(nt + fx.Int32(future_block // SCALE_BLOCKS), _g_blk) + _sc_gather = [lambda: scale_gather.gather(_g_blk, _g_slot)] + + wait_barrier(_top_vmcnt(kc)) + il = ( + _riffle( + _g2s_thunks(b0_g2s, bst[0], b_off, NB), + _s2r_thunks(a_s2r, ac1, _a1, NA, False), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, + b0f, + c00f, + saR0, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + # two G2S loaders are never riffled: each one's steps share m0 (s_mov, then s_add per step), so + # interleaving them corrupts the LDS destinations. a0's loads go with c10f. + il = _g2s_thunks(b1_g2s, bst[1], b_off, NB) + _rd_scn[2:] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a1f = _a1 + + wait_barrier(_mid_vmcnt(kc)) + # a0(kc+2) may overwrite ac0 here: its last reads retired before the top barrier, and the s2r reads + # below take an0. Concatenated, not riffled. + il = ( + _g2s_thunks(a0_g2s, ac0, a_off, NA) + + _s2r_thunks(a_s2r, an0, _a0n, NA, False) + + _sc_gather + ) + c10f = mfma.call( + a1f, + b0f, + c10f, + saR1, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a0nf = _a0n + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, NA), + _s2r_thunks(b_s2r, bnx[0], _b0n, NB, True) + + _s2r_thunks(b_s2r, bnx[1], _b1n, NB, True), + ) + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) + return ( + a0nf, + _b0n, + _b1n, + sc_next, + (c00f, c01f, c10f, c11f), + (an0, an1, ac0, ac1), + ) + + def _stage(accs, h, ti): + """Routing-weighted BF16 of the 16 rows (half ``h``, a-tile ``ti``) x the wave's 128 columns into its + staging buffer, 8 B per lane per tile; one wave's LDS ops run in order, so the writes follow the last + flush's reads.""" + w = row_w[h * NA + ti] + for hf in range_constexpr(2): + cq = accs[h * 2 + hf] + for j in range_constexpr(NB): + v = Vec(cq[mfma.idx(ti, j)]) + d0 = _bf16x2(fx.Float32(v[0]) * w, fx.Float32(v[1]) * w) + d1 = _bf16x2(fx.Float32(v[2]) * w, fx.Float32(v[3]) * w) + # cols hf*128 + j*16 + 4*g4 of the wave's 128 -> chunk + chunk = fx.Int32(hf * 8 + j * 2) + g4 // 2 + _lds_store_vec(_v2i32(d0, d1), _stg_addr(r16, chunk, g4 % 2), 2) + + def _flush(nt, h, ti, sids): + """The staged 16 rows -> full 128-B output lines, 8 per store (non-temporal): a lane takes row + 4*(lane//16) + k, segment (lane//8)%2, 16 B at chunk lane%8. ``sids``: the group's 4 sorted ids.""" + col_wave = (_n_glob(nt) * fx.Int32(BN) + wave_j * (N_TILES_B * 16)) * 2 + hf_l = (lane_id // 8) % 2 + ch_l = lane_id % 8 + _asm_void([], "s_waitcnt lgkmcnt(0)", "") + # the group's 4 reads, then its 4 stores: one LDS drain per group, not one lgkmcnt(0) per store + datas = [] + for k in range_constexpr(N_FLUSH): + row = (lane_id // 16) * 4 + fx.Int32(k) + datas.append(_lds_load_vec(_stg_addr(row, hf_l * 8 + ch_l, 0), 4)) + for k in range_constexpr(N_FLUSH): + buffer_ops.buffer_store( + datas[k], + out_rsrc, + _orow(sids[k]) * fx.Int32(OUT_ROW_BYTES) + + col_wave + + hf_l * fx.Int32(LDS_BLOCK_N * 2) + + ch_l * fx.Int32(16), + offset_is_bytes=True, + cache_modifier=_STORE_CPOL, + ) + + def _epilogue(nt, accs, act): + # Aligned 4-id LDS reads cover the lane's rows (lgkmcnt only). + sid_row = wave_i * (N_TILES_A * 16) + (lane_id // 16) * 4 + sids_all = [] + for h in range_constexpr(act): + for ti in range_constexpr(NA): + sids_all.append( + Vec( + _lds_load_vec( + sid_lds + + (sid_row + fx.Int32(h * LDS_BLOCK_M + ti * 16)) + * 4, + 4, + ) + ) + ) + for h in range_constexpr(act): + for ti in range_constexpr(NA): + _stage(accs, h, ti) + _flush( + nt, + h, + ti, + sids_all[h * NA + ti], + ) + + abufs0 = (a_cur0, a_cur1, a_next0, a_next1) + n_a = 2 * NA + n_b = 2 * NB + n_sc = 2 * (NA // 2) + 2 * (NB // 2) + _R = fx.as_ir_value + + def _flat_sc(sc): + return [_R(v) for part in sc for v in part] + + def _unflat_sc(flat): + sizes = (NA // 2, NA // 2, NB // 2, NB // 2) + out, o = [], 0 + for n in sizes: + out.append(list(flat[o : o + n])) + o += n + return tuple(out) + + init_state = ( + _flat_frag(a0_frag) + + _flat_frag(b0_frag) + + _flat_frag(b1_frag) + + _flat_sc(sc0) + ) + def _iteration(nt_idx, state, act): + nt0 = fx.Int32(nt_idx) + off = 0 + a0f = _unflat_frag(state[off : off + n_a], NA) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + abufs = abufs0 + for f in range_constexpr(FLAT): + nt = nt0 + fx.Int32(f // K_ITERS) + kc = f % K_ITERS + if const_expr(kc == 0): + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # the gathers run three steps ahead: blocks 3-5 of n-tile nt, then blocks 0-2 of nt + 1 + _set_scale_tile(nt) + if const_expr(kc == K_ITERS - 3): + _set_scale_tile(nt + fx.Int32(1)) + a0f, b0f, b1f, sc, accs, abufs = _one_step( + nt, kc, f, a0f, b0f, b1f, sc, accs, abufs + ) + if const_expr(kc == K_ITERS - 1): + _epilogue(nt, accs, act) + return _flat_frag(a0f) + _flat_frag(b0f) + _flat_frag(b1f) + _flat_sc(sc) + + # m_active is workgroup-uniform; both loops issue every load, MFMA and wait, the short one omits only row + # half 1's staging and its out-of-bounds stores + if const_expr(pad_skip): + if m_active > fx.Int32(LDS_BLOCK_M): + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 1) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + # the redundant loads of the tile after the last one must land before the LDS is handed on + wait_barrier(0) + + @flyc.kernel(name="kernel_gemm2_mxfp4_h6144_i1536_bm128" + probe_suffix) + def kernel_gemm2_mxfp4_h6144_i1536_bm128( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.stage_lds.ptr, + lds.sids.ptr, + fx.block_idx.x, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm2_mxfp4_h6144_i1536_bm128( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm2_mxfp4_h6144_i1536_bm128( + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid_size, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm2_mxfp4_h6144_i1536_bm128.kernel_name = "kernel_gemm2_mxfp4_h6144_i1536_bm128" + probe_suffix + launch_gemm2_mxfp4_h6144_i1536_bm128.tile = tile + launch_gemm2_mxfp4_h6144_i1536_bm128.lds_bytes = ( + LDS_TILES_BYTES, SCALE_LDS_BYTES2, STG_LDS_BYTES, BM + ) + return launch_gemm2_mxfp4_h6144_i1536_bm128 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_adaptive.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_adaptive.py new file mode 100644 index 0000000..c8cd02d --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_adaptive.py @@ -0,0 +1,801 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm2_a4w4_bm128`` for a 128-row sort with the column split (2, 4, 6 or 12 chunks) chosen on device from the +routed block count: no host readback, no split-K, one writer per output element. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .a16w4_utils import READY_STRIDE, poll_ready, wait_ready_from + +from .common.epilogue import ( + _NUM_XCDS, + _STORE_CPOL, + _bf16x2, + _lds_load_vec, + _lds_store_vec, + _v2i32, +) +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOT_BYTES, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _asm_void, + _Buf, + _divmod_nonneg, + _flat_frag, + _g2s_thunks, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .i3072.gemm1_prefill_h6144_i3072_persistent import Mfma16x16x128Fp4 +from .mxfp8.gemm1_prefill_h6144_i3072 import BLOCK_K +from amd_kernels.warm_up import launcher + + +_SCALE_SLOTS2 = 4 + + +def pad_half_epilogue_skip(n_tokens: int) -> bool: + """Whether the kernel served at ``n_tokens`` skips row half 1's epilogue on m-tiles of at most 64 routed rows.""" + return n_tokens <= PAD_SKIP_MAX or n_tokens >= PAD_SKIP_FROM + + +# the skip: 513-640 tokens -0.6..-3.9 us, 3584-4096 -1.1..-2.4 us; 704-1344 up to +2 us +PAD_SKIP_MAX, PAD_SKIP_FROM = 640, 3584 + + +@launcher +def compile_moe_gemm2_mxfp4_h6144_i1536( + *, + E: int, + topk: int, + n_tokens: int = 513, + pad_skip: bool | None = None, + wait_for: int = 0, +): + """The launcher; ``launch.tile`` is the workgroup body of the one-launch GEMM1 + GEMM2 kernels. + + The split is 12 only up to 21 routed blocks (252 workgroups) and at most 6 above, so a six-chunk grid of at least + 21 blocks covers every choice. + + Args: + n_tokens: picks the row-probe form and, with ``pad_skip`` None, the padded-half epilogue skip. + pad_skip: skip row half 1's epilogue on m-tiles of at most 64 routed rows. + wait_for: > 0: before loading the intermediate, the body waits until its m-block's ready count reaches it; the + runtime split then gives the number of consumers that reset the counters. + """ + assert E == 128 and topk == 4 + sort_block_m = 128 + setup_cost = 1 # n-tile units per round; 0 and 2 measured slower + # one wave ballots the eight row-group leaders from 1537 tokens; at <= 1536 the SMEM probes measured no slower + wave_probes = n_tokens >= 1537 + if pad_skip is None: + pad_skip = pad_half_epilogue_skip(n_tokens) + probe_suffix = ("_wave_probes" if wave_probes else "") + ("_pad_skip" if pad_skip else "") + probe_suffix += f"_wait{wait_for}" if wait_for else "" + H, I = 6144, 1536 # noqa: E741 + BM = 128 + BN = 256 + K = I + K_BYTES = K // 2 # A row bytes (packed e2m1) + KW_BYTES = K // 2 # W2 row bytes + BLOCK_K_BYTES = BLOCK_K + EID_SHIFT = (sort_block_m // BM).bit_length() - 1 # 0 or 1 + assert sort_block_m in (BM, 2 * BM) + N_TILES_ALL = H // BN + assert H % BN == 0 + K_ITERS = K_BYTES // BLOCK_K + assert K_ITERS == 6 and K % 256 == 0 + FLAT = K_ITERS # K-steps per iteration: one n-tile, an even count that brings the A / B buffers back + SCALE_BLOCKS = K_ITERS # one 256-B scale block (8 K-groups = 256 K) per step + LDS_BLOCK_M = BM // 2 + LDS_BLOCK_N = BN // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 4 + N_TILES_B = LDS_BLOCK_N // 2 // 16 # 4 + N_ACCUMS = N_TILES_A * N_TILES_B + NB = N_TILES_B + NA = N_TILES_A + OUT_ROW_BYTES = H * 2 + W2_BYTES = H * KW_BYTES + assert W2_BYTES <= 0xFFFFFFFF + B_TILE_BYTES = BN * KW_BYTES # one n-tile of W2 (256 rows) + + B_STAGES = 2 + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES # 16 KB + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES # 16 KB + A_BUFS = 4 * a_lds_size # 64 KB: 2 stages x 2 halves + LDS_TILES_BYTES = A_BUFS + 2 * B_STAGES * b_lds_size # 128 KB: 64 A + 64 W2 + SCALE_LDS_BYTES2 = _SCALE_SLOTS2 * _SCALE_SLOT_BYTES # 16 KB + # epilogue staging, wave-private: 16 rows x (2 halves x 64 cols) bf16, 256 B rows + STG_ROW = 2 * N_TILES_B * 16 * 2 # 256 B: the wave's two 128-B line segments + STG_CH = STG_ROW // 16 # 16 chunks of 16 B per staged row + STG_ROWS = 16 # one (row half, a-tile) at a time + STG_WAVE = STG_ROWS * STG_ROW # 4 KB + STG_LDS_BYTES = _N_WAVES * STG_WAVE # 16 KB + N_FLUSH = (STG_ROWS * 2) // 8 # dwordx4 stores per round per lane: 32 lines, 8 each + SID_LDS_BYTES = BM * 4 # the m-tile's sorted ids, read by every n-tile's epilogue + assert LDS_TILES_BYTES + SCALE_LDS_BYTES2 + STG_LDS_BYTES + SID_LDS_BYTES <= 160 * 1024 + + A_WAVE_GROUPS = N_TILES_A // 2 # 2 + A_HALF_GROUPS = LDS_BLOCK_M // 32 # 4 + B_WAVE_GROUPS = 2 # 32-row groups per wave (64 W2 rows) + B_HALF_GROUPS = LDS_BLOCK_N // 32 # 4: half 1 = 128 rows further + SC_COLS = K // 32 + W2_SC_BYTES = E * H * SC_COLS + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_LDS_BYTES2, 16] + stage_lds: fx.Array[fx.Int8, STG_LDS_BYTES, 16] + sids: fx.Array[fx.Int32, BM, 16] + + @flyc.jit + def tile( + _base_ptr, _scale_base_ptr, _stg_ptr, sids_ptr, bx, + A, W2, OUT, A_scale, W2_scale, sorted_ids, sorted_expert_ids, + sorted_weights, num_valid_ids, n_tokens, num_m_blocks, ready, + ): + """Workgroup ``bx``'s (m-tile, n-chunk) on the given LDS regions; ``ready``: the one-launch kernel's m-block + counters (``wait_for`` > 0).""" + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + # B: 2 stages x 2 halves + b_st = [ + [_Buf(_base_ptr, A_BUFS + (2 * st + hf) * b_lds_size) for hf in range(2)] + for st in range(B_STAGES) + ] + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + g4 = lane_id // 16 + r16 = lane_id % 16 + + # ---- work item: (m-tile, n-chunk); consecutive m-tiles share an XCD ---- + nv_rsrc = buffer_ops.create_buffer_resource( + num_valid_ids, max_size=False, num_records_bytes=4 + ) + num_valid = fx.Int32( + buffer_ops.buffer_load( + nv_rsrc, fx.Int32(0), vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + blocks = num_valid // BM + # Cost in n-tile units: whole 256-CU rounds times setup + tiles. + split = fx.Int32(6) + nt_count = fx.Int32(4) + best = ((blocks * 6 + 255) // 256) * (4 + setup_cost) + for ns in range_constexpr(3): + ns = (2, 4, 12)[ns] + cost = ((blocks * ns + 255) // 256) * (24 // ns + setup_cost) + better = cost < best + split = better.select(fx.Int32(ns), split) + nt_count = better.select(fx.Int32(24 // ns), nt_count) + best = better.select(cost, best) + @flyc.jit + def run_chunk(nt_count, split): + n_work = blocks * split + per_xcd = (n_work + fx.Int32(_NUM_XCDS - 1)) // fx.Int32(_NUM_XCDS) + intra, xcd = _divmod_nonneg(bx, _NUM_XCDS) + work = xcd * per_xcd + intra + block_valid = (intra < per_xcd) & (work < n_work) + work_safe = block_valid.select(work, fx.Int32(0)) + tile_i = (work_safe * nt_count) // 24 + chunk = work_safe - tile_i * split + m_base = tile_i * BM + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + expert = fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, + tile_i >> EID_SHIFT, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BM * 4) + ) + if const_expr(EID_SHIFT > 0): + # a 2*BM-row sort pads each expert to 2*BM rows: a tile led by the sentinel (tok == n_tokens) is empty + first_sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + block_valid = block_valid & ((first_sid & fx.Int32(0x00FFFFFF)) < n_tokens) + chunk_n0 = chunk * nt_count # first n-tile (global index) of this CTA + # no n-tile rotation: the physical tile is the logical one + nt_rot = fx.Int32(0) + + def _pn(nt): + """CTA-local logical n-tile (sweep order) -> physical n-tile of the chunk""" + x = nt + nt_rot + return (x >= nt_count).select(x - nt_count, x) + + def _n_glob(nt): + """Global n-tile of ``nt``, clamped: the redundant loads after the last tile re-fetch the last one.""" + ntc = (nt >= nt_count).select((nt_count - 1), nt) + return chunk_n0 + _pn(ntc) + + if block_valid: + if const_expr(wait_for > 0): + ready_addr = ready + fx.Int64(tile_i * READY_STRIDE) + got0 = poll_ready(ready_addr) + if const_expr(wave_probes): + # padding is a row suffix: lanes 0-7 probe the eight 16-row group leaders and the low 8 ballot bits + # count the valid prefix in 16-row groups, in every wave the same count as the SMEM probes + probe_sid = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, m_base + (lane_id % 8) * 16, + vec_width=1, dtype=fx.Int32)) + probe_mask = fx.Int64(fx.rocdl.ballot( + T.i64, (probe_sid & 0x00FFFFFF) < n_tokens)) + m_active = fx.math.ctpop(probe_mask.to(fx.Int32) & 255).to(fx.Int32) * 16 + else: + # the sort ends each expert's rows with the sentinel, so padding is a row suffix of the m-tile: one + # scalar load (lgkmcnt, not vmcnt) per 16-row group leader counts the valid rows in 16-row groups + m_active = fx.Int32(0) + for g in range_constexpr(BM // 16): + sid_g = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + g * 16, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + m_active = m_active + ( + (sid_g & fx.Int32(0x00FFFFFF)) < n_tokens + ).select(fx.Int32(16), fx.Int32(0)) + + sw_rsrc = buffer_ops.create_buffer_resource( + sorted_weights, + max_size=False, + num_records_bytes=num_m_blocks * (BM * 4), + ) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=num_m_blocks * (BM * K_BYTES) + ) + b_rsrc = buffer_ops.create_buffer_resource( + W2, + max_size=False, + num_records_bytes=W2_BYTES, + base_byte_offset=fx.Int64(expert) * W2_BYTES, + ) + # padded sorted rows (tok == n_tokens) fall outside the resource: dropped + out_rows = n_tokens * fx.Int32(topk) + out_rsrc = buffer_ops.create_buffer_resource( + OUT, max_size=False, num_records_bytes=out_rows * OUT_ROW_BYTES + ) + + # ---- A rows (sorted, contiguous): row * K_BYTES + swizzled col ---- + def _a_offsets(half): + offs = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + grow = m_base + half * LDS_BLOCK_M + row + offs.append(grow * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + # padding rows (row_in_tile >= m_active) get an OOB offset: no memory access, zeros, still counted in + # vmcnt + a_oob = num_m_blocks * fx.Int32(BM * K_BYTES) # == a_rsrc records + + def _oob_wrap(offs, half): + out = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + row_in_tile = fx.Int32(half * LDS_BLOCK_M) + row + out.append((row_in_tile >= m_active).select(a_oob, offs[rnd])) + return out + + gl_off_a0 = _oob_wrap(_a_offsets(0), 0) + gl_off_a1 = _oob_wrap(_a_offsets(1), 1) + + # ---- W2: expert row n*256 + hf*128 + r, 16-row blocks of KW_BYTES*16 B (a8 preshuffle inside); + # the n-tile goes in soffset (B_TILE_BYTES per tile) ---- + e_row0 = expert * fx.Int32(H) + + def _b_offsets(hf): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + wrow = fx.Int32(hf * LDS_BLOCK_N + row) + off = (wrow // fx.Int32(16)) * fx.Int32(KW_BYTES * 16) + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + offs.append(off) + return offs + + gl_off_b0 = _b_offsets(0) + gl_off_b1 = _b_offsets(1) + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = BLOCK_K_BYTES * 16 + + mfma = Mfma16x16x128Fp4(N_TILES_A, N_TILES_B) + sb_index = lambda j, hf: (j // 2, j % 2) # noqa: E731 (32-row groups) + + scale_gather = ScaleGatherMoE( + A_scale, + W2_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + num_m_blocks * (BM * SC_COLS), + W2_SC_BYTES, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + + def _set_scale_tile(nt): + scale_gather.set_wave_base(m_base, e_row0 + _n_glob(nt) * fx.Int32(BN)) + + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(nt, g): + return (fx.Int32(nt) * fx.Int32(SCALE_BLOCKS) + fx.Int32(g)) % fx.Int32( + _SCALE_SLOTS2 + ) + + a0_g2s = G2SLoaderAsm(a_rsrc, gl_off_a0, N_TILES_A, wave_id) + a1_g2s = G2SLoaderAsm(a_rsrc, gl_off_a1, N_TILES_A, wave_id) + b0_g2s = G2SLoaderAsm(b_rsrc, gl_off_b0, N_TILES_B, wave_id) + b1_g2s = G2SLoaderAsm(b_rsrc, gl_off_b1, N_TILES_B, wave_id) + for ld in (a0_g2s, a1_g2s, b0_g2s, b1_g2s): + ld.set_wave_base(_base_ptr) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + + # ---- output rows / weights of the 4 rows this lane writes ---- + def _orow(sid): + tok = sid & fx.Int32(0x00FFFFFF) # padded: tok == n_tokens -> OOB + slot = (sid >> 24) & fx.Int32(0xFF) + return tok * fx.Int32(topk) + slot + + # routing weights of the lane's rows (row half h, tile ti, r16). The 32 output-row offsets are not held + # across the main loop: they spilled, and each scratch reload before a store forced vmcnt(0) (stores + # count in vmcnt). + wave_row0 = m_base + wave_i * (N_TILES_A * 16) + row_w = [] + for h in range_constexpr(2): + for ti in range_constexpr(N_TILES_A): + row = wave_row0 + fx.Int32(h * LDS_BLOCK_M + ti * 16) + r16 + row_w.append( + fx.Float32( + buffer_ops.buffer_load( + sw_rsrc, row, vec_width=1, dtype=fx.Float32 + ) + ) + ) + # the m-tile's sorted ids go to LDS before the prologue: a global load in each epilogue would wait + # (in-order vmcnt) behind every inline-asm A / W2 prefetch in flight. Two threads write each id (the + # same value); the prologue's wait_barrier publishes them. + sid_t = fx.thread_idx.x % BM + sid_slot = sids_ptr + sid_slot[sid_t] = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base + sid_t, vec_width=1, dtype=fx.Int32 + ) + ) + sid_lds = fx.Int32(fx.ptrtoint(sid_slot)) + stg_base = fx.Int32(fx.ptrtoint(_stg_ptr)) + wave_id * fx.Int32(STG_WAVE) + + def _stg_addr(row, chunk, half): + """Staging address of (row 0..15, 16-B chunk, 8-B half); chunks are XOR-swizzled by the row so the + tile-wise writes and the line-wise reads spread over the banks.""" + return ( + stg_base + + row * fx.Int32(STG_ROW) + + ((chunk ^ (row % STG_CH)) * 16 + half * 8) + ) + + if const_expr(wait_for > 0): + wait_ready_from(ready_addr, wait_for, split, got0) + + # ---- prologue: steps -2 and -1 of the steady-state order (B and A two steps ahead) ---- + # g0 B(0) a0(0) a1(0) g1 B(1) a0(1) g2 a1(1); scale block b is read at step b - 1 + _set_scale_tile(fx.Int32(0)) + scale_gather.gather(0, _slot(0, 0)) + b_soff0 = _n_glob(fx.Int32(0)) * fx.Int32(B_TILE_BYTES) + b0_g2s.load(b_st[0][0], b_soff0 + fx.Int32(0 * B_K_STEP)) + b1_g2s.load(b_st[0][1], b_soff0 + fx.Int32(0 * B_K_STEP)) + a0_g2s.load(a_cur0, fx.Int32(0 * A_K_STEP)) + a1_g2s.load(a_cur1, fx.Int32(0 * A_K_STEP)) + scale_gather.gather(1, _slot(0, 1)) + b0_g2s.load(b_st[1][0], b_soff0 + fx.Int32(1 * B_K_STEP)) + b1_g2s.load(b_st[1][1], b_soff0 + fx.Int32(1 * B_K_STEP)) + a0_g2s.load(a_next0, fx.Int32(1 * A_K_STEP)) + scale_gather.gather(2, _slot(0, 2)) + a1_g2s.load(a_next1, fx.Int32(1 * A_K_STEP)) + + # g0, B(0) and a0(0) landed: a1(0), g1, B(1), a0(1), g2, a1(1) may fly + wait_barrier(NA + 1 + 2 * NB + NA + 1 + NA) + a0_frag = a_s2r.load(a_cur0) + b0_frag = b_s2r.load(b_st[0][0], preshuffled=True) + b1_frag = b_s2r.load(b_st[0][1], preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0, 0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0, 0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + + # Step kc issues B(kc+2) (b0, b1), then after the MID wait a0(kc+2), the scale gather and a1(kc+2), into + # stage (step % 2). The top wait lands a1(kc), the MID wait a0(kc+1). The counts take loads only: stores + # retire out of order with loads, so counting the epilogue's stores races. + def _top_vmcnt(kc): + return 2 * NB + 2 * NA + 1 # one gather per step + + def _mid_vmcnt(kc): + # exact: step kc-1's gather and a1(kc+1), then B(kc+2), are the loads younger than a0(kc+1); a + # larger count would release with a0(kc+1) in flight and c10f's s2r would read stale LDS + return 2 * NB + NA + 1 + + def _read_scale_thunks(nt, kc_next, holder): + # the next step's scales: block kc_next of this tile or block 0 of the next (Python ints only: an + # ``if`` here would trace to an scf.if and lose the binding) + s = _slot(nt + fx.Int32(kc_next // K_ITERS), kc_next % K_ITERS) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step(nt, kc, f, a0f, b0f, b1f_in, sc, accs, abufs): + """One K-step: constexpr ``kc`` of n-tile ``nt``; the parity of ``f``, the iteration's constexpr + flat step, picks the B stage.""" + k2 = f % 2 + ac0, ac1, an0, an1 = abufs + bst = b_st[f % B_STAGES] # consumed this step, refilled for step f + 2 + bnx = b_st[(f + 1) % B_STAGES] # step f + 1's halves, read this step + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + zero_acc = kc == 0 + + _a1 = [None] * NA + _a0n = [None] * NA + _b0n = [None] * NB + _b1n = [None] * NB + # A and B of step kc + 2 (B of tile nt + 1 from kc = K_ITERS - 2 on) + ka = (kc + 2) % K_ITERS + kb = (kc + B_STAGES) % K_ITERS + nt_b = nt + fx.Int32((kc + B_STAGES) // K_ITERS) + a_off = fx.Int32(ka * A_K_STEP) + b_off = _n_glob(nt_b) * fx.Int32(B_TILE_BYTES) + fx.Int32(kb * B_K_STEP) + + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(nt, kc + 1, _scn) + # Gather scales three steps ahead, crossing n-tiles as needed. + future_block = kc + 3 + _g_blk = future_block % SCALE_BLOCKS + _g_slot = _slot(nt + fx.Int32(future_block // SCALE_BLOCKS), _g_blk) + _sc_gather = [lambda: scale_gather.gather(_g_blk, _g_slot)] + + wait_barrier(_top_vmcnt(kc)) + il = ( + _riffle( + _g2s_thunks(b0_g2s, bst[0], b_off, NB), + _s2r_thunks(a_s2r, ac1, _a1, NA, False), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, + b0f, + c00f, + saR0, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + # two G2S loaders are never riffled: each one's steps share m0 (s_mov, then s_add per step), so + # interleaving them corrupts the LDS destinations. a0's loads go with c10f. + il = _g2s_thunks(b1_g2s, bst[1], b_off, NB) + _rd_scn[2:] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a1f = _a1 + + wait_barrier(_mid_vmcnt(kc)) + # a0(kc+2) may overwrite ac0 here: its last reads retired before the top barrier, and the s2r reads + # below take an0. Concatenated, not riffled. + il = ( + _g2s_thunks(a0_g2s, ac0, a_off, NA) + + _s2r_thunks(a_s2r, an0, _a0n, NA, False) + + _sc_gather + ) + c10f = mfma.call( + a1f, + b0f, + c10f, + saR1, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a0nf = _a0n + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, NA), + _s2r_thunks(b_s2r, bnx[0], _b0n, NB, True) + + _s2r_thunks(b_s2r, bnx[1], _b1n, NB, True), + ) + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) + return ( + a0nf, + _b0n, + _b1n, + sc_next, + (c00f, c01f, c10f, c11f), + (an0, an1, ac0, ac1), + ) + + def _stage(accs, h, ti): + """Routing-weighted BF16 of the 16 rows (half ``h``, a-tile ``ti``) x the wave's 128 columns into + its staging buffer, 8 B per lane per tile; one wave's LDS ops run in order, so the writes follow the + last flush's reads.""" + w = row_w[h * NA + ti] + for hf in range_constexpr(2): + cq = accs[h * 2 + hf] + for j in range_constexpr(NB): + v = Vec(cq[mfma.idx(ti, j)]) + d0 = _bf16x2(fx.Float32(v[0]) * w, fx.Float32(v[1]) * w) + d1 = _bf16x2(fx.Float32(v[2]) * w, fx.Float32(v[3]) * w) + # cols hf*128 + j*16 + 4*g4 of the wave's 128 -> chunk + chunk = fx.Int32(hf * 8 + j * 2) + g4 // 2 + _lds_store_vec(_v2i32(d0, d1), _stg_addr(r16, chunk, g4 % 2), 2) + + def _flush(nt, h, ti, sids): + """The staged 16 rows -> full 128-B output lines, 8 per store (non-temporal): a lane takes row + 4*(lane//16) + k, segment (lane//8)%2, 16 B at chunk lane%8. ``sids``: the group's 4 sorted ids.""" + col_wave = (_n_glob(nt) * fx.Int32(BN) + wave_j * (N_TILES_B * 16)) * 2 + hf_l = (lane_id // 8) % 2 + ch_l = lane_id % 8 + _asm_void([], "s_waitcnt lgkmcnt(0)", "") + # the group's 4 reads, then its 4 stores: one LDS drain per group, not one lgkmcnt(0) per store + datas = [] + for k in range_constexpr(N_FLUSH): + row = (lane_id // 16) * 4 + fx.Int32(k) + datas.append(_lds_load_vec(_stg_addr(row, hf_l * 8 + ch_l, 0), 4)) + for k in range_constexpr(N_FLUSH): + buffer_ops.buffer_store( + datas[k], + out_rsrc, + _orow(sids[k]) * fx.Int32(OUT_ROW_BYTES) + + col_wave + + hf_l * fx.Int32(LDS_BLOCK_N * 2) + + ch_l * fx.Int32(16), + offset_is_bytes=True, + cache_modifier=_STORE_CPOL, + ) + + def _epilogue(nt, accs, act): + # Aligned 4-id LDS reads cover the lane's rows (lgkmcnt only). + sid_row = wave_i * (N_TILES_A * 16) + (lane_id // 16) * 4 + sids_all = [] + for h in range_constexpr(act): + for ti in range_constexpr(NA): + sids_all.append( + Vec( + _lds_load_vec( + sid_lds + + (sid_row + fx.Int32(h * LDS_BLOCK_M + ti * 16)) + * 4, + 4, + ) + ) + ) + for h in range_constexpr(act): + for ti in range_constexpr(NA): + _stage(accs, h, ti) + _flush( + nt, + h, + ti, + sids_all[h * NA + ti], + ) + + abufs0 = (a_cur0, a_cur1, a_next0, a_next1) + n_a = 2 * NA + n_b = 2 * NB + n_sc = 2 * (NA // 2) + 2 * (NB // 2) + _R = fx.as_ir_value + + def _flat_sc(sc): + return [_R(v) for part in sc for v in part] + + def _unflat_sc(flat): + sizes = (NA // 2, NA // 2, NB // 2, NB // 2) + out, o = [], 0 + for n in sizes: + out.append(list(flat[o : o + n])) + o += n + return tuple(out) + + init_state = ( + _flat_frag(a0_frag) + + _flat_frag(b0_frag) + + _flat_frag(b1_frag) + + _flat_sc(sc0) + ) + def _iteration(nt_idx, state, act): + nt0 = fx.Int32(nt_idx) + off = 0 + a0f = _unflat_frag(state[off : off + n_a], NA) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + abufs = abufs0 + for f in range_constexpr(FLAT): + nt = nt0 + fx.Int32(f // K_ITERS) + kc = f % K_ITERS + if const_expr(kc == 0): + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # the gathers run three steps ahead: blocks 3-5 of n-tile nt, then blocks 0-2 of nt + 1 + _set_scale_tile(nt) + if const_expr(kc == K_ITERS - 3): + _set_scale_tile(nt + fx.Int32(1)) + a0f, b0f, b1f, sc, accs, abufs = _one_step( + nt, kc, f, a0f, b0f, b1f, sc, accs, abufs + ) + if const_expr(kc == K_ITERS - 1 and act > 0): + _epilogue(nt, accs, act) + return ( + _flat_frag(a0f) + _flat_frag(b0f) + _flat_frag(b1f) + _flat_sc(sc) + ) + + # m_active is workgroup-uniform; both loops issue every load, MFMA and wait, the short one omits only + # row half 1's staging and its out-of-bounds stores. FlyDSL rewrites range(init=...) to a loop carrying + # state. + if const_expr(pad_skip): + if m_active > fx.Int32(LDS_BLOCK_M): + for nt_idx, state in range( # type: ignore[call-overload] + 0, nt_count, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, nt_count, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 1) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, nt_count, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + # the redundant loads of the tile after the last one must land before the LDS is handed on + wait_barrier(0) + + @flyc.jit + def dispatch_chunk(): + if split == 6: + run_chunk(fx.Int32(4), fx.Int32(6)) + else: + run_chunk(nt_count, split) + + dispatch_chunk() + + @flyc.kernel(name="kernel_gemm2_mxfp4_h6144_i1536_bm128_adaptive" + probe_suffix) + def kernel_gemm2_mxfp4_h6144_i1536_bm128_adaptive( + A: fx.Tensor, W2: fx.Tensor, OUT: fx.Tensor, + A_scale: fx.Tensor, W2_scale: fx.Tensor, sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, n_tokens: fx.Int32, num_m_blocks: fx.Int32, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile(lds.all_lds.ptr, lds.scale_lds.ptr, lds.stage_lds.ptr, lds.sids.ptr, + fx.block_idx.x, A, W2, OUT, A_scale, W2_scale, sorted_ids, + sorted_expert_ids, sorted_weights, num_valid_ids, n_tokens, num_m_blocks, + fx.Int64(0)) + + @flyc.jit + def launch_gemm2_mxfp4_h6144_i1536_bm128_adaptive( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm2_mxfp4_h6144_i1536_bm128_adaptive( + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid_size, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm2_mxfp4_h6144_i1536_bm128_adaptive.kernel_name = "kernel_gemm2_mxfp4_h6144_i1536_bm128_adaptive" + probe_suffix + launch_gemm2_mxfp4_h6144_i1536_bm128_adaptive.tile = tile + launch_gemm2_mxfp4_h6144_i1536_bm128_adaptive.lds_bytes = ( + LDS_TILES_BYTES, SCALE_LDS_BYTES2, STG_LDS_BYTES, BM, + ) + return launch_gemm2_mxfp4_h6144_i1536_bm128_adaptive diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_epilogue_groups.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_epilogue_groups.py new file mode 100644 index 0000000..0642717 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm128_epilogue_groups.py @@ -0,0 +1,808 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm2_a4w4_bm128`` (128-row sort, at most 1536 tokens) in which a wave skips the epilogue of each of its 16-row +groups that is all padding; every MFMA, load and CTA barrier stays unconditional. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .common.epilogue import ( + _NUM_XCDS, + _STORE_CPOL, + _bf16x2, + _lds_load_vec, + _lds_store_vec, + _v2i32, +) +from .a16w4_utils import READY_STRIDE, poll_ready, wait_ready_from +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOT_BYTES, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _asm_void, + _Buf, + _divmod_nonneg, + _flat_frag, + _g2s_thunks, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .i3072.gemm1_prefill_h6144_i3072_persistent import Mfma16x16x128Fp4 +from .mxfp8.gemm1_prefill_h6144_i3072 import BLOCK_K +from amd_kernels.warm_up import launcher + + +_SCALE_SLOTS2 = 4 + + +@launcher +def compile_moe_gemm2_mxfp4_h6144_i1536( + *, + E: int, + topk: int, + n_tokens: int = 513, + n_split: int = 6, + sort_block_m: int = 128, + wait_for: int = 0, +): + """The launcher; ``launch.tile`` is the workgroup body of the one-launch GEMM1 + GEMM2 kernels. + + Args: + n_tokens: picks the row-probe and padded-half epilogue forms; the launch takes the runtime M. + wait_for: > 0: before loading the intermediate, the body waits until its m-block's ready count reaches it. + """ + assert n_tokens <= 1536 and sort_block_m == 128 + # one wave ballots the eight row-group leaders from 1537 tokens; at <= 1536 the SMEM probes measured no slower + wave_probes = n_tokens >= 1537 + group_skip = True + # skip row half 1's epilogue when all its rows are padding: a gain at 1537-2048 tokens, noise at 3072 with GEMM1 + pad_skip = 1537 <= n_tokens <= 2048 and sort_block_m == 128 + probe_suffix = ("_wave_probes" if wave_probes else "") + ("_pad_skip" if pad_skip else "") + H, I = 6144, 1536 # noqa: E741 + BM = 128 + BN = 256 + K = I + K_BYTES = K // 2 # A row bytes (packed e2m1) + KW_BYTES = K // 2 # W2 row bytes + BLOCK_K_BYTES = BLOCK_K + EID_SHIFT = (sort_block_m // BM).bit_length() - 1 # 0 or 1 + assert sort_block_m in (BM, 2 * BM) + N_TILES_ALL = H // BN + assert H % BN == 0 and N_TILES_ALL % n_split == 0 + NT = N_TILES_ALL // n_split # n-tiles per CTA + K_ITERS = K_BYTES // BLOCK_K + assert K_ITERS == 6 and K % 256 == 0 + FLAT = K_ITERS # K-steps per iteration: one n-tile, an even count that brings the A / B buffers back + SCALE_BLOCKS = K_ITERS # one 256-B scale block (8 K-groups = 256 K) per step + LDS_BLOCK_M = BM // 2 + LDS_BLOCK_N = BN // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 4 + N_TILES_B = LDS_BLOCK_N // 2 // 16 # 4 + N_ACCUMS = N_TILES_A * N_TILES_B + NB = N_TILES_B + NA = N_TILES_A + OUT_ROW_BYTES = H * 2 + W2_BYTES = H * KW_BYTES + assert W2_BYTES <= 0xFFFFFFFF + B_TILE_BYTES = BN * KW_BYTES # one n-tile of W2 (256 rows) + + B_STAGES = 2 + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES # 16 KB + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES # 16 KB + A_BUFS = 4 * a_lds_size # 64 KB: 2 stages x 2 halves + LDS_TILES_BYTES = A_BUFS + 2 * B_STAGES * b_lds_size # 128 KB: 64 A + 64 W2 + SCALE_LDS_BYTES2 = _SCALE_SLOTS2 * _SCALE_SLOT_BYTES # 16 KB + # epilogue staging, wave-private: 16 rows x (2 halves x 64 cols) bf16, 256 B rows + STG_ROW = 2 * N_TILES_B * 16 * 2 # 256 B: the wave's two 128-B line segments + STG_CH = STG_ROW // 16 # 16 chunks of 16 B per staged row + STG_ROWS = 16 # one (row half, a-tile) at a time + STG_WAVE = STG_ROWS * STG_ROW # 4 KB + STG_LDS_BYTES = _N_WAVES * STG_WAVE # 16 KB + N_FLUSH = (STG_ROWS * 2) // 8 # dwordx4 stores per round per lane: 32 lines, 8 each + SID_LDS_BYTES = BM * 4 # the m-tile's sorted ids, read by every n-tile's epilogue + assert LDS_TILES_BYTES + SCALE_LDS_BYTES2 + STG_LDS_BYTES + SID_LDS_BYTES <= 160 * 1024 + + A_WAVE_GROUPS = N_TILES_A // 2 # 2 + A_HALF_GROUPS = LDS_BLOCK_M // 32 # 4 + B_WAVE_GROUPS = 2 # 32-row groups per wave (64 W2 rows) + B_HALF_GROUPS = LDS_BLOCK_N // 32 # 4: half 1 = 128 rows further + SC_COLS = K // 32 + W2_SC_BYTES = E * H * SC_COLS + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_LDS_BYTES2, 16] + stage_lds: fx.Array[fx.Int8, STG_LDS_BYTES, 16] + sids: fx.Array[fx.Int32, BM, 16] + + @flyc.jit + def tile( + _base_ptr, + _scale_base_ptr, + _stg_ptr, + sid_slot, + bx, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + ready, + ): + """Workgroup ``bx``'s (m-tile, n-chunk) on the given LDS regions; ``ready``: the one-launch kernel's m-block + counters (``wait_for`` > 0).""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + # B: 2 stages x 2 halves + b_st = [ + [_Buf(_base_ptr, A_BUFS + (2 * st + hf) * b_lds_size) for hf in range(2)] + for st in range(B_STAGES) + ] + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + g4 = lane_id // 16 + r16 = lane_id % 16 + + # ---- work item: (m-tile, n-chunk); consecutive m-tiles share an XCD ---- + nv_rsrc = buffer_ops.create_buffer_resource( + num_valid_ids, max_size=False, num_records_bytes=4 + ) + num_valid = fx.Int32( + buffer_ops.buffer_load( + nv_rsrc, fx.Int32(0), vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + n_work = (num_valid // fx.Int32(BM)) * fx.Int32(n_split) + per_xcd = (n_work + fx.Int32(_NUM_XCDS - 1)) // fx.Int32(_NUM_XCDS) + intra, xcd = _divmod_nonneg(bx, _NUM_XCDS) + work = xcd * per_xcd + intra + block_valid = (intra < per_xcd) & (work < n_work) + work_safe = block_valid.select(work, fx.Int32(0)) + tile_i, chunk = _divmod_nonneg(work_safe, n_split) + m_base = tile_i * BM + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + expert = fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, + tile_i >> EID_SHIFT, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BM * 4) + ) + if const_expr(EID_SHIFT > 0): + # a 2*BM-row sort pads each expert to 2*BM rows: a tile led by the sentinel (tok == n_tokens) is empty + first_sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + block_valid = block_valid & ((first_sid & fx.Int32(0x00FFFFFF)) < n_tokens) + chunk_n0 = chunk * NT # first n-tile (global index) of this CTA + # no n-tile rotation: the physical tile is the logical one + nt_rot = fx.Int32(0) + + def _pn(nt): + """CTA-local logical n-tile (sweep order) -> physical n-tile of the chunk""" + x = nt + nt_rot + return (x >= fx.Int32(NT)).select(x - fx.Int32(NT), x) + + def _n_glob(nt): + """Global n-tile of ``nt``, clamped: the redundant loads after the last tile re-fetch the last one.""" + ntc = (nt >= fx.Int32(NT)).select(fx.Int32(NT - 1), nt) + return chunk_n0 + _pn(ntc) + + if block_valid: + if const_expr(wait_for > 0): + # thread 0's first poll overlaps the setup's loads + ready_addr = ready + fx.Int64(tile_i * READY_STRIDE) + got0 = poll_ready(ready_addr) + if const_expr(wave_probes): + # padding is a row suffix: lanes 0-7 probe the eight 16-row group leaders and the low 8 ballot bits + # count the valid prefix in 16-row groups, in every wave the same count as the SMEM probes + probe_sid = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, m_base + (lane_id % 8) * 16, + vec_width=1, dtype=fx.Int32)) + probe_mask = fx.Int64(fx.rocdl.ballot( + T.i64, (probe_sid & 0x00FFFFFF) < n_tokens)) + m_active = fx.math.ctpop(probe_mask.to(fx.Int32) & 255).to(fx.Int32) * 16 + else: + # the sort ends each expert's rows with the sentinel, so padding is a row suffix of the m-tile: one + # scalar load (lgkmcnt, not vmcnt) per 16-row group leader counts the valid rows in 16-row groups + m_active = fx.Int32(0) + for g in range_constexpr(BM // 16): + sid_g = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + g * 16, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + m_active = m_active + ( + (sid_g & fx.Int32(0x00FFFFFF)) < n_tokens + ).select(fx.Int32(16), fx.Int32(0)) + + sw_rsrc = buffer_ops.create_buffer_resource( + sorted_weights, + max_size=False, + num_records_bytes=num_m_blocks * (BM * 4), + ) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=num_m_blocks * (BM * K_BYTES) + ) + b_rsrc = buffer_ops.create_buffer_resource( + W2, + max_size=False, + num_records_bytes=W2_BYTES, + base_byte_offset=fx.Int64(expert) * W2_BYTES, + ) + # padded sorted rows (tok == n_tokens) fall outside the resource: dropped + out_rows = n_tokens * fx.Int32(topk) + out_rsrc = buffer_ops.create_buffer_resource( + OUT, max_size=False, num_records_bytes=out_rows * OUT_ROW_BYTES + ) + + # ---- A rows (sorted, contiguous): row * K_BYTES + swizzled col ---- + def _a_offsets(half): + offs = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + grow = m_base + half * LDS_BLOCK_M + row + offs.append(grow * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + # padding rows (row_in_tile >= m_active) get an OOB offset: no memory access, zeros, still counted in vmcnt + a_oob = num_m_blocks * fx.Int32(BM * K_BYTES) # == a_rsrc records + + def _oob_wrap(offs, half): + out = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + row_in_tile = fx.Int32(half * LDS_BLOCK_M) + row + out.append((row_in_tile >= m_active).select(a_oob, offs[rnd])) + return out + + gl_off_a0 = _oob_wrap(_a_offsets(0), 0) + gl_off_a1 = _oob_wrap(_a_offsets(1), 1) + + # ---- W2: expert row n*256 + hf*128 + r, 16-row blocks of KW_BYTES*16 B (a8 preshuffle inside); + # the n-tile goes in soffset (B_TILE_BYTES per tile) ---- + e_row0 = expert * fx.Int32(H) + + def _b_offsets(hf): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + wrow = fx.Int32(hf * LDS_BLOCK_N + row) + off = (wrow // fx.Int32(16)) * fx.Int32(KW_BYTES * 16) + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + offs.append(off) + return offs + + gl_off_b0 = _b_offsets(0) + gl_off_b1 = _b_offsets(1) + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = BLOCK_K_BYTES * 16 + + mfma = Mfma16x16x128Fp4(N_TILES_A, N_TILES_B) + sb_index = lambda j, hf: (j // 2, j % 2) # noqa: E731 (32-row groups) + + scale_gather = ScaleGatherMoE( + A_scale, + W2_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + num_m_blocks * (BM * SC_COLS), + W2_SC_BYTES, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + + def _set_scale_tile(nt): + scale_gather.set_wave_base(m_base, e_row0 + _n_glob(nt) * fx.Int32(BN)) + + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(nt, g): + return (fx.Int32(nt) * fx.Int32(SCALE_BLOCKS) + fx.Int32(g)) % fx.Int32( + _SCALE_SLOTS2 + ) + + a0_g2s = G2SLoaderAsm(a_rsrc, gl_off_a0, N_TILES_A, wave_id) + a1_g2s = G2SLoaderAsm(a_rsrc, gl_off_a1, N_TILES_A, wave_id) + b0_g2s = G2SLoaderAsm(b_rsrc, gl_off_b0, N_TILES_B, wave_id) + b1_g2s = G2SLoaderAsm(b_rsrc, gl_off_b1, N_TILES_B, wave_id) + for ld in (a0_g2s, a1_g2s, b0_g2s, b1_g2s): + ld.set_wave_base(_base_ptr) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + + # ---- output rows / weights of the 4 rows this lane writes ---- + def _orow(sid): + tok = sid & fx.Int32(0x00FFFFFF) # padded: tok == n_tokens -> OOB + slot = (sid >> 24) & fx.Int32(0xFF) + return tok * fx.Int32(topk) + slot + + # routing weights of the lane's rows (row half h, tile ti, r16). The 32 output-row offsets are not held + # across the main loop: they spilled, and each scratch reload before a store forced vmcnt(0) (stores count + # in vmcnt). + wave_row0 = m_base + wave_i * (N_TILES_A * 16) + row_w = [] + for h in range_constexpr(2): + for ti in range_constexpr(N_TILES_A): + row = wave_row0 + fx.Int32(h * LDS_BLOCK_M + ti * 16) + r16 + row_w.append( + fx.Float32( + buffer_ops.buffer_load( + sw_rsrc, row, vec_width=1, dtype=fx.Float32 + ) + ) + ) + # the m-tile's sorted ids go to LDS before the prologue: a global load in each epilogue would wait (in-order + # vmcnt) behind every inline-asm A / W2 prefetch in flight. Two threads write each id (the same value); the + # prologue's wait_barrier publishes them. + sid_t = fx.thread_idx.x % BM + sid_slot[sid_t] = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base + sid_t, vec_width=1, dtype=fx.Int32 + ) + ) + sid_lds = fx.Int32(fx.ptrtoint(sid_slot)) + stg_base = fx.Int32(fx.ptrtoint(_stg_ptr)) + wave_id * fx.Int32(STG_WAVE) + + def _stg_addr(row, chunk, half): + """Staging address of (row 0..15, 16-B chunk, 8-B half); chunks are XOR-swizzled by the row so the + tile-wise writes and the line-wise reads spread over the banks.""" + return ( + stg_base + + row * fx.Int32(STG_ROW) + + ((chunk ^ (row % STG_CH)) * 16 + half * 8) + ) + + if const_expr(wait_for > 0): + # until GEMM1 has stored all the m-tile's intermediate rows + wait_ready_from(ready_addr, wait_for, n_split, got0) + + # ---- prologue: steps -2 and -1 of the steady-state order (B and A two steps ahead) ---- + # g0 B(0) a0(0) a1(0) g1 B(1) a0(1) g2 a1(1); scale block b is read at step b - 1 + _set_scale_tile(fx.Int32(0)) + scale_gather.gather(0, _slot(0, 0)) + b_soff0 = _n_glob(fx.Int32(0)) * fx.Int32(B_TILE_BYTES) + b0_g2s.load(b_st[0][0], b_soff0 + fx.Int32(0 * B_K_STEP)) + b1_g2s.load(b_st[0][1], b_soff0 + fx.Int32(0 * B_K_STEP)) + a0_g2s.load(a_cur0, fx.Int32(0 * A_K_STEP)) + a1_g2s.load(a_cur1, fx.Int32(0 * A_K_STEP)) + scale_gather.gather(1, _slot(0, 1)) + b0_g2s.load(b_st[1][0], b_soff0 + fx.Int32(1 * B_K_STEP)) + b1_g2s.load(b_st[1][1], b_soff0 + fx.Int32(1 * B_K_STEP)) + a0_g2s.load(a_next0, fx.Int32(1 * A_K_STEP)) + scale_gather.gather(2, _slot(0, 2)) + a1_g2s.load(a_next1, fx.Int32(1 * A_K_STEP)) + + # g0, B(0) and a0(0) landed: a1(0), g1, B(1), a0(1), g2, a1(1) may fly + wait_barrier(NA + 1 + 2 * NB + NA + 1 + NA) + a0_frag = a_s2r.load(a_cur0) + b0_frag = b_s2r.load(b_st[0][0], preshuffled=True) + b1_frag = b_s2r.load(b_st[0][1], preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0, 0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0, 0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + + # Step kc issues B(kc+2) (b0, b1), then after the MID wait a0(kc+2), the scale gather and a1(kc+2), into + # stage (step % 2). The top wait lands a1(kc), the MID wait a0(kc+1). The counts take loads only: stores + # retire out of order with loads, so counting the epilogue's stores races. + def _top_vmcnt(kc): + return 2 * NB + 2 * NA + 1 # one gather per step + + def _mid_vmcnt(kc): + # exact: step kc-1's gather and a1(kc+1), then B(kc+2), are the loads younger than a0(kc+1); a larger + # count would release with a0(kc+1) in flight and c10f's s2r would read stale LDS + return 2 * NB + NA + 1 + + def _read_scale_thunks(nt, kc_next, holder): + # the next step's scales: block kc_next of this tile or block 0 of the next (Python ints only: an ``if`` + # here would trace to an scf.if and lose the binding) + s = _slot(nt + fx.Int32(kc_next // K_ITERS), kc_next % K_ITERS) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step(nt, kc, f, a0f, b0f, b1f_in, sc, accs, abufs): + """One K-step: constexpr ``kc`` of n-tile ``nt``; the parity of ``f``, the iteration's constexpr flat + step, picks the B stage.""" + k2 = f % 2 + ac0, ac1, an0, an1 = abufs + bst = b_st[f % B_STAGES] # consumed this step, refilled for step f + 2 + bnx = b_st[(f + 1) % B_STAGES] # step f + 1's halves, read this step + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + zero_acc = kc == 0 + + _a1 = [None] * NA + _a0n = [None] * NA + _b0n = [None] * NB + _b1n = [None] * NB + # A and B of step kc + 2 (B of tile nt + 1 from kc = K_ITERS - 2 on) + ka = (kc + 2) % K_ITERS + kb = (kc + B_STAGES) % K_ITERS + nt_b = nt + fx.Int32((kc + B_STAGES) // K_ITERS) + a_off = fx.Int32(ka * A_K_STEP) + b_off = _n_glob(nt_b) * fx.Int32(B_TILE_BYTES) + fx.Int32(kb * B_K_STEP) + + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(nt, kc + 1, _scn) + # Gather scales three steps ahead, crossing n-tiles as needed. + future_block = kc + 3 + _g_blk = future_block % SCALE_BLOCKS + _g_slot = _slot(nt + fx.Int32(future_block // SCALE_BLOCKS), _g_blk) + _sc_gather = [lambda: scale_gather.gather(_g_blk, _g_slot)] + + wait_barrier(_top_vmcnt(kc)) + il = ( + _riffle( + _g2s_thunks(b0_g2s, bst[0], b_off, NB), + _s2r_thunks(a_s2r, ac1, _a1, NA, False), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, + b0f, + c00f, + saR0, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + # two G2S loaders are never riffled: each one's steps share m0 (s_mov, then s_add per step), so + # interleaving them corrupts the LDS destinations. a0's loads go with c10f. + il = _g2s_thunks(b1_g2s, bst[1], b_off, NB) + _rd_scn[2:] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a1f = _a1 + + wait_barrier(_mid_vmcnt(kc)) + # a0(kc+2) may overwrite ac0 here: its last reads retired before the top barrier, and the s2r reads + # below take an0. Concatenated, not riffled. + il = ( + _g2s_thunks(a0_g2s, ac0, a_off, NA) + + _s2r_thunks(a_s2r, an0, _a0n, NA, False) + + _sc_gather + ) + c10f = mfma.call( + a1f, + b0f, + c10f, + saR1, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a0nf = _a0n + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, NA), + _s2r_thunks(b_s2r, bnx[0], _b0n, NB, True) + + _s2r_thunks(b_s2r, bnx[1], _b1n, NB, True), + ) + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) + return ( + a0nf, + _b0n, + _b1n, + sc_next, + (c00f, c01f, c10f, c11f), + (an0, an1, ac0, ac1), + ) + + def _stage(accs, h, ti): + """Routing-weighted BF16 of the 16 rows (half ``h``, a-tile ``ti``) x the wave's 128 columns into its + staging buffer, 8 B per lane per tile; one wave's LDS ops run in order, so the writes follow the last + flush's reads.""" + w = row_w[h * NA + ti] + for hf in range_constexpr(2): + cq = accs[h * 2 + hf] + for j in range_constexpr(NB): + v = Vec(cq[mfma.idx(ti, j)]) + d0 = _bf16x2(fx.Float32(v[0]) * w, fx.Float32(v[1]) * w) + d1 = _bf16x2(fx.Float32(v[2]) * w, fx.Float32(v[3]) * w) + # cols hf*128 + j*16 + 4*g4 of the wave's 128 -> chunk + chunk = fx.Int32(hf * 8 + j * 2) + g4 // 2 + _lds_store_vec(_v2i32(d0, d1), _stg_addr(r16, chunk, g4 % 2), 2) + + def _flush(nt, h, ti, sids): + """The staged 16 rows -> full 128-B output lines, 8 per store (non-temporal): a lane takes row + 4*(lane//16) + k, segment (lane//8)%2, 16 B at chunk lane%8. ``sids``: the group's 4 sorted ids.""" + col_wave = (_n_glob(nt) * fx.Int32(BN) + wave_j * (N_TILES_B * 16)) * 2 + hf_l = (lane_id // 8) % 2 + ch_l = lane_id % 8 + _asm_void([], "s_waitcnt lgkmcnt(0)", "") + # the group's 4 reads, then its 4 stores: one LDS drain per group, not one lgkmcnt(0) per store + datas = [] + for k in range_constexpr(N_FLUSH): + row = (lane_id // 16) * 4 + fx.Int32(k) + datas.append(_lds_load_vec(_stg_addr(row, hf_l * 8 + ch_l, 0), 4)) + for k in range_constexpr(N_FLUSH): + buffer_ops.buffer_store( + datas[k], + out_rsrc, + _orow(sids[k]) * fx.Int32(OUT_ROW_BYTES) + + col_wave + + hf_l * fx.Int32(LDS_BLOCK_N * 2) + + ch_l * fx.Int32(16), + offset_is_bytes=True, + cache_modifier=_STORE_CPOL, + ) + + @flyc.jit + def _epilogue(nt, accs, act): + # Aligned 4-id LDS reads cover the lane's rows (lgkmcnt only). + sid_row = wave_i * (N_TILES_A * 16) + (lane_id // 16) * 4 + sids_all = [] + for h in range_constexpr(act): + for ti in range_constexpr(NA): + sids_all.append( + Vec( + _lds_load_vec( + sid_lds + + (sid_row + fx.Int32(h * LDS_BLOCK_M + ti * 16)) + * 4, + 4, + ) + ) + ) + for h in range_constexpr(act): + for ti in range_constexpr(NA): + if const_expr(group_skip): + # wave-uniform branch: the staging is wave-private and the group holds no CTA barrier + if m_active > wave_i * (NA * 16) + h * LDS_BLOCK_M + ti * 16: + _stage(accs, h, ti) + _flush( + nt, + h, + ti, + sids_all[h * NA + ti], + ) + else: + _stage(accs, h, ti) + _flush( + nt, + h, + ti, + sids_all[h * NA + ti], + ) + + abufs0 = (a_cur0, a_cur1, a_next0, a_next1) + n_a = 2 * NA + n_b = 2 * NB + n_sc = 2 * (NA // 2) + 2 * (NB // 2) + _R = fx.as_ir_value + + def _flat_sc(sc): + return [_R(v) for part in sc for v in part] + + def _unflat_sc(flat): + sizes = (NA // 2, NA // 2, NB // 2, NB // 2) + out, o = [], 0 + for n in sizes: + out.append(list(flat[o : o + n])) + o += n + return tuple(out) + + init_state = ( + _flat_frag(a0_frag) + + _flat_frag(b0_frag) + + _flat_frag(b1_frag) + + _flat_sc(sc0) + ) + def _iteration(nt_idx, state, act): + nt0 = fx.Int32(nt_idx) + off = 0 + a0f = _unflat_frag(state[off : off + n_a], NA) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + abufs = abufs0 + for f in range_constexpr(FLAT): + nt = nt0 + fx.Int32(f // K_ITERS) + kc = f % K_ITERS + if const_expr(kc == 0): + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # the gathers run three steps ahead: blocks 3-5 of n-tile nt, then blocks 0-2 of nt + 1 + _set_scale_tile(nt) + if const_expr(kc == K_ITERS - 3): + _set_scale_tile(nt + fx.Int32(1)) + a0f, b0f, b1f, sc, accs, abufs = _one_step( + nt, kc, f, a0f, b0f, b1f, sc, accs, abufs + ) + if const_expr(kc == K_ITERS - 1): + _epilogue(nt, accs, act) + return _flat_frag(a0f) + _flat_frag(b0f) + _flat_frag(b1f) + _flat_sc(sc) + + # m_active is workgroup-uniform; both loops issue every load, MFMA and wait, the short one omits only row + # half 1's staging and its out-of-bounds stores + if const_expr(pad_skip): + if m_active > fx.Int32(LDS_BLOCK_M): + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 1) + else: + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + state = yield _iteration(nt_idx, state, 2) + # the redundant loads of the tile after the last one must land before the LDS is handed on + wait_barrier(0) + + @flyc.kernel(name="kernel_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups" + probe_suffix) + def kernel_gemm2_mxfp4_h6144_i1536_bm128( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.stage_lds.ptr, + lds.sids.ptr, + fx.block_idx.x, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm2_mxfp4_h6144_i1536_bm128( + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid_size, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups.kernel_name = "kernel_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups" + probe_suffix + launch_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups.tile = tile + launch_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups.lds_bytes = ( + LDS_TILES_BYTES, SCALE_LDS_BYTES2, STG_LDS_BYTES, BM + ) + return launch_gemm2_mxfp4_h6144_i1536_bm128_epilogue_groups diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_epilogue_groups.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_epilogue_groups.py new file mode 100644 index 0000000..46df811 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_epilogue_groups.py @@ -0,0 +1,776 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm2_a4w4_bm256_half_rows`` with SMEM row probes and sorted ids in LDS at every chunk count, in which a wave +skips the epilogue of each of its 16-row groups that is all padding; every load, wait, MFMA and CTA barrier stays +unconditional and in order. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import Vector as Vec + +from .i3072.gemm1_prefill_h6144_i3072_persistent import Mfma16x16x128Fp4 + +from .common.epilogue import ( + _NUM_XCDS, + _STORE_CPOL, + _bf16x2, + _lds_load_vec, + _lds_store_vec, + _v2i32, +) +from .a16w4_utils import READY_STRIDE, poll_ready, wait_ready_from +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOT_BYTES, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _asm_void, + _Buf, + _divmod_nonneg, + _flat_frag, + _g2s_thunks, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + BLOCK_K, +) +from amd_kernels.warm_up import launcher + + +# a scale block is read one step ahead and drained at the next barrier before gather(step + 3) reuses its slot: three +# slots suffice, which leaves LDS room for the m-tile's ids +_SCALE_SLOTS2 = 3 + + +@launcher +def compile_moe_gemm2_mxfp4_h6144_i1536( + *, + E: int, + topk: int, + n_split: int = 6, + sort_block_m: int = 256, + wait_for: int = 0, +): + """The launcher; ``launch.tile`` is the workgroup body of the one-launch GEMM1 + GEMM2 kernel. + + Args: + wait_for: > 0: before loading the intermediate, the body waits until its m-block's ready count reaches it. + """ + H, I = 6144, 1536 # noqa: E741 + BM = 256 + BN = 256 + K = I + K_BYTES = K // 2 # A row bytes (packed e2m1) + KW_BYTES = K // 2 # W2 row bytes + BLOCK_K_BYTES = BLOCK_K + EID_SHIFT = (sort_block_m // BM).bit_length() - 1 # 0 or 1 + assert sort_block_m in (BM, 2 * BM) + N_TILES_ALL = H // BN + assert H % BN == 0 and N_TILES_ALL % n_split == 0 + NT = N_TILES_ALL // n_split # n-tiles per CTA + K_ITERS = K_BYTES // BLOCK_K + assert K_ITERS == 6 and K % 256 == 0 + FLAT = K_ITERS # K-steps per iteration: one n-tile, an even count that brings the A / B buffers back + SCALE_BLOCKS = K_ITERS # one 256-B scale block (8 K-groups = 256 K) per step + LDS_BLOCK_M = BM // 2 + LDS_BLOCK_N = BN // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 4 + N_TILES_B = LDS_BLOCK_N // 2 // 16 # 4 + N_ACCUMS = N_TILES_A * N_TILES_B + NB = N_TILES_B + NA = N_TILES_A + OUT_ROW_BYTES = H * 2 + W2_BYTES = H * KW_BYTES + assert W2_BYTES <= 0xFFFFFFFF + B_TILE_BYTES = BN * KW_BYTES # one n-tile of W2 (256 rows) + + B_STAGES = 2 + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES # 16 KB + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES # 16 KB + A_BUFS = 4 * a_lds_size # 64 KB: 2 stages x 2 halves + LDS_TILES_BYTES = A_BUFS + 2 * B_STAGES * b_lds_size # 128 KB: 64 A + 64 W2 + SCALE_LDS_BYTES2 = _SCALE_SLOTS2 * _SCALE_SLOT_BYTES # 12 KB + # epilogue staging, wave-private: 16 rows x (2 halves x 64 cols) bf16, 256 B rows + STG_ROW = 2 * N_TILES_B * 16 * 2 # 256 B: the wave's two 128-B line segments + STG_CH = STG_ROW // 16 # 16 chunks of 16 B per staged row + STG_ROWS = 16 # one (row half, a-tile) at a time + STG_WAVE = STG_ROWS * STG_ROW # 4 KB + STG_LDS_BYTES = _N_WAVES * STG_WAVE # 16 KB + N_FLUSH = (STG_ROWS * 2) // 8 # dwordx4 stores per round per lane: 32 lines, 8 each + assert LDS_TILES_BYTES + SCALE_LDS_BYTES2 + STG_LDS_BYTES + BM * 4 <= 160 * 1024 + + A_WAVE_GROUPS = N_TILES_A // 2 # 2 + A_HALF_GROUPS = LDS_BLOCK_M // 32 # 4 + B_WAVE_GROUPS = 2 # 32-row groups per wave (64 W2 rows) + B_HALF_GROUPS = LDS_BLOCK_N // 32 # 4: half 1 = 128 rows further + SC_COLS = K // 32 + W2_SC_BYTES = E * H * SC_COLS + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_LDS_BYTES2, 16] + stage_lds: fx.Array[fx.Int8, STG_LDS_BYTES, 16] + sids: fx.Array[fx.Int32, BM, 16] + + @flyc.jit + def tile( + _base_ptr, + _scale_base_ptr, + _stg_ptr, + sid_slot, + bx, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + ready, + ): + """Workgroup ``bx``'s (m-tile, n-chunk) on the given LDS regions; ``ready``: the one-launch kernel's m-block + counters (``wait_for`` > 0).""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + # B: 2 stages x 2 halves + b_st = [ + [_Buf(_base_ptr, A_BUFS + (2 * st + hf) * b_lds_size) for hf in range(2)] + for st in range(B_STAGES) + ] + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + g4 = lane_id // 16 + r16 = lane_id % 16 + + # ---- work item: (m-tile, n-chunk); consecutive m-tiles share an XCD ---- + nv_rsrc = buffer_ops.create_buffer_resource( + num_valid_ids, max_size=False, num_records_bytes=4 + ) + num_valid = fx.Int32( + buffer_ops.buffer_load( + nv_rsrc, fx.Int32(0), vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + n_work = (num_valid // fx.Int32(BM)) * fx.Int32(n_split) + per_xcd = (n_work + fx.Int32(_NUM_XCDS - 1)) // fx.Int32(_NUM_XCDS) + intra, xcd = _divmod_nonneg(bx, _NUM_XCDS) + work = xcd * per_xcd + intra + block_valid = (intra < per_xcd) & (work < n_work) + work_safe = block_valid.select(work, fx.Int32(0)) + tile_i, chunk = _divmod_nonneg(work_safe, n_split) + m_base = tile_i * BM + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + expert = fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, + tile_i >> EID_SHIFT, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BM * 4) + ) + if const_expr(EID_SHIFT > 0): + # a 2*BM-row sort pads each expert to 2*BM rows: a tile led by the sentinel (tok == n_tokens) is empty + first_sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + block_valid = block_valid & ((first_sid & fx.Int32(0x00FFFFFF)) < n_tokens) + chunk_n0 = chunk * NT # first n-tile (global index) of this CTA + # no n-tile rotation: the physical tile is the logical one + nt_rot = fx.Int32(0) + + def _pn(nt): + """CTA-local logical n-tile (sweep order) -> physical n-tile of the chunk""" + x = nt + nt_rot + return (x >= fx.Int32(NT)).select(x - fx.Int32(NT), x) + + def _n_glob(nt): + """Global n-tile of ``nt``, clamped: the redundant loads after the last tile re-fetch the last one.""" + ntc = (nt >= fx.Int32(NT)).select(fx.Int32(NT - 1), nt) + return chunk_n0 + _pn(ntc) + + if block_valid: + if const_expr(wait_for > 0): + # thread 0's first poll overlaps the setup's loads + ready_addr = ready + fx.Int64(tile_i * READY_STRIDE) + got0 = poll_ready(ready_addr) + # the sort ends each expert's rows with the sentinel, so padding is a row suffix of the m-tile: one scalar + # load (lgkmcnt, not vmcnt) per 16-row group leader counts the valid rows in 16-row groups + m_active = fx.Int32(0) + for g in range_constexpr(BM // 16): + sid_g = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, + m_base + g * 16, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + m_active = m_active + ( + (sid_g & fx.Int32(0x00FFFFFF)) < n_tokens + ).select(fx.Int32(16), fx.Int32(0)) + + def _compute(half=False): + sw_rsrc = buffer_ops.create_buffer_resource( + sorted_weights, + max_size=False, + num_records_bytes=num_m_blocks * (BM * 4), + ) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=num_m_blocks * (BM * K_BYTES) + ) + b_rsrc = buffer_ops.create_buffer_resource( + W2, + max_size=False, + num_records_bytes=W2_BYTES, + base_byte_offset=fx.Int64(expert) * W2_BYTES, + ) + # padded sorted rows (tok == n_tokens) fall outside the resource: dropped + out_rows = n_tokens * fx.Int32(topk) + out_rsrc = buffer_ops.create_buffer_resource( + OUT, max_size=False, num_records_bytes=out_rows * OUT_ROW_BYTES + ) + + # ---- A rows (sorted, contiguous): row * K_BYTES + swizzled col ---- + def _a_offsets(half): + offs = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + grow = m_base + half * LDS_BLOCK_M + row + offs.append(grow * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + # padding rows (row_in_tile >= m_active) get an OOB offset: no memory access, zeros, still counted in + # vmcnt + a_oob = num_m_blocks * fx.Int32(BM * K_BYTES) # == a_rsrc records + + def _oob_wrap(offs, half): + out = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + row_in_tile = fx.Int32(half * LDS_BLOCK_M) + row + out.append((row_in_tile >= m_active).select(a_oob, offs[rnd])) + return out + + gl_off_a0 = _oob_wrap(_a_offsets(0), 0) + gl_off_a1 = _oob_wrap(_a_offsets(1), 1) + + # ---- W2: expert row n*256 + hf*128 + r, 16-row blocks of KW_BYTES*16 B (a8 preshuffle inside); + # the n-tile goes in soffset (B_TILE_BYTES per tile) ---- + e_row0 = expert * fx.Int32(H) + + def _b_offsets(hf): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + wrow = fx.Int32(hf * LDS_BLOCK_N + row) + off = (wrow // fx.Int32(16)) * fx.Int32(KW_BYTES * 16) + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + offs.append(off) + return offs + + gl_off_b0 = _b_offsets(0) + gl_off_b1 = _b_offsets(1) + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = BLOCK_K_BYTES * 16 + + mfma = Mfma16x16x128Fp4(N_TILES_A, N_TILES_B) + sb_index = lambda j, hf: (j // 2, j % 2) # noqa: E731 (32-row groups) + + scale_gather = ScaleGatherMoE( + A_scale, + W2_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + num_m_blocks * (BM * SC_COLS), + W2_SC_BYTES, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + + def _set_scale_tile(nt): + scale_gather.set_wave_base(m_base, e_row0 + _n_glob(nt) * fx.Int32(BN)) + + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(nt, g): + # runtime ring indices: constant ones made LLVM spill 48 registers (+20 us at 4097 rows) + return (fx.Int32(nt) * fx.Int32(SCALE_BLOCKS) + fx.Int32(g)) % fx.Int32( + _SCALE_SLOTS2 + ) + + a0_g2s = G2SLoaderAsm(a_rsrc, gl_off_a0, N_TILES_A, wave_id) + a1_g2s = G2SLoaderAsm(a_rsrc, gl_off_a1, N_TILES_A, wave_id) + b0_g2s = G2SLoaderAsm(b_rsrc, gl_off_b0, N_TILES_B, wave_id) + b1_g2s = G2SLoaderAsm(b_rsrc, gl_off_b1, N_TILES_B, wave_id) + for ld in (a0_g2s, a1_g2s, b0_g2s, b1_g2s): + ld.set_wave_base(_base_ptr) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + + # ---- output rows / weights of the 4 rows this lane writes ---- + def _orow(sid): + tok = sid & fx.Int32(0x00FFFFFF) # padded: tok == n_tokens -> OOB + slot = (sid >> 24) & fx.Int32(0xFF) + return tok * fx.Int32(topk) + slot + + # routing weights stay in VGPRs, the 32 output rows in LDS: those in VGPRs too would spill + wave_row0 = m_base + wave_i * (N_TILES_A * 16) + row_w = [] + for h in range_constexpr(2): + for ti in range_constexpr(N_TILES_A): + row = wave_row0 + fx.Int32(h * LDS_BLOCK_M + ti * 16) + r16 + row_w.append( + fx.Float32( + buffer_ops.buffer_load( + sw_rsrc, row, vec_width=1, dtype=fx.Float32 + ) + ) + ) + # the m-tile's sorted ids go to LDS before the prologue: a global load in each epilogue would wait + # (in-order vmcnt) behind every inline-asm A / W2 prefetch in flight. One thread writes each id. + sid_t = fx.thread_idx.x + sid_slot[sid_t] = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base + sid_t, vec_width=1, dtype=fx.Int32 + ) + ) + sid_lds = fx.Int32(fx.ptrtoint(sid_slot)) + stg_base = fx.Int32(fx.ptrtoint(_stg_ptr)) + wave_id * fx.Int32(STG_WAVE) + + def _stg_addr(row, chunk, half): + """Staging address of (row 0..15, 16-B chunk, 8-B half); chunks are XOR-swizzled by the row so the + tile-wise writes and the line-wise reads spread over the banks.""" + return ( + stg_base + + row * fx.Int32(STG_ROW) + + ((chunk ^ (row % STG_CH)) * 16 + half * 8) + ) + + if const_expr(wait_for > 0): + # until GEMM1 has stored all the m-tile's intermediate rows + wait_ready_from(ready_addr, wait_for, n_split, got0) + + # ---- prologue: steps -2 and -1 of the steady-state order (B and A two steps ahead) ---- + # g0 B(0) a0(0) a1(0) g1 B(1) a0(1) g2 a1(1); scale block b is read at step b - 1 + _set_scale_tile(fx.Int32(0)) + scale_gather.gather(0, _slot(0, 0)) + b_soff0 = _n_glob(fx.Int32(0)) * fx.Int32(B_TILE_BYTES) + b0_g2s.load(b_st[0][0], b_soff0 + fx.Int32(0 * B_K_STEP)) + b1_g2s.load(b_st[0][1], b_soff0 + fx.Int32(0 * B_K_STEP)) + a0_g2s.load(a_cur0, fx.Int32(0 * A_K_STEP)) + a1_g2s.load(a_cur1, fx.Int32(0 * A_K_STEP)) + scale_gather.gather(1, _slot(0, 1)) + b0_g2s.load(b_st[1][0], b_soff0 + fx.Int32(1 * B_K_STEP)) + b1_g2s.load(b_st[1][1], b_soff0 + fx.Int32(1 * B_K_STEP)) + a0_g2s.load(a_next0, fx.Int32(1 * A_K_STEP)) + scale_gather.gather(2, _slot(0, 2)) + a1_g2s.load(a_next1, fx.Int32(1 * A_K_STEP)) + + # g0, B(0) and a0(0) landed: a1(0), g1, B(1), a0(1), g2, a1(1) may fly + wait_barrier(NA + 1 + 2 * NB + NA + 1 + NA) + a0_frag = a_s2r.load(a_cur0) + b0_frag = b_s2r.load(b_st[0][0], preshuffled=True) + b1_frag = b_s2r.load(b_st[0][1], preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0, 0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0, 0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + + # Step kc issues B(kc+2) (b0, b1), then after the MID wait a0(kc+2), the scale gather and a1(kc+2), into + # stage (step % 2). The top wait lands a1(kc), the MID wait a0(kc+1). The counts take loads only: stores + # retire out of order with loads, so counting the epilogue's stores races. + def _top_vmcnt(kc): + return 2 * NB + 2 * NA + 1 # one gather per step + + def _mid_vmcnt(kc): + # exact: step kc-1's gather and a1(kc+1), then B(kc+2), are the loads younger than a0(kc+1); a + # larger count would release with a0(kc+1) in flight and c10f's s2r would read stale LDS + return 2 * NB + NA + 1 + + def _read_scale_thunks(nt, kc_next, holder): + # the next step's scales: block kc_next of this tile or block 0 of the next (Python ints only: an + # ``if`` here would trace to an scf.if and lose the binding) + s = _slot(nt + fx.Int32(kc_next // K_ITERS), kc_next % K_ITERS) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step(nt, kc, f, a0f, b0f, b1f_in, sc, accs, abufs): + """One K-step: constexpr ``kc`` of n-tile ``nt``; the parity of ``f``, the iteration's constexpr + flat step, picks the B stage.""" + k2 = f % 2 + ac0, ac1, an0, an1 = abufs + bst = b_st[f % B_STAGES] # consumed this step, refilled for step f + 2 + bnx = b_st[(f + 1) % B_STAGES] # step f + 1's halves, read this step + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + zero_acc = kc == 0 + + _a1 = [None] * NA + _a0n = [None] * NA + _b0n = [None] * NB + _b1n = [None] * NB + # A and B of step kc + 2 (B of tile nt + 1 from kc = K_ITERS - 2 on) + ka = (kc + 2) % K_ITERS + kb = (kc + B_STAGES) % K_ITERS + nt_b = nt + fx.Int32((kc + B_STAGES) // K_ITERS) + a_off = fx.Int32(ka * A_K_STEP) + b_off = _n_glob(nt_b) * fx.Int32(B_TILE_BYTES) + fx.Int32(kb * B_K_STEP) + + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(nt, kc + 1, _scn) + # Gather scales three steps ahead, crossing n-tiles as needed. + future_block = kc + 3 + _g_blk = future_block % SCALE_BLOCKS + _g_slot = _slot(nt + fx.Int32(future_block // SCALE_BLOCKS), _g_blk) + _sc_gather = [lambda: scale_gather.gather(_g_blk, _g_slot)] + + wait_barrier(_top_vmcnt(kc)) + il = ( + _riffle( + _g2s_thunks(b0_g2s, bst[0], b_off, NB), + ([] if half else _s2r_thunks(a_s2r, ac1, _a1, NA, False)), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, + b0f, + c00f, + saR0, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + # two G2S loaders are never riffled: each one's steps share m0 (s_mov, then s_add per step), so + # interleaving them corrupts the LDS destinations. a0's loads go with c10f. + il = _g2s_thunks(b1_g2s, bst[1], b_off, NB) + _rd_scn[2:] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a1f = _a1 + + wait_barrier(_mid_vmcnt(kc)) + # a0(kc+2) may overwrite ac0 here: its last reads retired before the top barrier, and the s2r reads + # below take an0. Concatenated, not riffled. + il = ( + _g2s_thunks(a0_g2s, ac0, a_off, NA) + + _s2r_thunks(a_s2r, an0, _a0n, NA, False) + + _sc_gather + ) + if const_expr(not half): + c10f = mfma.call( + a1f, + b0f, + c10f, + saR1, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + else: + for th in il: + th() + c10f = c00f + a0nf = _a0n + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, NA), + _s2r_thunks(b_s2r, bnx[0], _b0n, NB, True) + + _s2r_thunks(b_s2r, bnx[1], _b1n, NB, True), + ) + if const_expr(not half): + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + else: + for th in il: + th() + c11f = c01f + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) + return ( + a0nf, + _b0n, + _b1n, + sc_next, + (c00f, c01f, c10f, c11f), + (an0, an1, ac0, ac1), + ) + + def _stage(accs, h, ti): + """Routing-weighted BF16 of the 16 rows (half ``h``, a-tile ``ti``) x the wave's 128 columns into + its staging buffer, 8 B per lane per tile; one wave's LDS ops run in order, so the writes follow the + last flush's reads.""" + w = row_w[h * NA + ti] + for hf in range_constexpr(2): + cq = accs[h * 2 + hf] + for j in range_constexpr(NB): + v = Vec(cq[mfma.idx(ti, j)]) + d0 = _bf16x2(fx.Float32(v[0]) * w, fx.Float32(v[1]) * w) + d1 = _bf16x2(fx.Float32(v[2]) * w, fx.Float32(v[3]) * w) + # cols hf*128 + j*16 + 4*g4 of the wave's 128 -> chunk + chunk = fx.Int32(hf * 8 + j * 2) + g4 // 2 + _lds_store_vec(_v2i32(d0, d1), _stg_addr(r16, chunk, g4 % 2), 2) + + def _flush(nt, h, ti, sids): + """The staged 16 rows -> full 128-B output lines, 8 per store (non-temporal): a lane takes row + 4*(lane//16) + k, segment (lane//8)%2, 16 B at chunk lane%8. ``sids``: the group's 4 sorted ids.""" + col_wave = (_n_glob(nt) * fx.Int32(BN) + wave_j * (N_TILES_B * 16)) * 2 + hf_l = (lane_id // 8) % 2 + ch_l = lane_id % 8 + _asm_void([], "s_waitcnt lgkmcnt(0)", "") + # the group's 4 reads, then its 4 stores: one LDS drain per group, not one lgkmcnt(0) per store + datas = [] + for k in range_constexpr(N_FLUSH): + row = (lane_id // 16) * 4 + fx.Int32(k) + datas.append(_lds_load_vec(_stg_addr(row, hf_l * 8 + ch_l, 0), 4)) + for k in range_constexpr(N_FLUSH): + buffer_ops.buffer_store( + datas[k], + out_rsrc, + _orow(sids[k]) * fx.Int32(OUT_ROW_BYTES) + + col_wave + + hf_l * fx.Int32(LDS_BLOCK_N * 2) + + ch_l * fx.Int32(16), + offset_is_bytes=True, + cache_modifier=_STORE_CPOL, + ) + + @flyc.jit + def _epilogue(nt, accs): + # one group's ids at a time: all 32 live compete with the accumulators at the 512-register limit + sid_row = wave_i * (N_TILES_A * 16) + (lane_id // 16) * 4 + for h in range_constexpr(1 if half else 2): + for ti in range_constexpr(NA): + # wave-uniform branch: the staging is wave-private and the group holds no CTA barrier + if m_active > wave_i * (NA * 16) + h * LDS_BLOCK_M + ti * 16: + sids = Vec( + _lds_load_vec( + sid_lds + + (sid_row + fx.Int32(h * LDS_BLOCK_M + ti * 16)) * 4, + 4, + ) + ) + _stage(accs, h, ti) + _flush(nt, h, ti, sids) + + abufs0 = (a_cur0, a_cur1, a_next0, a_next1) + n_a = 2 * NA + n_b = 2 * NB + n_sc = 2 * (NA // 2) + 2 * (NB // 2) + _R = fx.as_ir_value + + def _flat_sc(sc): + return [_R(v) for part in sc for v in part] + + def _unflat_sc(flat): + sizes = (NA // 2, NA // 2, NB // 2, NB // 2) + out, o = [], 0 + for n in sizes: + out.append(list(flat[o : o + n])) + o += n + return tuple(out) + + init_state = ( + _flat_frag(a0_frag) + + _flat_frag(b0_frag) + + _flat_frag(b1_frag) + + _flat_sc(sc0) + ) + # FlyDSL rewrites range(init=...) to a loop carrying state. + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + nt0 = fx.Int32(nt_idx) + off = 0 + a0f = _unflat_frag(state[off : off + n_a], NA) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + abufs = abufs0 + for f in range_constexpr(FLAT): + nt = nt0 + fx.Int32(f // K_ITERS) + kc = f % K_ITERS + if const_expr(kc == 0): + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # the gathers run three steps ahead: blocks 3-5 of n-tile nt, then blocks 0-2 of nt + 1 + _set_scale_tile(nt) + if const_expr(kc == K_ITERS - 3): + _set_scale_tile(nt + fx.Int32(1)) + a0f, b0f, b1f, sc, accs, abufs = _one_step( + nt, kc, f, a0f, b0f, b1f, sc, accs, abufs + ) + if const_expr(kc == K_ITERS - 1): + _epilogue(nt, accs) + state = ( + yield _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + ) + # the redundant loads of the tile after the last one must land before the LDS is handed on + wait_barrier(0) + + if m_active <= BM // 2: + _compute(half=True) + else: + _compute(half=False) + + @flyc.kernel(name="kernel_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups") + def kernel_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.stage_lds.ptr, + lds.sids.ptr, + fx.block_idx.x, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups( + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid_size, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups.kernel_name = "kernel_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups" + launch_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups.tile = tile + launch_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups.lds_bytes = (LDS_TILES_BYTES, SCALE_LDS_BYTES2, STG_LDS_BYTES, BM) + return launch_gemm2_mxfp4_h6144_i1536_bm256_epilogue_groups diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_half_rows.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_half_rows.py new file mode 100644 index 0000000..f6ce4d6 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_bm256_half_rows.py @@ -0,0 +1,771 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MoE down projection (GEMM2) for H 6144, I 1536, MXFP4 x MXFP4, on 256-row m-tiles of a 256-row sort: +out[tok*topk + slot] = bf16((h[row] @ W2[e]^T) * sorted_weights[row]), run-to-run bitwise. One n-tile per main-loop +iteration: K 1536 is six 128-B steps, an even count, so the B stage and A buffers return to their start every n-tile +(``n_split`` 6 column chunks of four). The scale gathers run three steps ahead: steps 0-2 gather the n-tile's blocks +3-5, steps 3-5 the next n-tile's blocks 0-2. The m-tile's output rows sit in LDS as byte offsets; +ballot row probes and a short main loop serve 256-row tiles of at most 128 valid rows (the same loads, gathers and +waits without the upper half's LDS reads, MFMAs and epilogue). One writer per valid output element. + +Layouts (bytes): A [num_m_blocks*BM, I/2] e2m1 sorted rows; A_scale [pad32(num_m_blocks*BM), I/32] e8m0-shuffled; +W2 [E, H, I/2] aiter shuffle_weight (16, 16): half h of K-step t is MX block 8t + 4h + klane, whose e8m0 is byte +(row half, h) of cell pair t's dword in W2_scale [E*H, I/32] (aiter e8m0_shuffle); OUT [n_tokens*topk, H] bf16. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T, Vector as Vec + +from .i3072.gemm1_prefill_h6144_i3072_persistent import Mfma16x16x128Fp4 + +from .common.epilogue import ( + _NUM_XCDS, + _STORE_CPOL, + _bf16x2, + _lds_load_vec, + _lds_store_vec, + _v2i32, +) +from .a16w4_utils import READY_STRIDE, poll_ready, wait_ready_from +from .common.loaders import ( + _N_WAVES, + _SCALE_A_REGION, + _SCALE_B_REGION, + _SCALE_SLOT_BYTES, + G2SLoaderAsm, + S2RLoader128B, + ScaleGatherMoE, + ScaleLoaderLDS, + _Buf, + _divmod_nonneg, + _flat_frag, + _g2s_thunks, + _riffle, + _s2r_thunks, + _swizzled_col, + _unflat_frag, + wait_barrier, +) +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + BLOCK_K, +) +from amd_kernels.warm_up import launcher + + +# a scale block is read one step ahead and drained at the next barrier before gather(step + 3) reuses its slot: three +# slots suffice, which leaves LDS room for the m-tile's ids +_SCALE_SLOTS2 = 3 + + +@launcher +def compile_moe_gemm2_mxfp4_h6144_i1536( + *, + E: int, + topk: int, + n_split: int = 6, + sort_block_m: int = 256, + wait_for: int = 0, +): + """The launcher; ``launch.tile`` is the workgroup body of the one-launch GEMM1 + GEMM2 kernel. + + Args: + wait_for: > 0: before loading the intermediate, the body waits until its m-block's ready count reaches it. + """ + H, I = 6144, 1536 # noqa: E741 + BM = 256 + BN = 256 + K = I + K_BYTES = K // 2 # A row bytes (packed e2m1) + KW_BYTES = K // 2 # W2 row bytes + BLOCK_K_BYTES = BLOCK_K + EID_SHIFT = (sort_block_m // BM).bit_length() - 1 # 0 or 1 + assert sort_block_m in (BM, 2 * BM) + N_TILES_ALL = H // BN + assert H % BN == 0 and N_TILES_ALL % n_split == 0 + NT = N_TILES_ALL // n_split # n-tiles per CTA + K_ITERS = K_BYTES // BLOCK_K + assert K_ITERS == 6 and K % 256 == 0 + FLAT = K_ITERS # K-steps per iteration: one n-tile, an even count that brings the A / B buffers back + SCALE_BLOCKS = K_ITERS # one 256-B scale block (8 K-groups = 256 K) per step + LDS_BLOCK_M = BM // 2 + LDS_BLOCK_N = BN // 2 + N_TILES_A = LDS_BLOCK_M // 2 // 16 # 4 + N_TILES_B = LDS_BLOCK_N // 2 // 16 # 4 + N_ACCUMS = N_TILES_A * N_TILES_B + NB = N_TILES_B + NA = N_TILES_A + OUT_ROW_BYTES = H * 2 + W2_BYTES = H * KW_BYTES + assert W2_BYTES <= 0xFFFFFFFF + B_TILE_BYTES = BN * KW_BYTES # one n-tile of W2 (256 rows) + + B_STAGES = 2 + a_lds_size = LDS_BLOCK_M * BLOCK_K_BYTES # 16 KB + b_lds_size = LDS_BLOCK_N * BLOCK_K_BYTES # 16 KB + A_BUFS = 4 * a_lds_size # 64 KB: 2 stages x 2 halves + LDS_TILES_BYTES = A_BUFS + 2 * B_STAGES * b_lds_size # 128 KB: 64 A + 64 W2 + SCALE_LDS_BYTES2 = _SCALE_SLOTS2 * _SCALE_SLOT_BYTES # 12 KB + # epilogue staging, wave-private: 16 rows x (2 halves x 64 cols) bf16, 256 B rows + STG_ROW = 2 * N_TILES_B * 16 * 2 # 256 B: the wave's two 128-B line segments + STG_CH = STG_ROW // 16 # 16 chunks of 16 B per staged row + STG_ROWS = 16 # one (row half, a-tile) at a time + STG_WAVE = STG_ROWS * STG_ROW # 4 KB + STG_LDS_BYTES = _N_WAVES * STG_WAVE # 16 KB + N_FLUSH = (STG_ROWS * 2) // 8 # dwordx4 stores per round per lane: 32 lines, 8 each + assert LDS_TILES_BYTES + SCALE_LDS_BYTES2 + STG_LDS_BYTES + BM * 4 <= 160 * 1024 + + A_WAVE_GROUPS = N_TILES_A // 2 # 2 + A_HALF_GROUPS = LDS_BLOCK_M // 32 # 4 + B_WAVE_GROUPS = 2 # 32-row groups per wave (64 W2 rows) + B_HALF_GROUPS = LDS_BLOCK_N // 32 # 4: half 1 = 128 rows further + SC_COLS = K // 32 + W2_SC_BYTES = E * H * SC_COLS + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, LDS_TILES_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_LDS_BYTES2, 16] + stage_lds: fx.Array[fx.Int8, STG_LDS_BYTES, 16] + sids: fx.Array[fx.Int32, BM, 16] + + @flyc.jit + def tile( + _base_ptr, + _scale_base_ptr, + _stg_ptr, + sid_slot, + bx, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + ready, + ): + """Workgroup ``bx``'s (m-tile, n-chunk) on the given LDS regions; ``ready``: the one-launch kernel's m-block + counters (``wait_for`` > 0).""" + + a_cur0 = _Buf(_base_ptr, 0 * a_lds_size) + a_cur1 = _Buf(_base_ptr, 1 * a_lds_size) + a_next0 = _Buf(_base_ptr, 2 * a_lds_size) + a_next1 = _Buf(_base_ptr, 3 * a_lds_size) + # B: 2 stages x 2 halves + b_st = [ + [_Buf(_base_ptr, A_BUFS + (2 * st + hf) * b_lds_size) for hf in range(2)] + for st in range(B_STAGES) + ] + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + wave_i = wave_id // 2 + wave_j = wave_id % 2 + g4 = lane_id // 16 + r16 = lane_id % 16 + + # ---- work item: (m-tile, n-chunk); consecutive m-tiles share an XCD ---- + nv_rsrc = buffer_ops.create_buffer_resource( + num_valid_ids, max_size=False, num_records_bytes=4 + ) + num_valid = fx.Int32( + buffer_ops.buffer_load( + nv_rsrc, fx.Int32(0), vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + n_work = (num_valid // fx.Int32(BM)) * fx.Int32(n_split) + per_xcd = (n_work + fx.Int32(_NUM_XCDS - 1)) // fx.Int32(_NUM_XCDS) + intra, xcd = _divmod_nonneg(bx, _NUM_XCDS) + work = xcd * per_xcd + intra + block_valid = (intra < per_xcd) & (work < n_work) + work_safe = block_valid.select(work, fx.Int32(0)) + tile_i, chunk = _divmod_nonneg(work_safe, n_split) + m_base = tile_i * BM + eid_rsrc = buffer_ops.create_buffer_resource( + sorted_expert_ids, max_size=False, num_records_bytes=num_m_blocks * 4 + ) + expert = fx.Int32( + buffer_ops.buffer_load( + eid_rsrc, + tile_i >> EID_SHIFT, + vec_width=1, + dtype=fx.Int32, + is_scalar=True, + ) + ) + ids_rsrc = buffer_ops.create_buffer_resource( + sorted_ids, max_size=False, num_records_bytes=num_m_blocks * (BM * 4) + ) + if const_expr(EID_SHIFT > 0): + # a 2*BM-row sort pads each expert to 2*BM rows: a tile led by the sentinel (tok == n_tokens) is empty + first_sid = fx.Int32( + buffer_ops.buffer_load( + ids_rsrc, m_base, vec_width=1, dtype=fx.Int32, is_scalar=True + ) + ) + block_valid = block_valid & ((first_sid & fx.Int32(0x00FFFFFF)) < n_tokens) + chunk_n0 = chunk * NT # first n-tile (global index) of this CTA + # no n-tile rotation: the physical tile is the logical one + nt_rot = fx.Int32(0) + + def _pn(nt): + """CTA-local logical n-tile (sweep order) -> physical n-tile of the chunk""" + x = nt + nt_rot + return (x >= fx.Int32(NT)).select(x - fx.Int32(NT), x) + + def _n_glob(nt): + """Global n-tile of ``nt``, clamped: the redundant loads after the last tile re-fetch the last one.""" + ntc = (nt >= fx.Int32(NT)).select(fx.Int32(NT - 1), nt) + return chunk_n0 + _pn(ntc) + + if block_valid: + if const_expr(wait_for > 0): + # thread 0's first poll overlaps the setup's loads + ready_addr = ready + fx.Int64(tile_i * READY_STRIDE) + got0 = poll_ready(ready_addr) + # padding is a row suffix: lanes 0-15 probe the sixteen 16-row group leaders and the low 16 ballot bits + # count the valid prefix in 16-row groups, the same in every wave (a CTA-uniform main-loop choice) + probe_sid = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, m_base + (lane_id % 16) * 16, + vec_width=1, dtype=fx.Int32)) + probe_mask = fx.Int64(fx.rocdl.ballot( + T.i64, (probe_sid & 0x00FFFFFF) < n_tokens)) + m_active = fx.math.ctpop(probe_mask.to(fx.Int32) & 65535).to(fx.Int32) * 16 + + def _compute(half=False): + sw_rsrc = buffer_ops.create_buffer_resource( + sorted_weights, + max_size=False, + num_records_bytes=num_m_blocks * (BM * 4), + ) + a_rsrc = buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=num_m_blocks * (BM * K_BYTES) + ) + b_rsrc = buffer_ops.create_buffer_resource( + W2, + max_size=False, + num_records_bytes=W2_BYTES, + base_byte_offset=fx.Int64(expert) * W2_BYTES, + ) + # padded sorted rows (tok == n_tokens) fall outside the resource: dropped + out_rows = n_tokens * fx.Int32(topk) + out_rsrc = buffer_ops.create_buffer_resource( + OUT, max_size=False, num_records_bytes=out_rows * OUT_ROW_BYTES + ) + + # ---- A rows (sorted, contiguous): row * K_BYTES + swizzled col ---- + def _a_offsets(half): + offs = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + grow = m_base + half * LDS_BLOCK_M + row + offs.append(grow * fx.Int32(K_BYTES) + _swizzled_col(row, col)) + return offs + + # padding rows (row_in_tile >= m_active) get an OOB offset: no memory access, zeros, still counted in + # vmcnt + a_oob = num_m_blocks * fx.Int32(BM * K_BYTES) # == a_rsrc records + + def _oob_wrap(offs, half): + out = [] + for rnd in range_constexpr(N_TILES_A): + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + row_in_tile = fx.Int32(half * LDS_BLOCK_M) + row + out.append((row_in_tile >= m_active).select(a_oob, offs[rnd])) + return out + + gl_off_a0 = _oob_wrap(_a_offsets(0), 0) + gl_off_a1 = _oob_wrap(_a_offsets(1), 1) + + # ---- W2: expert row n*256 + hf*128 + r, 16-row blocks of KW_BYTES*16 B (a8 preshuffle inside); + # the n-tile goes in soffset (B_TILE_BYTES per tile) ---- + e_row0 = expert * fx.Int32(H) + + def _b_offsets(hf): + offs = [] + for rnd in range_constexpr(N_TILES_B): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + wrow = fx.Int32(hf * LDS_BLOCK_N + row) + off = (wrow // fx.Int32(16)) * fx.Int32(KW_BYTES * 16) + fx.Int32( + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + offs.append(off) + return offs + + gl_off_b0 = _b_offsets(0) + gl_off_b1 = _b_offsets(1) + A_K_STEP = BLOCK_K_BYTES + B_K_STEP = BLOCK_K_BYTES * 16 + + mfma = Mfma16x16x128Fp4(N_TILES_A, N_TILES_B) + sb_index = lambda j, hf: (j // 2, j % 2) # noqa: E731 (32-row groups) + + scale_gather = ScaleGatherMoE( + A_scale, + W2_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + num_m_blocks * (BM * SC_COLS), + W2_SC_BYTES, + A_WAVE_GROUPS, + A_HALF_GROUPS, + B_WAVE_GROUPS, + B_HALF_GROUPS, + ) + + def _set_scale_tile(nt): + scale_gather.set_wave_base(m_base, e_row0 + _n_glob(nt) * fx.Int32(BN)) + + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(nt, g): + # runtime ring indices: constant ones made LLVM spill 48 registers (+20 us at 4097 rows) + return (fx.Int32(nt) * fx.Int32(SCALE_BLOCKS) + fx.Int32(g)) % fx.Int32( + _SCALE_SLOTS2 + ) + + a0_g2s = G2SLoaderAsm(a_rsrc, gl_off_a0, N_TILES_A, wave_id) + a1_g2s = G2SLoaderAsm(a_rsrc, gl_off_a1, N_TILES_A, wave_id) + b0_g2s = G2SLoaderAsm(b_rsrc, gl_off_b0, N_TILES_B, wave_id) + b1_g2s = G2SLoaderAsm(b_rsrc, gl_off_b1, N_TILES_B, wave_id) + for ld in (a0_g2s, a1_g2s, b0_g2s, b1_g2s): + ld.set_wave_base(_base_ptr) + a_s2r = S2RLoader128B(wave_i, N_TILES_A) + b_s2r = S2RLoader128B(wave_j, N_TILES_B) + + # ---- output rows / weights of the 4 rows this lane writes ---- + def _orow(sid): + tok = sid & fx.Int32(0x00FFFFFF) # padded: tok == n_tokens -> OOB + slot = (sid >> 24) & fx.Int32(0xFF) + return tok * fx.Int32(topk) + slot + + # routing weights stay in VGPRs, the 32 output rows in LDS: those in VGPRs too would spill + wave_row0 = m_base + wave_i * (N_TILES_A * 16) + row_w = [] + for h in range_constexpr(2): + for ti in range_constexpr(N_TILES_A): + row = wave_row0 + fx.Int32(h * LDS_BLOCK_M + ti * 16) + r16 + row_w.append( + fx.Float32( + buffer_ops.buffer_load( + sw_rsrc, row, vec_width=1, dtype=fx.Float32 + ) + ) + ) + # the m-tile's output rows go to LDS before the prologue (LDS reads leave the A / W2 prefetches in + # flight), one thread per entry, as byte offsets decoded once (padding still points past out_rsrc). + sid_t = fx.thread_idx.x + sid_value = fx.Int32(buffer_ops.buffer_load( + ids_rsrc, m_base + sid_t, vec_width=1, dtype=fx.Int32)) + sid_slot[sid_t] = _orow(sid_value) * fx.Int32(OUT_ROW_BYTES) + sid_lds = fx.Int32(fx.ptrtoint(sid_slot)) + stg_base = fx.Int32(fx.ptrtoint(_stg_ptr)) + wave_id * fx.Int32(STG_WAVE) + + def _stg_addr(row, chunk, half): + """Staging address of (row 0..15, 16-B chunk, 8-B half); chunks are XOR-swizzled by the row so the + tile-wise writes and the line-wise reads spread over the banks.""" + return ( + stg_base + + row * fx.Int32(STG_ROW) + + ((chunk ^ (row % STG_CH)) * 16 + half * 8) + ) + + if const_expr(wait_for > 0): + # until GEMM1 has stored all the m-tile's intermediate rows + wait_ready_from(ready_addr, wait_for, n_split, got0) + + # ---- prologue: steps -2 and -1 of the steady-state order (B and A two steps ahead) ---- + # g0 B(0) a0(0) a1(0) g1 B(1) a0(1) g2 a1(1); scale block b is read at step b - 1 + _set_scale_tile(fx.Int32(0)) + scale_gather.gather(0, _slot(0, 0)) + b_soff0 = _n_glob(fx.Int32(0)) * fx.Int32(B_TILE_BYTES) + b0_g2s.load(b_st[0][0], b_soff0 + fx.Int32(0 * B_K_STEP)) + b1_g2s.load(b_st[0][1], b_soff0 + fx.Int32(0 * B_K_STEP)) + a0_g2s.load(a_cur0, fx.Int32(0 * A_K_STEP)) + a1_g2s.load(a_cur1, fx.Int32(0 * A_K_STEP)) + scale_gather.gather(1, _slot(0, 1)) + b0_g2s.load(b_st[1][0], b_soff0 + fx.Int32(1 * B_K_STEP)) + b1_g2s.load(b_st[1][1], b_soff0 + fx.Int32(1 * B_K_STEP)) + a0_g2s.load(a_next0, fx.Int32(1 * A_K_STEP)) + scale_gather.gather(2, _slot(0, 2)) + a1_g2s.load(a_next1, fx.Int32(1 * A_K_STEP)) + + # g0, B(0) and a0(0) landed: a1(0), g1, B(1), a0(1), g2, a1(1) may fly + wait_barrier(NA + 1 + 2 * NB + NA + 1 + NA) + a0_frag = a_s2r.load(a_cur0) + b0_frag = b_s2r.load(b_st[0][0], preshuffled=True) + b1_frag = b_s2r.load(b_st[0][1], preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0, 0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0, 0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + + # Step kc issues B(kc+2) (b0, b1), then after the MID wait a0(kc+2), the scale gather and a1(kc+2), into + # stage (step % 2). The top wait lands a1(kc), the MID wait a0(kc+1). The counts take loads only: stores + # retire out of order with loads, so counting the epilogue's stores races. + def _top_vmcnt(kc): + return 2 * NB + 2 * NA + 1 # one gather per step + + def _mid_vmcnt(kc): + # exact: step kc-1's gather and a1(kc+1), then B(kc+2), are the loads younger than a0(kc+1); a + # larger count would release with a0(kc+1) in flight and c10f's s2r would read stale LDS + return 2 * NB + NA + 1 + + def _read_scale_thunks(nt, kc_next, holder): + # the next step's scales: block kc_next of this tile or block 0 of the next (Python ints only: an + # ``if`` here would trace to an scf.if and lose the binding) + s = _slot(nt + fx.Int32(kc_next // K_ITERS), kc_next % K_ITERS) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _one_step(nt, kc, f, a0f, b0f, b1f_in, sc, accs, abufs): + """One K-step: constexpr ``kc`` of n-tile ``nt``; the parity of ``f``, the iteration's constexpr + flat step, picks the B stage.""" + k2 = f % 2 + ac0, ac1, an0, an1 = abufs + bst = b_st[f % B_STAGES] # consumed this step, refilled for step f + 2 + bnx = b_st[(f + 1) % B_STAGES] # step f + 1's halves, read this step + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + zero_acc = kc == 0 + + _a1 = [None] * NA + _a0n = [None] * NA + _b0n = [None] * NB + _b1n = [None] * NB + # A and B of step kc + 2 (B of tile nt + 1 from kc = K_ITERS - 2 on) + ka = (kc + 2) % K_ITERS + kb = (kc + B_STAGES) % K_ITERS + nt_b = nt + fx.Int32((kc + B_STAGES) // K_ITERS) + a_off = fx.Int32(ka * A_K_STEP) + b_off = _n_glob(nt_b) * fx.Int32(B_TILE_BYTES) + fx.Int32(kb * B_K_STEP) + + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(nt, kc + 1, _scn) + # Gather scales three steps ahead, crossing n-tiles as needed. + future_block = kc + 3 + _g_blk = future_block % SCALE_BLOCKS + _g_slot = _slot(nt + fx.Int32(future_block // SCALE_BLOCKS), _g_blk) + _sc_gather = [lambda: scale_gather.gather(_g_blk, _g_slot)] + + wait_barrier(_top_vmcnt(kc)) + il = ( + _riffle( + _g2s_thunks(b0_g2s, bst[0], b_off, NB), + ([] if half else _s2r_thunks(a_s2r, ac1, _a1, NA, False)), + ) + + _rd_scn[:2] + ) + c00f = mfma.call( + a0f, + b0f, + c00f, + saR0, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + # two G2S loaders are never riffled: each one's steps share m0 (s_mov, then s_add per step), so + # interleaving them corrupts the LDS destinations. a0's loads go with c10f. + il = _g2s_thunks(b1_g2s, bst[1], b_off, NB) + _rd_scn[2:] + c01f = mfma.call( + a0f, + b1f_in, + c01f, + saR0, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + a1f = _a1 + + wait_barrier(_mid_vmcnt(kc)) + # a0(kc+2) may overwrite ac0 here: its last reads retired before the top barrier, and the s2r reads + # below take an0. Concatenated, not riffled. + il = ( + _g2s_thunks(a0_g2s, ac0, a_off, NA) + + _s2r_thunks(a_s2r, an0, _a0n, NA, False) + + _sc_gather + ) + if const_expr(not half): + c10f = mfma.call( + a1f, + b0f, + c10f, + saR1, + sbC0, + 0, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + else: + for th in il: + th() + c10f = c00f + a0nf = _a0n + il = _riffle( + _g2s_thunks(a1_g2s, ac1, a_off, NA), + _s2r_thunks(b_s2r, bnx[0], _b0n, NB, True) + + _s2r_thunks(b_s2r, bnx[1], _b1n, NB, True), + ) + if const_expr(not half): + c11f = mfma.call( + a1f, + b1f_in, + c11f, + saR1, + sbC1, + 1, + k2, + interleave=il, + zero_acc=zero_acc, + sb_index=sb_index, + ) + else: + for th in il: + th() + c11f = c01f + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) + return ( + a0nf, + _b0n, + _b1n, + sc_next, + (c00f, c01f, c10f, c11f), + (an0, an1, ac0, ac1), + ) + + def _stage(accs, h, ti): + """Routing-weighted BF16 of the 16 rows (half ``h``, a-tile ``ti``) x the wave's 128 columns into + its staging buffer, 8 B per lane per tile; one wave's LDS ops run in order, so the writes follow the + last flush's reads.""" + w = row_w[h * NA + ti] + for hf in range_constexpr(2): + cq = accs[h * 2 + hf] + for j in range_constexpr(NB): + v = Vec(cq[mfma.idx(ti, j)]) + d0 = _bf16x2(fx.Float32(v[0]) * w, fx.Float32(v[1]) * w) + d1 = _bf16x2(fx.Float32(v[2]) * w, fx.Float32(v[3]) * w) + # cols hf*128 + j*16 + 4*g4 of the wave's 128 -> chunk + chunk = fx.Int32(hf * 8 + j * 2) + g4 // 2 + _lds_store_vec(_v2i32(d0, d1), _stg_addr(r16, chunk, g4 % 2), 2) + + def _flush(nt, h, ti, sids): + """The staged 16 rows -> full 128-B output lines, 8 per store (non-temporal): a lane takes row + 4*(lane//16) + k, segment (lane//8)%2, 16 B at chunk lane%8. ``sids``: the group's 4 LDS entries, + byte offsets.""" + col_wave = (_n_glob(nt) * fx.Int32(BN) + wave_j * (N_TILES_B * 16)) * 2 + hf_l = (lane_id // 8) % 2 + ch_l = lane_id % 8 + # no LDS drain: the staging is wave-private and one wave's LDS ops run in order; LLVM waits for the + # read results before the stores. The 4 reads go before the 4 stores. + datas = [] + for k in range_constexpr(N_FLUSH): + row = (lane_id // 16) * 4 + fx.Int32(k) + datas.append(_lds_load_vec(_stg_addr(row, hf_l * 8 + ch_l, 0), 4)) + for k in range_constexpr(N_FLUSH): + row_byte = fx.Int32(sids[k]) + buffer_ops.buffer_store( + datas[k], + out_rsrc, + row_byte + col_wave + + hf_l * fx.Int32(LDS_BLOCK_N * 2) + + ch_l * fx.Int32(16), + offset_is_bytes=True, + cache_modifier=_STORE_CPOL, + ) + + def _epilogue(nt, accs): + # one group's ids at a time: all 32 live compete with the accumulators at the 512-register limit + sid_row = wave_i * (N_TILES_A * 16) + (lane_id // 16) * 4 + for h in range_constexpr(1 if half else 2): + for ti in range_constexpr(NA): + sids = Vec( + _lds_load_vec( + sid_lds + + (sid_row + fx.Int32(h * LDS_BLOCK_M + ti * 16)) * 4, + 4, + ) + ) + _stage(accs, h, ti) + _flush(nt, h, ti, sids) + + abufs0 = (a_cur0, a_cur1, a_next0, a_next1) + n_a = 2 * NA + n_b = 2 * NB + n_sc = 2 * (NA // 2) + 2 * (NB // 2) + _R = fx.as_ir_value + + def _flat_sc(sc): + return [_R(v) for part in sc for v in part] + + def _unflat_sc(flat): + sizes = (NA // 2, NA // 2, NB // 2, NB // 2) + out, o = [], 0 + for n in sizes: + out.append(list(flat[o : o + n])) + o += n + return tuple(out) + + init_state = ( + _flat_frag(a0_frag) + + _flat_frag(b0_frag) + + _flat_frag(b1_frag) + + _flat_sc(sc0) + ) + # FlyDSL rewrites range(init=...) to a loop carrying state. + for nt_idx, state in range( # type: ignore[call-overload] + 0, NT, 1, init=init_state + ): + nt0 = fx.Int32(nt_idx) + off = 0 + a0f = _unflat_frag(state[off : off + n_a], NA) + off += n_a + b0f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + b1f = _unflat_frag(state[off : off + n_b], NB) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + abufs = abufs0 + for f in range_constexpr(FLAT): + nt = nt0 + fx.Int32(f // K_ITERS) + kc = f % K_ITERS + if const_expr(kc == 0): + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # the gathers run three steps ahead: blocks 3-5 of n-tile nt, then blocks 0-2 of nt + 1 + _set_scale_tile(nt) + if const_expr(kc == K_ITERS - 3): + _set_scale_tile(nt + fx.Int32(1)) + a0f, b0f, b1f, sc, accs, abufs = _one_step( + nt, kc, f, a0f, b0f, b1f, sc, accs, abufs + ) + if const_expr(kc == K_ITERS - 1): + _epilogue(nt, accs) + state = ( + yield _flat_frag(a0f) + + _flat_frag(b0f) + + _flat_frag(b1f) + + _flat_sc(sc) + ) + # the redundant loads of the tile after the last one must land before the LDS is handed on + wait_barrier(0) + + if m_active <= BM // 2: + _compute(half=True) + else: + _compute(half=False) + + @flyc.kernel(name="kernel_gemm2_mxfp4_h6144_i1536_bm256_output_offsets") + def kernel_gemm2_mxfp4_h6144_i1536_bm256_output_offsets( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + ): + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + tile( + lds.all_lds.ptr, + lds.scale_lds.ptr, + lds.stage_lds.ptr, + lds.sids.ptr, + fx.block_idx.x, + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + fx.Int64(0), + ) + + @flyc.jit + def launch_gemm2_mxfp4_h6144_i1536_bm256_output_offsets( + A: fx.Tensor, + W2: fx.Tensor, + OUT: fx.Tensor, + A_scale: fx.Tensor, + W2_scale: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_expert_ids: fx.Tensor, + sorted_weights: fx.Tensor, + num_valid_ids: fx.Tensor, + n_tokens: fx.Int32, + num_m_blocks: fx.Int32, + grid_size: fx.Int32, + stream: fx.Stream, + ): + kernel_gemm2_mxfp4_h6144_i1536_bm256_output_offsets( + A, + W2, + OUT, + A_scale, + W2_scale, + sorted_ids, + sorted_expert_ids, + sorted_weights, + num_valid_ids, + n_tokens, + num_m_blocks, + grid_size, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": "256,256", + }, + ).launch(grid=(grid_size, 1, 1), block=(256, 1, 1), stream=stream) + + launch_gemm2_mxfp4_h6144_i1536_bm256_output_offsets.kernel_name = "kernel_gemm2_mxfp4_h6144_i1536_bm256_output_offsets" + launch_gemm2_mxfp4_h6144_i1536_bm256_output_offsets.tile = tile + launch_gemm2_mxfp4_h6144_i1536_bm256_output_offsets.lds_bytes = (LDS_TILES_BYTES, SCALE_LDS_BYTES2, STG_LDS_BYTES, BM) + return launch_gemm2_mxfp4_h6144_i1536_bm256_output_offsets diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_id_rows.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_id_rows.py new file mode 100644 index 0000000..c17186e --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_id_rows.py @@ -0,0 +1,124 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm2_a4w4_small.py`` taking the block's expert and its 16 packed rows from ``gemm1_a4w4_id_rows`` (no sort +launch, no counters).""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .gemm1_a4w4_small import _mfma, _hold, _raw, _lds_ptr +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 128 + + +@functools.cache +def compile_gemm2(*, tn=256, bm=16, pf=2): + assert tn in (128, 256, 512) and bm == 16 + ni, mr = tn // 64, bm // 16 + + @flyc.jit + def routed_tile(base, rb, A, AS, W, WS, OUT, IDS, WT, EIDS, NV, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + mb, nb = rb // (H // tn), rb % (H // tn) + record = fx.Int32(EIDS[mb]) + blocks = fx.Int32(EIDS[rows // 16]) + fx.rocdl.sched_barrier(0) + if mb < blocks: + expert = record & 127 + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=rows * (I // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=rows * (I // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=H * I // 2, + base_byte_offset=fx.Int64(expert) * (H * I // 2)) + sw = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * H * (I // 32)) + out = buffer_ops.create_buffer_resource(OUT, max_size=False, num_records_bytes=m * 4 * H * 2) + ad = [((mb * bm + mi * 16 + l16) * (I // 2) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [(mb * bm + mi * 16 + l16) * (I // 32) // 4 for mi in range_constexpr(mr)] + wd = [((nb * (tn // 16) + wave * ni + j) * (I // 128) * 1024 + q16 * 256 + l16 * 16) // 4 for j in range_constexpr(ni)] + swd = [((expert * (H // 32) + nb * (tn // 32) + (wave * ni + j) // 2) * (I // 256)) * 64 + q16 * 16 + l16 + for j in range_constexpr(ni)] + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = [buffer_ops.buffer_load(wr, wd[j], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 1024, cache_modifier=2) for j in range_constexpr(ni)] + sb = [buffer_ops.buffer_load(sw, swd[j], vec_width=1, dtype=fx.Int32, soffset_bytes=(k // 2) * 256) for j in range_constexpr(ni)] + return aa, sa, bb, sb + acc = [[None] * ni for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(I // 128): + if fx.const_expr(k + pf < I // 128): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + acc[mi][j] = _mfma(aa[mi], bb[j], acc[mi][j], sa[mi], sb[j], (k % 2) * 2 + j % 2) + flat = _hold([acc[mi][j] for mi in range(mr) for j in range(ni)]) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + v = Vec(flat[mi * ni + j]) + col = wave * (tn // 4) + j * 16 + l16 + for ii in range_constexpr(4): + off = ((mi * 16 + q16 * 4 + ii) * tn + col) * 4 + llvm.StoreOp(_raw(v[ii]), _lds_ptr(base, off)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + for rr in range_constexpr(bm // 8): + row = rr * 8 + tx // 32 + packed = fx.Int32(IDS[mb * 16 + row]) + token, slot = packed & 0xFFFFFF, packed.shrui(fx.Int32(24)) + pair = (token < m).select(token * 4 + slot, fx.Int32(0)) + weight = fx.Float32(WT[pair]) + for c in range_constexpr(tn // 64): + col = tx % 32 * 2 + c * 64 + v = Vec(llvm.load(T.vec(2, T.f32), _lds_ptr(base, (row * tn + col) * 4), alignment=8)) + pk = (v * weight).to(fx.BFloat16).bitcast(fx.Int32)[0] + buffer_ops.buffer_store(pk, out, ((token * 4 + slot) * H + nb * tn + col) // 2, mask=token < m) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm2_shared_first(*, tn=256, bm=16, pf=2, sh_tn=256, sh_bm=16, view=False): + """``compile_gemm2`` whose first ``nsh`` workgroups run the shared expert's MXFP8 down + tiles (``shared_gemm2_a8w8.compile_gemm2(tn=sh_tn, bm=sh_bm)``, its own output); routed as alone.""" + from .shared_gemm2_a8w8 import compile_gemm2 as compile_shared_gemm2 + assert bm in (16, 32) and sh_bm in (16, 32, 64) + sh_name = ("" if sh_bm == 16 else f"_bm{sh_bm}") + ("_view" if view else "") + name = f"moe_gemm2_id_rows_a4w4_partials_h6144_i1536_bm{bm}_tn{tn}_pf{pf}_shared_a8w8{sh_name}_tn{sh_tn}_first" + sh = compile_shared_gemm2(tn=sh_tn, bm=sh_bm, pf=2, view=view) + sh_tile, routed_tile = sh, compile_gemm2(tn=tn, bm=bm, pf=pf) + lds = max(bm * tn * 4, sh.lds_bytes) + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Tensor, WT: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, OUT8, m) + if bid >= nsh: + routed_tile(base, bid - nsh, A, AS, W, WS, OUT, IDS, WT, EIDS, NV, m, rows) + + @flyc.jit + def launch_id_rows_gemm2_a4w4_small_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Tensor, WT: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, stream: fx.Stream): + kernel(A, AS, W, WS, OUT, IDS, WT, EIDS, NV, A8, AS8, W8, WS8, OUT8, m, rows, nsh).launch( + grid=(rows // bm * (H // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_id_rows_gemm2_a4w4_small_shared_first.kernel_name = name + return launch_id_rows_gemm2_a4w4_small_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_inline_sort.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_inline_sort.py new file mode 100644 index 0000000..54f16fc --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_inline_sort.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm2_a4w4_small.py`` whose blocks build their rows from ``topk_ids`` (``a16w4_utils.inline_sort_table``) +instead of the sort's; the first weight step is loaded before the table's barriers.""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .gemm1_a4w4_small import _mfma, _hold, _raw, _lds_ptr + +from .a16w4_utils import inline_sort_table +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 128 + + +@functools.cache +def compile_gemm2(*, tn=256, bm=16, pf=2): + assert tn in (128, 256, 512) and bm == 16 + ni, mr = tn // 64, bm // 16 + + @flyc.jit + def routed_tile(base, tab, rb, A, AS, W, WS, OUT, IDS, WT, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + mb, nb = rb // (H // tn), rb % (H // tn) + expert, owner, count, build_tab = inline_sort_table( + IDS, m, 4, mb, lane, tab, max_pairs=64, bm=16, multi_block=True) + if owner: + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=rows * (I // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=rows * (I // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=H * I // 2, + base_byte_offset=fx.Int64(expert) * (H * I // 2)) + sw = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * H * (I // 32)) + out = buffer_ops.create_buffer_resource(OUT, max_size=False, num_records_bytes=m * 4 * H * 2) + wd = [((nb * (tn // 16) + wave * ni + j) * (I // 128) * 1024 + q16 * 256 + l16 * 16) // 4 for j in range_constexpr(ni)] + swd = [((expert * (H // 32) + nb * (tn // 32) + (wave * ni + j) // 2) * (I // 256)) * 64 + q16 * 16 + l16 + for j in range_constexpr(ni)] + # the expert is known: the first weight step goes out before the table's two barriers, the activation + # addresses wait for it + bpre = [buffer_ops.buffer_load(wr, wd[j], vec_width=4, dtype=fx.Int32, + cache_modifier=2) for j in range_constexpr(ni)] + spre = [buffer_ops.buffer_load(sw, swd[j], vec_width=1, dtype=fx.Int32) + for j in range_constexpr(ni)] + build_tab() + ad = [((mb * bm + mi * 16 + l16) * (I // 2) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [(mb * bm + mi * 16 + l16) * (I // 32) // 4 for mi in range_constexpr(mr)] + + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = bpre if k == 0 else [buffer_ops.buffer_load(wr, wd[j], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 1024, cache_modifier=2) for j in range_constexpr(ni)] + sb = spre if k == 0 else [buffer_ops.buffer_load(sw, swd[j], vec_width=1, dtype=fx.Int32, soffset_bytes=(k // 2) * 256) for j in range_constexpr(ni)] + return aa, sa, bb, sb + acc = [[None] * ni for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(I // 128): + if fx.const_expr(k + pf < I // 128): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + acc[mi][j] = _mfma(aa[mi], bb[j], acc[mi][j], sa[mi], sb[j], (k % 2) * 2 + j % 2) + flat = _hold([acc[mi][j] for mi in range(mr) for j in range(ni)]) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + v = Vec(flat[mi * ni + j]) + col = wave * (tn // 4) + j * 16 + l16 + for ii in range_constexpr(4): + off = ((mi * 16 + q16 * 4 + ii) * tn + col) * 4 + llvm.StoreOp(_raw(v[ii]), _lds_ptr(base, off)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + for rr in range_constexpr(bm // 8): + row = rr * 8 + tx // 32 + packed = fx.Int32(tab[row]) + token, slot = packed & 0xFFFFFF, packed.shrui(fx.Int32(24)) + pair = token * 4 + slot + pair = (token < m).select(pair, fx.Int32(0)) + weight = fx.Float32(WT[pair]) + for c in range_constexpr(tn // 64): + col = tx % 32 * 2 + c * 64 + v = Vec(llvm.load(T.vec(2, T.f32), _lds_ptr(base, (row * tn + col) * 4), alignment=8)) + pk = (v * weight).to(fx.BFloat16).bitcast(fx.Int32)[0] + buffer_ops.buffer_store(pk, out, ((token * 4 + slot) * H + nb * tn + col) // 2, mask=token < m) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm2_shared_first(*, tn=256, bm=16, pf=2, sh_tn=256, view=False): + """``compile_gemm2`` whose first ``nsh`` workgroups run the shared expert's MXFP8 down + tiles (``shared_gemm2_a8w8.compile_gemm2(tn=sh_tn)``, its own output); routed as alone.""" + from .shared_gemm2_a8w8 import compile_gemm2 as compile_shared_gemm2 + assert bm == 16 + name = (f"moe_gemm2_inline_sort_a4w4_partials_h6144_i1536_bm{bm}_tn{tn}_pf{pf}_weight_prefetch_shared_a8w8_tn{sh_tn}_first" + + ("_view" if view else "")) + sh = compile_shared_gemm2(tn=sh_tn, bm=16, pf=2, view=view) + sh_tile, routed_tile = sh, compile_gemm2(tn=tn, bm=bm, pf=pf) + lds = max(bm * tn * 4, sh.lds_bytes) + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + tab: fx.Array[fx.Int32, 33] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Int64, WT: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, OUT8, m) + if bid >= nsh: + routed_tile(base, smem.tab.ptr, bid - nsh, A, AS, W, WS, OUT, IDS, WT, m, rows) + + @flyc.jit + def launch_gemm2_a4w4_inline_sort_weight_prefetch_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Int64, WT: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, stream: fx.Stream): + kernel(A, AS, W, WS, OUT, IDS, WT, A8, AS8, W8, WS8, OUT8, m, rows, nsh).launch( + grid=(rows // bm * (H // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_gemm2_a4w4_inline_sort_weight_prefetch_shared_first.kernel_name = name + return launch_gemm2_a4w4_inline_sort_weight_prefetch_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_small.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_small.py new file mode 100644 index 0000000..8cc54ed --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/gemm2_a4w4_small.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""16/32-row MXFP4 down GEMM with direct operands and one BF16 writer per slot.""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .gemm1_a4w4_small import _mfma, _hold, _raw, _lds_ptr +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 128 + + +@functools.cache +def compile_gemm2(*, tn=256, bm=16, pf=2): + assert tn in (128, 256, 512) and bm in (16, 32) + ni, mr = tn // 64, bm // 16 + + @flyc.jit + def routed_tile(base, rb, A, AS, W, WS, OUT, IDS, WT, EIDS, NV, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + mb, nb = rb // (H // tn), rb % (H // tn) + nr = fx.Int32(NV[0]) + if mb * bm < nr: + expert = fx.Int32(EIDS[mb]) + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=rows * (I // 2)) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=rows * (I // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=H * I // 2, + base_byte_offset=fx.Int64(expert) * (H * I // 2)) + sw = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=E * H * (I // 32)) + out = buffer_ops.create_buffer_resource(OUT, max_size=False, num_records_bytes=m * 4 * H * 2) + ad = [((mb * bm + mi * 16 + l16) * (I // 2) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [(mb * bm + mi * 16 + l16) * (I // 32) // 4 for mi in range_constexpr(mr)] + wd = [((nb * (tn // 16) + wave * ni + j) * (I // 128) * 1024 + q16 * 256 + l16 * 16) // 4 for j in range_constexpr(ni)] + swd = [((expert * (H // 32) + nb * (tn // 32) + (wave * ni + j) // 2) * (I // 256)) * 64 + q16 * 16 + l16 + for j in range_constexpr(ni)] + def load(k): + aa = [buffer_ops.buffer_load(ar, ad[mi], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 64) for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = [buffer_ops.buffer_load(wr, wd[j], vec_width=4, dtype=fx.Int32, soffset_bytes=k * 1024, cache_modifier=2) for j in range_constexpr(ni)] + sb = [buffer_ops.buffer_load(sw, swd[j], vec_width=1, dtype=fx.Int32, soffset_bytes=(k // 2) * 256) for j in range_constexpr(ni)] + return aa, sa, bb, sb + acc = [[None] * ni for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(I // 128): + if fx.const_expr(k + pf < I // 128): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + acc[mi][j] = _mfma(aa[mi], bb[j], acc[mi][j], sa[mi], sb[j], (k % 2) * 2 + j % 2) + flat = _hold([acc[mi][j] for mi in range(mr) for j in range(ni)]) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + v = Vec(flat[mi * ni + j]) + col = wave * (tn // 4) + j * 16 + l16 + for ii in range_constexpr(4): + off = ((mi * 16 + q16 * 4 + ii) * tn + col) * 4 + llvm.StoreOp(_raw(v[ii]), _lds_ptr(base, off)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + for rr in range_constexpr(bm // 8): + row = rr * 8 + tx // 32 + packed = fx.Int32(IDS[mb * bm + row]) + token, slot = packed & 0xFFFFFF, packed.shrui(fx.Int32(24)) + weight = fx.Float32(WT[mb * bm + row]) + for c in range_constexpr(tn // 64): + col = tx % 32 * 2 + c * 64 + v = Vec(llvm.load(T.vec(2, T.f32), _lds_ptr(base, (row * tn + col) * 4), alignment=8)) + pk = (v * weight).to(fx.BFloat16).bitcast(fx.Int32)[0] + buffer_ops.buffer_store(pk, out, ((token * 4 + slot) * H + nb * tn + col) // 2, mask=token < m) + return routed_tile + + +@functools.cache +@launcher +def compile_gemm2_shared_first(*, tn=256, bm=16, pf=2, sh_tn=256, sh_bm=16, view=False): + """``compile_gemm2`` whose first ``nsh`` workgroups run the shared expert's MXFP8 down + tiles (``shared_gemm2_a8w8.compile_gemm2(tn=sh_tn, bm=sh_bm)``, its own output); routed as alone. ``sh_bm`` 0: the + routed tiles only.""" + from .shared_gemm2_a8w8 import compile_gemm2 as compile_shared_gemm2 + assert bm in (16, 32) and sh_bm in (0, 16, 32, 64) + sh_name = ("" if sh_bm == 16 else f"_bm{sh_bm}") + ("_view" if view else "") + name = f"moe_gemm2_a4w4_partials_h6144_i1536_bm{bm}_tn{tn}_pf{pf}_shared_a8w8{sh_name}_tn{sh_tn}_first" + sh = compile_shared_gemm2(tn=sh_tn, bm=sh_bm, pf=2, view=view) if sh_bm else None + sh_tile, routed_tile = sh, compile_gemm2(tn=tn, bm=bm, pf=pf) + lds = max(bm * tn * 4, sh.lds_bytes if sh_bm else 0) + if sh_bm == 0: + name = f"moe_gemm2_a4w4_partials_h6144_i1536_bm{bm}_tn{tn}_pf{pf}" + + @fx.struct + class Shared: + raw: fx.Array[fx.Uint8, lds, 16] + + @flyc.kernel(name=name, known_block_size=[256, 1, 1]) + def kernel(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Tensor, WT: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32): + smem = fx.SharedAllocator().allocate(Shared).peek() + base = fx.Int32(fx.ptrtoint(smem.raw.ptr)) + bid = fx.Int32(fx.block_idx.x) + if fx.const_expr(sh_bm == 0): + routed_tile(base, bid, A, AS, W, WS, OUT, IDS, WT, EIDS, NV, m, rows) + else: + if bid < nsh: + sh_tile(base, bid, A8, AS8, W8, WS8, OUT8, m) + if bid >= nsh: + routed_tile(base, bid - nsh, A, AS, W, WS, OUT, IDS, WT, EIDS, NV, m, rows) + + @flyc.jit + def launch_gemm2_a4w4_small_shared_first(A: fx.Tensor, AS: fx.Tensor, W: fx.Tensor, WS: fx.Tensor, + OUT: fx.Tensor, IDS: fx.Tensor, WT: fx.Tensor, EIDS: fx.Tensor, NV: fx.Tensor, + A8: fx.Tensor, AS8: fx.Tensor, W8: fx.Tensor, WS8: fx.Tensor, OUT8: fx.Tensor, + m: fx.Int32, rows: fx.Int32, nsh: fx.Int32, stream: fx.Stream): + kernel(A, AS, W, WS, OUT, IDS, WT, EIDS, NV, A8, AS8, W8, WS8, OUT8, m, rows, nsh).launch( + grid=(rows // bm * (H // tn) + nsh, 1, 1), block=(256, 1, 1), stream=stream) + launch_gemm2_a4w4_small_shared_first.kernel_name = name + return launch_gemm2_a4w4_small_shared_first diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/__init__.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/__init__.py new file mode 100644 index 0000000..74d432e --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``gemm1_prefill_h6144_i3072_persistent`` holds helpers the I 1536 kernels import (no kernel of its own): the +row-stationary FP4 MFMA class, the e2m1 / e8m0 converters, the SwiGLU-OAI epilogue, the per-XCD claim helpers and +``interleave_w13_scales``.""" diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/gemm1_prefill_h6144_i3072_persistent.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/gemm1_prefill_h6144_i3072_persistent.py new file mode 100644 index 0000000..897d18c --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/i3072/gemm1_prefill_h6144_i3072_persistent.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Helpers the I 1536 GEMM1s and the router / quant kernels import (gfx950): the row-stationary FP4 MFMA class, the +e2m1 / e8m0 converters, the SwiGLU-OAI epilogue, the per-XCD claim helpers and ``interleave_w13_scales``.""" + +import flydsl.expr as fx +import torch +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import arith as _arith +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from ..common.epilogue import _maxf_nn +from ..common.loaders import _as_f32, _bits, _permlane16_swap +from ..common.tile_map import CLAIM_COUNTERS +from ..mxfp8.gemm1_prefill_h6144_i3072 import SWIGLU_ALPHA, SWIGLU_LIMIT, Mfma16x16x128Fp8, _fmed3 +from ..common.mfma_order import row_stationary_order + + +class Mfma16x16x128Fp4(Mfma16x16x128Fp8): + """e2m1 x e2m1 16x16x128 scaled MFMAs (``cbsz:4 blgp:4``, 4-VGPR operands): two per tile pair and 128-B step, on + half h of the fragments (``a[i][h]``) with scale byte (row half, h); every pair's half 0 goes first, then half 1, so + a pair's two dependent accumulates are a whole quadrant of MFMAs apart.""" + + def call( + self, + a, + b, + c, + sa, + sb, + gu, + k2, + interleave=None, + zero_acc=False, + late=None, + late_start=4, + sb_index=None, + ): + if sb_index is None: + sb_index = lambda j, gu: (j, gu) # noqa: E731 + thunks = list(interleave) if interleave else [] + assert not late + order = self._order() + seq = [(h, i, j) for h in range(2) for (i, j) in order] + n_mfma = len(seq) + by_slot: dict[int, list] = {} + for t, th in enumerate(thunks): + by_slot.setdefault((t * n_mfma) // len(thunks), []).append(th) + for m, (h, i, j) in enumerate(seq): + acc = None if (zero_acc and h == 0) else c[self.idx(i, j)] + sbj, sbb = sb_index(j, gu) + c[self.idx(i, j)] = self._mfma4( + a[i][h], b[j][h], acc, sa[i // 2], sb[sbj], i % 2, sbb, h + ) + for th in by_slot.get(m, ()): + th() + for m in range(n_mfma, n_mfma + len(thunks) + 1): + for th in by_slot.get(m, ()): + th() + return c + + def _mfma4(self, a_op, b_op, acc, sa_v, sb_v, ia, jb, h): + a_op, b_op = b_op, a_op + sa_v, sb_v = sb_v, sa_v + ia, jb = jb, ia + src2 = "$0" if acc is not None else "0" + asm = ( + f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {src2}, $3, $4 " + f"op_sel:[{ia},{jb},0] op_sel_hi:[{h},{h},0] cbsz:4 blgp:4" + ) + ops = [ + fx.as_ir_value(a_op), + fx.as_ir_value(b_op), + fx.as_ir_value(sa_v), + fx.as_ir_value(sb_v), + ] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(fx.as_ir_value(acc)) + cons += ",0" + return _llvm.inline_asm(self.res_ty, ops, asm, cons, has_side_effects=True) + + +class MfmaRowStationaryFp4(Mfma16x16x128Fp4): + """``Mfma16x16x128Fp4`` issuing each half's quadrant row by row.""" + + def _order(self): + return row_stationary_order(self.n_tiles_a, self.n_tiles_b) + + +def _cvt_pk_fp4(old, a, b, scale_f32, sel): + """v_cvt_scalef32_pk_fp4_f32: two f32 / scale -> 2 e2m1 into byte ``sel``.""" + return fx.Int32( + _llvm.call_intrinsic( + _T.i32, + "llvm.amdgcn.cvt.scalef32.pk.fp4.f32", + [ + fx.as_ir_value(old), + fx.as_ir_value(a), + fx.as_ir_value(b), + fx.as_ir_value(scale_f32), + fx.as_ir_value(fx.Int32(sel)), + ], + [], + [], + ) + ) + + +def _fmin_nn(a, b): + """v_min_f32 with nnan: the select-based clamp lowers to minnum on a canonicalized input (one more ``v_max x, x`` + per value); an MFMA accumulator holds no signalling NaN, so the value is the same.""" + fm = _ir.Attribute.parse("#arith.fastmath") + return fx.Float32( + _arith.MinNumFOp(fx.as_ir_value(a), fx.as_ir_value(b), fastmath=fm).result + ) + + +def _swiglu_oai_x4(gs, us): + """``reference.swiglu_oai`` on four values: g clamped above (``_fmin_nn``), u both sides, t = (g * alpha) * + (-log2 e) (one folded multiply changes the last bit), sigmoid = rcp(1 + exp2(t)), g * sig * (u + 1); the multiplies + and adds as v_pk_mul_f32 / v_pk_add_f32 on two pairs, the two pairs interleaved in program order so a v_exp / v_rcp + sits next to the other pair's work, not right after the packed op that produces its input (gfx950 puts a wait state + between them).""" + lim = fx.Float32(SWIGLU_LIMIT) + nlim = fx.Float32(-SWIGLU_LIMIT) + g = [ + Vec.from_elements( + [_fmin_nn(gs[2 * q], lim), _fmin_nn(gs[2 * q + 1], lim)], fx.Float32 + ) + for q in range(2) + ] + u = [ + Vec.from_elements( + [_fmed3(us[2 * q], nlim, lim), _fmed3(us[2 * q + 1], nlim, lim)], fx.Float32 + ) + for q in range(2) + ] + t = [g[q] * fx.Float32(SWIGLU_ALPHA) for q in range(2)] + t = [t[q] * fx.Float32(-1.4426950408889634) for q in range(2)] + e = [ + fx.Float32(_rocdl.exp2(_T.f32, fx.as_ir_value(fx.Float32(t[q][k])))) + for k in range(2) + for q in range(2) + ] # order e00 e10 e01 e11 + d = [ + Vec.from_elements([e[q], e[2 + q]], fx.Float32) + fx.Float32(1.0) + for q in range(2) + ] + r = [ + fx.Float32(_rocdl.rcp(_T.f32, fx.as_ir_value(fx.Float32(d[q][k])))) + for k in range(2) + for q in range(2) + ] + u1 = [u[q] + fx.Float32(1.0) for q in range(2)] + gs_ = [g[q] * Vec.from_elements([r[q], r[2 + q]], fx.Float32) for q in range(2)] + h = [gs_[q] * u1[q] for q in range(2)] + return [fx.Float32(h[q][k]) for q in range(2) for k in range(2)] + + +def _permlane32_swap(d_a, d_b): + pair_ty = _ir.Type.parse("!llvm.struct<(i32, i32)>") + res = _rocdl.permlane32_swap( + pair_ty, fx.as_ir_value(d_a), fx.as_ir_value(d_b), False, False + ) + return fx.Int32(_llvm.extractvalue(_T.i32, res, [0])), fx.Int32( + _llvm.extractvalue(_T.i32, res, [1]) + ) + + +def _e8m0_even_e2m1(amax): + """Quark FP4 even scale of a nonnegative f32 amax, exponent byte >= 1: exponent(amax + half an e2m1 mantissa ulp, + 0x200000) - 2, the subtraction folded into the bias (a signed shift for tiny inputs).""" + t = (_bits(amax) - fx.Int32(0xE00000)) >> 23 + return (t > fx.Int32(1)).select(t, fx.Int32(1)) + + +def _e8m0_pair(amax_a, amax_b): + """The e8m0 of two row tiles (ti 2 tp, 2 tp + 1) from their per-lane amaxes: a permlane16 swap of a with b puts rows + (0, 1) / (2, 3) of each side by side, a permlane32 swap of that with a copy gives a in rows 0 / 2 and b in + rows 1 / 3, one e8m0 per lane covers both tiles, and a permlane16 swap of it with a copy broadcasts each tile's + to all rows. Per tile the same max over the same 4 lanes (l, l ^ 16, l ^ 32, l ^ 48) as two shuffle_xor steps + (max is exact and order-free).""" + x, y = _permlane16_swap(_bits(amax_a), _bits(amax_b)) + m = _maxf_nn(_as_f32(x), _as_f32(y)) + x, y = _permlane32_swap(_bits(m), _bits(m)) + e = _e8m0_even_e2m1(_maxf_nn(_as_f32(x), _as_f32(y))) + return _permlane16_swap(e, e) + + +N_XCDS = 8 +# the per-XCD claim counters: the tile map table's last entries +assert CLAIM_COUNTERS >= N_XCDS + + +def _cu_count() -> int: + import torch + + return torch.cuda.get_device_properties( + torch.accelerator.current_device_index() + ).multi_processor_count + + +def _buffer_atomic_add(rsrc, voffset_bytes, value): + """``old = rsrc[voffset]; rsrc[voffset] += value`` (i32, per lane; lanes with an out-of-range offset do nothing and + return 0).""" + return fx.Int32( + _llvm.call_intrinsic( + _T.i32, + "llvm.amdgcn.raw.ptr.buffer.atomic.add.i32", + [ + fx.as_ir_value(fx.Int32(value)), + fx.as_ir_value(rsrc), + fx.as_ir_value(fx.Int32(voffset_bytes)), + fx.as_ir_value(fx.Int32(0)), + fx.as_ir_value(fx.Int32(0)), + ], + [], + [], + ) + ) + + +def interleave_w13_scales(w13_scale_shuffled: torch.Tensor) -> torch.Tensor: + """Aiter's e8m0_shuffle'd W13 scales [E, 2I, H/32] (gate rows, then up rows) -> the gate/up-interleaved shuffle + GEMM1 reads (the MXFP8 chain's layout).""" + from aiter.ops.shuffle import shuffle_scale + + e, n, k32 = w13_scale_shuffled.shape + t = w13_scale_shuffled.reshape(e * n // 32, k32 // 8, 4, 16, 2, 2) + raw = t.permute(0, 5, 3, 1, 4, 2).reshape(e, n, k32) # row-major again + return shuffle_scale( + raw.reshape(e * n, k32), e, is_guinterleave=True, gate_up=True + ).view(torch.uint8) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/layout.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/layout.py new file mode 100644 index 0000000..193104e --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/layout.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The routed experts' MXFP4 values in the layout the W4A4 kernels read: a pure permutation of the checkpoint's bytes, +within a 16-row block aiter's ``shuffle_weight`` (16, 16). The scales go through +``prefill_h6144_i1536.Mxfp4PrefillScales.from_row_major``. + + W13 [E, I/16, 2 (gate, up), K/128, klane 4, nlane 16, 16 B] K = H + W2 [E, H/16, I/128, klane 4, nlane 16, 16 B] +""" + +import torch + +TILE_K = 128 # K per 1 KB block of 16 columns + + +def _shuffle_values(packed: torch.Tensor) -> torch.Tensor: + """[E, N, K/2] u8 -> [E, N/16, K/128, klane 4, nlane 16, 16 B].""" + e, n, half_k = packed.shape + k = half_k * 2 + v = packed.view(e, n // 16, 16, k // TILE_K, 4, 16) # e nb nl kt kl j + return v.permute(0, 1, 3, 4, 2, 5).contiguous() + + +def w13_layout(packed: torch.Tensor) -> torch.Tensor: + """Gate / up e2m1 codes ``[E, 2I, H/2]`` (gate rows first) -> W13 (module docstring): per gate / up half, then the + halves interleaved per 16 rows.""" + e, n2, half_k = packed.shape + inter = n2 // 2 + assert n2 % 32 == 0 and (half_k * 2) % 256 == 0, packed.shape + w = _shuffle_values(packed.reshape(e * 2, inter, half_k)) + return w.view(e, 2, inter // 16, *w.shape[2:]).transpose(1, 2).contiguous() + + +def w2_layout(packed: torch.Tensor) -> torch.Tensor: + """Down e2m1 codes ``[E, H, I/2]`` -> W2 (module docstring).""" + e, h, half_k = packed.shape + assert h % 16 == 0 and (half_k * 2) % 256 == 0, packed.shape + return _shuffle_values(packed) + + +def router_line_layout(weight: torch.Tensor) -> torch.Tensor: + """FP32 [128,6144] -> [192,8,2,4,16,4] for contiguous router loads. + + A wave's 64 lanes read 16 bytes each: eight complete 128-byte lines. + This pure permutation runs once at weight loading, before graph capture. + """ + assert weight.shape == (128, 6144) and weight.dtype == torch.float32 + return weight.reshape(8, 16, 192, 4, 2, 4).permute(2, 0, 4, 3, 1, 5).contiguous() diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp4_from_mxfp8.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp4_from_mxfp8.py new file mode 100644 index 0000000..f23a56b --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp4_from_mxfp8.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP8 -> MXFP4 requant of the all-gathered MoE input under sequence parallelism: e4m3 codes [M, K] + e8m0 [M, K/32] +-> ``quant.mxfp4_quant``'s layouts, equal to ``mxfp4_quant`` of the dequantized input in bf16 (an e4m3 value times a +power of two is exact in bf16). One lane per 32-value block: two 16-B code loads and the scale byte, an exact decode to +f32 (``v_cvt_scalef32_pk_f32_fp8``), then ``quant``'s |max|, Quark even e8m0 and RNE e2m1 (gfx950 has no fp8 -> fp4 +conversion). +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .common.launch import _run_compiled +from .common.loaders import _as_f32 +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + _fmax3_abs, +) + +from .i3072.gemm1_prefill_h6144_i3072_persistent import _cvt_pk_fp4, _e8m0_even_e2m1 +from amd_kernels.warm_up import launcher + +_THREADS = 256 + + +def _e4m3x2(word, scale_f, hi): + """Bytes 0, 1 (``hi``: 2, 3) of ``word`` as f32 times ``scale_f``.""" + pk = _rocdl_ops.cvt_scalef32_pk_f32_fp8( + T.vec(2, T.f32), fx.as_ir_value(word), fx.as_ir_value(scale_f), hi + ) + v = Vec(pk) + return fx.Float32(v[0]), fx.Float32(v[1]) + + +@functools.cache +@launcher +def compile_mxfp4_from_mxfp8(): + @flyc.kernel(name="mxfp4_from_mxfp8_even") + def kernel_mxfp4_from_mxfp8( + C: fx.Tensor, CS: fx.Tensor, Q: fx.Tensor, S: fx.Tensor, n_blocks: fx.Int32 + ): + blk = fx.block_idx.x * _THREADS + fx.thread_idx.x + c_rsrc = buffer_ops.create_buffer_resource( + C, max_size=False, num_records_bytes=n_blocks * 32 + ) + cs_rsrc = buffer_ops.create_buffer_resource( + CS, max_size=False, num_records_bytes=n_blocks + ) + q_rsrc = buffer_ops.create_buffer_resource( + Q, max_size=False, num_records_bytes=n_blocks * 16 + ) + s_rsrc = buffer_ops.create_buffer_resource( + S, max_size=False, num_records_bytes=n_blocks + ) + # the block's scale byte, from the dword four neighbouring lanes share + sw = fx.Int32( + buffer_ops.buffer_load(cs_rsrc, blk >> 2, vec_width=1, dtype=fx.Int32) + ) + s8 = (sw >> ((blk & fx.Int32(3)) * fx.Int32(8))) & fx.Int32(0xFF) + scale_in = _as_f32(s8 << 23) + vals = [] + for j in range_constexpr(2): + v = Vec( + buffer_ops.buffer_load( + c_rsrc, blk * 8 + j * 4, vec_width=4, dtype=fx.Int32 + ) + ) + for k in range_constexpr(4): + w = fx.Int32(v[k]) + for hi in (False, True): + vals.extend(_e4m3x2(w, scale_in, hi)) + amax = _fmax3_abs(vals[0], vals[1], vals[2]) + for i in range_constexpr(3, 31, 2): + amax = _fmax3_abs(vals[i], vals[i + 1], amax) + amax = _fmax3_abs(vals[31], vals[31], amax) + e8m0 = _e8m0_even_e2m1(amax) + scale_f = _as_f32(e8m0 << 23) + out = [] + for d in range_constexpr(4): + p = fx.Int32(0) + for k in range_constexpr(4): + p = _cvt_pk_fp4( + p, vals[8 * d + 2 * k], vals[8 * d + 2 * k + 1], scale_f, k + ) + out.append(p) + buffer_ops.buffer_store(Vec.from_elements(out, fx.Int32), q_rsrc, blk * 4) + buffer_ops.buffer_store(fx.Int32(e8m0).to(fx.Int8), s_rsrc, blk) + + @flyc.jit + def launch_mxfp4_from_mxfp8( + C: fx.Tensor, + CS: fx.Tensor, + Q: fx.Tensor, + S: fx.Tensor, + n_blocks: fx.Int32, + grid: fx.Int32, + stream: fx.Stream, + ): + kernel_mxfp4_from_mxfp8(C, CS, Q, S, n_blocks).launch( + grid=(grid, 1, 1), block=(_THREADS, 1, 1), stream=stream + ) + + return launch_mxfp4_from_mxfp8 + + +def mxfp4_from_mxfp8( + codes: torch.Tensor, scales: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """MXFP8 ``(codes [M, K] e4m3, scales [M, K/32] e8m0)`` -> MXFP4 ``(codes [M, K/2], scales [M, K/32])``, all + row-major u8 views.""" + m, k = codes.shape + assert codes.is_contiguous() and scales.is_contiguous() and k % 128 == 0 + assert scales.shape == (m, k // 32), scales.shape + q = torch.empty((m, k // 2), dtype=torch.uint8, device=codes.device) + s = torch.empty((m, k // 32), dtype=torch.uint8, device=codes.device) + n_blocks = m * (k // 32) + _run_compiled( + compile_mxfp4_from_mxfp8(), + codes.view(torch.uint8).view(-1), + scales.view(torch.uint8).view(-1), + q.view(-1), + s.view(-1), + n_blocks, + (n_blocks + _THREADS - 1) // _THREADS, + torch.cuda.current_stream(), + ) + return q, s diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/__init__.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/__init__.py new file mode 100644 index 0000000..1388e42 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP8 code of the entry: ``shared_dense_4wave`` (the shared expert's 4-wave GEMMs) and ``gemm1_prefill_h6144_i3072`` +(helpers the MXFP4 kernels import, no kernel of its own). +""" diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/gemm1_prefill_h6144_i3072.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/gemm1_prefill_h6144_i3072.py new file mode 100644 index 0000000..fb52b86 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/gemm1_prefill_h6144_i3072.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Helpers the entry's GEMMs import (gfx950): the 16x16x128 FP8 MFMA class and its operand packing, the scale +loader ``_G2SCM``, the SwiGLU-OAI constants, ``v_med3`` / ``v_max3`` helpers and the pipeline step ``BLOCK_K``.""" + +import flydsl.expr as fx +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from ..common.loaders import G2SLoaderAsm, _asm_void, _uniform_i32 + +SWIGLU_ALPHA = 1.702 +SWIGLU_LIMIT = 7.0 + +BLOCK_K = 128 # K per pipeline step (128 B per row) + + +def _fmed3(a, b, c): + """v_med3_f32 (rocdl.fmed3): the exact median of finite inputs, so max(min(u, lim), -lim) in one op.""" + return fx.Float32( + _rocdl.fmed3(_T.f32, *[fx.as_ir_value(fx.Float32(v)) for v in (a, b, c)]) + ) + + +def _fmax3_abs(a, b, c): + """Max of 3 absolute values by v_max3_f32 with |.| source modifiers (asm: the MLIR bridge lacks amdgcn.fmax3).""" + return fx.Float32( + _llvm.inline_asm( + _T.f32, + [fx.as_ir_value(fx.Float32(v)) for v in (a, b, c)], + "v_max3_f32 $0, |$1|, |$2|, |$3|", + "=v,v,v,v", + has_side_effects=False, + ) + ) + + +# The MFMAs are inline asm, so LLVM's hazard recognizer misses the XDL write -> v_accvgpr_read hazard and may copy a +# final accumulator out of the AGPRs before its last MFMA wrote it (BLOCK_M 256 with VGPR spills lost the last +# K-step). The last step hands each group's accumulators to the epilogue through ``Mfma16x16x128Fp8.hold`` only after +# the next group's 16 MFMAs (the last group after these wait states: 24 cover a 16x16x128 fp8 MFMA). +_MFMA_RESULT_WAIT = "s_nop 7\n s_nop 7\n s_nop 7" + + +def _pack8(h0, h1): + """Two 4 x i32 halves -> one 8 x i32 MFMA operand: K [16 klane, +16) and K [64 + 16 klane, +16).""" + a, b = Vec(h0), Vec(h1) + return Vec.from_elements( + [a[k] for k in range(4)] + [b[k] for k in range(4)], fx.Int32 + ) + + +def _pack6(h0, h1): + """MXFP6 even step: c1 (16 B) + the low 8 B of c2 (``S2RLoader128B`` fp6), the lane's 32 e2m3 -> 6 VGPRs.""" + a, b = Vec(h0), Vec(h1) + return Vec.from_elements([a[k] for k in range(4)] + [b[0], b[1]], fx.Int32) + + +def _pack6_odd(h0, h1): + """MXFP6 odd step: the high 8 B of c2 + c3 (16 B).""" + a, b = Vec(h0), Vec(h1) + return Vec.from_elements([a[0], a[1]] + [b[k] for k in range(4)], fx.Int32) + + +class Mfma16x16x128Fp8: + """fp8 x fp8 16x16x128 scaled MFMAs over an n_tiles_a x n_tiles_b quadrant, accumulators pinned in AGPRs. + + One MFMA per tile pair per 128-K step; the scale byte is (row half / gate-up, step parity). ``fp6``: e2m3 x e2m3 + (``cbsz:2 blgp:2``, 6-VGPR operands) on ``S2RLoader128B`` fragments (``fp6_paired``: the paired-step layout's + pieces, which differ by step parity; else head + tail).""" + + def __init__(self, n_tiles_a, n_tiles_b, fp6=False, fp6_paired=False): + assert n_tiles_a % 2 == 0 + self.n_tiles_a = n_tiles_a + self.n_tiles_b = n_tiles_b + self.fp6 = fp6 + self.paired = fp6_paired + self.res_ty = Vec.make_type(4, fx.Float32) + + def idx(self, i, j): + return i * self.n_tiles_b + j + + def _order(self): + order = [] + j0s = list(range(0, self.n_tiles_b, 2)) + for n, i0 in enumerate(range(0, self.n_tiles_a, 2)): + for j0 in reversed(j0s) if n % 2 else j0s: + order += [(i0 + di, j0 + dj) for di in range(2) for dj in range(2)] + return order + + def call( + self, + a, + b, + c, + sa, + sb, + gu, + k2, + interleave=None, + zero_acc=False, + late=None, + late_start=4, + sb_index=None, + ): + """The quadrant's MFMAs with the ``interleave`` / ``late`` (from MFMA ``late_start``) thunks spread over them. + + Args: + a: ``a[i] = [half0, half1]`` (4 x i32 each); ``b[j]`` likewise. + sa: ``sa[i // 2]``: the A scale dword of a 32-row group (byte = row half i % 2 + 2 k2). + sb_index: ``(j, gu) -> (dword index into sb, byte)``; the default is the gate/up-interleaved W13 + (``sb[j]``, byte ``gu``); 32-row groups use ``(j // 2, j % 2)``. + """ + if sb_index is None: + sb_index = lambda j, gu: (j, gu) # noqa: E731 + thunks = list(interleave) if interleave else [] + late_thunks = list(late) if late else [] + order = self._order() + n_mfma = len(order) + # thunk t goes right after MFMA (t * n_mfma) // n_thunks: with more thunks than MFMAs several share a slot, + # so none trails the last MFMA + by_slot = {} # type: dict[int, list] + for t, th in enumerate(thunks): + by_slot.setdefault((t * n_mfma) // len(thunks), []).append(th) + n_late_slots = max(n_mfma - late_start, 1) + for t, th in enumerate(late_thunks): + by_slot.setdefault( + late_start + (t * n_late_slots) // len(late_thunks), [] + ).append(th) + # fp6's paired-step layout: the pieces differ by step parity + pack = _pack8 + if self.fp6: + pack = _pack6_odd if self.paired and k2 else _pack6 + for m, (i, j) in enumerate(order): + a_op = pack(a[i][0], a[i][1]) + b_op = pack(b[j][0], b[j][1]) + acc = None if zero_acc else c[self.idx(i, j)] + sbj, sbb = sb_index(j, gu) + c[self.idx(i, j)] = self._mfma_agpr( + a_op, b_op, acc, sa[i // 2], sb[sbj], i % 2, sbb, k2 + ) + for th in by_slot.get(m, ()): + th() + for m in range( + n_mfma, n_mfma + max(len(thunks), len(late_thunks)) + late_start + 1 + ): + for th in by_slot.get(m, ()): + th() + return c + + def hold(self, c, wait=""): + """``c`` through a side-effecting asm that ties each AGPR quad in place (``wait``: asm run first, e.g. wait + states), so reads of the results cannot move above this point of the MFMA stream.""" + if wait: + _llvm.inline_asm(None, [], wait, "", has_side_effects=True) + return [ + _llvm.inline_asm( + self.res_ty, [fx.as_ir_value(v)], "", "=a,0", has_side_effects=True + ) + for v in c + ] + + def _mfma_agpr(self, a_op, b_op, acc, sa_v, sb_v, ia, jb, k2): + # (B, A) instead of (A, B): C^T = B^T A^T, so lane L holds C[row L % 16, cols 4 (L // 16) .. + 4] + a_op, b_op = b_op, a_op + sa_v, sb_v = sb_v, sa_v + ia, jb = jb, ia + opsel = f"op_sel:[{ia},{jb},0]" + opsel_hi = f"op_sel_hi:[{k2},{k2},0]" + src2 = "$0" if acc is not None else "0" + fmt = 2 if self.fp6 else 0 + # LLVM adds no VALU write -> MFMA read wait states for asm: the fp6 operands must come straight from LDS + # loads, as ``_pack6`` arranges + asm = ( + f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {src2}, $3, $4 " + f"{opsel} {opsel_hi} cbsz:{fmt} blgp:{fmt}" + ) + ops = [ + fx.as_ir_value(a_op), + fx.as_ir_value(b_op), + fx.as_ir_value(sa_v), + fx.as_ir_value(sb_v), + ] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(fx.as_ir_value(acc)) + cons += ",0" + return _llvm.inline_asm(self.res_ty, ops, asm, cons, has_side_effects=True) + + +class _G2SCM(G2SLoaderAsm): + """``G2SLoaderAsm`` with a cache-policy suffix on the LDS loads (gfx950: sc0 / nt go before ``lds``).""" + + def __init__(self, *args, cm="", **kwargs): + super().__init__(*args, **kwargs) + self._CM = cm + + def _emit(self, lds_dst, k_offset, step): + voff = self._voffset(step) + soff = _uniform_i32(k_offset) + stride = self._step_stride + if step == 0: + m0 = self._lds_base_sgpr(lds_dst) + asm = ( + "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen" + + self._CM + + " lds" + ) + _asm_void([m0, voff, self.rsrc, soff], asm, "s,v,s,s", "~{scc}") + else: + asm = ( + f"s_add_u32 m0, {stride}, m0\nbuffer_load_dwordx4 $0, $1, $2 offen" + + self._CM + + " lds" + ) + _asm_void([voff, self.rsrc, soff], asm, "v,s,s", "~{scc}") diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/shared_dense_4wave.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/shared_dense_4wave.py new file mode 100644 index 0000000..58d85f5 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/mxfp8/shared_dense_4wave.py @@ -0,0 +1,2284 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Persistent 4-wave MXFP8 GEMMs of the shared expert (gfx950): ``compile_mxfp8_gemm_4w`` C[M, N] bf16 = +A[M, K] @ W[N, K]^T, ``compile_mxfp8_gemm_4w_gateup`` the gate/up GEMM with a SwiGLU + MXFP8 epilogue. + +A: row-major e4m3 [M, K]; W: e4m3 ``shuffle_weight(.., (16, 16))``; e8m0 scales per 32 K of both in AITER's shuffled +layout (``_scale_off``, ``shuffle_scale_a16w4(.., 1, False)``). K % 256 == 0; N % 256 == 0 (``n_tail``: 128; +gate/up: N % 512 == 0); M any, the buffer bounds drop rows beyond M (the A scales padded to 32 rows). +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops as _buffer_ops +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr import rocdl as _rocdl +from flydsl.expr.typing import T as _T +from flydsl.expr.typing import Vector as Vec + +from ..common.loaders import _asm_void, _g2s_thunks, _lds_ptr_t, _riffle, _uniform_i32, wait_barrier +from ..common.mfma_order import row_stationary_order +from amd_kernels.warm_up import launcher + +_N_WAVES = 4 +PARENT_M = 256 # the tile of the 4x4 layout; the 4x2 / 2x2 layouts run sub-tiles of it +PARENT_N = 256 +BLOCK_K = 128 # one MFMA K-block per step; a scale block is two steps +_PACK = 2 # 16-row tiles per 32-row scale group +_LDS_BUF_BYTES = 16384 + + +def _ceildiv(a, b): + return (a + b - 1) // b + + +def _swizzle_128(row, col): + """XOR bank swizzle of a 128-byte LDS row (16-byte chunks).""" + offset = row * 128 + col + swizzle = ((offset % (16 * 128)) >> 8) << 4 + swizzled = offset ^ swizzle + return swizzled // 128, swizzled % 128 + + +def _global_swizzle(lane_id, wave_id, K, n_rounds, preshuffled): + """Per-lane global byte offsets of a wave's g2s pieces (the LDS side is the linear m0 stream: swizzled here).""" + offsets = [] + for rnd in range_constexpr(n_rounds): + if const_expr(preshuffled): + row = lane_id % 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id // 8) * 16 + offsets.append( + (row // 16) * (K * 16) + + (row % 16) * 16 + + (col // 64) * 1024 + + ((col % 64) // 16) * 256 + + (col % 16) + ) + else: + row = lane_id // 8 + wave_id * 8 + rnd * (_N_WAVES * 8) + col = (lane_id % 8) * 16 + r, c = _swizzle_128(row, col) + offsets.append(r * K + c) + return offsets + + +class _Buf: + def __init__(self, base_ptr, byte_off): + self.base_ptr = base_ptr + self.byte_off = byte_off + + +_gep = _buffer_ops.get_element_ptr + + +def _cvt_pk_bf16(a, b): + """Two f32 -> one i32 of two bf16 (RNE), ``a`` in the low half.""" + v2f32 = _ir.VectorType.get([2], fx.Float32.ir_type) + vec = Vec.from_elements([fx.Float32(a), fx.Float32(b)], fx.Float32) + src = fx.as_ir_value(vec) + if src.type != v2f32: + src = fx.arith.bitcast(v2f32, src) + v2bf16 = _ir.VectorType.get([2], fx.BFloat16.ir_type) + return _llvm.BitcastOp(fx.Int32.ir_type, fx.arith.trunc_f(v2bf16, src)).result + + +_M0_CLOBBER = "~{m0}" + + +# ---- gate/up epilogue helpers: SwiGLU-OAI with NaN-dropping v_min / v_max_f32 clamps, MXFP8 quant of bf16 dwords + + +def _swap16(v): + """(this i32 from lane l ^ 16, to lane l ^ 16) of a 16-lane row pair.""" + pair_ty = _ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = fx.as_ir_value(fx.Int32(v)) + res = fx.rocdl.permlane16_swap(pair_ty, raw, raw, False, False) + return ( + fx.Int32(_llvm.extractvalue(_T.i32, res, [0])), + fx.Int32(_llvm.extractvalue(_T.i32, res, [1])), + ) + + +def _swap32(v): + """(low-half value, high-half value): the values of lanes l and l ^ 32.""" + pair_ty = _ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = fx.as_ir_value(fx.Int32(v)) + res = fx.rocdl.permlane32_swap(pair_ty, raw, raw, False, False) + return ( + fx.Int32(_llvm.extractvalue(_T.i32, res, [0])), + fx.Int32(_llvm.extractvalue(_T.i32, res, [1])), + ) + + +def _pk_max_u16(a, b): + """Per-16-bit unsigned max (v_pk_max_u16): the value order on sign-cleared bf16 patterns.""" + va = _llvm.BitcastOp(_T.i16x2, fx.as_ir_value(fx.Int32(a))).result + vb = _llvm.BitcastOp(_T.i16x2, fx.as_ir_value(fx.Int32(b))).result + return fx.Int32(_llvm.BitcastOp(_T.i32, _llvm.intr_umax(va, vb)).result) + + +def _amax_bf16(dwords): + """max |x| over the 2 * len(dwords) bf16 values, as a 16-bit bf16 pattern in an i32.""" + m = None + for d in dwords: + a = fx.Int32(d) & fx.Int32(0x7FFF7FFF) + m = a if m is None else _pk_max_u16(m, a) + lo, hi = m & fx.Int32(0xFFFF), m >> 16 + return (lo > hi).select(lo, hi) + + +def _i32_max(a, b): + return (a > b).select(a, b) + + +def _e8m0_bf16(amax_bits): + """(scale byte, conversion scale) of a block from the bf16 pattern of its maximum.""" + t = (fx.Int32(amax_bits) + fx.Int32(0x1F)) >> 7 + sb = _i32_max(t, fx.Int32(8)) - fx.Int32(8) + scale = ((_i32_max(t, fx.Int32(9)) - fx.Int32(8)) << 23).bitcast(fx.Float32) + return sb, scale + + +def _quant_bf16(dwords, scale): + """bf16-pair dwords -> half as many dwords of e4m3 codes of x / scale, RNE (v_cvt_scalef32_pk_fp8_bf16).""" + out = [] + for i in range(0, len(dwords), 2): + pair0 = _llvm.BitcastOp(_T.bf16x2, fx.as_ir_value(fx.Int32(dwords[i]))).result + pair1 = _llvm.BitcastOp( + _T.bf16x2, fx.as_ir_value(fx.Int32(dwords[i + 1])) + ).result + # the first conversion's other half is overwritten by the second + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16( + _T.i16x2, _llvm.mlir_undef(_T.i16x2), pair0, fx.as_ir_value(scale), False + ) + w = fx.rocdl.cvt_scalef32_pk_fp8_bf16( + _T.i16x2, w, pair1, fx.as_ir_value(scale), True + ) + out.append(fx.Int32(_llvm.BitcastOp(_T.i32, w).result)) + return out + + +def _bf16x2(dw): + """2 fp32 of one bf16 dword (low half first).""" + w = fx.Int32(dw) + return [(w << 16).bitcast(fx.Float32), (w & -65536).bitcast(fx.Float32)] + + +def _minnum_f32(a, b): + """v_min_f32 (minnum: NaN-dropping) — Triton tl.minimum's AMDGPU lowering.""" + return fx.Float32( + _llvm.inline_asm( + _T.f32, + [fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b))], + "v_min_f32 $0, $1, $2", + "=v,v,v", + has_side_effects=False, + ) + ) + + +def _maxnum_f32(a, b): + """v_max_f32 (maxnum: NaN-dropping) — Triton tl.maximum's AMDGPU lowering.""" + return fx.Float32( + _llvm.inline_asm( + _T.f32, + [fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b))], + "v_max_f32 $0, $1, $2", + "=v,v,v", + has_side_effects=False, + ) + ) + + +def _swigluq_act(g, u, f32_alpha, f32_beta, f32_limit): + """One SwiGLU-OAI value in fp32: clamp, alpha * gate, sigmoid, gate * sig * (up + beta), in this op order.""" + g = _minnum_f32(g, f32_limit) + u = _minnum_f32(_maxnum_f32(u, -f32_limit), f32_limit) + t = f32_alpha * g + sig = fx.Float32(1.0) / (fx.Float32(1.0) + fx.exp(-t)) + return g * sig * (u + f32_beta) + + +def _scale_off(m, g, scale_k1): + """Byte of (row m, group g) in AITER's shuffled scale layout [M/32, K/256, KLane 4, NLane 16, KPack 2, MPack 2].""" + return ( + ((((m // 32) * scale_k1 + g // 8) * 4 + g % 4) * 16 + m % 16) * 2 + (g % 8) // 4 + ) * 2 + (m % 32) // 16 + + +class G2SLoaderAsm: + """Global -> LDS by ``buffer_load_dwordx4 ... lds`` (m0 = the LDS destination), 1 KB per wave per instruction.""" + + def __init__(self, rsrc, gl_offsets, n_load_steps, wave_id): + self.rsrc = fx.as_ir_value(rsrc) + self.gl_offsets = gl_offsets + self.n_load_steps = n_load_steps + self.wave_id = wave_id + + @property + def _step_stride(self): + return _N_WAVES * 1024 + + def set_wave_base(self, base_ptr): + wb = fx.Int32(fx.ptrtoint(base_ptr)) + fx.Int32(self.wave_id * 1024) + self._wave_base_s = _rocdl.readfirstlane(_T.i32, fx.as_ir_value(wb)) + + def _emit(self, lds_dst, k_offset, step): + voff = fx.as_ir_value(fx.Int32(self.gl_offsets[step])) + soff = _uniform_i32(k_offset) + stride = self._step_stride + if step == 0: + m0 = fx.as_ir_value( + fx.Int32(self._wave_base_s) + fx.Int32(lds_dst.byte_off) + ) + asm = "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds" + _asm_void([m0, voff, self.rsrc, soff], asm, "s,v,s,s", _M0_CLOBBER) + else: + # s_add_u32 writes SCC: without the clobber a loop compare's SCC stays live across the asm (the tile loop + # then exits after one tile) + asm = ( + f"s_add_u32 m0, {stride}, m0\nbuffer_load_dwordx4 $0, $1, $2 offen lds" + ) + _asm_void([voff, self.rsrc, soff], asm, "v,s,s", "~{scc}") + + def load(self, lds_dst, k_offset): + for step in range_constexpr(self.n_load_steps): + self._emit(lds_dst, k_offset, step) + + def load_one(self, lds_dst, k_offset, step): + self._emit(lds_dst, k_offset, step) + + +class S2RLoader: + """LDS -> the 32-byte MFMA operand of lane (row lane % 16, klane lane // 16): K bytes 16 klane and 64 + 16 klane.""" + + def __init__(self, wave_idx, n_tiles): + self.lane_id = fx.thread_idx.x % 64 + self.wave_idx = wave_idx + self.n_tiles = n_tiles + + def _vec_load_16(self, lds_src, dyn_offset, const_offset): + total_off = lds_src.byte_off + const_offset + window_base = (total_off // 0x10000) * 0x10000 + imm = total_off - window_base + assert 0 <= imm <= 0xFFFF + vaddr = fx.Int32(fx.ptrtoint(lds_src.base_ptr)) + fx.Int32( + window_base + dyn_offset + ) + lds_ptr = _llvm.inttoptr(_lds_ptr_t(), fx.as_ir_value(vaddr)) + if imm != 0: + lds_ptr = _gep(lds_ptr, static_byte_offset=imm) + vec4_i32 = _ir.VectorType.get([4], fx.Int32.ir_type) + return Vec(_llvm.LoadOp(vec4_i32, lds_ptr, alignment=16).result) + + def _dyn_offset(self, seg, preshuffled): + row = self.wave_idx * (self.n_tiles * 16) + self.lane_id % 16 + col = (self.lane_id // 16) * 16 + seg * 64 + if const_expr(preshuffled): + return (row // 8) * 1024 + (row % 8) * 16 + (col // 16) * 128 + row_swz, col_swz = _swizzle_128(row, col) + return row_swz * 128 + col_swz + + def load_seg(self, lds_src, i, seg, preshuffled=False): + """One 16-byte segment (tile i, K segment seg) as i32x4.""" + return self._vec_load_16( + lds_src, self._dyn_offset(seg, preshuffled), i * 2048 + ).bitcast(fx.Int32) + + def load(self, lds_src, preshuffled=False): + return [_pack8(self.load_seg(lds_src, i, 0, preshuffled), self.load_seg(lds_src, i, 1, preshuffled)) + for i in range_constexpr(self.n_tiles)] # fmt: skip + + +def _pack8(lo, hi): + """Two i32x4 K segments -> one i32x8 MFMA operand.""" + return lo.shuffle(hi, list(range(8))) + + +def _s2r_thunks(s2r, src, holder, n, pre): + """Thunks loading tile i's two segments into holder[i] (packed after the second).""" + ts = [] + for i in range(n): + for seg in range(2): + + def f(i=i, seg=seg): + v = s2r.load_seg(src, i, seg, preshuffled=pre) + if seg == 0: + holder[i] = v + else: + holder[i] = _pack8(holder[i], v) + + ts.append(f) + return ts + + +def _min(a, b): + return fx.arith.select(a < b, a, b) + + +def _divmod_nonneg(a, b): + """divmod of a non-negative value; a power-of-two Python int ``b`` becomes a shift and a mask.""" + if const_expr(isinstance(b, int) and b > 0 and (b & (b - 1)) == 0): + sh = b.bit_length() - 1 + return (a >> sh, a & (b - 1)) if const_expr(sh > 0) else (a, 0) + return (a // b, a % b) + + +def _xcd_swizzle(num_pid_m, num_pid_n, wgid): + """Tile index -> (tile_i, tile_j) in aiter's XCD-aware order: workgroup ``w`` runs on XCD ``w % 8``, and XCD ``x`` + walks its own eighth of the tiles column-major in groups of 4 rows, so its CUs share W and A tiles in L2. An index + beyond the grid lands on row ``num_pid_m``, beyond M, where the buffer bounds drop its stores.""" + NUM_XCDS = 8 + WGM = 4 + num_wg = num_pid_m * num_pid_n + in_grid = (wgid >= 0) & (wgid < num_wg) + wgid = fx.arith.select(in_grid, wgid, 0) + intra_xcd, xcd = _divmod_nonneg(wgid, NUM_XCDS) + base, extra = _divmod_nonneg(num_wg, NUM_XCDS) + wgid_remap = xcd * base + _min(xcd, extra) + intra_xcd + group_id, intra_group = _divmod_nonneg(wgid_remap, WGM * num_pid_n) + first_pid_m = group_id * WGM + group_size_m = _min(num_pid_m - first_pid_m, WGM) + pid_n, intra_group_m = _divmod_nonneg(intra_group, group_size_m) + pid_m = fx.arith.select(in_grid, first_pid_m + intra_group_m, num_pid_m) + return pid_m, pid_n + + +class Mfma16x16x128Fp8: + """Scaled fp8 16x16x128 MFMAs over a quadrant, accumulators pinned in AGPRs, in the 2x2 zig-zag order. + + Operands go in as (W, A), so lane (klane, r) accumulates out[row r, 4 klane ..] (C^T = W A^T); ``sa`` / ``sb`` are + the packed e8m0 dwords of the 32-row / 32-column groups, op_sel (tile parity) picks the row half of the scale + byte, op_sel_hi (K-step parity) its K half.""" + + def __init__(self, n_a, n_b): + self.res_ty = Vec.make_type(4, fx.Float32) + self.n_a = n_a + self.n_b = n_b + + def idx(self, i, j): + return i * self.n_b + j + + def _order(self): + order = [] + j0s = list(range(0, self.n_b, 2)) + for n, i0 in enumerate(range(0, self.n_a, 2)): + for j0 in reversed(j0s) if n % 2 else j0s: + order += [(i0 + di, j0 + dj) for di in range(2) for dj in range(2)] + return order + + def call(self, a, b, c, sa, sb, khalf, interleave=None, zero_acc=False): + """The quadrant's MFMAs with the ``interleave`` thunks spread evenly between them (the rest after the last).""" + thunks = list(interleave) if interleave else [] + nth = [0] + mth = [0] + order = self._order() + n_mfma = len(order) + slots = ( + {(t * n_mfma) // len(thunks) for t in range(len(thunks))} + if thunks + else set() + ) + for i, j in order: + acc = None if zero_acc else c[self.idx(i, j)] + c[self.idx(i, j)] = self._mfma( + a[i], + b[j], + acc, + sa[i // _PACK], + sb[j // _PACK], + khalf, + i % _PACK, + j % _PACK, + ) + if nth[0] < len(thunks) and mth[0] in slots: + thunks[nth[0]]() + nth[0] += 1 + mth[0] += 1 + while nth[0] < len(thunks): + thunks[nth[0]]() + nth[0] += 1 + return c + + def _mfma(self, a_op, b_op, acc, sa_v, sb_v, khalf, ia, jb): + a_op, b_op = b_op, a_op + sa_v, sb_v = sb_v, sa_v + ia, jb = jb, ia + src2 = "$0" if acc is not None else "0" + asm = (f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {src2}, $3, $4 " + f"op_sel:[{ia},{jb},0] op_sel_hi:[{khalf},{khalf},0] cbsz:0 blgp:0") # fmt: skip + ops = [ + fx.as_ir_value(a_op), + fx.as_ir_value(b_op), + fx.as_ir_value(sa_v), + fx.as_ir_value(sb_v), + ] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(fx.as_ir_value(acc)) + cons += ",0" + return _llvm.inline_asm(self.res_ty, ops, asm, cons, has_side_effects=True) + + +class MfmaRowStationary(Mfma16x16x128Fp8): + """``Mfma16x16x128Fp8`` issuing a quadrant row by row (``row_stationary_order``, bitwise the zig-zag's).""" + + def _order(self): + return row_stationary_order(self.n_a, self.n_b) + + +_SCALE_QUARTER_BYTES = 1024 # one wave's gather: 4 groups x 256 B of one 256-K block +_SCALE_REGION_BYTES = 2 * _SCALE_QUARTER_BYTES # all of A (or all of W) for one block +_SCALE_SLOT_BYTES = _N_WAVES * _SCALE_QUARTER_BYTES # 4 KB +_SCALE_SLOTS = 2 # the block in use (VGPR carry) and the next one +_SCALE_A_REGION = 0 +_SCALE_B_REGION = _SCALE_REGION_BYTES + + +class ScaleGatherLDS: + """One ``buffer_load_dwordx4 ... lds`` per wave per 256-K scale block: waves 0 / 1 fetch the A scales of their own + rows (groups {0, 1, 4, 5} / {2, 3, 6, 7} of the tile's eight 32-row groups), waves 2 / 3 the W scales alike.""" + + def __init__( + self, + a_scale, + b_scale, + K, + lane_id, + wave_id, + lds_base_ptr, + a_rows, + b_rows, + n_a, + n_b, + fused_gu=False, + b_k=None, + up_rows=None, + ): + b_k = K if b_k is None else b_k + self.row_i32 = (K // 256) * 64 # dwords per 32-row group + self.b_row_i32 = (b_k // 256) * 64 + self.wave_id = wave_id + self.n_a = ( + n_a # 16-row tiles per wave quadrant; an LDS half holds n_a 32-row groups + ) + self.n_b = n_b + self.fused_gu = fused_gu + # fused_gu: the tile's W halves are the gate rows [tj*128, +128) and the up rows [N/2 + tj*128, +128); the + # up panel starts (N/2) / 32 groups in + self._up_groups = ((b_rows // 64) if up_rows is None else fx.Int32(up_rows // 32)) if fused_gu else None + row_bytes = K // 32 + if up_rows is not None: + b_rows = up_rows + b_rows // 2 + # the shuffled layout interleaves a 32-row group, so the bound is whole groups (a bound of M rows zeroes the + # last group's scales; the quantizer pads the rows to 32) + a_rows = (a_rows + 31) // 32 * 32 + # create_buffer_resource returns the raw !llvm.ptr<8> resource + self.a_rsrc = fx.as_ir_value( + _buffer_ops.create_buffer_resource( + a_scale, max_size=False, num_records_bytes=a_rows * row_bytes + ) + ) + self.b_rsrc = fx.as_ir_value( + _buffer_ops.create_buffer_resource( + b_scale, max_size=False, num_records_bytes=b_rows * (b_k // 32) + ) + ) + self._blk = lane_id // 16 + self._in16 = lane_id % 16 + self._lds_base = fx.Int32(fx.ptrtoint(lds_base_ptr)) + + def set_wave_base(self): + wid = fx.Int32(_rocdl.readfirstlane(_T.i32, fx.as_ir_value(self.wave_id))) + self._wave_base_s = fx.as_ir_value( + self._lds_base + wid * fx.Int32(_SCALE_QUARTER_BYTES) + ) + self._is_a = wid < fx.Int32(2) + self._q = wid % fx.Int32(2) + self._rsrc = fx.arith.select(self._is_a, self.a_rsrc, self.b_rsrc) + self._soff0 = _uniform_i32(fx.Int32(0)) + + def _row_i32(self): + if const_expr(self.b_row_i32 == self.row_i32): + return fx.Int32(self.row_i32) + return fx.arith.select(self._is_a, fx.Int32(self.row_i32), fx.Int32(self.b_row_i32)) + + def tile_group(self, a_base_tile, b_base_tile): + """This wave's first 32-row group of a tile (wave-uniform).""" + if const_expr(self.fused_gu): + # the gate panel starts at W row tj*128 = b_base_tile / 2 + b_base_tile = b_base_tile // 2 + base_tile = fx.arith.select( + self._is_a, + a_base_tile + self._q * fx.Int32(16 * self.n_a), + b_base_tile + self._q * fx.Int32(16 * self.n_b), + ) + return _uniform_i32(base_tile // fx.Int32(32)) + + def gather(self, G, block, slot): + """Scale block ``block`` of the tile whose first group is ``G`` into LDS slot ``slot``.""" + # lane blocks 0 / 1 = the wave's groups of the first tile half, 2 / 3 of the second (a half holds n_a / n_b + # groups); blocks beyond the wave's groups fetch neighbours. fused_gu: W's second half is the up panel, + # (N/2) / 32 groups away. + if const_expr(self.fused_gu): + half = fx.arith.select(self._is_a, fx.Int32(self.n_a), self._up_groups) + else: + half = fx.arith.select(self._is_a, fx.Int32(self.n_a), fx.Int32(self.n_b)) + grp = fx.Int32(G) + (self._blk // 2) * half + (self._blk % 2) + i32_off = ( + grp * self._row_i32() + + fx.Int32(block) * fx.Int32(64) + + self._in16 * fx.Int32(4) + ) + voff = fx.as_ir_value(i32_off * fx.Int32(4)) + addr = fx.Int32(self._wave_base_s) + fx.Int32(slot) * fx.Int32( + _SCALE_SLOT_BYTES + ) + asm = "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds" + _asm_void( + [fx.as_ir_value(addr), voff, self._rsrc, self._soff0], + asm, + "s,v,s,s", + _M0_CLOBBER, + ) + + def set_tile_voff(self, G): + """This tile's per-lane gather offset without the K block (that rides the uniform soffset), once per tile.""" + if const_expr(self.fused_gu): + half = fx.arith.select(self._is_a, fx.Int32(self.n_a), self._up_groups) + else: + half = fx.arith.select(self._is_a, fx.Int32(self.n_a), fx.Int32(self.n_b)) + grp = fx.Int32(G) + (self._blk // 2) * half + (self._blk % 2) + self._voff_g = fx.as_ir_value( + (grp * self._row_i32() + self._in16 * fx.Int32(4)) * fx.Int32(4) + ) + + def gather_soff(self, m0_s, soff): + """``gather`` with the K-block byte offset in soffset and the slot's LDS address ``m0_s`` precomputed.""" + asm = "s_mov_b32 m0, $0\nbuffer_load_dwordx4 $1, $2, $3 offen lds" + soff_v = fx.Int32(soff) if isinstance(soff, int) else soff + _asm_void( + [fx.as_ir_value(m0_s), self._voff_g, self._rsrc, _uniform_i32(soff_v)], + asm, + "s,v,s,s", + _M0_CLOBBER, + ) + + +class ScaleLoaderLDS: + """The wave's scale dwords of a block: lane (klane, r) reads group (half, gi)'s dword at its (klane, mlane).""" + + def __init__(self, n_tiles, lane_id, quarter, lds_base_ptr, region_off): + assert n_tiles % _PACK == 0 + self.n_groups = n_tiles // _PACK + self.lane_id = lane_id + self._region_base = ( + fx.Int32(fx.ptrtoint(lds_base_ptr)) + + fx.Int32(region_off) + + quarter * fx.Int32(_SCALE_QUARTER_BYTES) + ) + + def read_half(self, slot, half): + L = self.lane_id + base = ( + self._region_base + + fx.Int32(slot) * fx.Int32(_SCALE_SLOT_BYTES) + + fx.Int32((L // 4) * 16 + (L % 4) * 4) + ) + out = [] + for gi in range_constexpr(self.n_groups): + blk = half * 2 + gi + lds_ptr = _llvm.inttoptr( + _lds_ptr_t(), fx.as_ir_value(base + fx.Int32(blk * 256)) + ) + out.append( + fx.Int32(_llvm.LoadOp(fx.Int32.ir_type, lds_ptr, alignment=4).result) + ) + return out + + def read(self, slot): + return self.read_half(slot, 0), self.read_half(slot, 1) + + +def _acc_read(vec4, i): + """``v_accvgpr_read_b32`` as side-effecting asm: LLVM does not know the inline-asm MFMA's latency and would hoist + a plain read up to the MFMA that writes the accumulator (a stale read); the asm keeps its place in the stream.""" + el = fx.as_ir_value(Vec(vec4)[i]) + return fx.Float32( + _llvm.inline_asm( + fx.Float32.ir_type, + [el], + "v_accvgpr_read_b32 $0, $1", + "=v,a", + has_side_effects=True, + ) + ) + + +class StoreCLds: + """bf16 output through a per-wave LDS transpose (2 buffers). + + A 16-row block of a quadrant is written in the MFMA layout (4 columns of one row per lane per tile) and read back + as slabs of (32 / n_b) rows, so each ``buffer_store_dwordx4`` writes whole rows instead of 64 scattered 16-byte + pieces (~3x the issue cost). The 16-byte pieces of a row are XOR-swizzled by (row % pieces) against bank conflicts. + """ + + def __init__( + self, + C, + c_rows, + c_cols, + c_idx_fn, + lds_base_ptr, + wave_id, + n_a, + n_b, + nt=False, + n_tail=False, + ): + self.c_cols = c_cols + self.c_rows = c_rows + self.n_tail = n_tail + self.c_idx_fn = c_idx_fn + self.n_a = n_a + self.n_b = n_b + self.pieces = 2 * n_b # 16-byte pieces per block row (32 n_b bytes) + self.rows_per_read = 64 // self.pieces # rows one 64-lane read / store covers + self.slabs = 16 // self.rows_per_read # reads per block + self.buf_bytes = 16 * 32 * n_b + self.cpol = ( + 2 if nt else 0 + ) # gfx950 buffer_store cache policy 0x2 = nt + self.rsrc = _buffer_ops.create_buffer_resource( + C, max_size=False, num_records_bytes=c_rows * c_cols * 2 + ) + lane = fx.thread_idx.x % 64 + g = lane // 16 + r = lane % 16 + row_b = 32 * n_b + base = fx.Int32(fx.ptrtoint(lds_base_ptr)) + wave_id * fx.Int32( + 2 * self.buf_bytes + ) + # write side: lane (g, r) holds columns 4 g .. 4 g + 3 of row r of each 16x16 tile + self.w_off = [ + base + + r * fx.Int32(row_b) + + (((fx.Int32(2 * tj) + g // 2) ^ (r % self.pieces)) * fx.Int32(16)) + + (g % 2) * fx.Int32(8) + for tj in range_constexpr(n_b) + ] + # read side: lane (r2 = lane // pieces, q = lane % pieces) takes piece q of row r2 of each slab + q = lane % self.pieces + r2 = lane // self.pieces + self.r_off = [ + base + + (r2 + h * self.rows_per_read) * fx.Int32(row_b) + + ((q ^ (r2 % self.pieces)) * fx.Int32(16)) + for h in range_constexpr(self.slabs) + ] + self.st_off = ( + r2 * c_cols + q * 8 + ) * 2 # the per-lane part of the store byte offset + + @property + def n_thunks(self): + return self.n_a * (1 + self.slabs) + + def _write(self, c_frag, ti, tj, buf): + v = c_frag[self.c_idx_fn(ti, tj)] + rd = _acc_read + d0 = _cvt_pk_bf16(rd(v, 0), rd(v, 1)) + d1 = _cvt_pk_bf16(rd(v, 2), rd(v, 3)) + vec = Vec.from_elements([fx.Int32(d0), fx.Int32(d1)], fx.Int32) + ptr = _llvm.inttoptr( + _lds_ptr_t(), + fx.as_ir_value(self.w_off[tj] + fx.Int32(buf * self.buf_bytes)), + ) + _llvm.StoreOp(fx.as_ir_value(vec), ptr, alignment=8) + + def _read_store(self, base_row, base_col, ti, h, buf): + ptr = _llvm.inttoptr( + _lds_ptr_t(), fx.as_ir_value(self.r_off[h] + fx.Int32(buf * self.buf_bytes)) + ) + vec4 = _ir.VectorType.get([4], fx.Int32.ir_type) + data = Vec(_llvm.LoadOp(vec4, ptr, alignment=16).result) + off = ( + (base_row + ti * 16 + h * self.rows_per_read) * self.c_cols + base_col + ) * 2 + self.st_off + _buffer_ops.buffer_store( + data, self.rsrc, off, offset_is_bytes=True, cache_modifier=self.cpol + ) + + def thunks(self, c_frag, base_row, base_col): + """n_a (1 + slabs) thunks per quadrant: the blocks through the two buffers as W0 W1 R0 W2 R1 ... + + A wave's LDS ops run in order, so a buffer's re-write after its reads needs no wait; quadrants sharing the + buffers must be concatenated, not interleaved. ``n_tail``: a quadrant beyond N (N % 256 == 128) is stored at + row M, where the buffer bounds drop it.""" + if const_expr(self.n_tail): + base_row = fx.arith.select( + base_col < self.c_cols, base_row, fx.Int32(self.c_rows) + ) + order = [("w", 0)] + for b in range(1, self.n_a): + order += [("w", b), ("r", b - 1)] + order.append(("r", self.n_a - 1)) + ts = [] + for kind, ti in order: + if kind == "w": + ts.append( + lambda ti=ti: [ + self._write(c_frag, ti, tj, ti % 2) for tj in range(self.n_b) + ] + ) + else: + for h in range(self.slabs): + ts.append( + lambda ti=ti, h=h: self._read_store( + base_row, base_col, ti, h, ti % 2 + ) + ) + return ts + + def store(self, c_frag, base_row, base_col): + for t in self.thunks(c_frag, base_row, base_col): + t() + + +class StoreSwigluQ: + """The gate/up epilogue: SwiGLU-OAI + MXFP8 quant into the down GEMM's A operand, e4m3 codes row-major [M, N/2] + plus e8m0 scale bytes in AITER's shuffled layout; bitwise the bf16 C store followed by SwiGLU and MXFP8 quant. + + The tile's b0 half holds the gate columns and b1 the matching up columns (``tile_bases``), so the quadrant pairs + (c00, c01) / (c10, c11) cover a [64-row, 64-column] output window and every 32-column quant group is local; a + group's 4 lanes (r, r + 16, r + 32, r + 48) combine with permlane16 / 32 swaps. ``thunks``: read + math + store + per 16-row chunk (the top pair, in the last step's groups 2-3); ``thunks_rm`` splits the accumulator reads, due + before the next tile's step-0 MFMAs zero the AGPRs (groups 0-1), from the math and stores (groups 2-3).""" + + def __init__( + self, Q, S, c_m, out_n, c_idx_fn, lane_id, n_a, n_b, alpha, beta, limit + ): + self.c_idx_fn = c_idx_fn + self.n_a = n_a + self.n_b = n_b + self.out_n = out_n + self.scale_k1 = out_n // 256 + self.alpha = alpha + self.beta = beta + self.limit = limit + self.qg = lane_id // 16 # the lane's 4-column quarter of a 16-column tile row + self.r = lane_id % 16 + self.q_rsrc = _buffer_ops.create_buffer_resource( + Q, max_size=False, num_records_bytes=c_m * out_n + ) + self.s_rsrc = _buffer_ops.create_buffer_resource( + S, max_size=False, num_records_bytes=((c_m + 31) // 32 * 32) * (out_n // 32) + ) + + def _read_acts(self, c_gate, c_up, ti): + """Row-tile ti's gate + up accumulators -> 2 n_b bf16 dwords of SwiGLU (the only phase reading AGPRs).""" + dws = [] + for tj in range_constexpr(self.n_b): + vg = c_gate[self.c_idx_fn(ti, tj)] + vu = c_up[self.c_idx_fn(ti, tj)] + gv = _bf16x2(_cvt_pk_bf16(_acc_read(vg, 0), _acc_read(vg, 1))) + _bf16x2( + _cvt_pk_bf16(_acc_read(vg, 2), _acc_read(vg, 3)) + ) + uv = _bf16x2(_cvt_pk_bf16(_acc_read(vu, 0), _acc_read(vu, 1))) + _bf16x2( + _cvt_pk_bf16(_acc_read(vu, 2), _acc_read(vu, 3)) + ) + act = [ + _swigluq_act(gv[j], uv[j], self.alpha, self.beta, self.limit) + for j in range_constexpr(4) + ] + dws.append(_cvt_pk_bf16(act[0], act[1])) + dws.append(_cvt_pk_bf16(act[2], act[3])) + return dws + + def _quant_store(self, dws, base_row, base_col, ti): + """Codes + scale bytes of row-tile ti from its packed activations (VGPRs only).""" + row = base_row + fx.Int32(ti * 16) + self.r + for h in range_constexpr(self.n_b // 2): + gdw = dws[4 * h : 4 * h + 4] + m1a, m1b = _swap16(_amax_bf16(gdw)) + m2a, m2b = _swap32(_i32_max(m1a, m1b)) + sb, scale = _e8m0_bf16(_i32_max(m2a, m2b)) + codes = _quant_bf16(gdw, scale) + col = base_col + fx.Int32(32 * h) + self.qg * fx.Int32(4) + row_off = row * self.out_n + col + for tt in range_constexpr(2): + _buffer_ops.buffer_store( + codes[tt], + self.q_rsrc, + row_off + fx.Int32(16 * tt), + offset_is_bytes=True, + ) + _buffer_ops.buffer_store( + sb.to(fx.Int8), + self.s_rsrc, + _scale_off(row, base_col // fx.Int32(32) + fx.Int32(h), self.scale_k1), + mask=self.qg == 0, + offset_is_bytes=True, + ) + + def thunks(self, c_gate, c_up, base_row, base_col): + return [ + ( + lambda ti=ti: self._quant_store( + self._read_acts(c_gate, c_up, ti), base_row, base_col, ti + ) + ) + for ti in range_constexpr(self.n_a) + ] + + def thunks_rm(self, c_gate, c_up, base_row, base_col): + dws = [None] * self.n_a + + def _r(ti): + dws[ti] = self._read_acts(c_gate, c_up, ti) + + def _m(ti): + self._quant_store(dws[ti], base_row, base_col, ti) + + return ( + [(lambda ti=ti: _r(ti)) for ti in range_constexpr(self.n_a)], + [(lambda ti=ti: _m(ti)) for ti in range_constexpr(self.n_a)], + ) + + +@launcher +def compile_mxfp8_gemm_4w( + *, + K: int, + use_xcd_remap: bool = True, + MN: tuple | None = None, + store_nt: bool = False, + tiles: tuple = (4, 4), + n_tail: bool = False, + grid_max: int = 256, + row_stationary: bool = True, + salu_diet: bool = True, + w_k: int | None = None, +): + """The bf16-output GEMM; the launcher's ``.tile`` is the workgroup body for composed launches. + + A workgroup of 4 waves computes a tile, each wave a 2x2 of quadrants (c00 / c01 / c10 / c11) of n_a x n_b MFMA + tiles; A and W go through 8 LDS buffers filled two K-steps ahead. Persistent: min(n_work, ``grid_max``) + workgroups walk the work items w, w + grid, ...; a tile's first two steps' loads and scale gather go out in the + previous tile's last two steps, and the stores of c00 / c01 interleave with the last step's c10 / c11 MFMAs, those + of c10 / c11 with the next tile's first c00 / c01 MFMAs (old and new accumulators share the 256 AGPRs half and + half). Scale blocks are gathered one block ahead (at even steps) and read at the following odd step. + + Args: + MN: (M, N) baked in (every divide folds). + store_nt: non-temporal output stores (pays when the output exceeds the 256 MB infinity cache). + tiles: (n_a, n_b) 16-row / 16-column MFMA tiles per wave quadrant: (4, 4) is the 256x256 parent tile, + (4, 2) / (2, 2) its 256x128 / 128x128 sub-tiles; a parent is 1 / 2 / 4 work items. + n_tail: N % 256 == 128: the last tile column computes its missing half against zero weights, stores dropped. + grid_max: workgroups of the launch (256: one per CU, persistent; larger: a plain grid, one per work item, + which no co-resident kernel holds back). + row_stationary: issue each quadrant row by row (``MfmaRowStationary``, bitwise the 2x2 zig-zag's). + salu_diet: (with ``row_stationary``) carry the g2s / gather addresses in uniform accumulators instead of + recomputing them every K-step (bitwise). + w_k: the K of W's layout when the GEMM reads a K slice of a wider weight (16-row block and 32-row scale + group strides); W and its scales start at the slice. + """ + salu_diet = salu_diet and row_stationary + W_K = K if w_k is None else w_k + assert W_K % 256 == 0 and W_K >= K + N_TILES_A, N_TILES_B = tiles + assert N_TILES_A in (2, 4) and N_TILES_B in (2, 4) + BLOCK_M = 64 * N_TILES_A + BLOCK_N = 64 * N_TILES_B + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + SUB_M = PARENT_M // BLOCK_M # sub-tiles per parent tile + SUB_N = PARENT_N // BLOCK_N + SUB_SPLIT = SUB_M * SUB_N + assert K % 256 == 0 + K_ITERS = K // BLOCK_K # even + KB = K // 256 # scale blocks + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + assert ( + LDS_BLOCK_M * BLOCK_K <= _LDS_BUF_BYTES + and LDS_BLOCK_N * BLOCK_K <= _LDS_BUF_BYTES + ) + GRID_MAX = grid_max + order = "zigzag" if not row_stationary else ("rowstationary_salu" if salu_diet else "rowstationary") + name = (f"shared_dense_mxfp8_h6144_i1536_{order}_k{K}" + (f"_m{MN[0]}_n{MN[1]}" if MN else "") + ("_nt" if store_nt else "") + ("" if grid_max == 256 else f"_g{grid_max}") + + (f"_t{N_TILES_A}x{N_TILES_B}" if tiles != (4, 4) else "") + ("_ntail" if n_tail else "") + + (f"_wk{W_K}" if W_K != K else "")) # fmt: skip + SCALE_SLOTS = _SCALE_SLOTS + OUT_LDS_BYTES = _N_WAVES * 2 * 16 * 32 * N_TILES_B + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, 8 * _LDS_BUF_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_SLOTS * _SCALE_SLOT_BYTES, 16] + out_lds: fx.Array[fx.Int8, OUT_LDS_BYTES, 16] + + @flyc.jit + def workgroup_tile(base_ptr, scale_ptr, out_ptr, wg, + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + ): + _base_ptr = base_ptr + + def _buf(idx): + return _Buf(_base_ptr, idx * _LDS_BUF_BYTES) + + a_cur0, a_cur1, a_next0, a_next1 = _buf(0), _buf(1), _buf(2), _buf(3) + b_cur0, b_cur1, b_next0, b_next1 = _buf(4), _buf(5), _buf(6), _buf(7) + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + if const_expr(MN is not None): + c_m, c_n = const_expr(MN[0]), const_expr(MN[1]) + n_pid_m = _ceildiv(c_m, PARENT_M) # the parent tiles' grid + n_blocks = _ceildiv(c_n, PARENT_N) + grid = _min(fx.Int32(n_work), fx.Int32(GRID_MAX)) + my_tiles = (fx.Int32(n_work) - wg + grid - fx.Int32(1)) // grid + wave_i = wave_id // 2 + wave_j = wave_id % 2 + A_K_STEP = BLOCK_K + B_K_STEP = 2 * 1024 # preshuffled (16, 16): a 16-row block spans 2 KB per 128 K + + def tile_coords(t): + """Work item -> (tile_i, tile_j) in this kernel's tile units, the parents in ``_xcd_swizzle``'s order. + + Item ``t = 8 q + x`` is sub-tile ``q % SUB_SPLIT`` of parent ``tile_off + 8 (q / SUB_SPLIT) + x`` + (``tile_off`` a multiple of 8), so a sub-tile launch keeps a workgroup on its XCD's parents; a negative or + out-of-range index lands beyond M, where the buffer bounds drop its stores.""" + if const_expr(use_xcd_remap and SUB_SPLIT > 1): + q, x = _divmod_nonneg(t, 8) + p_intra, sub = _divmod_nonneg(q, SUB_SPLIT) + parent = fx.Int32(tile_off) + p_intra * 8 + x + else: + parent, sub = _divmod_nonneg(t, SUB_SPLIT) + parent = fx.Int32(tile_off) + parent + if const_expr(use_xcd_remap): + pi, pj = _xcd_swizzle(n_pid_m, n_blocks, parent) + else: + pi, pj = parent // n_blocks, parent % n_blocks + si, sj = _divmod_nonneg(sub, SUB_N) + return pi * SUB_M + si, pj * SUB_N + sj + + def tile_bases(ti, tj): + """The g2s byte bases of a tile's two A halves (row-major) and two W halves (preshuffled).""" + return ( + (ti * BLOCK_M) * K, + (ti * BLOCK_M + LDS_BLOCK_M) * K, + (tj * BLOCK_N) * W_K, + (tj * BLOCK_N + LDS_BLOCK_N) * W_K, + ) + + mfma = ( + MfmaRowStationary(N_TILES_A, N_TILES_B) + if row_stationary + else Mfma16x16x128Fp8(N_TILES_A, N_TILES_B) + ) + _scale_base_ptr = scale_ptr + scale_gather = ScaleGatherLDS( + A_scale, + B_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + c_m, + c_n, + N_TILES_A, + N_TILES_B, + b_k=W_K, + ) + scale_gather.set_wave_base() + if const_expr(salu_diet): + # the two scale slots' LDS addresses, once; the gather's K-block offset rides the uniform soffset + _gm0_s = ( + scale_gather._wave_base_s, + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + fx.Int32(_SCALE_SLOT_BYTES) + ), + ) + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(b): + return fx.Int32(b) % fx.Int32(SCALE_SLOTS) + + gl_off_a = _global_swizzle(lane_id, wave_id, K, N_TILES_A, False) + gl_off_b = _global_swizzle(lane_id, wave_id, W_K, N_TILES_B, True) + a_rsrc = _buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=c_m * K + ) + b_rsrc = _buffer_ops.create_buffer_resource( + B_T, max_size=False, num_records_bytes=c_n * W_K + ) + a_g2s = G2SLoaderAsm(a_rsrc, gl_off_a, N_TILES_A, wave_id) + b_g2s = G2SLoaderAsm(b_rsrc, gl_off_b, N_TILES_B, wave_id) + a_g2s.set_wave_base(_base_ptr) + b_g2s.set_wave_base(_base_ptr) + a_s2r = S2RLoader(wave_i, N_TILES_A) + b_s2r = S2RLoader(wave_j, N_TILES_B) + store_c = StoreCLds( + C, + c_m, + c_n, + mfma.idx, + out_ptr, + wave_id, + N_TILES_A, + N_TILES_B, + nt=store_nt, + n_tail=n_tail, + ) + + # a step issues [ac0 (n_a) | bc0 (n_b) | bc1 (n_b) | ac1 (n_a)] loads per wave: at a step's top the previous + # step's may all be in flight; mid-step the previous step's ac1 and this step's first two batches + _MAIN_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(block, holder): + # salu_diet: the caller passes the LDS slot itself (no block -> slot math) + s = block if salu_diet else _slot(block) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _g2s_bases(bases, step): + a0, a1, b0, b1 = bases + st = fx.Int32(step) + return ( + fx.Int32(a0) + st * fx.Int32(A_K_STEP), + fx.Int32(a1) + st * fx.Int32(A_K_STEP), + fx.Int32(b0) + st * fx.Int32(B_K_STEP), + fx.Int32(b1) + st * fx.Int32(B_K_STEP), + ) + + def _one_step( + parity, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + load, + gather, + read_block, + zero_acc=False, + extra=None, + ): + """One K-step: its MFMAs with the next step's LDS reads and the step after's global loads interleaved. + + Args: + load: ``(bases, step)``, or the (a0, a1, b0, b1) soffsets with ``salu_diet``. + gather: even steps: the next scale block's ``(G, block)``, or (m0, soff) with ``salu_diet``. + read_block: odd steps: the scale block read into the VGPR carry (its LDS slot with ``salu_diet``). + extra: ``extra(g, c00f, c01f)`` before group g = 0..3 (the accumulators finished so far, else None) + returns more thunks for that group: the stores of this tile's c00 / c01 in the last step, of + the previous tile's c10 / c11 in the first. + """ + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + if const_expr(salu_diet): + a0_off, a1_off, b0_off, b1_off = load + else: + a0_off, a1_off, b0_off, b1_off = _g2s_bases(*load) + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(read_block, _scn) if parity == 1 else [] + if const_expr(salu_diet): + _sc_gather = ( + [] + if parity == 1 + else [lambda: scale_gather.gather_soff(gather[0], gather[1])] + ) + else: + _sc_gather = ( + [] + if parity == 1 + else [ + lambda: scale_gather.gather( + gather[0], gather[1], _slot(gather[1]) + ) + ] + ) + ex = extra if extra is not None else (lambda g, c00, c01: ()) + + wait_barrier(_MAIN_VMCNT) + il = _riffle( + _g2s_thunks(a_g2s, ac0, a0_off, N_TILES_A), + _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False), + ) + list(ex(0, None, None)) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + il = _riffle( + _g2s_thunks(b_g2s, bc0, b0_off, N_TILES_B), list(ex(1, c00f, None)) + ) + c01f = mfma.call( + a0f, b1f_in, c01f, saR0, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + a1f = _a1 + + wait_barrier(_SEG2_VMCNT) + il = (_riffle(_g2s_thunks(b_g2s, bc1, b1_off, N_TILES_B), _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False)) + + _sc_gather + _rd_scn[:2] + list(ex(2, c00f, c01f))) # fmt: skip + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + a0nf = _a0n + il = ( + _riffle( + _g2s_thunks(a_g2s, ac1, a1_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + + _rd_scn[2:] + + list(ex(3, c00f, c01f)) + ) + c11f = mfma.call( + a1f, b1f_in, c11f, saR1, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + b0nf, b1nf = _b0n, _b1n + + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) if parity == 1 else sc + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = N_TILES_A + n_b = N_TILES_B + n_ga = N_TILES_A // _PACK + n_gb = N_TILES_B // _PACK + n_sc = 2 * n_ga + 2 * n_gb + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC0, sbC1 = sc + return ( + [_R(v) for v in saR0] + + [_R(v) for v in saR1] + + [_R(v) for v in sbC0] + + [_R(v) for v in sbC1] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC0 = list(flat[o : o + n_gb]) + o += n_gb + sbC1 = list(flat[o : o + n_gb]) + return (saR0, saR1, sbC0, sbC1) + + # ---- prologue: the first tile's steps 0 / 1 and its scale blocks 0 / 1 ---- + t0 = wg + ti0, tj0 = tile_coords(t0) + bases0 = tile_bases(ti0, tj0) + G0 = scale_gather.tile_group(ti0 * BLOCK_M, tj0 * BLOCK_N) + if const_expr(salu_diet): + scale_gather.set_tile_voff(G0) + scale_gather.gather_soff(_gm0_s[0], 0) + scale_gather.gather_soff(_gm0_s[1], 256) + else: + scale_gather.gather(G0, 0, _slot(0)) + scale_gather.gather(G0, 1, _slot(1)) + s0 = _g2s_bases(bases0, 0) + s1 = _g2s_bases(bases0, 1) + a_g2s.load(a_cur0, s0[0]) + b_g2s.load(b_cur0, s0[2]) + b_g2s.load(b_cur1, s0[3]) + a_g2s.load(a_cur1, s0[1]) + a_g2s.load(a_next0, s1[0]) + b_g2s.load(b_next0, s1[2]) + b_g2s.load(b_next1, s1[3]) + a_g2s.load(a_next1, s1[1]) + # 2 gathers + 32 loads in flight: a_cur0 landed at vmcnt(28), b_cur0 / b_cur1 at 20 + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + zero4 = Vec.filled(4, 0.0, fx.Float32) + c10_prev = [zero4] * N_ACCUMS + c11_prev = [zero4] * N_ACCUMS + + # ---- the tile loop: state = the tile's step-0 fragments and scale block 0, the previous tile's c10 / c11 + # (stored during this tile's first step), the tile indices + tile_state = ( + [_R(x) for x in a0_frag] + [_R(x) for x in b0_frag] + [_R(x) for x in b1_frag] + _flat_sc(sc0) + + [_R(x) for x in c10_prev] + [_R(x) for x in c11_prev] + [_R(fx.Int32(t0)), _R(fx.Int32(-1))] + ) # fmt: skip + for jt, tstate in range(0, my_tiles, 1, init=tile_state): + off = 0 + a0f = list(tstate[off : off + n_a]) + off += n_a + b0f = list(tstate[off : off + n_b]) + off += n_b + b1f = list(tstate[off : off + n_b]) + off += n_b + sc = _unflat_sc(tstate[off : off + n_sc]) + off += n_sc + c10p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_cur = fx.Int32(tstate[off]) + t_prev = fx.Int32(tstate[off + 1]) + t_next = t_cur + grid + ti, tj = tile_coords(t_cur) + bases = tile_bases(ti, tj) + G = scale_gather.tile_group(ti * BLOCK_M, tj * BLOCK_N) + if const_expr(salu_diet): + scale_gather.set_tile_voff(G) + # the next tile (after the last tile one beyond the grid: its rows beyond M read zero, nothing is stored) + tin, tjn = tile_coords(t_next) + bases_n = tile_bases(tin, tjn) + Gn = scale_gather.tile_group(tin * BLOCK_M, tjn * BLOCK_N) + # the previous tile's c10 / c11 store bases (row beyond M when there is none) + tip, tjp = tile_coords(t_prev) + has_prev = t_prev >= fx.Int32(0) + prev_row1 = fx.arith.select( + has_prev, + tip * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M, + fx.Int32(c_m), + ) + prev_col0 = fx.arith.select( + has_prev, tjp * BLOCK_N + wave_j * (N_TILES_B * 16), fx.Int32(0) + ) + prev_col1 = fx.arith.select(has_prev, prev_col0 + LDS_BLOCK_N, fx.Int32(0)) + base_row = ti * BLOCK_M + wave_i * (N_TILES_A * 16) + base_col = tj * BLOCK_N + wave_j * (N_TILES_B * 16) + + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # step 0: zero accumulators, the previous tile's c10 / c11 stores spread over the groups before the new + # c10 (born in group 2) / c11 (group 3), gather block 1 + prev_thunks = store_c.thunks(c10p, prev_row1, prev_col0) + store_c.thunks( + c11p, prev_row1, prev_col1 + ) + n_prev3 = ( + len(prev_thunks) + 2 + ) // 3 # over the three groups before the new c11 + if const_expr(salu_diet): + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, _g2s_bases(bases, 2), (_gm0_s[1], 256), None, zero_acc=True, + extra=lambda g, c00, c01: prev_thunks[n_prev3 * g : n_prev3 * (g + 1)], + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, _g2s_bases(bases, 3), None, 1 + ) + # the four g2s streams advance by uniform strides and the gather block by 256 B: uniform + # accumulators, incremented per step, instead of base | (step << s) every K-step + ka0 = fx.Int32(bases[0]) + fx.Int32(4 * A_K_STEP) + ka1 = fx.Int32(bases[1]) + fx.Int32(4 * A_K_STEP) + kb0 = fx.Int32(bases[2]) + fx.Int32(4 * B_K_STEP) + kb1 = fx.Int32(bases[3]) + fx.Int32(4 * B_K_STEP) + kg = fx.Int32(512) # gather block soff at kk=2: block 2 << 8 + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg)] + ) # fmt: skip + else: + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, (bases, 2), (G, 1), None, zero_acc=True, + extra=lambda g, c00, c01: prev_thunks[n_prev3 * g : n_prev3 * (g + 1)], + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, (bases, 3), None, 1 + ) + + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + ) # fmt: skip + for kk, state in range(2, K_ITERS - 2, UNROLL, init=init_state): + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + bufs = bufs0 + if const_expr(salu_diet): + off += N_ACCUMS # past c11f: the uniform address accumulators + ka0 = fx.Int32(state[off]) + ka1 = fx.Int32(state[off + 1]) + kb0 = fx.Int32(state[off + 2]) + kb1 = fx.Int32(state[off + 3]) + kg = fx.Int32(state[off + 4]) + for u in range_constexpr(UNROLL): + if const_expr(salu_diet): + if const_expr(UNROLL == 4): + # kk == 2 (mod 4): u 0 gathers an even block (slot 0), u 2 an odd one (slot 1), so + # the gather slots fold; the read slots stay runtime + lds_off = (ka0, ka1, kb0, kb1) + gath = None + if const_expr(u % 2 == 0): + gath = ( + _gm0_s[u // 2], + kg if u == 0 else kg + fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + # a runtime value: a constant slot makes LLVM split the scale ds_read2st64 + # into 2 ds_read_b32 + rslot = ( + fx.Int32(kk) // fx.Int32(2) + fx.Int32(1 + u // 2) + ) % fx.Int32(2) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + fx.Int32(1) + lds_off = _g2s_bases(bases, kc + fx.Int32(2)) + gath = None + if const_expr(u % 2 == 0): + gath = ( + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + + (block % fx.Int32(2)) + * fx.Int32(_SCALE_SLOT_BYTES) + ), + block * fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + rslot = block % fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + lds_off, + gath, + rslot, + ) + ka0 = ka0 + fx.Int32(A_K_STEP) + ka1 = ka1 + fx.Int32(A_K_STEP) + kb0 = kb0 + fx.Int32(B_K_STEP) + kb1 = kb1 + fx.Int32(B_K_STEP) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, a0f, b0f, b1f, sc, accs, bufs, (bases, kc + fx.Int32(2)), (G, block + fx.Int32(1)), + block + fx.Int32(1), + ) # fmt: skip + if const_expr(salu_diet): + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg + fx.Int32(UNROLL * 128))] + ) # fmt: skip + else: + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + ) # fmt: skip + state = yield new_state + + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + # step K_ITERS - 2 (even): loads of the next tile's step 0, gather of its block 0 + if const_expr(salu_diet): + scale_gather.set_tile_voff(Gn) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, + a0f, + b0f, + b1f, + sc, + accs, + bufs0, + _g2s_bases(bases_n, 0), + (_gm0_s[0], 0), + None, + ) + else: + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, (bases_n, 0), (Gn, 0), None + ) + # step K_ITERS - 1 (odd): loads of the next tile's step 1, read of its block 0; this tile's c00 / c01 + # stored over the groups after their last MFMAs (c00: 1..3, c01: 2..3; the two quadrants' thunks stay + # concatenated, they share the LDS buffers) + n_q3 = ( + 2 * store_c.n_thunks + 2 + ) // 3 # thunks per group: two quadrants over 3 groups + + def _last_extra(g, c00, c01): + # compile-time Python only: no mutable state (the DSL rewriter turns body-level literals into + # None), every `if` under const_expr + if const_expr(g == 0): + return () + t00 = store_c.thunks(c00, base_row, base_col) + if const_expr(g == 1): + return t00[:n_q3] + t01 = store_c.thunks(c01, base_row, base_col + LDS_BLOCK_N) + if const_expr(g == 2): + return t00[n_q3:] + t01[: 2 * n_q3 - len(t00)] + return t01[2 * n_q3 - len(t00) :] + + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + _g2s_bases(bases_n, 1) if salu_diet else (bases_n, 1), + None, + 0, + extra=_last_extra, + ) + new_tile_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + [_R(t_next), _R(t_cur)] + ) # fmt: skip + tstate = yield new_tile_state + + # ---- epilogue: the last tile's c10 / c11 ---- + off = n_a + 2 * n_b + n_sc + c10_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_last = fx.Int32(tstate[off + 1]) + til, tjl = tile_coords(t_last) + last_row1 = til * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M + last_col0 = tjl * BLOCK_N + wave_j * (N_TILES_B * 16) + store_c.store(c10_last, last_row1, last_col0) + store_c.store(c11_last, last_row1, last_col0 + LDS_BLOCK_N) + + @flyc.kernel(name=name) + def kernel_gemm( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + ): + + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + workgroup_tile(lds.all_lds.ptr, lds.scale_lds.ptr, lds.out_lds.ptr, + fx.Int32(fx.block_idx.x), A, B_T, C, A_scale, B_scale, c_m, c_n, tile_off, n_work) + + @flyc.jit + def launch_shared_dense_mxfp8( + A: fx.Tensor, + B_T: fx.Tensor, + C: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + stream: fx.Stream, + ): + """``n_work`` work items from parent tile ``tile_off``; all: (0, ceil(M / 256) ceil(N / 256) SUB_SPLIT).""" + grid_x = _min(n_work, fx.Int32(GRID_MAX)) + kernel_gemm( + A, B_T, C, A_scale, B_scale, c_m, c_n, tile_off, n_work, + value_attrs={"rocdl.waves_per_eu": 1, "rocdl.flat_work_group_size": "256,256"}, + ).launch(grid=(grid_x, 1, 1), block=(256, 1, 1), stream=stream) # fmt: skip + + launch_shared_dense_mxfp8.kernel_name = name + launch_shared_dense_mxfp8.sub_split = SUB_SPLIT + launch_shared_dense_mxfp8.tile = workgroup_tile + launch_shared_dense_mxfp8.lds_bytes = (8 * _LDS_BUF_BYTES, SCALE_SLOTS * _SCALE_SLOT_BYTES, OUT_LDS_BYTES) + return launch_shared_dense_mxfp8 + + +@launcher +def compile_mxfp8_gemm_4w_gateup( + *, + K: int, + alpha: float, + beta: float, + limit: float, + use_xcd_remap: bool = True, + tiles: tuple = (4, 4), + grid_max: int = 256, + row_stationary: bool = True, + salu_diet: bool = True, + epilogue_type=StoreSwigluQ, + kernel_prefix: str = "flydsl_mxfp8_4wave_", + up_rows: int | None = None, +): + """The gate/up form of ``compile_mxfp8_gemm_4w``; the launcher's ``.tile`` is the workgroup body. + + The tile's two W halves are the gate rows [tj h, +h) and the up rows [N/2 + tj h, +h), h = BLOCK_N / 2 (a 16-row + remap, free on the preshuffled weight), and ``epilogue_type`` writes SwiGLU-OAI(gate, up) as MXFP8: e4m3 codes + row-major [M, N/2] plus e8m0 scales in AITER's shuffled layout, the down GEMM's A operand. N % 512 == 0. + + Args: + alpha, beta, limit: the SwiGLU constants, baked in. + epilogue_type: ``StoreSwigluQ`` or a subclass; ``kernel_prefix`` starts its kernel name. + use_xcd_remap, tiles, grid_max, row_stationary, salu_diet: as for ``compile_mxfp8_gemm_4w``. + up_rows: the up panel's first W row (and scale row) from the gate panel's, when W is a slice of a wider + [gate; up] weight (default N / 2). + """ + salu_diet = salu_diet and row_stationary + N_TILES_A, N_TILES_B = tiles + assert N_TILES_A in (2, 4) and N_TILES_B in (2, 4) + BLOCK_M = 64 * N_TILES_A + BLOCK_N = 64 * N_TILES_B + LDS_BLOCK_M = BLOCK_M // 2 + LDS_BLOCK_N = BLOCK_N // 2 + SUB_M = PARENT_M // BLOCK_M # sub-tiles per parent tile + SUB_N = PARENT_N // BLOCK_N + SUB_SPLIT = SUB_M * SUB_N + assert K % 256 == 0 + K_ITERS = K // BLOCK_K # even + KB = K // 256 # scale blocks + UNROLL = 4 if (K_ITERS - 4) % 4 == 0 else 2 + assert (K_ITERS - 4) % UNROLL == 0, K_ITERS + N_ACCUMS = N_TILES_A * N_TILES_B + assert ( + LDS_BLOCK_M * BLOCK_K <= _LDS_BUF_BYTES + and LDS_BLOCK_N * BLOCK_K <= _LDS_BUF_BYTES + ) + GRID_MAX = grid_max + order = "zigzag" if not row_stationary else ("rowstationary_salu" if salu_diet else "rowstationary") + name = (f"{kernel_prefix}{order}_k{K}" + ("" if grid_max == 256 else f"_g{grid_max}") + + (f"_t{N_TILES_A}x{N_TILES_B}" if tiles != (4, 4) else "") + "_gusq" + + (f"_up{up_rows}" if up_rows is not None else "")) # fmt: skip + assert up_rows is None or up_rows % 32 == 0 + SCALE_SLOTS = _SCALE_SLOTS + OUT_LDS_BYTES = _N_WAVES * 2 * 16 * 32 * N_TILES_B + + @fx.struct + class SharedStorage: + all_lds: fx.Array[fx.Int8, 8 * _LDS_BUF_BYTES, 16] + scale_lds: fx.Array[fx.Int8, SCALE_SLOTS * _SCALE_SLOT_BYTES, 16] + out_lds: fx.Array[ + fx.Int8, OUT_LDS_BYTES, 16 + ] # unused by the gate/up epilogue; keeps the plain kernel's LDS map and occupancy + + @flyc.jit + def workgroup_tile(base_ptr, scale_ptr, out_ptr, wg, + A: fx.Tensor, + B_T: fx.Tensor, + Q: fx.Tensor, + S: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + ): + _base_ptr = base_ptr + + def _buf(idx): + return _Buf(_base_ptr, idx * _LDS_BUF_BYTES) + + a_cur0, a_cur1, a_next0, a_next1 = _buf(0), _buf(1), _buf(2), _buf(3) + b_cur0, b_cur1, b_next0, b_next1 = _buf(4), _buf(5), _buf(6), _buf(7) + + lane_id = fx.thread_idx.x % 64 + wave_id = fx.thread_idx.x // 64 + n_pid_m = _ceildiv(c_m, PARENT_M) # the parent tiles' grid + n_blocks = _ceildiv(c_n, PARENT_N) + grid = _min(fx.Int32(n_work), fx.Int32(GRID_MAX)) + my_tiles = (fx.Int32(n_work) - wg + grid - fx.Int32(1)) // grid + wave_i = wave_id // 2 + wave_j = wave_id % 2 + A_K_STEP = BLOCK_K + B_K_STEP = 2 * 1024 # preshuffled (16, 16): a 16-row block spans 2 KB per 128 K + + def tile_coords(t): + """Work item -> (tile_i, tile_j) in this kernel's tile units, the parents in ``_xcd_swizzle``'s order. + + Item ``t = 8 q + x`` is sub-tile ``q % SUB_SPLIT`` of parent ``tile_off + 8 (q / SUB_SPLIT) + x`` + (``tile_off`` a multiple of 8), so a sub-tile launch keeps a workgroup on its XCD's parents; a negative or + out-of-range index lands beyond M, where the buffer bounds drop its stores.""" + if const_expr(use_xcd_remap and SUB_SPLIT > 1): + q, x = _divmod_nonneg(t, 8) + p_intra, sub = _divmod_nonneg(q, SUB_SPLIT) + parent = fx.Int32(tile_off) + p_intra * 8 + x + else: + parent, sub = _divmod_nonneg(t, SUB_SPLIT) + parent = fx.Int32(tile_off) + parent + if const_expr(use_xcd_remap): + pi, pj = _xcd_swizzle(n_pid_m, n_blocks, parent) + else: + pi, pj = parent // n_blocks, parent % n_blocks + si, sj = _divmod_nonneg(sub, SUB_N) + return pi * SUB_M + si, pj * SUB_N + sj + + def tile_bases(ti, tj): + """The g2s byte bases of a tile's two A halves (row-major) and its gate / up W halves (preshuffled).""" + gate0 = tj * LDS_BLOCK_N + return ( + (ti * BLOCK_M) * K, + (ti * BLOCK_M + LDS_BLOCK_M) * K, + gate0 * K, + (gate0 + (c_n // 2 if up_rows is None else up_rows)) * K, + ) + + mfma = ( + MfmaRowStationary(N_TILES_A, N_TILES_B) + if row_stationary + else Mfma16x16x128Fp8(N_TILES_A, N_TILES_B) + ) + _scale_base_ptr = scale_ptr + scale_gather = ScaleGatherLDS( + A_scale, + B_scale, + K, + lane_id, + wave_id, + _scale_base_ptr, + c_m, + c_n, + N_TILES_A, + N_TILES_B, + fused_gu=True, + up_rows=up_rows, + ) + scale_gather.set_wave_base() + if const_expr(salu_diet): + # the two scale slots' LDS addresses, once; the gather's K-block offset rides the uniform soffset + _gm0_s = ( + scale_gather._wave_base_s, + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + fx.Int32(_SCALE_SLOT_BYTES) + ), + ) + a_scale_ld = ScaleLoaderLDS( + N_TILES_A, lane_id, wave_i, _scale_base_ptr, _SCALE_A_REGION + ) + b_scale_ld = ScaleLoaderLDS( + N_TILES_B, lane_id, wave_j, _scale_base_ptr, _SCALE_B_REGION + ) + + def _slot(b): + return fx.Int32(b) % fx.Int32(SCALE_SLOTS) + + gl_off_a = _global_swizzle(lane_id, wave_id, K, N_TILES_A, False) + gl_off_b = _global_swizzle(lane_id, wave_id, K, N_TILES_B, True) + a_rsrc = _buffer_ops.create_buffer_resource( + A, max_size=False, num_records_bytes=c_m * K + ) + b_rsrc = _buffer_ops.create_buffer_resource( + B_T, max_size=False, num_records_bytes=(c_n if up_rows is None else c_n // 2 + up_rows) * K + ) + a_g2s = G2SLoaderAsm(a_rsrc, gl_off_a, N_TILES_A, wave_id) + b_g2s = G2SLoaderAsm(b_rsrc, gl_off_b, N_TILES_B, wave_id) + a_g2s.set_wave_base(_base_ptr) + b_g2s.set_wave_base(_base_ptr) + a_s2r = S2RLoader(wave_i, N_TILES_A) + b_s2r = S2RLoader(wave_j, N_TILES_B) + out_n = c_n // 2 + store_sq = epilogue_type( + Q, + S, + c_m, + out_n, + mfma.idx, + lane_id, + N_TILES_A, + N_TILES_B, + fx.Float32(alpha), + fx.Float32(beta), + fx.Float32(limit), + ) + + # a step issues [ac0 (n_a) | bc0 (n_b) | bc1 (n_b) | ac1 (n_a)] loads per wave: at a step's top the previous + # step's may all be in flight; mid-step the previous step's ac1 and this step's first two batches + _MAIN_VMCNT = 2 * N_TILES_A + 2 * N_TILES_B + _SEG2_VMCNT = 2 * N_TILES_A + N_TILES_B + + def _read_scale_thunks(block, holder): + # salu_diet: the caller passes the LDS slot itself (no block -> slot math) + s = block if salu_diet else _slot(block) + + def _r(dst, ld, half, _s=s): + holder[dst] = ld.read_half(_s, half) + + return [ + lambda: _r(0, a_scale_ld, 0), + lambda: _r(1, a_scale_ld, 1), + lambda: _r(2, b_scale_ld, 0), + lambda: _r(3, b_scale_ld, 1), + ] + + def _g2s_bases(bases, step): + a0, a1, b0, b1 = bases + st = fx.Int32(step) + return ( + fx.Int32(a0) + st * fx.Int32(A_K_STEP), + fx.Int32(a1) + st * fx.Int32(A_K_STEP), + fx.Int32(b0) + st * fx.Int32(B_K_STEP), + fx.Int32(b1) + st * fx.Int32(B_K_STEP), + ) + + def _one_step( + parity, + a0f, + b0f, + b1f_in, + sc, + accs, + bufs, + load, + gather, + read_block, + zero_acc=False, + extra=None, + ): + """One K-step: its MFMAs with the next step's LDS reads and the step after's global loads interleaved. + + Args: + load: ``(bases, step)``, or the (a0, a1, b0, b1) soffsets with ``salu_diet``. + gather: even steps: the next scale block's ``(G, block)``, or (m0, soff) with ``salu_diet``. + read_block: odd steps: the scale block read into the VGPR carry (its LDS slot with ``salu_diet``). + extra: ``extra(g, c00f, c01f)`` before group g = 0..3 (the accumulators finished so far, else None) + returns more thunks for that group: the epilogue of this tile's (c00, c01) pair in the last + step, of the previous tile's (c10, c11) pair in the first. + """ + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + saR0, saR1, sbC0, sbC1 = sc + c00f, c01f, c10f, c11f = accs + _a1 = [None] * N_TILES_A + _a0n = [None] * N_TILES_A + _b0n = [None] * N_TILES_B + _b1n = [None] * N_TILES_B + if const_expr(salu_diet): + a0_off, a1_off, b0_off, b1_off = load + else: + a0_off, a1_off, b0_off, b1_off = _g2s_bases(*load) + _scn = [None, None, None, None] + _rd_scn = _read_scale_thunks(read_block, _scn) if parity == 1 else [] + if const_expr(salu_diet): + _sc_gather = ( + [] + if parity == 1 + else [lambda: scale_gather.gather_soff(gather[0], gather[1])] + ) + else: + _sc_gather = ( + [] + if parity == 1 + else [ + lambda: scale_gather.gather( + gather[0], gather[1], _slot(gather[1]) + ) + ] + ) + ex = extra if extra is not None else (lambda g, c00, c01: ()) + + wait_barrier(_MAIN_VMCNT) + il = _riffle( + _g2s_thunks(a_g2s, ac0, a0_off, N_TILES_A), + _s2r_thunks(a_s2r, ac1, _a1, N_TILES_A, False), + ) + list(ex(0, None, None)) + c00f = mfma.call( + a0f, b0f, c00f, saR0, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + il = _riffle( + _g2s_thunks(b_g2s, bc0, b0_off, N_TILES_B), list(ex(1, c00f, None)) + ) + c01f = mfma.call( + a0f, b1f_in, c01f, saR0, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + a1f = _a1 + + wait_barrier(_SEG2_VMCNT) + il = (_riffle(_g2s_thunks(b_g2s, bc1, b1_off, N_TILES_B), _s2r_thunks(a_s2r, an0, _a0n, N_TILES_A, False)) + + _sc_gather + _rd_scn[:2] + list(ex(2, c00f, c01f))) # fmt: skip + c10f = mfma.call( + a1f, b0f, c10f, saR1, sbC0, parity, interleave=il, zero_acc=zero_acc + ) + a0nf = _a0n + il = ( + _riffle( + _g2s_thunks(a_g2s, ac1, a1_off, N_TILES_A), + _s2r_thunks(b_s2r, bn0, _b0n, N_TILES_B, True) + + _s2r_thunks(b_s2r, bn1, _b1n, N_TILES_B, True), + ) + + _rd_scn[2:] + + list(ex(3, c00f, c01f)) + ) + c11f = mfma.call( + a1f, b1f_in, c11f, saR1, sbC1, parity, interleave=il, zero_acc=zero_acc + ) + b0nf, b1nf = _b0n, _b1n + + sc_next = (_scn[0], _scn[1], _scn[2], _scn[3]) if parity == 1 else sc + new_bufs = (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + return a0nf, b0nf, b1nf, sc_next, (c00f, c01f, c10f, c11f), new_bufs + + bufs0 = (a_cur0, a_cur1, a_next0, a_next1, b_cur0, b_cur1, b_next0, b_next1) + + def _swap_bufs(bufs): + ac0, ac1, an0, an1, bc0, bc1, bn0, bn1 = bufs + return (an0, an1, ac0, ac1, bn0, bn1, bc0, bc1) + + n_a = N_TILES_A + n_b = N_TILES_B + n_ga = N_TILES_A // _PACK + n_gb = N_TILES_B // _PACK + n_sc = 2 * n_ga + 2 * n_gb + _R = fx.as_ir_value + + def _flat_sc(sc): + saR0, saR1, sbC0, sbC1 = sc + return ( + [_R(v) for v in saR0] + + [_R(v) for v in saR1] + + [_R(v) for v in sbC0] + + [_R(v) for v in sbC1] + ) + + def _unflat_sc(flat): + o = 0 + saR0 = list(flat[o : o + n_ga]) + o += n_ga + saR1 = list(flat[o : o + n_ga]) + o += n_ga + sbC0 = list(flat[o : o + n_gb]) + o += n_gb + sbC1 = list(flat[o : o + n_gb]) + return (saR0, saR1, sbC0, sbC1) + + # ---- prologue: the first tile's steps 0 / 1 and its scale blocks 0 / 1 ---- + t0 = wg + ti0, tj0 = tile_coords(t0) + bases0 = tile_bases(ti0, tj0) + G0 = scale_gather.tile_group(ti0 * BLOCK_M, tj0 * BLOCK_N) + if const_expr(salu_diet): + scale_gather.set_tile_voff(G0) + scale_gather.gather_soff(_gm0_s[0], 0) + scale_gather.gather_soff(_gm0_s[1], 256) + else: + scale_gather.gather(G0, 0, _slot(0)) + scale_gather.gather(G0, 1, _slot(1)) + s0 = _g2s_bases(bases0, 0) + s1 = _g2s_bases(bases0, 1) + a_g2s.load(a_cur0, s0[0]) + b_g2s.load(b_cur0, s0[2]) + b_g2s.load(b_cur1, s0[3]) + a_g2s.load(a_cur1, s0[1]) + a_g2s.load(a_next0, s1[0]) + b_g2s.load(b_next0, s1[2]) + b_g2s.load(b_next1, s1[3]) + a_g2s.load(a_next1, s1[1]) + # 2 gathers + 32 loads in flight: a_cur0 landed at vmcnt(28), b_cur0 / b_cur1 at 20 + wait_barrier((3 * N_TILES_A) + (4 * N_TILES_B)) + a0_frag = a_s2r.load(a_cur0) + wait_barrier((3 * N_TILES_A) + (2 * N_TILES_B)) + b0_frag = b_s2r.load(b_cur0, preshuffled=True) + b1_frag = b_s2r.load(b_cur1, preshuffled=True) + sc0_saR0, sc0_saR1 = a_scale_ld.read(_slot(0)) + sc0_sbC0, sc0_sbC1 = b_scale_ld.read(_slot(0)) + sc0 = (sc0_saR0, sc0_saR1, sc0_sbC0, sc0_sbC1) + zero4 = Vec.filled(4, 0.0, fx.Float32) + c10_prev = [zero4] * N_ACCUMS + c11_prev = [zero4] * N_ACCUMS + + # ---- the tile loop: state = the tile's step-0 fragments and scale block 0, the previous tile's c10 / c11 + # (their epilogue runs during this tile's first step), the tile indices + tile_state = ( + [_R(x) for x in a0_frag] + [_R(x) for x in b0_frag] + [_R(x) for x in b1_frag] + _flat_sc(sc0) + + [_R(x) for x in c10_prev] + [_R(x) for x in c11_prev] + [_R(fx.Int32(t0)), _R(fx.Int32(-1))] + ) # fmt: skip + for jt, tstate in range(0, my_tiles, 1, init=tile_state): + off = 0 + a0f = list(tstate[off : off + n_a]) + off += n_a + b0f = list(tstate[off : off + n_b]) + off += n_b + b1f = list(tstate[off : off + n_b]) + off += n_b + sc = _unflat_sc(tstate[off : off + n_sc]) + off += n_sc + c10p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11p = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_cur = fx.Int32(tstate[off]) + t_prev = fx.Int32(tstate[off + 1]) + t_next = t_cur + grid + ti, tj = tile_coords(t_cur) + bases = tile_bases(ti, tj) + G = scale_gather.tile_group(ti * BLOCK_M, tj * BLOCK_N) + if const_expr(salu_diet): + scale_gather.set_tile_voff(G) + # the next tile (after the last tile one beyond the grid: its rows beyond M read zero, nothing is stored) + tin, tjn = tile_coords(t_next) + bases_n = tile_bases(tin, tjn) + Gn = scale_gather.tile_group(tin * BLOCK_M, tjn * BLOCK_N) + # the previous tile's pair window (rows beyond M when there is none) + tip, tjp = tile_coords(t_prev) + has_prev = t_prev >= fx.Int32(0) + prev_row1 = fx.arith.select( + has_prev, + tip * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M, + fx.Int32(c_m), + ) + prev_col0 = fx.arith.select( + has_prev, tjp * LDS_BLOCK_N + wave_j * (N_TILES_B * 16), fx.Int32(0) + ) + base_row = ti * BLOCK_M + wave_i * (N_TILES_A * 16) + base_col = tj * LDS_BLOCK_N + wave_j * (N_TILES_B * 16) + + accs = ( + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + [None] * N_ACCUMS, + ) + # step 0: zero accumulators; the previous tile's (c10, c11) epilogue: the accumulator reads in groups 0-1 + # (the group-2 MFMAs zero the new c10, group 3 c11), the VGPR math + stores in groups 2-3; gather block 1 + prev_r, prev_m = store_sq.thunks_rm(c10p, c11p, prev_row1, prev_col0) + n_r1 = (len(prev_r) + 1) // 2 + n_m1 = (len(prev_m) + 1) // 2 + + def _prev_extra(g, c00, c01, _r=prev_r, _m=prev_m, _nr=n_r1, _nm=n_m1): + if const_expr(g == 0): + return _r[:_nr] + if const_expr(g == 1): + return _r[_nr:] + if const_expr(g == 2): + return _m[:_nm] + return _m[_nm:] + + if const_expr(salu_diet): + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, _g2s_bases(bases, 2), (_gm0_s[1], 256), None, zero_acc=True, + extra=_prev_extra, + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, _g2s_bases(bases, 3), None, 1 + ) + # the four g2s streams advance by uniform strides and the gather block by 256 B: uniform + # accumulators, incremented per step, instead of base | (step << s) every K-step + ka0 = fx.Int32(bases[0]) + fx.Int32(4 * A_K_STEP) + ka1 = fx.Int32(bases[1]) + fx.Int32(4 * A_K_STEP) + kb0 = fx.Int32(bases[2]) + fx.Int32(4 * B_K_STEP) + kb1 = fx.Int32(bases[3]) + fx.Int32(4 * B_K_STEP) + kg = fx.Int32(512) # gather block soff at kk=2: block 2 << 8 + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg)] + ) # fmt: skip + else: + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, (bases, 2), (G, 1), None, zero_acc=True, + extra=_prev_extra, + ) # fmt: skip + # step 1: read block 1 + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, a0f, b0f, b1f, sc, accs, bufs, (bases, 3), None, 1 + ) + + init_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + ) # fmt: skip + for kk, state in range(2, K_ITERS - 2, UNROLL, init=init_state): + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + bufs = bufs0 + if const_expr(salu_diet): + off += N_ACCUMS # past c11f: the uniform address accumulators + ka0 = fx.Int32(state[off]) + ka1 = fx.Int32(state[off + 1]) + kb0 = fx.Int32(state[off + 2]) + kb1 = fx.Int32(state[off + 3]) + kg = fx.Int32(state[off + 4]) + for u in range_constexpr(UNROLL): + if const_expr(salu_diet): + if const_expr(UNROLL == 4): + # kk == 2 (mod 4): u 0 gathers an even block (slot 0), u 2 an odd one (slot 1), so + # the gather slots fold; the read slots stay runtime + lds_off = (ka0, ka1, kb0, kb1) + gath = None + if const_expr(u % 2 == 0): + gath = ( + _gm0_s[u // 2], + kg if u == 0 else kg + fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + # a runtime value: a constant slot makes LLVM split the scale ds_read2st64 + # into 2 ds_read_b32 + rslot = ( + fx.Int32(kk) // fx.Int32(2) + fx.Int32(1 + u // 2) + ) % fx.Int32(2) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + fx.Int32(1) + lds_off = _g2s_bases(bases, kc + fx.Int32(2)) + gath = None + if const_expr(u % 2 == 0): + gath = ( + fx.as_ir_value( + fx.Int32(scale_gather._wave_base_s) + + (block % fx.Int32(2)) + * fx.Int32(_SCALE_SLOT_BYTES) + ), + block * fx.Int32(256), + ) + rslot = None + if const_expr(u % 2 == 1): + rslot = block % fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + lds_off, + gath, + rslot, + ) + ka0 = ka0 + fx.Int32(A_K_STEP) + ka1 = ka1 + fx.Int32(A_K_STEP) + kb0 = kb0 + fx.Int32(B_K_STEP) + kb1 = kb1 + fx.Int32(B_K_STEP) + else: + kc = fx.Int32(kk) + fx.Int32(u) + block = kc // fx.Int32(2) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + u % 2, a0f, b0f, b1f, sc, accs, bufs, (bases, kc + fx.Int32(2)), (G, block + fx.Int32(1)), + block + fx.Int32(1), + ) # fmt: skip + if const_expr(salu_diet): + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + + [_R(ka0), _R(ka1), _R(kb0), _R(kb1), _R(kg + fx.Int32(UNROLL * 128))] + ) # fmt: skip + else: + new_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[0]] + [_R(x) for x in accs[1]] + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + ) # fmt: skip + state = yield new_state + + off = 0 + a0f = list(state[off : off + n_a]) + off += n_a + b0f = list(state[off : off + n_b]) + off += n_b + b1f = list(state[off : off + n_b]) + off += n_b + sc = _unflat_sc(state[off : off + n_sc]) + off += n_sc + c00f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c01f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c10f = list(state[off : off + N_ACCUMS]) + off += N_ACCUMS + c11f = list(state[off : off + N_ACCUMS]) + accs = (c00f, c01f, c10f, c11f) + # step K_ITERS - 2 (even): loads of the next tile's step 0, gather of its block 0 + if const_expr(salu_diet): + scale_gather.set_tile_voff(Gn) + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, + a0f, + b0f, + b1f, + sc, + accs, + bufs0, + _g2s_bases(bases_n, 0), + (_gm0_s[0], 0), + None, + ) + else: + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 0, a0f, b0f, b1f, sc, accs, bufs0, (bases_n, 0), (Gn, 0), None + ) + + # step K_ITERS - 1 (odd): loads of the next tile's step 1, read of its block 0; this tile's (c00, c01) + # epilogue in groups 2-3 (c00 is final after group 0, c01 after group 1) + def _last_extra(g, c00, c01): + # compile-time Python only: no mutable state (the DSL rewriter turns body-level literals into + # None), every `if` under const_expr + if const_expr(g < 2): + return () + ts = store_sq.thunks(c00, c01, base_row, base_col) + n_h1 = (len(ts) + 1) // 2 + if const_expr(g == 2): + return ts[:n_h1] + return ts[n_h1:] + + a0f, b0f, b1f, sc, accs, bufs = _one_step( + 1, + a0f, + b0f, + b1f, + sc, + accs, + bufs, + _g2s_bases(bases_n, 1) if salu_diet else (bases_n, 1), + None, + 0, + extra=_last_extra, + ) + new_tile_state = ( + [_R(x) for x in a0f] + [_R(x) for x in b0f] + [_R(x) for x in b1f] + _flat_sc(sc) + + [_R(x) for x in accs[2]] + [_R(x) for x in accs[3]] + [_R(t_next), _R(t_cur)] + ) # fmt: skip + tstate = yield new_tile_state + + # ---- epilogue: the last tile's c10 / c11 pair ---- + off = n_a + 2 * n_b + n_sc + c10_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + c11_last = list(tstate[off : off + N_ACCUMS]) + off += N_ACCUMS + t_last = fx.Int32(tstate[off + 1]) + til, tjl = tile_coords(t_last) + last_row1 = til * BLOCK_M + wave_i * (N_TILES_A * 16) + LDS_BLOCK_M + last_col0 = tjl * LDS_BLOCK_N + wave_j * (N_TILES_B * 16) + for t in store_sq.thunks(c10_last, c11_last, last_row1, last_col0): + t() + + @flyc.kernel(name=name) + def kernel_gemm_gusq( + A: fx.Tensor, + B_T: fx.Tensor, + Q: fx.Tensor, + S: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + ): + + lds = fx.SharedAllocator().allocate(SharedStorage).peek() + workgroup_tile(lds.all_lds.ptr, lds.scale_lds.ptr, lds.out_lds.ptr, + fx.Int32(fx.block_idx.x), A, B_T, Q, S, A_scale, B_scale, c_m, c_n, tile_off, n_work) + + @flyc.jit + def launch_shared_dense_gateup_mxfp8( + A: fx.Tensor, + B_T: fx.Tensor, + Q: fx.Tensor, + S: fx.Tensor, + A_scale: fx.Tensor, + B_scale: fx.Tensor, + c_m: fx.Int32, + c_n: fx.Int32, + tile_off: fx.Int32, + n_work: fx.Int32, + stream: fx.Stream, + ): + """``n_work`` work items from parent tile ``tile_off``; all: (0, ceil(M / 256) ceil(N / 256) SUB_SPLIT).""" + grid_x = _min(n_work, fx.Int32(GRID_MAX)) + kernel_gemm_gusq( + A, B_T, Q, S, A_scale, B_scale, c_m, c_n, tile_off, n_work, + value_attrs={"rocdl.waves_per_eu": 1, "rocdl.flat_work_group_size": "256,256"}, + ).launch(grid=(grid_x, 1, 1), block=(256, 1, 1), stream=stream) # fmt: skip + + launch_shared_dense_gateup_mxfp8.kernel_name = name + launch_shared_dense_gateup_mxfp8.sub_split = SUB_SPLIT + launch_shared_dense_gateup_mxfp8.tile = workgroup_tile + launch_shared_dense_gateup_mxfp8.lds_bytes = (8 * _LDS_BUF_BYTES, SCALE_SLOTS * _SCALE_SLOT_BYTES, OUT_LDS_BYTES) + return launch_shared_dense_gateup_mxfp8 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prefill_h6144_i1536.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prefill_h6144_i1536.py new file mode 100644 index 0000000..a1609f4 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prefill_h6144_i1536.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""What the BM128 and BM256 expert hosts share (H 6144, I 1536 per rank): ``Mxfp4PrefillScales``, the expert scales every +host reads (the weights are the bytes of ``layout``, one copy per layer), and GEMM2's column chunks and grid. +""" + +from dataclasses import dataclass + +import torch + +from .common.epilogue import _NUM_XCDS + +H, I = 6144, 1536 # noqa: E741 + +# GEMM2's column chunks per m-tile at every row count: 4 n-tiles of 256 columns each +GEMM2_N_SPLIT = 6 + + +def gemm2_grid(num_m_blocks: int, n_split: int) -> int: + """Workgroups to launch: every (m-tile, chunk) of the allocation rounded up to the 8 XCDs (the kernel drops the + fully padded tail tiles at run time).""" + return (num_m_blocks * n_split + _NUM_XCDS - 1) // _NUM_XCDS * _NUM_XCDS + + +@dataclass +class Mxfp4PrefillScales: + """A layer's routed-expert e8m0 scales for the FP4 kernels: W13 gate / up interleaved and aiter-shuffled + (``i3072.gemm1_prefill_h6144_i3072_persistent.interleave_w13_scales``), W2 aiter-shuffled (``e8m0_shuffle``).""" + + w13: torch.Tensor + w2: torch.Tensor + + @classmethod + def from_row_major( + cls, w13_scale: torch.Tensor, w2_scale: torch.Tensor + ) -> "Mxfp4PrefillScales": + """From the checkpoint's row-major e8m0 ``[E, 2I, H/32]`` (gate rows, then up rows) and ``[E, H, I/32]``.""" + from aiter.utility.fp4_utils import e8m0_shuffle + + from .i3072.gemm1_prefill_h6144_i3072_persistent import interleave_w13_scales + + e = w13_scale.shape[0] + w13 = e8m0_shuffle(w13_scale.reshape(e * 2 * I, -1)).view(e, 2 * I, -1) + w2 = e8m0_shuffle(w2_scale.reshape(e * H, -1)).view(e, H, -1) + return cls(interleave_w13_scales(w13.view(torch.uint8)), w2.view(torch.uint8)) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prepared_input.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prepared_input.py new file mode 100644 index 0000000..095a5e5 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/prepared_input.py @@ -0,0 +1,234 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MoE entry's sequence-parallel input: ``prepare_mxfp4_moe`` quantizes and routes a local bf16 shard; the caller +concatenates the row-major payloads of every rank in rank order (padding rows included) and builds the matching input +type from them. Both formats quantize straight from the bf16 input; the compact one carries the FP4 scales and an exact +correction mask instead of the FP4 codes. The routed-only input (``MoeInput.routed``, 3296 B/token) carries the routed +operands alone: the owner of the rows runs the full shared expert on them (``shared_expert_full``). +""" +from dataclasses import dataclass + +import torch + +from amd_kernels.warm_up import entry + + +@dataclass(frozen=True) +class MoeInput: + """Quantized operands and routing, no bf16 activations or expert weights. + + ``tensors()`` returns the gather payload in constructor order, without copies. The scales are e8m0 bytes per 32 + values, the routing weights include the routed scaling factor; the shared payloads are the shared expert's MXFP8 + input. + """ + routed_codes: torch.Tensor + routed_scales: torch.Tensor + topk_weights: torch.Tensor + topk_ids: torch.Tensor + shared_codes: torch.Tensor | None = None + shared_scales: torch.Tensor | None = None + + def tensors(self): + return (self.routed_codes, self.routed_scales, self.topk_weights, self.topk_ids, self.shared_codes, + self.shared_scales) + + @property + def routed_only(self) -> bool: + return self.shared_codes is None and self.shared_scales is None + + def routed(self): + """The routed operands alone (shared fields None): the 3296 B/token gather payload.""" + return MoeInput(self.routed_codes, self.routed_scales, self.topk_weights, self.topk_ids) + + def compact(self): + """The 7328 B/token payload (packing costs a launch); release this object's FP4 codes after packing.""" + from .sidecar_pack import pack_mxfp4_sidecar + if self.shared_codes is None or self.shared_scales is None: + raise ValueError("compact input requires shared MXFP8 operands") + if not isinstance(self.routed_codes,torch.Tensor) or self.routed_codes.ndim!=2: + raise ValueError("expected routed codes [M,3072]") + n=self.routed_codes.shape[0] + compact=MoeInputCompact(self.routed_scales,self.topk_weights,self.topk_ids, + self.shared_codes,self.shared_scales,None) + compact._validate(check_mask=False) + if (self.routed_codes.shape!=(n,3072) or self.routed_codes.dtype!=torch.uint8 + or not self.routed_codes.is_contiguous() + or self.routed_codes.device!=self.shared_codes.device): + raise ValueError("expected contiguous uint8 routed codes [M,3072]") + mask=pack_mxfp4_sidecar(self.routed_codes,self.routed_scales, + self.shared_codes,self.shared_scales) + return MoeInputCompact(*compact.tensors()[:-1],mask) + + def validate(self, w): + from .chain import H, TOPK, variant_for + if not isinstance(self.routed_codes, torch.Tensor) or self.routed_codes.ndim != 2: + raise ValueError("routed_codes must be row-major [M,3072]") + n = self.routed_codes.shape[0] + variant_for(n) + fields = ((self.routed_codes, (n, H // 2), torch.uint8), + (self.routed_scales, (n, H // 32), torch.uint8), + (self.topk_weights, (n, TOPK), torch.float32), + (self.topk_ids, (n, TOPK), torch.int32)) + if not self.routed_only: + fields += ((self.shared_codes, (n, H), torch.uint8), (self.shared_scales, (n, H // 32), torch.uint8)) + device = w.router_weight.device + for tensor, shape, dtype in fields: + if (not isinstance(tensor, torch.Tensor) or tensor.shape != shape or tensor.dtype != dtype + or not tensor.is_contiguous() or tensor.device != device): + raise ValueError(f"expected contiguous {dtype} {shape} on {device}") + + +@dataclass(frozen=True) +class MoeInputCompact: + """The lossless gather payload, in ``tensors()`` order, all row-major. + + The FP4 scales and one correction bit per value rebuild the direct FP4 codes from the shared FP8 ones (signed zero + and the quantization kept): 2304 B/token less than ``MoeInput`` with a shared expert, at one launch to pack and one + to rebuild. + """ + routed_scales: torch.Tensor + topk_weights: torch.Tensor + topk_ids: torch.Tensor + shared_codes: torch.Tensor + shared_scales: torch.Tensor + correction_mask: torch.Tensor + + def tensors(self): + return (self.routed_scales,self.topk_weights,self.topk_ids,self.shared_codes, + self.shared_scales,self.correction_mask) + + def _validate(self, *, check_mask=True): + from .chain import variant_for + if not isinstance(self.shared_codes,torch.Tensor) or self.shared_codes.ndim!=2: + raise ValueError("expected shared codes [M,6144]") + n=self.shared_codes.shape[0] + if self.shared_codes.device.type!="cuda": + raise ValueError("compact input requires CUDA operands") + variant_for(n) + specs=(((n,192),torch.uint8),((n,4),torch.float32),((n,4),torch.int32), + ((n,6144),torch.uint8),((n,192),torch.uint8),((n,768),torch.uint8)) + for t,(shape,dtype) in zip(self.tensors() if check_mask else self.tensors()[:-1],specs): + if (not isinstance(t,torch.Tensor) or t.shape!=shape or t.dtype!=dtype + or not t.is_contiguous() or t.device!=self.shared_codes.device): + raise ValueError(f"expected contiguous {dtype} {shape} on {self.shared_codes.device}") + + def expand(self): + """Reconstruct FP4 codes bit for bit; keep routing, scales and shared codes.""" + from .sidecar_unpack import unpack_mxfp4_sidecar + self._validate() + q=unpack_mxfp4_sidecar(self.correction_mask,self.routed_scales, + self.shared_codes,self.shared_scales) + return MoeInput(q,self.routed_scales,self.topk_weights,self.topk_ids, + self.shared_codes,self.shared_scales) + + +def prepare_mxfp4_moe(x, w): + """One local bf16 [M, 6144] shard prepared before the caller's all-gather. + + The router MFMAs read bf16, the routed experts MXFP4 and the shared expert MXFP8, with the unsharded entry's + quantizers at every M; both scales are row-major, even if M is not a multiple of 32. Returns only tensor payloads; + the caller owns the gather and the fused tail after it. + """ + from .chain import H, ROUTER_QUANT_FROM, _check_weights, _route_parts, _route_quant_small, variant_for + if (not isinstance(x, torch.Tensor) or x.ndim != 2 or x.shape[1] != H + or x.dtype != torch.bfloat16 or not x.is_contiguous()): + raise ValueError("expected contiguous BF16 [M,6144]") + n = x.shape[0] + variant_for(n) + _check_weights(w) + if n < ROUTER_QUANT_FROM: + (q, s), (q8, s8), wt, ids = _route_quant_small(x, w) + else: + from .router_mxfp4_mxfp8_quant_row_scales import router_mxfp4_mxfp8_quant_row_scales + (q, s), (q8, s8), parts = router_mxfp4_mxfp8_quant_row_scales(x, w.router_weight_lines) + wt, ids = _route_parts(parts, w) + return MoeInput(q, s.view(torch.uint8), wt, ids, q8, s8) + + +def _prepare_mode(x, w): + """What a preparation fixes about its kernels besides the token count (``amd_kernels.warm_up``).""" + return w.shared_weights is not None, w.router_weight_lines is not None + + +def _prepare_sizes(mode): + from .chain import WARM_UP_TOKENS + return WARM_UP_TOKENS + + +def _prepare_inputs(mode, n, x, w): + """One warm-up call at ``n`` tokens (zeros: the dispatch depends on shapes only).""" + from .chain import H + return (torch.zeros((n, H), device=w.router_weight.device, dtype=torch.bfloat16), w), {} + + +prepare_mxfp4_moe = entry("moe/mxfp4_h6144_i1536_e128_top4:prepare", mode=_prepare_mode, sizes=_prepare_sizes, + inputs=_prepare_inputs)(prepare_mxfp4_moe) + + +def _mxfp4_moe_prepared(x, w, *, return_partials): + from .chain import _small_experts, mxfp4_moe_experts, variant_for + x.validate(w) + q4 = (x.routed_codes, x.routed_scales) + if x.routed_only: + # the routed launches alone, the BF16 entry's routed hosts by rows (its slots bit for bit) + if not return_partials: + raise ValueError("a routed-only input has no shared expert: call with return_partials=True") + experts = _small_experts if variant_for(x.routed_codes.shape[0]) == "a4w4_small" else mxfp4_moe_experts + return experts(q4, x.topk_weights, x.topk_ids, w), None + from .shared_expert import shared_expert, shared_expert_parallel, shared_folded_for, shared_parallel_for + n = x.routed_codes.shape[0] + sw = w.shared_weights + q8 = (x.shared_codes, x.shared_scales) + # the BF16 entry's schedule (its bits): below 96 rows the shared tiles lead the routed small-tile launches + small = variant_for(n) == "a4w4_small" + folded = small and shared_folded_for(n) + parallel = shared_parallel_for(n) and not folded + current = torch.cuda.current_stream(x.routed_codes.device) + if folded: + from .experts_a4w4_small import a4w4_experts + parts, shared = a4w4_experts(q4, w.w13, w.w2, w.a4w4_scales, x.topk_weights, x.topk_ids, shared=(*q8, sw)) + else: + if parallel: + shared = shared_expert_parallel(x.shared_codes, sw, current, x_quant=q8) + else: + shared = shared_expert(x.shared_codes, sw, x_quant=q8) + experts = _small_experts if small else mxfp4_moe_experts + parts = experts(q4, x.topk_weights, x.topk_ids, w) + if parallel: + current.wait_stream(sw.stream) + shared.record_stream(current) + if return_partials: + return parts, shared + from .common.reduce_slots_shared import reduce_slots_shared + return reduce_slots_shared(parts, shared) + + +# gathered rows at most: the entry's prefill chunk (16384 rows per rank at two ranks, 8192 at four) +GATHER_MAX_ROWS = 32768 + + +def gather_mxfp4_moe(local, comm, *, world=2, routed_only=False): + """A prepared shard gathered losslessly for the same public MoE entry: compact (``MoeInputCompact``), or with + ``routed_only`` the routed operands alone (a routed-only ``MoeInput``: codes, scales, top-4 weights and ids, 3296 + B/token, no packing launch). + + Args: + local: ``prepare_mxfp4_moe``'s explicit result (compact: with shared operands); every rank supplies the same + positive row count, zero padding included. + comm: a warmed ``AllGatherComm`` on the ``world``-rank TP group whose outputs concatenate the ranks along + the first axis (it owns the IPC staging; this helper owns its result buffers). + """ + if not isinstance(local, MoeInput): + raise ValueError("expected an explicit prepared MoeInput") + rows = local.routed_codes.shape[0] + if rows * world > GATHER_MAX_ROWS: + raise ValueError(f"the prepared gather takes at most {GATHER_MAX_ROWS // world} rows per rank at {world} ranks") + if routed_only: + fields = local.routed().tensors()[:4] + else: + if local.shared_codes is None or local.shared_scales is None: + raise ValueError("lossless mixed-format gather requires shared MXFP8 operands") + fields = local.compact().tensors() + outs = [torch.empty((world*rows,*t.shape[1:]),dtype=t.dtype,device=t.device) for t in fields] + comm.all_gather(outs,list(fields)) + return MoeInput(*outs) if routed_only else MoeInputCompact(*outs) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/quant.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/quant.py new file mode 100644 index 0000000..7276c05 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/quant.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MXFP4 activation quant: bf16 [M, K] -> e2m1 codes [M, K/2] (low nibble first) + e8m0 scales [M, K/32] (token +order, what ``scale_sort_tile_map`` sorts), all row-major. One lane per 32-value block: four 16-B loads (a wave reads +4 KB contiguous), the block's |max|, Quark even e8m0 (floored at 1), RNE e2m1 by ``v_cvt_scalef32_pk_fp4_f32``, one +16-B store of codes and one scale byte. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +from .common.launch import _run_compiled +from .common.loaders import _as_f32 +from .mxfp8.gemm1_prefill_h6144_i3072 import ( + _fmax3_abs, +) + +from .i3072.gemm1_prefill_h6144_i3072_persistent import _cvt_pk_fp4, _e8m0_even_e2m1 +from amd_kernels.warm_up import launcher + +_THREADS = 256 + + +@functools.cache +@launcher +def compile_mxfp4_quant(): + @flyc.kernel(name="mxfp4_quant_even") + def kernel_mxfp4_quant( + X: fx.Tensor, Q: fx.Tensor, S: fx.Tensor, n_blocks: fx.Int32 + ): + blk = fx.block_idx.x * _THREADS + fx.thread_idx.x + x_rsrc = buffer_ops.create_buffer_resource( + X, max_size=False, num_records_bytes=n_blocks * 64 + ) + q_rsrc = buffer_ops.create_buffer_resource( + Q, max_size=False, num_records_bytes=n_blocks * 16 + ) + s_rsrc = buffer_ops.create_buffer_resource( + S, max_size=False, num_records_bytes=n_blocks + ) + vals = [] + for j in range_constexpr(4): + v = Vec( + buffer_ops.buffer_load( + x_rsrc, blk * 16 + j * 4, vec_width=4, dtype=fx.Int32 + ) + ) + for k in range_constexpr(4): + w = fx.Int32(v[k]) + vals.append(_as_f32(w << 16)) + vals.append(_as_f32(w & fx.Int32(-65536))) + amax = _fmax3_abs(vals[0], vals[1], vals[2]) + for i in range_constexpr(3, 31, 2): + amax = _fmax3_abs(vals[i], vals[i + 1], amax) + amax = _fmax3_abs(vals[31], vals[31], amax) + e8m0 = _e8m0_even_e2m1(amax) + scale_f = _as_f32(e8m0 << 23) + out = [] + for d in range_constexpr(4): + p = fx.Int32(0) + for k in range_constexpr(4): + p = _cvt_pk_fp4( + p, vals[8 * d + 2 * k], vals[8 * d + 2 * k + 1], scale_f, k + ) + out.append(p) + buffer_ops.buffer_store(Vec.from_elements(out, fx.Int32), q_rsrc, blk * 4) + buffer_ops.buffer_store(fx.Int32(e8m0).to(fx.Int8), s_rsrc, blk) + + @flyc.jit + def launch_mxfp4_quant_i1536( + X: fx.Tensor, + Q: fx.Tensor, + S: fx.Tensor, + n_blocks: fx.Int32, + grid: fx.Int32, + stream: fx.Stream, + ): + kernel_mxfp4_quant(X, Q, S, n_blocks).launch( + grid=(grid, 1, 1), block=(_THREADS, 1, 1), stream=stream + ) + + return launch_mxfp4_quant_i1536 + + +def mxfp4_quant(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + m, k = x.shape + assert x.dtype == torch.bfloat16 and x.is_contiguous() and k % 32 == 0 + q = torch.empty((m, k // 2), dtype=torch.uint8, device=x.device) + s = torch.empty((m, k // 32), dtype=torch.uint8, device=x.device) + n_blocks = m * (k // 32) + _run_compiled( + compile_mxfp4_quant(), + x.view(-1), + q.view(-1), + s.view(-1), + n_blocks, + (n_blocks + _THREADS - 1) // _THREADS, + torch.cuda.current_stream(), + ) + return q, s diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/reference.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/reference.py new file mode 100644 index 0000000..bb92615 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/reference.py @@ -0,0 +1,235 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Torch references of this entry's kernels: the contract the tests check (never called in serving). + +MXFP4: e2m1 codes packed two per byte along K (low nibble first), one e8m0 scale per 32 K. The activation quant +(``quant.mxfp4_quant``, GEMM1's intermediate quant, ``mxfp4_from_mxfp8``) uses Quark's even rule: e8m0 = +exponent(amax + half an e2m1 mantissa ulp) - 2, at least 1; values round to the e2m1 grid {0, 0.5, 1, 1.5, 2, 3, 4, 6} +to nearest, ties to the even code, |v| > 6 saturating to 6. ``mxfp4_quant_ref`` is bitwise what the kernels write for +an f32 / bf16 input. + +``w4a4_moe_ref`` is one MoE call on the dequantized operands in f32: per (token, top-k slot) gate/up = x @ W13[e]^T, +h = SwiGLU-OAI(gate, up) (alpha 1.702, limit 7, up + 1) quantized by the same rule, out = sum_k w * (h @ W2[e]^T). The +kernels accumulate in f32 in another order and round each slot to bf16 before the top-k sum, so a few codes of h differ +near rounding ties: the tests bound the relative L2 error (< 4e-3) and check bitwise repeatability separately. +""" + +import torch + +ALPHA, LIMIT = 1.702, 7.0 +_GRID = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]) + + +def dequant_mxfp4(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """Row-major MXFP4 ([.., K/2] u8, [.., K/32] e8m0 u8) -> f32 [.., K].""" + lut = torch.cat([_GRID, -_GRID]).to(codes.device) + v = torch.stack( + [lut[(codes & 0xF).long()], lut[(codes >> 4).long()]], dim=-1 + ).flatten(-2) + return v * torch.exp2(scales.float() - 127.0).repeat_interleave(32, dim=-1) + + +def mxfp4_quant_ref(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """f32 / bf16 [M, K] -> (codes [M, K/2], e8m0 [M, K/32]), Quark even.""" + m, k = x.shape + v = x.float().view(m, k // 32, 32) + amax = v.abs().amax(dim=-1) + e = ((amax.view(torch.int32) - 0xE00000) >> 23).clamp(min=1) + a = v.abs() / torch.exp2(e.float() - 127.0).unsqueeze(-1) + # the code is the number of midpoints below a; a tie goes to the even code: the lower neighbour at + # 0.25 / 1.25 / 2.5 / 5, the upper one at 0.75 / 1.75 / 3.5 + code = torch.zeros_like(a, dtype=torch.uint8) + for i, mid in enumerate((0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)): + code += (a >= mid if i % 2 else a > mid).to(torch.uint8) + code = (code | torch.signbit(v).to(torch.uint8) << 3).view(m, k) + packed = code[:, 0::2] | (code[:, 1::2] << 4) + return packed.contiguous(), e.to(torch.uint8).contiguous() + + +def swiglu_oai(g: torch.Tensor, u: torch.Tensor) -> torch.Tensor: + g = g.clamp(max=LIMIT) + u = u.clamp(-LIMIT, LIMIT) + return g * torch.sigmoid(ALPHA * g) * (u + 1.0) + + +def w4a4_moe_ref( + x_quant: tuple[torch.Tensor, torch.Tensor], + w13: torch.Tensor, + w13_scale: torch.Tensor, + w2: torch.Tensor, + w2_scale: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, +) -> torch.Tensor: + """f32 [M, H]; weights row-major as stored: ``w13`` [E, 2I, H/2] (gate rows, then up rows), ``w13_scale`` + [E, 2I, H/32], ``w2`` [E, H, I/2], ``w2_scale`` [E, H, I/32].""" + x = dequant_mxfp4(*x_quant) + inter = w13.shape[1] // 2 + out = torch.zeros((x.shape[0], w2.shape[1]), device=x.device) + for e in topk_ids.unique().tolist(): + t, s = torch.nonzero(topk_ids == e, as_tuple=True) + gu = x[t] @ dequant_mxfp4(w13[e], w13_scale[e]).t() + h = dequant_mxfp4(*mxfp4_quant_ref(swiglu_oai(gu[:, :inter], gu[:, inter:]))) + y = h @ dequant_mxfp4(w2[e], w2_scale[e]).t() + out.index_add_(0, t, y * topk_weights[t, s].float().unsqueeze(1)) + return out + + +def a16w4_moe_ref(x, w13, w13_scale, w2, w2_scale, topk_weights, topk_ids, *, return_partials=False): + """BF16 activations x MXFP4 weights (checkpoint layouts): FP32 GEMMs, SwiGLU-OAI rounded to BF16. + + Each slot is weighted (routed scale included) in FP32 and rounded once to BF16; the reduced output sums slots 0..3 + in FP32, then rounds to BF16. Tests: cosine >= 0.9999 overall and per token (another FP32 MFMA order). + """ + m, topk = topk_ids.shape + inter = w13.shape[1] // 2 + parts = torch.empty((m, topk, w2.shape[1]), device=x.device, dtype=torch.bfloat16) + for e in topk_ids.unique().tolist(): + t, slot = torch.nonzero(topk_ids == e, as_tuple=True) + gu = x[t].float() @ dequant_mxfp4(w13[e], w13_scale[e]).t() + h = swiglu_oai(gu[:, :inter], gu[:, inter:]).bfloat16().float() + y = h @ dequant_mxfp4(w2[e], w2_scale[e]).t() + parts[t, slot] = (y * topk_weights[t, slot, None]).bfloat16() + if return_partials: + return parts + out = torch.zeros((m, w2.shape[1]), device=x.device) + for slot in range(topk): + out = out + parts[:, slot].float() + return out.bfloat16() + + +def router_gemm_partials_ref(x, weight, split_k=16): + """FP64 split-K router partials of the FP32 weight and their absolute-dot bounds, [split, M, 128]. + + The BF16x3 kernel keeps high and low terms in separate FP32 chains: tests bound each partial's absolute error by + 2**-18 times its bound (MFMA rounding is not reproduced) and check bitwise determinism separately. + """ + x = x.double().reshape(x.shape[0], split_k, -1) + w = weight.double().reshape(weight.shape[0], split_k, -1) + exact = torch.stack([x[:, k] @ w[:, k].t() for k in range(split_k)]) + bound = torch.stack([x[:, k].abs() @ w[:, k].abs().t() for k in range(split_k)]) + return exact, bound + + +TIE_EPS = 1e-6 +WEIGHT_RTOL = 2.0**-20 # measured <= 2.6e-7 (2^-21.9) + + +def splitk_logits_reference(partials: torch.Tensor) -> torch.Tensor: + """fp32 logits ``[M, E]`` from partials ``[M, S, E]``, summed from 0.0 in split order (the top-k kernels' order).""" + p = [partials[:, k].float() for k in range(partials.shape[1])] + out = torch.zeros_like(p[0]) + for k in range(len(p)): + out = out + p[k] + return out + + +def biased_topk_reference( + logits: torch.Tensor, bias: torch.Tensor, topk: int, scale: float +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """(weights ``[M, topk]`` fp64, ids ``[M, topk]`` int64, sorted choice values ``[M, topk + 1]`` fp64) of the biased + top-k on fp32 ``logits``: ``s = sigmoid(logit)``, ``choice = s + bias``, the ``topk`` largest choices in descending + order, weight ``s * scale / sum(s over the chosen)``. The kernels evaluate the sigmoid with the hardware exp2 and + reciprocal, so ids match where the deciding choices differ by more than ``TIE_EPS`` (``decided``) and weights + within ``WEIGHT_RTOL`` relative; the ``topk + 1`` largest choices let a test tell a decided id from a near tie.""" + s = torch.sigmoid(logits.double()) + choice = s + bias.double() + vals, ids = torch.topk(choice, topk + 1, dim=-1, sorted=True) + chosen = torch.gather(s, 1, ids[:, :topk]) + weights = chosen * scale / chosen.sum(-1, keepdim=True) + return weights, ids[:, :topk], vals + + +def decided(vals: torch.Tensor) -> torch.Tensor: + """``[M, topk]`` bool: the reference id at position j is decided (its choice is more than ``TIE_EPS`` from the + neighbours j - 1 and j + 1).""" + gaps = vals[:, :-1] - vals[:, 1:] # [M, topk]: gap below position j + above = torch.cat([torch.full_like(gaps[:, :1], float("inf")), gaps[:, :-1]], 1) + return (gaps > TIE_EPS) & (above > TIE_EPS) + + +def reduce_slots_bf16_ref(partials): + """BF16 [M, 4, H] -> BF16 [M, H]: +0, then slots 0..3 in FP32, rounded once. + + Finite results match bit for bit, signed zero and subnormals included; NaN payloads are unspecified. + """ + acc = torch.zeros_like(partials[:, 0], dtype=torch.float32) + for k in range(partials.shape[1]): + acc = acc + partials[:, k].float() + return acc.to(partials.dtype) + + +def shared_mxfp8_quant_ref(x): + """Row-major E4M3 codes and e8m0 ``ceil_pow2(amax / 448)`` (exponent >= 1) per 32, no BF16 rounding first. + + The rule of the shared expert's input and of its FP32 SwiGLU output, at every M. + """ + v=x.float().reshape(*x.shape[:-1],x.shape[-1]//32,32) + u=(v.abs().amax(-1)/448).contiguous().view(torch.int32) + e=(u>>23)&255 + e=(e+(((u&0x7fffff)!=0)&(e<255)).int()).clamp(min=1) + q=(v/torch.exp2(e.float()-127).unsqueeze(-1)).to(torch.float8_e4m3fn) + return q.reshape(x.shape).view(torch.uint8),e.to(torch.uint8) + + +def dequant_shared_mxfp8(q,s): + return q.view(torch.float8_e4m3fn).float()*torch.exp2(s.view(torch.uint8).float()-127).repeat_interleave(32,-1) + + +def shared_mxfp8_ref(x,w13,s13,w2,s2,*,quantized=True): + """The shared expert in FP32 sums (the kernels round the output to BF16); tests: cosine >= .999 per token (FP8 + rounding ties can differ). + + ``quantized=False``: BF16 input and an unquantized FP32 intermediate, for the high-precision comparison. + """ + x=dequant_shared_mxfp8(*shared_mxfp8_quant_ref(x)) if quantized else x.float() + gu=x @ dequant_shared_mxfp8(w13,s13).t() + h=swiglu_oai(gu[:,:gu.shape[1]//2],gu[:,gu.shape[1]//2:]) + if quantized:h=dequant_shared_mxfp8(*shared_mxfp8_quant_ref(h)) + return h @ dequant_shared_mxfp8(w2,s2).t() + + +def shared_mxfp8_operand_ref(x_quant, w13, s13, w2, s2): + """``shared_mxfp8_ref`` on an already quantized row-major FP8 / e8m0 input (a gathered SP payload).""" + x = dequant_shared_mxfp8(*x_quant) + gu = x @ dequant_shared_mxfp8(w13, s13).t() + h = swiglu_oai(gu[:, :gu.shape[1] // 2], gu[:, gu.shape[1] // 2:]) + h = dequant_shared_mxfp8(*shared_mxfp8_quant_ref(h)) + return h @ dequant_shared_mxfp8(w2, s2).t() + + +def _mxfp4_via_shared_codes_ref(s4,q8,s8): + """The FP4 codes of the FP8 values at the FP4 scales (computed in FP8 units, no FTZ), and their magnitudes.""" + ratio=torch.exp2((s4.int()-s8.int()).float()).repeat_interleave(32,-1) + values=q8.view(torch.float8_e4m3fn).float() + magnitude=values.abs()/ratio + codes=torch.zeros_like(q8) + for j,mid in enumerate((.25,.75,1.25,1.75,2.5,3.5,5.)): + codes+=(magnitude>=mid if j%2 else magnitude>mid).to(torch.uint8) + return codes|((q8>>7)<<3),magnitude + + +def mxfp4_sidecar_pack_ref(q4,s4,q8,s8): + """The mask, row-major [M, 768] (bit i % 8 is value i, low bit first): set where the direct FP4 code differs from + the FP4 rounding of the shared FP8 code. + + Both inputs come from the same finite BF16 rows through this entry's quantizers, each with its own scales. + """ + candidate,_=_mxfp4_via_shared_codes_ref(s4,q8,s8) + direct=torch.stack((q4&15,q4>>4),-1).flatten(-2) + bits=(candidate!=direct).reshape(q8.shape[0],-1,8).long() + return (bits*(1<>torch.arange(8,device=mask.device))&1 + down=(magnitude==.75)|(magnitude==1.75)|(magnitude==3.5) + other=(candidate&8)|(((candidate&7).int()+torch.where(down,-1,1))&7).to(torch.uint8) + codes=torch.where(bits.flatten(-2).bool(),other,candidate) + return (codes[:,::2]|(codes[:,1::2]<<4)).contiguous() diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm.py new file mode 100644 index 0000000..a3f7fe5 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm.py @@ -0,0 +1,467 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router GEMM split-K partials of bf16 x [m, 6144] and the fp32 gate [128, 6144] as three bf16 terms (gfx950): +acc_hi = x . w0, acc_lo = per BLOCK_K x . w1 then x . w2, in the tier's MFMA (16x16x32 or 32x32x16); also the host +dispatch over the router tiers. A split's workgroups run on one XCD, which then reads its x slice once.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +H, E = 6144, 128 +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} + + +def _pk_bf16(a, b): + return fx.Int32( + fx.rocdl.cvt_pk_bf16_f32( + fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b)) + ) + ) + + +def _split3(w): + """4 fp32 (i32 bits) -> three terms of 2 bf16-pair dwords: RN(w), RN(w - w0), RN(w - w0 - w1), exact remainders.""" + parts = [[], [], []] + for i in range_constexpr(2): + a = fx.Int32(w[2 * i]).bitcast(fx.Float32) + b = fx.Int32(w[2 * i + 1]).bitcast(fx.Float32) + for p in range_constexpr(3): + pk = _pk_bf16(a, b) + parts[p].append(pk) + if p < 2: + a = a - (pk << 16).bitcast(fx.Float32) + b = b - (pk & -65536).bitcast(fx.Float32) + return parts + + +@launcher +def compile_router_gemm( + mfma: int, split_k: int, block_k: int, waves: int, helpers: int +): + """One m-tile per wave, ceil(tiles / waves) m-groups per (n-block, split). + + Args: + mfma: 16 or 32, the MFMA's N / M width (k 32 or 16 per step). + helpers: extra waves that only load and split weights. + """ + assert mfma in (16, 32) and split_k % 8 == 0 + NW = waves + NT = waves + helpers # waves per workgroup + NB = mfma # experts per n-block = MFMA rows (src0 = weight) + MT = mfma # tokens per m-tile = MFMA columns + SUBK = 32 if mfma == 16 else 16 + KC = H // split_k + SPB = block_k // SUBK # MFMA steps per BLOCK_K + NSTEP = KC // block_k + assert KC % block_k == 0 and block_k % SUBK == 0 + NACC = MT * NB // 64 + NBLK = E // NB + W_PIECES = NB * block_k // 4 // (64 * NT) # 16-byte weight pieces per thread + X_PIECES = MT * block_k // 8 // 64 # 16-byte x pieces per lane + assert NB * block_k // 4 == W_PIECES * 64 * NT and X_PIECES >= 1 + W_SLOT = 3 * SPB * 64 # 16-byte chunks: (term, step, lane) + X_SLOT = SPB * 64 + PPR = block_k // 8 # 16-byte bf16 pieces per row and step + assert PPR <= 16 + Q8 = SUBK // 8 # lane groups (8 k each) of an MFMA step + + def swz(chunk, kp): + """LDS chunk of operand chunk ``chunk`` (sub, lane (row, q)), kp = sub * Q8 + q: the low 4 bits xor kp * 16 / + PPR, so the row-contiguous writes and the lane-order reads both hit 16 distinct 16-byte bank groups.""" + return chunk ^ (kp * (16 // PPR)) + + name = f"router_gemm_bf16x3_h6144_e128_mfma{mfma}_s{split_k}_bk{block_k}_w{NW}_hw{helpers}" + + @fx.struct + class Shared: + w: fx.Array[fx.Int32, NSTEP * W_SLOT * 4, 16] + x: fx.Array[fx.Int32, 2 * NW * X_SLOT * 4, 16] + + def mfma_op(a, b, acc): + av = Vec(a).bitcast(fx.BFloat16) + bv = Vec(b).bitcast(fx.BFloat16) + fn = ( + fx.rocdl.mfma_f32_16x16x32_bf16 + if mfma == 16 + else fx.rocdl.mfma_f32_32x32x16_bf16 + ) + return Vec(fn(Vec.make_type(NACC, fx.Float32), [av, bv, acc, 0, 0, 0])) + + @flyc.kernel(name=name, known_block_size=[64 * NT, 1, 1]) + def kernel( + arg_x: fx.Int64, # [m, H] bf16, row stride i32_sx + arg_w: fx.Int64, # [E, H] fp32 + arg_p: fx.Int64, # partials [split_k, m, E] fp32 + i32_m: fx.Int32, + i32_sx: fx.Int32, + ): + tx, bid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + # bid = (g * NBLK + nb) * split_k + s: a split's workgroups share an XCD (split_k is a multiple of 8) + s = bid % split_k + nb = (bid // split_k) % NBLK + g = bid // (split_k * NBLK) + t0 = (g * NW + wave) * MT # this wave's first token + smem = fx.SharedAllocator().allocate(Shared).peek() + + def view16(arr, n): + return fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, arr.ptr), fx.make_layout(n, 1)), + fx.make_layout(4, 1), + ) + + w16 = view16(smem.w, NSTEP * W_SLOT * 4) + w8 = fx.logical_divide( + fx.make_view( + fx.recast_iter(fx.Int32, smem.w.ptr), + fx.make_layout(NSTEP * W_SLOT * 4, 1), + ), + fx.make_layout(2, 1), + ) + x16 = view16(smem.x, 2 * NW * X_SLOT * 4) + atom16 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=E * H * 4 + ) + xr = buffer_ops.create_buffer_resource_from_addr( + arg_x, num_records_bytes=fx.Int64(i32_m) * fx.Int64(i32_sx * 2) + ) + pr = buffer_ops.create_buffer_resource_from_addr( + arg_p, + num_records_bytes=fx.Int64(i32_m) * (split_k * E * 4), + ) + k_base = s * KC + + # every load first, row-contiguous: x piece u = (row u // (block_k / 8), k 8 (u % (block_k / 8))), w piece = + # (expert, 4 k) + def x_piece(it): + u = it * 64 + lane + return u // (block_k // 8), u % (block_k // 8) + + xl = [] + for st in range_constexpr(NSTEP): + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + row = t0 + r + xl.append( + Vec( + buffer_ops.buffer_load( + xr, + (row * i32_sx + k_base + st * block_k + kp * 8) >> 1, + vec_width=4, + dtype=fx.Int32, + mask=(row < i32_m) & (wave < NW), + ) + ) + ) + + def w_piece(it): + u = it * 64 * NT + tx + return u // (block_k // 4), u % (block_k // 4) + + wl = [] + for st in range_constexpr(NSTEP): + for it in range_constexpr(W_PIECES): + n, kp = w_piece(it) + wl.append( + Vec( + buffer_ops.buffer_load( + wr, + (nb * NB + n) * H + k_base + st * block_k + kp * 4, + vec_width=4, + dtype=fx.Int32, + ) + ) + ) + + def stage_w(st): + """Step ``st``'s weights -> 3 bf16 terms, chunk (term, sub, lane (n, q)).""" + slot = st * W_SLOT + for it in range_constexpr(W_PIECES): + n, kp = w_piece(it) + kq = kp // 2 # sub * Q8 + q of the 4 weights + chunk = slot + swz((kq // Q8) * 64 + n + NB * (kq % Q8), kq) + parts = _split3( + [fx.Int32(wl[st * W_PIECES + it][e]) for e in range_constexpr(4)] + ) + for p in range_constexpr(3): + r8 = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + r8.store(Vec.from_elements(parts[p], fx.Int32)) + dst = fx.slice(w8, (None, (chunk + p * SPB * 64) * 2 + kp % 2)) + fx.copy(atom8, r8, dst) + + def stage_x(st): + """This wave's x of step ``st`` -> its private slot (ping-pong).""" + xslot = ((st % 2) * NW + wave) * X_SLOT + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r16.store(xl[st * X_PIECES + it]) + c = swz((kp // Q8) * 64 + r + MT * (kp % Q8), kp) + fx.copy(atom16, r16, fx.slice(x16, (None, xslot + c))) + + def operands(st): + """(x, weight terms) of step ``st`` in the MFMA lane order.""" + slot = st * W_SLOT + xslot = ((st % 2) * NW + wave) * X_SLOT + xs, ws = [], [] + for j in range_constexpr(SPB): + c = swz(j * 64 + lane, j * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(x16, (None, xslot + c)), r16) + xs.append(Vec(r16.load())) + for p in range_constexpr(3): + for j in range_constexpr(SPB): + c = swz(j * 64 + lane, j * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(w16, (None, slot + p * SPB * 64 + c)), r16) + ws.append(Vec(r16.load())) + return xs, ws + + # the whole weight slice once (one barrier); x in private slots per step + for st in range_constexpr(NSTEP): + stage_w(st) + fx.gpu.barrier() + if wave < NW: + stage_x(0) + acc_hi = Vec.filled(NACC, 0.0, fx.Float32) + acc_lo = Vec.filled(NACC, 0.0, fx.Float32) + nxt = operands(0) + for st in range_constexpr(NSTEP): + # the next step's operands are read before this step's MFMAs + xs, ws = nxt + if const_expr(st + 1 < NSTEP): + stage_x(st + 1) + nxt = operands(st + 1) + # keep the reads ahead of the MFMAs (LLVM would sink them to their uses) + fx.rocdl.sched_barrier(0) + for j in range_constexpr(SPB): + acc_hi = mfma_op(ws[j], xs[j], acc_hi) + for p in range_constexpr(1, 3): + for j in range_constexpr(SPB): + acc_lo = mfma_op(ws[p * SPB + j], xs[j], acc_lo) + out = [ + fx.Float32(acc_hi[i]) + fx.Float32(acc_lo[i]) + for i in range_constexpr(NACC) + ] + # D[n][m]: lane (m, q) holds n = 8 j + 4 q + i (32x32) / 4 q + i (16x16) + m = t0 + lane % MT + q = lane // MT + for j in range_constexpr(NACC // 4): + n = nb * NB + j * 8 + q * 4 + buffer_ops.buffer_store( + Vec.from_elements(out[4 * j : 4 * j + 4], fx.Float32), + pr, + (s * i32_m + m) * E + n, + mask=m < i32_m, + ) + + @flyc.jit + def launch_router_gemm_mxfp4_moe( + arg_x: fx.Int64, + arg_w: fx.Int64, + arg_p: fx.Int64, + i32_m: fx.Int32, + i32_sx: fx.Int32, + stream: fx.Stream, + ): + groups = (i32_m + (MT * NW - 1)) // (MT * NW) + kernel(arg_x, arg_w, arg_p, i32_m, i32_sx).launch( + grid=(fx.Int64(groups * (NBLK * split_k)), 1, 1), + block=(64 * NT, 1, 1), + stream=stream, + ) + + launch_router_gemm_mxfp4_moe.kernel_name = name + # LDS allows 4 (32x32) / 8 (16x16) waves per CU: let LLVM use the registers + launch_router_gemm_mxfp4_moe.compile_hints = {**PRELOAD_HINTS, "waves_per_eu": 1 if mfma == 32 else 2} + return launch_router_gemm_mxfp4_moe + + +# ---- host: compile cache, tier dispatch +import functools # noqa: E402 + +import torch # noqa: E402 + + +def _run_compiled(exe, *args): + """The first call compiles and runs (``flyc.compile``); later calls dispatch the cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +# Larger calls use the fused BF16x3 router and MXFP4 input quantizer. +MAX_TOKENS = 2047 +# inclusive token bound -> (MFMA width, split_k, BLOCK_K): the partials' K split and sum order (the top-k kernels and +# the router + quant launches read the split) +SPLIT_TIERS = ((64, (16, 48, 64)), (192, (16, 16, 128)), (MAX_TOKENS, (32, 16, 64))) + + +def split_tier(num_tokens: int) -> tuple[int, int, int] | None: + """(MFMA width, split_k, BLOCK_K) for ``num_tokens``, None above ``MAX_TOKENS``.""" + for bound, tier in SPLIT_TIERS: + if num_tokens <= bound: + return tier + return None + + +@functools.cache +def _get_kernel(mfma: int, split_k: int, block_k: int, waves: int, helpers: int): + return compile_router_gemm(mfma, split_k, block_k, waves, helpers) + + +def _waves(mfma: int, num_tokens: int) -> tuple[int, int]: + """(m-tile waves, weight-only helper waves) per workgroup, cold gate: 16-row tiles 8 + 0 above 8 tiles (172 rows + 4.9 vs 5.2 us with 4); 32-row tiles 2 + 2 to 8 tiles (240 rows 6.1 vs 6.9), 4 + 0 to 16.""" + # 513-2047 (64 workgroups per m-group, 256 CUs): the fewest token waves for the same rounds, the rest stage weights + if mfma == 32 and 513 <= num_tokens <= 2047: + rounds = (num_tokens + 1023) // 1024 + waves = (num_tokens + 128 * rounds - 1) // (128 * rounds) + # seven waves at 769-896: no clear gain (A/B ranges overlap at 896); with two rounds seven wins + if num_tokens <= 1024 and waves == 7: + return 8, 0 + return waves, 8 - waves + tiles = (num_tokens + mfma - 1) // mfma + if mfma == 16: + return (8, 0) if tiles > 8 else (4, 0) + if tiles <= 8: + return 2, 2 + return (4, 0) if tiles <= 16 else (8, 0) + + +def _kernel_config(num_tokens): + tier = split_tier(num_tokens) + if tier is None: + return None + mfma, split_k, block_k = tier + if 193 <= num_tokens <= 384: + # 16-row MFMA, 4 waves: up to three groups per CU instead of one partly idle 32-row group; bitwise equal + return 16, split_k, block_k, 4, 0 + return mfma, split_k, block_k, *_waves(mfma, num_tokens) + + +# router_gemm_nblocks at 1-384 rows, two waves to 32: -0.15..-0.30 us at 8-32 rows, -0.05..-0.15 at 64-256 +NBLOCKS_TWO_WAVES_MAX = 32 +# router_gemm_direct (no LDS) on the line-layout gate copy to one m-tile: router + top-k chain -0.011..-0.054 us at +# 1-16 rows, +0.02 at 24-32 +DIRECT_LINES_MAX = 16 +# nblocks ``x_direct`` (four waves): router -0.21..-0.01 us at 33-56 rows, +0.03 at 64, +0.04..+0.08 with two waves +X_DIRECT_FROM, X_DIRECT_MAX = 33, 57 + + +# router_gemm_nblocks_lines (two waves, line-layout gate): faster only at 17-32 rows +NBLOCKS_LINES_FROM, NBLOCKS_LINES_MAX = 17, 32 + +# split-48 partials token-major [m, 48, 128] to here: the top-k reads them at constant split offsets +TOKEN_MAJOR_MAX = 64 + + +def nblocks_config(num_tokens: int) -> tuple[int, int, int, int] | None: + """(split_k, BLOCK_K, waves, n-blocks per workgroup) of ``router_gemm_nblocks`` where it serves ``num_tokens``, else + None (this kernel's 32-row MFMA / helper-wave tiers).""" + config = _kernel_config(num_tokens) + if config is None: + return None + mfma, split_k, block_k, waves, helpers = config + if mfma != 16 or helpers: + return None + if num_tokens <= NBLOCKS_TWO_WAVES_MAX: + waves = 2 + return split_k, block_k, waves, 1 + + +def router_gemm_partials( + hidden_states: torch.Tensor, router_weight: torch.Tensor, + router_weight_lines: torch.Tensor | None = None, +) -> torch.Tensor | None: + """FP32 partials [m, split_k, 128] (``split_tier``'s split and order), contiguous to 64 rows and a view of + [split_k, m, 128] above; None outside the covered shapes and row counts. + + Args: + router_weight_lines: the gate in ``layout.router_line_layout``, used by the 1-32 row tiers when given. + """ + m = hidden_states.shape[0] + if ( + m == 0 + or hidden_states.dim() != 2 + or hidden_states.shape[1] != H + or hidden_states.dtype != torch.bfloat16 + or hidden_states.stride(1) != 1 + or hidden_states.stride(0) % 2 + or router_weight.shape != (E, H) + or router_weight.dtype != torch.float32 + or not router_weight.is_contiguous() + ): + return None + config = _kernel_config(m) + if config is None: + return None + _, split_k, _, _, _ = config + token_major = m <= TOKEN_MAJOR_MAX + partials = torch.empty( + (split_k, m, E), dtype=torch.float32, device=hidden_states.device + ) + if (router_weight_lines is not None and m <= DIRECT_LINES_MAX and config[1:3] == (48, 64) + and router_weight_lines.dtype == torch.float32 and router_weight_lines.is_contiguous() + and router_weight_lines.numel() == E * H): + from .router_gemm_direct import _get_kernel as _get_direct_kernel + tiles = (m + 15) // 16 + _run_compiled( + _get_direct_kernel(48, 64, tiles, 2, True, token_major), + hidden_states.data_ptr(), + router_weight_lines.data_ptr(), + partials.data_ptr(), + m, + hidden_states.stride(0), + torch.cuda.current_stream(), + ) + return partials.view(m, split_k, E) if token_major else partials.permute(1, 0, 2) + if (router_weight_lines is not None and NBLOCKS_LINES_FROM <= m <= NBLOCKS_LINES_MAX + and router_weight_lines.dtype == torch.float32 and router_weight_lines.is_contiguous() + and router_weight_lines.numel() == E * H): + from .router_gemm_nblocks_lines import _get_kernel as _get_lines_kernel + _run_compiled(_get_lines_kernel(token_major), hidden_states.data_ptr(), router_weight_lines.data_ptr(), + partials.data_ptr(), m, hidden_states.stride(0), torch.cuda.current_stream()) + return partials.view(m, split_k, E) if token_major else partials.permute(1, 0, 2) + nblocks = nblocks_config(m) + if nblocks is not None: + from .router_gemm_nblocks import _get_kernel as _get_nblocks_kernel + x_direct = X_DIRECT_FROM <= m <= X_DIRECT_MAX + _run_compiled( + _get_nblocks_kernel(*nblocks, 0, x_direct, token_major), + hidden_states.data_ptr(), + router_weight.data_ptr(), + partials.data_ptr(), + m, + hidden_states.stride(0), + 0, 0, 0, 0, + torch.cuda.current_stream(), + ) + return partials.view(m, split_k, E) if token_major else partials.permute(1, 0, 2) + _run_compiled( + _get_kernel(*config), + hidden_states.data_ptr(), + router_weight.data_ptr(), + partials.data_ptr(), + m, + hidden_states.stride(0), + torch.cuda.current_stream(), + ) + return partials.view(m, split_k, E) if token_major else partials.permute(1, 0, 2) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_direct.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_direct.py new file mode 100644 index 0000000..01c7ee3 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_direct.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router GEMM split-K partials for up to 64 rows without LDS (gfx950): ``router_gemm.py``'s 16-row MFMA partials bit +for bit. Each lane loads its MFMA operands from global memory in the lane order (lane (i, q): expert / token i, k 8 q +.. 8 q + 7 of a 32-k step) and splits the fp32 weight into the three bf16 terms in registers. ``lines``: the weight +from the line-layout gate copy, 1 KB contiguous per wave load. A split's workgroups share an XCD as in router_gemm.""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +from .router_gemm import PRELOAD_HINTS, _run_compiled, _split3 # noqa: F401 + +H, E = 6144, 128 + + +@launcher +def compile_router_gemm_direct(split_k: int, block_k: int, tiles: int, mode: int = 0, lines: bool = False, token_major: bool = False): + """``tiles`` (<= 4) 16-row m-tiles of one (n-block, split). + + Args: + mode: 0 one wave takes the ``tiles`` m-tiles, 1 one wave per m-tile in a workgroup, 2 one workgroup per m-tile. + lines: the weight is the ``layout.router_line_layout`` copy [192 k-steps, 8 n-blocks, 2, 4, 16, 4]. + """ + # split 48 only: S16 / S24 give wrong partials (logits off by 0.7-0.95 at 8 rows) + assert split_k == 48 and 1 <= tiles <= 4 and mode in (0, 1, 2) + NWV = tiles if mode == 1 else 1 + TPW = tiles if mode == 0 else 1 # m-tiles per wave + NB = MT = 16 + SUBK = 32 + KC = H // split_k + NK = KC // SUBK # 32-k MFMA steps of the split + SPB = block_k // SUBK # MFMA steps per BLOCK_K + assert KC % block_k == 0 and block_k % SUBK == 0 + NBLK = E // NB + name = (f"router_gemm_bf16x3_direct_h6144_e128_mfma16_s{split_k}_bk{block_k}_t{tiles}_m{mode}" + + ("_lines" if lines else "") + ("_token_partials" if token_major else "")) + + def mfma_op(a, b, acc): + av = Vec(a).bitcast(fx.BFloat16) + bv = Vec(b).bitcast(fx.BFloat16) + return Vec(fx.rocdl.mfma_f32_16x16x32_bf16(Vec.make_type(4, fx.Float32), [av, bv, acc, 0, 0, 0])) + + @flyc.kernel(name=name, known_block_size=[64 * NWV, 1, 1]) + def kernel( + arg_x: fx.Int64, # [m, H] bf16, row stride i32_sx + arg_w: fx.Int64, # [E, H] fp32, or its line layout (``lines``) + arg_p: fx.Int64, # partials [split_k, m, E] fp32 + i32_m: fx.Int32, + i32_sx: fx.Int32, + ): + tx, bid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + s = bid % split_k + nb = (bid // split_k) % NBLK + if const_expr(mode == 1): + t_first = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) * MT + elif const_expr(mode == 2): + t_first = bid // (split_k * NBLK) * MT + else: + t_first = fx.Int32(0) + i = lane % 16 + q = lane // 16 + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=E * H * 4) + xr = buffer_ops.create_buffer_resource_from_addr( + arg_x, num_records_bytes=fx.Int64(i32_m) * fx.Int64(i32_sx * 2) + ) + pr = buffer_ops.create_buffer_resource_from_addr( + arg_p, num_records_bytes=fx.Int64(i32_m) * (split_k * E * 4) + ) + k0 = s * KC + q * 8 + # weight: expert nb * 16 + i, 8 fp32 per 32-k step (two dwordx4) + wl = [] + for kk in range_constexpr(NK): + for h in range_constexpr(2): + if const_expr(lines): + # [k-step, n-block, h, q, i, 4]: k-step s * NK + kk, lane = q * 16 + i + off = (s * NK + kk) * (E * SUBK) + nb * (NB * SUBK) + h * 256 + lane * 4 + else: + off = (nb * NB + i) * H + k0 + kk * SUBK + h * 4 + wl.append(Vec(buffer_ops.buffer_load(wr, off, vec_width=4, dtype=fx.Int32))) + xl = [] + for tl in range_constexpr(TPW): + row = t_first + tl * MT + i + for kk in range_constexpr(NK): + xl.append(Vec(buffer_ops.buffer_load( + xr, (row * i32_sx + k0 + kk * SUBK) >> 1, vec_width=4, dtype=fx.Int32, mask=row < i32_m))) + # three bf16 terms of each step's 8 weights, 4 dwords each in k order + terms = [] + for kk in range_constexpr(NK): + a = _split3([fx.Int32(wl[2 * kk][e]) for e in range_constexpr(4)]) + b = _split3([fx.Int32(wl[2 * kk + 1][e]) for e in range_constexpr(4)]) + terms.append([Vec.from_elements(a[p] + b[p], fx.Int32) for p in range_constexpr(3)]) + for tl in range_constexpr(TPW): + acc_hi = Vec.filled(4, 0.0, fx.Float32) + acc_lo = Vec.filled(4, 0.0, fx.Float32) + for st in range_constexpr(NK // SPB): + for j in range_constexpr(SPB): + kk = st * SPB + j + acc_hi = mfma_op(terms[kk][0], xl[tl * NK + kk], acc_hi) + for p in range_constexpr(1, 3): + for j in range_constexpr(SPB): + kk = st * SPB + j + acc_lo = mfma_op(terms[kk][p], xl[tl * NK + kk], acc_lo) + out = [fx.Float32(acc_hi[e]) + fx.Float32(acc_lo[e]) for e in range_constexpr(4)] + m = t_first + tl * MT + i + buffer_ops.buffer_store( + Vec.from_elements(out, fx.Float32), pr, ((m * split_k + s) if const_expr(token_major) else (s * i32_m + m)) * E + nb * NB + q * 4, mask=m < i32_m + ) + + @flyc.jit + def launch_router_gemm_direct_mxfp4_moe( + arg_x: fx.Int64, + arg_w: fx.Int64, + arg_p: fx.Int64, + i32_m: fx.Int32, + i32_sx: fx.Int32, + stream: fx.Stream, + ): + kernel(arg_x, arg_w, arg_p, i32_m, i32_sx).launch( + grid=(fx.Int64(NBLK * split_k * (tiles if mode == 2 else 1)), 1, 1), block=(64 * NWV, 1, 1), + stream=stream + ) + + launch_router_gemm_direct_mxfp4_moe.kernel_name = name + launch_router_gemm_direct_mxfp4_moe.compile_hints = PRELOAD_HINTS + return launch_router_gemm_direct_mxfp4_moe + + +@functools.cache +def _get_kernel(split_k: int, block_k: int, tiles: int, mode: int = 0, lines: bool = False, token_major: bool = False): + return compile_router_gemm_direct(split_k, block_k, tiles, mode, lines, token_major) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks.py new file mode 100644 index 0000000..d429e9b --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks.py @@ -0,0 +1,325 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router GEMM split-K partials with ``nbw`` 16-expert n-blocks per workgroup (gfx950): ``router_gemm.py``'s 16-row MFMA +partials bit for bit, the x traffic and the grid divided by ``nbw``. ``quant`` 1 / 2 appends / prepends workgroups for +the MXFP4 + MXFP8 input quant (prepended ones a multiple of 8, so split s stays on XCD s % 8); ``x_direct`` loads x in +the MFMA lane order (``router_gemm_direct``'s x loads) instead of staging it through LDS.""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +from .common.loaders import _as_f32 +from .i3072.gemm1_prefill_h6144_i3072_persistent import _cvt_pk_fp4, _e8m0_even_e2m1 +from .mxfp8.gemm1_prefill_h6144_i3072 import _fmax3_abs +from .router_gemm import PRELOAD_HINTS, _run_compiled, _split3 +from .shared_quant import _fp8, _e8m0_fp8 + +H, E = 6144, 128 +BLOCKS = H // 32 # 32-value groups per row + + +def _dual_quant_group(i32_m, arg_x, arg_q, arg_s, arg_q8, arg_s8, blk): + """One lane: 32-value group ``blk`` of x -> MXFP4 (even e8m0) and MXFP8 codes and scales, + ``router_topk_mxfp4_mxfp8_quant.py``'s arithmetic op for op.""" + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=fx.Int64(i32_m) * (H * 2)) + xv = [Vec(buffer_ops.buffer_load(xr, blk * 16 + j * 4, vec_width=4, dtype=fx.Int32)) for j in range_constexpr(4)] + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=fx.Int64(i32_m) * (H // 2)) + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=fx.Int64(i32_m) * (H // 32)) + q8r = buffer_ops.create_buffer_resource_from_addr(arg_q8, num_records_bytes=fx.Int64(i32_m) * H) + s8r = buffer_ops.create_buffer_resource_from_addr(arg_s8, num_records_bytes=fx.Int64(i32_m) * (H // 32)) + vals = [] + for j in range_constexpr(4): + for kk in range_constexpr(4): + w = fx.Int32(xv[j][kk]) + vals.append(_as_f32(w << 16)) + vals.append(_as_f32(w & fx.Int32(-65536))) + amax = _fmax3_abs(vals[0], vals[1], vals[2]) + for i in range_constexpr(3, 31, 2): + amax = _fmax3_abs(vals[i], vals[i + 1], amax) + amax = _fmax3_abs(vals[31], vals[31], amax) + e8 = _e8m0_fp8(amax) + sf8 = (e8 << 23).bitcast(fx.Float32) + for half in range_constexpr(2): + words = [] + for dw in range_constexpr(4): + j = half * 16 + dw * 4 + pk = _fp8(fx.Int32(0), vals[j], vals[j + 1], sf8, False) + pk = _fp8(pk, vals[j + 2], vals[j + 3], sf8, True) + words.append(pk) + buffer_ops.buffer_store(Vec.from_elements(words, fx.Int32), q8r, blk * 8 + half * 4) + buffer_ops.buffer_store(e8.to(fx.Int8), s8r, blk) + e8m0 = _e8m0_even_e2m1(amax) + scale_f = _as_f32(e8m0 << 23) + qo = [] + for d in range_constexpr(4): + p = fx.Int32(0) + for kk in range_constexpr(4): + p = _cvt_pk_fp4(p, vals[8 * d + 2 * kk], vals[8 * d + 2 * kk + 1], scale_f, kk) + qo.append(p) + buffer_ops.buffer_store(Vec.from_elements(qo, fx.Int32), qr, blk * 4) + buffer_ops.buffer_store(fx.Int32(e8m0).to(fx.Int8), sr, blk) + + +@launcher +def compile_router_gemm_nblocks(split_k: int, block_k: int, waves: int, nbw: int, quant: int = 0, + x_direct: bool = False, token_major: bool = False): + """``router_gemm.compile_router_gemm(16, ...)``'s chains: ``waves`` m-tile waves, ``nbw`` of the 8 n-blocks per + workgroup, ``quant`` 0 none, 1 appended, 2 prepended.""" + assert split_k % 8 == 0 and nbw in (1, 2, 4, 8) and quant in (0, 1, 2) + NW = NT = waves + NB = MT = 16 + SUBK = 32 + KC = H // split_k + SPB = block_k // SUBK + NSTEP = KC // block_k + assert KC % block_k == 0 and block_k % SUBK == 0 + NACC = MT * NB // 64 + NBLK = E // NB + NBG = NBLK // nbw # n-block groups + W_PIECES = nbw * NB * block_k // 4 // (64 * NT) + X_PIECES = MT * block_k // 8 // 64 + assert nbw * NB * block_k // 4 == W_PIECES * 64 * NT and X_PIECES >= 1 + W_SLOT = 3 * SPB * 64 + X_SLOT = SPB * 64 + PPR = block_k // 8 + assert PPR <= 16 + Q8 = SUBK // 8 + + def swz(chunk, kp): + return chunk ^ (kp * (16 // PPR)) + + name = (f"router_gemm_bf16x3_h6144_e128_mfma16_s{split_k}_bk{block_k}_w{NW}_nb{nbw}" + + (f"_dual_quant{quant}" if quant else "") + ("_xd" if x_direct else "") + ("_token_partials" if token_major else "")) + + @fx.struct + class Shared: + w: fx.Array[fx.Int32, nbw * NSTEP * W_SLOT * 4, 16] + x: fx.Array[fx.Int32, 2 * NW * X_SLOT * 4, 16] + + def mfma_op(a, b, acc): + av = Vec(a).bitcast(fx.BFloat16) + bv = Vec(b).bitcast(fx.BFloat16) + return Vec(fx.rocdl.mfma_f32_16x16x32_bf16(Vec.make_type(NACC, fx.Float32), [av, bv, acc, 0, 0, 0])) + + @flyc.kernel(name=name, known_block_size=[64 * NT, 1, 1]) + def kernel( + arg_x: fx.Int64, # [m, H] bf16, row stride i32_sx + arg_w: fx.Int64, # [E, H] fp32 + arg_p: fx.Int64, # partials [split_k, m, E] fp32 + i32_m: fx.Int32, + i32_sx: fx.Int32, + arg_q: fx.Int64, # quant: e2m1 codes [M, H / 2], e8m0 [M, H / 32], e4m3 [M, H], e8m0 [M, H / 32] + arg_s: fx.Int64, + arg_q8: fx.Int64, + arg_s8: fx.Int64, + ): + tx, bid0 = fx.thread_idx.x, fx.block_idx.x + n_router = (i32_m + (MT * NW - 1)) // (MT * NW) * (NBG * split_k) + n_q = (i32_m * BLOCKS + (64 * NT * 8 - 1)) // (64 * NT * 8) * 8 + if const_expr(quant == 1): + if bid0 >= n_router: + blk = (bid0 - n_router) * (64 * NT) + tx + if blk < i32_m * BLOCKS: + _dual_quant_group(i32_m, arg_x, arg_q, arg_s, arg_q8, arg_s8, blk) + if const_expr(quant == 2): + if bid0 < n_q: + blk = bid0 * (64 * NT) + tx + if blk < i32_m * BLOCKS: + _dual_quant_group(i32_m, arg_x, arg_q, arg_s, arg_q8, arg_s8, blk) + bid = bid0 - n_q if const_expr(quant == 2) else bid0 + if (bid >= 0) & (bid < n_router): + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + # bid = (g * NBG + nbg) * split_k + s: a split's workgroups share an XCD + s = bid % split_k + nbg = (bid // split_k) % NBG + g = bid // (split_k * NBG) + t0 = (g * NW + wave) * MT + smem = fx.SharedAllocator().allocate(Shared).peek() + + def view16(arr, n): + return fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, arr.ptr), fx.make_layout(n, 1)), + fx.make_layout(4, 1), + ) + + w16 = view16(smem.w, nbw * NSTEP * W_SLOT * 4) + w8 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.w.ptr), fx.make_layout(nbw * NSTEP * W_SLOT * 4, 1)), + fx.make_layout(2, 1), + ) + x16 = view16(smem.x, 2 * NW * X_SLOT * 4) + atom16 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=E * H * 4) + xr = buffer_ops.create_buffer_resource_from_addr( + arg_x, num_records_bytes=fx.Int64(i32_m) * fx.Int64(i32_sx * 2) + ) + pr = buffer_ops.create_buffer_resource_from_addr( + arg_p, num_records_bytes=fx.Int64(i32_m) * (split_k * E * 4) + ) + k_base = s * KC + + def x_piece(it): + u = it * 64 + lane + return u // (block_k // 8), u % (block_k // 8) + + xl = [] + for st in range_constexpr(NSTEP if x_direct else 0): + # x_direct: each wave's own m-tile in the MFMA lane order (row lane % 16, k piece lane // 16) + for jj in range_constexpr(SPB): + row = t0 + lane % MT + xl.append(Vec(buffer_ops.buffer_load( + xr, (row * i32_sx + k_base + st * block_k + jj * SUBK + (lane // MT) * 8) >> 1, + vec_width=4, dtype=fx.Int32, mask=row < i32_m))) + for st in range_constexpr(0 if x_direct else NSTEP): + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + row = t0 + r + xl.append( + Vec( + buffer_ops.buffer_load( + xr, + (row * i32_sx + k_base + st * block_k + kp * 8) >> 1, + vec_width=4, + dtype=fx.Int32, + mask=row < i32_m, + ) + ) + ) + + def w_piece(it): + """(n-block j of the group, expert of the n-block, 4-k piece).""" + u = it * 64 * NT + tx + n, kp = u // (block_k // 4), u % (block_k // 4) + return n // NB, n % NB, kp + + wl = [] + for st in range_constexpr(NSTEP): + for it in range_constexpr(W_PIECES): + j, n, kp = w_piece(it) + wl.append( + Vec( + buffer_ops.buffer_load( + wr, + ((nbg * nbw + j) * NB + n) * H + k_base + st * block_k + kp * 4, + vec_width=4, + dtype=fx.Int32, + ) + ) + ) + + def stage_w(st): + for it in range_constexpr(W_PIECES): + j, n, kp = w_piece(it) + slot = (j * NSTEP + st) * W_SLOT + kq = kp // 2 + chunk = slot + swz((kq // Q8) * 64 + n + NB * (kq % Q8), kq) + parts = _split3([fx.Int32(wl[st * W_PIECES + it][e]) for e in range_constexpr(4)]) + for p in range_constexpr(3): + r8 = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + r8.store(Vec.from_elements(parts[p], fx.Int32)) + dst = fx.slice(w8, (None, (chunk + p * SPB * 64) * 2 + kp % 2)) + fx.copy(atom8, r8, dst) + + def stage_x(st): + if const_expr(x_direct): + return + xslot = ((st % 2) * NW + wave) * X_SLOT + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r16.store(xl[st * X_PIECES + it]) + c = swz((kp // Q8) * 64 + r + MT * (kp % Q8), kp) + fx.copy(atom16, r16, fx.slice(x16, (None, xslot + c))) + + def operands(st): + """x of step ``st`` and each n-block's weight terms, in the MFMA lane order.""" + xslot = ((st % 2) * NW + wave) * X_SLOT + xs, ws = [], [] + for jj in range_constexpr(SPB if not x_direct else 0): + c = swz(jj * 64 + lane, jj * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(x16, (None, xslot + c)), r16) + xs.append(Vec(r16.load())) + if const_expr(x_direct): + xs = xl[st * SPB:(st + 1) * SPB] + for j in range_constexpr(nbw): + slot = (j * NSTEP + st) * W_SLOT + for p in range_constexpr(3): + for jj in range_constexpr(SPB): + c = swz(jj * 64 + lane, jj * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(w16, (None, slot + p * SPB * 64 + c)), r16) + ws.append(Vec(r16.load())) + return xs, ws + + for st in range_constexpr(NSTEP): + stage_w(st) + fx.gpu.barrier() + stage_x(0) + acc_hi = [Vec.filled(NACC, 0.0, fx.Float32) for _ in range_constexpr(nbw)] + acc_lo = [Vec.filled(NACC, 0.0, fx.Float32) for _ in range_constexpr(nbw)] + nxt = operands(0) + for st in range_constexpr(NSTEP): + xs, ws = nxt + if const_expr(st + 1 < NSTEP): + stage_x(st + 1) + nxt = operands(st + 1) + fx.rocdl.sched_barrier(0) + for j in range_constexpr(nbw): + wj = ws[j * 3 * SPB:(j + 1) * 3 * SPB] + for jj in range_constexpr(SPB): + acc_hi[j] = mfma_op(wj[jj], xs[jj], acc_hi[j]) + for p in range_constexpr(1, 3): + for jj in range_constexpr(SPB): + acc_lo[j] = mfma_op(wj[p * SPB + jj], xs[jj], acc_lo[j]) + m = t0 + lane % MT + q = lane // MT + for j in range_constexpr(nbw): + out = [fx.Float32(acc_hi[j][i]) + fx.Float32(acc_lo[j][i]) for i in range_constexpr(NACC)] + n = (nbg * nbw + j) * NB + q * 4 + buffer_ops.buffer_store( + Vec.from_elements(out, fx.Float32), pr, ((m * split_k + s) if const_expr(token_major) else (s * i32_m + m)) * E + n, mask=m < i32_m + ) + + @flyc.jit + def launch_router_gemm_nblocks_mxfp4_moe( + arg_x: fx.Int64, + arg_w: fx.Int64, + arg_p: fx.Int64, + i32_m: fx.Int32, + i32_sx: fx.Int32, + arg_q: fx.Int64, + arg_s: fx.Int64, + arg_q8: fx.Int64, + arg_s8: fx.Int64, + stream: fx.Stream, + ): + groups = (i32_m + (MT * NW - 1)) // (MT * NW) + nq = 0 + if const_expr(quant == 1): + nq = (i32_m * BLOCKS + (64 * NT - 1)) // (64 * NT) + if const_expr(quant == 2): + nq = (i32_m * BLOCKS + (64 * NT * 8 - 1)) // (64 * NT * 8) * 8 + kernel(arg_x, arg_w, arg_p, i32_m, i32_sx, arg_q, arg_s, arg_q8, arg_s8).launch( + grid=(fx.Int64(groups * (NBG * split_k) + nq), 1, 1), + block=(64 * NT, 1, 1), + stream=stream, + ) + + launch_router_gemm_nblocks_mxfp4_moe.kernel_name = name + launch_router_gemm_nblocks_mxfp4_moe.compile_hints = {**PRELOAD_HINTS, "waves_per_eu": 2} + return launch_router_gemm_nblocks_mxfp4_moe + + +@functools.cache +def _get_kernel(split_k: int, block_k: int, waves: int, nbw: int, quant: int = 0, x_direct: bool = False, token_major: bool = False): + return compile_router_gemm_nblocks(split_k, block_k, waves, nbw, quant, x_direct, token_major) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks_lines.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks_lines.py new file mode 100644 index 0000000..c354b76 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_nblocks_lines.py @@ -0,0 +1,242 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``router_gemm_nblocks``'s split-48 two-wave router (17-32 rows) on the line-layout gate copy (gfx950): a wave load +reads consecutive 16-byte pieces of a 32-K gate step, and each piece's expert and k place its three bf16 terms in the +same swizzled LDS layout, so the partials are the same.""" +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +from .router_gemm import PRELOAD_HINTS, _split3 + +H, E = 6144, 128 + + +@launcher +def compile_router_gemm_nblocks_lines(token_major: bool = False): + """The split-48, BLOCK_K-64 router with two 16-row m-tile waves.""" + split_k, block_k, waves, nbw = 48, 64, 2, 1 + NW = NT = waves + NB = MT = 16 + SUBK = 32 + KC = H // split_k + SPB = block_k // SUBK + NSTEP = KC // block_k + assert KC % block_k == 0 and block_k % SUBK == 0 + NACC = MT * NB // 64 + NBLK = E // NB + NBG = NBLK // nbw # n-block groups + W_PIECES = nbw * NB * block_k // 4 // (64 * NT) + X_PIECES = MT * block_k // 8 // 64 + assert nbw * NB * block_k // 4 == W_PIECES * 64 * NT and X_PIECES >= 1 + W_SLOT = 3 * SPB * 64 + X_SLOT = SPB * 64 + PPR = block_k // 8 + assert PPR <= 16 + Q8 = SUBK // 8 + + def swz(chunk, kp): + return chunk ^ (kp * (16 // PPR)) + + name = (f"router_gemm_nblocks_lines_bf16x3_h6144_e128_s{split_k}_bk{block_k}_w{NW}" + + ("_token_partials" if token_major else "")) + + @fx.struct + class Shared: + w: fx.Array[fx.Int32, nbw * NSTEP * W_SLOT * 4, 16] + x: fx.Array[fx.Int32, 2 * NW * X_SLOT * 4, 16] + + def mfma_op(a, b, acc): + av = Vec(a).bitcast(fx.BFloat16) + bv = Vec(b).bitcast(fx.BFloat16) + return Vec(fx.rocdl.mfma_f32_16x16x32_bf16(Vec.make_type(NACC, fx.Float32), [av, bv, acc, 0, 0, 0])) + + @flyc.kernel(name=name, known_block_size=[64 * NT, 1, 1]) + def kernel( + arg_x: fx.Int64, # [m, H] bf16, row stride i32_sx + arg_w: fx.Int64, # layout.router_line_layout gate, fp32 + arg_p: fx.Int64, # partials [split_k, m, E] fp32 + i32_m: fx.Int32, + i32_sx: fx.Int32, + ): + tx, bid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + # bid = (g * NBG + nbg) * split_k + s: a split's workgroups share an XCD + s = bid % split_k + nbg = (bid // split_k) % NBG + g = bid // (split_k * NBG) + t0 = (g * NW + wave) * MT + smem = fx.SharedAllocator().allocate(Shared).peek() + + def view16(arr, n): + return fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, arr.ptr), fx.make_layout(n, 1)), + fx.make_layout(4, 1), + ) + + w16 = view16(smem.w, nbw * NSTEP * W_SLOT * 4) + w8 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.w.ptr), fx.make_layout(nbw * NSTEP * W_SLOT * 4, 1)), + fx.make_layout(2, 1), + ) + x16 = view16(smem.x, 2 * NW * X_SLOT * 4) + atom16 = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + atom8 = fx.make_copy_atom(fx.UniversalCopy64b(), fx.Int32) + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=E * H * 4) + xr = buffer_ops.create_buffer_resource_from_addr( + arg_x, num_records_bytes=fx.Int64(i32_m) * fx.Int64(i32_sx * 2) + ) + pr = buffer_ops.create_buffer_resource_from_addr( + arg_p, num_records_bytes=fx.Int64(i32_m) * (split_k * E * 4) + ) + k_base = s * KC + + def x_piece(it): + u = it * 64 + lane + return u // (block_k // 8), u % (block_k // 8) + + xl = [] + for st in range_constexpr(NSTEP): + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + row = t0 + r + xl.append( + Vec( + buffer_ops.buffer_load( + xr, + (row * i32_sx + k_base + st * block_k + kp * 8) >> 1, + vec_width=4, + dtype=fx.Int32, + mask=row < i32_m, + ) + ) + ) + + def w_piece(it): + """(n-block j of the group, expert of the n-block, 4-k piece) of line-layout piece ``it``.""" + u = it * 64 * NT + tx + j = u // (NB * block_k // 4) + piece = u % (NB * block_k // 4) + step = piece // 128 + h = (piece // 64) % 2 + lane_w = piece % 64 + return j, lane_w % 16, step * 8 + (lane_w // 16) * 2 + h + + wl = [] + for st in range_constexpr(NSTEP): + for it in range_constexpr(W_PIECES): + j, n, kp = w_piece(it) + wl.append( + Vec( + buffer_ops.buffer_load( + wr, + ((k_base + st * block_k) // 32 + kp // 8) * (E * 32) + + (nbg * nbw + j) * (NB * 32) + (kp % 2) * 256 + + (kp % 8 // 2 * 16 + n) * 4, + vec_width=4, + dtype=fx.Int32, + ) + ) + ) + + def stage_w(st): + for it in range_constexpr(W_PIECES): + j, n, kp = w_piece(it) + slot = (j * NSTEP + st) * W_SLOT + kq = kp // 2 + chunk = slot + swz((kq // Q8) * 64 + n + NB * (kq % Q8), kq) + parts = _split3([fx.Int32(wl[st * W_PIECES + it][e]) for e in range_constexpr(4)]) + for p in range_constexpr(3): + r8 = fx.make_rmem_tensor(fx.make_layout(2, 1), fx.Int32) + r8.store(Vec.from_elements(parts[p], fx.Int32)) + dst = fx.slice(w8, (None, (chunk + p * SPB * 64) * 2 + kp % 2)) + fx.copy(atom8, r8, dst) + + def stage_x(st): + xslot = ((st % 2) * NW + wave) * X_SLOT + for it in range_constexpr(X_PIECES): + r, kp = x_piece(it) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r16.store(xl[st * X_PIECES + it]) + c = swz((kp // Q8) * 64 + r + MT * (kp % Q8), kp) + fx.copy(atom16, r16, fx.slice(x16, (None, xslot + c))) + + def operands(st): + """x of step ``st`` and each n-block's weight terms, in the MFMA lane order.""" + xslot = ((st % 2) * NW + wave) * X_SLOT + xs, ws = [], [] + for jj in range_constexpr(SPB): + c = swz(jj * 64 + lane, jj * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(x16, (None, xslot + c)), r16) + xs.append(Vec(r16.load())) + for j in range_constexpr(nbw): + slot = (j * NSTEP + st) * W_SLOT + for p in range_constexpr(3): + for jj in range_constexpr(SPB): + c = swz(jj * 64 + lane, jj * Q8 + lane // MT) + r16 = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(atom16, fx.slice(w16, (None, slot + p * SPB * 64 + c)), r16) + ws.append(Vec(r16.load())) + return xs, ws + + for st in range_constexpr(NSTEP): + stage_w(st) + fx.gpu.barrier() + stage_x(0) + acc_hi = [Vec.filled(NACC, 0.0, fx.Float32) for _ in range_constexpr(nbw)] + acc_lo = [Vec.filled(NACC, 0.0, fx.Float32) for _ in range_constexpr(nbw)] + nxt = operands(0) + for st in range_constexpr(NSTEP): + xs, ws = nxt + if const_expr(st + 1 < NSTEP): + stage_x(st + 1) + nxt = operands(st + 1) + fx.rocdl.sched_barrier(0) + for j in range_constexpr(nbw): + wj = ws[j * 3 * SPB:(j + 1) * 3 * SPB] + for jj in range_constexpr(SPB): + acc_hi[j] = mfma_op(wj[jj], xs[jj], acc_hi[j]) + for p in range_constexpr(1, 3): + for jj in range_constexpr(SPB): + acc_lo[j] = mfma_op(wj[p * SPB + jj], xs[jj], acc_lo[j]) + m = t0 + lane % MT + q = lane // MT + for j in range_constexpr(nbw): + out = [fx.Float32(acc_hi[j][i]) + fx.Float32(acc_lo[j][i]) for i in range_constexpr(NACC)] + n = (nbg * nbw + j) * NB + q * 4 + buffer_ops.buffer_store( + Vec.from_elements(out, fx.Float32), pr, ((m * split_k + s) if const_expr(token_major) else (s * i32_m + m)) * E + n, mask=m < i32_m + ) + + @flyc.jit + def launch_router_gemm_nblocks_lines_mxfp4_moe( + arg_x: fx.Int64, + arg_w: fx.Int64, + arg_p: fx.Int64, + i32_m: fx.Int32, + i32_sx: fx.Int32, + stream: fx.Stream, + ): + groups = (i32_m + (MT * NW - 1)) // (MT * NW) + kernel(arg_x, arg_w, arg_p, i32_m, i32_sx).launch( + grid=(fx.Int64(groups * (NBG * split_k)), 1, 1), + block=(64 * NT, 1, 1), + stream=stream, + ) + + launch_router_gemm_nblocks_lines_mxfp4_moe.kernel_name = name + launch_router_gemm_nblocks_lines_mxfp4_moe.compile_hints = {**PRELOAD_HINTS, "waves_per_eu": 2} + return launch_router_gemm_nblocks_lines_mxfp4_moe + + +@functools.cache +def _get_kernel(token_major: bool = False): + return compile_router_gemm_nblocks_lines(token_major) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_topk_mxfp4_mxfp8_quant.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_topk_mxfp4_mxfp8_quant.py new file mode 100644 index 0000000..21958c1 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_gemm_topk_mxfp4_mxfp8_quant.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The router GEMM with the MXFP4 + MXFP8 input quant workgroups in its launch (``router_gemm_nblocks`` ``quant``), +then ``router_topk_sorted``'s top-4: the chains, sum order, tie order and quantizers of the separate launches. The +quant workgroups write disjoint outputs and need no synchronization.""" +import struct + +import torch + +from .router_gemm import nblocks_config, _run_compiled +from .router_gemm_nblocks import _get_kernel as _get_router +from .router_topk_sorted import _get_kernel as _get_topk + +H, E, K = 6144, 128, 4 + + +# (first row, last row, quant 1 appended / 2 prepended): router + top-k + sort chain -0.042..-0.053 us at 58-64 rows +QUANT_TIERS = ((58, 64, 1), (65, 128, 2), (193, 256, 1)) + + +def router_quant_placement(m: int) -> int: + """0 (the quant stays in the top-k launch), 1 appended, 2 prepended.""" + for lo, hi, placement in QUANT_TIERS: + if lo <= m <= hi: + return placement + return 0 + + +def router_dual_quant_for(m: int) -> bool: + """Whether ``m`` takes this launch pair (a ``QUANT_TIERS`` row).""" + return router_quant_placement(m) != 0 + + +def router_gemm_topk_mxfp4_mxfp8_quant(x, router_weight, correction_bias, routed_scaling_factor): + """-> ((MXFP4 codes, scales), (MXFP8 codes, scales), topk weights, topk ids, router partials [m, split, 128]).""" + m = x.shape[0] + assert router_dual_quant_for(m) + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.shape[1] == H + if (router_weight.shape != (E, H) or router_weight.dtype != torch.float32 + or not router_weight.is_contiguous()): + raise ValueError("dual input quant requires the split-K router range") + assert correction_bias.dtype == torch.float32 and correction_bias.is_contiguous() + config = nblocks_config(m) + split = config[0] + token_major = split == 48 or 65 <= m <= 128 + partials = torch.empty((split, m, E), dtype=torch.float32, device=x.device) + ids = torch.empty((m, K), dtype=torch.int32, device=x.device) + weights = torch.empty((m, K), dtype=torch.float32, device=x.device) + q = torch.empty((m, H // 2), dtype=torch.uint8, device=x.device) + s = torch.empty((m, H // 32), dtype=torch.uint8, device=x.device) + q8 = torch.empty((m, H), dtype=torch.uint8, device=x.device) + s8 = torch.empty((m, H // 32), dtype=torch.uint8, device=x.device) + stream = torch.cuda.current_stream(x.device) + _run_compiled(_get_router(*config, router_quant_placement(m), False, token_major), + x.data_ptr(), router_weight.data_ptr(), partials.data_ptr(), m, x.stride(0), + q.data_ptr(), s.data_ptr(), q8.data_ptr(), s8.data_ptr(), stream) + _run_compiled((_get_topk(E, K, split, True) if token_major else _get_topk(E, K, split)), partials.data_ptr(), correction_bias.data_ptr(), + weights.data_ptr(), ids.data_ptr(), partials.numel() * 4, m, split * E if token_major else E, E if token_major else m * E, K, + struct.unpack(" row p // SEGS, segment p % SEGS + piece_rows = [(tx + j * N_THREADS) // SEGS for j in range(PIECES_PER_LANE)] + piece_segs = [(tx + j * N_THREADS) % SEGS for j in range(PIECES_PER_LANE)] + piece_gdw = [ + (row0 + piece_rows[j]) * ROW_DW + k0_dw + piece_segs[j] * 4 + for j in range(PIECES_PER_LANE) + ] + piece_lds = [ + lds_base + piece_rows[j] * LDS_ROW_B + piece_segs[j] * 16 + for j in range(PIECES_PER_LANE) + ] + # this lane's A fragment addresses in a slot (row rt * 16 + r16, K group g) + frag_lds = [ + [ + lds_base + (rt * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + for rt in range(N_RT) + ] + # fp32 B [K/32, wave, half, g, r16, 4]: a wave load reads 64 adjacent 16-byte pieces, whole 128-byte lines + b_row_dw = k0_dw * 2 * E + wave * (2 * 64 * 4) + lane * 4 + # the quant row this wave owns: row tile ``wave`` (row wave * 16 + r16) + qrow = row0 + wave * 16 + r16 + qfrag_lds = [ + lds_base + (wave * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + + def load_a(c): + """This lane's staged pieces of chunk ``c`` (global -> registers); none past the CTA's last chunk.""" + k_dw = c * (KC * 16) + return [ + buffer_ops.buffer_load( + y_rsrc, + piece_gdw[j] + k_dw, + vec_width=4, + dtype=fx.Int32, + mask=(c < NCHUNK).ir_value(), + cache_modifier=_LD_NT, + ) + for j in range(PIECES_PER_LANE) + ] + + def store_a(pieces, slot): + for j in range_constexpr(PIECES_PER_LANE): + _lds_store_vec(pieces[j], piece_lds[j] + slot * SLOT_B, 4) + + def load_b(c): + """fp32 B fragments [half][kt] of chunk ``c``, this wave's n-tile (two 4 x f32 halves of the lane's 8 K).""" + k_dw = c * (KC * 16) * 2 * E + return [ + [ + buffer_ops.buffer_load( + w_rsrc, + b_row_dw + k_dw + kt * 32 * E + hf * 256, + vec_width=4, + dtype=fx.Float32, + ) + for kt in range(KC) + ] + for hf in range(2) + ] + + def split_b(b, kt): + """The k-tile's fp32 B as bf16 hi = RN(w), mid = RN(w - hi), lo = RN(w - hi - mid): the router's split.""" + w = Vec.from_elements( + [fx.Float32(Vec(b[0][kt])[i]) for i in range(4)] + + [fx.Float32(Vec(b[1][kt])[i]) for i in range(4)], + fx.Float32, + ) + hi = w.to(fx.BFloat16) + r1 = w - _bf16_to_f32(hi) + mid = r1.to(fx.BFloat16) + lo = (r1 - _bf16_to_f32(mid)).to(fx.BFloat16) + return hi, mid, lo + + def mfma(a4, b8, acc): + return Vec( + rocdl.mfma_f32_16x16x32_bf16( + T.vec(4, T.f32), + [ + Vec(a4).bitcast(fx.BFloat16).ir_value(), + b8.ir_value(), + acc.ir_value(), + 0, + 0, + 0, + ], + ) + ) + + sb_acc = [] + + def quant_tile(c, hq, kt, a4): + """Quantizes this wave's row tile at k-tile ``kt`` of chunk ``c`` (``a4``: this lane's A fragment of it).""" + v = Vec(a4) # 4 x i32 = 8 bf16 of this lane's K + dw = [fx.Int32(v[d]) for d in range(4)] + am = _amax_bf16(dw) + lo32, hi32 = _swap32(am) + am = _i32_max(lo32, hi32) + ev16, od16 = _swap16(am) + am = _i32_max(ev16, od16) + grp = khalf * (KH // 32) + c * KC + kt + sb4 = _e8m0_e2m1_even_bf16(am) + scale4 = _as_f32(sb4 << 23) + code4 = fx.Int32(0) + for j in range_constexpr(4): + code4 = _cvt_pk_fp4_bf16(code4, dw[j], scale4, j) + _lds_store_vec( + Vec.from_elements([code4], fx.Int32).ir_value(), + q4st_base + r16 * Q4ST_ROW_B + (hq * KC + kt) * 16 + g * 4, + 1, + ) + sb_acc.append(sb4) + # The amax is already common to the four K lanes of this 32-value group. + e8 = _e8m0_fp8(_as_f32(am << 16)) + scale8 = _as_f32(e8 << 23) + pk8 = [] + for j in range_constexpr(2): + a, b = dw[2*j], dw[2*j+1] + z = _fp8(fx.Int32(0), _as_f32(a << 16), _as_f32(a & fx.Int32(-65536)), scale8, False) + z = _fp8(z, _as_f32(b << 16), _as_f32(b & fx.Int32(-65536)), scale8, True) + pk8.append(z) + buffer_ops.buffer_store(Vec.from_elements(pk8, fx.Int32), q8_rsrc, + (qrow * H + grp * 32 + g * 8) // 4) + r32, c8 = qrow % 32, grp % 8 + scale_off = ( + ((qrow // 32) * (H // 256) + grp // 8) * 256 + + (c8 % 4) * 64 + (r32 % 16) * 4 + (c8 // 4) * 2 + r32 // 16 + ) + # The separate scale shuffle reads zero outside the valid input rows. + e8 = (qrow < n_rows).select(e8, fx.Int32(0)) + buffer_ops.buffer_store(e8.to(fx.Int8), s8_rsrc, scale_off, + mask=(g == 0), offset_is_bytes=True) + + def step(c, slot, hq, b, acc_hi, acc_lo): + """One chunk: A fragments from LDS slot ``slot``, MFMAs; each wave also quantizes its row tile (16 B more + LDS reads per k-tile).""" + for kt in range_constexpr(KC): + a_kt = [ + _lds_load_vec(frag_lds[rt][kt] + slot * SLOT_B, 4) + for rt in range(N_RT) + ] + aq = _lds_load_vec(qfrag_lds[kt] + slot * SLOT_B, 4) + # the k-tile's 9 LDS reads ahead of its MFMAs: LLVM sinks each to its MFMAs behind an lgkmcnt(0) (12 + # full waits per chunk); pinned 5 waits, 194 VGPRs, -7..-12% at 2048-32768 rows + fx.rocdl.sched_barrier(0) + b_hi, b_mid, b_lo = split_b(b, kt) + for rt in range_constexpr(N_RT): + acc_hi[rt] = mfma(a_kt[rt], b_hi, acc_hi[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_mid, acc_lo[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_lo, acc_lo[rt]) + quant_tile(c, hq, kt, aq) + return acc_hi, acc_lo + + def store_q(c): + """After chunk ``c``, the last of a group of QC: the wave's 16 rows x QB * 16 B of MXFP4 codes (QB lanes per + row) and QB scale bytes per row in dwords (lanes g == 0).""" + assert len(sb_acc) == QB + grp0 = khalf * (KH // 32) + (c - (QC - 1)) * KC + words = [] + for w4 in range_constexpr(QB // 4): + v = sb_acc[4 * w4] + for j in range_constexpr(1, 4): + v = v | (sb_acc[4 * w4 + j] << (8 * j)) + words.append(v) + sb_acc.clear() + buffer_ops.buffer_store( + Vec.from_elements(words, fx.Int32).ir_value() if len(words) > 1 else words[0], + s4_rsrc, + qrow * SROW_B + grp0, + mask=(g == fx.Int32(0)).ir_value(), + offset_is_bytes=True, + ) + fx.rocdl.s_waitcnt(lgkmcnt=0) + k4_b = grp0 * 16 + RPP = 64 // QB # rows per pass + rq = lane // QB + for ps in range_constexpr(16 // RPP): + v4 = _lds_load_vec(q4st_base + (ps * RPP + rq) * Q4ST_ROW_B + (lane % QB) * 16, 4) + buffer_ops.buffer_store( + v4, + q4_rsrc, + (row0 + wave * 16 + ps * RPP + rq) * (H // 2) + k4_b + (lane % QB) * 16, + offset_is_bytes=True, + ) + + def flat_b(b): + return [b[hf][kt] for hf in range(2) for kt in range(KC)] + + def unflat_b(vals): + return [[vals[hf * KC + kt] for kt in range(KC)] for hf in range(2)] + + # prologue: stage chunk 0; A(1 .. DA) and B(0) in registers + DA = A_AHEAD + store_a(load_a(fx.Int32(0)), 0) + a_q = [load_a(fx.Int32(1 + j)) for j in range_constexpr(DA)] + b0 = load_b(fx.Int32(0)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + zero4 = Vec.filled(4, 0.0, fx.Float32) + n_b = 2 * KC + n_a = PIECES_PER_LANE + init_state = ( + [zero4.ir_value() for _ in range(2 * N_RT)] + + flat_b(b0) + + [v for aa in a_q for v in aa] + ) + def chunk(c, h, hq, acc_hi, acc_lo, b, a_q): + """Chunk c (parity h): write A(c + 1) to slot 1 - h (last read in chunk c - 1, behind the barrier), load + A(c + 1 + DA) (after the compute with ``A_AFTER_COMPUTE``) and B(c + 1), compute chunk c, store a finished + group's codes and scales, barrier.""" + store_a(a_q[0], 1 - h) + a_new = a_q[0] if A_AFTER_COMPUTE else load_a(c + 1 + DA) + b_next = load_b(c + 1) + acc_hi, acc_lo = step(c, h, hq, b, acc_hi, acc_lo) + if const_expr(hq == QC - 1): + store_q(c) + a_new = load_a(c + 1 + DA) if A_AFTER_COMPUTE else a_new + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + return acc_hi, acc_lo, b_next, a_q[1:] + [a_new] + + def unpack(state): + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + o = 2 * N_RT + n_b + a_q = [state[o + j * n_a : o + (j + 1) * n_a] for j in range(DA)] + return acc_hi, acc_lo, b, a_q + + def pack(acc_hi, acc_lo, b, a_q): + return ( + [v.ir_value() for v in acc_hi + acc_lo] + + flat_b(b) + + [v for aa in a_q for v in aa] + ) + + # loop-carried: the accumulators, this chunk's B and the next DA chunks' A. QC (even) chunks per iteration bring + # the A queue and B back to their own registers (one chunk per iteration: 20 v_mov_b64 per chunk) and keep the + # LDS slots and the group's store static; the last QC chunks follow the loop. + for pi, state in range( # type: ignore[call-overload] + 0, (NCHUNK - QC) // QC, 1, init=init_state + ): + acc_hi, acc_lo, b, a_q = unpack(state) + for h in range_constexpr(QC): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(pi) * QC + h, h % 2, h, acc_hi, acc_lo, b, a_q + ) + state = yield pack(acc_hi, acc_lo, b, a_q) + acc_hi, acc_lo, b, a_q = unpack(state) + for j in range_constexpr(QC - 1): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(NCHUNK - QC + j), j % 2, j, acc_hi, acc_lo, b, a_q + ) + state = pack(acc_hi, acc_lo, b, a_q) + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + acc_hi, acc_lo = step( + fx.Int32(NCHUNK - 1), (NCHUNK - 1) % LDS_SLOTS, QC - 1, b, acc_hi, acc_lo + ) + store_q(fx.Int32(NCHUNK - 1)) + # logits of this K slice: the lane holds rows g * 4 + i of each row tile, expert wave * 16 + r16 + for rt in range_constexpr(N_RT): + out = acc_hi[rt] + acc_lo[rt] + for i in range_constexpr(4): + row = row0 + rt * 16 + g * 4 + i + # Mask excess rows: buffer bounds alone include the next K slice. + buffer_ops.buffer_store( + fx.Float32(out[i]), + l_rsrc, + (khalf * n_rows + row) * E + wave * 16 + r16, + mask=(row < n_rows).ir_value(), + cache_modifier=_ST_NT, + ) + + @flyc.jit + def launch_router_mxfp4_mxfp8_quant_line_layout_i1536( + Y: fx.Tensor, + W: fx.Tensor, + Q4: fx.Tensor, + S4: fx.Tensor, + Q8: fx.Tensor, + S8: fx.Tensor, + LOGITS: fx.Tensor, + n_rows: fx.Int32, + stream: fx.Stream, + ): + grid = ((n_rows + BLOCK_M - 1) // BLOCK_M) * K_SPLIT + kernel_router_quant( + Y, + W, + Q4, + S4, + Q8, + S8, + LOGITS, + n_rows, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": f"{N_THREADS},{N_THREADS}", + }, + ).launch(grid=(grid, 1, 1), block=(N_THREADS, 1, 1), stream=stream) + + launch_router_mxfp4_mxfp8_quant_line_layout_i1536.kernel_name = name + return launch_router_mxfp4_mxfp8_quant_line_layout_i1536 + + +def router_mxfp4_mxfp8_quant_line_layout(x: torch.Tensor, gate_weight: torch.Tensor): + """BF16 [M, 6144] -> (MXFP4 codes, scales), (FP8 codes, scales), fp32 [M, S, 128] router partials. + + ``gate_weight``: ``layout.router_line_layout``'s copy. The FP8 scales are 32-row / 256-K tiled, padding rows zero + (as the dense scale shuffle). + """ + m, h = x.shape + assert h == 6144 and x.dtype == torch.bfloat16 and x.is_contiguous() + ks = router_k_split(m) + q4 = torch.empty((m, h // 2), dtype=torch.uint8, device=x.device) + s4 = torch.empty((m, h // 32), dtype=torch.uint8, device=x.device) + q8 = torch.empty((m, h), dtype=torch.uint8, device=x.device) + s8 = torch.empty(((m + 31) // 32 * 32, h // 32), dtype=torch.uint8, device=x.device) + parts = torch.empty((ks, m, 128), dtype=torch.float32, device=x.device) + _run_compiled( + compile_router_mxfp4_mxfp8_quant_line_layout(ks), + x.view(torch.int32).view(-1), gate_weight.view(-1), + q4.view(-1), s4.view(-1), q8.view(-1), s8.view(-1), parts.view(-1), m, + torch.cuda.current_stream(), + ) + return (q4, s4), (q8, s8), parts.permute(1, 0, 2) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_mxfp8_quant_row_scales.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_mxfp8_quant_row_scales.py new file mode 100644 index 0000000..9c77faf --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_mxfp8_quant_row_scales.py @@ -0,0 +1,449 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``router_mxfp4_quant_line_layout.py`` plus the shared expert's MXFP8 quant of the same bf16 fragments (one amax per +32 values), its scales row-major, so token shards concatenate before the dense GEMM's shuffle (gfx950).""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr, rocdl +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .common.epilogue import ( + _lds_load_vec, + _lds_store_vec, +) +from .common.launch import _run_compiled +from .router_mxfp4_quant import ( + _amax_bf16, + _bf16_to_f32, + _cvt_pk_fp4_bf16, + _e8m0_e2m1_even_bf16, + _i32_max, + _swap16, + _swap32, + router_k_split, +) +from .shared_quant import _e8m0_fp8, _fp8 +from .common.loaders import _as_f32 +from amd_kernels.warm_up import launcher + + +BLOCK_M = 128 # rows per CTA (8 MFMA row tiles) +N_WAVES = 8 # one 16-expert n-tile per wave: E = 128 +KT_PER_CHUNK = 2 # k-tiles (32 K each) per LDS stage / quant chunk / load batch: 153 vs 184 us with 4 at 16384 rows +LDS_ROW_B = KT_PER_CHUNK * 64 + 16 # padded A row in LDS (conflict-free reads) +LDS_SLOTS = 2 # A stages: write c + 1 while reading c +_LD_NT = 2 # non-temporal loads of y (read once per K half) +_ST_NT = 0x2 # non-temporal stores for the write-once logits +# A chunks in flight in registers: 2 is -0.5..-2% vs 3 (4: +1%) and keeps every loop-carried value in place +A_AHEAD = 2 + + +@functools.cache +@launcher +def compile_router_mxfp4_mxfp8_quant_row_scales(K_SPLIT: int): + """Per k-tile the hi, mid and lo terms' MFMAs, K split ``K_SPLIT`` ways.""" + H, E = 6144, 128 + KC = KT_PER_CHUNK + assert E == 16 * N_WAVES and H % (K_SPLIT * 32 * KC) == 0 + KH = H // K_SPLIT # K per CTA + NCHUNK = KH // (32 * KC) + ROW_DW = H // 2 # bf16 row in dwords + SROW_B = H // 32 # scale row in bytes + N_RT = BLOCK_M // 16 + N_THREADS = 64 * N_WAVES + PIECES = BLOCK_M * KC * 4 # 16 B pieces of an A stage (row x 64 B per k-tile) + PIECES_PER_LANE = PIECES // N_THREADS + SEGS = KC * 4 # 16 B segments per row per stage + SLOT_B = BLOCK_M * LDS_ROW_B + assert PIECES % N_THREADS == 0 and LDS_SLOTS * SLOT_B <= 160 * 1024 + name = f"router_bf16x3_mxfp4_even_mxfp8_quant_h6144_e128_ks{K_SPLIT}" + name += "_shared_scales_row_major" + assert NCHUNK % 2 == 0 # the 2-chunk loop body + # chunks per code / scale store, 4 (a row's 128 B of codes = one line) or 2: -6% at 32768 rows vs per-k-tile stores + QC = 4 if NCHUNK % 4 == 0 else 2 + assert NCHUNK % QC == 0 + name += f"_store{QC * KC * 16}" + # the next A load after the compute, except in the short split-16 body (same chunks and queue) + A_AFTER_COMPUTE = K_SPLIT != 16 + name += "_weight_lines" + ("_a_after_compute" if A_AFTER_COMPUTE else "_a_ahead") + QB = QC * KC # scale bytes / 16 B code pieces per row and group + Q4ST_ROW_B = QB * 16 + 16 + Q4ST_WAVE_B = 16 * Q4ST_ROW_B + + @fx.struct + class Shared: + a_lds: fx.Array[fx.Int8, LDS_SLOTS * SLOT_B, 16] + q4_lds: fx.Array[fx.Int8, N_WAVES * Q4ST_WAVE_B, 16] + + @flyc.kernel(name=name, known_block_size=[N_THREADS, 1, 1]) + def kernel_router_quant( + Y: fx.Tensor, + W: fx.Tensor, + Q4: fx.Tensor, + S4: fx.Tensor, + Q8: fx.Tensor, + S8: fx.Tensor, + LOGITS: fx.Tensor, + n_rows: fx.Int32, + ): + tx = fx.thread_idx.x + lane = tx % 64 + wave = tx // 64 + g = lane // 16 # K group of the A / B fragment, row group of C + r16 = lane % 16 + row0 = (fx.block_idx.x // K_SPLIT) * BLOCK_M + khalf = fx.block_idx.x % K_SPLIT + k0_dw = khalf * (KH // 2) # this CTA's K start in dwords + lds = fx.SharedAllocator().allocate(Shared).peek() + lds_base = fx.Int32(fx.ptrtoint(lds.a_lds.ptr)) + q4st_base = fx.Int32(fx.ptrtoint(lds.q4_lds.ptr)) + wave * Q4ST_WAVE_B + y_rsrc = buffer_ops.create_buffer_resource( + Y, max_size=False, num_records_bytes=n_rows * (H * 2) + ) + w_rsrc = buffer_ops.create_buffer_resource(W, max_size=True) + q4_rsrc = buffer_ops.create_buffer_resource( + Q4, max_size=False, num_records_bytes=n_rows * (H // 2) + ) + s4_rsrc = buffer_ops.create_buffer_resource( + S4, max_size=False, num_records_bytes=n_rows * SROW_B + ) + q8_rsrc = buffer_ops.create_buffer_resource( + Q8, max_size=False, num_records_bytes=n_rows * H + ) + s8_rsrc = buffer_ops.create_buffer_resource( + S8, max_size=False, num_records_bytes=n_rows * SROW_B + ) + l_rsrc = buffer_ops.create_buffer_resource( + LOGITS, max_size=False, num_records_bytes=n_rows * (K_SPLIT * E * 4) + ) + # the A pieces this lane stages: piece p = tx + j * N_THREADS -> row p // SEGS, segment p % SEGS + piece_rows = [(tx + j * N_THREADS) // SEGS for j in range(PIECES_PER_LANE)] + piece_segs = [(tx + j * N_THREADS) % SEGS for j in range(PIECES_PER_LANE)] + piece_gdw = [ + (row0 + piece_rows[j]) * ROW_DW + k0_dw + piece_segs[j] * 4 + for j in range(PIECES_PER_LANE) + ] + piece_lds = [ + lds_base + piece_rows[j] * LDS_ROW_B + piece_segs[j] * 16 + for j in range(PIECES_PER_LANE) + ] + # this lane's A fragment addresses in a slot (row rt * 16 + r16, K group g) + frag_lds = [ + [ + lds_base + (rt * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + for rt in range(N_RT) + ] + # fp32 B [K/32, wave, half, g, r16, 4]: a wave load reads 64 adjacent 16-byte pieces, whole 128-byte lines + b_row_dw = k0_dw * 2 * E + wave * (2 * 64 * 4) + lane * 4 + # the quant row this wave owns: row tile ``wave`` (row wave * 16 + r16) + qrow = row0 + wave * 16 + r16 + qfrag_lds = [ + lds_base + (wave * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + + def load_a(c): + """This lane's staged pieces of chunk ``c`` (global -> registers); none past the CTA's last chunk.""" + k_dw = c * (KC * 16) + return [ + buffer_ops.buffer_load( + y_rsrc, + piece_gdw[j] + k_dw, + vec_width=4, + dtype=fx.Int32, + mask=(c < NCHUNK).ir_value(), + cache_modifier=_LD_NT, + ) + for j in range(PIECES_PER_LANE) + ] + + def store_a(pieces, slot): + for j in range_constexpr(PIECES_PER_LANE): + _lds_store_vec(pieces[j], piece_lds[j] + slot * SLOT_B, 4) + + def load_b(c): + """fp32 B fragments [half][kt] of chunk ``c``, this wave's n-tile (two 4 x f32 halves of the lane's 8 K).""" + k_dw = c * (KC * 16) * 2 * E + return [ + [ + buffer_ops.buffer_load( + w_rsrc, + b_row_dw + k_dw + kt * 32 * E + hf * 256, + vec_width=4, + dtype=fx.Float32, + ) + for kt in range(KC) + ] + for hf in range(2) + ] + + def split_b(b, kt): + """The k-tile's fp32 B as bf16 hi = RN(w), mid = RN(w - hi), lo = RN(w - hi - mid): the router's split.""" + w = Vec.from_elements( + [fx.Float32(Vec(b[0][kt])[i]) for i in range(4)] + + [fx.Float32(Vec(b[1][kt])[i]) for i in range(4)], + fx.Float32, + ) + hi = w.to(fx.BFloat16) + r1 = w - _bf16_to_f32(hi) + mid = r1.to(fx.BFloat16) + lo = (r1 - _bf16_to_f32(mid)).to(fx.BFloat16) + return hi, mid, lo + + def mfma(a4, b8, acc): + return Vec( + rocdl.mfma_f32_16x16x32_bf16( + T.vec(4, T.f32), + [ + Vec(a4).bitcast(fx.BFloat16).ir_value(), + b8.ir_value(), + acc.ir_value(), + 0, + 0, + 0, + ], + ) + ) + + sb_acc = [] + + def quant_tile(c, hq, kt, a4): + """Quantizes this wave's row tile at k-tile ``kt`` of chunk ``c`` (``a4``: this lane's A fragment of it).""" + v = Vec(a4) # 4 x i32 = 8 bf16 of this lane's K + dw = [fx.Int32(v[d]) for d in range(4)] + am = _amax_bf16(dw) + lo32, hi32 = _swap32(am) + am = _i32_max(lo32, hi32) + ev16, od16 = _swap16(am) + am = _i32_max(ev16, od16) + grp = khalf * (KH // 32) + c * KC + kt + sb4 = _e8m0_e2m1_even_bf16(am) + scale4 = _as_f32(sb4 << 23) + code4 = fx.Int32(0) + for j in range_constexpr(4): + code4 = _cvt_pk_fp4_bf16(code4, dw[j], scale4, j) + _lds_store_vec( + Vec.from_elements([code4], fx.Int32).ir_value(), + q4st_base + r16 * Q4ST_ROW_B + (hq * KC + kt) * 16 + g * 4, + 1, + ) + sb_acc.append(sb4) + # The amax is already common to the four K lanes of this 32-value group. + e8 = _e8m0_fp8(_as_f32(am << 16)) + scale8 = _as_f32(e8 << 23) + pk8 = [] + for j in range_constexpr(2): + a, b = dw[2*j], dw[2*j+1] + z = _fp8(fx.Int32(0), _as_f32(a << 16), _as_f32(a & fx.Int32(-65536)), scale8, False) + z = _fp8(z, _as_f32(b << 16), _as_f32(b & fx.Int32(-65536)), scale8, True) + pk8.append(z) + buffer_ops.buffer_store(Vec.from_elements(pk8, fx.Int32), q8_rsrc, + (qrow * H + grp * 32 + g * 8) // 4) + scale_off = qrow * SROW_B + grp + # Buffer bounds drop stores outside the valid input rows. + e8 = (qrow < n_rows).select(e8, fx.Int32(0)) + buffer_ops.buffer_store(e8.to(fx.Int8), s8_rsrc, scale_off, + mask=(g == 0), offset_is_bytes=True) + + def step(c, slot, hq, b, acc_hi, acc_lo): + """One chunk: A fragments from LDS slot ``slot``, MFMAs; each wave also quantizes its row tile (16 B more + LDS reads per k-tile).""" + for kt in range_constexpr(KC): + a_kt = [ + _lds_load_vec(frag_lds[rt][kt] + slot * SLOT_B, 4) + for rt in range(N_RT) + ] + aq = _lds_load_vec(qfrag_lds[kt] + slot * SLOT_B, 4) + # the k-tile's 9 LDS reads ahead of its MFMAs: LLVM sinks each to its MFMAs behind an lgkmcnt(0) (12 + # full waits per chunk); pinned 5 waits, 194 VGPRs, -7..-12% at 2048-32768 rows + fx.rocdl.sched_barrier(0) + b_hi, b_mid, b_lo = split_b(b, kt) + for rt in range_constexpr(N_RT): + acc_hi[rt] = mfma(a_kt[rt], b_hi, acc_hi[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_mid, acc_lo[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_lo, acc_lo[rt]) + quant_tile(c, hq, kt, aq) + return acc_hi, acc_lo + + def store_q(c): + """After chunk ``c``, the last of a group of QC: the wave's 16 rows x QB * 16 B of MXFP4 codes (QB lanes per + row) and QB scale bytes per row in dwords (lanes g == 0).""" + assert len(sb_acc) == QB + grp0 = khalf * (KH // 32) + (c - (QC - 1)) * KC + words = [] + for w4 in range_constexpr(QB // 4): + v = sb_acc[4 * w4] + for j in range_constexpr(1, 4): + v = v | (sb_acc[4 * w4 + j] << (8 * j)) + words.append(v) + sb_acc.clear() + buffer_ops.buffer_store( + Vec.from_elements(words, fx.Int32).ir_value() if len(words) > 1 else words[0], + s4_rsrc, + qrow * SROW_B + grp0, + mask=(g == fx.Int32(0)).ir_value(), + offset_is_bytes=True, + ) + fx.rocdl.s_waitcnt(lgkmcnt=0) + k4_b = grp0 * 16 + RPP = 64 // QB # rows per pass + rq = lane // QB + for ps in range_constexpr(16 // RPP): + v4 = _lds_load_vec(q4st_base + (ps * RPP + rq) * Q4ST_ROW_B + (lane % QB) * 16, 4) + buffer_ops.buffer_store( + v4, + q4_rsrc, + (row0 + wave * 16 + ps * RPP + rq) * (H // 2) + k4_b + (lane % QB) * 16, + offset_is_bytes=True, + ) + + def flat_b(b): + return [b[hf][kt] for hf in range(2) for kt in range(KC)] + + def unflat_b(vals): + return [[vals[hf * KC + kt] for kt in range(KC)] for hf in range(2)] + + # prologue: stage chunk 0; A(1 .. DA) and B(0) in registers + DA = A_AHEAD + store_a(load_a(fx.Int32(0)), 0) + a_q = [load_a(fx.Int32(1 + j)) for j in range_constexpr(DA)] + b0 = load_b(fx.Int32(0)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + zero4 = Vec.filled(4, 0.0, fx.Float32) + n_b = 2 * KC + n_a = PIECES_PER_LANE + init_state = ( + [zero4.ir_value() for _ in range(2 * N_RT)] + + flat_b(b0) + + [v for aa in a_q for v in aa] + ) + def chunk(c, h, hq, acc_hi, acc_lo, b, a_q): + """Chunk c (parity h): write A(c + 1) to slot 1 - h (last read in chunk c - 1, behind the barrier), load + A(c + 1 + DA) (after the compute with ``A_AFTER_COMPUTE``) and B(c + 1), compute chunk c, store a finished + group's codes and scales, barrier.""" + store_a(a_q[0], 1 - h) + a_new = a_q[0] if A_AFTER_COMPUTE else load_a(c + 1 + DA) + b_next = load_b(c + 1) + acc_hi, acc_lo = step(c, h, hq, b, acc_hi, acc_lo) + if const_expr(hq == QC - 1): + store_q(c) + a_new = load_a(c + 1 + DA) if A_AFTER_COMPUTE else a_new + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + return acc_hi, acc_lo, b_next, a_q[1:] + [a_new] + + def unpack(state): + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + o = 2 * N_RT + n_b + a_q = [state[o + j * n_a : o + (j + 1) * n_a] for j in range(DA)] + return acc_hi, acc_lo, b, a_q + + def pack(acc_hi, acc_lo, b, a_q): + return ( + [v.ir_value() for v in acc_hi + acc_lo] + + flat_b(b) + + [v for aa in a_q for v in aa] + ) + + # loop-carried: the accumulators, this chunk's B and the next DA chunks' A. QC (even) chunks per iteration bring + # the A queue and B back to their own registers (one chunk per iteration: 20 v_mov_b64 per chunk) and keep the + # LDS slots and the group's store static; the last QC chunks follow the loop. + for pi, state in range( # type: ignore[call-overload] + 0, (NCHUNK - QC) // QC, 1, init=init_state + ): + acc_hi, acc_lo, b, a_q = unpack(state) + for h in range_constexpr(QC): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(pi) * QC + h, h % 2, h, acc_hi, acc_lo, b, a_q + ) + state = yield pack(acc_hi, acc_lo, b, a_q) + acc_hi, acc_lo, b, a_q = unpack(state) + for j in range_constexpr(QC - 1): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(NCHUNK - QC + j), j % 2, j, acc_hi, acc_lo, b, a_q + ) + state = pack(acc_hi, acc_lo, b, a_q) + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + acc_hi, acc_lo = step( + fx.Int32(NCHUNK - 1), (NCHUNK - 1) % LDS_SLOTS, QC - 1, b, acc_hi, acc_lo + ) + store_q(fx.Int32(NCHUNK - 1)) + # logits of this K slice: the lane holds rows g * 4 + i of each row tile, expert wave * 16 + r16 + for rt in range_constexpr(N_RT): + out = acc_hi[rt] + acc_lo[rt] + for i in range_constexpr(4): + row = row0 + rt * 16 + g * 4 + i + # Mask excess rows: buffer bounds alone include the next K slice. + buffer_ops.buffer_store( + fx.Float32(out[i]), + l_rsrc, + (khalf * n_rows + row) * E + wave * 16 + r16, + mask=(row < n_rows).ir_value(), + cache_modifier=_ST_NT, + ) + + @flyc.jit + def launch_router_mxfp4_mxfp8_quant_row_scales_i1536( + Y: fx.Tensor, + W: fx.Tensor, + Q4: fx.Tensor, + S4: fx.Tensor, + Q8: fx.Tensor, + S8: fx.Tensor, + LOGITS: fx.Tensor, + n_rows: fx.Int32, + stream: fx.Stream, + ): + grid = ((n_rows + BLOCK_M - 1) // BLOCK_M) * K_SPLIT + kernel_router_quant( + Y, + W, + Q4, + S4, + Q8, + S8, + LOGITS, + n_rows, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": f"{N_THREADS},{N_THREADS}", + }, + ).launch(grid=(grid, 1, 1), block=(N_THREADS, 1, 1), stream=stream) + + launch_router_mxfp4_mxfp8_quant_row_scales_i1536.kernel_name = name + return launch_router_mxfp4_mxfp8_quant_row_scales_i1536 + + +def router_mxfp4_mxfp8_quant_row_scales(x: torch.Tensor, gate_weight: torch.Tensor): + """BF16 [M, 6144] -> (MXFP4 codes, scales), (FP8 codes, scales), fp32 [M, S, 128] router partials. + + ``gate_weight``: ``layout.router_line_layout``'s copy. The FP8 scales are row-major [M, 192], without padding rows. + """ + m, h = x.shape + assert h == 6144 and x.dtype == torch.bfloat16 and x.is_contiguous() + ks = router_k_split(m) + q4 = torch.empty((m, h // 2), dtype=torch.uint8, device=x.device) + s4 = torch.empty((m, h // 32), dtype=torch.uint8, device=x.device) + q8 = torch.empty((m, h), dtype=torch.uint8, device=x.device) + s8 = torch.empty((m, h // 32), dtype=torch.uint8, device=x.device) + parts = torch.empty((ks, m, 128), dtype=torch.float32, device=x.device) + _run_compiled( + compile_router_mxfp4_mxfp8_quant_row_scales(ks), + x.view(torch.int32).view(-1), gate_weight.view(-1), + q4.view(-1), s4.view(-1), q8.view(-1), s8.view(-1), parts.view(-1), m, + torch.cuda.current_stream(), + ) + return (q4, s4), (q8, s8), parts.permute(1, 0, 2) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant.py new file mode 100644 index 0000000..f2600ba --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Helpers of the fused router + input quant kernels (``router_mxfp4_quant_line_layout``, +``router_mxfp4_mxfp8_quant_line_layout``, ``router_mxfp4_mxfp8_quant_row_scales``): the bf16 amax, the e2m1 converter, +the Quark-even e8m0 of a bf16 amax, the lane swaps, and the K split per row count.""" + +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .a16w4_utils import _raw +from .common.loaders import _as_f32 + + +def _pk_max_u16(a, b): + """v_pk_max_u16, per 16-bit half: on sign-cleared bf16 patterns the unsigned order is the value order.""" + va = _llvm.BitcastOp(T.i16x2, _raw(fx.Int32(a))).result + vb = _llvm.BitcastOp(T.i16x2, _raw(fx.Int32(b))).result + return fx.Int32(_llvm.BitcastOp(T.i32, _llvm.intr_umax(va, vb)).result) + +def _amax_bf16(dwords): + """max |x| over the 2 * len(dwords) bf16 values, as a 16-bit bf16 pattern in an i32.""" + m = None + for d in dwords: + a = fx.Int32(d) & fx.Int32(0x7FFF7FFF) + m = a if m is None else _pk_max_u16(m, a) + lo, hi = m & fx.Int32(0xFFFF), m >> 16 + return (lo > hi).select(lo, hi) + +def _i32_max(a, b): + return (a > b).select(a, b) + +def _swap16(v): + """(even-row value, odd-row value) of this lane's 16-lane row pair (klane q, q ^ 1).""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane16_swap(pair_ty, raw, raw, False, False) + return fx.Int32(_llvm.extractvalue(T.i32, res, [0])), fx.Int32( + _llvm.extractvalue(T.i32, res, [1]) + ) + +def _swap32(v): + """(low-half value, high-half value): the values of klanes 0/1 and 2/3.""" + pair_ty = ir.Type.parse("!llvm.struct<(i32, i32)>") + raw = _raw(fx.Int32(v)) + res = fx.rocdl.permlane32_swap(pair_ty, raw, raw, False, False) + return fx.Int32(_llvm.extractvalue(T.i32, res, [0])), fx.Int32( + _llvm.extractvalue(T.i32, res, [1]) + ) + + +def _cvt_pk_fp4_bf16(old, dword, scale_f32, sel): + """v_cvt_scalef32_pk_fp4_bf16: the two bf16 of ``dword`` (low first) / scale -> 2 e2m1 in byte ``sel``, the f32 + form's codes (a bf16 is exact in f32) without unpacking the pair.""" + pair = _llvm.BitcastOp(T.bf16x2, fx.Int32(dword).ir_value()).result + return fx.Int32( + _llvm.call_intrinsic( + T.i32, + "llvm.amdgcn.cvt.scalef32.pk.fp4.bf16", + [ + fx.Int32(old).ir_value(), + pair, + scale_f32.ir_value(), + fx.Int32(sel).ir_value(), + ], + [], + [], + ) + ) + + +def _bf16_to_f32(v): + """8 x bf16 (2 per dword) -> 8 x f32 by a shift / a mask per value; ``.to(fx.Float32)`` takes 16 VALU per 8 (a + second v_cvt_pk_bf16_f32 of every source value plus the shift).""" + d = v.bitcast(fx.Int32) + return Vec.from_elements( + [ + _as_f32(x) + for i in range(4) + for x in (fx.Int32(d[i]) << 16, fx.Int32(d[i]) & fx.Int32(-65536)) + ], + fx.Float32, + ) + + +def _e8m0_e2m1_even_bf16(amax_bits): + """Quark even scale of a BF16 amax: carry at mantissa 0x60, byte >= 1.""" + t = (fx.Int32(amax_bits) - fx.Int32(0xE0)) >> 7 + return _i32_max(t, fx.Int32(1)) + + +# (max rows, K split) that fill whole CU rounds: router + top-k -2..-18 us vs a neighbour split at 2049-21760 rows +# (above the largest DBO microbatch only a BF16 call with sequence parallelism off comes: the 10881-16384 split) +K_SPLITS = ((2048, 16), (2688, 12), (4096, 8), (5376, 6), (8192, 4), (10880, 3), (16384, 2), (21760, 3)) +K_SPLIT_ABOVE = 2 + + +def router_k_split(m: int) -> int: + for bound, ks in K_SPLITS: + if m <= bound: + return ks + return K_SPLIT_ABOVE diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant_line_layout.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant_line_layout.py new file mode 100644 index 0000000..f80ed91 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_mxfp4_quant_line_layout.py @@ -0,0 +1,417 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``router_mxfp4_quant.py`` on the line-layout gate copy (gfx950): a wave reads the fp32 gate in whole 128-byte lines, +and the next x chunk loads after the compute except in the short split-16 body.""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr, rocdl +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .common.epilogue import ( + _lds_load_vec, + _lds_store_vec, +) +from .common.launch import _run_compiled +from .router_mxfp4_quant import ( + _amax_bf16, + _bf16_to_f32, + _cvt_pk_fp4_bf16, + _e8m0_e2m1_even_bf16, + _i32_max, + _swap16, + _swap32, + router_k_split, +) +from .common.loaders import _as_f32 +from amd_kernels.warm_up import launcher + + +BLOCK_M = 128 # rows per CTA (8 MFMA row tiles) +N_WAVES = 8 # one 16-expert n-tile per wave: E = 128 +KT_PER_CHUNK = 2 # k-tiles (32 K each) per LDS stage / quant chunk / load batch: 153 vs 184 us with 4 at 16384 rows +LDS_ROW_B = KT_PER_CHUNK * 64 + 16 # padded A row in LDS (conflict-free reads) +LDS_SLOTS = 2 # A stages: write c + 1 while reading c +_LD_NT = 2 # non-temporal loads of y (read once per K half) +_ST_NT = 0x2 # non-temporal stores for the write-once logits +# A chunks in flight in registers: 2 is -0.5..-2% vs 3 (4: +1%) and keeps every loop-carried value in place +A_AHEAD = 2 + + +@functools.cache +@launcher +def compile_router_mxfp4_quant_line_layout(K_SPLIT: int): + """Per k-tile the hi, mid and lo terms' MFMAs, K split ``K_SPLIT`` ways.""" + H, E = 6144, 128 + KC = KT_PER_CHUNK + assert E == 16 * N_WAVES and H % (K_SPLIT * 32 * KC) == 0 + KH = H // K_SPLIT # K per CTA + NCHUNK = KH // (32 * KC) + ROW_DW = H // 2 # bf16 row in dwords + SROW_B = H // 32 # scale row in bytes + N_RT = BLOCK_M // 16 + N_THREADS = 64 * N_WAVES + PIECES = BLOCK_M * KC * 4 # 16 B pieces of an A stage (row x 64 B per k-tile) + PIECES_PER_LANE = PIECES // N_THREADS + SEGS = KC * 4 # 16 B segments per row per stage + SLOT_B = BLOCK_M * LDS_ROW_B + assert PIECES % N_THREADS == 0 and LDS_SLOTS * SLOT_B <= 160 * 1024 + name = f"router_bf16x3_mxfp4_even_quant_h6144_e128_ks{K_SPLIT}" + assert NCHUNK % 2 == 0 # the 2-chunk loop body + # chunks per code / scale store, 4 (a row's 128 B of codes = one line) or 2: -6% at 32768 rows vs per-k-tile stores + QC = 4 if NCHUNK % 4 == 0 else 2 + assert NCHUNK % QC == 0 + name += f"_store{QC * KC * 16}" + # the next A load after the compute, except in the short split-16 body (same chunks and queue) + A_AFTER_COMPUTE = K_SPLIT != 16 + name += "_weight_lines" + ("_a_after_compute" if A_AFTER_COMPUTE else "_a_ahead") + QB = QC * KC # scale bytes / 16 B code pieces per row and group + Q4ST_ROW_B = QB * 16 + 16 + Q4ST_WAVE_B = 16 * Q4ST_ROW_B + + @fx.struct + class Shared: + a_lds: fx.Array[fx.Int8, LDS_SLOTS * SLOT_B, 16] + q4_lds: fx.Array[fx.Int8, N_WAVES * Q4ST_WAVE_B, 16] + + @flyc.kernel(name=name, known_block_size=[N_THREADS, 1, 1]) + def kernel_router_quant( + Y: fx.Tensor, + W: fx.Tensor, + Q4: fx.Tensor, + S4: fx.Tensor, + LOGITS: fx.Tensor, + n_rows: fx.Int32, + ): + tx = fx.thread_idx.x + lane = tx % 64 + wave = tx // 64 + g = lane // 16 # K group of the A / B fragment, row group of C + r16 = lane % 16 + row0 = (fx.block_idx.x // K_SPLIT) * BLOCK_M + khalf = fx.block_idx.x % K_SPLIT + k0_dw = khalf * (KH // 2) # this CTA's K start in dwords + lds = fx.SharedAllocator().allocate(Shared).peek() + lds_base = fx.Int32(fx.ptrtoint(lds.a_lds.ptr)) + q4st_base = fx.Int32(fx.ptrtoint(lds.q4_lds.ptr)) + wave * Q4ST_WAVE_B + y_rsrc = buffer_ops.create_buffer_resource( + Y, max_size=False, num_records_bytes=n_rows * (H * 2) + ) + w_rsrc = buffer_ops.create_buffer_resource(W, max_size=True) + q4_rsrc = buffer_ops.create_buffer_resource( + Q4, max_size=False, num_records_bytes=n_rows * (H // 2) + ) + s4_rsrc = buffer_ops.create_buffer_resource( + S4, max_size=False, num_records_bytes=n_rows * SROW_B + ) + l_rsrc = buffer_ops.create_buffer_resource( + LOGITS, max_size=False, num_records_bytes=n_rows * (K_SPLIT * E * 4) + ) + # the A pieces this lane stages: piece p = tx + j * N_THREADS -> row p // SEGS, segment p % SEGS + piece_rows = [(tx + j * N_THREADS) // SEGS for j in range(PIECES_PER_LANE)] + piece_segs = [(tx + j * N_THREADS) % SEGS for j in range(PIECES_PER_LANE)] + piece_gdw = [ + (row0 + piece_rows[j]) * ROW_DW + k0_dw + piece_segs[j] * 4 + for j in range(PIECES_PER_LANE) + ] + piece_lds = [ + lds_base + piece_rows[j] * LDS_ROW_B + piece_segs[j] * 16 + for j in range(PIECES_PER_LANE) + ] + # this lane's A fragment addresses in a slot (row rt * 16 + r16, K group g) + frag_lds = [ + [ + lds_base + (rt * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + for rt in range(N_RT) + ] + # fp32 B [K/32, wave, half, g, r16, 4]: a wave load reads 64 adjacent 16-byte pieces, whole 128-byte lines + b_row_dw = k0_dw * 2 * E + wave * (2 * 64 * 4) + lane * 4 + # the quant row this wave owns: row tile ``wave`` (row wave * 16 + r16) + qrow = row0 + wave * 16 + r16 + qfrag_lds = [ + lds_base + (wave * 16 + r16) * LDS_ROW_B + kt * 64 + g * 16 + for kt in range(KC) + ] + + def load_a(c): + """This lane's staged pieces of chunk ``c`` (global -> registers); none past the CTA's last chunk.""" + k_dw = c * (KC * 16) + return [ + buffer_ops.buffer_load( + y_rsrc, + piece_gdw[j] + k_dw, + vec_width=4, + dtype=fx.Int32, + mask=(c < NCHUNK).ir_value(), + cache_modifier=_LD_NT, + ) + for j in range(PIECES_PER_LANE) + ] + + def store_a(pieces, slot): + for j in range_constexpr(PIECES_PER_LANE): + _lds_store_vec(pieces[j], piece_lds[j] + slot * SLOT_B, 4) + + def load_b(c): + """fp32 B fragments [half][kt] of chunk ``c``, this wave's n-tile (two 4 x f32 halves of the lane's 8 K).""" + k_dw = c * (KC * 16) * 2 * E + return [ + [ + buffer_ops.buffer_load( + w_rsrc, + b_row_dw + k_dw + kt * 32 * E + hf * 256, + vec_width=4, + dtype=fx.Float32, + ) + for kt in range(KC) + ] + for hf in range(2) + ] + + def split_b(b, kt): + """The k-tile's fp32 B as bf16 hi = RN(w), mid = RN(w - hi), lo = RN(w - hi - mid): the router's split.""" + w = Vec.from_elements( + [fx.Float32(Vec(b[0][kt])[i]) for i in range(4)] + + [fx.Float32(Vec(b[1][kt])[i]) for i in range(4)], + fx.Float32, + ) + hi = w.to(fx.BFloat16) + r1 = w - _bf16_to_f32(hi) + mid = r1.to(fx.BFloat16) + lo = (r1 - _bf16_to_f32(mid)).to(fx.BFloat16) + return hi, mid, lo + + def mfma(a4, b8, acc): + return Vec( + rocdl.mfma_f32_16x16x32_bf16( + T.vec(4, T.f32), + [ + Vec(a4).bitcast(fx.BFloat16).ir_value(), + b8.ir_value(), + acc.ir_value(), + 0, + 0, + 0, + ], + ) + ) + + sb_acc = [] + + def quant_tile(c, hq, kt, a4): + """Quantizes this wave's row tile at k-tile ``kt`` of chunk ``c`` (``a4``: this lane's A fragment of it).""" + v = Vec(a4) # 4 x i32 = 8 bf16 of this lane's K + dw = [fx.Int32(v[d]) for d in range(4)] + am = _amax_bf16(dw) + lo32, hi32 = _swap32(am) + am = _i32_max(lo32, hi32) + ev16, od16 = _swap16(am) + am = _i32_max(ev16, od16) + grp = khalf * (KH // 32) + c * KC + kt + sb4 = _e8m0_e2m1_even_bf16(am) + scale4 = _as_f32(sb4 << 23) + code4 = fx.Int32(0) + for j in range_constexpr(4): + code4 = _cvt_pk_fp4_bf16(code4, dw[j], scale4, j) + _lds_store_vec( + Vec.from_elements([code4], fx.Int32).ir_value(), + q4st_base + r16 * Q4ST_ROW_B + (hq * KC + kt) * 16 + g * 4, + 1, + ) + sb_acc.append(sb4) + + def step(c, slot, hq, b, acc_hi, acc_lo): + """One chunk: A fragments from LDS slot ``slot``, MFMAs; each wave also quantizes its row tile (16 B more + LDS reads per k-tile).""" + for kt in range_constexpr(KC): + a_kt = [ + _lds_load_vec(frag_lds[rt][kt] + slot * SLOT_B, 4) + for rt in range(N_RT) + ] + aq = _lds_load_vec(qfrag_lds[kt] + slot * SLOT_B, 4) + # the k-tile's 9 LDS reads ahead of its MFMAs: LLVM sinks each to its MFMAs behind an lgkmcnt(0) (12 + # full waits per chunk); pinned 5 waits, 194 VGPRs, -7..-12% at 2048-32768 rows + fx.rocdl.sched_barrier(0) + b_hi, b_mid, b_lo = split_b(b, kt) + for rt in range_constexpr(N_RT): + acc_hi[rt] = mfma(a_kt[rt], b_hi, acc_hi[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_mid, acc_lo[rt]) + acc_lo[rt] = mfma(a_kt[rt], b_lo, acc_lo[rt]) + quant_tile(c, hq, kt, aq) + return acc_hi, acc_lo + + def store_q(c): + """After chunk ``c``, the last of a group of QC: the wave's 16 rows x QB * 16 B of MXFP4 codes (QB lanes per + row) and QB scale bytes per row in dwords (lanes g == 0).""" + assert len(sb_acc) == QB + grp0 = khalf * (KH // 32) + (c - (QC - 1)) * KC + words = [] + for w4 in range_constexpr(QB // 4): + v = sb_acc[4 * w4] + for j in range_constexpr(1, 4): + v = v | (sb_acc[4 * w4 + j] << (8 * j)) + words.append(v) + sb_acc.clear() + buffer_ops.buffer_store( + Vec.from_elements(words, fx.Int32).ir_value() if len(words) > 1 else words[0], + s4_rsrc, + qrow * SROW_B + grp0, + mask=(g == fx.Int32(0)).ir_value(), + offset_is_bytes=True, + ) + fx.rocdl.s_waitcnt(lgkmcnt=0) + k4_b = grp0 * 16 + RPP = 64 // QB # rows per pass + rq = lane // QB + for ps in range_constexpr(16 // RPP): + v4 = _lds_load_vec(q4st_base + (ps * RPP + rq) * Q4ST_ROW_B + (lane % QB) * 16, 4) + buffer_ops.buffer_store( + v4, + q4_rsrc, + (row0 + wave * 16 + ps * RPP + rq) * (H // 2) + k4_b + (lane % QB) * 16, + offset_is_bytes=True, + ) + + def flat_b(b): + return [b[hf][kt] for hf in range(2) for kt in range(KC)] + + def unflat_b(vals): + return [[vals[hf * KC + kt] for kt in range(KC)] for hf in range(2)] + + # prologue: stage chunk 0; A(1 .. DA) and B(0) in registers + DA = A_AHEAD + store_a(load_a(fx.Int32(0)), 0) + a_q = [load_a(fx.Int32(1 + j)) for j in range_constexpr(DA)] + b0 = load_b(fx.Int32(0)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + zero4 = Vec.filled(4, 0.0, fx.Float32) + n_b = 2 * KC + n_a = PIECES_PER_LANE + init_state = ( + [zero4.ir_value() for _ in range(2 * N_RT)] + + flat_b(b0) + + [v for aa in a_q for v in aa] + ) + def chunk(c, h, hq, acc_hi, acc_lo, b, a_q): + """Chunk c (parity h): write A(c + 1) to slot 1 - h (last read in chunk c - 1, behind the barrier), load + A(c + 1 + DA) (after the compute with ``A_AFTER_COMPUTE``) and B(c + 1), compute chunk c, store a finished + group's codes and scales, barrier.""" + store_a(a_q[0], 1 - h) + a_new = a_q[0] if A_AFTER_COMPUTE else load_a(c + 1 + DA) + b_next = load_b(c + 1) + acc_hi, acc_lo = step(c, h, hq, b, acc_hi, acc_lo) + if const_expr(hq == QC - 1): + store_q(c) + a_new = load_a(c + 1 + DA) if A_AFTER_COMPUTE else a_new + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + return acc_hi, acc_lo, b_next, a_q[1:] + [a_new] + + def unpack(state): + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + o = 2 * N_RT + n_b + a_q = [state[o + j * n_a : o + (j + 1) * n_a] for j in range(DA)] + return acc_hi, acc_lo, b, a_q + + def pack(acc_hi, acc_lo, b, a_q): + return ( + [v.ir_value() for v in acc_hi + acc_lo] + + flat_b(b) + + [v for aa in a_q for v in aa] + ) + + # loop-carried: the accumulators, this chunk's B and the next DA chunks' A. QC (even) chunks per iteration bring + # the A queue and B back to their own registers (one chunk per iteration: 20 v_mov_b64 per chunk) and keep the + # LDS slots and the group's store static; the last QC chunks follow the loop. + for pi, state in range( # type: ignore[call-overload] + 0, (NCHUNK - QC) // QC, 1, init=init_state + ): + acc_hi, acc_lo, b, a_q = unpack(state) + for h in range_constexpr(QC): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(pi) * QC + h, h % 2, h, acc_hi, acc_lo, b, a_q + ) + state = yield pack(acc_hi, acc_lo, b, a_q) + acc_hi, acc_lo, b, a_q = unpack(state) + for j in range_constexpr(QC - 1): + acc_hi, acc_lo, b, a_q = chunk( + fx.Int32(NCHUNK - QC + j), j % 2, j, acc_hi, acc_lo, b, a_q + ) + state = pack(acc_hi, acc_lo, b, a_q) + acc_hi = [Vec(state[rt]) for rt in range(N_RT)] + acc_lo = [Vec(state[N_RT + rt]) for rt in range(N_RT)] + b = unflat_b(state[2 * N_RT : 2 * N_RT + n_b]) + acc_hi, acc_lo = step( + fx.Int32(NCHUNK - 1), (NCHUNK - 1) % LDS_SLOTS, QC - 1, b, acc_hi, acc_lo + ) + store_q(fx.Int32(NCHUNK - 1)) + # logits of this K slice: the lane holds rows g * 4 + i of each row tile, expert wave * 16 + r16 + for rt in range_constexpr(N_RT): + out = acc_hi[rt] + acc_lo[rt] + for i in range_constexpr(4): + row = row0 + rt * 16 + g * 4 + i + # Mask excess rows: buffer bounds alone include the next K slice. + buffer_ops.buffer_store( + fx.Float32(out[i]), + l_rsrc, + (khalf * n_rows + row) * E + wave * 16 + r16, + mask=(row < n_rows).ir_value(), + cache_modifier=_ST_NT, + ) + + @flyc.jit + def launch_router_mxfp4_quant_line_layout_i1536( + Y: fx.Tensor, + W: fx.Tensor, + Q4: fx.Tensor, + S4: fx.Tensor, + LOGITS: fx.Tensor, + n_rows: fx.Int32, + stream: fx.Stream, + ): + grid = ((n_rows + BLOCK_M - 1) // BLOCK_M) * K_SPLIT + kernel_router_quant( + Y, + W, + Q4, + S4, + LOGITS, + n_rows, + value_attrs={ + "rocdl.waves_per_eu": 1, + "rocdl.flat_work_group_size": f"{N_THREADS},{N_THREADS}", + }, + ).launch(grid=(grid, 1, 1), block=(N_THREADS, 1, 1), stream=stream) + + launch_router_mxfp4_quant_line_layout_i1536.kernel_name = name + return launch_router_mxfp4_quant_line_layout_i1536 + + +def router_mxfp4_quant_line_layout(x: torch.Tensor, gate_weight: torch.Tensor): + """BF16 [M, 6144] -> MXFP4 codes, row scales and fp32 [M, S, 128] router partials (the MFMAs read the bf16 x). + + ``gate_weight``: ``layout.router_line_layout``'s copy; ``router_mxfp4_quant``'s outputs bit for bit. + """ + m, h = x.shape + assert h == 6144 and x.dtype == torch.bfloat16 and x.is_contiguous() + ks = router_k_split(m) + q4 = torch.empty((m, h // 2), dtype=torch.uint8, device=x.device) + s4 = torch.empty((m, h // 32), dtype=torch.uint8, device=x.device) + parts = torch.empty((ks, m, 128), dtype=torch.float32, device=x.device) + _run_compiled( + compile_router_mxfp4_quant_line_layout(ks), + x.view(torch.int32).view(-1), gate_weight.view(-1), + q4.view(-1), s4.view(-1), parts.view(-1), m, + torch.cuda.current_stream(), + ) + return q4, s4, parts.permute(1, 0, 2) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk.py new file mode 100644 index 0000000..9d972b3 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router split-K partials -> AITER's biased top-k (``grouped_topk_kernel``, one group) in one launch (gfx950): lane l +of a token's wave sums experts 4 l .. 4 l + 3 in split order from 0.0, takes the sigmoid and the biased choice, runs k +rounds of a lane-local strict argmax and AITER's DPP tree read at lane 63, and renormalizes the unbiased sigmoids by +scale / sum: AITER's ids and weights bit for bit (ties included).""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +# one token (wave) per workgroup spreads the partial loads over more CUs: 172 rows 3.75 -> 3.33 us after the router +NW = 1 +# kernel arguments preloaded into SGPRs instead of s_load'ed first +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} +NEG_LOG2E = -1.44269504088896340736 +# AITER's wave_reduce on gfx9: quad_perm [1,0,3,2], quad_perm [2,3,0,1], +# row_half_mirror, row_mirror, row_bcast:15, row_bcast:31 +DPP_TREE = (0xB1, 0x4E, 0x141, 0x140, 0x142, 0x143) + + +def _ir(v): + return v.ir_value() if hasattr(v, "ir_value") else v + + +def _dpp(v, ctrl): + """v_mov_b32_dpp, all rows / banks: an invalid source lane keeps its value (the lane-63 result never reads one).""" + return fx.Int32(_rocdl_ops.update_dpp(T.i32, _ir(v), _ir(v), ctrl, 0xF, 0xF, False)) + + +@launcher +def compile_router_topk(num_experts: int, topk: int, split_k: int, tpw: int = 1): + """``tpw``: tokens per wave, 1 or 2 (lanes 0-31 and 32-63).""" + assert num_experts % 4 == 0 and num_experts <= 256 and 1 <= topk <= 64 + assert tpw == 1 or (tpw == 2 and num_experts <= 128 and topk <= 32) + E, K, S = num_experts, topk, split_k + LPT = 64 // tpw # lanes per token + # tpw 2: the first five steps never move a value across lanes 31 / 32 toward the lanes read, so lanes 31 and 63 + # hold the one-token result without the last step (row_bcast:31) + TREE = DPP_TREE if tpw == 1 else DPP_TREE[:-1] + name = f"router_splitk_biased_top{K}_e{E}_s{S}" + ("" if tpw == 1 else f"_tpw{tpw}") + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_part: fx.Int64, # split-K partials, fp32, [M, S, E] by strides + arg_bias: fx.Int64, # correction bias [E] fp32 + arg_w: fx.Int64, # top-k weights fp32, row stride i32_stk + arg_ids: fx.Int64, # top-k ids int32, row stride i32_stk + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, # partial strides (elements): token, split + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, # routed_scaling_factor (fp32 bits) + ): + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + t0 = (pid * NW + wave) * tpw + if t0 < i32_m: + pr = buffer_ops.create_buffer_resource_from_addr( + arg_part, num_records_bytes=i64_part_bytes + ) + br = buffer_ops.create_buffer_resource_from_addr( + arg_bias, num_records_bytes=E * 4 + ) + out_bytes = fx.Int64(i32_m) * fx.Int64(i32_stk) * 4 + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=out_bytes + ) + ir = buffer_ops.create_buffer_resource_from_addr( + arg_ids, num_records_bytes=out_bytes + ) + if const_expr(tpw == 1): + t, lh = t0, lane + has = lane < E // 4 + else: + t, lh = t0 + lane // LPT, lane % LPT + has = (lh < E // 4) & (t < i32_m) + base = t * i32_sm + lh * 4 + parts = [ + Vec( + buffer_ops.buffer_load( + pr, base + k * i32_sk, vec_width=4, dtype=fx.Float32, mask=has + ) + ) + for k in range_constexpr(S) + ] + bias = Vec( + buffer_ops.buffer_load( + br, lh * 4, vec_width=4, dtype=fx.Float32, mask=lh < E // 4 + ) + ) + neg_inf = fx.Float32(float("-inf")) + sig, choice = [], [] + for i in range_constexpr(4): + x = fx.Float32(0.0) + for k in range_constexpr(S): + x = x + fx.Float32(parts[k][i]) + # AITER's -log2(e) is a double literal: the product is a double rounded to fp32 + y = (x.to(fx.Float64) * fx.Float64(NEG_LOG2E)).to(fx.Float32) + s = fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(1.0) + y.exp2())) + sig.append(s) + choice.append(has.select(s + fx.Float32(bias[i]), neg_inf)) + total = fx.Float32(0.0) + out_id = fx.Int32(0) + out_val = fx.Float32(0.0) + for k in range_constexpr(K): + mv = neg_inf + mi = fx.Int32(k) + for i in range_constexpr(4): + take = choice[i] > mv + mv = take.select(choice[i], mv) + mi = take.select(lh * 4 + i, mi) + for ctrl in TREE: + rv = _dpp(mv.bitcast(fx.Int32), ctrl).bitcast(fx.Float32) + ri = _dpp(mi, ctrl) + take = rv > mv + mv = take.select(rv, mv) + mi = take.select(ri, mi) + picks = [] + for h in range_constexpr(tpw): + hidx = fx.Int32( + fx.rocdl.readlane(T.i32, mi, fx.Int32(LPT * h + LPT - 1)) + ) + howner = hidx // 4 + LPT * h + hcomp = hidx % 4 + # the selected expert's unbiased sigmoid, from its owner lane + sv = [ + fx.Int32( + fx.rocdl.readlane(T.i32, sig[i].bitcast(fx.Int32), howner) + ).bitcast(fx.Float32) + for i in range_constexpr(4) + ] + hw = sv[0] + for i in range_constexpr(1, 4): + hw = (hcomp == i).select(sv[i], hw) + picks.append((hidx, howner, hw)) + idx, owner, w = picks[0] + if const_expr(tpw == 2): + upper = lane >= LPT + idx = upper.select(picks[1][0], idx) + owner = upper.select(picks[1][1], owner) + w = upper.select(picks[1][2], w) + comp = idx % 4 + for i in range_constexpr(4): + choice[i] = ((lane == owner) & (comp == i)).select( + neg_inf, choice[i] + ) + total = total + w + out_id = (lh == k).select(idx, out_id) + out_val = (lh == k).select(w, out_val) + scale = i32_scale.bitcast(fx.Float32) + factor = scale / total + off = t * i32_stk + lh + keep = lh < K + if const_expr(tpw == 2): + keep = keep & (t < i32_m) + buffer_ops.buffer_store(out_val * factor, wr, off, mask=keep) + buffer_ops.buffer_store(out_id, ir, off, mask=keep) + + @flyc.jit + def launch_router_topk_mxfp4_moe( + arg_part: fx.Int64, + arg_bias: fx.Int64, + arg_w: fx.Int64, + arg_ids: fx.Int64, + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_part, + arg_bias, + arg_w, + arg_ids, + i64_part_bytes, + i32_m, + i32_sm, + i32_sk, + i32_stk, + i32_scale, + ).launch( + grid=(fx.Int64((i32_m + (NW * tpw - 1)) // (NW * tpw)), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch_router_topk_mxfp4_moe.kernel_name = name + launch_router_topk_mxfp4_moe.compile_hints = PRELOAD_HINTS + return launch_router_topk_mxfp4_moe + + +# ---- host: compile cache, launch +import functools # noqa: E402 +import struct # noqa: E402 + +import torch # noqa: E402 + + +def _run_compiled(exe, *args): + """The first call compiles and runs (``flyc.compile``); later calls dispatch the cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +# two tokens per wave from 4097 tokens: -0.3 .. -5.9 us at 4097-32768, +0.35 at 4096 (split 8) +TPW2_FROM = 4097 + + +@functools.cache +def _get_kernel(num_experts: int, topk: int, split_k: int, tpw: int = 1): + from .router_topk_vec2 import VEC2_SPLITS, compile_router_topk_vec2 + if (num_experts, topk, tpw) == (128, 4, 1) and split_k in VEC2_SPLITS: + # the one-token router splits: two experts per lane, the same bits + return compile_router_topk_vec2(num_experts, topk, split_k, tpw) + return compile_router_topk(num_experts, topk, split_k, tpw) + + +def splitk_biased_topk( + partials: torch.Tensor, # [M, split_k, E] fp32 (any strides, last dim dense) + correction_bias: torch.Tensor, # [E] fp32 + topk_weights: torch.Tensor, # [M, topk] fp32 (row stride free) + topk_ids: torch.Tensor, # [M, topk] int32, same row stride + routed_scaling_factor: float, +) -> None: + """Sums the router's split-K partials and runs AITER's biased top-k (one group, renormalized) on the logits: AITER's + ids and weights bit for bit.""" + m, split_k, num_experts = partials.shape + if m == 0: + return + assert partials.dtype == torch.float32 and partials.stride(2) == 1 + assert correction_bias.dtype == torch.float32 and correction_bias.is_contiguous() + assert topk_weights.stride(1) == 1 and topk_ids.stride(1) == 1 + assert topk_weights.stride(0) == topk_ids.stride(0) + end = ( + (m - 1) * partials.stride(0) + (split_k - 1) * partials.stride(1) + num_experts + ) + tpw = 2 if m >= TPW2_FROM and num_experts <= 128 else 1 + _run_compiled( + _get_kernel(num_experts, topk_ids.shape[1], split_k, tpw), + partials.data_ptr(), + correction_bias.data_ptr(), + topk_weights.data_ptr(), + topk_ids.data_ptr(), + end * 4, + m, + partials.stride(0), + partials.stride(1), + topk_ids.stride(0), + struct.unpack("= i32_m: + if t < m32: + zr = buffer_ops.create_buffer_resource_from_addr(arg_s8, num_records_bytes=fx.Int64(m32) * (H // 32)) + for b in range_constexpr(BPL): + buffer_ops.buffer_store(fx.Int32(0).to(fx.Int8), zr, + _scale_off(t, lane + 64 * b, H // 256), offset_is_bytes=True) + if t < i32_m: + pr = buffer_ops.create_buffer_resource_from_addr( + arg_part, num_records_bytes=i64_part_bytes + ) + br = buffer_ops.create_buffer_resource_from_addr( + arg_bias, num_records_bytes=E * 4 + ) + out_bytes = fx.Int64(i32_m) * fx.Int64(i32_stk) * 4 + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=out_bytes + ) + ir = buffer_ops.create_buffer_resource_from_addr( + arg_ids, num_records_bytes=out_bytes + ) + has = lane < E // 4 + base = t * i32_sm + lane * 4 + parts = [ + Vec( + buffer_ops.buffer_load( + pr, base + k * i32_sk, vec_width=4, dtype=fx.Float32, mask=has + ) + ) + for k in range_constexpr(S) + ] + # token t's row, loaded behind the partials: lane l takes 32-value blocks l + 64 b (4 x 16 B each) + row_bytes = fx.Int64(i32_m) * (H * 2) + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=row_bytes) + xv = [ + [ + Vec( + buffer_ops.buffer_load( + xr, (t * BLOCKS + lane + 64 * b) * 16 + j * 4, vec_width=4, dtype=fx.Int32 + ) + ) + for j in range_constexpr(4) + ] + for b in range_constexpr(BPL) + ] + bias = Vec( + buffer_ops.buffer_load( + br, lane * 4, vec_width=4, dtype=fx.Float32, mask=has + ) + ) + neg_inf = fx.Float32(float("-inf")) + sig, choice = [], [] + for i in range_constexpr(4): + x = fx.Float32(0.0) + for k in range_constexpr(S): + x = x + fx.Float32(parts[k][i]) + y = (x.to(fx.Float64) * fx.Float64(NEG_LOG2E)).to(fx.Float32) + s = fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(1.0) + y.exp2())) + sig.append(s) + choice.append(has.select(s + fx.Float32(bias[i]), neg_inf)) + total = fx.Float32(0.0) + out_id = fx.Int32(0) + out_val = fx.Float32(0.0) + for k in range_constexpr(K): + mv = neg_inf + mi = fx.Int32(k) + for i in range_constexpr(4): + take = choice[i] > mv + mv = take.select(choice[i], mv) + mi = take.select(lane * 4 + i, mi) + for ctrl in DPP_TREE: + rv = _dpp(mv.bitcast(fx.Int32), ctrl).bitcast(fx.Float32) + ri = _dpp(mi, ctrl) + take = rv > mv + mv = take.select(rv, mv) + mi = take.select(ri, mi) + idx = fx.Int32(fx.rocdl.readlane(T.i32, mi, fx.Int32(63))) + owner = idx // 4 + comp = idx % 4 + # the selected expert's unbiased sigmoid, from its owner lane + sv = [ + fx.Int32( + fx.rocdl.readlane(T.i32, sig[i].bitcast(fx.Int32), owner) + ).bitcast(fx.Float32) + for i in range_constexpr(4) + ] + w = sv[0] + for i in range_constexpr(1, 4): + w = (comp == i).select(sv[i], w) + for i in range_constexpr(4): + choice[i] = ((lane == owner) & (comp == i)).select( + neg_inf, choice[i] + ) + total = total + w + out_id = (lane == k).select(idx, out_id) + out_val = (lane == k).select(w, out_val) + scale = i32_scale.bitcast(fx.Float32) + factor = scale / total + off = t * i32_stk + lane + buffer_ops.buffer_store(out_val * factor, wr, off, mask=lane < K) + buffer_ops.buffer_store(out_id, ir, off, mask=lane < K) + # both quants, quant.py's and shared_quant's arithmetic op for op + qr = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=fx.Int64(i32_m) * (H // 2) + ) + sr = buffer_ops.create_buffer_resource_from_addr( + arg_s, num_records_bytes=fx.Int64(i32_m) * (H // 32) + ) + q8r = buffer_ops.create_buffer_resource_from_addr(arg_q8, num_records_bytes=fx.Int64(i32_m) * H) + s8_rows = i32_m + if const_expr(s8_tiles): + s8_rows = (i32_m + 31) // 32 * 32 + s8r = buffer_ops.create_buffer_resource_from_addr(arg_s8, num_records_bytes=fx.Int64(s8_rows) * (H//32)) + for b in range_constexpr(BPL): + blk = t * BLOCKS + lane + 64 * b + vals = [] + for j in range_constexpr(4): + for kk in range_constexpr(4): + w = fx.Int32(xv[b][j][kk]) + vals.append(_as_f32(w << 16)) + vals.append(_as_f32(w & fx.Int32(-65536))) + amax = _fmax3_abs(vals[0], vals[1], vals[2]) + for i in range_constexpr(3, 31, 2): + amax = _fmax3_abs(vals[i], vals[i + 1], amax) + amax = _fmax3_abs(vals[31], vals[31], amax) + e8 = _e8m0_fp8(amax) + sf8 = (e8 << 23).bitcast(fx.Float32) + for half in range_constexpr(2): + words = [] + for dw in range_constexpr(4): + j = half * 16 + dw * 4 + pk = _fp8(fx.Int32(0), vals[j], vals[j+1], sf8, False) + pk = _fp8(pk, vals[j+2], vals[j+3], sf8, True) + words.append(pk) + buffer_ops.buffer_store(Vec.from_elements(words, fx.Int32), q8r, blk*8+half*4) + if const_expr(s8_tiles): + buffer_ops.buffer_store(e8.to(fx.Int8), s8r, _scale_off(t, lane + 64 * b, H // 256), + offset_is_bytes=True) + else: + buffer_ops.buffer_store(e8.to(fx.Int8), s8r, blk) + e8m0 = _e8m0_even_e2m1(amax) + scale_f = _as_f32(e8m0 << 23) + qo = [] + for d in range_constexpr(4): + p = fx.Int32(0) + for kk in range_constexpr(4): + p = _cvt_pk_fp4(p, vals[8 * d + 2 * kk], vals[8 * d + 2 * kk + 1], scale_f, kk) + qo.append(p) + buffer_ops.buffer_store(Vec.from_elements(qo, fx.Int32), qr, blk * 4) + buffer_ops.buffer_store(fx.Int32(e8m0).to(fx.Int8), sr, blk) + + @flyc.jit + def launch_router_topk_mxfp4_mxfp8_quant( + arg_part: fx.Int64, + arg_bias: fx.Int64, + arg_w: fx.Int64, + arg_ids: fx.Int64, + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, + arg_x: fx.Int64, + arg_q: fx.Int64, + arg_s: fx.Int64, + arg_q8: fx.Int64, + arg_s8: fx.Int64, + stream: fx.Stream, + ): + rows = i32_m + if const_expr(s8_tiles): + rows = (i32_m + 31) // 32 * 32 + kernel( + arg_part, + arg_bias, + arg_w, + arg_ids, + i64_part_bytes, + i32_m, + i32_sm, + i32_sk, + i32_stk, + i32_scale, + arg_x, + arg_q, + arg_s, + arg_q8, + arg_s8, + ).launch( + grid=(fx.Int64((rows + (NW - 1)) // NW), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch_router_topk_mxfp4_mxfp8_quant.kernel_name = name + launch_router_topk_mxfp4_mxfp8_quant.compile_hints = PRELOAD_HINTS + return launch_router_topk_mxfp4_mxfp8_quant + + +@functools.cache +def _get_kernel(num_experts: int, topk: int, split_k: int, s8_tiles: bool = False): + return compile_router_topk_mxfp4_mxfp8_quant(num_experts, topk, split_k, s8_tiles) + + +def splitk_biased_topk_mxfp4_mxfp8_quant( + partials: torch.Tensor, # [M, split_k, E] fp32 (any strides, last dim dense) + correction_bias: torch.Tensor, # [E] fp32 + topk_weights: torch.Tensor, # [M, topk] fp32 + topk_ids: torch.Tensor, # [M, topk] int32, same row stride + routed_scaling_factor: float, + x: torch.Tensor, # bf16 [M, H] contiguous + *, + s8_tiles: bool = False, +) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]: + """Top-k and both input quants from one bf16 read -> ((FP4 codes, e8m0 scales), (FP8 codes, e8m0 scales)). + + ``s8_tiles`` (> 256 rows): the FP8 scales [ceil32(M), H/32] in the shared GEMMs' tiled layout, padding zero. + """ + m, split_k, num_experts = partials.shape + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.shape == (m, H) + assert partials.dtype == torch.float32 and partials.stride(2) == 1 + assert correction_bias.dtype == torch.float32 and correction_bias.is_contiguous() + assert topk_weights.stride(1) == 1 and topk_ids.stride(1) == 1 + assert topk_weights.stride(0) == topk_ids.stride(0) + q = torch.empty((m, H // 2), dtype=torch.uint8, device=x.device) + s = torch.empty((m, H // 32), dtype=torch.uint8, device=x.device) + q8 = torch.empty((m,H),dtype=torch.uint8,device=x.device) + s8 = torch.empty((((m + 31) // 32 * 32) if s8_tiles else m, H // 32), dtype=torch.uint8, device=x.device) + end = (m - 1) * partials.stride(0) + (split_k - 1) * partials.stride(1) + num_experts + get_kernel = _get_kernel + # separate top-k and quant workgroups to 256 rows (the 16-row expert tier): the whole entry faster; above, noise + if 0 < m <= 256: + assert not s8_tiles + from .router_topk_mxfp4_mxfp8_quant_sorted_tasks import _get_kernel as get_kernel + token_major = split_k == 48 and partials.stride(0) == split_k * num_experts and partials.stride(1) == num_experts + launch = (get_kernel(num_experts, topk_ids.shape[1], split_k, True) if token_major + else get_kernel(num_experts, topk_ids.shape[1], split_k)) + else: + launch = get_kernel(num_experts, topk_ids.shape[1], split_k, s8_tiles) + _run_compiled( + launch, + partials.data_ptr(), + correction_bias.data_ptr(), + topk_weights.data_ptr(), + topk_ids.data_ptr(), + end * 4, + m, + partials.stride(0), + partials.stride(1), + topk_ids.stride(0), + struct.unpack("") + return fx.Float32(_arith.MaxNumFOp(fx.as_ir_value(a), fx.as_ir_value(b), fastmath=fm).result) + + +@launcher +def compile_router_topk_mxfp4_mxfp8_quant_sorted_tasks(num_experts: int, topk: int, split_k: int, token_major: bool = False): + """Each lane sorts its four choices once; the winning lane pops its head after each pick. + + Strict adjacent swaps keep the lower component first on ties. The tree's sixth DPP step only copies lane 31's + result to lane 63, so the tree stops at five steps and reads lane 31. + + Args: + token_major: contiguous [m, S, 128] partials, read at constant split offsets. + """ + assert num_experts == 128 and topk == 4 + E, K, S = num_experts, topk, split_k + assert not token_major or S == 48 + name = (f"router_splitk_biased_top{K}_mxfp4_mxfp8_quant_sorted_tasks_h{H}_e{E}_s{S}" + + "_half_tree_maxnum_direct_weight" + ("_token_partials" if token_major else "")) + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_part: fx.Int64, # split-K partials, fp32, [M, S, E] by strides + arg_bias: fx.Int64, # correction bias [E] fp32 + arg_w: fx.Int64, # top-k weights fp32, row stride i32_stk + arg_ids: fx.Int64, # top-k ids int32, row stride i32_stk + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, # partial strides (elements): token, split + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, # routed_scaling_factor (fp32 bits) + arg_x: fx.Int64, # bf16 [M, H] row-major (the router GEMM's input) + arg_q: fx.Int64, # e2m1 codes [M, H / 2] + arg_s: fx.Int64, # routed e8m0 [M,H/32] + arg_q8: fx.Int64, + arg_s8: fx.Int64, + ): + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + t = pid * NW + wave + if t < i32_m: + pr = buffer_ops.create_buffer_resource_from_addr( + arg_part, num_records_bytes=i64_part_bytes + ) + br = buffer_ops.create_buffer_resource_from_addr( + arg_bias, num_records_bytes=E * 4 + ) + out_bytes = fx.Int64(i32_m) * fx.Int64(i32_stk) * 4 + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=out_bytes + ) + ir = buffer_ops.create_buffer_resource_from_addr( + arg_ids, num_records_bytes=out_bytes + ) + has = lane < E // 4 + el = lane % 32 * 4 + lane // 32 * 2 + base = t * (S * E if const_expr(token_major) else i32_sm) + el + parts = [Vec(buffer_ops.buffer_load(pr, base + k * (E if const_expr(token_major) else i32_sk), vec_width=2, dtype=fx.Float32)) + for k in range_constexpr(S)] + bias = Vec( + buffer_ops.buffer_load( + br, lane * 4, vec_width=4, dtype=fx.Float32, mask=has + ) + ) + neg_inf = fx.Float32(float("-inf")) + ss = [] + for i in range_constexpr(2): + x = fx.Float32(0.0) + for k in range_constexpr(S): + x = x + fx.Float32(parts[k][i]) + y = (x.to(fx.Float64) * fx.Float64(NEG_LOG2E)).to(fx.Float32) + ss.append(fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(1.0) + y.exp2()))) + # components 2 / 3 back to the quartet's lower lane, for the same argmax tree and ties + sig = [ss[0], ss[1], + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[0].bitcast(fx.Int32))).bitcast(fx.Float32), + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[1].bitcast(fx.Int32))).bitcast(fx.Float32)] + choice = [has.select(sig[i] + fx.Float32(bias[i]), neg_inf) for i in range_constexpr(4)] + # Strict argmax ignores NaNs and -inf; keep that behavior in the queue. + choice = [(v > neg_inf).select(v, neg_inf) for v in choice] + local_ids = [lane * 4 + i for i in range_constexpr(4)] + # stable insertion network: ties keep the component order + for a, b in ((0, 1), (1, 2), (0, 1), (2, 3), (1, 2), (0, 1)): + take = choice[b] > choice[a] + va, vb = choice[a], choice[b] + ia, ib = local_ids[a], local_ids[b] + choice[a], choice[b] = take.select(vb, va), take.select(va, vb) + local_ids[a], local_ids[b] = take.select(ib, ia), take.select(ia, ib) + total = fx.Float32(0.0) + out_id = fx.Int32(0) + out_val = fx.Float32(0.0) + for k in range_constexpr(K): + mv = choice[0] + mi = (mv > neg_inf).select(local_ids[0], fx.Int32(k)) + for ctrl in DPP_TREE[:-1]: + dpp = _dpp_full if ctrl in FULL_DPP else _dpp + rv = dpp(mv.bitcast(fx.Int32), ctrl).bitcast(fx.Float32) + ri = dpp(mi, ctrl) + take = rv > mv + mv = _maxf_nn(rv, mv) + mi = take.select(ri, mi) + idx = fx.Int32(fx.rocdl.readlane(T.i32, mi, fx.Int32(31))) + owner, comp = idx >> 2, idx & 3 + # expert idx: element comp % 2 of lane owner + 32 (comp // 2), also the strict argmax's fallback index + source_lane = owner + ((comp & 2) << 4) + local_w = ((comp & 1) == 1).select(ss[1], ss[0]) + w = fx.Int32(fx.rocdl.readlane( + T.i32, local_w.bitcast(fx.Int32), source_lane + )).bitcast(fx.Float32) + # Only the lane that supplied the global pick advances its queue. + pick = lane == owner + for i in range_constexpr(3): + choice[i] = pick.select(choice[i + 1], choice[i]) + local_ids[i] = pick.select(local_ids[i + 1], local_ids[i]) + choice[3] = pick.select(neg_inf, choice[3]) + total = total + w + out_id = (lane == k).select(idx, out_id) + out_val = (lane == k).select(w, out_val) + scale = i32_scale.bitcast(fx.Float32) + factor = scale / total + off = t * i32_stk + lane + buffer_ops.buffer_store(out_val * factor, wr, off, mask=lane < K) + buffer_ops.buffer_store(out_id, ir, off, mask=lane < K) + if pid >= i32_m: + blk_base = (pid - i32_m) * 64 + if blk_base < i32_m * BLOCKS: + # one 32-value group per lane + row_bytes = fx.Int64(i32_m) * (H * 2) + xr = buffer_ops.create_buffer_resource_from_addr(arg_x, num_records_bytes=row_bytes) + xv = [ + [ + Vec( + buffer_ops.buffer_load( + xr, (blk_base + lane + 64 * b) * 16 + j * 4, vec_width=4, dtype=fx.Int32 + ) + ) + for j in range_constexpr(4) + ] + for b in range_constexpr(1) + ] + # both quants, quant.py's and shared_quant's arithmetic op for op + qr = buffer_ops.create_buffer_resource_from_addr( + arg_q, num_records_bytes=fx.Int64(i32_m) * (H // 2) + ) + sr = buffer_ops.create_buffer_resource_from_addr( + arg_s, num_records_bytes=fx.Int64(i32_m) * (H // 32) + ) + q8r = buffer_ops.create_buffer_resource_from_addr(arg_q8, num_records_bytes=fx.Int64(i32_m) * H) + s8r = buffer_ops.create_buffer_resource_from_addr(arg_s8, num_records_bytes=fx.Int64(i32_m) * (H//32)) + for b in range_constexpr(1): + blk = blk_base + lane + 64 * b + vals = [] + for j in range_constexpr(4): + for kk in range_constexpr(4): + w = fx.Int32(xv[b][j][kk]) + vals.append(_as_f32(w << 16)) + vals.append(_as_f32(w & fx.Int32(-65536))) + amax = _fmax3_abs(vals[0], vals[1], vals[2]) + for i in range_constexpr(3, 31, 2): + amax = _fmax3_abs(vals[i], vals[i + 1], amax) + amax = _fmax3_abs(vals[31], vals[31], amax) + e8 = _e8m0_fp8(amax) + sf8 = (e8 << 23).bitcast(fx.Float32) + for half in range_constexpr(2): + words = [] + for dw in range_constexpr(4): + j = half * 16 + dw * 4 + pk = _fp8(fx.Int32(0), vals[j], vals[j+1], sf8, False) + pk = _fp8(pk, vals[j+2], vals[j+3], sf8, True) + words.append(pk) + buffer_ops.buffer_store(Vec.from_elements(words, fx.Int32), q8r, blk*8+half*4) + buffer_ops.buffer_store(e8.to(fx.Int8), s8r, blk) + e8m0 = _e8m0_even_e2m1(amax) + scale_f = _as_f32(e8m0 << 23) + qo = [] + for d in range_constexpr(4): + p = fx.Int32(0) + for kk in range_constexpr(4): + p = _cvt_pk_fp4(p, vals[8 * d + 2 * kk], vals[8 * d + 2 * kk + 1], scale_f, kk) + qo.append(p) + buffer_ops.buffer_store(Vec.from_elements(qo, fx.Int32), qr, blk * 4) + buffer_ops.buffer_store(fx.Int32(e8m0).to(fx.Int8), sr, blk) + + @flyc.jit + def launch_router_topk_mxfp4_mxfp8_quant_sorted_tasks( + arg_part: fx.Int64, + arg_bias: fx.Int64, + arg_w: fx.Int64, + arg_ids: fx.Int64, + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, + arg_x: fx.Int64, + arg_q: fx.Int64, + arg_s: fx.Int64, + arg_q8: fx.Int64, + arg_s8: fx.Int64, + stream: fx.Stream, + ): + kernel( + arg_part, + arg_bias, + arg_w, + arg_ids, + i64_part_bytes, + i32_m, + i32_sm, + i32_sk, + i32_stk, + i32_scale, + arg_x, + arg_q, + arg_s, + arg_q8, + arg_s8, + ).launch( + grid=(fx.Int64(i32_m + (i32_m * BLOCKS + 63) // 64), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch_router_topk_mxfp4_mxfp8_quant_sorted_tasks.kernel_name = name + launch_router_topk_mxfp4_mxfp8_quant_sorted_tasks.compile_hints = PRELOAD_HINTS + return launch_router_topk_mxfp4_mxfp8_quant_sorted_tasks + + +@functools.cache +def _get_kernel(num_experts: int, topk: int, split_k: int, token_major: bool = False): + return compile_router_topk_mxfp4_mxfp8_quant_sorted_tasks(num_experts, topk, split_k, token_major) + + +def splitk_biased_topk_mxfp4_mxfp8_quant( + partials: torch.Tensor, # [M, split_k, E] fp32 (any strides, last dim dense) + correction_bias: torch.Tensor, # [E] fp32 + topk_weights: torch.Tensor, # [M, topk] fp32 + topk_ids: torch.Tensor, # [M, topk] int32, same row stride + routed_scaling_factor: float, + x: torch.Tensor, # bf16 [M, H] contiguous +) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]: + """Top-k and both input quants from one bf16 read -> ((FP4 codes, e8m0 scales), (FP8 codes, e8m0 scales)).""" + m, split_k, num_experts = partials.shape + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.shape == (m, H) + assert partials.dtype == torch.float32 and partials.stride(2) == 1 + assert correction_bias.dtype == torch.float32 and correction_bias.is_contiguous() + assert topk_weights.stride(1) == 1 and topk_ids.stride(1) == 1 + assert topk_weights.stride(0) == topk_ids.stride(0) + q = torch.empty((m, H // 2), dtype=torch.uint8, device=x.device) + s = torch.empty((m, H // 32), dtype=torch.uint8, device=x.device) + q8 = torch.empty((m,H),dtype=torch.uint8,device=x.device) + s8 = torch.empty((m,H//32),dtype=torch.uint8,device=x.device) + end = (m - 1) * partials.stride(0) + (split_k - 1) * partials.stride(1) + num_experts + _run_compiled( + _get_kernel(num_experts, topk_ids.shape[1], split_k), + partials.data_ptr(), + correction_bias.data_ptr(), + topk_weights.data_ptr(), + topk_ids.data_ptr(), + end * 4, + m, + partials.stride(0), + partials.stride(1), + topk_ids.stride(0), + struct.unpack("") + return fx.Float32(_arith.MaxNumFOp(fx.as_ir_value(a), fx.as_ir_value(b), fastmath=fm).result) + + +@launcher +def compile_router_topk_sorted(num_experts: int, topk: int, split_k: int, token_major: bool = False): + """Each lane sorts its four choices once; the winning lane pops its head after each pick. + + Strict adjacent swaps keep the lower component first on ties. The tree's sixth DPP step only copies lane 31's + result to lane 63, so the tree stops at five steps and reads lane 31. + + Args: + token_major: contiguous [m, S, 128] partials, read at constant split offsets. + """ + assert num_experts == 128 and topk == 4 and split_k in (16, 48) + E, K, S = num_experts, topk, split_k + assert not token_major or S in (16, 48) + name = (f"router_splitk_biased_top{K}_sorted_h{H}_e{E}_s{S}" + + "_half_tree_maxnum_direct_weight" + ("_token_partials" if token_major else "")) + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_part: fx.Int64, # split-K partials, fp32, [M, S, E] by strides + arg_bias: fx.Int64, # correction bias [E] fp32 + arg_w: fx.Int64, # top-k weights fp32, row stride i32_stk + arg_ids: fx.Int64, # top-k ids int32, row stride i32_stk + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, # partial strides (elements): token, split + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, # routed_scaling_factor (fp32 bits) + ): + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + t = pid * NW + wave + if t < i32_m: + pr = buffer_ops.create_buffer_resource_from_addr( + arg_part, num_records_bytes=i64_part_bytes + ) + br = buffer_ops.create_buffer_resource_from_addr( + arg_bias, num_records_bytes=E * 4 + ) + out_bytes = fx.Int64(i32_m) * fx.Int64(i32_stk) * 4 + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=out_bytes + ) + ir = buffer_ops.create_buffer_resource_from_addr( + arg_ids, num_records_bytes=out_bytes + ) + has = lane < E // 4 + el = lane % 32 * 4 + lane // 32 * 2 + base = t * (S * E if const_expr(token_major) else i32_sm) + el + parts = [Vec(buffer_ops.buffer_load(pr, base + k * (E if const_expr(token_major) else i32_sk), vec_width=2, dtype=fx.Float32)) + for k in range_constexpr(S)] + bias = Vec( + buffer_ops.buffer_load( + br, lane * 4, vec_width=4, dtype=fx.Float32, mask=has + ) + ) + neg_inf = fx.Float32(float("-inf")) + ss = [] + for i in range_constexpr(2): + x = fx.Float32(0.0) + for k in range_constexpr(S): + x = x + fx.Float32(parts[k][i]) + y = (x.to(fx.Float64) * fx.Float64(NEG_LOG2E)).to(fx.Float32) + ss.append(fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(1.0) + y.exp2()))) + # components 2 / 3 back to the quartet's lower lane, for the same argmax tree and ties + sig = [ss[0], ss[1], + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[0].bitcast(fx.Int32))).bitcast(fx.Float32), + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[1].bitcast(fx.Int32))).bitcast(fx.Float32)] + choice = [has.select(sig[i] + fx.Float32(bias[i]), neg_inf) for i in range_constexpr(4)] + # Strict argmax ignores NaNs and -inf; keep that behavior in the queue. + choice = [(v > neg_inf).select(v, neg_inf) for v in choice] + local_ids = [lane * 4 + i for i in range_constexpr(4)] + # stable insertion network: ties keep the component order + for a, b in ((0, 1), (1, 2), (0, 1), (2, 3), (1, 2), (0, 1)): + take = choice[b] > choice[a] + va, vb = choice[a], choice[b] + ia, ib = local_ids[a], local_ids[b] + choice[a], choice[b] = take.select(vb, va), take.select(va, vb) + local_ids[a], local_ids[b] = take.select(ib, ia), take.select(ia, ib) + total = fx.Float32(0.0) + out_id = fx.Int32(0) + out_val = fx.Float32(0.0) + for k in range_constexpr(K): + mv = choice[0] + mi = (mv > neg_inf).select(local_ids[0], fx.Int32(k)) + for ctrl in DPP_TREE[:-1]: + dpp = _dpp_full if ctrl in FULL_DPP else _dpp + rv = dpp(mv.bitcast(fx.Int32), ctrl).bitcast(fx.Float32) + ri = dpp(mi, ctrl) + take = rv > mv + mv = _maxf_nn(rv, mv) + mi = take.select(ri, mi) + idx = fx.Int32(fx.rocdl.readlane(T.i32, mi, fx.Int32(31))) + owner, comp = idx >> 2, idx & 3 + # expert idx: element comp % 2 of lane owner + 32 (comp // 2), also the strict argmax's fallback index + source_lane = owner + ((comp & 2) << 4) + local_w = ((comp & 1) == 1).select(ss[1], ss[0]) + w = fx.Int32(fx.rocdl.readlane( + T.i32, local_w.bitcast(fx.Int32), source_lane + )).bitcast(fx.Float32) + # Only the lane that supplied the global pick advances its queue. + pick = lane == owner + for i in range_constexpr(3): + choice[i] = pick.select(choice[i + 1], choice[i]) + local_ids[i] = pick.select(local_ids[i + 1], local_ids[i]) + choice[3] = pick.select(neg_inf, choice[3]) + total = total + w + out_id = (lane == k).select(idx, out_id) + out_val = (lane == k).select(w, out_val) + scale = i32_scale.bitcast(fx.Float32) + factor = scale / total + off = t * i32_stk + lane + buffer_ops.buffer_store(out_val * factor, wr, off, mask=lane < K) + buffer_ops.buffer_store(out_id, ir, off, mask=lane < K) + + @flyc.jit + def launch_router_topk_sorted( + arg_part: fx.Int64, + arg_bias: fx.Int64, + arg_w: fx.Int64, + arg_ids: fx.Int64, + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_part, + arg_bias, + arg_w, + arg_ids, + i64_part_bytes, + i32_m, + i32_sm, + i32_sk, + i32_stk, + i32_scale, + ).launch( + grid=(fx.Int64(i32_m), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch_router_topk_sorted.kernel_name = name + launch_router_topk_sorted.compile_hints = PRELOAD_HINTS + return launch_router_topk_sorted + + +@functools.cache +def _get_kernel(num_experts: int, topk: int, split_k: int, token_major: bool = False): + return compile_router_topk_sorted(num_experts, topk, split_k, token_major) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_vec2.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_vec2.py new file mode 100644 index 0000000..1721d08 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/router_topk_vec2.py @@ -0,0 +1,173 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``router_topk.py``'s top-4 of 128 experts (one token per wave) with two experts per lane (gfx950): all 64 lanes sum +two experts and take their sigmoids, and two bpermutes restore the four-expert layout in lanes 0-31 before the same +argmax tree: the same ids and weights bit for bit.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from amd_kernels.warm_up import launcher + +# one token (wave) per workgroup spreads the partial loads over more CUs: 172 rows 3.75 -> 3.33 us after the router +NW = 1 +# kernel arguments preloaded into SGPRs instead of s_load'ed first +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} +NEG_LOG2E = -1.44269504088896340736 +# AITER's wave_reduce on gfx9: quad_perm [1,0,3,2], quad_perm [2,3,0,1], +# row_half_mirror, row_mirror, row_bcast:15, row_bcast:31 +DPP_TREE = (0xB1, 0x4E, 0x141, 0x140, 0x142, 0x143) +# the router splits this variant serves; S12 (2049-2688 rows): -0.15..-0.27 us vs four experts per lane +VEC2_SPLITS = (8, 12, 16, 48) + + +def _ir(v): + return v.ir_value() if hasattr(v, "ir_value") else v + + +def _dpp(v, ctrl): + """v_mov_b32_dpp, all rows / banks: an invalid source lane keeps its value (the lane-63 result never reads one).""" + return fx.Int32(_rocdl_ops.update_dpp(T.i32, _ir(v), _ir(v), ctrl, 0xF, 0xF, False)) + + +@launcher +def compile_router_topk_vec2(num_experts: int, topk: int, split_k: int, tpw: int = 1): + assert (num_experts, topk, tpw) == (128, 4, 1) and split_k in VEC2_SPLITS + E, K, S = num_experts, topk, split_k + name = f"router_splitk_biased_top{K}_vec2_e{E}_s{S}" + + @flyc.kernel(name=name, known_block_size=[64 * NW, 1, 1]) + def kernel( + arg_part: fx.Int64, # split-K partials, fp32, [M, S, E] by strides + arg_bias: fx.Int64, # correction bias [E] fp32 + arg_w: fx.Int64, # top-k weights fp32, row stride i32_stk + arg_ids: fx.Int64, # top-k ids int32, row stride i32_stk + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, # partial strides (elements): token, split + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, # routed_scaling_factor (fp32 bits) + ): + tx, pid = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + t0 = pid + if t0 < i32_m: + pr = buffer_ops.create_buffer_resource_from_addr( + arg_part, num_records_bytes=i64_part_bytes + ) + br = buffer_ops.create_buffer_resource_from_addr( + arg_bias, num_records_bytes=E * 4 + ) + out_bytes = fx.Int64(i32_m) * fx.Int64(i32_stk) * 4 + wr = buffer_ops.create_buffer_resource_from_addr( + arg_w, num_records_bytes=out_bytes + ) + ir = buffer_ops.create_buffer_resource_from_addr( + arg_ids, num_records_bytes=out_bytes + ) + t, lh = t0, lane + has = lane < E // 4 + # lane l: experts 4 (l % 32) + 2 (l // 32) + 0 / 1, so lanes l and l + 32 hold one quartet + el = lane % 32 * 4 + lane // 32 * 2 + base = t * i32_sm + el + parts = [Vec(buffer_ops.buffer_load( + pr, base + k * i32_sk, vec_width=2, dtype=fx.Float32 + )) for k in range_constexpr(S)] + bias = Vec(buffer_ops.buffer_load(br, lane * 4, vec_width=4, dtype=fx.Float32, mask=has)) + neg_inf = fx.Float32(float("-inf")) + ss = [] + for i in range_constexpr(2): + x = fx.Float32(0.0) + for k in range_constexpr(S): + x = x + fx.Float32(parts[k][i]) + y = (x.to(fx.Float64) * fx.Float64(NEG_LOG2E)).to(fx.Float32) + ss.append(fx.Float32(fx.rocdl.rcp(T.f32, fx.Float32(1.0) + y.exp2()))) + # components 2 / 3 back to the quartet's lower lane, for the same argmax tree and ties + sig = [ss[0], ss[1], + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[0].bitcast(fx.Int32))).bitcast(fx.Float32), + fx.Int32(fx.rocdl.ds_bpermute(T.i32, (lane ^ 32) * 4, ss[1].bitcast(fx.Int32))).bitcast(fx.Float32)] + choice = [has.select(sig[i] + fx.Float32(bias[i]), neg_inf) for i in range_constexpr(4)] + total = fx.Float32(0.0) + out_id = fx.Int32(0) + out_val = fx.Float32(0.0) + for k in range_constexpr(K): + mv = neg_inf + mi = fx.Int32(k) + for i in range_constexpr(4): + take = choice[i] > mv + mv = take.select(choice[i], mv) + mi = take.select(lh * 4 + i, mi) + for ctrl in DPP_TREE: + rv = _dpp(mv.bitcast(fx.Int32), ctrl).bitcast(fx.Float32) + ri = _dpp(mi, ctrl) + take = rv > mv + mv = take.select(rv, mv) + mi = take.select(ri, mi) + idx = fx.Int32(fx.rocdl.readlane(T.i32, mi, fx.Int32(63))) + owner = idx // 4 + comp = idx % 4 + # the component first: one readlane per pick instead of four + local_w = sig[0] + for i in range_constexpr(1, 4): + local_w = (comp == i).select(sig[i], local_w) + w = fx.Int32(fx.rocdl.readlane( + T.i32, local_w.bitcast(fx.Int32), owner + )).bitcast(fx.Float32) + for i in range_constexpr(4): + choice[i] = ((lane == owner) & (comp == i)).select( + neg_inf, choice[i] + ) + total = total + w + out_id = (lh == k).select(idx, out_id) + out_val = (lh == k).select(w, out_val) + scale = i32_scale.bitcast(fx.Float32) + factor = scale / total + off = t * i32_stk + lh + keep = lh < K + buffer_ops.buffer_store(out_val * factor, wr, off, mask=keep) + buffer_ops.buffer_store(out_id, ir, off, mask=keep) + + @flyc.jit + def launch_router_topk_vec2_mxfp4_moe( + arg_part: fx.Int64, + arg_bias: fx.Int64, + arg_w: fx.Int64, + arg_ids: fx.Int64, + i64_part_bytes: fx.Int64, + i32_m: fx.Int32, + i32_sm: fx.Int32, + i32_sk: fx.Int32, + i32_stk: fx.Int32, + i32_scale: fx.Int32, + stream: fx.Stream, + ): + kernel( + arg_part, + arg_bias, + arg_w, + arg_ids, + i64_part_bytes, + i32_m, + i32_sm, + i32_sk, + i32_stk, + i32_scale, + ).launch( + grid=(fx.Int64(i32_m), 1, 1), + block=(64 * NW, 1, 1), + stream=stream, + ) + + launch_router_topk_vec2_mxfp4_moe.kernel_name = name + launch_router_topk_vec2_mxfp4_moe.compile_hints = PRELOAD_HINTS + return launch_router_topk_vec2_mxfp4_moe diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/routing_ids.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/routing_ids.py new file mode 100644 index 0000000..b234875 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/routing_ids.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One wave's 16-row expert block straight from ``topk_ids`` (128 experts, top-4, <= 96 tokens), without a sort launch +or global atomics: LDS-atomic counts of the M * 4 pairs, the canonical block order (experts ascending, ceil(count / 16) +blocks each), the block's rows by ballot in pair order (token, slot: deterministic), padding token M. Every wave of a +workgroup computes the same block, so no barrier; other pairs go to per-lane dummy slots (no same-address atomic or +store within an instruction).""" +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import arith, range_constexpr, rocdl +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .common.utils import wave_incl_scan_i32 + +E, TOPK, MAX_PAIRS = 128, 4, 256 +HIST = 256 # 128 experts, 64 per-lane dummy slots; 64 lanes x 16 B zeroed in one store +TAB = 80 # 16 rows, 64 per-lane dummy slots +WAVE_BYTES = (HIST + TAB) * 4 # one wave's LDS slice + + +def _raw(x): + return x.ir_value() if hasattr(x, "ir_value") else x + + +def _ptr(addr): + """``!llvm.ptr<3>`` at LDS address ``addr``: the raw LDS atomic adds, slot stores and loads below take it.""" + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(addr))) + + +def wave_expert(ids, m, block, lane, hb, pairs=MAX_PAIRS): + """(expert of canonical 16-row block ``block``, its first rank, number of blocks, the lane's pair ids). + + Args: + hb: the wave's ``WAVE_BYTES`` LDS slice. + pairs: the pair slots scanned, a compile-time multiple of 64, >= M * 4. + """ + n_pairs = m * TOPK + ir_ = buffer_ops.create_buffer_resource(ids, max_size=False, num_records_bytes=n_pairs * 4) + vals = [] + for c in range_constexpr(pairs // 64): + q = lane + c * 64 + v = fx.Int32(buffer_ops.buffer_load(ir_, q, vec_width=1, dtype=fx.Int32)) + vals.append((q < n_pairs).select(v, fx.Int32(-1))) + llvm.StoreOp(_raw(Vec.from_elements([fx.Int32(0)] * 4, fx.Int32)), _ptr(hb + lane * 16), alignment=16) + llvm.StoreOp(_raw(fx.Int32(m)), _ptr(hb + HIST * 4 + lane * 4), alignment=4) + fx.rocdl.s_waitcnt(lgkmcnt=0) + for c in range_constexpr(len(vals)): + slot = (vals[c] >= 0).select(vals[c], E + lane) + llvm.AtomicRMWOp(llvm.AtomicBinOp.add, _ptr(hb + slot * 4), _raw(fx.Int32(1)), + llvm.AtomicOrdering.monotonic, syncscope="workgroup", alignment=4) + fx.rocdl.s_waitcnt(lgkmcnt=0) + cc = Vec(llvm.load(T.vec(2, T.i32), _ptr(hb + lane * 8), alignment=8)) + c0, c1 = fx.Int32(cc[0]), fx.Int32(cc[1]) + # the canonical block order on the counts + b0, b1 = (c0 + 15) >> 4, (c1 + 15) >> 4 + end = wave_incl_scan_i32(b0 + b1) + start = end - b0 - b1 + total = fx.Int32(fx.rocdl.readlane(T.i32, end, fx.Int32(63))) + has = (block >= start) & (block < end) + mask = fx.Int64(fx.rocdl.ballot(T.i64, has)) + winner = (mask != fx.Int64(0)).select(fx.math.cttz(mask).to(fx.Int32), fx.Int32(0)) + second = block >= start + b0 + local_e = lane * 2 + second.select(fx.Int32(1), fx.Int32(0)) + local_s = second.select(start + b0, start) + expert = fx.Int32(fx.rocdl.readlane(T.i32, local_e, winner)) + origin = fx.Int32(fx.rocdl.readlane(T.i32, local_s, winner)) + return expert, (block - origin) * 16, total, vals + + +def wave_block_rows(vals, expert, first, lane, hb): + """This lane's row ``lane % 16`` of the block (token | slot << 24, token M = padding): the expert's pairs of rank + ``first`` .. ``first + 15`` in pair order, by ballot.""" + run = fx.Int32(0) + for c in range_constexpr(len(vals)): + q = lane + c * 64 + match = vals[c] == expert + bal = rocdl.ballot(T.i64, _raw(match)) + lo = arith.trunci(T.i32, bal) + hi = arith.trunci(T.i32, arith.shrui(bal, arith.constant(32, type=T.i64))) + below = fx.Int32(rocdl.mbcnt_hi(T.i32, hi, rocdl.mbcnt_lo(T.i32, lo, _raw(fx.Int32(0))))) + r = run + below - first + slot = (match & (r >= 0) & (r < 16)).select(r, 16 + lane) + llvm.StoreOp(_raw((q >> 2) | ((q & 3) << 24)), _ptr(hb + HIST * 4 + slot * 4), alignment=4) + tot = run + below + match.select(fx.Int32(1), fx.Int32(0)) + run = fx.Int32(fx.rocdl.readlane(T.i32, tot, fx.Int32(63))) + fx.rocdl.s_waitcnt(lgkmcnt=0) + return fx.Int32(llvm.load(T.i32, _ptr(hb + HIST * 4 + (lane % 16) * 4), alignment=4)) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_dense.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_dense.py new file mode 100644 index 0000000..0a407de --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_dense.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The shared expert's MXFP8 gate/up + SwiGLU + quant and down GEMMs on the 4-wave LDS-pipelined kernels +(``mxfp8/shared_dense_4wave.py``; ``shared_expert`` from 96 rows), on the weight layout the direct tiles read.""" +import functools +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from .mxfp8.shared_dense_4wave import StoreSwigluQ, compile_mxfp8_gemm_4w_gateup, compile_mxfp8_gemm_4w, _acc_read, _scale_off +from .shared_gemm1_a8w8 import VIEW_UP_ROWS, _swiglu +from .shared_gemm2_a8w8 import VIEW_W_K +from .shared_quant import _fp8, _e8m0_fp8 +from .common.launch import _run_compiled +from amd_kernels.warm_up import launcher + + +class StoreSharedFp32(StoreSwigluQ): + """``StoreSwigluQ`` with the SwiGLU on the fp32 accumulators (no bf16 rounding), ``shared_quant``'s MXFP8 rule.""" + + def _read_acts(self, cg, cu, ti): + values=[] + for j in range_constexpr(self.n_b): + g=cg[self.c_idx_fn(ti,j)];u=cu[self.c_idx_fn(ti,j)] + values.extend([_swiglu(fx.Float32(_acc_read(g,k)),fx.Float32(_acc_read(u,k))) for k in range_constexpr(4)]) + return values + + def _quant_store(self, values, base_row, base_col, ti): + row=base_row+ti*16+self.r + for h in range_constexpr(self.n_b//2): + v=values[h*8:h*8+8] + amax=fx.math.absf(v[0]) + for j in range_constexpr(1,8): + z=fx.math.absf(v[j]);amax=(amax>z).select(amax,z) + for sh in range_constexpr(2): + z=amax.shuffle_xor(16<z).select(amax,z) + e=_e8m0_fp8(amax);scale=(e<<23).bitcast(fx.Float32) + col=base_col+32*h+self.qg*4 + for j in range_constexpr(2): + pk=_fp8(fx.Int32(0),v[j*4],v[j*4+1],scale,False) + pk=_fp8(pk,v[j*4+2],v[j*4+3],scale,True) + buffer_ops.buffer_store(pk,self.q_rsrc,(row*self.out_n+col+j*16)//4) + buffer_ops.buffer_store(e.to(fx.Int8),self.s_rsrc,_scale_off(row,base_col//32+h,self.scale_k1), + offset_is_bytes=True,mask=self.qg==0) + + +@functools.cache +@launcher +def _shuffle_scales(k): + """Row-major e8m0 scales [m, k / 32] -> AITER's shuffled layout (``_scale_off``), rows padded to 32.""" + sc=k//32;kt=k//256 + @flyc.kernel(name=f'shared_scale_row_to_tiles_k{k}',known_block_size=[256,1,1]) + def kernel(S:fx.Tensor,O:fx.Tensor,m:fx.Int32): + t=fx.thread_idx.x;b=fx.block_idx.x + row=(b//kt)*32+(t//4)%16+(t%2)*16 + col=(b%kt)*8+(t//64)+(t%4//2)*4 + sr=buffer_ops.create_buffer_resource(S,max_size=False,num_records_bytes=m*sc) + out=buffer_ops.create_buffer_resource(O,max_size=False,num_records_bytes=((m+31)//32*32)*sc) + v=buffer_ops.buffer_load(sr,row*sc+col,vec_width=1,dtype=fx.Int8) + buffer_ops.buffer_store(v,out,b*256+t,offset_is_bytes=True) + @flyc.jit + def launch_shared_scales_to_tiles(S:fx.Tensor,O:fx.Tensor,m:fx.Int32,stream:fx.Stream): + kernel(S,O,m).launch(grid=(((m+31)//32)*kt,1,1),block=(256,1,1),stream=stream) + return launch_shared_scales_to_tiles + + +@functools.cache +def _gemm1(tiles=(4,4), *, schedule="deferred", view=False): + """The gate/up GEMM; ``schedule``: single (one tile per workgroup), early or deferred previous-tile epilogue; + ``view``: the rank's shard read from the full weight (up rows ``VIEW_UP_ROWS`` after the gate rows).""" + up_rows = VIEW_UP_ROWS if view else None + if schedule == "single" and tiles == (4,4): + from .shared_gemm1_a8w8_single_tile import compile_shared_gemm1_a8w8_single_tile + return compile_shared_gemm1_a8w8_single_tile(up_rows=up_rows) + if schedule == "early" and tiles == (4,4): + from .shared_gemm1_a8w8_epilogue_early import compile_shared_gemm1_a8w8_epilogue_early + return compile_shared_gemm1_a8w8_epilogue_early(up_rows=up_rows) + return compile_mxfp8_gemm_4w_gateup(K=6144,alpha=1.702,beta=1.0,limit=7.0, + epilogue_type=StoreSharedFp32,kernel_prefix='shared_fp32_swiglu_mxfp8_h6144_i1536_',tiles=tiles, + up_rows=up_rows) + + +@functools.cache +def _gemm2(tiles=(4,4), *, nt=False, view=False): + return compile_mxfp8_gemm_4w(K=1536,tiles=tiles,store_nt=nt,w_k=VIEW_W_K if view else None) + + +# nt bf16 output stores (same bits, kernel suffix _nt): entry -0.4..-1.0% at 4096-32768, -0.1..-0.5% at 1025-3072 +SHARED_GEMM2_NT_FROM = 2048 + + +def _gemm2_nt_for(m): + return m >= SHARED_GEMM2_NT_FROM + + +def _gemm1_schedule_for(m): + # twelve 256-column gate/up parent tiles per 256 rows: at most 252 tiles through 5376 rows, above that a + # workgroup owns two + return "single" if 1025 <= m <= 5376 else "early" if m > 5376 else "deferred" + + +def _tiled_scales(s, m, stream, side, keep=None): + ss=torch.empty(((m+31)//32*32,192),device=s.device,dtype=torch.uint8) + _hold(stream, side, keep, ss) + _run_compiled(_shuffle_scales(6144),s.view(-1),ss.view(-1),m,stream) + return ss + + +def _hold(stream, side, keep, *tensors): + """Ties the current stream's ``tensors``, used on the side ``stream``, to its work: appended to ``keep`` (the + caller frees them after the current stream waited for ``stream``) or else recorded on ``stream``.""" + if not side: + return + if keep is not None: + keep.extend(tensors) + else: + for t in tensors: + t.record_stream(stream) + + +def shared_dense(q,s,w,*,config=((4,4),(4,4)), scales_tiled=False, stream=None): + """The shared expert's bf16 output [m, 6144] from ``shared_quant``'s codes ``q`` and scales ``s``: the TP shard + (``SharedWeights.tp_operands``). + + Args: + scales_tiled: ``s`` is already in the shuffled layout (``_shuffle_scales``). + stream: launch there; the buffers still come from the current stream's allocator pool and are recorded on + ``stream`` (the shared expert's side stream). + """ + m=q.shape[0] + side=stream is not None and stream != torch.cuda.current_stream() + if stream is None: + stream=torch.cuda.current_stream() + ss=s if scales_tiled else _tiled_scales(s,m,stream,side) + hq=torch.empty((m,1536),device=q.device,dtype=torch.uint8) + hs=torch.empty(((m+31)//32*32,48),device=q.device,dtype=torch.uint8) + out=torch.empty((m,6144),device=q.device,dtype=torch.bfloat16) + if side: + for t in (hq,hs,out): + t.record_stream(stream) + i8=lambda t:t.view(torch.int8).view(-1) + w13,s13,w2,s2=w.tp_operands() + g1=_gemm1(config[0],schedule=_gemm1_schedule_for(m),view=w.full) + g2=_gemm2(config[1],nt=_gemm2_nt_for(m),view=w.full) + p1=((m+255)//256)*12;p2=((m+255)//256)*24 + n1=((p1+7)//8*8 if g1.sub_split>1 else p1)*g1.sub_split + n2=((p2+7)//8*8 if g2.sub_split>1 else p2)*g2.sub_split + _run_compiled(g1,i8(q),i8(w13),i8(hq),hs.view(-1),ss.view(-1),s13,m,3072,0,n1,stream) + _run_compiled(g2,i8(hq),i8(w2),out.view(-1),hs.view(-1),s2,m,6144,0,n2,stream) + return out + + +# The full expert (I 3072) on its own rows (sequence-parallel prefill): gate/up N 6144 K 6144, down N 6144 K 3072. +@functools.cache +def _full_gemm1(tiles=(4,4), *, schedule="deferred"): + if tiles == (4,4) or schedule == "deferred": + return _gemm1(tiles, schedule=schedule) + if schedule == "single": + from .shared_gemm1_a8w8_single_tile import compile_shared_gemm1_a8w8_single_tile + return compile_shared_gemm1_a8w8_single_tile(tiles=tiles) + if schedule == "early": + from .shared_gemm1_a8w8_epilogue_early import compile_shared_gemm1_a8w8_epilogue_early + return compile_shared_gemm1_a8w8_epilogue_early(tiles=tiles) + return compile_mxfp8_gemm_4w_gateup(K=6144,alpha=1.702,beta=1.0,limit=7.0, + epilogue_type=StoreSharedFp32,kernel_prefix='shared_fp32_swiglu_mxfp8_h6144_i1536_',tiles=tiles) + + +@functools.cache +def _full_gemm2(tiles=(4,4), *, nt=False): + return compile_mxfp8_gemm_4w(K=2*1536,tiles=tiles,store_nt=nt) + + +# rows -> ((gate/up tiles, schedule), (down tiles, nt stores)); "single" holds one work item per workgroup (at most +# 256). Sweep (benchmark_shared_full.py --sweep): vs the TP shard on 2 m rows -27% at 1024, -12% at 2048, -6% at +# 4096, -6% at 8192, -5% at 16384. +FULL_CONFIGS = ( + (1280, (((2,4),"single"),((2,4),True))), + (2560, (((4,4),"single"),((4,4),True))), + (4096, (((2,4),"deferred"),((2,4),True))), + (16384, (((4,4),"early"),((4,4),True))), +) + + +def full_config_for(m): + """The full expert's GEMM configuration at ``m`` rows (``FULL_CONFIGS``; beyond its last bound, the last).""" + for bound, config in FULL_CONFIGS: + if m <= bound: + return config + return FULL_CONFIGS[-1][1] + + +def _full_parents(m): + return ((m+255)//256)*24 + + +def shared_dense_full(q,s,w,*,stream=None,config=None,keep=None): + """The full expert's bf16 output [m, 6144] from MXFP8 codes ``q`` [m, 6144] and row-major e8m0 ``s`` [m, 192] + (``shared_dense``'s stream rule; with a list ``keep``, the buffers go there instead of being recorded).""" + m=q.shape[0] + side=stream is not None and stream != torch.cuda.current_stream() + if stream is None: + stream=torch.cuda.current_stream() + (t1,sched),(t2,nt)=full_config_for(m) if config is None else config + assert sched != "single" or _full_parents(m) * (4 // t1[0]) * (4 // t1[1]) <= 256, (m, t1) + ss=_tiled_scales(s,m,stream,side,keep) + hq=torch.empty((m,3072),device=q.device,dtype=torch.uint8) + hs=torch.empty(((m+31)//32*32,96),device=q.device,dtype=torch.uint8) + out=torch.empty((m,6144),device=q.device,dtype=torch.bfloat16) + _hold(stream,side,keep,hq,hs,out) + i8=lambda t:t.view(torch.int8).view(-1) + g1,g2=_full_gemm1(t1,schedule=sched),_full_gemm2(t2,nt=nt) + p1=((m+255)//256)*24;p2=((m+255)//256)*24 + n1=((p1+7)//8*8 if g1.sub_split>1 else p1)*g1.sub_split + n2=((p2+7)//8*8 if g2.sub_split>1 else p2)*g2.sub_split + _run_compiled(g1,i8(q),i8(w.w13),i8(hq),hs.view(-1),ss.view(-1),w.s13.view(-1),m,6144,0,n1,stream) + _run_compiled(g2,i8(hq),i8(w.w2),out.view(-1),hs.view(-1),w.s2.view(-1),m,6144,0,n2,stream) + return out diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_expert.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_expert.py new file mode 100644 index 0000000..886bee2 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_expert.py @@ -0,0 +1,231 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared H6144 I1536 expert: MXFP8 activations, weights and MFMA at every M. + +The weights are either a TP2 rank's shard (I 1536) or the full expert (I 3072, ``SharedWeights.full``). With the full +copy the BF16 entry still computes the rank's shard (``tp_rank``): every shared kernel reads it in place, the gate and +up rows as two contiguous ranges of the gate/up weight and the down columns as a K slice (16-row block stride 2 I K +bytes, scale group stride 2 I bytes), the same bytes in the same order as the shard copy, so the same bits. +""" +from dataclasses import dataclass +import torch +from amd_kernels.warm_up import entry +from .shared_quant import shared_quant + +H, I = 6144, 1536 # noqa: E741 + + +# the one-launch MXFP4 + MXFP8 input quant wins at these rows only in the reduced (summed) mode +SHARED_QUANT_REDUCED_ONLY = (4032, 4096) +# first row of the combined routed / shared launch (below it the small-tile host) +SHARED_PERSISTENT_FROM = 641 + +# One side stream per device (``shared_parallel_for``), shared by every layer's SharedWeights: torch's caching +# allocator keeps one block pool per stream, and per-layer streams hold ~20 GiB of shared-expert outputs per rank +# under DBO until the allocator retries (sync + free every segment). +_SIDE_STREAMS = {} + + +def side_stream(device): + """The shared expert's side stream of ``device`` (created on first use).""" + device = torch.device(device) + index = torch.cuda.current_device() if device.index is None else device.index + stream = _SIDE_STREAMS.get(index) + if stream is None: + stream = _SIDE_STREAMS[index] = torch.cuda.Stream(device=torch.device("cuda", index)) + return stream + + +# Inside a DBO microbatch the shared expert stays on the caller's stream: DBO already overlaps one microbatch's +# compute with the other's collectives (serial, the 32768-token DBO steps took a stable 466-472 ms). A server passes +# its query with ``set_dbo_query`` (vLLM: ``vllm.v1.worker.ubatching.dbo_enabled``); without one ``DBO`` answers. +DBO = False +_dbo_query = None + + +def set_dbo_query(query) -> None: + """``query()`` -> whether the current call runs inside a DBO microbatch; None returns to ``DBO``.""" + global _dbo_query + _dbo_query = query + + +def _in_dbo() -> bool: + return bool(_dbo_query()) if _dbo_query is not None else DBO + + +def _weight_layout(w): + """Row-major e4m3 [N, K] -> AITER ``shuffle_weight(.., (16, 16))``: 16-row blocks of [K / 64, 4, 16 rows, 16 B].""" + n, k = w.shape + return w.view(torch.uint8).reshape(n // 16, 16, k // 64, 4, 16).permute(0, 2, 3, 1, 4).contiguous() + + +def _scale_layout(s): + """Row-major e8m0 [N, K / 32] -> AITER ``shuffle_scale_a16w4(.., 1, False)``: 32-row groups of [K / 256, 4, 16, 2, + 2].""" + n, g = s.shape + return s.view(torch.uint8).reshape(n // 32, 2, 16, g // 8, 2, 4).permute(0, 3, 5, 2, 4, 1).contiguous() + + +@dataclass +class SharedWeights: + """The shared expert's MXFP8 weights and scales in the kernels' layout, and the device's side stream. + + ``full``: the whole expert (I 3072); the BF16 entry computes TP2 rank ``tp_rank``'s shard of it in place + (``tp_operands``), ``shared_expert_full`` the whole expert.""" + + w13: torch.Tensor + s13: torch.Tensor + w2: torch.Tensor + s2: torch.Tensor + stream: object + full: bool = False + tp_rank: int | None = None + + @classmethod + def from_row_major(cls, w13, s13, w2, s2, *, tp_rank=None): + """From row-major e4m3 [N, K] weights and e8m0 [N, K / 32] scales (gate rows, then up rows in ``w13``): a + rank's shard (w13 [3072, 6144], w2 [6144, 1536]) or the full expert (w13 [6144, 6144], w2 [6144, 3072]) with + the TP2 rank ``tp_rank`` (0 or 1) whose shard the BF16 entry computes.""" + full = tuple(w13.shape) == (4 * I, H) + i = 2 * I if full else I + assert w13.shape == (2 * i, H) and w2.shape == (H, i), (w13.shape, w2.shape) + assert s13.shape == (2 * i, H // 32) and s2.shape == (H, i // 32), (s13.shape, s2.shape) + if full: + assert tp_rank in (0, 1), f"the full shared expert needs tp_rank 0 or 1, got {tp_rank}" + else: + assert tp_rank in (None, 0, 1), tp_rank + # A bit-exact permutation. The returned object holds only these layouts. + return cls(_weight_layout(w13), _scale_layout(s13), _weight_layout(w2), _scale_layout(s2), + side_stream(w13.device), full, tp_rank) + + def tp_operands(self): + """(w13, s13, w2, s2) flat u8 views of the TP shard the BF16 entry computes: the tensors themselves for a + shard copy; for the full copy, each starting at rank ``tp_rank``'s first byte (gate rows r I, scale groups + r I / 32, down K columns r I, scale K groups r I / 32), one length for both ranks.""" + flat = [t.view(torch.uint8).view(-1) for t in (self.w13, self.s13, self.w2, self.s2)] + if not self.full: + return tuple(flat) + r = self.tp_rank + spans = ((r * I * H, 3 * I * H), (r * I * (H // 32), 3 * I * (H // 32)), + (r * 16 * I, H * 2 * I - 16 * I), (r * I, H * (2 * I // 32) - I)) + return tuple(t[o:o + n] for t, (o, n) in zip(flat, spans)) + + +def shared_expert(x, w, *, x_quant=None, scales_tiled=False, stream=None): + """The shared expert of bf16 ``x`` [n, 6144] (``shared_dense``, from 96 rows) -> bf16 [n, 6144]; below 96 rows the + shared tiles run folded into the routed small-tile launches. + + Args: + x_quant: ``shared_quant(x)``'s (codes, scales) if already computed. + stream: launch there instead of on the current stream; the buffers still come from the current stream's + allocator pool and are recorded on ``stream`` (``shared_expert_parallel``). + """ + n = x.shape[0] + if not shared_dense_for(n): + raise ValueError(f"the standalone shared expert takes >= 96 rows, got {n}") + q, s = shared_quant(x, stream=stream) if x_quant is None else x_quant + from .shared_dense import shared_dense + return shared_dense(q, s, w, config=shared_dense_config_for(n), scales_tiled=scales_tiled, stream=stream) + + +def shared_expert_full(codes, scales, w, *, stream=None, keep=None): + """The full shared expert (``w.full``) of MXFP8 codes u8 [n, 6144] and row-major e8m0 u8 [n, 192] + (``prepare_mxfp4_moe``'s ``shared_codes`` / ``shared_scales``) -> bf16 [n, 6144]. + + Args: + stream: launch there instead of on the current stream (the caller orders it after the inputs' producer); the + inputs and every buffer, which come from the current stream's allocator pool, are recorded on it. + keep: a list; with a side ``stream``, the inputs and buffers are appended to it instead of recorded, and the + caller drops them only after the current stream waited for ``stream``'s work. A recorded block is + reusable only once the allocator sees the side stream's event complete, which lags while the host runs + layers ahead: recorded, the pool grew by 17-20 GiB over 57 sequence-parallel layers at 32768 tokens. + """ + from .shared_dense import shared_dense_full + if not w.full: + raise ValueError("shared_expert_full needs the full shared expert (SharedWeights.full)") + n = codes.shape[0] + if (codes.shape != (n, H) or scales.shape != (n, H // 32) or codes.dtype != torch.uint8 + or scales.dtype != torch.uint8 or not codes.is_contiguous() or not scales.is_contiguous() or n < 1): + raise ValueError(f"expected contiguous u8 codes [n, {H}] and scales [n, {H // 32}], got " + f"{tuple(codes.shape)} / {tuple(scales.shape)}") + if keep is None: + _record(stream, codes, scales) + elif stream is not None and stream != torch.cuda.current_stream(): + keep.extend((codes, scales)) + return shared_dense_full(codes, scales, w, stream=stream, keep=keep) + + +# one row count per tier of ``shared_dense.FULL_CONFIGS`` (the sequence-parallel own rows reach 1..16384) +FULL_WARM_UP_ROWS = (1, 1281, 2561, 4097) + + +def _full_inputs(mode, n, codes, scales, w, *, stream=None): + dev = codes.device + def zeros(cols): + return torch.zeros((n, cols), dtype=torch.uint8, device=dev) + + return (zeros(H), zeros(H // 32), w), {} + + +shared_expert_full = entry("moe/mxfp4_h6144_i1536_e128_top4:shared_full", mode=lambda *a, **k: ("full",), + sizes=lambda mode: FULL_WARM_UP_ROWS, inputs=_full_inputs)(shared_expert_full) + + +def _record(stream, *tensors): + """Records the current stream's ``tensors`` on ``stream``, so their blocks wait for its work before reuse.""" + if stream is not None and stream != torch.cuda.current_stream(): + for t in tensors: + t.record_stream(stream) + + +def shared_expert_parallel(x, w, current, **kwargs): + """``shared_expert`` on ``w.stream`` (the device's side stream) after the work queued on ``current`` so far. + + Every buffer comes from ``current``'s allocator pool (recorded on the side stream), so the allocator caches nothing + under the side stream. The caller joins with ``current.wait_stream(w.stream)`` before it reads the result.""" + side = w.stream + side.wait_stream(current) + _record(side, x, *(kwargs.get("x_quant") or ())) + return shared_expert(x, w, stream=side, **kwargs) + + +def shared_folded_for(n): + """Whether the shared tiles lead both routed 16 / 32-row GEMM launches (below 96 rows). From 96 rows the routed + launches run alone, at the routed tiles' registers and LDS (the folded 64-row shared tiles hold them to two waves + per SIMD), next to the shared expert on the side stream: -4..-15% at 128-512 rows.""" + return n < 96 + + +# second stream vs serial: -2.1..-6.6% at 16385-28672 rows, -0.5..-1.5% to 30720, noise at 31744 / 32768 +SHARED_PARALLEL_MAX = 30720 + + +def shared_parallel_for(n): + """Whether the shared expert runs on the side stream (96..``SHARED_PARALLEL_MAX`` rows, not inside DBO).""" + return 96 <= n <= SHARED_PARALLEL_MAX and not _in_dbo() + + +def shared_dual_quant_for(n): + # the MXFP4 + MXFP8 input quant in one launch on these rows (the same bits as two) + return (n <= 1024 or SHARED_QUANT_REDUCED_ONLY[0] <= n <= SHARED_QUANT_REDUCED_ONLY[1] + or 7425 <= n <= 8192 + or 10240 <= n <= 10880 or 13568 <= n <= 32768) + + +def shared_dense_for(n): + return n >= 96 + + +def shared_dense_config_for(n): + if n <= 512: + return (2,2),(2,2) + if n <= 1024: + return (2,4),(4,4) + return (4,4),(4,4) + + +def shared_in_persistent_for(n): + """Whether routed and shared GEMMs share one launch (the BM128 host) on BF16-entry rows (641-3583; below, the + small-tile host's routed launches next to the shared expert on the second stream).""" + # the adaptive 3584-4096 tier did not pass the entry's noise gate + return SHARED_PERSISTENT_FROM<=n<=3583 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8.py new file mode 100644 index 0000000..e84e8b8 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared-expert MXFP8 gate/up tile bodies with direct operands (``SharedWeights`` layout), FP32 SwiGLU and MXFP8 +quant, which the routed small-tile GEMM1s run first in their launch (``compile_gemm1_tile``). + +One wave owns 32 intermediate columns with their whole quant group; K-waves sum their FP32 partials through LDS in +rank order. +""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .shared_quant import _e8m0_fp8, _fp8 +from amd_kernels.warm_up import launcher + +H, I, E = 6144, 1536, 1 +# the TP2 rank's shard read in place from the full [gate 2I; up 2I] weight (``SharedWeights.full``): the up rows start +# 2I rows after the gate rows +VIEW_UP_ROWS = 2 * I + + +def _raw(x): + """The IR value of ``x`` for LLVM intrinsics and inline asm.""" + return x.ir_value() if hasattr(x, "ir_value") else x + + +def _mfma(a, b, acc, sa, sb, b_byte, transpose=False): + """Native gfx950 MXFP8 scaled MFMA into AGPRs; transpose selects GEMM1 C layout.""" + ty = T.vec(4, T.f32) + a_byte = 0 + if transpose: + a, b, sa, sb = b, a, sb, sa + a_byte, b_byte = b_byte, 0 + asm = (f"v_mfma_scale_f32_16x16x128_f8f6f4 $0, $1, $2, {'$0' if acc is not None else '0'}, $3, $4 " + f"op_sel:[{a_byte % 2},{b_byte % 2},0] op_sel_hi:[{a_byte // 2},{b_byte // 2},0] cbsz:0 blgp:0") + ops = [_raw(a), _raw(b), _raw(sa), _raw(sb)] + cons = "=a,v,v,v,v" + if acc is not None: + ops.append(_raw(acc)) + cons += ",0" + return llvm.inline_asm(ty, ops, asm, cons, has_side_effects=True) + + +def _hold(xs): + """Keep the required inline-MFMA latency before any accumulator reads.""" + if len(xs) == 1: # one output cannot be an LLVM inline-asm struct + return [llvm.inline_asm(T.vec(4, T.f32), [_raw(xs[0])], "s_nop 7\ns_nop 7\ns_nop 7", "=a,0", + has_side_effects=True)] + ty = ir.Type.parse("!llvm.struct<(" + ", ".join(["vector<4xf32>"] * len(xs)) + ")>") + cons = ",".join(["=a"] * len(xs) + [str(i) for i in range(len(xs))]) + v = llvm.inline_asm(ty, [_raw(x) for x in xs], "s_nop 7\ns_nop 7\ns_nop 7", cons, has_side_effects=True) + return [llvm.extractvalue(T.vec(4, T.f32), v, [i]) for i in range(len(xs))] + + +def _lds_ptr(base, off): + """Byte-address the aligned LDS exchange and transpose regions.""" + return llvm.inttoptr(ir.Type.parse("!llvm.ptr<3>"), _raw(fx.Int64(base + off))) + + +def _swiglu(g, u): + """SwiGLU-OAI in fp32: g <= 7, -7 <= u <= 7, g * sigmoid(1.702 g) * (u + 1), the sigmoid by exp2 + rcp.""" + g = (g < fx.Float32(7)).select(g, fx.Float32(7)) + u = (u < fx.Float32(7)).select(u, fx.Float32(7)) + u = (u > fx.Float32(-7)).select(u, fx.Float32(-7)) + t = (g * fx.Float32(1.702)) * fx.Float32(-1.4426950408889634) + e = fx.Float32(fx.rocdl.exp2(T.f32, _raw(t))) + r = fx.Float32(fx.rocdl.rcp(T.f32, _raw(fx.Float32(1) + e))) + return (g * r) * (u + fx.Float32(1)) + + +def _load8(rsrc, off, soff, plane=16): + """Two 16-byte loads ``plane`` dwords apart as one 8-dword MFMA operand.""" + lo = Vec(buffer_ops.buffer_load(rsrc, off, vec_width=4, dtype=fx.Int32, soffset_bytes=soff)) + hi = Vec(buffer_ops.buffer_load(rsrc, off + plane, vec_width=4, dtype=fx.Int32, soffset_bytes=soff)) + return Vec.from_elements([lo[j] for j in range(4)] + [hi[j] for j in range(4)], fx.Int32) + + +def _swiglu_quant_store(vals, mr, row0, col, l16, q16, tok, m, qr, ss): + """FP32 gate/up sums of one wave's 32 columns -> SwiGLU -> MXFP8 codes + e8m0.""" + for mi in range_constexpr(mr): + y = [[_swiglu(fx.Float32(vals[mi * 4 + ni * 2][v]), fx.Float32(vals[mi * 4 + ni * 2 + 1][v])) + for v in range_constexpr(4)] for ni in range_constexpr(2)] + amax = fx.math.absf(y[0][0]) + for ni in range_constexpr(2): + for v in range_constexpr(4): + z = fx.math.absf(y[ni][v]) + amax = (amax > z).select(amax, z) + for sh in range_constexpr(2): + z = amax.shuffle_xor(16 << sh, 64) + amax = (amax > z).select(amax, z) + e = _e8m0_fp8(amax) + scale = (e << 23).bitcast(fx.Float32) + row = row0 + mi * 16 + l16 + for ni in range_constexpr(2): + pk = _fp8(fx.Int32(0), y[ni][0], y[ni][1], scale, False) + pk = _fp8(pk, y[ni][2], y[ni][3], scale, True) + off = row * I + col + ni * 16 + q16 * 4 + buffer_ops.buffer_store(pk, qr, off // 4, mask=tok[mi] < m) + buffer_ops.buffer_store(e.to(fx.Int8), ss, row * (I // 32) + col // 32, + offset_is_bytes=True, mask=(tok[mi] < m) & (q16 == 0)) + + +@functools.cache +def compile_gemm1(*, kw=1, pf=3, bm=16, view=False): + """The body of a gate/up tile of ``bm`` rows x 128 / ``kw`` columns: ``kw`` K-waves, ``pf`` K-steps in flight; + ``view``: W / WS start at the rank's gate rows of the full weight (up rows ``VIEW_UP_ROWS`` further).""" + # a K range starts on an even 128-K step: one weight-scale word covers two steps + assert kw in (1, 2, 4) and bm in (16, 32, 64) and 48 % kw == 0 and 48 // kw % 2 == 0 + nw = 4 // kw + tn, kt, mr = nw * 32, 48 // kw, bm // 16 + up = VIEW_UP_ROWS if view else I + + @flyc.jit + def tile(base, bid, A, AS, W, WS, Q, S, m, rows): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + nk, kk = wave % nw, wave // nw + tb, kg = bid, kk + mb, nb = tb // (I // tn), tb % (I // tn) + if mb * bm < m: + col = nb * tn + nk * 32 + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * H) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=m * (H // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=(up + I) * H) + ws = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=(up + I) * (H // 32)) + qr = buffer_ops.create_buffer_resource(Q, max_size=False, num_records_bytes=m * I) + ss = buffer_ops.create_buffer_resource(S, max_size=False, num_records_bytes=m * (I // 32)) + tok = [mb * bm + mi * 16 + l16 for mi in range_constexpr(mr)] + ad = [(tok[mi] * H + kg * (kt * 128) + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [tok[mi] * (H // 32) // 4 + kg * kt for mi in range_constexpr(mr)] + wd = [[((col // 16 + ni + gu * (up // 16)) * 48 * 2048 + q16 * 256 + l16 * 16) // 4 + kg * kt * 512 + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + swd = [((col // 32 + gu * (up // 32)) * 24) * 64 + q16 * 16 + l16 + kg * (kt // 2) * 64 + for gu in range_constexpr(2)] + + def load(k): + aa = [_load8(ar, ad[mi], k * 128) for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, + soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = [[_load8(wr, wd[ni][gu], k * 2048, 256) + for gu in range_constexpr(2)] for ni in range_constexpr(2)] + sb = [buffer_ops.buffer_load(ws, swd[gu], vec_width=1, dtype=fx.Int32, + soffset_bytes=(k // 2) * 256) for gu in range_constexpr(2)] + return aa, sa, bb, sb + + acc = [[[None] * 2 for _ in range(2)] for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(kt): + if const_expr(k + pf < kt): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for ni in range_constexpr(2): + for gu in range_constexpr(2): + acc[mi][ni][gu] = _mfma(aa[mi], bb[ni][gu], acc[mi][ni][gu], + sa[mi], sb[gu], (k % 2) * 2 + ni, True) + flat = _hold([acc[mi][ni][gu] for mi in range(mr) for ni in range(2) for gu in range(2)]) + if const_expr(kw > 1): + for j in range_constexpr(mr * 4): + llvm.StoreOp(_raw(flat[j]), _lds_ptr(base, ((wave * mr * 4 + j) * 64 + lane) * 16), alignment=16) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + if kk == 0: + vals = [Vec(v) for v in flat] + for k in range_constexpr(1, kw): + for j in range_constexpr(mr * 4): + other = Vec(llvm.load(T.vec(4, T.f32), _lds_ptr(base, (((nk + k * nw) * mr * 4 + j) * 64 + lane) * 16), alignment=16)) + vals[j] = vals[j] + other + _swiglu_quant_store(vals, mr, mb * bm, col, l16, q16, tok, m, qr, ss) + + return tile + + +@launcher +def compile_gemm1_tile(*, kw=4, pf=2, bm=16, view=False): + """The gate/up tile of ``compile_gemm1(kw, pf, bm, view)`` as a body another launch runs on its workgroup + ``bid``: ``tile(base, bid, A, AS, W, WS, Q, S, m, rows)``, ``base`` = its LDS.""" + return compile_gemm1(kw=kw, pf=pf, bm=bm, view=view) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_epilogue_early.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_epilogue_early.py new file mode 100644 index 0000000..9fab8e9 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_epilogue_early.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared gate/up GEMM (``StoreSharedFp32``'s bits) storing each previous-tile 16-row group right after its +accumulator reads, so no activated FP32 tile stays live across the next tile's first MFMA step.""" +import functools + +from .shared_dense import StoreSharedFp32 +from .mxfp8.shared_dense_4wave import compile_mxfp8_gemm_4w_gateup +from amd_kernels.warm_up import launcher + + +class StoreSharedFp32Early(StoreSharedFp32): + def thunks_rm(self, c_gate, c_up, base_row, base_col): + # both previous-tile quadrants are read before step 0's groups 2 / 3 zero their AGPRs: store in that phase + return self.thunks(c_gate, c_up, base_row, base_col), [] + + +@functools.cache +@launcher +def compile_shared_gemm1_a8w8_epilogue_early(*, up_rows=None, tiles=(4, 4)): + return compile_mxfp8_gemm_4w_gateup( + K=6144, alpha=1.702, beta=1.0, limit=7.0, + epilogue_type=StoreSharedFp32Early, + kernel_prefix='shared_gemm1_a8w8_swiglu_quant_early_h6144_i1536_', + tiles=tiles, up_rows=up_rows, + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_single_tile.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_single_tile.py new file mode 100644 index 0000000..61d5997 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_single_tile.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared gate/up GEMM (``StoreSharedFp32``'s bits) when each workgroup owns one 256x256 parent tile, so there is no +previous-tile epilogue; the grid must have at most 256 parents: ceil(M / 256) * 12 <= 256.""" +import functools + +from .shared_dense import StoreSharedFp32 +from .mxfp8.shared_dense_4wave import compile_mxfp8_gemm_4w_gateup +from amd_kernels.warm_up import launcher + + +class StoreSharedFp32SingleTile(StoreSharedFp32): + def thunks_rm(self, c_gate, c_up, base_row, base_col): + return [], [] + + +@functools.cache +@launcher +def compile_shared_gemm1_a8w8_single_tile(*, up_rows=None, tiles=(4, 4)): + return compile_mxfp8_gemm_4w_gateup( + K=6144, alpha=1.702, beta=1.0, limit=7.0, + epilogue_type=StoreSharedFp32SingleTile, + kernel_prefix='shared_gemm1_a8w8_swiglu_quant_single_tile_h6144_i1536_', + tiles=tiles, up_rows=up_rows, + ) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_write_through.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_write_through.py new file mode 100644 index 0000000..9b99b15 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm1_a8w8_write_through.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared gate/up GEMM (``StoreSharedFp32``'s bits) for the composed routed + shared launch: one tile per workgroup, +codes and scales stored with cache policy 16 (write-through) for the down-GEMM consumers of the same launch.""" +import functools +import flydsl.expr as fx +from flydsl.expr import range_constexpr +from amd_kernels.common import buffer_ops +from .shared_dense import StoreSharedFp32 +from .mxfp8.shared_dense_4wave import _scale_off, compile_mxfp8_gemm_4w_gateup +from .shared_quant import _fp8, _e8m0_fp8 +from amd_kernels.warm_up import launcher + +class StoreSharedFp32WriteThrough(StoreSharedFp32): + def thunks_rm(self,*args): + return [],[] + def _quant_store(self,values,base_row,base_col,ti): + row=base_row+ti*16+self.r + for h in range_constexpr(self.n_b//2): + v=values[h*8:h*8+8] + amax=fx.math.absf(v[0]) + for j in range_constexpr(1,8): + z=fx.math.absf(v[j]);amax=(amax>z).select(amax,z) + for sh in range_constexpr(2): + z=amax.shuffle_xor(16<z).select(amax,z) + e=_e8m0_fp8(amax);scale=(e<<23).bitcast(fx.Float32) + col=base_col+32*h+self.qg*4 + for j in range_constexpr(2): + pk=_fp8(fx.Int32(0),v[j*4],v[j*4+1],scale,False) + pk=_fp8(pk,v[j*4+2],v[j*4+3],scale,True) + buffer_ops.buffer_store(pk,self.q_rsrc,(row*self.out_n+col+j*16)//4,cache_modifier=16) + buffer_ops.buffer_store(e.to(fx.Int8),self.s_rsrc,_scale_off(row,base_col//32+h,self.scale_k1), + offset_is_bytes=True,mask=self.qg==0,cache_modifier=16) + +@functools.cache +@launcher +def compile_shared_gemm1_a8w8_write_through(*,tiles,up_rows=None): + # the composed host issues at most 168 shared workgroups (<= grid_max 256): one tile each + return compile_mxfp8_gemm_4w_gateup(K=6144,alpha=1.702,beta=1.0,limit=7.0, + epilogue_type=StoreSharedFp32WriteThrough, + kernel_prefix='shared_gemm1_a8w8_swiglu_quant_write_through_h6144_i1536_', + tiles=tiles,up_rows=up_rows) diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm2_a8w8.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm2_a8w8.py new file mode 100644 index 0000000..17aac5e --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_gemm2_a8w8.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared MXFP8 down-GEMM tile bodies with direct operands and one BF16 writer per output, which the routed small-tile +GEMM2s run first in their launch.""" +import functools +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .shared_gemm1_a8w8 import _mfma, _hold, _raw, _lds_ptr, _load8 + +H, I, E = 6144, 1536, 128 +# the TP2 rank's K slice read in place from the full [H, 2I] down weight (``SharedWeights.full``) +VIEW_W_K = 2 * I + + +@functools.cache +def compile_gemm2(*, tn=256, bm=16, pf=2, view=False): + """The body ``tile(base, bid, A, AS, W, WS, OUT, m)`` of a down-GEMM tile of ``bm`` rows x ``tn`` columns with + ``pf`` K-steps in flight; the output goes through its ``lds_bytes`` of LDS from ``base``. ``view``: W / WS start + at the rank's K slice of the full weight (row stride ``VIEW_W_K``).""" + # tn 64: one 16-column subtile per wave, so the wave's half of a 32-column scale word is runtime (wave parity); + # its bytes are shifted into the even slot + assert tn in (64, 128, 256, 512) and bm in (16, 32, 64) + ni, mr = tn // 64, bm // 16 + wk = VIEW_W_K if view else I + + @flyc.jit + def tile(base, bid, A, AS, W, WS, OUT, m): + tx = fx.thread_idx.x + lane, wave = tx % 64, tx // 64 + l16, q16 = lane % 16, lane // 16 + mb, nb = bid // (H // tn), bid % (H // tn) + if mb * bm < m: + ar = buffer_ops.create_buffer_resource(A, max_size=False, num_records_bytes=m * I) + sr = buffer_ops.create_buffer_resource(AS, max_size=False, num_records_bytes=m * (I // 32)) + wr = buffer_ops.create_buffer_resource(W, max_size=False, num_records_bytes=H * wk) + sw = buffer_ops.create_buffer_resource(WS, max_size=False, num_records_bytes=H * (wk // 32)) + out = buffer_ops.create_buffer_resource(OUT, max_size=False, num_records_bytes=m * H * 2) + ad = [((mb * bm + mi * 16 + l16) * I + q16 * 16) // 4 for mi in range_constexpr(mr)] + sd = [(mb * bm + mi * 16 + l16) * (I // 32) // 4 for mi in range_constexpr(mr)] + wd = [((nb * (tn // 16) + wave * ni + j) * (wk // 128) * 2048 + q16 * 256 + l16 * 16) // 4 + for j in range_constexpr(ni)] + swd = [((nb * (tn // 32) + (wave * ni + j) // 2) * (wk // 256)) * 64 + q16 * 16 + l16 + for j in range_constexpr(ni)] + def load(k): + aa = [_load8(ar, ad[mi], k * 128) for mi in range_constexpr(mr)] + sa = [(fx.Int32(buffer_ops.buffer_load(sr, sd[mi], vec_width=1, dtype=fx.Int32, soffset_bytes=k * 4)) >> (q16 * 8)) & 255 + for mi in range_constexpr(mr)] + bb = [_load8(wr, wd[j], k * 2048, 256) for j in range_constexpr(ni)] + sb = [buffer_ops.buffer_load(sw, swd[j], vec_width=1, dtype=fx.Int32, soffset_bytes=(k // 2) * 256) for j in range_constexpr(ni)] + if fx.const_expr(ni == 1): + sb = [fx.Int32(sb[0]) >> ((wave % 2) * 8)] + return aa, sa, bb, sb + acc = [[None] * ni for _ in range(mr)] + ring = [load(k) for k in range_constexpr(pf)] + for k in range_constexpr(I // 128): + if fx.const_expr(k + pf < I // 128): + ring.append(load(k + pf)) + aa, sa, bb, sb = ring.pop(0) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + acc[mi][j] = _mfma(aa[mi], bb[j], acc[mi][j], sa[mi], sb[j], (k % 2) * 2 + j % 2) + flat = _hold([acc[mi][j] for mi in range(mr) for j in range(ni)]) + for mi in range_constexpr(mr): + for j in range_constexpr(ni): + v = Vec(flat[mi * ni + j]) + col = wave * (tn // 4) + j * 16 + l16 + for ii in range_constexpr(4): + off = ((mi * 16 + q16 * 4 + ii) * tn + col) * 4 + llvm.StoreOp(_raw(v[ii]), _lds_ptr(base, off)) + fx.rocdl.s_waitcnt(lgkmcnt=0) + fx.gpu.barrier() + for rr in range_constexpr(bm // 8): + row = rr * 8 + tx // 32 + token = mb * bm + row + for c in range_constexpr(tn // 64): + col = tx % 32 * 2 + c * 64 + v = Vec(llvm.load(T.vec(2, T.f32), _lds_ptr(base, (row * tn + col) * 4), alignment=8)) + pk = v.to(fx.BFloat16).bitcast(fx.Int32)[0] + buffer_ops.buffer_store(pk, out, (token * H + nb * tn + col) // 2, mask=token < m) + + tile.lds_bytes = bm * tn * 4 + return tile diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_quant.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_quant.py new file mode 100644 index 0000000..1e44c94 --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/shared_quant.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One MXFP8 activation rule for the shared expert at every row count.""" +import functools +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec +from .common.launch import _run_compiled +from amd_kernels.warm_up import launcher + + +def _e8m0_fp8(amax): + """e8m0 exponent of ceil_pow2(amax / 448), floored at 1; 255 (NaN / Inf) is not bumped.""" + u = (amax * fx.Float32(1.0 / 448.0)).bitcast(fx.Int32) + e = (u >> 23) & 255 + bump = ((u & 0x7FFFFF) != 0) & (e < 255) + e = bump.select(e + 1, e) + return (e < 1).select(fx.Int32(1), e) + + +def _fp8(old, a, b, scale, hi): + """v_cvt_scalef32_pk_fp8_f32: (a, b) / scale -> 2 e4m3 into the low (``hi`` False) or high half of ``old``.""" + ty = ir.VectorType.get([2], ir.IntegerType.get_signless(16)) + old_v = llvm.bitcast(ty, fx.as_ir_value(old)) + res = llvm.call_intrinsic(ty, "llvm.amdgcn.cvt.scalef32.pk.fp8.f32", + [old_v, fx.as_ir_value(a), fx.as_ir_value(b), fx.as_ir_value(scale), fx.Boolean(hi).ir_value()], [], []) + return fx.Int32(llvm.bitcast(T.i32, res)) + + +@functools.cache +@launcher +def compile_quant(): + """bf16 [m, 6144] -> e4m3 codes + row-major e8m0 scales per 32 columns, a workgroup per 2048 columns of a row.""" + @flyc.kernel(name="shared_mxfp8_quant_h6144", known_block_size=[256,1,1]) + def kernel(X:fx.Tensor,Q:fx.Tensor,S:fx.Tensor,m:fx.Int32): + tx=fx.thread_idx.x + row,chunk=fx.block_idx.x // 3,fx.block_idx.x % 3 + col=chunk*2048+tx*8 + xr=buffer_ops.create_buffer_resource(X,max_size=False,num_records_bytes=m*6144*2) + qr=buffer_ops.create_buffer_resource(Q,max_size=False,num_records_bytes=m*6144) + sr=buffer_ops.create_buffer_resource(S,max_size=False,num_records_bytes=m*192) + raw=Vec(buffer_ops.buffer_load(xr,(row*6144+col)//2,vec_width=4,dtype=fx.Int32)) + v=[] + for j in range_constexpr(4): + dw=fx.Int32(raw[j]) + v.append((dw << 16).bitcast(fx.Float32)) + v.append((dw & fx.Int32(-65536)).bitcast(fx.Float32)) + amax=fx.math.absf(v[0]) + for j in range_constexpr(1,8): + z=fx.math.absf(v[j]);amax=(amax>z).select(amax,z) + for sh in range_constexpr(2): + z=amax.shuffle_xor(1<z).select(amax,z) + e=_e8m0_fp8(amax);scale=(e<<23).bitcast(fx.Float32) + lo=_fp8(fx.Int32(0),v[0],v[1],scale,False);lo=_fp8(lo,v[2],v[3],scale,True) + hi=_fp8(fx.Int32(0),v[4],v[5],scale,False);hi=_fp8(hi,v[6],v[7],scale,True) + buffer_ops.buffer_store(Vec.from_elements([lo,hi],fx.Int32),qr,(row*6144+col)//4) + buffer_ops.buffer_store(e.to(fx.Int8),sr,row*192+col//32,offset_is_bytes=True,mask=(tx%4)==0) + @flyc.jit + def launch_shared_mxfp8_quant(X:fx.Tensor,Q:fx.Tensor,S:fx.Tensor,m:fx.Int32,stream:fx.Stream): + kernel(X,Q,S,m).launch(grid=(m*3,1,1),block=(256,1,1),stream=stream) + return launch_shared_mxfp8_quant + + +def shared_quant(x, stream=None): + """MXFP8 codes [m, 6144] and row-major e8m0 scales [m, 192] of bf16 ``x``. + + Args: + stream: launch there; the codes and scales still come from the current stream's allocator pool and are + recorded on ``stream`` (the shared expert's side stream). + """ + assert x.dtype == torch.bfloat16 and x.is_contiguous() and x.shape[1] == 6144 + m=x.shape[0] + q=torch.empty((m,6144),device=x.device,dtype=torch.uint8) + s=torch.empty((m,192),device=x.device,dtype=torch.uint8) + if stream is None: + stream=torch.cuda.current_stream() + elif stream != torch.cuda.current_stream(): + q.record_stream(stream);s.record_stream(stream) + _run_compiled(compile_quant(),x.view(-1),q.view(-1),s.view(-1),m,stream) + return q,s diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_pack.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_pack.py new file mode 100644 index 0000000..8b54d8a --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_pack.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The correction mask of the lossless MXFP4 sidecar: one bit per value, set where FP4 rounding of the shared MXFP8 +value (with the source FP4 scale) differs from the direct MXFP4 code. + +That happens only where FP8 rounding lands on an FP4 midpoint; the sign, negative zero included, stays in the FP8 code. +Both operands come straight from finite bf16: the transport changes, never the quantization rule. +""" +import functools +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec +from amd_kernels.common import buffer_ops +from .common.launch import _run_compiled +from .mxfp4_from_mxfp8 import _e4m3x2 +from .quant import _cvt_pk_fp4 +from amd_kernels.warm_up import launcher + +@functools.cache +@launcher +def compile_sidecar_pack(): + name='mxfp4_mxfp8_sidecar_pack_h6144' + @flyc.kernel(name=name,known_block_size=[256,1,1]) + def kernel(Q8:fx.Tensor,S8:fx.Tensor,S4:fx.Tensor,Q4:fx.Tensor,MASK:fx.Tensor,n:fx.Int32): + b=fx.block_idx.x*256+fx.thread_idx.x + q8r=buffer_ops.create_buffer_resource(Q8,max_size=False,num_records_bytes=n*32) + s8r=buffer_ops.create_buffer_resource(S8,max_size=False,num_records_bytes=n) + s4r=buffer_ops.create_buffer_resource(S4,max_size=False,num_records_bytes=n) + q4r=buffer_ops.create_buffer_resource(Q4,max_size=False,num_records_bytes=n*16) + mr=buffer_ops.create_buffer_resource(MASK,max_size=False,num_records_bytes=n*4) + e8=fx.Int32(buffer_ops.buffer_load(s8r,b,vec_width=1,dtype=fx.Int8)).to(fx.Int32)&255 + e4=fx.Int32(buffer_ops.buffer_load(s4r,b,vec_width=1,dtype=fx.Int8)).to(fx.Int32)&255 + # unscaled FP8 values times the FP4 / FP8 scale ratio: midpoint comparisons stay normal for tiny bf16 values + ratio=((e4-e8+127)<<23).bitcast(fx.Float32) + mask=fx.Int32(0) + for d in range_constexpr(4): + words=Vec(buffer_ops.buffer_load(q8r,b*8+d*2,vec_width=2,dtype=fx.Int32)) + vals=[] + for w in range_constexpr(2): + for hi in (False,True): + vals.extend(_e4m3x2(fx.Int32(words[w]),fx.Float32(1.0),hi)) + candidate=fx.Int32(0) + for j in range_constexpr(4): + candidate=_cvt_pk_fp4(candidate,vals[j*2],vals[j*2+1],ratio,j) + direct=fx.Int32(buffer_ops.buffer_load(q4r,b*4+d,vec_width=1,dtype=fx.Int32)) + for j in range_constexpr(8): + diff=((direct>>(j*4))&15)!=((candidate>>(j*4))&15) + mask=mask|diff.select(fx.Int32(1<<(d*8+j)),fx.Int32(0)) + buffer_ops.buffer_store(mask,mr,b) + @flyc.jit + def launch_mxfp4_mxfp8_sidecar_pack_h6144(Q8:fx.Tensor,S8:fx.Tensor,S4:fx.Tensor,Q4:fx.Tensor,MASK:fx.Tensor,n:fx.Int32,stream:fx.Stream): + kernel(Q8,S8,S4,Q4,MASK,n).launch(grid=((n+255)//256,1,1),block=(256,1,1),stream=stream) + return launch_mxfp4_mxfp8_sidecar_pack_h6144 + +def pack_mxfp4_sidecar(q4,s4,q8,s8): + n=s4.numel();m=q4.shape[0] + mask=torch.empty((m,q4.shape[1]//4),dtype=torch.uint8,device=q4.device) + _run_compiled(compile_sidecar_pack(),q8.view(-1),s8.view(-1),s4.view(-1),q4.view(-1),mask.view(-1),n,torch.cuda.current_stream()) + return mask diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_unpack.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_unpack.py new file mode 100644 index 0000000..77ed7de --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sidecar_unpack.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The direct MXFP4 codes rebuilt from the shared MXFP8 codes, the source FP4 scales and ``sidecar_pack``'s correction +bit per value: FP4 rounding of the FP8 value, moved to the other neighbour where the bit is set. + +The bit is set only where FP8 rounding lands on an FP4 midpoint; the sign, negative zero included, stays in the FP8 +code. Both operands come straight from finite bf16: the transport changes, never the quantization rule. +""" +import functools +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec +from amd_kernels.common import buffer_ops +from .common.launch import _run_compiled +from .mxfp4_from_mxfp8 import _e4m3x2 +from .quant import _cvt_pk_fp4 +from amd_kernels.warm_up import launcher + +@functools.cache +@launcher +def compile_sidecar_unpack(): + name='mxfp4_mxfp8_sidecar_unpack_bit_direction_h6144' + @flyc.kernel(name=name,known_block_size=[256,1,1]) + def kernel(Q8:fx.Tensor,S8:fx.Tensor,S4:fx.Tensor,Q4:fx.Tensor,MASK:fx.Tensor,n:fx.Int32): + b=fx.block_idx.x*256+fx.thread_idx.x + q8r=buffer_ops.create_buffer_resource(Q8,max_size=False,num_records_bytes=n*32) + s8r=buffer_ops.create_buffer_resource(S8,max_size=False,num_records_bytes=n) + s4r=buffer_ops.create_buffer_resource(S4,max_size=False,num_records_bytes=n) + q4r=buffer_ops.create_buffer_resource(Q4,max_size=False,num_records_bytes=n*16) + mr=buffer_ops.create_buffer_resource(MASK,max_size=False,num_records_bytes=n*4) + e8=fx.Int32(buffer_ops.buffer_load(s8r,b,vec_width=1,dtype=fx.Int8)).to(fx.Int32)&255 + e4=fx.Int32(buffer_ops.buffer_load(s4r,b,vec_width=1,dtype=fx.Int8)).to(fx.Int32)&255 + # unscaled FP8 values times the FP4 / FP8 scale ratio: midpoint comparisons stay normal for tiny bf16 values + ratio=((e4-e8+127)<<23).bitcast(fx.Float32) + mask=fx.Int32(buffer_ops.buffer_load(mr,b,vec_width=1,dtype=fx.Int32)) + out=[] + for d in range_constexpr(4): + words=Vec(buffer_ops.buffer_load(q8r,b*8+d*2,vec_width=2,dtype=fx.Int32)) + vals=[] + for w in range_constexpr(2): + for hi in (False,True): + vals.extend(_e4m3x2(fx.Int32(words[w]),fx.Float32(1.0),hi)) + candidate=fx.Int32(0) + for j in range_constexpr(4): + candidate=_cvt_pk_fp4(candidate,vals[j*2],vals[j*2+1],ratio,j) + fixed=fx.Int32(0) + for j in range_constexpr(8): + code=(candidate>>(j*4))&15 + # only set mask bits reach the correction; at the seven midpoints FP8 mantissa bit 2 selects upper-even + # (.75 / 1.75 / 3.5) + twice_down=(fx.Int32(words[j//4])>>((j%4)*8+1))&2 + other=code+1-twice_down + code=(((mask>>(d*8+j))&1)!=0).select(other,code) + fixed=fixed|(code<<(j*4)) + out.append(fixed) + buffer_ops.buffer_store(Vec.from_elements(out,fx.Int32),q4r,b*4) + @flyc.jit + def launch_mxfp4_mxfp8_sidecar_unpack_bit_direction_h6144(Q8:fx.Tensor,S8:fx.Tensor,S4:fx.Tensor,Q4:fx.Tensor,MASK:fx.Tensor,n:fx.Int32,stream:fx.Stream): + kernel(Q8,S8,S4,Q4,MASK,n).launch(grid=((n+255)//256,1,1),block=(256,1,1),stream=stream) + return launch_mxfp4_mxfp8_sidecar_unpack_bit_direction_h6144 + +def unpack_mxfp4_sidecar(mask,s4,q8,s8): + q4=torch.empty((q8.shape[0],q8.shape[1]//2),dtype=torch.uint8,device=q8.device) + _run_compiled(compile_sidecar_unpack(),q8.view(-1),s8.view(-1),s4.view(-1),q4.view(-1),mask.view(-1),s4.numel(),torch.cuda.current_stream()) + return q4 diff --git a/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sort_a16w4.py b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sort_a16w4.py new file mode 100644 index 0000000..58d727a --- /dev/null +++ b/amd_kernels/moe/mxfp4_h6144_i1536_e128_top4/sort_a16w4.py @@ -0,0 +1,242 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One-workgroup expert sort for at most 640 tokens (16 / 32-row blocks, at most 256 of them); no output clearing. + +The sorted table is built in LDS: every row is written as padding under the id loads, the pairs land on their rows +after one wave's scan of the 64 expert pairs, and rows [0, num_valid) leave in 16 B stores after all LDS reads (one +CU's memory pipeline carries every store, so scattered dword stores would be most of the kernel). A block's first row +always holds a pair, which writes the block -> expert entry. Each pair owns its GEMM2 output row, so placement order +cannot change its bits. +""" + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from amd_kernels.common import buffer_ops +from .common.launch import _run_compiled +from .common.sort import max_sorted_rows +from .common.utils import _lds_atomic_add_i32, wave_incl_scan_i32 +from amd_kernels.warm_up import launcher + + +threads = 256 # one thread per histogram/scan entry +# rank placement from 17 (inline routing below): chain -0.05..-0.20 us at 64-512, sort alone -0.03 us at 17-32 +SORT_RANK_FROM = 17 +# compile tiers (max tokens; pairs per thread = 4 * tier / 256), each reached by a WARM_UP_TOKENS size of the entry +SORT_TOKEN_TIERS = (64, 128, 192, 256, 384, 448, 512, 640) + + +@functools.cache +@launcher +def compile_a16w4_sort( + *, E: int, topk: int, block_m: int, max_tokens: int, rank: bool = True +): + """Histogram, padded-count scan and placement, with no cross-CTA state.""" + assert E == 128 + assert (block_m & (block_m - 1)) == 0 and threads >= E + bm_shift = block_m.bit_length() - 1 + PPT = (max_tokens * topk + threads - 1) // threads + n_waves = threads // 64 + pad_epw = 64 // block_m # experts per wave and padding pass + assert block_m <= 64 and E % (pad_epw * n_waves) == 0 + tag = f"e{E}_top{topk}_bm{block_m}_ppt{PPT}_t{threads}" + ("_rank" if rank else "") + # rank: invalid pairs count into their own slot past E (distinct addresses, never read) + COUNT = E + threads * PPT if rank else E + # cursor: invalid pairs advance their own slot past E too (no branch around the atomic) + CURSOR = E if rank else E + threads * PPT + # the sorted table in LDS (ids rows, then weight bits): padding everywhere first, the pairs on top, then + # [0, num_valid) leaves in 16 B stores + MS = max_sorted_rows(max_tokens, E, topk, block_m) + CH = (MS // 4 + threads - 1) // threads # 16 B tiles per thread and array + TAB = CH * threads * 4 # rows per array + assert TAB > MS # row MS takes the invalid pairs + NB = MS // block_m # blocks of the bound: one block -> expert entry per thread + assert NB <= threads + + @fx.struct + class Shared: + count: fx.Array[fx.Int32, COUNT] # rows per expert + cursor: fx.Array[fx.Int32, CURSOR] # next free sorted row per expert + scan: fx.Array[fx.Int32, E] # ping-pong scan arrays + nvs: fx.Array[fx.Int32, 4] # num_valid for every wave + tab: fx.Array[fx.Int32, 2 * TAB, 16] # sorted ids rows, then weight bits (16 B aligned) + blk: fx.Array[fx.Int32, threads + threads * PPT] # block -> expert; past threads: one dummy per pair + + @flyc.kernel(name=f"sort_a16w4_pairs_{tag}", known_block_size=[threads, 1, 1]) + def sort_a16w4( + topk_ids: fx.Tensor, # [n_tokens * topk] i32 + topk_w: fx.Tensor, # [n_tokens * topk] f32 + sorted_ids: fx.Tensor, # [max_sorted] i32 + sorted_w: fx.Tensor, # [max_sorted] f32 + sorted_eids: fx.Tensor, # [max_sorted / block_m] i32 + num_valid: fx.Tensor, # [2] i32 + n_tok: fx.Int32, + ): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + if bx == 0: + count, cursor, blk = smem.count.ptr, smem.cursor.ptr, smem.blk.ptr + scan = [smem.scan.ptr] + n_pairs = n_tok * topk + last_pair = n_pairs - 1 + t_lt_E = tx < E + active = (n_pairs < E).select(n_pairs, fx.Int32(E)) + ms_rows = ((n_pairs + active * (block_m - 1) + (block_m - 1)) >> bm_shift) << bm_shift + ids_r = buffer_ops.create_buffer_resource(sorted_ids, max_size=False, num_records_bytes=ms_rows * 4) + w_r = buffer_ops.create_buffer_resource(sorted_w, max_size=False, num_records_bytes=ms_rows * 4) + e_r = buffer_ops.create_buffer_resource(sorted_eids, max_size=False, + num_records_bytes=(ms_rows >> bm_shift) * 4) + + # 0. this thread's pairs: one batch of loads (clamped index, validity) + pairs = [] + for k in range_constexpr(PPT): + p = tx + k * threads + valid = p < n_pairs + pc = valid.select(p, last_pair) + pairs.append((p, valid, fx.Int32(topk_ids[pc]), fx.Float32(topk_w[pc]))) + + lds16 = fx.logical_divide( + fx.make_view(fx.recast_iter(fx.Int32, smem.tab.ptr), fx.make_layout(2 * TAB, 1)), + fx.make_layout(4, 1), + ) + lds_atom = fx.make_copy_atom(fx.UniversalCopy128b(), fx.Int32) + + def lds_store16(tile, vec4): + r = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + r.store(vec4) + fx.copy(lds_atom, r, fx.slice(lds16, (None, tile))) + + def lds_load16(tile): + r = fx.make_rmem_tensor(fx.make_layout(4, 1), fx.Int32) + fx.copy(lds_atom, fx.slice(lds16, (None, tile)), r) + return Vec(r.load()) + + # every table row as padding (token n_tokens, slot 0, weight 0), under the id loads + pad_ids = Vec.from_elements([n_tok] * 4, fx.Int32) + pad_w = Vec.filled(4, 0, fx.Int32) + for j in range_constexpr(CH): + lds_store16(tx + j * threads, pad_ids) + lds_store16(tx + (j + CH) * threads, pad_w) + tab = smem.tab.ptr + + # 1. zero the counters + if t_lt_E: + count[tx] = 0 + fx.gpu.barrier() + + # 2. histogram (by scan slot); rank: the returned count is the pair's rank + ranks = [] + for k, (p, valid, e, _w) in enumerate(pairs): + if const_expr(rank): + slot = valid.select(e, fx.Int32(E + k * threads) + tx) + ranks.append(_lds_atomic_add_i32(count + slot, 1)) + else: + if valid: + _lds_atomic_add_i32(count + e, 1) + fx.gpu.barrier() + + # One wave scans 64 consecutive expert pairs; starts/ends stay in expert order. + if wave == 0: + c0, c1 = fx.Int32(count[lane*2]), fx.Int32(count[lane*2+1]) + p0 = (c0 + (block_m-1)) & ~(block_m-1) + p1 = (c1 + (block_m-1)) & ~(block_m-1) + pair_end = wave_incl_scan_i32(p0+p1) + pair_start = pair_end-(p0+p1) + for j in range_constexpr(2): + e_t = lane*2+j + start = pair_start + (p0 if const_expr(j == 1) else fx.Int32(0)) + end = start + (p0 if const_expr(j == 0) else p1) + scan[0][e_t] = start if const_expr(rank) else end + if const_expr(not rank): + cursor[e_t] = start + if lane == 63: + num_valid[0] = pair_end + smem.nvs.ptr[0] = pair_end + fx.gpu.barrier() + rows = [] + for k, (p, valid, e, w) in enumerate(pairs): + if const_expr(rank): + rows.append(fx.Int32(scan[0][e]) + ranks[k]) + else: + slot = valid.select(e, fx.Int32(E + k * threads) + tx) + rows.append(_lds_atomic_add_i32(cursor + slot, 1)) + nv = fx.Int32(smem.nvs.ptr[0]) + # 5. place the pairs into the table (rows of a pair's expert, disjoint from its padding) + for k, (p, valid, e, w) in enumerate(pairs): + r = valid.select(rows[k], fx.Int32(MS)) # invalid pairs: a row past MS, never copied + tab[r] = ((p // topk) & 0x00FFFFFF) | ((p % topk) << 24) + tab[r + TAB] = w.bitcast(fx.Int32) + # a block's first row is always a pair (padding < block_m rows): its pair names the block + first = valid & ((r & (block_m - 1)) == 0) + blk[first.select(r >> bm_shift, fx.Int32(threads + k * threads) + tx)] = e + fx.gpu.barrier() + # 6. rows [0, num_valid) in 16 B stores: every LDS read first, then the stores + tiles = [] + for j in range_constexpr(CH): + t = tx + j * threads + tiles.append((t, lds_load16(t), lds_load16(t + CH * threads))) + blk_e = fx.Int32(blk[tx]) + buffer_ops.buffer_store(blk_e, e_r, tx, mask=tx < (nv >> bm_shift)) + for t, vi, vw in tiles: + in_rows = t * 4 < nv + buffer_ops.buffer_store(vi, ids_r, t * 4, mask=in_rows) + buffer_ops.buffer_store(vw, w_r, t * 4, mask=in_rows) + + @flyc.jit + def launch_sort_pairs_mxfp4_a16w4( + topk_ids: fx.Tensor, + topk_w: fx.Tensor, + sorted_ids: fx.Tensor, + sorted_w: fx.Tensor, + sorted_eids: fx.Tensor, + num_valid: fx.Tensor, + n_tokens: fx.Int32, + stream: fx.Stream, + ): + sort_a16w4( + topk_ids, + topk_w, + sorted_ids, + sorted_w, + sorted_eids, + num_valid, + n_tokens, + ).launch(grid=(1, 1, 1), block=(threads, 1, 1), stream=stream) + + return launch_sort_pairs_mxfp4_a16w4 + + +def moe_sort_a16w4(topk_ids, topk_weights, E, block_m, *, rank=None): + """Packed rows, sorted weights, block experts and the padded valid row count; ``rank`` (None: from + ``SORT_RANK_FROM`` tokens) picks ``compile_a16w4_sort``'s placement.""" + n_tokens, topk = topk_ids.shape + if rank is None: + rank = n_tokens >= SORT_RANK_FROM + dev = topk_ids.device + ms = max_sorted_rows(n_tokens, E, topk, block_m) + sorted_ids = torch.empty(ms, dtype=torch.int32, device=dev) + sorted_w = torch.empty(ms, dtype=torch.float32, device=dev) + sorted_eids = torch.empty(ms // block_m, dtype=torch.int32, device=dev) + num_valid = torch.empty(1, dtype=torch.int32, device=dev) + assert n_tokens <= SORT_TOKEN_TIERS[-1] + launch = compile_a16w4_sort( + E=E, topk=topk, block_m=block_m, + max_tokens=next(t for t in SORT_TOKEN_TIERS if n_tokens <= t), rank=rank, + ) + # through the cached handle: a direct @flyc.jit call enters the JIT on every call (FlyDSL 0.5: a full trace) + _run_compiled( + launch, + topk_ids.contiguous().int().view(-1), + topk_weights.contiguous().float().view(-1), + sorted_ids, sorted_w, sorted_eids, num_valid, int(n_tokens), + torch.cuda.current_stream(), + ) + return sorted_ids, sorted_w, sorted_eids, num_valid diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/README.md b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/README.md new file mode 100644 index 0000000..2b82412 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/README.md @@ -0,0 +1,47 @@ +# Two-rank all-reduce + Gemma norm, bf16-row MXFP8, hidden 6144 + +## What it does + +`Bf16ArNormComm` fuses the TP2 all-reduce of a bf16 partial with the residual add, Gemma RMSNorm and (optionally) the +MXFP8 quant, in one launch, for the MiniMax-M3.1-Flash DSpark draft on TP2 (the draft's o_proj / down stay TP +partials; nine quantized tails and one final bf16 tail per draft step). Input bf16 `[rows, 6144]` contiguous, rows +1-2048 (the draft runs 7 per request); residual bf16, unit column stride, row stride divisible by 8; weight bf16 +`[6144]` contiguous. Codes row-major or (16, 16)-tiled, padded to `code_tile_rows`; scales in AITER's shuffled layout, +padded to 32 rows; padding rows are not written. Contract (`reference.py`): `bf16(ftz(ftz(p0) + ftz(p1)))` at every +row (commutative: AITER's one- and two-stage bits at two ranks); the residual add is fp32; the norm (weight + 1) uses +the 12-wave square tree of the plain norm; the quant reads the bf16-rounded norm (the draft's rule). Deterministic; +atomics only on flags. + +## Entry points + +- `Bf16ArNormComm`: the communicator, a collective on two ranks with the device current; ~50 MiB shared per rank. +- `Bf16ArNormComm.warm_up`: collective; compiles every row tier, both code layouts, bf16 out, both residual forms. +- `Bf16ArNormComm.ar_add_rmsnorm_mxfp8`: all-reduce + add + norm + quant -> (codes, scales, residual out). +- `Bf16ArNormComm.ar_add_rmsnorm`: the same with bf16 out -> (out, residual out). +- The owned-residual form of both (from `host.OWNER_MIN_ROWS`): each rank reduces, norms and quantizes its half of + the rows and stages the result (codes + scales, or the bf16 row) for the peer, so the residual is read and written on + `owned_rows(rows)` only. Pass a full residual or the residual out of an owned call with the same row count. Outputs + equal the all-row form bit for bit. +- `Bf16ArNormComm.capture`: a no-op context matching the four-rank API (eager and graph calls run the same kernel). +- `Bf16ArNormComm.close`: collective, after graph use has ended. + +One communicator per stream, the same calls in the same order on every rank. `host.config` picks rows per batch and +the role grid by row count. + +## Performance + +MI355X TP2 (one xGMI link, ~56 GB/s per direction). Graphs over distinct inputs beyond the 256 MB MALL, MAX over +ranks, median us. Baseline: AITER custom all-reduce + the one-GPU bf16 norm + the FlyDSL quantizer. MXFP8, tile 16. + +| rows | 7 | 28 | 56 | 224 | 448 | 896 | 1792 | +|---|---:|---:|---:|---:|---:|---:|---:| +| baseline | 11.17 | 16.35 | 23.39 | 63.16 | 114.95 | 218.35 | 422.80 | +| this entry, all-row | 6.86 | 11.39 | 17.71 | 57.15 | 107.71 | 211.52 | 415.82 | +| this entry, owned residual | (all-row) | 11.06 | 15.37 | 45.45 | 86.90 | 168.76 | 336.26 | + +## Tests + +```bash +python3 -m pytest tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2 -q # 2 GPUs +python3 benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py # 2 GPUs +``` diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/__init__.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/__init__.py new file mode 100644 index 0000000..f2cfd00 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/__init__.py @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +from .host import Bf16ArNormComm + +__all__ = ['Bf16ArNormComm'] diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/common.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/common.py new file mode 100644 index 0000000..d9ab9b6 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/common.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-rank one-shot protocol: flags, buffer layout and the bf16 rank sum. + +Every rank owns one uncached buffer mapped by its peer: row slots ``[2, MAX_ROWS, 12288 B]`` (two copies by the row's +call-count parity) that it writes itself, then one 128-byte flag line per row that the peer writes (words 0-11: the +peer's partial rows of a batch staged, wave v; words 16-27: the peer's owner results of a batch staged, wave v). A +writer wave drains its stores into its own slots, then stores the batch's call count into the peer's flag word; the +reader wave with the same columns polls it in its own buffer and reads the rows from the peer's slots (the result's +scale bytes are local byte stores, the link carries whole-line reads). A rank starts call n + 1 only after its call n +saw a flag of the peer's call n, so the peer has finished reading call n - 1: two copies suffice. In one call a row's +slot holds either a partial or a result. + +Each role (stagers, reducers, receivers) keeps its own per-row call counts and advances them once per call for every +row of the call, after its last read: equal on both ranks (same calls in the same order), and a role that finishes +early never changes a count another role of the same call reads. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .norm_math import H, pk_bf16 + +WORLD = 2 +SYS = 0x11 # sc0 | sc1: system-coherent loads, write-through stores +ROWB = H * 2 +MAX_ROWS = 2048 +LINE = 32 # flag words per row: each row's flags in their own 128-byte line +RESULT_FLAG = 16 # first result flag word of a line +DATA_BYTES = 2 * MAX_ROWS * ROWB +FLAG_OFF = DATA_BYTES +BUF_BYTES = FLAG_OFF + MAX_ROWS * LINE * 4 +COUNT_WORDS = MAX_ROWS + 64 # per role; masked rows of a last batch read past MAX_ROWS + + +def _gptr(addr): + return fx.inttoptr( + fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + fx.Int64(addr), + ) + + +def flag_store(addr, v): + """An atomic exchange at system scope (write-through).""" + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.xchg, + _gptr(addr).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ) + + +def _flag_read(addr): + """An atomic add of 0 at system scope (a spin on it is never hoisted).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _gptr(addr).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +@flyc.jit +def spin(a, e): + f = _flag_read(a) + while (f - e) < 0: # signed difference: the counts may wrap + f = _flag_read(a) + + +def flag_addr(base, row, word): + return base + fx.Int64(FLAG_OFF) + fx.Int64((row * LINE + word) * 4) + + +def slot_addr(base, row, e): + """Row ``row``'s slot of call count ``e`` in ``base``'s buffer.""" + return base + (fx.Int64(e & 1) * MAX_ROWS + fx.Int64(row)) * ROWB + + +def _as_f32(i): + return fx.Int32(i).bitcast(fx.Float32) + + +def _ftz(u): + """fp32 bits with a denormal flushed to a zero of its sign.""" + return ((u & 0x7F800000) == 0).select(u & fx.Int32(-0x80000000), u) + + +def sum2(a, b): + """bf16(ftz(ftz(a) + ftz(b))) per element of two 16 B packs in fp32 (AITER's denormal flush; commutative).""" + out = [] + for i in range_constexpr(4): + wa, wb = fx.Int32(a[i]), fx.Int32(b[i]) + lo = _ftz((_as_f32(_ftz(wa << 16)) + _as_f32(_ftz(wb << 16))).bitcast(fx.Int32)) + hi = _ftz((_as_f32(_ftz(wa & fx.Int32(-65536))) + _as_f32(_ftz(wb & fx.Int32(-65536)))).bitcast(fx.Int32)) + out.append(pk_bf16(_as_f32(lo), _as_f32(hi))) + return Vec.from_elements(out, fx.Int32) diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/host.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/host.py new file mode 100644 index 0000000..d4a28db --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/host.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-rank communicator: one IPC-mapped staging buffer per rank, the role call counts and the row tiers.""" +import contextlib +import ctypes +import functools +import struct + +import torch +import torch.distributed as dist + +from amd_kernels.common import hip_runtime as ops +from . import common as C +from .kernel import compile_allreduce_gemma_bf16_mxfp8 + +# owner form from here: 21 rows all-row 10.20 vs owner 10.48 us, 28: 11.26 vs 10.98 (benchmark_allreduce_ranks2.py) +OWNER_MIN_ROWS = 24 + + +@functools.cache +def _kernel(rpb, tiled, quantized, owner): + return compile_allreduce_gemma_bf16_mxfp8(rpb, tiled, quantized, owner) + + +def _run_compiled(exe, *args): + """The first call compiles and runs (``flyc.compile``), later calls run the cached CompiledFunction.""" + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +def _f32_bits(v: float) -> int: + return struct.unpack(" list: + """Every rank's ``obj`` (broadcasts on the CPU group: all_gather_object fails on gloo under inference mode).""" + ranks = sorted(dist.get_process_group_ranks(group=group)) + out = [[None] for _ in ranks] + out[rank][0] = obj + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +def owned_rows(rows, rank): + """[start, stop) of the rows whose sum, residual and norm ``rank`` owns.""" + q = (rows + 1) // 2 + return min(rank * q, rows), min((rank + 1) * q, rows) + + +def config(rows, owner): + """(rows per batch, stagers, reducers, receivers) of a call, measured (benchmark_allreduce_ranks2.py --sweep).""" + if owner: + q = (rows + 1) // 2 + rpb = 1 if q <= 32 else 2 if q <= 256 else 4 if q <= 512 else 8 + nb = 2 * -(-q // rpb) + nred = min(nb, 128 if rows <= 512 else 224) + return rpb, min(nb, 128), nred, nred + if rows <= 256: + rpb = 1 if rows <= 64 else 2 + nb = -(-rows // rpb) + return rpb, nb, nb, 0 + rpb, nstage, nred = (2, 32, 128) if rows <= 512 else (4, 224, 224) if rows <= 1024 else (8, 32, 128) + nb = -(-rows // rpb) + return rpb, min(nb, nstage), min(nb, nred), 0 + + +def warm_calls(): + """(rows, owned_residual) of the first call of every compiled variant (rows per batch, form).""" + seen, out = set(), [] + for owner in (False, True): + for rows in range(1, C.MAX_ROWS + 1): + form = owner and rows >= OWNER_MIN_ROWS + key = (config(rows, form)[0], form) + if key not in seen: + seen.add(key) + out.append((rows, owner)) + return out + + +class Bf16ArNormComm: + """One communicator per stream, created collectively on two ranks. + + Eager and graph calls run the same kernel: each rank stages its partial rows in its own buffer and reads the + peer's from there, so peer inputs are never read in place (``capture()`` is a no-op matching the four-rank API). + All ranks call the same operations in the same order; close only after graph work completes.""" + + config = None # benchmarks may override ``config`` + + def __init__(self, rank, group, device): + if dist.get_world_size(group) != C.WORLD: + raise ValueError("two ranks required") + device = torch.device(device) + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + if torch.cuda.current_device() != device.index or dist.get_rank(group) != rank: + raise ValueError("select this rank device before creating the communicator") + self.rank, self.group, self.device = rank, group, device + ptr, handle = ops.allocate_shared_buffer_and_handle(C.BUF_BYTES) + every = exchange(handle, rank, group) + self.mine, self.peer = ptr, ops.open_mem_handle(every[1 - rank]) + self.counts = torch.zeros(3 * C.COUNT_WORDS, dtype=torch.int32, device=device) + torch.cuda.synchronize(device) + dist.barrier(group=group) + + @contextlib.contextmanager + def capture(self): + yield + + def _launch(self, x, residual, weight, eps, tile, quantized, owner=False): + if self.mine is None: + raise RuntimeError("communicator is closed") + m = x.shape[0] + if not (0 < m <= C.MAX_ROWS and tuple(x.shape) == (m, C.H)): + raise ValueError("expected 1..2048 rows of hidden 6144") + if x.device != self.device or residual.device != self.device or weight.device != self.device: + raise ValueError("all tensors must be on the communicator device") + if x.dtype != torch.bfloat16 or not x.is_contiguous(): + raise ValueError("partial must be contiguous bf16") + if residual.shape != x.shape or residual.dtype != x.dtype or residual.stride(1) != 1 or residual.stride(0) % 8: + raise ValueError("residual must be bf16 with a row stride divisible by eight") + if tuple(weight.shape) != (C.H,) or weight.dtype != x.dtype or not weight.is_contiguous(): + raise ValueError("weight must be contiguous bf16 [6144]") + if tile < 0 or tile % 16: + raise ValueError("code tile rows must be zero or a multiple of 16") + padded = (m + max(tile, 1) - 1) // max(tile, 1) * max(tile, 1) + out = torch.empty((padded if quantized else m, C.H), dtype=torch.float8_e4m3fn if quantized else torch.bfloat16, + device=self.device) + scales = torch.empty(((padded + 31) // 32 * 32, C.H // 32), dtype=torch.uint8, + device=self.device) if quantized else None + res = torch.empty_like(x) + owner = owner and m >= OWNER_MIN_ROWS + rpb, nstage, nred, nrecv = (self.config or config)(m, owner) + assert 0 < nstage < 1024 and 0 < nred < 1024 and 0 <= nrecv < 1024 + _run_compiled(_kernel(rpb, tile > 0, quantized, owner), m, self.rank, nstage | nred << 10 | nrecv << 20, + residual.stride(0), x.data_ptr(), self.mine, self.peer, self.counts.data_ptr(), + residual.data_ptr(), weight.data_ptr(), out.data_ptr(), scales.data_ptr() if quantized else 0, + res.data_ptr(), _f32_bits(eps), torch.cuda.current_stream()) + return (out, scales, res) if quantized else (out, res) + + def ar_add_rmsnorm(self, x, residual, weight, eps=1e-6, owned_residual=False): + """``owned_residual``: from ``OWNER_MIN_ROWS`` rows on, only this rank's ``owned_rows`` of ``residual`` are read + and of the residual output written; the owner norms its rows and stages the bf16 result for the peer. Fewer rows + run the all-row form. So pass a full residual or the residual output of an ``owned_residual`` call of the same + row count. Outputs equal the default form's bit for bit.""" + return self._launch(x, residual, weight, eps, 0, False, owned_residual) + + def ar_add_rmsnorm_mxfp8(self, x, residual, weight, eps=1e-6, code_tile_rows=16, owned_residual=False): + """``owned_residual``: as in ``ar_add_rmsnorm``; the owner stages its codes and scale bytes (about half the + bytes of a bf16 row).""" + return self._launch(x, residual, weight, eps, code_tile_rows, True, owned_residual) + + def owned_rows(self, rows): + return owned_rows(rows, self.rank) + + def warm_up(self, weight, eps=1e-6): + """Compile every row tier, both code layouts, bf16 output and both residual forms collectively.""" + for rows, owner in warm_calls(): + x = torch.zeros((rows, C.H), dtype=torch.bfloat16, device=self.device) + self.ar_add_rmsnorm(x, x, weight, eps, owner) + for tile in (0, 16): + self.ar_add_rmsnorm_mxfp8(x, x, weight, eps, tile, owner) + torch.cuda.synchronize(self.device) + + def close(self): + if self.mine is None: + return + torch.cuda.synchronize(self.device) + dist.barrier(group=self.group) + _release(self.peer, True) + _release(self.mine, False) + self.mine = self.peer = None diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/kernel.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/kernel.py new file mode 100644 index 0000000..bd5cc44 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/kernel.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank one-shot all-reduce with the 12-wave norm and the bf16-row quant rule. + +Rows go in batches of ``rpb``; the grid is stagers, then reducers, then (``owner``) receivers, each role striding over +the batches. All-row form: a stager writes its partial rows into its own slots, a reducer reads the peer's over the +link, sums them with its own per row and adds, norms (and quantizes) every row. ``owner``: rows split into chunk c = +rows [c q, (c + 1) q), q = ceil(rows / 2), owned by rank c: a stager writes only the peer's chunk; a reducer reduces +its own chunk (residual read and written there only) and stages the result (6144 code + 192 scale bytes, or the bf16 +row) in its own slots; a receiver copies the peer's results into its outputs. Stagers never wait and reducers wait only +for peer stagers, so with that dispatch order the grid needs no residency guarantee.""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .common import COUNT_WORDS, RESULT_FLAG, SYS, flag_addr, flag_store, slot_addr, spin, sum2 +from .norm_math import PRELOAD_HINTS, H, _block_sum, bf16x8, pk_bf16, rsqrt_rn, seq_sum_sq, swap_xor1, wave_sum_lane63 +from .quant_math import _code_off, _codes, _e8m0, _scale_off, _set_fp16_ovfl + +NW = H // 8 // 64 # 12 waves of 8 columns per lane (the plain norm's layout) +NT = 64 * NW + + +def compile_allreduce_gemma_bf16_mxfp8(rpb: int, shuffled: bool, quantized: bool, owner: bool = False): + B = rpb + name = (f"allreduce_gemma_bf16_{'mxfp8' if quantized else 'out'}_{'tiled' if shuffled else 'rowmajor'}" + f"_h6144_ranks2_b{B}" + ("_owner" if owner else "")) + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, 16] + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + i32_m: fx.Int32, + i32_me: fx.Int32, + i32_grid: fx.Int32, # stagers | reducers << 10 | receivers << 20 + i32_sr: fx.Int32, # residual row stride (elements) + arg_x: fx.Int64, # [m, H] bf16 partial, contiguous + arg_mine: fx.Int64, + arg_peer: fx.Int64, + arg_cnt: fx.Int64, # int32 [3, COUNT_WORDS]: stagers, reducers, receivers + arg_r: fx.Int64, + arg_w: fx.Int64, + arg_q: fx.Int64, # codes, or the bf16 out + arg_s: fx.Int64, + arg_res: fx.Int64, + i32_eps: fx.Int32, + ): + if const_expr(quantized): + _set_fp16_ovfl() + tx, b = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + m, me = i32_m, i32_me + nstage = i32_grid & 0x3FF + nred = (i32_grid >> 10) & 0x3FF + nrecv = i32_grid >> 20 + nb = (m + (B - 1)) // B + q = (m + 1) // 2 + nb0 = (q + (B - 1)) // B + if const_expr(owner): + nb = nb0 + (m - q + (B - 1)) // B + nbytes = H * 2 + off = tx * 4 + cr = buffer_ops.create_buffer_resource_from_addr(arg_cnt, num_records_bytes=3 * COUNT_WORDS * 4) + + def batch(k): + """(first row, end of its chunk, its chunk) of batch k.""" + if const_expr(owner): + c = (k >= nb0).select(fx.Int32(1), fx.Int32(0)) + r0 = (c == 0).select(k * B, q + (k - nb0) * B) + return r0, (c == 0).select(q, m), c + return k * B, m, fx.Int32(0) + + def count(role, r): + return fx.Int32(buffer_ops.buffer_load(cr, role * COUNT_WORDS + r, vec_width=1, dtype=fx.Int32, + is_scalar=True)) + 1 + + def advance(roles, first, step): + """After the workgroup's last read: count + 1 for every row of its batches.""" + fx.gpu.barrier() + for k in range(first, nb, step): + r0, end, _c = batch(k) + r = r0 + tx + ok = (tx < B) & (r < end) + for role in roles: + cur = fx.Int32(buffer_ops.buffer_load(cr, role * COUNT_WORDS + r, vec_width=1, dtype=fx.Int32, + mask=ok)) + buffer_ops.buffer_store(cur + 1, cr, role * COUNT_WORDS + r, mask=ok) + + def read_row(base, cm, ok): + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=nbytes) + return Vec(buffer_ops.buffer_load(rs, off, vec_width=4, dtype=fx.Int32, mask=ok, cache_modifier=cm)) + + if const_expr(quantized): + q_bytes = ((m + 15) // 16 * 16 if shuffled else m) * H + else: + q_bytes = m * nbytes + s_bytes = (m + 31) // 32 * 32 * (H // 32) + + def store_q(codes, row): + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=q_bytes) + if const_expr(shuffled): + # (16, 16) layout: even lanes store their piece's low 8 B plus the odd lane's high 8 B as one 16 B + # store; odd lanes idle + recv = swap_xor1(codes) + piece = Vec.from_elements([codes[0], codes[1], recv[0], recv[1]], fx.Int32) + buffer_ops.buffer_store(piece, qr, _code_off(row, tx // 2) // 4, mask=(tx % 2) == 0) + else: + buffer_ops.buffer_store(Vec.from_elements(codes, fx.Int32), qr, (row * (H // 4)) + tx * 2) + + def store_s(sbyte, row, g): + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=s_bytes) + buffer_ops.buffer_store(sbyte.to(fx.Int8), sr, _scale_off(row, g), offset_is_bytes=True) + + def store_out(v, row): + qr = buffer_ops.create_buffer_resource_from_addr(arg_q, num_records_bytes=q_bytes) + buffer_ops.buffer_store(v, qr, row * (H // 2) + off) + + if b < nstage: + for k in range(b, nb, nstage): + r0, end, c = batch(k) + if (c != me) | fx.Boolean(not owner): + for i in range_constexpr(B): + r = r0 + i + ok = r < end + dst = buffer_ops.create_buffer_resource_from_addr( + slot_addr(arg_mine, r, count(0, r)), num_records_bytes=nbytes) + buffer_ops.buffer_store(read_row(arg_x + fx.Int64(r) * nbytes, 0, ok), dst, off, mask=ok, + cache_modifier=SYS) + fx.rocdl.s_waitcnt(vmcnt=0) + if lane == 0: + flag_store(flag_addr(arg_peer, r0, wave), count(0, r0)) + advance((0,), b, nstage) + elif b < nstage + nred: + smem = fx.SharedAllocator().allocate(Shared).peek() + part = smem.part.ptr + wr = buffer_ops.create_buffer_resource_from_addr(arg_w, num_records_bytes=nbytes) + wv = bf16x8(Vec(buffer_ops.buffer_load(wr, off, vec_width=4, dtype=fx.Int32))) + for k in range(b - nstage, nb, nred): + r0, end, c = batch(k) + if (c == me) | fx.Boolean(not owner): + oks = [(r0 + i) < end for i in range_constexpr(B)] + owns = [read_row(arg_x + fx.Int64(r0 + i) * nbytes, 0, oks[i]) for i in range_constexpr(B)] + rvs = [read_row(arg_r + fx.Int64(r0 + i) * fx.Int64(i32_sr * 2), 0, oks[i]) + for i in range_constexpr(B)] + # a poll waits for the wave's outstanding vector loads: the batch's own rows and residuals + if lane == 0: + spin(flag_addr(arg_mine, r0, wave), count(1, r0)) + es = [count(1, r0 + i) for i in range_constexpr(B)] + staged = [read_row(slot_addr(arg_peer, r0 + i, es[i]), SYS, oks[i]) for i in range_constexpr(B)] + for i in range_constexpr(B): + row = r0 + i + if row < end: + xv = bf16x8(sum2(owns[i], staged[i])) + rv = bf16x8(rvs[i]) + s = [xv[n] + rv[n] for n in range_constexpr(8)] + resr = buffer_ops.create_buffer_resource_from_addr( + arg_res + fx.Int64(row) * nbytes, num_records_bytes=nbytes) + res = [pk_bf16(s[2 * n], s[2 * n + 1]) for n in range_constexpr(4)] + buffer_ops.buffer_store(Vec.from_elements(res, fx.Int32), resr, off) + acc = wave_sum_lane63(seq_sum_sq(s)) + if lane == 63: + part[wave] = acc + fx.gpu.barrier() + rstd = rsqrt_rn(_block_sum([part[n] for n in range_constexpr(NW)]), + i32_eps.bitcast(fx.Float32)) + y = [(s[n] * rstd) * (wv[n] + fx.Float32(1.0)) for n in range_constexpr(8)] + ypack = Vec.from_elements([pk_bf16(y[2 * n], y[2 * n + 1]) for n in range_constexpr(4)], + fx.Int32) + result = buffer_ops.create_buffer_resource_from_addr(slot_addr(arg_mine, row, es[i]), + num_records_bytes=nbytes) + if const_expr(quantized): + # the standalone quantizer reads the bf16-rounded normed value + sb, rescale = _e8m0(bf16x8(ypack)) + codes = _codes(bf16x8(ypack), rescale) # 2 dwords, column order + sbyte = fx.Int32(sb.to(fx.Int32)) + store_q(codes, row) + if (tx % 4) == 0: + store_s(sbyte, row, tx // 4) + if const_expr(owner): + # the staged result: 6144 code bytes, then 192 scale bytes + buffer_ops.buffer_store(Vec.from_elements(codes, fx.Int32), result, tx * 2, + cache_modifier=SYS) + buffer_ops.buffer_store(sbyte.to(fx.Int8), result, H + tx // 4, + mask=(tx % 4) == 0, offset_is_bytes=True, + cache_modifier=SYS) + else: + store_out(ypack, row) + if const_expr(owner): + buffer_ops.buffer_store(ypack, result, off, cache_modifier=SYS) + fx.gpu.barrier() + if const_expr(owner): + fx.rocdl.s_waitcnt(vmcnt=0) + if lane == 0: + flag_store(flag_addr(arg_peer, r0, RESULT_FLAG + wave), count(1, r0)) + advance((1,) if owner else (1, 2), b - nstage, nred) + else: + for k in range(b - nstage - nred, nb, nrecv): + r0, end, c = batch(k) + if c != me: + if lane == 0: + spin(flag_addr(arg_mine, r0, RESULT_FLAG + wave), count(2, r0)) + for i in range_constexpr(B): + row = r0 + i + ok = row < end + src = buffer_ops.create_buffer_resource_from_addr(slot_addr(arg_peer, row, count(2, row)), + num_records_bytes=nbytes) + if const_expr(quantized): + cv = Vec(buffer_ops.buffer_load(src, tx * 2, vec_width=2, dtype=fx.Int32, mask=ok, + cache_modifier=SYS)) + # lane 16 l copies the scale bytes of groups 4 l .. 4 l + 3 (its wave wrote them) + sw = fx.Int32(buffer_ops.buffer_load(src, H // 4 + tx // 16, vec_width=1, dtype=fx.Int32, + mask=ok & ((tx % 16) == 0), cache_modifier=SYS)) + if ok: + store_q([fx.Int32(cv[0]), fx.Int32(cv[1])], row) + if (tx % 16) == 0: + for j in range_constexpr(4): + store_s((sw >> (8 * j)) & 0xFF, row, tx // 4 + j) + else: + v = Vec(buffer_ops.buffer_load(src, off, vec_width=4, dtype=fx.Int32, mask=ok, + cache_modifier=SYS)) + if ok: + store_out(v, row) + advance((2,), b - nstage - nred, nrecv) + + @flyc.jit + def launch_allreduce_gemma_bf16_mxfp8_ranks2( + i32_m: fx.Int32, i32_me: fx.Int32, i32_grid: fx.Int32, i32_sr: fx.Int32, arg_x: fx.Int64, + arg_mine: fx.Int64, arg_peer: fx.Int64, arg_cnt: fx.Int64, arg_r: fx.Int64, arg_w: fx.Int64, + arg_q: fx.Int64, arg_s: fx.Int64, arg_res: fx.Int64, i32_eps: fx.Int32, stream: fx.Stream, + ): + kernel(i32_m, i32_me, i32_grid, i32_sr, arg_x, arg_mine, arg_peer, arg_cnt, arg_r, arg_w, arg_q, arg_s, + arg_res, i32_eps).launch( + grid=(fx.Int64((i32_grid & 0x3FF) + ((i32_grid >> 10) & 0x3FF) + (i32_grid >> 20)), 1, 1), + block=(NT, 1, 1), stream=stream, + ) + + launch_allreduce_gemma_bf16_mxfp8_ranks2.kernel_name = name + launch_allreduce_gemma_bf16_mxfp8_ranks2.compile_hints = PRELOAD_HINTS + return launch_allreduce_gemma_bf16_mxfp8_ranks2 diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/norm_math.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/norm_math.py new file mode 100644 index 0000000..7a281a1 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/norm_math.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared pieces of the Gemma add + RMSNorm kernels: bf16 packing and the sum of +squares in the reduction order of Triton's ``tl.sum`` (gfx950, wave64).""" + +import flydsl.expr as fx +from flydsl._mlir.dialects import arith as _arith +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr.typing import T + +H = 6144 + +# kernel arguments preloaded into SGPRs (as Triton does), not s_load'ed first +PRELOAD_HINTS = { + "llvm_options": { + "amdgpu-kernarg-preload": True, + "amdgpu-kernarg-preload-count": 16, + } +} + + +def bf16x8(v): + """8 fp32 of 4 dwords of bf16 (low half first).""" + out = [] + for i in range(4): + w = fx.Int32(v[i]) + out += [(w << 16).bitcast(fx.Float32), (w & -65536).bitcast(fx.Float32)] + return out + + +def pk_bf16(a, b): + """Two fp32 -> two bf16 in a dword (v_cvt_pk_bf16_f32, RNE), ``a`` low.""" + return fx.Int32( + fx.rocdl.cvt_pk_bf16_f32( + fx.as_ir_value(fx.Float32(a)), fx.as_ir_value(fx.Float32(b)) + ) + ) + + +def _dpp_f(v, ctrl, row_mask=0xF, old=None): + """v_mov_b32_dpp of an fp32; source lanes outside the row read 0 (bound_ctrl), + rows ``row_mask`` leaves out keep ``old``.""" + o = _llvm.mlir_undef(T.i32) if old is None else fx.as_ir_value(_bits(old)) + r = _rocdl_ops.update_dpp( + T.i32, o, fx.as_ir_value(_bits(v)), ctrl, row_mask, 0xF, True + ) + return fx.Int32(r).bitcast(fx.Float32) + + +def _bits(x): + return fx.Float32(x).bitcast(fx.Int32) + + +def maxnum(a, b): + return fx.Float32(_arith.MaxNumFOp(fx.as_ir_value(a), fx.as_ir_value(b)).result) + + +def minnum(a, b): + return fx.Float32(_arith.MinNumFOp(fx.as_ir_value(a), fx.as_ir_value(b)).result) + + +def max_dpp(a, ctrl): + """maxnum(a, DPP lane of a): the group amax steps (quad_perm).""" + b = fx.Int32( + _rocdl_ops.update_dpp( + T.i32, + _llvm.mlir_undef(T.i32), + fx.as_ir_value(_bits(a)), + ctrl, + 0xF, + 0xF, + True, + ) + ).bitcast(fx.Float32) + return maxnum(a, b) + + +def swap_xor1(words): + """Each i32 of ``words`` from lane l ^ 1 (quad_perm [1, 0, 3, 2]).""" + return [ + fx.Int32( + _rocdl_ops.update_dpp( + T.i32, _llvm.mlir_undef(T.i32), fx.as_ir_value(w), 0xB1, 0xF, 0xF, True + ) + ) + for w in words + ] + + +def seq_sum_sq(vals): + """((v0^2 + v1^2) + v2^2) + ...: each square rounded, no FMA (Triton's per-thread + accumulation).""" + sq = [v * v for v in vals] + acc = sq[0] + sq[1] + for q in sq[2:]: + acc = acc + q + return acc + + +def wave_sum_lane63(acc): + """Triton's wave reduction: row_shr 8/4/2/1 adds (lane 15 of each row holds the + row sum), row_bcast 15 into rows 1 and 3, row_bcast 31; the result is valid in + lane 63: (R0 + R1) + (R2 + R3).""" + for sh in (8, 4, 2, 1): + acc = acc + _dpp_f(acc, 0x110 + sh) + acc = _dpp_f(acc, 0x142, row_mask=0xA, old=acc) + acc + return acc + _dpp_f(acc, 0x143) + + +def rsqrt_rn(ss, eps): + """1 / v_sqrt_f32(ss / 6144 + eps), both divisions correctly rounded (Triton's + ``1.0 / tl.sqrt(var + eps)`` with ``var = sum / n_cols``).""" + var = ss / fx.Float32(float(H)) + sq = fx.Float32(_rocdl_ops.sqrt(T.f32, fx.as_ir_value(var + eps))) + return fx.Float32(1.0) / sq + + +def xcd_row(bid, m): + """(row, live) of workgroup ``bid``: workgroup b runs on XCD b % 8, and XCD x + takes rows [x * (m // 8) + min(x, m % 8), ...), a contiguous run. Grid: + ``xcd_grid(m)``.""" + x, k = bid % 8, bid // 8 + q8, r8 = m // 8, m % 8 + extra = x < r8 + row = x * q8 + extra.select(x, r8) + k + return row, k < q8 + extra.select(fx.Int32(1), fx.Int32(0)) + + +def xcd_grid(m): + return (m + 7) // 8 * 8 + +def _block_sum(w): + """Triton's fold of 16 wave sums (lane 0 of xor 8, 4, 2, 1), waves 12-15 = +0.""" + return (((w[0] + w[8]) + w[4]) + ((w[2] + w[10]) + w[6])) + ( + ((w[1] + w[9]) + w[5]) + ((w[3] + w[11]) + w[7]) + ) diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/quant_math.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/quant_math.py new file mode 100644 index 0000000..a0ad6a9 --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/quant_math.py @@ -0,0 +1,89 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The bf16-row MXFP8 scale rule and layouts; copied unchanged from the norm family.""" +import flydsl.expr as fx +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl._mlir.dialects import rocdl as _rocdl_ops +from flydsl.expr.typing import T + +from .norm_math import H, max_dpp, maxnum, minnum + +SCALE_K1 = H // 256 # AITER scale layout: 256-column blocks per row +TINY = 0x00800000 # fp32 tiny, the amax floor +HWREG_MODE_FP16_OVFL = 1 | (23 << 6) # hwreg(HW_REG_MODE, 23, 1) + + +def _f32(bits): + return fx.Int32(bits).bitcast(fx.Float32) + + +def _set_fp16_ovfl(): + """MODE.fp16_ovfl = 1, which Triton sets before its fp8 conversions.""" + _llvm.call_intrinsic( + None, + "llvm.amdgcn.s.setreg", + [fx.Int32(HWREG_MODE_FP16_OVFL).ir_value(), fx.Int32(1).ir_value()], + [], + [], + ) + + +def _cvt_pk_fp8(old, a, b, hi: bool): + """v_cvt_scalef32_pk_fp8_f32 (scale 1): a, b -> 2 e4m3 in the low / high half.""" + v2i16 = _ir.VectorType.get([2], _ir.IntegerType.get_signless(16)) + res = _llvm.call_intrinsic( + v2i16, + "llvm.amdgcn.cvt.scalef32.pk.fp8.f32", + [ + _llvm.bitcast(v2i16, fx.as_ir_value(old)), + fx.as_ir_value(a), + fx.as_ir_value(b), + fx.as_ir_value(fx.Float32(1.0)), + fx.Boolean(hi).ir_value(), + ], + [], + [], + ) + return fx.Int32(_llvm.bitcast(T.i32, res)) + + +def _e8m0(y): + """(scale byte as fp32, 2^(127 - byte)) of a 32-column group whose 4 lanes hold + ``y``: Triton's amax / log2 / ceil / clamp sequence.""" + a = fx.math.absf(y[0]) + for v in y[1:]: + a = maxnum(a, fx.math.absf(v)) + a = max_dpp(a, 0x4E) # quad_perm [2, 3, 0, 1] + a = maxnum(max_dpp(a, 0xB1), _f32(TINY)) # quad_perm [1, 0, 3, 2] + d = a / fx.Float32(448.0) + small = d < _f32(TINY) + lg = fx.Float32( + _rocdl_ops.log(T.f32, fx.as_ir_value(small.select(d * fx.Float32(2.0**32), d))) + ) - small.select(fx.Float32(32.0), fx.Float32(0.0)) + sb = minnum( + maxnum(fx.math.ceil(lg) + fx.Float32(127.0), fx.Float32(0.0)), + fx.Float32(254.0), + ) + rescale = fx.Float32(_rocdl_ops.exp2(T.f32, fx.as_ir_value(fx.Float32(127.0) - sb))) + return sb, rescale + + +def _codes(y, rescale): + """8 e4m3 codes of y * rescale as 2 dwords (column order, low byte first).""" + q = [v * rescale for v in y] + lo = _cvt_pk_fp8(_cvt_pk_fp8(fx.Int32(0), q[0], q[1], False), q[2], q[3], True) + hi = _cvt_pk_fp8(_cvt_pk_fp8(fx.Int32(0), q[4], q[5], False), q[6], q[7], True) + return [lo, hi] + + +def _scale_off(m, g): + """Byte of (row, group) in AITER's shuffled scale layout.""" + return ( + ((((m // 32) * SCALE_K1 + g // 8) * 4 + g % 4) * 16 + m % 16) * 2 + (g % 8) // 4 + ) * 2 + (m % 32) // 16 + + +def _code_off(m, p): + """Byte of (row, 16-column piece) in the (16, 16) code layout.""" + return (m // 16) * (16 * H) + (p // 4) * 1024 + (p % 4) * 256 + (m % 16) * 16 diff --git a/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/reference.py b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/reference.py new file mode 100644 index 0000000..9d0b6da --- /dev/null +++ b/amd_kernels/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/reference.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Contract for two-rank bf16 all-reduce, residual add, Gemma norm and optional MXFP8. + +The reduction is ``bf16(ftz(ftz(p0) + ftz(p1)))`` in fp32 at every row: each operand and the sum flush denormals to +a zero of their sign, one bf16 rounding. With two operands the sum is commutative, so it equals AITER's one-stage and +two-stage all-reduce at two ranks and does not depend on the row's owner. Residual addition is fp32 and writes a bf16 +residual. The norm uses the unrounded fp32 addition, weight offset one and epsilon 1e-6. Its bf16 output must have +per-row cosine >= 0.99999 against the fp64 norm below. MXFP8 always quantizes the bf16-rounded norm output, in blocks +of 32, using the family reference's scale rule. Codes are row-major or (16, 16) tiled; scales use AITER's shuffled +layout. Padding rows are unwritten. Determinism is bitwise. +""" +import torch + + +def reduce_reference(parts): + assert len(parts) == 2 + + def ftz(v): + return torch.where(v.abs() < 2.0**-126, v * 0.0, v) + + return ftz(ftz(parts[0].float()) + ftz(parts[1].float())).bfloat16() + + +def allreduce_norm_reference(parts, residual, weight, eps=1e-6): + """Return the fp64 normalized value and exact bf16 residual of the reduction.""" + return add_rmsnorm_reference(reduce_reference(parts), residual, weight, eps) + + +def add_rmsnorm_reference( + x: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, eps: float +) -> tuple[torch.Tensor, torch.Tensor]: + """(y in fp64, the residual out bf16) of ``x + residual``; ``bf16(y)`` is the bf16 output.""" + s = x.float() + residual.float() + s64 = s.double() + eps32 = torch.tensor(eps, dtype=torch.float32).item() + ss = (s64 * s64).sum(-1, keepdim=True) + ss = torch.where(ss > torch.finfo(torch.float32).max, torch.full_like(ss, float("inf")), ss) + y = s64 * torch.rsqrt(ss / 6144 + eps32) * (weight.double() + 1.0) + return y, s.bfloat16() diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/README.md b/amd_kernels/norm/gemma_add_rmsnorm_h6144/README.md index 141a516..a5b6a19 100644 --- a/amd_kernels/norm/gemma_add_rmsnorm_h6144/README.md +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/README.md @@ -1,15 +1,16 @@ -# Gemma add + RMSNorm, hidden 6144, and the TP4 all-reduce fused in front +# Gemma add + RMSNorm, hidden 6144, and the TP4 / TP2 all-reduce fused in front ## What it does Residual add + Gemma RMSNorm of MiniMax-M3.1-Flash in FlyDSL (bf16 x, residual, weight as w + 1; fp32 eps; hidden 6144). `add_rmsnorm` (one GPU): bf16 or MXFP8 out (AITER-layout E8M0 scales; codes row-major or in the decode GEMM's -(16, 16) tiles). `allreduce_add_rmsnorm`: the TP4 all-reduce in front, for the attention tail (one bf16 o_proj -partial per rank) and the MoE tail (which first sums each rank's 4 weighted slots + shared in fp32). Contract -(`reference.py`): chunk c = rows `[c q, (c + 1) q)`, q = ceil(rows / 4), summed over ranks c..c+3 in fp32 with -denormals flushed, one bf16 rounding (AITER's two-stage bits); `s = x + residual` in fp32, residual out `bf16(s)`; -`y = s * rstd * (w + 1)` in Triton's square-sum order, within 1 bf16 ulp of fp64; MXFP8 of the fp32 y at every M. -Fused = one-GPU kernels on `allreduce_rows_reference`. +(16, 16) tiles). `allreduce_add_rmsnorm` (4 ranks) and `allreduce_add_rmsnorm_ranks2` (2 ranks): the TP all-reduce in +front, for the attention tail (one bf16 o_proj partial per rank) and the MoE tail (which first sums each rank's 4 +weighted slots + shared in fp32). Contract (`reference.py`): 4 ranks: chunk c = rows `[c q, (c + 1) q)`, q = +ceil(rows / 4), summed over ranks c..c+3 in fp32 with denormals flushed, one bf16 rounding (AITER's two-stage bits); +2 ranks: `bf16(ftz(ftz(x0) + ftz(x1)))` at every row (AITER's one- and two-stage bits); `s = x + residual` in fp32, +residual out `bf16(s)`; `y = s * rstd * (w + 1)` in Triton's square-sum order, within 1 bf16 ulp of fp64; MXFP8 of +the fp32 y at every M. Fused = one-GPU kernels on `allreduce_rows_reference`. ## Entry points @@ -27,27 +28,33 @@ Fused = one-GPU kernels on `allreduce_rows_reference`. - `allreduce_add_rmsnorm.unsupported_reason`: why this process cannot run the communicators (None: it can). - `allreduce_add_rmsnorm.DecoderArNorm`: both communicators (+ one layer's MoE modes): `add_rmsnorm`, a layer's tail `post_attention` (norm + FFN -> `FfnPartial`) and `pre_attention` (the next norm); `DenseMlp`: a dense layer's MLP. +- `allreduce_add_rmsnorm_ranks2.*`: the same names at 2 ranks (with the i1536 MoE and the i6144 ranks2 dense MLP). -Both communicators: 1-4095 rows (larger prefills take the sequence-parallel hand-offs), eager and graphs, one stream -each, `warm_up` on every rank before serving, the same calls in order on all ranks, IPC buffers for the process -lifetime. Tiers by row count: one-shot (each rank sums all), two-stage one row per workgroup, streamed R rows per -workgroup, some exchanges in a lossless 12-bit code; `allreduce_add_rmsnorm/staged_host.py` and `host.py` pick them. +TP4: 1-4095 rows (larger prefills take the sequence-parallel hand-offs); one-shot, two-stage and streamed tiers, some in +a lossless 12-bit code (`allreduce_add_rmsnorm/staged_host.py`, `host.py`). TP2: to 65536 rows (3840-row launches above +4095), one-shot, from 49 rows in the 12-bit code (`allreduce_add_rmsnorm_ranks2/host.py`). All: eager and graphs, one +stream each, `warm_up` on every rank first, the same calls in order on all ranks, IPC buffers for the process life. ## Performance -Graphs to 512 rows, eager above; MAX over ranks, median us; baseline = AITER's custom all-reduce (RCCL above its -cap), then the norm. Coded tiers (real layer-39 producer + tail, against the same stream uncoded): attention 2048 / -3253 rows 219.1 / 338.2 us, -16..-18%; MoE 1024 rows 340.9 us, -4%; the coded reduce-scatter -3..-65 us more. +Graphs (TP4: to 512 rows), MAX over ranks, median us; baseline = AITER's custom all-reduce (RCCL above its cap; TP2 +bf16 to 80 rows its one-stage fused all-reduce + norm), then the norm; TP2 inputs beyond the 256 MB MALL. -| M | bf16: baseline | entry | MXFP8: baseline | entry | MoE tail: baseline | entry | -|---:|---:|---:|---:|---:|---:|---:| -| 8 | 7.94 | 5.99 | 10.83 | 7.04 | 15.43 | 8.09 | -| 1024 | 128.91 | 119.45 | 129.64 | 118.60 | 148.13 | 124.03 | +| ranks | M | bf16: baseline | entry | MXFP8: baseline | entry | MoE tail: baseline | entry | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 | 8 | 7.94 | 5.99 | 10.83 | 7.04 | 15.43 | 8.09 | +| 4 | 1024 | 128.91 | 119.45 | 129.64 | 118.60 | 148.13 | 124.03 | +| 2 | 8 | 7.69 | 6.34 | 10.05 | 7.33 | 9.93 | 8.39 | +| 2 | 64 | 19.97 | 17.52 | 23.54 | 21.29 | 23.30 | 22.50 | +| 2 | 256 | 66.36 | 53.80 | 66.99 | 58.44 | 67.16 | 63.24 | +| 2 | 1024 | 240.00 | 201.75 | 239.40 | 198.80 | 240.03 | 201.44 | +| 2 | 4095 | 922.28 | 749.88 | 917.39 | 738.30 | 917.06 | 771.50 | ## Tests ```bash -python3 -m pytest tests/norm/gemma_add_rmsnorm_h6144 -q # all-reduce tests: 4 GPUs +python3 -m pytest tests/norm/gemma_add_rmsnorm_h6144 -q # all-reduce tests: 4 GPUs (test_ranks2_*: 2 GPUs) python3 benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_rows.py --kinds bf16,mxfp8 --rounds 7 8 128 1024 4095 python3 benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_slots4_ar_add_rmsnorm.py result.json # also benchmark_add_rmsnorm.py +python3 benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py # 2 GPUs, all three tails ``` diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/__init__.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/__init__.py index 240bbd1..b36104e 100644 --- a/amd_kernels/norm/gemma_add_rmsnorm_h6144/__init__.py +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """Residual add + Gemma RMSNorm at hidden 6144 on gfx950, FlyDSL: ``add_rmsnorm`` (one GPU; bf16 or MXFP8 out, the -split-K GEMM's slab reduce optionally in front) and ``allreduce_add_rmsnorm`` (the TP4 all-reduce in front, eager and -graphs). The torch contract is ``reference.py``.""" +split-K GEMM's slab reduce optionally in front), ``allreduce_add_rmsnorm`` (the TP4 all-reduce in front) and +``allreduce_add_rmsnorm_ranks2`` (the TP2 one), eager and graphs. The torch contract is ``reference.py``.""" diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/__init__.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/__init__.py new file mode 100644 index 0000000..2c28fbe --- /dev/null +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/__init__.py @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-rank all-reduce fused in front of the residual add + Gemma RMSNorm (bf16 or MXFP8 out), FlyDSL (gfx950): +``staged_comm`` (the attention tail), ``slots4_comm`` (the MoE combine tail) and ``DecoderArNorm`` (both, and a decoder +layer's FFN between them); the API of ``allreduce_add_rmsnorm`` at two ranks.""" + +import dataclasses +import functools + +import torch + + +def _expandable_segments() -> bool: + """PyTorch's allocator hands out expandable segments (no ``hipIpcGetMemHandle`` on them).""" + try: + from aiter.dist.device_communicators.custom_all_reduce import _expandable_segments_enabled + + return _expandable_segments_enabled() + except ImportError: + import os + + conf = ",".join( + os.environ.get(v, "") for v in ("PYTORCH_ALLOC_CONF", "PYTORCH_CUDA_ALLOC_CONF", "PYTORCH_HIP_ALLOC_CONF") + ) + return "expandable_segments:true" in conf.replace(" ", "").lower() + + +@functools.cache +def unsupported_reason(world_size: int, hidden_size: int = 6144) -> str | None: + """None when this process can run the communicators, else why not: 2 ranks, hidden 6144, gfx950 with FlyDSL.""" + from .. import add_rmsnorm + + if world_size != 2: + return f"needs 2 ranks (got {world_size})" + if hidden_size != 6144: + return f"needs hidden size 6144 (got {hidden_size})" + if not add_rmsnorm.available(): + return "needs gfx950 with FlyDSL" + if _expandable_segments(): + return "not with PyTorch expandable segments (the staging buffers are shared by IPC handle)" + return None + + +def staged_comm(rank: int, cpu_group, device: torch.device, buffers=None): + """The attention tail's communicator (``host.StagedArNormComm``: one bf16 partial per rank, bf16 or MXFP8 out; a + collective on the CPU group, once per rank): to ``host.LIMIT_ROWS`` rows, eager or captured. Call ``warm_up`` on + every rank before serving.""" + from .host import StagedArNormComm + + return StagedArNormComm(rank, cpu_group, device, buffers) + + +def slots4_comm(rank: int, cpu_group, device: torch.device, buffers=None): + """The MoE combine tail's communicator (``host.SlotsArNormComm``; a collective on the CPU group, once per rank): + eager and graph calls alike; call its ``warm_up`` on every rank before serving.""" + from .host import SlotsArNormComm + + return SlotsArNormComm(rank, cpu_group, device, buffers) + + +@dataclasses.dataclass +class DenseMlp: + """A dense layer's MLP for ``DecoderArNorm.post_attention``: preshuffled MXFP8 gate_up [12288, 6144] and down + [6144, 6144] with their scales, and SwiGLU-OAI's alpha, beta, limit.""" + + gate_up: torch.Tensor + gate_up_scale: torch.Tensor + down: torch.Tensor + down_scale: torch.Tensor + alpha: float + beta: float + limit: float + + def __post_init__(self): + if not dense_mlp_supported(self.gate_up, self.down): + raise ValueError(f"dense MLP weights {tuple(self.gate_up.shape)} / {tuple(self.down.shape)} not supported") + + +def _dense_package(): + """``decoder.dense_mxfp8_h6144_i6144_ranks2``, or None where it is not installed (``post_attention`` then returns + gate_up's codes for the caller's MLP).""" + try: + from amd_kernels.decoder import dense_mxfp8_h6144_i6144_ranks2 + except ImportError: + return None + return dense_mxfp8_h6144_i6144_ranks2 + + +def dense_mlp_supported(gate_up: torch.Tensor, down: torch.Tensor) -> bool: + """Whether ``post_attention`` takes a dense layer's MLP on these weights.""" + D = _dense_package() + if D is not None: + return D.mlp_supported(gate_up, down) + return (tuple(gate_up.shape) == (12288, 6144) and tuple(down.shape) == (6144, 6144) + and gate_up.dtype == down.dtype == torch.float8_e4m3fn) + + +@dataclasses.dataclass +class FfnPartial: + """A decoder layer's FFN output before its TP all-reduce, which the next ``pre_attention`` runs: the MoE's four + weighted slots [M, 4, 6144] + shared expert [M, 6144], or one bf16 partial [M, 6144].""" + + slots: torch.Tensor | None = None + shared: torch.Tensor | None = None + partial: torch.Tensor | None = None + + +class DecoderArNorm: + """Both two-rank norm tails of a decoder layer on one stream (sharing one buffer pair): the staged + communicator (one bf16 partial per rank) and the slots4 one (four weighted MoE slots + the shared expert), built + and warmed up on every rank (a collective on the CPU group); with ``moe`` (one layer's ``MoeWeights`` of + ``moe.mxfp4_h6144_i1536_e128_top4``) also the MoE modes ``post_attention`` runs.""" + + def __init__(self, rank: int, cpu_group, device: torch.device, weight: torch.Tensor, eps: float, *, moe=None): + from .host import SharedBuffers + + buffers = SharedBuffers(rank, cpu_group, device) + self.staged = staged_comm(rank, cpu_group, device, buffers) + self.slots = slots4_comm(rank, cpu_group, device, buffers) + self.staged.warm_up(weight, eps) + self.slots.warm_up(weight, eps) + if moe is not None: + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import warm_up_mxfp4_moe + + warm_up_mxfp4_moe(moe, partials=True) + torch.accelerator.synchronize(device) + + def capture(self): + """Enter around every graph capture of the group (the four-rank API; nothing to register).""" + return self.staged.capture() + + def supports(self, rows: int) -> bool: + return self.staged.supports(rows) and self.slots.supports(rows) + + def accepts(self, x: torch.Tensor, residual: torch.Tensor) -> bool: + return self.staged.accepts(x, residual) + + def add_rmsnorm(self, residual, weight, eps, *, partial=None, slots=None, shared=None, gemm_weight=None): + """all-reduce(``partial``, or ``slots`` summed + ``shared``) + residual -> Gemma RMSNorm: (out, residual). + + ``out`` is bf16 without ``gemm_weight``, else (codes, scales) in the layout that K 6144 GEMM reads at this row + count.""" + if slots is not None: + assert gemm_weight is not None, "the slots4 tail writes MXFP8 only" + q, s, residual = self.slots.slots4_ar_add_rmsnorm_mxfp8( + slots, shared, residual, weight, eps, gemm_weight=gemm_weight) + return (q, s), residual + if gemm_weight is None: + return self.staged.ar_add_rmsnorm(partial, residual, weight, eps) + q, s, residual = self.staged.ar_add_rmsnorm_mxfp8(partial, residual, weight, eps, gemm_weight=gemm_weight) + return (q, s), residual + + def post_attention(self, o_partial, residual, weight, eps, ffn, *, final=False): + """all-reduce(``o_partial``) + residual -> Gemma RMSNorm -> ``ffn`` on this rank (``MoeWeights`` with its + shared expert, or ``DenseMlp``): (``FfnPartial``, residual) for the next ``pre_attention`` (``final``: the one + without a GEMM, bf16 out). None outside the communicators' rows; above the MLP's rows or without the dense MLP + package, (gate_up's codes, scales) for the caller's MLP.""" + rows = o_partial.shape[0] + if not (self.accepts(o_partial, residual) and self.supports(rows)): + return None + if isinstance(ffn, DenseMlp): + D = _dense_package() + (q, s), residual = self.add_rmsnorm(residual, weight, eps, partial=o_partial, gemm_weight=ffn.gate_up) + if D is None or rows > D.MAX_ROWS: + return (q, s), residual + out = D.mxfp8_swiglu_oai_mlp(q, s, rows, ffn.gate_up, ffn.gate_up_scale, ffn.down, ffn.down_scale, + ffn.alpha, ffn.beta, ffn.limit, form="fp32") + return FfnPartial(partial=out), residual + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + + x, residual = self.add_rmsnorm(residual, weight, eps, partial=o_partial) + if final: # the slots4 tail writes MXFP8 only + return FfnPartial(partial=mxfp4_moe(x, ffn)), residual + slots, shared = mxfp4_moe(x, ffn, return_partials=True) + return FfnPartial(slots=slots, shared=shared), residual + + def pre_attention(self, ffn: FfnPartial, residual, weight, eps, gemm_weight=None): + """all-reduce(``ffn``) + residual -> Gemma RMSNorm: ``add_rmsnorm``'s (out, residual).""" + return self.add_rmsnorm(residual, weight, eps, partial=ffn.partial, slots=ffn.slots, shared=ffn.shared, + gemm_weight=gemm_weight) diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/common.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/common.py new file mode 100644 index 0000000..ad54605 --- /dev/null +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/common.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The one-shot protocol of the two-rank all-reduce + add + Gemma RMSNorm kernels. + +Every rank owns one uncached buffer mapped by its peer: its own staged partial rows (two copies, by the row's call +count parity), one 128-byte flag line per row, then the raw slots of coded rows. A stager wave writes its columns of a +batch of rows into its own copy, drains its stores and stores the batch's call count into flag (first row, wave) on the +peer; the reducer wave with the same columns polls that word in its own buffer and reads the rows from the peer's +buffer. Local writes keep the code's byte and short header stores off the link, which then carries whole-line reads. +A rank starts call n + 1 only after its call n saw the peer's call n flags, so the peer has finished reading call +n - 1: two copies suffice. + +Each rank keeps two per-row call-count arrays (stagers, reducers), advanced once per call for every row the call +touches; they are equal on both ranks because the ranks make the same calls in the same order. Separate arrays: a +reducer that finishes its batch must not change the count a stager of the same rows reads. + +The rank sum is fp32 with denormal operands and sums flushed to a zero of their sign, one bf16 rounding. With two +operands it is commutative: AITER's one-stage and two-stage all-reduce give the same bits at two ranks. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..add_rmsnorm.add_rmsnorm_mxfp8 import NT as CODE_NT +from ..add_rmsnorm.add_rmsnorm_mxfp8 import _code_off, _scale_off +from ..add_rmsnorm.common import H, bf16x8, pk_bf16, swap_xor1 + +SYS = 0x11 # sc0 | sc1: system-coherent loads, write-through stores +ROWB = H * 2 +MAX_ROWS = 4096 +DATA_BYTES = 2 * MAX_ROWS * ROWB +LINE = 32 # flag words per row (a 128-byte line) +FLAG_OFF = DATA_BYTES +FLAG_BYTES = MAX_ROWS * LINE * 4 +RAW_OFF = FLAG_OFF + FLAG_BYTES # the coded rows' raw virtual waves (``lossless_bf16``), one slot per staged row +BUF_BYTES = RAW_OFF + DATA_BYTES +COUNT_PAD = 64 # count words past MAX_ROWS: masked rows of a last batch read them + + +def _gptr(addr): + return fx.inttoptr( + fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + fx.Int64(addr), + ) + + +def flag_store(addr, v): + """An atomic exchange at system scope (write-through).""" + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.xchg, + _gptr(addr).llvm_ptr, + fx.Int32(v).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ) + + +def _flag_read(addr): + """An atomic add of 0 at system scope (a spin on it is never hoisted).""" + return fx.Int32( + _llvm.AtomicRMWOp( + _llvm.AtomicBinOp.add, + _gptr(addr).llvm_ptr, + fx.Int32(0).ir_value(), + _llvm.AtomicOrdering.monotonic, + syncscope="", + alignment=4, + ).result + ) + + +@flyc.jit +def spin(a, e): + f = _flag_read(a) + while (f - e) < 0: # signed difference: the counts may wrap + f = _flag_read(a) + + +def flag_addr(base, row, wave): + return base + fx.Int64(FLAG_OFF) + fx.Int64((row * LINE + wave) * 4) + + +def data_addr(base, row, e, raw=False): + """Row ``row``'s staged copy of call count ``e`` in ``base``'s buffer (``raw``: its raw-wave slot).""" + return base + fx.Int64(RAW_OFF if raw else 0) + (fx.Int64(e & 1) * MAX_ROWS + fx.Int64(row)) * ROWB + + +def _ftz(u): + """fp32 bits with a denormal flushed to a zero of its sign.""" + return ((u & 0x7F800000) == 0).select(u & fx.Int32(-0x80000000), u) + + +def _as_f32(i): + return fx.Int32(i).bitcast(fx.Float32) + + +def sum2(a, b): + """bf16(ftz(ftz(a) + ftz(b))) per element of two 16 B packs (fp32, AITER's denormal flush).""" + out = [] + for i in range_constexpr(4): + wa, wb = fx.Int32(a[i]), fx.Int32(b[i]) + lo = _ftz((_as_f32(_ftz(wa << 16)) + _as_f32(_ftz(wb << 16))).bitcast(fx.Int32)) + hi = _ftz((_as_f32(_ftz(wa & fx.Int32(-65536))) + _as_f32(_ftz(wb & fx.Int32(-65536)))).bitcast(fx.Int32)) + out.append(pk_bf16(_as_f32(lo), _as_f32(hi))) + return Vec.from_elements(out, fx.Int32) + + +def local_sum(slots, shared): + """bf16((((s0 + s1) + s2) + s3) + shared) of 16 B packs, fp32 adds.""" + vs = [bf16x8(p) for p in slots] + [bf16x8(shared)] + acc = vs[0] + for v in vs[1:]: + acc = [acc[n] + v[n] for n in range_constexpr(8)] + return Vec.from_elements([pk_bf16(acc[2 * n], acc[2 * n + 1]) for n in range_constexpr(4)], fx.Int32) + + +def store_tiled(tx, rw, codes, sbs, has1, ok, qr, sr): + """One row's codes in the (16, 16) tiles and its scale bytes (AITER layout), 8-wave lane map.""" + lane = tx % 64 + odd = (lane & 1) == 1 + p = tx // 2 + odd.select(fx.Int32(256), fx.Int32(0)) + q4 = lane % 4 + g = tx // 4 + (q4 == 1).select(fx.Int32(128), fx.Int32(0)) + # lane pairs: the even lane stores chunk 0's 16 columns, the odd chunk 1's + recv = swap_xor1([odd.select(codes[0][n], codes[1][n]) for n in range_constexpr(2)]) + piece = Vec.from_elements( + [ + odd.select(recv[0], codes[0][0]), + odd.select(recv[1], codes[0][1]), + odd.select(codes[1][0], recv[0]), + odd.select(codes[1][1], recv[1]), + ], + fx.Int32, + ) + buffer_ops.buffer_store( + piece, qr, _code_off(rw, p) // 4, mask=(((lane & 1) == 0) | has1) & ok, cache_modifier=CODE_NT + ) + sb = (q4 == 1).select(sbs[1], sbs[0]) + buffer_ops.buffer_store( + sb.to(fx.Int8), sr, _scale_off(rw, g), mask=((q4 == 0) | ((q4 == 1) & has1)) & ok, offset_is_bytes=True + ) + + +def store_rowmajor(tx, rw, codes, sbs, has1, ok, qr, sr): + """One row's row-major codes and its scale bytes (AITER layout), 8-wave lane map.""" + for h in range_constexpr(2): + mask = ok if h == 0 else ok & has1 + buffer_ops.buffer_store( + Vec.from_elements(codes[h], fx.Int32), qr, (rw * H + tx * 8 + h * 4096) // 4, mask=mask, + cache_modifier=CODE_NT, + ) + buffer_ops.buffer_store( + sbs[h].to(fx.Int8), sr, _scale_off(rw, tx // 4 + h * 128), mask=mask & (tx % 4 == 0), + offset_is_bytes=True, + ) diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/host.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/host.py new file mode 100644 index 0000000..bf482d2 --- /dev/null +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/host.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Communicators of the two-rank one-shot all-reduce + add + Gemma RMSNorm kernels: ``SharedBuffers`` (the IPC-mapped +buffer pair and the call counts of one stream), ``StagedArNormComm`` (the attention tail: one bf16 partial per rank) +and ``SlotsArNormComm`` (the MoE tail: four weighted slots + the shared expert per rank).""" + +import contextlib +import functools + +import torch +import torch.distributed as dist + +from amd_kernels.common import hip_runtime as ops +from ..add_rmsnorm.host import _f32_bits, _out_or_empty, _run_compiled, code_tile_rows_for +from . import common as C +from . import oneshot as O + +H = C.H +WORLD = 2 +MAX_ROWS = C.MAX_ROWS - 1 # rows of one launch +CHUNK_ROWS = 3840 # larger calls launch per chunk: a multiple of every code tile (<= 256) and of the 32-row scales +LIMIT_ROWS = 1 << 16 + + +@functools.cache +def _kernel(out: str, slots: bool, rpb: int, coded: bool): + return O.compile_oneshot_ar_add_rmsnorm(out, slots, rpb, coded) + + +def exchange(obj, rank: int, group) -> list: + """Every rank's ``obj`` (broadcasts on the CPU group: all_gather_object fails on gloo under inference mode).""" + ranks = sorted(dist.get_process_group_ranks(group=group)) + out = [[None] for _ in ranks] + out[rank][0] = obj + for i, r in enumerate(ranks): + dist.broadcast_object_list(out[i], src=r, group=group, device="cpu") + return [o[0] for o in out] + + +def config(rows: int, quant: bool, slots: bool) -> tuple[int, int, int, bool]: + """(rows per batch, stagers, reducers, coded) of a call, measured for every output (benchmark_ar_add_rmsnorm_ranks2 + .py --sweep): the lossless code wins from 49-64 rows (48 rows: bf16 coded 14.82 / raw 15.25 us, MXFP8 16.54 / + 16.05).""" + if rows <= 48: + return 1, rows, rows, False + rpb = 1 if rows <= 1024 else 2 if rows <= 2048 else 4 + nb = (rows + rpb - 1) // rpb + nred = 32 if not quant and rows <= 96 else 64 if rows <= 256 else 128 + return rpb, min(nb, 32), min(nb, nred), True + + +def warm_rows(quant: bool, slots: bool) -> list[int]: + """The first row count of every compiled variant (rows per batch, coded) of ``config``.""" + seen, out = set(), [] + for rows in range(1, MAX_ROWS + 1): + rpb, _, _, coded = config(rows, quant, slots) + if (rpb, coded) not in seen: + seen.add((rpb, coded)) + out.append(rows) + return out + + +def _overlaps(a: torch.Tensor, *others: torch.Tensor) -> bool: + lo, hi = a.data_ptr(), a.data_ptr() + a.numel() * a.element_size() + return any(t is not None and t.data_ptr() < hi and lo < t.data_ptr() + t.numel() * t.element_size() + for t in others) + + +class SharedBuffers: + """One uncached buffer per rank (its staged rows and the flags the peer stores), mapped by the peer, and the per-row + call counts: a collective on the 2-rank CPU group, once per rank and stream; the communicators of one stream may + share it (calls in the same order on both ranks).""" + + def __init__(self, rank: int, group, device: torch.device): + if dist.get_world_size(group) != WORLD: + raise ValueError("two ranks required") + device = torch.device(device) + if device.index is None: + device = torch.device("cuda", torch.cuda.current_device()) + self.rank, self.group, self.device = rank, group, device + ptr, handle = ops.allocate_shared_buffer_and_handle(C.BUF_BYTES) + every = exchange(handle, rank, group) + self.mine = ptr + self.peer = ops.open_mem_handle(every[1 - rank]) + self.counts = torch.zeros(2 * O.COUNT_WORDS, dtype=torch.int32, device=device) + torch.cuda.synchronize(device) + dist.barrier(group=group) + + config = None # benchmarks may override ``config`` + + def launch(self, out_kind, slots, rows, sx, sr, x, sh, residual, weight, out, scales, res_out, eps): + """One launch per ``CHUNK_ROWS`` rows above ``MAX_ROWS`` (row-wise op, the chunks' outputs are contiguous).""" + chunk = rows if rows <= MAX_ROWS else CHUNK_ROWS + xrow = (4 * C.ROWB) if slots else sx * 2 + orow = H if out_kind != "bf16" else C.ROWB + for r0 in range(0, rows, chunk): + n = min(chunk, rows - r0) + rpb, nstage, nred, coded = (self.config or config)(n, out_kind != "bf16", slots) + assert 0 < nstage < 1 << 15 and 0 < nred < 1 << 15 + _run_compiled( + _kernel(out_kind, slots, rpb, coded), n, nstage | (nred << 16), sx, sr, x + r0 * xrow, + sh + r0 * C.ROWB if sh else 0, self.mine, self.peer, residual + r0 * sr * 2, weight, out + r0 * orow, + scales + r0 * (H // 32) if scales else 0, res_out + r0 * C.ROWB, self.counts.data_ptr(), + _f32_bits(eps), torch.cuda.current_stream(), + ) + + +def _check_rows(x: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, rows: int): + assert 0 < rows <= LIMIT_ROWS, rows + assert residual.shape == (rows, H) and residual.dtype == torch.bfloat16 + assert residual.stride(1) == 1 and residual.stride(0) % 8 == 0 + assert weight.shape == (H,) and weight.dtype == torch.bfloat16 and weight.is_contiguous() + assert x.device == residual.device == weight.device + + +def _mxfp8_outputs(rows, code_tile_rows, q_out, scale_out, res_out, device): + assert code_tile_rows >= 0 and code_tile_rows % 16 == 0 + tiled = (rows + code_tile_rows - 1) // code_tile_rows * code_tile_rows if code_tile_rows else rows + q = _out_or_empty(q_out, (tiled, H), torch.float8_e4m3fn, device) + scale = _out_or_empty(scale_out, ((tiled + 31) // 32 * 32, H // 32), torch.uint8, device) + res_out = _out_or_empty(res_out, (rows, H), torch.bfloat16, device) + return q, scale, res_out + + +class StagedArNormComm: + """The attention tail at two ranks: all-reduce of one bf16 [rows, H] partial per rank + residual add + Gemma + RMSNorm, bf16 or MXFP8 out; eager and graph calls alike (``capture()`` is a no-op matching the four-rank API). + One communicator per stream; ``buffers``: share another communicator's (the same stream).""" + + def __init__(self, rank: int, group, device: torch.device, buffers: SharedBuffers | None = None): + self.buffers = buffers or SharedBuffers(rank, group, device) + self.rank, self.group, self.device = rank, group, self.buffers.device + + @contextlib.contextmanager + def capture(self): + yield + + def supports(self, rows: int, mxfp8: bool = False) -> bool: + """All-reduce row counts supported for either output, eager and graphs.""" + return 0 < rows <= LIMIT_ROWS + + def accepts(self, x: torch.Tensor, residual: torch.Tensor) -> bool: + """Whether ``ar_add_rmsnorm`` / ``ar_add_rmsnorm_mxfp8`` take these tensors: bf16 [rows, H] with unit column + stride and 16-byte rows.""" + return ( + x.dim() == 2 + and x.shape[1] == H + and self.supports(x.shape[0]) + and residual.shape == x.shape + and x.dtype == residual.dtype == torch.bfloat16 + and x.stride(1) == 1 + and residual.stride(1) == 1 + and x.stride(0) % 8 == 0 + and residual.stride(0) % 8 == 0 + ) + + def _input(self, x, *outputs): + # a reducer may write a row before this rank's stager of it reads it: outputs must not overlap the input + return x.clone() if _overlaps(x, *outputs) else x + + def ar_add_rmsnorm(self, x, residual, weight, eps, out=None, res_out=None): + """``add_rmsnorm.gemma_add_rmsnorm`` of all-reduce(x) + residual: (bf16 out, residual out); every rank passes + its own bf16 [rows, H] partial (``reference.allreduce_rows_reference``).""" + rows = x.shape[0] + assert self.accepts(x, residual), (tuple(x.shape), x.stride(), residual.stride()) + _check_rows(x, residual, weight, rows) + out = _out_or_empty(out, (rows, H), torch.bfloat16, self.device) + res_out = _out_or_empty(res_out, (rows, H), torch.bfloat16, self.device) + x = self._input(x, out, res_out) + self.buffers.launch("bf16", False, rows, x.stride(0), residual.stride(0), x.data_ptr(), 0, + residual.data_ptr(), weight.data_ptr(), out.data_ptr(), 0, res_out.data_ptr(), eps) + return out, res_out + + def ar_add_rmsnorm_mxfp8(self, x, residual, weight, eps, code_tile_rows=None, q_out=None, scale_out=None, + res_out=None, *, gemm_weight=None): + """``add_rmsnorm.gemma_add_rmsnorm_mxfp8`` of all-reduce(x) + residual; every rank passes its own bf16 + [rows, H] partial. Returns (codes in the (16, 16) layout padded to ``code_tile_rows`` when > 0, row-major at + 0, the tiled norm's arithmetic in both; AITER-layout scales; residual out). ``gemm_weight`` instead of + ``code_tile_rows``: the layout that GEMM reads at this row count.""" + rows = x.shape[0] + if gemm_weight is not None: + code_tile_rows = code_tile_rows_for(rows, gemm_weight) + assert code_tile_rows is not None + assert self.accepts(x, residual) + _check_rows(x, residual, weight, rows) + assert rows <= MAX_ROWS or CHUNK_ROWS % max(code_tile_rows, 1) == 0, code_tile_rows + q, scale, res_out = _mxfp8_outputs(rows, code_tile_rows, q_out, scale_out, res_out, self.device) + x = self._input(x, q, scale, res_out) + self.buffers.launch("tiled" if code_tile_rows else "rowmajor", False, rows, x.stride(0), residual.stride(0), + x.data_ptr(), 0, residual.data_ptr(), weight.data_ptr(), q.data_ptr(), scale.data_ptr(), + res_out.data_ptr(), eps) + return q, scale, res_out + + def warm_up(self, weight: torch.Tensor, eps: float, code_tile_rows: int = 16) -> None: + """Compile every variant before serving, both code layouts (a collective: every rank calls it).""" + for rows in sorted(set(warm_rows(False, False)) | set(warm_rows(True, False))): + x = torch.zeros((rows, H), dtype=torch.bfloat16, device=self.device) + r = torch.zeros_like(x) + self.ar_add_rmsnorm(x, r, weight, eps) + for tile in (0, code_tile_rows or 16): + self.ar_add_rmsnorm_mxfp8(x, r, weight, eps, tile) + torch.cuda.synchronize(self.device) + + +class SlotsArNormComm: + """The MoE tail at two ranks: all-reduce of each rank's ``bf16((((s0 + s1) + s2) + s3) + shared)`` + residual add + + Gemma RMSNorm + MXFP8 quant; eager and graph calls alike. One communicator per stream; ``buffers``: share another + communicator's (the same stream).""" + + def __init__(self, rank: int, group, device: torch.device, buffers: SharedBuffers | None = None): + self.buffers = buffers or SharedBuffers(rank, group, device) + self.rank, self.group, self.device = rank, group, self.buffers.device + + def supports(self, rows: int) -> bool: + """Row counts ``slots4_ar_add_rmsnorm_mxfp8`` takes, eager and captured.""" + return 0 < rows <= LIMIT_ROWS + + def slots4_ar_add_rmsnorm_mxfp8(self, slots, shared, residual, weight, eps, code_tile_rows=None, q_out=None, + scale_out=None, res_out=None, *, gemm_weight=None): + """``add_rmsnorm.gemma_add_rmsnorm_mxfp8`` of all-reduce(slot 0 + .. + slot 3 + shared) + residual. Every rank + passes its own contiguous bf16 [rows, 4, H] slots and [rows, H] shared output. Returns (codes in the (16, 16) + layout when ``code_tile_rows`` > 0, row-major at 0; AITER-layout scales; residual out); ``gemm_weight`` instead + of ``code_tile_rows``: the layout that GEMM reads at this row count.""" + rows = slots.shape[0] + if gemm_weight is not None: + code_tile_rows = code_tile_rows_for(rows, gemm_weight) + assert code_tile_rows is not None + assert slots.shape == (rows, 4, H) and slots.dtype == torch.bfloat16 and slots.is_contiguous() + assert shared.shape == (rows, H) and shared.dtype == torch.bfloat16 and shared.is_contiguous() + _check_rows(slots, residual, weight, rows) + assert rows <= MAX_ROWS or CHUNK_ROWS % max(code_tile_rows, 1) == 0, code_tile_rows + q, scale, res_out = _mxfp8_outputs(rows, code_tile_rows, q_out, scale_out, res_out, self.device) + if _overlaps(slots, q, scale, res_out) or _overlaps(shared, q, scale, res_out): + slots, shared = slots.clone(), shared.clone() + self.buffers.launch("tiled" if code_tile_rows else "rowmajor", True, rows, 0, residual.stride(0), + slots.data_ptr(), shared.data_ptr(), residual.data_ptr(), weight.data_ptr(), q.data_ptr(), + scale.data_ptr(), res_out.data_ptr(), eps) + return q, scale, res_out + + def warm_up(self, weight: torch.Tensor, eps: float, code_tile_rows: int = 16) -> None: + """Compile every variant before serving, both code layouts (a collective: every rank calls it).""" + for rows in warm_rows(True, True): + slots = torch.zeros((rows, 4, H), dtype=torch.bfloat16, device=self.device) + row = torch.zeros((rows, H), dtype=torch.bfloat16, device=self.device) + for tile in (0, code_tile_rows or 16): + self.slots4_ar_add_rmsnorm_mxfp8(slots, row, row.clone(), weight, eps, tile) + torch.cuda.synchronize(self.device) diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/lossless_x3.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/lossless_x3.py new file mode 100644 index 0000000..18cfb69 --- /dev/null +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/lossless_x3.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``lossless_bf16``'s exact 12-bit code in 12-byte lane records, for readers across the link. + +The code is ``allreduce_add_rmsnorm/lossless_bf16``'s (per 32 values a group exponent, per value sign, 7 mantissa bits +and a 4-bit exponent delta; zeros, exceptions and raw virtual waves as there); only the layout changes. Lane t's 8 +sign/mantissa bytes and its 8 delta nibbles are one record at byte 12 t, then 12 headers of 32 B at 9216 (16 group +exponent bytes, the count word, 6 exception values): 9600 B = 0.781x the bf16 row. A reader fetches a record with one +dwordx3 and all 12 headers with one wave's 384-byte load, shared through LDS: whole-line reads (per-wave 32-byte header +reads cost ~10% of the link, which is request-bound). Raw virtual waves go to the caller's raw slot (1 KB at 1024 v). +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..allreduce_add_rmsnorm.lossless_bf16 import RAWF, SPECF, _dec_fast, _dec_fix, enc +from .common import ROWB, SYS + +HDR = 9216 # header bytes of a coded row +HDR_DW = 96 # header dwords of a row + + +@flyc.jit +def enc_store(value, base, raw, tid, mask): + """Write one lane's 8 values of a coded row (or its virtual wave's raw fallback), without publishing.""" + lane, wave = tid % 64, tid // 64 + a0, a1, nib, gmax, cwv, ev, idx, has, rawi = enc(value, lane) + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=ROWB) + rr = buffer_ops.create_buffer_resource_from_addr(raw, num_records_bytes=ROWB) + buffer_ops.buffer_store(value, rr, tid * 4, mask=mask & (rawi != 0), cache_modifier=SYS) + buffer_ops.buffer_store(Vec.from_elements([a0, a1, nib], fx.Int32), rs, tid * 3, mask=mask, cache_modifier=SYS) + buffer_ops.buffer_store(gmax.to(fx.Int8), rs, HDR + 32 * wave + lane // 4, mask=mask & (lane % 4 == 0), + cache_modifier=SYS, offset_is_bytes=True) + buffer_ops.buffer_store(cwv, rs, HDR // 4 + 8 * wave + 4, mask=mask & (lane == 0), cache_modifier=SYS) + buffer_ops.buffer_store(ev.to(fx.Int16), rs, HDR + 32 * wave + 20 + 2 * (idx - 1), + mask=mask & has & (rawi == 0), cache_modifier=SYS, offset_is_bytes=True) + + +def fetch(base, tid, mask): + """A lane's record (3 dwords); issue every row's before decoding any.""" + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=ROWB) + rec = Vec(buffer_ops.buffer_load(rs, tid * 3, vec_width=3, dtype=fx.Int32, mask=mask, cache_modifier=SYS)) + return [fx.Int32(rec[t]) for t in range_constexpr(3)] + + +def fetch_headers(base, lane, mask): + """Lane l < 24's 4 header dwords 4 l .. 4 l + 3 of a row (``mask``: the loading wave's lanes).""" + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=ROWB) + hv = Vec(buffer_ops.buffer_load(rs, HDR // 4 + 4 * lane, vec_width=4, dtype=fx.Int32, mask=mask & (lane < 24), + cache_modifier=SYS)) + return [fx.Int32(hv[t]) for t in range_constexpr(4)] + + +@flyc.jit +def finish(rec, hw, raw, tid, mask): + """Decode a lane's four packed bf16 pairs from its record and its virtual wave's 8 header dwords ``hw``; a raw + virtual wave reads the caller's raw slot. Masked lanes return zeros (their ``hw`` may be any LDS words).""" + lane = tid % 64 + pw, pb = rec[:2], rec[2] + cw = fx.Int32(fx.rocdl.readfirstlane(T.i32, hw[4])) + d0, d1, d2, d3 = _dec_fast(pw, pb, hw, lane) + if (cw & SPECF) != 0: + d0, d1, d2, d3 = _dec_fix([d0, d1, d2, d3], pw, pb, hw) + if (cw & RAWF) != 0: + rr = buffer_ops.create_buffer_resource_from_addr(raw, num_records_bytes=ROWB) + r = Vec(buffer_ops.buffer_load(rr, tid * 4, vec_width=4, dtype=fx.Int32, mask=mask, cache_modifier=SYS)) + d0, d1, d2, d3 = [fx.Int32(r[t]) for t in range_constexpr(4)] + return Vec.from_elements([mask.select(d, fx.Int32(0)) for d in (d0, d1, d2, d3)], fx.Int32) diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/oneshot.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/oneshot.py new file mode 100644 index 0000000..c665cb7 --- /dev/null +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/allreduce_add_rmsnorm_ranks2/oneshot.py @@ -0,0 +1,272 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank all-reduce + residual add + Gemma RMSNorm (bf16 out, or MXFP8 with tiled / row-major codes), one-shot, +eager or in graphs. + +Rows go in batches of ``rpb``. The first ``ns`` workgroups are stagers: batch k by stager k % ns, in order, each wave +writing its columns of the batch's partial rows into this rank's buffer, then the batch's flag into the peer's +(``common``; one per wave, coded rows one per batch). The other ``nr`` are reducers: batch k by reducer k % nr; each +wave loads its own partial and residual rows, polls the flag, reads the peer's staged rows over the link, sums them in +fp32 per row and runs ``add_rmsnorm``'s norm (12 waves, bf16 out) or ``add_rmsnorm_mxfp8``'s norm + quant (8 waves). +Stagers dispatch first and never wait on the peer, so the grid needs no residency guarantee, and a reducer's rows cross +the link while earlier batches are reduced. ``slots``: the partial is the MoE combine +``bf16((((s0 + s1) + s2) + s3) + shared)`` of this rank's [rows, 4, H] slots and [rows, H] shared output (stagers and +reducers both compute it: the same bits). The inputs are read where they are: no registration. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from ..add_rmsnorm import add_rmsnorm as _bf16_norm +from ..add_rmsnorm import add_rmsnorm_mxfp8 as _mx_norm +from ..add_rmsnorm.common import ( + PRELOAD_HINTS, + H, + bf16x8, + pk_bf16, + rsqrt_rn, + seq_sum_sq, + wave_sum_lane63, +) +from . import lossless_x3 as _x3 +from .common import ( + COUNT_PAD, + MAX_ROWS, + ROWB, + SYS, + data_addr, + flag_addr, + flag_store, + local_sum, + spin, + store_rowmajor, + store_tiled, + sum2, +) + +SLOTS = 4 +COUNT_WORDS = MAX_ROWS + COUNT_PAD # per count array: stagers at 0, reducers after + + +def compile_oneshot_ar_add_rmsnorm(out: str = "bf16", slots: bool = False, rpb: int = 1, coded: bool = False): + """``out``: "bf16", "tiled" or "rowmajor" (MXFP8 codes); ``rpb`` rows per batch (one flag per batch and wave); + ``coded``: rows cross the link in ``lossless_x3``'s exact 12-bit code (0.781x the bytes, one flag per batch; a + virtual wave of 512 values is the 8 columns of 64 lanes, so the MXFP8 lane map's second chunk is virtual waves + 8-11).""" + B = rpb + quant = out != "bf16" + NW = 8 if quant else 12 + NT = 64 * NW + NPK = 2 if quant else 1 # 16-byte packs per lane and row + block_sum = _mx_norm._block_sum if quant else _bf16_norm._block_sum + name = ( + ("sum_slots4_shared_" if slots else "") + + "allreduce_oneshot_ranks2_gemma_add_rmsnorm" + + (f"_mxfp8_{out}" if quant else "") + + f"_h6144_b{B}" + + ("_coded" if coded else "") + ) + + @fx.struct + class Shared: + part: fx.Array[fx.Float32, 16] + hdr: fx.Array[fx.Int32, B * _x3.HDR_DW] + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + i32_rows: fx.Int32, + i32_grid: fx.Int32, # stagers | reducers << 16 + i32_sx: fx.Int32, # partial row stride (elements, a multiple of 8) + i32_sr: fx.Int32, # residual row stride (elements, a multiple of 8) + arg_x: fx.Int64, # [rows, H] bf16 partial, or [rows, 4, H] bf16 slots + arg_sh: fx.Int64, # [rows, H] bf16 shared-expert output (slots) + arg_mine: fx.Int64, # this rank's buffer + arg_peer: fx.Int64, # the peer's buffer, mapped here + arg_r: fx.Int64, # residual [rows, H] bf16 + arg_w: fx.Int64, # norm weight [H] bf16 + arg_out: fx.Int64, # [rows, H] bf16 out, or the e4m3 codes + arg_s: fx.Int64, # scales u8, AITER layout (MXFP8) + arg_res: fx.Int64, # residual out [rows, H] bf16 + arg_cnt: fx.Int64, # int32 [2, COUNT_WORDS] call counts: stagers, reducers + i32_eps: fx.Int32, # fp32 bits + ): + if const_expr(quant): + _mx_norm._set_fp16_ovfl() + tx, b = fx.thread_idx.x, fx.block_idx.x + lane = tx % 64 + wave = fx.Int32(fx.rocdl.readfirstlane(T.i32, tx // 64)) + rows = i32_rows + nstage = i32_grid & 0xFFFF + nred = i32_grid >> 16 + nb = (rows + (B - 1)) // B + cr = buffer_ops.create_buffer_resource_from_addr(arg_cnt, num_records_bytes=2 * COUNT_WORDS * 4) + has1 = tx < 256 # MXFP8: second chunk (columns 4096 + 8 t) inside the row + offs = [tx * 4, tx * 4 + 2048] + always = fx.Boolean(True) + + def ld(base, cm, ok): + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=ROWB) + return [ + Vec(buffer_ops.buffer_load(rs, offs[h], vec_width=4, dtype=fx.Int32, + mask=ok if h == 0 else ok & has1, cache_modifier=cm)) + for h in range_constexpr(NPK) + ] + + def st(vs, base, ok, cm): + rs = buffer_ops.create_buffer_resource_from_addr(base, num_records_bytes=ROWB) + for h in range_constexpr(NPK): + buffer_ops.buffer_store(vs[h], rs, offs[h], mask=ok if h == 0 else ok & has1, cache_modifier=cm) + + def own_row(r, ok): + if const_expr(slots): + sbase = arg_x + fx.Int64(r) * (SLOTS * ROWB) + sl = [ld(sbase + k * ROWB, 0, ok) for k in range_constexpr(SLOTS)] + sh = ld(arg_sh + fx.Int64(r) * ROWB, 0, ok) + return [local_sum([sl[k][h] for k in range_constexpr(SLOTS)], sh[h]) for h in range_constexpr(NPK)] + return ld(arg_x + fx.Int64(r) * fx.Int64(i32_sx * 2), 0, ok) + + def count(base, r): + return fx.Int32(buffer_ops.buffer_load(cr, base + r, vec_width=1, dtype=fx.Int32, is_scalar=True)) + 1 + + def advance(base, first, step): + """After the workgroup's last read: count + 1 for every row of its batches.""" + fx.gpu.barrier() + for k in range(first, nb, step): + r = k * B + tx + m = (tx < B) & (r < rows) + cur = fx.Int32(buffer_ops.buffer_load(cr, base + r, vec_width=1, dtype=fx.Int32, mask=m)) + buffer_ops.buffer_store(cur + 1, cr, base + r, mask=m) + + if b < nstage: + for k in range(b, nb, nstage): + r0 = k * B + for i in range_constexpr(B): + r = r0 + i + ok = r < rows + e = count(0, r) + if const_expr(coded): + v = own_row(r, ok) + for h in range_constexpr(NPK): + _x3.enc_store(v[h], data_addr(arg_mine, r, e), data_addr(arg_mine, r, e, True), + tx + 512 * h, ok if h == 0 else ok & has1) + else: + st(own_row(r, ok), data_addr(arg_mine, r, e), ok, SYS) + fx.rocdl.s_waitcnt(vmcnt=0) + if const_expr(coded): + # one flag per batch: a reader wave decodes with every wave's header + fx.gpu.barrier() + if tx == 0: + flag_store(flag_addr(arg_peer, r0, 0), count(0, r0)) + else: + # the wave's own stores landed: its flag for this batch + if lane == 0: + flag_store(flag_addr(arg_peer, r0, wave), count(0, r0)) + advance(0, b, nstage) + else: + smem = fx.SharedAllocator().allocate(Shared).peek() + part = smem.part.ptr + hdr = smem.hdr.ptr + wv = [bf16x8(p) for p in ld(arg_w, 0, always)] + for k in range(b - nstage, nb, nred): + r0 = k * B + oks = [(r0 + i) < rows for i in range_constexpr(B)] + masks = [[oks[i] if h == 0 else oks[i] & has1 for h in range_constexpr(NPK)] + for i in range_constexpr(B)] + owns = [own_row(r0 + i, oks[i]) for i in range_constexpr(B)] + rvs = [ld(arg_r + fx.Int64(r0 + i) * fx.Int64(i32_sr * 2), 0, oks[i]) for i in range_constexpr(B)] + # a poll waits for the wave's outstanding vector loads: the batch's own rows and residuals land first + if const_expr(coded): + if tx == 0: + spin(flag_addr(arg_mine, r0, 0), count(COUNT_WORDS, r0)) + fx.gpu.barrier() + elif lane == 0: + spin(flag_addr(arg_mine, r0, wave), count(COUNT_WORDS, r0)) + es = [count(COUNT_WORDS, r0 + i) for i in range_constexpr(B)] + if const_expr(coded): + recs = [[_x3.fetch(data_addr(arg_peer, r0 + i, es[i]), tx + 512 * h, masks[i][h]) + for h in range_constexpr(NPK)] for i in range_constexpr(B)] + hvs = [_x3.fetch_headers(data_addr(arg_peer, r0 + i, es[i]), lane, oks[i] & (wave == 0)) + for i in range_constexpr(B)] + if (wave == 0) & (lane < 24): + for i in range_constexpr(B): + for t in range_constexpr(4): + hdr[i * _x3.HDR_DW + 4 * lane + t] = hvs[i][t] + fx.gpu.barrier() + peers = [[_x3.finish(recs[i][h], [hdr[i * _x3.HDR_DW + 8 * (wave + 8 * h) + t] + for t in range_constexpr(8)], + data_addr(arg_peer, r0 + i, es[i], True), tx + 512 * h, masks[i][h]) + for h in range_constexpr(NPK)] for i in range_constexpr(B)] + else: + peers = [ld(data_addr(arg_peer, r0 + i, es[i]), SYS, oks[i]) for i in range_constexpr(B)] + for i in range_constexpr(B): + r = r0 + i + if r < rows: + own, peer, rv = owns[i], peers[i], rvs[i] + xv = [bf16x8(sum2(own[h], peer[h])) for h in range_constexpr(NPK)] + rvf = [bf16x8(rv[h]) for h in range_constexpr(NPK)] + s = [[xv[h][n] + rvf[h][n] for n in range_constexpr(8)] for h in range_constexpr(NPK)] + res = [ + Vec.from_elements([pk_bf16(s[h][2 * n], s[h][2 * n + 1]) for n in range_constexpr(4)], + fx.Int32) + for h in range_constexpr(NPK) + ] + st(res, arg_res + fx.Int64(r) * ROWB, always, 0) + sq = s[0] + s[1] if const_expr(quant) else s[0] + acc = wave_sum_lane63(seq_sum_sq(sq)) + if lane == 63: + part[wave] = acc + fx.gpu.barrier() + rstd = rsqrt_rn(block_sum([part[n] for n in range_constexpr(NW)]), + i32_eps.bitcast(fx.Float32)) + ys = [[(s[h][n] * rstd) * (wv[h][n] + fx.Float32(1.0)) for n in range_constexpr(8)] + for h in range_constexpr(NPK)] + if const_expr(quant): + codes, sbs = [], [] + for h in range_constexpr(2): + sb, rescale = _mx_norm._e8m0(ys[h]) + codes.append(_mx_norm._codes(ys[h], rescale)) + sbs.append(fx.Int32(sb.to(fx.Int32))) + q_bytes = (rows + 15) // 16 * 16 * H + s_bytes = (rows + 31) // 32 * 32 * (H // 32) + qr = buffer_ops.create_buffer_resource_from_addr(arg_out, num_records_bytes=q_bytes) + sr = buffer_ops.create_buffer_resource_from_addr(arg_s, num_records_bytes=s_bytes) + (store_tiled if out == "tiled" else store_rowmajor)(tx, r, codes, sbs, has1, always, + qr, sr) + else: + y = Vec.from_elements([pk_bf16(ys[0][2 * n], ys[0][2 * n + 1]) + for n in range_constexpr(4)], fx.Int32) + st([y], arg_out + fx.Int64(r) * ROWB, always, 0) + fx.gpu.barrier() + advance(COUNT_WORDS, b - nstage, nred) + + @flyc.jit + def launch_oneshot_ar_add_rmsnorm( + i32_rows: fx.Int32, + i32_grid: fx.Int32, + i32_sx: fx.Int32, + i32_sr: fx.Int32, + arg_x: fx.Int64, + arg_sh: fx.Int64, + arg_mine: fx.Int64, + arg_peer: fx.Int64, + arg_r: fx.Int64, + arg_w: fx.Int64, + arg_out: fx.Int64, + arg_s: fx.Int64, + arg_res: fx.Int64, + arg_cnt: fx.Int64, + i32_eps: fx.Int32, + stream: fx.Stream, + ): + kernel( + i32_rows, i32_grid, i32_sx, i32_sr, arg_x, arg_sh, arg_mine, arg_peer, arg_r, arg_w, arg_out, arg_s, + arg_res, arg_cnt, i32_eps, + ).launch(grid=(fx.Int64((i32_grid & 0xFFFF) + (i32_grid >> 16)), 1, 1), block=(NT, 1, 1), stream=stream) + + launch_oneshot_ar_add_rmsnorm.kernel_name = name + launch_oneshot_ar_add_rmsnorm.compile_hints = PRELOAD_HINTS + return launch_oneshot_ar_add_rmsnorm diff --git a/amd_kernels/norm/gemma_add_rmsnorm_h6144/reference.py b/amd_kernels/norm/gemma_add_rmsnorm_h6144/reference.py index fa322fa..3d6d349 100644 --- a/amd_kernels/norm/gemma_add_rmsnorm_h6144/reference.py +++ b/amd_kernels/norm/gemma_add_rmsnorm_h6144/reference.py @@ -10,6 +10,8 @@ in the rank order c, c + 1, c + 2, c + 3 (mod 4), operands and partial sums with denormals flushed to a zero of their sign, one RNE rounding (AITER's ``cross_device_reduce_2stage``, built with -fgpu-flush-denormals-to-zero), ``allreduce_rows_reference``; the MoE tail first sums each rank's slots, ``slots4_allreduce_reference``; + - two-rank all-reduce (``allreduce_add_rmsnorm_ranks2``): ``bf16(ftz(ftz(x0) + ftz(x1)))`` at every row, the same + functions with 2 partials (commutative: AITER's one-stage and two-stage results at two ranks); 2. ``s = fp32(x) + fp32(residual)`` (one fp32 rounding); the residual out is ``bf16(s)``; 3. ``y = (s * rstd) * (w + 1)``, ``rstd = 1 / sqrt(sum(s^2) / 6144 + eps)`` (Gemma's weight offset; the norm reads the fp32 s, not the rounded residual out). The kernels sum the squares in fp32 in the Triton kernels' order; the @@ -70,15 +72,17 @@ def allreduce_reference(xs: list[torch.Tensor]) -> torch.Tensor: def allreduce_rows_reference(xs: list[torch.Tensor]) -> torch.Tensor: - """The 4 ranks' ``[rows, H]`` bf16 partials -> their sum, bf16, at any row count (module docstring, step 1).""" + """The ranks' ``[rows, H]`` bf16 partials (4, or 2 for ``allreduce_add_rmsnorm_ranks2``) -> their sum, bf16, at any + row count (module docstring, step 1).""" + world = len(xs) rows = xs[0].shape[0] - q = (rows + WORLD - 1) // WORLD + q = (rows + world - 1) // world out = torch.empty_like(xs[0]) - for c in range(WORLD): + for c in range(world): blk = slice(min(c * q, rows), min((c + 1) * q, rows)) acc = _ftz(xs[c][blk].float()) - for t in range(1, WORLD): - acc = _ftz(acc + _ftz(xs[(c + t) % WORLD][blk].float())) + for t in range(1, world): + acc = _ftz(acc + _ftz(xs[(c + t) % world][blk].float())) out[blk] = acc.bfloat16() return out @@ -95,7 +99,7 @@ def slots4_local_reference(slots: torch.Tensor, shared: torch.Tensor) -> torch.T def slots4_allreduce_reference( slots: list[torch.Tensor], shared: list[torch.Tensor] ) -> torch.Tensor: - """The 4 ranks' slots and shared outputs -> the all-reduced bf16 rows of ``slots4_ar_add_rmsnorm_mxfp8``: the + """The ranks' slots and shared outputs -> the all-reduced bf16 rows of ``slots4_ar_add_rmsnorm_mxfp8``: the ranks' ``slots4_local_reference`` partials through ``allreduce_rows_reference`` (1-8191 rows). Its row-major codes use the tiled norm's arithmetic; only their addresses change.""" return allreduce_rows_reference([slots4_local_reference(s, h) for s, h in zip(slots, shared)]) diff --git a/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/README.md b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/README.md new file mode 100644 index 0000000..c49ed56 --- /dev/null +++ b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/README.md @@ -0,0 +1,58 @@ +# Batch-sharded sampling on two ranks: owned-row logits, output gather, Gumbel Markov draft + +## What it does + +Batch-sharded sampling on two TP ranks: each rank verifies, and drafts for, the requests it owns (batch position i on +rank i % 2, local row i // 2). Vocabulary 200064 (lm_head shards of 100032 in `shuffle_weight`'s layout), hidden 6144, +Markov rank 256 (`markov_w1` / `markov_w2` bf16 [200064, 256]); target 1-2048 logits rows, 2-256 requests, 1-8 logits +rows per request; the Markov draft 1-7 steps, up to 128 requests per rank. Caller: the MiniMax-M3.1 TP2 sampler and +DSpark Markov draft. Contract (`reference.py`): the per-kernel entries' references; the block equals the per-kernel +chain bit for bit (draft ids, every draft-cache byte, the logits). The owned rows' logits come from the vocabulary +shard exchange up to 96 rows per call and above from two local GEMMs (this rank's shard and a copy of the peer's) +with the same bits: one xGMI link carries M x 100 KB per direction, more than a second 1.23 GB weight read costs from +about 100 rows. + +## Entry points + +- `BatchShardedSampling`: the per-process object (communicators, owner tables, scratch), built collectively before + any CUDA graph capture; with `lm_weight` (the shuffled shard every call passes) it copies the peer's shard over IPC + (+1.23 GB per GPU); without it every call takes the exchange. +- `BatchShardedSampling.target_logits`: the `[owned rows, 200064]` bf16 logits of this rank's requests. +- `BatchShardedSampling.gather_sampler_output`: tokens and sampled / rejected / NaN counts of every request in batch + order. +- `BatchShardedSampling.setup_markov` (collective, before capture) and `markov_draft`: the draft ids of every request + in batch order and this rank's draft-cache rows; graph safe. +- `sampling_blocker`, `markov_blocker`: why the block or the draft cannot serve a shape (None: it can). +- `owner_tables`, `OwnerTables`: the index tables of one request count; `close`: collective teardown. + +Every rank makes every call with the same counts; `target_logits` and `markov_draft` are `amd_kernels.warm_up` entries. + +## Performance + +Two GPUs (one xGMI link) of one MI355X node. `benchmark_block.py`: graph replay of 12 distinct inputs, five alternated +rounds, MAX over ranks, median µs, dense owners. Target: the verify logits of R requests x 8 rows; vLLM = vLLM's +TP2 path (its lm_head GEMM over all rows + RCCL all-gather into [M, 200064] on both ranks), vLLM a2a = upstream +batch-sharded (the same GEMM + RCCL all-to-all). Draft: `markov_draft` (draft lm_head of 7 R rows, seven Markov steps, +id gather) against only the draft logits of vLLM's replicated draft (its GEMM + all-gather, before its Markov loop). + +| R | target M | vLLM | vLLM a2a | exchange | local | draft rows | vLLM logits | exchange | local | +|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| 1 | 8 | 253.1 | 238.7 | **213.4** | 356.0 | 7 | 244.1 | **389.8** | 538.1 | +| 4 | 32 | 370.5 | 309.3 | **247.0** | 363.8 | 28 | 354.4 | **426.5** | 548.3 | +| 8 | 64 | 548.2 | 416.1 | **308.8** | 367.8 | 56 | 510.3 | **506.9** | 561.3 | +| 12 | 96 | 673.4 | 488.7 | **372.7** | 373.8 | 84 | 635.0 | **549.7** | 574.5 | +| 16 | 128 | 816.7 | 557.7 | 431.2 | **380.4** | 112 | 743.0 | 604.7 | **580.5** | +| 24 | 192 | 1088.2 | 712.4 | 596.1 | **401.7** | 168 | 1001.3 | 780.8 | **605.1** | +| 32 | 256 | 1369.2 | 861.2 | 756.0 | **423.1** | 224 | 1232.2 | 1060.0 | **638.8** | +| 40 | 320 | 1626.1 | 971.7 | 1132.1 | **540.0** | 280 | 1447.7 | 1305.6 | **730.0** | +| 48 | 384 | 1914.0 | 1149.9 | 1325.7 | **595.3** | 336 | 1736.5 | 1347.7 | **830.6** | +| 52 | 416 | 2140.8 | 1290.1 | 1450.0 | **634.1** | 364 | 1823.5 | 1410.8 | **841.0** | + +Bold: the path `host.py` takes (exchange up to 96 rows, local above). + +## Tests + +```bash +python3 -m pytest tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2 -q # 2 GPUs +python3 benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py [R ...] # 2 GPUs +``` diff --git a/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py new file mode 100644 index 0000000..925e1b5 --- /dev/null +++ b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Batch-sharded sampling on two ranks (vocab 200064, hidden 6144, Markov rank 256): the target's verify logits of +the owned rows, the sampler-output gather and the Gumbel Markov draft chain, sequenced on the per-kernel entries.""" + +from .host import (LOCAL_MIN_ROWS, MAX_OWNED, MAX_REQS, MAX_ROWS, MAX_STEPS, BatchShardedSampling, OwnerTables, + markov_blocker, owner_tables, sampling_blocker) + +__all__ = ["BatchShardedSampling", "OwnerTables", "markov_blocker", "owner_tables", "sampling_blocker", + "LOCAL_MIN_ROWS", "MAX_OWNED", "MAX_REQS", "MAX_ROWS", "MAX_STEPS"] diff --git a/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/host.py b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/host.py new file mode 100644 index 0000000..eff809f --- /dev/null +++ b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/host.py @@ -0,0 +1,465 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Batch-sharded sampling on two ranks: each rank verifies, and drafts for, the requests it owns (batch position i +on rank i % 2). + +One object per process (``BatchShardedSampling``, built collectively, before any CUDA graph capture) owns the +communicators, the owner tables of every request count, and (``lm_weight`` given) a copy of the peer's lm_head shard. +The communicators compile and run their warm-up collectively when they are built; the two GEMM chains are entries +(``amd_kernels.warm_up``): the first call of each mode compiles every variant the mode can reach. Their warm-up calls +never touch a communicator or a caller's buffer: a variant compiled earlier in the process launches even in FlyDSL's +compile-only mode, so they publish into a private slot and skip the exchanges. + +Logits of the owned rows, two ways with the same bits (the GEMM's sums do not depend on its row tiling): +- exchange (up to ``LOCAL_MIN_ROWS`` rows, or without the peer copy): the lm_head GEMM over all owner-sorted rows + publishes into the exchange's send slot, the pull writes this rank's rows in vocabulary order. One xGMI link + carries (M / 2) x 100032 x 2 B per direction, 4x the per-link bytes of four ranks. +- local (above): the GEMM over the owned rows only, once on this rank's shard and once on the peer's copy, straight + into the two vocabulary halves of the output; no exchange. + +- ``target_logits``: the target's verify logits of this rank's rows. +- ``gather_sampler_output``: tokens / counts / NaN counts back in batch order (``SamplerAllGather``). +- ``markov_draft``: the Markov draft of every owned request: the draft lm_head, then per step the bias projection, + the Gumbel partials (+ draft-cache row) and the combine (+ draft id, next packed ``markov_w1`` row; two launches + above 64 owned rows), then the id gather into batch order (``I64C7AllGather``). + +The arithmetic is the per-kernel entries' (the tests compare bits with them); the chains only sequence them. +""" + +import ctypes +from dataclasses import dataclass + +import numpy as np +import torch +import torch.distributed as dist + +from amd_kernels.common.hip_runtime import load_hip_runtime +from amd_kernels.warm_up import _in_warm_up, entry + +V, H, RANK = 200064, 6144, 256 # vocabulary, hidden size (the lm_head's K), Markov rank +N = V // 2 # the lm_head's vocabulary shard +WORLD = 2 +MAX_ROWS = 2048 # the exchanges' and the lm_head GEMM's row bound +MAX_REQS = 256 # the small gathers' request bound +MAX_OWNED = 128 # requests per rank of the Markov bias GEMM (bf16_gemm_packed_rows) +COMBINE_ROWS = 64 # rows per launch of gumbel_combine_markov_embed_packed +MAX_STEPS = 7 # draft steps (I64C7AllGather's columns) +MAX_LOGITS_PER_REQ = 8 # SamplerAllGather's token columns +LOCAL_MIN_ROWS = 96 # above (total rows): the owned rows on both shards locally; crossover of benchmark_block.py + + +def sampling_blocker(world: int, max_reqs: int, logits_per_req: int) -> str | None: + """Why ``BatchShardedSampling`` cannot serve this batch shape (None: it can): ``world`` ranks, up to ``max_reqs`` + requests of up to ``logits_per_req`` logits rows each.""" + if world != WORLD: + return f"{world} ranks (two-rank kernels)" + if not WORLD <= max_reqs <= MAX_REQS: + return f"{max_reqs} requests outside [{WORLD}, {MAX_REQS}]" + if not 1 <= logits_per_req <= MAX_LOGITS_PER_REQ: + return f"{logits_per_req} logits rows per request (the gather has {MAX_LOGITS_PER_REQ} columns)" + if max_reqs * logits_per_req > MAX_ROWS: + return f"more than {MAX_ROWS} logits rows per step" + return None + + +def markov_blocker(steps: int, max_reqs: int, markov_w1: torch.Tensor, markov_w2: torch.Tensor, *, + cache: torch.Tensor | None = None, lm_weight_shuffled: bool = True, fp64_noise: bool = False, + confidences: bool = False, logit_transform: bool = False, + reduced_vocab: bool = False) -> str | None: + """Why ``setup_markov`` cannot serve this draft (None: it can). The keywords describe the caller's draft: its + draft-logits cache, whether its lm_head weight holds ``shuffle_weight``'s layout, and the features the chain does + not have (fp64 Gumbel noise, draft confidences, an output logit transform, a reduced draft vocabulary).""" + if not 1 <= steps <= MAX_STEPS: + return f"{steps} draft steps (the id gather has {MAX_STEPS} columns)" + if -(-max_reqs // WORLD) > MAX_OWNED: + return f"more than {MAX_OWNED} requests per rank" + for w in (markov_w1, markov_w2): + if tuple(w.shape) != (V, RANK) or w.dtype != torch.bfloat16 or not w.is_contiguous(): + return f"the Markov head is not bf16 [{V}, {RANK}]" + if cache is not None and not (cache.dtype == torch.bfloat16 and cache.dim() == 3 and cache.shape[1:] == (steps, V) + and cache.is_contiguous()): + return f"the draft-logits cache {tuple(cache.shape)} {cache.dtype} (contiguous bf16 [slots, {steps}, {V}])" + for flag, why in ((not lm_weight_shuffled, "the lm_head weight is not in shuffle_weight's layout"), + (fp64_noise, "fp64 Gumbel noise (the chain draws fp32 noise)"), + (confidences, "draft confidences (the chain writes ids and draft logits only)"), + (logit_transform, "an output logit transform"), + (reduced_vocab, f"a reduced draft vocabulary (the chain samples all {V} ids)")): + if flag: + return why + return None + + +@dataclass +class OwnerTables: + """Device index tables of one request count (``owner_tables``).""" + + cap: int # requests per rank, cdiv(num_reqs, 2) + send_rows: torch.Tensor # int64 [2 * cap * steps]: owner-sorted (request, step) rows, padding -> request 0 + own_rows: torch.Tensor # int64 [cap * steps]: this rank's part of send_rows + slot_rows: torch.Tensor # int64 [steps, cap]: this rank's (request, step) rows + valid: torch.Tensor # bool [1, cap]: the local request exists + anchor_rows: torch.Tensor # int64 [cap]: the anchor token's row in input_ids + gathered_src: torch.Tensor # int64 [num_reqs]: batch request -> (owner, local row) in the id gather + counts: torch.Tensor # int32 [2]: rows per rank in the exchange + + +def owner_tables(num_reqs: int, steps: int, rank: int, anchor_idx: np.ndarray, device) -> OwnerTables: + """Batch position i is owned by rank i % 2 (local row i // 2); a rank's padding rows read request 0.""" + cap = -(-num_reqs // WORLD) + send = np.arange(cap)[None, :] * WORLD + np.arange(WORLD)[:, None] # [2, cap] batch positions + valid = send < num_reqs + send_c = np.where(valid, send, 0) + step = np.arange(steps) + send_rows = (send_c[..., None] * steps + step).reshape(-1) + local = send_c[rank] + req = np.arange(num_reqs) + own = send_rows[rank * cap * steps:(rank + 1) * cap * steps] + return OwnerTables( + cap=cap, + send_rows=torch.from_numpy(send_rows.astype(np.int64)).to(device), + own_rows=torch.from_numpy(own.astype(np.int64)).to(device), + slot_rows=torch.from_numpy((local[None, :] * steps + step[:, None]).astype(np.int64)).to(device), + valid=torch.from_numpy(valid[rank][None, :].copy()).to(device), + anchor_rows=torch.from_numpy(anchor_idx[local].astype(np.int64)).to(device), + gathered_src=torch.from_numpy(((req % WORLD) * cap + req // WORLD).astype(np.int64)).to(device), + counts=torch.full((WORLD,), cap * steps, dtype=torch.int32, device=device), + ) + + +_CAPTURED_HOST: list[torch.Tensor] = [] + + +def _device_counts(counts, device: torch.device) -> torch.Tensor: + """counts[2] (host ints) -> device int32 through a pinned staging tensor (non-blocking; the caching host + allocator keeps the block until its copy has run). Under graph capture (a timing harness) the staging tensor is + kept, since every replay reads it.""" + host = torch.empty(WORLD, dtype=torch.int32, pin_memory=True) + host.numpy()[:] = counts + if torch.cuda.is_current_stream_capturing(): + _CAPTURED_HOST.append(host) + return host.to(device, non_blocking=True) + + +def _publish_slot(comm, rows: int, device): + """The exchange's send slot, or in a warm-up a private slot pair, counter and flag (no peer, no caller buffer).""" + if not _in_warm_up.get(): + return comm.slot(), None + slot_bytes = max(rows, 1) * N * 2 + keep = (torch.empty(2 * slot_bytes, dtype=torch.uint8, device=device), + torch.zeros(1, dtype=torch.int32, device=device), torch.zeros(16, dtype=torch.int32, device=device)) + return (keep[0].data_ptr(), slot_bytes, keep[1], keep[2].data_ptr()), keep + + +class _PeerShard: + """This rank's copy of the peer's lm_head shard (same layout), filled once over IPC: each rank writes its own + shard into the peer's copy (collective).""" + + def __init__(self, rank, group, device, lm_weight): + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.host import (_DevicePointer, _alloc_pool, + _exchange, _open_pool) + + assert lm_weight.shape == (N, H) and lm_weight.dtype == torch.bfloat16 and lm_weight.is_contiguous() + nbytes = N * H * 2 + self.ptr, handle, pool = _alloc_pool(nbytes, device) + self.weight = pool.view(torch.bfloat16).view(N, H) + self.own = lm_weight.data_ptr() + handles = _exchange(handle, rank, WORLD, group) + peer = _open_pool(handles[1 - rank], device) + dst = torch.as_tensor(_DevicePointer(peer, nbytes), device=device) + dst.copy_(lm_weight.view(-1).view(torch.uint8)) + torch.cuda.synchronize(device) + dist.barrier(group=group) + lib = load_hip_runtime() + lib.hipIpcCloseMemHandle.argtypes = [ctypes.c_void_p] + assert lib.hipIpcCloseMemHandle(ctypes.c_void_p(peer)) == 0 + dist.barrier(group=group) + + def close(self): + lib = load_hip_runtime() + lib.hipFree.argtypes = [ctypes.c_void_p] + torch.cuda.synchronize() + assert lib.hipFree(ctypes.c_void_p(self.ptr)) == 0 + self.weight = None + + +@dataclass +class _Markov: + steps: int + max_reqs: int + tables: dict + arange: torch.Tensor + w1: torch.Tensor + w2: torch.Tensor + + +class BatchShardedSampling: + """The communicators and tables of batch-sharded sampling on two ranks (module doc). Build it collectively + (every rank, the same order), before any CUDA graph capture; ``setup_markov`` adds the draft's exchange. + ``lm_weight``: this rank's lm_head shard in ``shuffle_weight``'s layout, the tensor every call passes; given, the + rank keeps a copy of the peer's shard (1.23 GB) and computes logits locally above ``LOCAL_MIN_ROWS`` rows.""" + + def __init__(self, rank: int, group, device, *, max_reqs: int, target_rows: int, + lm_weight: torch.Tensor | None = None): + from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2 import SamplerAllGather + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll + + assert WORLD <= max_reqs <= MAX_REQS and 1 <= target_rows <= MAX_ROWS + self.rank, self.group, self.device = rank, group, torch.device(device) + self.max_reqs, self.target_rows = max_reqs, target_rows + self.peer = None if lm_weight is None else _PeerShard(rank, group, self.device, lm_weight) + self.target_a2a = RowAllToAll(rank, group, self.device, max_rows=self._exchange_rows(target_rows)) + self.target_gather = SamplerAllGather(rank, group, self.device) + self.target_a2a.warmup() # collective: every rank, here + self.target_gather.warmup() + # the lm_head's publish (and local: rows) tiles at load (the entry's first call then finds them compiled) + from amd_kernels.gemm.bf16_n100032_k6144.host import warmup_bf16_gemm_n100032_k6144 + + warmup_bf16_gemm_n100032_k6144(self.device, shuffled=True, mode="publish") + if self.peer is not None: + warmup_bf16_gemm_n100032_k6144(self.device, shuffled=True, mode="rows") + torch.cuda.empty_cache() + self.markov: _Markov | None = None + self.draft_a2a = self.draft_gather = None + + def _exchange_rows(self, rows: int) -> int: + return rows if self.peer is None else min(rows, LOCAL_MIN_ROWS) + + def _local(self, m: int, lm_weight) -> bool: + if self.peer is None or m <= LOCAL_MIN_ROWS: + return False + assert lm_weight.data_ptr() == self.peer.own, "lm_weight is not the shard given at construction" + return True + + # ------------------------------------------------------------ the target's verify logits + def target_logits(self, hidden, lm_weight, sorted_rows, counts, num_local: int) -> torch.Tensor: + """[num_local, 200064] bf16: the logits of this rank's rows, bit for bit what the lm_head GEMM of every rank + over ``hidden[sorted_rows]`` + an all-to-all of the vocabulary shards give. ``hidden`` bf16 [T, 6144] + contiguous (the same on every rank); ``lm_weight`` this rank's [100032, 6144] shard in ``shuffle_weight``'s + layout; ``sorted_rows`` int64 [M] (the owner-sorted rows, 0 <= M <= target_rows, the same M on every rank); + ``counts`` host ints [2] (rows per owner; ``num_local == counts[rank]``).""" + if sorted_rows.shape[0] == 0: # the same on every rank: no exchange + return torch.empty((0, V), dtype=torch.bfloat16, device=hidden.device) + return _target_logits(self, hidden, lm_weight, sorted_rows, list(counts), num_local) + + # ------------------------------------------------------------ the sampler outputs in batch order + def gather_sampler_output(self, tokens, num_sampled, num_rejected, num_nans, cu, src, src_stride: int, + num_reqs: int, width: int, with_nans: bool): + """(tokens int64 [num_reqs, width], num_sampled int32, num_rejected int32, num_nans int32 or None) of every + request in batch order, from this rank's ``tokens`` [n, >= width] int64 / counts [n] / ``cu`` [n + 1] + (``tokens`` None: no local request) and the batch -> (owner, local row) map ``src`` int64 [num_reqs] + (row = owner * src_stride + local). Any integer dtype for the counts; ``num_nans`` is read with + ``with_nans`` only. Every rank calls, the same ``num_reqs``, ``width`` and ``with_nans``.""" + return _gather_sampler_output(self, tokens, num_sampled, num_rejected, num_nans, cu, src, src_stride, + num_reqs, width, with_nans) + + # ------------------------------------------------------------ the Markov draft + def setup_markov(self, steps: int, max_reqs: int, anchor_idx, markov_w1, markov_w2) -> None: + """The draft's exchange, id gather and owner tables of 1..max_reqs requests (collective, before capture). + ``anchor_idx`` [max_reqs] (host or device ints): request i's anchor token row in ``input_ids``.""" + from amd_kernels.collectives.all_gather_i64_c7_ranks2 import I64C7AllGather + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll + + why = markov_blocker(steps, max_reqs, markov_w1, markov_w2) + assert why is None, why + anchor = anchor_idx.cpu().numpy() if isinstance(anchor_idx, torch.Tensor) else np.asarray(anchor_idx) + cap = -(-max_reqs // WORLD) + rows = self._exchange_rows(min(MAX_ROWS, WORLD * cap * steps)) + self.draft_a2a = RowAllToAll(self.rank, self.group, self.device, max_rows=rows) + self.draft_gather = I64C7AllGather(self.rank, self.group, self.device) + self.draft_a2a.warmup() # collective: every rank, here + self.draft_gather.warmup() + self.markov = _Markov( + steps=steps, + max_reqs=max_reqs, + tables={n: owner_tables(n, steps, self.rank, anchor, self.device) for n in range(1, max_reqs + 1)}, + arange=torch.arange(cap, dtype=torch.int64, device=self.device), + w1=markov_w1, + w2=markov_w2, + ) + + def markov_draft(self, num_reqs: int, head_hidden, lm_weight, sample_indices, sample_idx_mapping, sample_pos, + input_ids, temperature, seeds, cache, draft_tokens) -> None: + """The Markov draft of ``num_reqs`` requests (1..max_reqs of ``setup_markov``), every rank the same call: + ``draft_tokens[:num_reqs, :steps]`` = every request's draft ids in batch order, and this rank's requests' + draft-cache rows ``cache[slot, step]``. + + ``head_hidden`` bf16 [T, 6144] contiguous (the draft head's output rows, the same on every rank), + ``lm_weight`` the shuffled lm_head shard, ``sample_indices`` int64 [>= num_reqs * steps] ((request, step) -> + row of head_hidden), ``sample_idx_mapping`` [>= num_reqs * steps] (-> request slot), ``sample_pos`` int64 + (-> draw position + 1), ``input_ids`` (the anchor tokens at ``anchor_idx``), ``temperature`` fp32 / ``seeds`` + int64 [slots], ``cache`` bf16 [slots, steps, 200064], ``draft_tokens`` int64 [>= num_reqs, steps]. The draws + are upstream vLLM's ``_sample_sequential_sharded`` (``gumbel_partials_bias`` at ``sample_pos - 1``).""" + _markov_draft(self, num_reqs, head_hidden, lm_weight, sample_indices, sample_idx_mapping, sample_pos, + input_ids, temperature, seeds, cache, draft_tokens) + + def close(self) -> None: + """Collective teardown after every graph that uses the communicators is retired.""" + for c in (self.target_a2a, self.target_gather, self.draft_a2a, self.draft_gather): + if c is not None: + c.close() + if self.peer is not None: + self.peer.close() + self.peer = None + + +def _local_logits(s: BatchShardedSampling, x, lm_weight, own_rows, out): + """out [n, 200064] = the logits of x[own_rows]: this rank's shard and the peer copy into their vocabulary halves.""" + from amd_kernels.gemm.bf16_n100032_k6144.host import bf16_gemm_n100032_k6144 + + for r, w in ((s.rank, lm_weight), (1 - s.rank, s.peer.weight)): + bf16_gemm_n100032_k6144(x, w, out[:, r * N:(r + 1) * N], shuffled=True, row_indices=own_rows) + + +# ---------------------------------------------------------------- the target logits +def _target_logits(s: BatchShardedSampling, hidden, lm_weight, sorted_rows, counts, num_local): + from amd_kernels.gemm.bf16_n100032_k6144.host import bf16_gemm_n100032_k6144 + + m = sorted_rows.shape[0] + assert 1 <= m <= s.target_rows and len(counts) == WORLD and sum(counts) == m and counts[s.rank] == num_local + dev = hidden.device + out = torch.empty((num_local, V), dtype=torch.bfloat16, device=dev) + if s._local(m, lm_weight): + if num_local: + lo = sum(counts[:s.rank]) + _local_logits(s, hidden, lm_weight, sorted_rows[lo:lo + num_local], out) + return out + cs = _device_counts(counts, dev) + slot, keep = _publish_slot(s.target_a2a, m, dev) + bf16_gemm_n100032_k6144(hidden, lm_weight, shuffled=True, row_indices=sorted_rows, publish=slot) + if keep is None: # the pull's grids compiled in the communicator's warm-up + s.target_a2a.pull(m, cs, out) + return out + + +def _target_mode(s, hidden, lm_weight, sorted_rows, counts, num_local): + return ("target", s.target_rows, s.peer is not None) + + +def _target_sizes(mode): + from amd_kernels.gemm.bf16_n100032_k6144.host import WARMUP_ROWS + + rows = mode[1] + # every lm_head tile variant of each path (the pull's grids compile with the communicator); a local call runs + # owned-row counts, about half of the total + sizes = {min(m, rows) for m in WARMUP_ROWS} + if mode[2]: + sizes |= {min(WORLD * m, rows) for m in WARMUP_ROWS if WORLD * m > LOCAL_MIN_ROWS} + return sorted(sizes) + + +def _target_inputs(mode, n, s, hidden, lm_weight, sorted_rows, counts, num_local): + x = hidden if hidden.shape[0] >= n else torch.zeros((n, H), dtype=torch.bfloat16, device=hidden.device) + c = [n // WORLD + int(r < n % WORLD) for r in range(WORLD)] + rows = torch.arange(n, dtype=torch.int64, device=hidden.device) + return (s, x, lm_weight, rows, c, c[s.rank]), {} + + +_target_logits = entry("sampling/batch_sharded_markov_v200064_h6144_r256_ranks2:target", mode=_target_mode, + sizes=_target_sizes, inputs=_target_inputs)(_target_logits) + + +# ---------------------------------------------------------------- the sampler outputs +def _gather_sampler_output(s, tokens, num_sampled, num_rejected, num_nans, cu, src, src_stride, num_reqs, width, + with_nans): + i32, i64, dev = torch.int32, torch.int64, s.device + assert 1 <= width <= MAX_LOGITS_PER_REQ and num_reqs == src.shape[0] <= MAX_REQS + if tokens is None: # no local requests + tokens = torch.zeros((0, 1), dtype=i64, device=dev) + num_sampled = num_rejected = torch.zeros(0, dtype=i32, device=dev) + num_nans = torch.zeros(0, dtype=i32, device=dev) if with_nans else None + cu = torch.zeros(1, dtype=i32, device=dev) + else: + if tokens.stride(-1) != 1 or tokens.shape[1] > MAX_LOGITS_PER_REQ: + tokens = tokens[:, :MAX_LOGITS_PER_REQ].contiguous() + num_sampled = num_sampled.to(i32).contiguous() + num_rejected = num_rejected.to(i32).contiguous() + num_nans = num_nans.to(i32).contiguous() if with_nans else None + cu = cu.to(i32).contiguous() + out_tokens = torch.empty((num_reqs, MAX_LOGITS_PER_REQ), dtype=i64, device=dev) + out_sampled = torch.empty(num_reqs, dtype=i32, device=dev) + out_rejected = torch.empty(num_reqs, dtype=i32, device=dev) + out_nans = torch.empty(num_reqs, dtype=i32, device=dev) if with_nans else None + s.target_gather(tokens, num_sampled, num_rejected, num_nans, cu, src, src_stride, out_tokens, out_sampled, + out_rejected, out_nans) + sampled = out_tokens if width == MAX_LOGITS_PER_REQ else out_tokens[:, :width].contiguous() + return sampled, out_sampled, out_rejected, out_nans + + +# ---------------------------------------------------------------- the Markov draft +def _markov_draft(s, num_reqs, head_hidden, lm_weight, sample_indices, sample_idx_mapping, sample_pos, input_ids, + temperature, seeds, cache, draft_tokens): + from amd_kernels.gemm.bf16_n100032_k6144.host import bf16_gemm_n100032_k6144 + from amd_kernels.gemm.bf16_n200064_k256.host import bf16_gemm_packed_rows, pack_bf16_rows, padded_rows + from amd_kernels.sampling.gumbel_combine_markov_embed_v200064_r256.host import ( + gumbel_combine_markov_embed_packed, + ) + from amd_kernels.sampling.gumbel_partials_bias_bf16_v200064.host import gumbel_partials_bias + from amd_kernels.sampling.gumbel_partials_bias_bf16_v200064.reference import NB + + d = s.markov + assert d is not None and 1 <= num_reqs <= d.max_reqs + assert head_hidden.dtype == torch.bfloat16 and head_hidden.dim() == 2 and head_hidden.is_contiguous() + t = d.tables[num_reqs] + cap, steps, dev = t.cap, d.steps, head_hidden.device + bf16 = torch.bfloat16 + # 1. the draft lm_head of this rank's requests' rows: over every owner's rows into the exchange, or locally + base = torch.empty((cap * steps, V), dtype=bf16, device=dev) + keep = None + if s._local(WORLD * cap * steps, lm_weight): + _local_logits(s, head_hidden, lm_weight, sample_indices[t.own_rows], base) + else: + rows = sample_indices[t.send_rows] + slot, keep = _publish_slot(s.draft_a2a, rows.numel(), dev) + bf16_gemm_n100032_k6144(head_hidden, lm_weight, shuffled=True, row_indices=rows, publish=slot) + if keep is None: # the pull's grids compiled in the communicator's warm-up + s.draft_a2a.pull(rows.numel(), t.counts, base) + base = base.view(cap, steps, V) + # 2. the local rows' request slots (-1: no request), draw positions and anchor tokens + slots = torch.where(t.valid, sample_idx_mapping[t.slot_rows], -1).to(torch.int32) + pos = sample_pos[t.slot_rows] + prev = input_ids[t.anchor_rows].to(torch.int64) + # 3. the Markov steps: bias GEMM, Gumbel partials (+ cache row), combine (+ id, next packed embedding row) + rows_ar = d.arange[:cap] + packed = torch.empty((padded_rows(cap), RANK), dtype=bf16, device=dev) + pack_bf16_rows(d.w1, prev, packed) + bias = torch.empty((cap, V), dtype=bf16, device=dev) + lmax = torch.empty((cap, NB), dtype=torch.float32, device=dev) + larg = torch.empty((cap, NB), dtype=torch.int64, device=dev) + tokens = torch.empty((cap, MAX_STEPS), dtype=torch.int64, device=dev) + prev_out = torch.empty(cap, dtype=torch.int64, device=dev) + for i in range(steps): + bf16_gemm_packed_rows(packed, d.w2, cap, bias) + gumbel_partials_bias(base[:, i], slots[i], temperature, seeds, pos[i], cache, i, bias, rows_ar, lmax, larg, + pos_offset=-1) # upstream draws at sample_pos - 1 + for lo in range(0, cap, COMBINE_ROWS): # packed rows [lo, lo + 64) are their own 64-row packed block + hi = min(cap, lo + COMBINE_ROWS) + gumbel_combine_markov_embed_packed(lmax[lo:hi], larg[lo:hi], rows_ar[lo:hi], d.w1, tokens[lo:hi], i, + prev_out, packed[lo:lo + padded_rows(hi - lo)]) + # 4. every rank gets every request's drafts, in batch order (the gather compiled in the communicator's warm-up) + if keep is not None: + return + out = draft_tokens[:num_reqs] + if steps == MAX_STEPS: + s.draft_gather(tokens, t.gathered_src, cap, out) + else: + full = torch.empty((num_reqs, MAX_STEPS), dtype=torch.int64, device=dev) + s.draft_gather(tokens, t.gathered_src, cap, full) + out.copy_(full[:, :steps]) + + +def _markov_mode(s, num_reqs, *args): + return ("markov", s.markov.steps, s.markov.max_reqs, s.peer is not None) + + +def _markov_inputs(mode, n, s, num_reqs, head_hidden, lm_weight, sample_indices, sample_idx_mapping, sample_pos, + input_ids, temperature, seeds, cache, draft_tokens): + # the caller's read-only buffers, no request slot (slot -1: the Gumbel kernel writes no cache row), a private + # output; a kernel compiled before the warm-up launches, so nothing it writes may be the caller's + no_slot = torch.full_like(sample_idx_mapping, -1) + out = torch.empty((n, mode[1]), dtype=torch.int64, device=s.device) + return (s, n, head_hidden, lm_weight, sample_indices, no_slot, sample_pos, input_ids, temperature, seeds, cache, + out), {} + + +_markov_draft = entry("sampling/batch_sharded_markov_v200064_h6144_r256_ranks2:markov", mode=_markov_mode, + sizes=lambda mode: range(1, mode[2] + 1), inputs=_markov_inputs)(_markov_draft) diff --git a/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/reference.py b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/reference.py new file mode 100644 index 0000000..7f58989 --- /dev/null +++ b/amd_kernels/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/reference.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The contract of ``BatchShardedSampling`` (``host.py``), on the whole batch in one process. + +The block only sequences the per-kernel entries, so their references are its contract: + +- ``target_logits``: rank r's rows are the owner-sorted rows ``hidden[sorted_rows]`` of owner r, each the full + vocabulary row ``bf16(fp32 hidden_row @ W^T)`` (``bf16_gemm_n100032_k6144``'s reference) with shard s at columns + [s * 100032, (s + 1) * 100032). The exchange and the local path give the same bits. +- ``gather_sampler_output``: batch request i is local row i // 2 of rank i % 2 (``owner_tables``); tokens, counts + and NaN counts are copied (exact). +- ``markov_draft``: for request i (owner i % 2) and step j, ``base = logits(head_hidden[sample_indices[i * steps + + j]])``, ``bias = markov_w2 @ markov_w1[prev]`` (fp32 accumulate, bf16), the cache row ``bf16(base + bias)`` and + the Gumbel draw of ``gumbel_partials_bias``'s reference at position ``sample_pos - 1``; ``prev`` = the drawn id + (step 0: ``input_ids[anchor_idx[i]]``). ``markov_draft_reference`` below is the greedy (temperature 0) case: the + draw is then exact (no noise), the first maximum. + +Tolerance: the cache rows cosine ~1 against this reference (the GEMMs' fp32 K order differs from torch's), the ids +equal where the rows have no near-tie; the tests compare the block with the per-kernel entries bit for bit. +""" + +import torch + +V, H, RANK, WORLD = 200064, 6144, 256, 2 + + +def owner_order_reference(num_reqs: int) -> list[list[int]]: + """The batch positions each rank owns, in its local row order.""" + return [list(range(r, num_reqs, WORLD)) for r in range(WORLD)] + + +def markov_draft_reference(rows: torch.Tensor, w: torch.Tensor, w1: torch.Tensor, w2: torch.Tensor, + prev: torch.Tensor, steps: int): + """Greedy Markov draft of n requests: ``rows`` bf16 [n, steps, 6144] (each step's head row), ``w`` the row-major + lm_head [200064, 6144], ``prev`` int64 [n] (the anchor tokens) -> (ids int64 [n, steps], cache rows bf16 + [n, steps, 200064]).""" + n = rows.shape[0] + ids = torch.empty((n, steps), dtype=torch.int64, device=rows.device) + cache = torch.empty((n, steps, V), dtype=torch.bfloat16, device=rows.device) + prev = prev.clone() + for j in range(steps): + base = (rows[:, j].float() @ w.float().T).to(torch.bfloat16) + bias = (w1[prev].float() @ w2.float().T).to(torch.bfloat16) + y = (base.float() + bias.float()).to(torch.bfloat16) + cache[:, j] = y + prev = y.float().argmax(dim=1) + ids[:, j] = prev + return ids, cache diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/README.md b/amd_kernels/sampling/top_p_mask_f32_v200064/README.md index 1440b3b..1f6ba4a 100644 --- a/amd_kernels/sampling/top_p_mask_f32_v200064/README.md +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/README.md @@ -1,39 +1,44 @@ -# Top-p mask of fp32 logits rows, vocab 200064: vLLM's split top-p chain in one launch +# Top-p mask of fp32 logits rows, vocab 200064: vLLM's top-p without top-k, bit for bit ## What it does Masks the columns outside the top-p nucleus of each row with -inf, in place, bit for bit what vLLM's -`apply_top_k_top_p_triton(logits, None, p)` does at <= 64 rows: its split top-p pipeline (`_topp_sb_stats_kernel`, -five `_topp_sb_step_kernel` rounds, `_topp_sb_mask_kernel`, Triton 3.8 on gfx950), seven launches and about twice -the time of this one. Per row (p < 1): S slices (32 at <= 8 rows, 16 above, by vLLM's -`_topp_split_count` on 256 CUs) give the max, the softmax sum and the min; five rounds of an 8-pivot geometric ladder -bracket the probability where the kept mass reaches p; the mask keeps `prob > pivot` or the max, and boundary -duplicates in column order. Contract (`reference.py`): the same fp32 operations in the same order (Triton's -`tl.exp` / `tl.log` expansions, each `tl.sum`'s butterfly, the two products the compiler fuses), compared exactly. -Rows with p >= 1 stay as they are. Bit equality with the Triton chain needs vLLM, so it is checked outside this -repository (all row counts 1-64; LM rows at T 0.6-1.5, integer ties, -inf columns, p 1 rows, flat rows). -Caller: the M3.1 TP4 sampler (target top-p, `--enable-batch-sharded-sampling` owned rows). +`apply_top_k_top_p_triton(logits, None, p)` does (Triton 3.8 on gfx950). At <= 64 rows that is its split top-p +pipeline (`_topp_sb_stats_kernel`, five `_topp_sb_step_kernel` rounds, `_topp_sb_mask_kernel`), here one launch: S +slices (32 at <= 8 rows, 16 above, vLLM's `_topp_split_count` on 256 CUs) give the max, the softmax sum and the min; +five rounds of an 8-pivot ladder bracket the probability where the kept mass reaches p. Above 64 rows it is the +one-program-per-row `_topk_topp_kernel`: tile-0 statistics, the softmax sum, the outliers (probability above a +Gaussian pivot) compacted in column order, up to 18 bisection rounds on their 4096-position chunks (or on every column +when the outliers hold less than p), then the mask in logit space; one workgroup per row, 1.4-3.4x faster (flat to +peaked rows, 65-640 rows). Both keep boundary duplicates in column order. Contract (`reference.py`): the same fp32 +operations in the same order (Triton's `tl.exp` / `tl.log` expansions, `v_sqrt_f32` for `tl.sqrt`, each `tl.sum`'s +butterfly, the products the compiler fuses), compared exactly. Rows with p >= 1 stay as they are. Bit equality with +vLLM needs vLLM, so it is checked outside this repository (1-64 rows; 65-300 rows: LM rows at T 0.6-1.5, heavy tails, +integer ties, -inf columns, p 1 rows, flat, constant and all -inf rows). Caller: the M3.1 sampler (target top-p, +batch-sharded owned rows) at TP4 and TP2. ## Entry points -- `top_p_mask`: masks the columns outside each row's top-p nucleus with -inf, in place (fp32 `[rows, 200064]`, 1-64 - rows); one workspace per device (as vLLM's), so launches go on one stream; graph safe. An `amd_kernels.warm_up` - entry: the first call compiles both slice counts. -- `top_p_blocker`: why the entry cannot stand in for a model's chain (None: it can). -- `warmup_top_p_mask`: compiles both variants and allocates the workspace; nothing compiles after it. -- `splits_for`: the slice count for a row count and CU count (vLLM's `_topp_split_count`); `host.py` dispatches on it. +- `top_p_mask`: masks the columns outside each row's top-p nucleus with -inf, in place (fp32 `[rows, 200064]`); one + workspace per device up to 64 rows (as vLLM's: launches on one stream), above a caller's buffer (vLLM's) or one + per device; graph safe. An `amd_kernels.warm_up` entry; `host.py` dispatches on the row count. +- `top_p_blocker`: why the entry cannot stand in for a model's top-p (None: it can). +- `warmup_top_p_mask`: compiles every variant; nothing compiles after it. +- `splits_for`: the slice count of the split pipeline (vLLM's `_topp_split_count`); `rows_grid`: the workgroups (and + buffer rows) above 64 rows. - `top_p_mask_reference`: the torch contract. `available`: gfx950 with FlyDSL importable. `V` = 200064. ## Performance -One MI355X; graph replay of 100 distinct inputs (real M3.1 logits rows, `--rows`, T 0.85-1.15, -p 0.95), each a bf16 -> fp32 copy and the mask, copy subtracted; rounds A B B A; us per call; A = vLLM's Triton chain -(7 launches). +One MI355X; graph replay of 100 distinct inputs (20 above 64 rows), each a bf16 -> fp32 copy and the mask, copy +subtracted; us per call; p 0.95. Up to 64 rows real M3.1 logits rows (`--rows`, T 0.85-1.15) against vLLM's split +chain (7 launches); above, LM-like rows (a N(0, 2) body, 64 heads at 6-18) against `_topk_topp_kernel` +(`benchmark_top_p_mask.py` for this entry; vLLM side measured outside the repository). -| Rows | 1 | 4 | 8 | 16 | 24 | 32 | 48 | 64 | -|---|---|---|---|---|---|---|---|---| -| Triton chain | 52.0 | 71.3 | 77.7 | 125.1 | 173.6 | 179.8 | 238.7 | 280.5 | -| this entry | 28.4 | 41.5 | 45.6 | 72.2 | 83.6 | 89.2 | 118.2 | 142.1 | +| Rows | 1 | 8 | 16 | 32 | 64 | 80 | 128 | 160 | 256 | 416 | 640 | +|---|---|---|---|---|---|---|---|---|---|---|---| +| vLLM | 52.0 | 77.7 | 125.1 | 179.8 | 280.5 | 426.3 | 444.9 | 445.8 | 472.1 | 865.5 | 1258.7 | +| this entry | 28.4 | 45.6 | 72.2 | 89.2 | 142.1 | 135.6 | 167.9 | 162.8 | 215.7 | 377.5 | 566.7 | ## Tests diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/__init__.py b/amd_kernels/sampling/top_p_mask_f32_v200064/__init__.py index 2fdf303..529cdd0 100644 --- a/amd_kernels/sampling/top_p_mask_f32_v200064/__init__.py +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/__init__.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Top-p mask of fp32 logits rows, vocab 200064: vLLM's split top-p pipeline (<= 64 rows) in one launch.""" +"""Top-p mask of fp32 logits rows, vocab 200064, bit for bit as vLLM: its split top-p pipeline (<= 64 rows) in one +launch, and its one-program-per-row top-p kernel above.""" def available() -> bool: @@ -16,8 +17,8 @@ def available() -> bool: return True -from .host import top_p_blocker, top_p_mask, warmup_top_p_mask +from .host import rows_grid, top_p_blocker, top_p_mask, warmup_top_p_mask from .reference import SPLIT_MAX_BATCH, V, splits_for, top_p_mask_reference -__all__ = ["SPLIT_MAX_BATCH", "V", "available", "splits_for", "top_p_blocker", "top_p_mask", "top_p_mask_reference", - "warmup_top_p_mask"] +__all__ = ["SPLIT_MAX_BATCH", "V", "available", "rows_grid", "splits_for", "top_p_blocker", "top_p_mask", + "top_p_mask_reference", "warmup_top_p_mask"] diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/host.py b/amd_kernels/sampling/top_p_mask_f32_v200064/host.py index cabbd78..847b2b0 100644 --- a/amd_kernels/sampling/top_p_mask_f32_v200064/host.py +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/host.py @@ -9,6 +9,7 @@ from amd_kernels.warm_up import entry from . import kernel as _kernel +from . import kernel_rows as _kernel_rows from .reference import SPLIT_MAX_BATCH, V, splits_for @@ -55,48 +56,87 @@ def top_p_blocker(vocab_size: int, mask_value: float) -> str | None: return None -def _top_p_mask(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor: +@functools.cache +def _rows_exe(): + return _kernel_rows.compile_top_p_mask_rows() + + +@functools.cache +def _sigma_table(device: torch.device) -> torch.Tensor: + from .sigma_table import NORMAL_CDF_TO_SIGMA + + return torch.tensor(NORMAL_CDF_TO_SIGMA, dtype=torch.float32, device=device) + + +_ROWS_WORKSPACE: dict = {} + + +def rows_grid(n: int, index: int) -> int: + """Workgroups of the > 64-row kernel (vLLM's program count: one per row up to the CU count).""" + return min(n, _num_cu(index)) + + +def _top_p_mask(logits: torch.Tensor, p: torch.Tensor, buffer: torch.Tensor | None = None) -> torch.Tensor: n = logits.shape[0] assert logits.dtype == torch.float32 and logits.dim() == 2 and logits.shape[1] == V and logits.stride(1) == 1 - assert 1 <= n <= SPLIT_MAX_BATCH, n + assert n >= 1, n assert p.dtype == torch.float32 and p.shape == (n,) and p.is_contiguous() + if n > SPLIT_MAX_BATCH: + grid = rows_grid(n, logits.device.index) + if buffer is None: + buffer = _ROWS_WORKSPACE.get(logits.device) + if buffer is None or buffer.shape[0] < grid: + buffer = torch.empty((grid, V), dtype=torch.float32, device=logits.device) + _ROWS_WORKSPACE[logits.device] = buffer + assert buffer.dtype == torch.float32 and buffer.shape[0] >= grid and buffer.shape[1] == V + assert buffer.is_contiguous() + _run_compiled(_rows_exe(), logits.data_ptr(), logits.stride(0), p.data_ptr(), + _sigma_table(logits.device).data_ptr(), buffer.data_ptr(), n, grid, + torch.cuda.current_stream()) + return logits s = splits_for(n, V, _num_cu(logits.device.index)) _run_compiled(_kernel_exe(s), logits.data_ptr(), logits.stride(0), p.data_ptr(), _workspace(logits.device).data_ptr(), n, 8 * s * ((n + 7) // 8), torch.cuda.current_stream()) return logits -def _mode(logits, p): - """One mode: the variant (slice count) follows the row count (``amd_kernels.warm_up``).""" +def _mode(logits, p, buffer=None): + """One mode: the variant (slice count, or the > 64-row kernel) follows the row count (``amd_kernels.warm_up``).""" return () def _sizes(mode): - """The first row count of each slice count.""" + """The first row count of each slice count, and the first of the > 64-row kernel.""" cu = _num_cu(torch.cuda.current_device()) first = {} for n in range(1, SPLIT_MAX_BATCH + 1): first.setdefault(splits_for(n, V, cu), n) - return tuple(first.values()) + return tuple(first.values()) + (SPLIT_MAX_BATCH + 1,) -def _inputs(mode, n, logits, p): +def _inputs(mode, n, logits, p, buffer=None): """Zero rows at p 0.95 (compile-only: nothing launches).""" x = torch.zeros(n, V, dtype=torch.float32, device=logits.device) return (x, torch.full((n,), 0.95, dtype=torch.float32, device=logits.device)), {} top_p_mask = entry("sampling/top_p_mask_f32_v200064", mode=_mode, sizes=_sizes, inputs=_inputs)(_top_p_mask) -top_p_mask.__doc__ = """vLLM's ``apply_top_k_top_p_triton(logits, None, p)`` at <= 64 rows (its split top-p pipeline: -stats, five pivot rounds, mask) in one launch: masks fp32 ``logits`` [n, 200064] (unit column stride, any row stride) -in place with -inf, bit for bit (``reference.py``); ``p`` fp32 [n]; rows with p >= 1 stay as they are. One workspace -per device: launches on one stream (or graphs replayed one after another). CUDA-graph safe. Returns ``logits``.""" +top_p_mask.__doc__ = """vLLM's ``apply_top_k_top_p_triton(logits, None, p)``: masks fp32 ``logits`` [n, 200064] (unit +column stride, any row stride) in place with -inf, bit for bit (``reference.py``); ``p`` fp32 [n]; rows with p >= 1 +stay as they are. At <= 64 rows its split top-p pipeline in one launch (one workspace per device: launches on one +stream, or graphs replayed one after another); above, its one-program-per-row kernel, with ``buffer`` (fp32 +[>= rows_grid(n), 200064], contiguous; vLLM's) as the workspace, else one per device. CUDA-graph safe. Returns +``logits``.""" def warmup_top_p_mask(device: torch.device) -> None: - """Compile every slice count (S 32 at <= 8 rows, S 16 above on 256 CUs) and allocate the workspace, before graph - capture; nothing compiles after it.""" + """Compile every variant (S 32 at <= 8 rows, S 16 above on 256 CUs, the > 64-row kernel) and allocate the split + pipeline's workspace, before graph capture; nothing compiles after it. The > 64-row workspace is freed (a caller + passes its own ``buffer``).""" for n in _sizes(()): x = torch.zeros(n, V, dtype=torch.float32, device=device) top_p_mask(x, torch.full((n,), 0.95, dtype=torch.float32, device=device)) torch.cuda.synchronize(device) + device = torch.device(device) + index = torch.cuda.current_device() if device.index is None else device.index + _ROWS_WORKSPACE.pop(torch.device("cuda", index), None) diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/kernel_rows.py b/amd_kernels/sampling/top_p_mask_f32_v200064/kernel_rows.py new file mode 100644 index 0000000..a1e1ca2 --- /dev/null +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/kernel_rows.py @@ -0,0 +1,900 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""vLLM's one-program-per-row top-p kernel (``_topk_topp_kernel``, top-k off) on more than 64 rows, bit for bit +(``reference.py``, ``top_p_mask_rows_reference``). + +One 512-thread workgroup per row, as Triton's program, so every ``tl.sum`` keeps its layout: thread t holds columns +2048 r + 4 t + j of an 8192-column tile, a 4096-position chunk of the compacted buffer 2048 r + 4 t + j for r < 2. +The tile and chunk sums are the in-thread pairs, the wave's DPP rows and the waves' LDS tree (``reference.py`` +offsets); the 25 tile sums of a pass meet in LDS after the pass and are chained in order. + +Pass 1 stores e = exp(x - max_sample) in the workgroup's workspace slice. Division by the row's exp sum is monotone, +so the outliers (e / se > outlier probability) are exactly e >= e_min, the least such e (found once by a few exact +divisions around outlier probability * se): pass 2 needs no division. It compacts the outliers' e in place (a tile's +positions lie below its columns, which every wave has loaded by the tile's barrier), keeps the maximum of every +256-position block and zeroes the rest of the last chunk (search loads read whole chunks, unmasked). Pass 2's +outlier mass decides the search's branch from a sum with a 1e-4 relative margin (its error bound is under 3e-5); +inside the margin a pass sums Triton's exact probabilities. A search round evaluates only the (chunk, wave) groups +whose maximum probability exceeds the pivot (a group below adds exactly what Triton's masked sum adds: 0.0, the 1.0 +sentinel, a count of 0); the K groups of largest maximum per wave stay in registers as probabilities, the others are +loaded in batches and divided on first use (written back: each wave owns its groups). Minima of probabilities are +integer minima of their bit patterns (non-negative). Rows whose exp sum or outlier probability is not a positive +finite number take Triton's division per element. The mask pass writes -inf without reading a tile whose maximum +cannot be kept. + +Names assigned inside a dynamic loop are unique in the kernel: FlyDSL carries every name a loop body assigns that +an enclosing scope also binds. +""" + +import flydsl.compiler as flyc +import flydsl.expr as fx +from amd_kernels.common import buffer_ops +from amd_kernels.warm_up import launcher +from flydsl._mlir import ir as _ir +from flydsl._mlir.dialects import arith as _arith +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import const_expr, range_constexpr +from flydsl.expr.typing import T +from flydsl.expr.typing import Vector as Vec + +from .kernel import EPS, LN2_HI, LN2_LO, _bf, _bi, _exp, _fbits, _fmax, _fmin, _launder, _lane_val, _popc, _uni +from .kernel import _dpp, _wmax, _wmin, _wsum +from .reference import CHUNK, ITERS, TILE, TILES, V + +F, I, I64 = fx.Float32, fx.Int32, fx.Int64 +NT = 512 +INF = float("inf") +NCH = -(-V // CHUNK) # 49 +NBLK = -(-V // 256) # 782 +PF = 2 # tiles in flight per pass +K = 4 # groups per wave held in registers through the search +BATCH = 4 # groups past them loaded together +LN2X32 = _fbits(0x41B17218) # fp32 32 ln2 +SIGN = -(2**31) +PART = NCH * 8 # search partials per parity: [chunk][wave] +MARGIN = 1e-4 # relative margin of the outlier-mass decision + + +def _sqrt(x): + """v_sqrt_f32 (Triton's tl.sqrt on gfx950).""" + return F(_llvm.call_intrinsic(T.f32, "llvm.amdgcn.sqrt.f32", [fx.as_ir_value(x)], [], [])) + + +def _log_any(x): + """Triton's fp32 ``tl.log`` for any x (below 2^-126 scaled by 2^32, minus 32 ln2; inf / nan pass through).""" + small = x < F(_fbits(0x00800000)) + y = fx.math.log2(small.select(x * F(4294967296.0), x), fastmath=fx.FastMathFlags.afn) + r0 = _launder(y * F(LN2_HI)) + f0 = fx.math.fma(y, F(LN2_HI), r0 * F(-1.0)) + f1 = fx.math.fma(y, F(LN2_LO), f0) + r = fx.math.fma(y, F(LN2_HI), f1) + r = (fx.math.absf(y) < F(INF)).select(r, y) + return r - small.select(F(LN2X32), F(0.0)) + + +def _wtree8(w): + """Eight wave values summed as Triton's cross-wave butterfly (4, 2, 1).""" + return ((w[0] + w[4]) + (w[2] + w[6])) + ((w[1] + w[5]) + (w[3] + w[7])) + + +def _tree16(e): + """In-thread sum of an 8192-tile's 16 registers e[4 r + j] (offsets 2, 2048, 4096, 1).""" + def half(j0): + return (e[j0] + e[j0 + 2] + (e[4 + j0] + e[6 + j0])) + (e[8 + j0] + e[10 + j0] + (e[12 + j0] + e[14 + j0])) + return half(0) + half(1) + + +def _tree8(e): + """In-thread sum of a 4096-chunk's 8 registers e[4 r + j] (offsets 2, 2048, 1).""" + return ((e[0] + e[2]) + (e[4] + e[6])) + ((e[1] + e[3]) + (e[5] + e[7])) + + +def _ult(a, b): + """Unsigned a < b on i32 bit patterns.""" + return (a ^ I(SIGN)) < (b ^ I(SIGN)) + + +def _u2f(v): + """uint32 -> fp32 (uitofp) of an i32 bit pattern.""" + return F(_arith.uitofp(T.f32, fx.as_ir_value(v))) + + +def _ballot(pred): + return I64(fx.rocdl.ballot(T.i64, pred)) + + +def _mbcnt(bal): + """Lanes below this one set in ``bal``.""" + mb = I(_llvm.call_intrinsic(T.i32, "llvm.amdgcn.mbcnt.lo", [bal.to(I).ir_value(), I(0).ir_value()], [], [])) + return I(_llvm.call_intrinsic(T.i32, "llvm.amdgcn.mbcnt.hi", [(bal >> I64(32)).to(I).ir_value(), mb.ir_value()], + [], [])) + + +def _lds_ptr(addr): + return _llvm.inttoptr(_ir.Type.parse("!llvm.ptr<3>"), fx.as_ir_value(I64(addr))) + + +def _lds_umax(addr, v): + _llvm.AtomicRMWOp(_llvm.AtomicBinOp.umax, _lds_ptr(addr), I(v).ir_value(), _llvm.AtomicOrdering.monotonic, + syncscope="workgroup", alignment=4) + + +def _lds_load8(addr): + """8 dwords at LDS byte address ``addr`` (16-byte aligned).""" + out = [] + for h in range(2): + v = Vec(_llvm.load(T.vec(4, T.i32), _lds_ptr(addr + I(16 * h)), alignment=16)) + out += [I(v[k]) for k in range(4)] + return out + + +def _imax(a, b): + return (a > b).select(a, b) + + +def _imin(a, b): + return (a < b).select(a, b) + + +def _wmin_bits(v): + """Wave minimum of non-negative fp32 bit patterns (integer min), uniform; lane 63's DPP chain stays in range.""" + for ctrl in (0x118, 0x114, 0x112, 0x111): + v = _imin(v, _dpp(v, ctrl)) + v = _imin(v, _dpp(v, 0x142, old=v, row_mask=0xA)) + v = _imin(v, _dpp(v, 0x143)) + return _lane_val(v, 63) + + +def _wmax_bits(v): + """Wave maximum of non-negative fp32 bit patterns (integer max; out-of-row DPP lanes read 0), uniform.""" + for ctrl in (0x118, 0x114, 0x112, 0x111): + v = _imax(v, _dpp(v, ctrl)) + v = _imax(v, _dpp(v, 0x142, old=v, row_mask=0xA)) + v = _imax(v, _dpp(v, 0x143)) + return _lane_val(v, 63) + + +def _scan32(v): + """Inclusive prefix sum over lanes 0-31 (integer, DPP rows then the row-15 broadcast).""" + for ctrl in (0x111, 0x112, 0x114, 0x118): + v = v + _dpp(v, ctrl) + return v + _dpp(v, 0x142, old=I(0), row_mask=0xA) + + +def _cols(i, tx): + """Column of each of a thread's 16 registers in tile i.""" + return [i * I(TILE) + I(2048 * (k // 4) + k % 4) + tx * I(4) for k in range(16)] + + +def _load_tile(rsrc, i, tx, fill): + """Tile i's 16 values of this thread, ``fill`` past V (a whole tile past V loads nothing).""" + out = [] + for r in range(4): + off = i * I(TILE) + I(2048 * r) + tx * I(4) + v = Vec(buffer_ops.buffer_load(rsrc, off, vec_width=4, dtype=I)) + for j in range(4): + out.append(((off + I(j)) < I(V)).select(I(v[j]).bitcast(F), F(fill))) + return out + + +def _load_raw(rsrc, i, tx): + """Tile i's 16 values of this thread as loaded (0 past V).""" + out = [] + for r in range(4): + v = Vec(buffer_ops.buffer_load(rsrc, i * I(TILE) + I(2048 * r) + tx * I(4), vec_width=4, dtype=I)) + out += [I(v[j]).bitcast(F) for j in range(4)] + return out + + +def _last_tile(vals, tx, fill): + """The last tile's values with ``fill`` past V.""" + cols = _cols(I(TILES - 1), tx) + return [(cols[k] < I(V)).select(vals[k], F(fill)) for k in range(16)] + + +def _store_tile(rsrc, vals, i, tx): + for r in range(4): + vec = Vec.from_elements([_bi(vals[4 * r + j]) for j in range(4)], I) + buffer_ops.buffer_store(vec, rsrc, i * I(TILE) + I(2048 * r) + tx * I(4)) + + +def _pass1_tile(xt, ms): + """A tile's e, their in-thread sum and the max (the last tile's columns past V are -inf: e 0).""" + e = [_exp(v - ms) for v in xt] + mx = F(-INF) + for v in xt: + mx = _fmax(mx, v) + return e, _tree16(e), mx + + +def _pass1_last(raw, ms, tx): + """The last tile: -inf past V, e 0 there.""" + xl = _last_tile(raw, tx, -INF) + cols = _cols(I(TILES - 1), tx) + e = [(cols[k] < I(V)).select(_exp(xl[k] - ms), F(0.0)) for k in range(16)] + mx = F(-INF) + for v in xl: + mx = _fmax(mx, v) + return e, _tree16(e), mx + + +def _part_counts(flags): + """Per 2048-column part r: the flagged registers of the lanes below this one, and of the wave.""" + before, tot = [], [] + for r in range(4): + b = I(0) + t = I(0) + for j in range(4): + bal = _ballot(flags[4 * r + j]) + b = b + _mbcnt(bal) + t = t + I(fx.math.ctpop(bal).to(I)) + before.append(b) + tot.append(t) + return before, tot + + +def _outliers_fast(et, e_min): + """A tile's outlier flags (e >= e_min > 0; e is 0 past V), this thread's outlier e sum (any order), and the part + counts.""" + om = [v >= e_min for v in et] + acc = F(0.0) + for k in range(16): + acc = acc + om[k].select(et[k], F(0.0)) + before, tot = _part_counts(om) + return om, acc, before, tot + + +def _outliers_exact(xt, ms, se, oprob, i, tx): + """Triton's probabilities of tile i, the outlier flags, the in-thread outlier sum and the part counts.""" + cols = _cols(i, tx) + pr = [_exp(v - ms) / se for v in xt] + om = [(pr[k] > oprob) & (cols[k] < I(V)) for k in range(16)] + ts = _tree16([om[k].select(pr[k], F(0.0)) for k in range(16)]) + before, tot = _part_counts(om) + return pr, om, ts, before, tot + + +def _write_counts(arr, par, wave, tot): + for r in range(4): + arr.ptr[par + I(8 * r) + wave] = tot[r] + + +def _bases(arr, par, run, lane, wave): + """Per part r: the first position of this wave's run and its length (from the tile's 32 wave counts [r][w] in + LDS, prefix-summed across lanes); and the new running total.""" + cw = (lane < I(32)).select(I(arr.ptr[par + (lane & I(31))]), I(0)) + inc = _scan32(cw) + exc = inc - cw + base = [run + _lane_val(exc, I(8 * r) + wave) for r in range(4)] + own = [_lane_val(cw, I(8 * r) + wave) for r in range(4)] + return base, own, run + _lane_val(inc, I(31)) + + +def _compact(buf, vals, om, before, base): + """Store the outliers at their compacted positions; per part the maximum of this wave's run (fp32 bits).""" + runs = [] + for r in range(4): + cnt = I(0) + m = I(0) + for j in range(4): + k = 4 * r + j + buffer_ops.buffer_store(_bi(vals[k]), buf, om[k].select(base[r] + before[r] + cnt, I(V))) + m = _imax(m, (om[k] & (vals[k] > F(0.0))).select(_bi(vals[k]), I(0))) + cnt = cnt + om[k].select(I(1), I(0)) + runs.append(_wmax_bits(m)) + return runs + + +def _block_maxima(bmax_base, runs, base, own): + """Lane 0: fold each run's maximum into the 256-position blocks it touches (an upper bound per block).""" + for r in range(4): + b_lo = base[r] >> I(8) + b_hi = (base[r] + own[r] - I(1)) >> I(8) + a = (own[r] > I(0)).select(bmax_base + b_lo * I(4), bmax_base + I((NBLK + 2) * 4)) + _lds_umax(a, runs[r]) + _lds_umax((b_hi != b_lo).select(a + I(4), bmax_base + I((NBLK + 2) * 4)), runs[r]) + + +def _load_group(buf, q, wave, lane, b1): + """Group (chunk q, wave): this lane's 8 (compacted) or 16 (full) registers; the buffer reads 0 past the data + (zero-filled to the chunk end, or past V).""" + vals = [] + size = CHUNK if b1 else TILE + for r in range(2 if b1 else 4): + v = Vec(buffer_ops.buffer_load(buf, q * I(size) + I(2048 * r) + wave * I(256) + lane * I(4), vec_width=4, + dtype=I)) + vals += [I(v[j]).bitcast(F) for j in range(4)] + return vals + + +def _store_group(buf, q, wave, lane, vals, b1): + size = CHUNK if b1 else TILE + for r in range(2 if b1 else 4): + vec = Vec.from_elements([_bi(vals[4 * r + j]) for j in range(4)], I) + buffer_ops.buffer_store(vec, buf, q * I(size) + I(2048 * r) + wave * I(256) + lane * I(4)) + + +def _bit(mask, q): + return ((mask >> I64(q)) & I64(1)) != I64(0) + + +def _probs(vals, se, div): + """Triton's probabilities of stored e (``div``) or stored probabilities.""" + return [div.select(v / se, v) for v in vals] + + +def _group_eval(vals, piv, b1): + """The wave's share of Triton's chunk statistics at ``piv``: the masked sum, the minimum above (1.0 if none; an + integer min of the non-negative bit patterns) and the count within 1e-9 of that minimum.""" + ab = [v > piv for v in vals] + contrib = [ab[k].select(vals[k], F(0.0)) for k in range(len(vals))] + sw = _wsum(_tree8(contrib) if b1 else _tree16(contrib)) + m = I(0x3F800000) + for k in range(len(vals)): + m = _imin(m, ab[k].select(_bi(vals[k]), I(0x3F800000))) + mw = _bf(_wmin_bits(m)) + c = I(0) + for k in range(len(vals)): + c = c + _popc(ab[k] & (fx.math.absf(vals[k] - mw) < F(EPS))) + return sw, mw, c + + +def _group_count(vals, piv, mref): + c = I(0) + for v in vals: + c = c + _popc((v > piv) & (fx.math.absf(v - mref) < F(EPS))) + return c + + +def _write_part(smem, o, s, m, c): + smem.psum.ptr[o] = _bi(s) + smem.pmin.ptr[o] = _bi(m) + smem.pcnt.ptr[o] = c + + +def _chunk_stats(ps, pm, pc, o, lane, nq): + """Lane q: chunk q's sum (the waves' tree), minimum, count, and whether a wave minimum lies within 1e-9 of the + chunk minimum without being equal (the count must then be taken against the chunk minimum). ``ps``, ``pm``, + ``pc``: LDS byte addresses of the [chunk][wave] sums, minima, counts; ``o`` the byte offset of chunk q's row.""" + ws = [_bf(v) for v in _lds_load8(ps + o)] + wm = [_bf(v) for v in _lds_load8(pm + o)] + wc = _lds_load8(pc + o) + m = F(1.0) + for w in range(8): + m = _fmin(m, wm[w]) + n = I(0) + near = I(0) + for w in range(8): + n = n + (wm[w] == m).select(wc[w], I(0)) + near = near | ((wm[w] != m) & (fx.math.absf(wm[w] - m) < F(EPS))).select(I(1), I(0)) + ok = lane < nq + return _wtree8(ws), ok.select(m, F(1.0)), ok.select(n, I(0)), ok.select(near, I(0)) + + +def _sum_rec(arr, o): + n = I(0) + for w in range(8): + n = n + I(arr.ptr[o + I(w)]) + return n + + +def _merge(s0, m0, n0, sq, mq, nq): + """One step of Triton's chunk loop: the sum, and ``_update_min_larger_stats``.""" + n0 = (mq < m0).select(nq, n0 + (fx.math.absf(mq - m0) < F(EPS)).select(nq, I(0))) + return s0 + sq, _fmin(m0, mq), n0 + + +def _decide(st, o, s0, m0, n0, piv, p, lo, hi, ppiv, mlp, nml, psum, it): + """One round's end in Triton's order (the found test, the bracket update, the stop test) into LDS ``st[o:o+7]``: + lo, hi, done, ppiv, mlp, nml, psum.""" + found = (s0 >= p) & (fx.math.fma(m0 * F(-1.0), _u2f(n0), s0) < p) + ppiv = found.select(piv, ppiv) + mlp = found.select(m0, mlp) + nml = found.select(n0, nml) + psum = found.select(s0, psum) + lo2 = (s0 > p).select(piv, lo) + hi2 = (~(s0 > p) & (s0 < p)).select(piv, hi) + stop = ((hi2 - lo2) < F(EPS)) | (it == I(ITERS - 1)) + ppiv = stop.select((hi2 + lo2) / F(2.0), ppiv) + mlp = stop.select(m0, mlp) + nml = stop.select(n0, nml) + psum = stop.select(s0, psum) + for k, v in enumerate((_bi(lo2), _bi(hi2), (found | stop).select(I(1), I(0)), _bi(ppiv), _bi(mlp), nml, + _bi(psum))): + st.ptr[o + I(k)] = v + + +def _take(mask): + """The lowest BATCH set lanes of ``mask`` (64 for none: a group past V loads zeros) and the rest of the mask.""" + qs = [] + for _ in range(BATCH): + qs.append((mask != I64(0)).select(I(fx.math.cttz(mask).to(I)), I(64))) + mask = mask & (mask - I64(1)) + return qs, mask + + +def _chain_start(): + return F(0.0), F(1.0), I(0) + + +def _publish_near(smem, o, lane, near, mq): + """Wave 0: the chunks needing a recount (a 64-bit lane mask in ``st[o+8:o+10]``) and every chunk minimum.""" + bal = _ballot(near != I(0)) + smem.st.ptr[o + I(8)] = bal.to(I) + smem.st.ptr[o + I(9)] = (bal >> I64(32)).to(I) + smem.cmin.ptr[lane] = _bi(mq) + + +def _read_flags(st, o): + lo = _uni(I(st.ptr[o + I(8)])) + hi = _uni(I(st.ptr[o + I(9)])) + return (I64(hi) << I64(32)) | (I64(lo) & I64(0xFFFFFFFF)) + + +def _read_state(st, o): + w = [I(st.ptr[o + I(k)]) for k in range(7)] + return (_uni(_bf(w[0])), _uni(_bf(w[1])), _uni(w[2]), _uni(_bf(w[3])), _uni(_bf(w[4])), _uni(w[5]), + _uni(_bf(w[6]))) + + +def _e_threshold(se, oprob): + """(e_min, ok): the least fp32 e with fl(e / se) > oprob, from fl(oprob * se) and at most 4 steps either way; + ok only for a positive finite se and oprob and a walk that ends at the boundary.""" + def q(e): + return e / se + + def nxt(e, d): + return _bf(_bi(e) + I(d)) + + e = oprob * se + for _ in range(4): + c = nxt(e, -1) + e = ((c > F(0.0)) & (q(c) > oprob)).select(c, e) + for _ in range(4): + e = (q(e) <= oprob).select(nxt(e, 1), e) + fin = (se > F(0.0)) & (se < F(INF)) & (oprob > F(0.0)) & (oprob < F(INF)) + ok = fin & (e > F(_fbits(0x00800000))) & (e < F(INF)) & (q(e) > oprob) & ~(q(nxt(e, -1)) > oprob) + return e, ok + + +def _dup_flags(xt, dup, i, tx): + cols = _cols(i, tx) + dm = [(fx.math.absf(xt[k] - dup) < F(EPS)) & (cols[k] < I(V)) for k in range(16)] + before, tot = _part_counts(dm) + return dm, before, tot + + +def _masked(xt, keep): + return [keep[k].select(xt[k], F(-INF)) for k in range(16)] + + +def _keep(xt, fpiv): + return [v > fpiv for v in xt] + + +def _keep_ties(keep, dm, before, base, nkeep): + out = list(keep) + for r in range(4): + own = I(0) + for j in range(4): + k = 4 * r + j + own = own + dm[k].select(I(1), I(0)) + out[k] = keep[k] & ~(dm[k] & ((base[r] + before[r] + own) > nkeep)) + return out + + +def _probs_tile(xt, ms, se, i, tx): + """Branch 2: Triton's probabilities of tile i, their maximum (columns past V excluded) and the finite min.""" + cols = _cols(i, tx) + pr = [_exp(v - ms) / se for v in xt] + gm = F(0.0) + mn = F(INF) + for k in range(16): + gm = _fmax(gm, ((pr[k] > F(0.0)) & (cols[k] < I(V))).select(pr[k], F(0.0))) + mn = _fmin(mn, (xt[k] > F(-INF)).select(xt[k], F(INF))) + return pr, gm, mn + + +@launcher +def compile_top_p_mask_rows(path: str = "auto"): + """``path`` (tests): "exact" takes Triton's division per element on every row, "mass" sums Triton's exact outlier + mass on every row.""" + assert path in ("auto", "exact", "mass") + name = "top_p_mask_rows_f32_v200064" + ("" if path == "auto" else "_" + path) + + @fx.struct + class Shared: + cnt: fx.Array[fx.Int32, 2 * 32] + psum: fx.Array[fx.Int32, 2 * PART] + pmin: fx.Array[fx.Int32, 2 * PART] + pcnt: fx.Array[fx.Int32, 2 * PART] + wp1: fx.Array[fx.Int32, TILES * 8] + wp2: fx.Array[fx.Int32, TILES * 8] + tmx: fx.Array[fx.Int32, TILES * 8] + red: fx.Array[fx.Int32, 6 * 8] + bmax: fx.Array[fx.Int32, NBLK + 4] + rec: fx.Array[fx.Int32, 2 * NCH * 8] + st: fx.Array[fx.Int32, 2 * 16] + cmin: fx.Array[fx.Int32, 64] + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel( + arg_x: fx.Int64, # fp32 [n, >= V] logits, masked in place + i32_sx: fx.Int32, # row stride (elements) + arg_p: fx.Int64, # fp32 [n] top-p + arg_tab: fx.Int64, # fp32 [200] sigma table + arg_buf: fx.Int64, # fp32 [grid, V] workspace + i32_n: fx.Int32, + i32_grid: fx.Int32, + ): + tx = I(fx.thread_idx.x) + lane = tx % 64 + wave = I(fx.rocdl.readfirstlane(T.i32, tx // 64)) + smem = fx.SharedAllocator().allocate(Shared).peek() + pid = I(fx.block_idx.x) + cnt_base = I(fx.ptrtoint(smem.cnt.ptr)) + ps_base = I(fx.ptrtoint(smem.psum.ptr)) + pm_base = I(fx.ptrtoint(smem.pmin.ptr)) + pc_base = I(fx.ptrtoint(smem.pcnt.ptr)) + bmax_base = I(fx.ptrtoint(smem.bmax.ptr)) + buf = buffer_ops.create_buffer_resource_from_addr(arg_buf + I64(pid) * (V * 4), num_records_bytes=V * 4) + tab = buffer_ops.create_buffer_resource_from_addr(arg_tab, num_records_bytes=200 * 4) + is_l0 = lane == 0 + + for row in range(pid, i32_n, I(i32_grid)): + rw = I(row) + pres = buffer_ops.create_buffer_resource_from_addr(arg_p + I64(rw) * 4, num_records_bytes=4) + p = _uni(I(buffer_ops.buffer_load(pres, 0, vec_width=1, dtype=I)).bitcast(F)) + xr = buffer_ops.create_buffer_resource_from_addr(arg_x + I64(rw) * I64(i32_sx) * 4, + num_records_bytes=V * 4) + if p < F(1.0): + # ---- tile 0 statistics -> max_sample + pend1 = [_load_raw(xr, I(t_), tx) for t_ in range_constexpr(PF)] + x0 = pend1[0] + fl = [(v > F(-INF)).select(v, F(0.0)) for v in x0] + nfw = I(0) + for v in x0: + nfw = nfw + _popc(v > F(-INF)) + s1w = _wsum(_tree16(fl)) + s2w = _wsum(_tree16([_launder(v * v) for v in fl])) + if is_l0: + smem.red.ptr[wave] = _bi(s1w) + smem.red.ptr[8 + wave] = _bi(s2w) + smem.red.ptr[16 + wave] = nfw + fx.gpu.barrier() + s1 = _uni(_wtree8([_bf(I(smem.red.ptr[w_])) for w_ in range_constexpr(8)])) + s2 = _uni(_wtree8([_bf(I(smem.red.ptr[8 + w_])) for w_ in range_constexpr(8)])) + nf = I(0) + for w_ in range_constexpr(8): + nf = nf + I(smem.red.ptr[16 + w_]) + nf = _uni(nf) + has = nf > I(0) + avg = has.select(s1 / nf.to(F), F(0.0)) + sq = has.select(s2 / nf.to(F), F(0.0)) + std = _uni(_sqrt(_fmax(fx.math.fma(avg * F(-1.0), avg, sq), F(0.0)))) + ms = _uni(fx.math.fma(std, F(10.0), avg)) + + # ---- pass 1: e to the workspace, tile sums of e (chained after the pass), max, finite min, tile max + for t1 in range(TILES - 1): + i1 = I(t1) + xt1 = pend1[0] + pend1 = pend1[1:] + [_load_raw(xr, i1 + I(PF), tx)] + e1, ts1, tmx1 = _pass1_tile(xt1, ms) + _store_tile(buf, e1, i1, tx) + sw1 = _wsum(ts1) + mw1 = _wmax(tmx1) + if is_l0: + smem.wp1.ptr[i1 * I(8) + wave] = _bi(sw1) + smem.tmx.ptr[i1 * I(8) + wave] = _bi(mw1) + el, tsl, tmxl = _pass1_last(pend1[0], ms, tx) + _store_tile(buf, el, I(TILES - 1), tx) + swl = _wsum(tsl) + mwl = _wmax(tmxl) + if is_l0: + smem.wp1.ptr[(TILES - 1) * 8 + wave] = _bi(swl) + smem.tmx.ptr[(TILES - 1) * 8 + wave] = _bi(mwl) + for b_ in range_constexpr(-(-(NBLK + 4) // NT)): + if tx + b_ * NT < NBLK + 4: + smem.bmax.ptr[tx + b_ * NT] = I(0) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + se = F(0.0) + for t_ in range_constexpr(TILES): + se = se + _wtree8([_bf(I(smem.wp1.ptr[t_ * 8 + w_])) for w_ in range_constexpr(8)]) + se = _uni(se) + tmaxl = F(-INF) + if lane < TILES: + for w_ in range_constexpr(8): + tmaxl = _fmax(tmaxl, _bf(I(smem.tmx.ptr[lane * 8 + w_]))) + max_logit = _uni(_wmax(tmaxl)) + idx = (p * F(200.0)).to(I) + idx = _uni((idx < I(0)).select(I(0), (idx > I(199)).select(I(199), idx))) + sig = I(buffer_ops.buffer_load(tab, idx, vec_width=1, dtype=I)).bitcast(F) + sig = _uni(sig + _launder(fx.math.absf(sig) * F(-0.25))) + oprob = _uni(_exp(fx.math.fma(std, sig, avg) - ms) / se) + hi0 = _uni(_exp(max_logit - ms) / se) + e_min, fast = _e_threshold(se, oprob) + if const_expr(path == "exact"): + fast = F(0.0) > F(1.0) + e_min = _uni(e_min) + + # ---- pass 2: the outliers, compacted in place (e, or Triton's probabilities off the fast path) + run2 = I(0) + acc2 = F(0.0) + if fast: + pend2 = [_load_raw(buf, I(t_), tx) for t_ in range_constexpr(PF)] + for t2 in range(TILES): + i2 = I(t2) + et2 = pend2[0] + pend2 = pend2[1:] + [_load_raw(buf, i2 + I(PF), tx)] + om2, a2, bef2, tot2 = _outliers_fast(et2, e_min) + acc2 = acc2 + a2 + par2 = (i2 % I(2)) * I(32) + if is_l0: + _write_counts(smem.cnt, par2, wave, tot2) + fx.gpu.barrier() + base2, own2, run2n = _bases(smem.cnt, par2, run2, lane, wave) + runs2 = _compact(buf, et2, om2, bef2, base2) + if is_l0: + _block_maxima(bmax_base, runs2, base2, own2) + run2 = run2n + else: + pend3 = [_load_tile(xr, I(t_), tx, -INF) for t_ in range_constexpr(PF)] + for t3 in range(TILES): + i3 = I(t3) + xt3 = pend3[0] + pend3 = pend3[1:] + [_load_tile(xr, i3 + I(PF), tx, -INF)] + pr3, om3, ts3, bef3, tot3 = _outliers_exact(xt3, ms, se, oprob, i3, tx) + sw3 = _wsum(ts3) + par3 = (i3 % I(2)) * I(32) + if is_l0: + smem.wp2.ptr[i3 * I(8) + wave] = _bi(sw3) + _write_counts(smem.cnt, par3, wave, tot3) + fx.gpu.barrier() + base3, own3, run3n = _bases(smem.cnt, par3, run2, lane, wave) + runs3 = _compact(buf, pr3, om3, bef3, base3) + if is_l0: + _block_maxima(bmax_base, runs3, base3, own3) + run2 = run3n + nout = _uni(run2) + accw = _wsum(acc2) + if is_l0: + smem.red.ptr[40 + wave] = _bi(accw) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + # zeros from the last outlier to the end of its chunk: the search loads whole chunks + for z_ in range_constexpr(CHUNK // NT): + zpos = nout + tx + I(z_ * NT) + buffer_ops.buffer_store(I(0), buf, (zpos < ((nout + I(CHUNK - 1)) // I(CHUNK)) * I(CHUNK)).select( + zpos, I(V))) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + # the outlier mass: Triton's (exact) off the fast path; on it, sum e / se decides outside the margin + so = F(0.0) + for t_ in range_constexpr(TILES): + so = so + _wtree8([_bf(I(smem.wp2.ptr[t_ * 8 + w_])) for w_ in range_constexpr(8)]) + se_out = F(0.0) + for w_ in range_constexpr(8): + se_out = se_out + _bf(I(smem.red.ptr[40 + w_])) + so_fast = se_out / se + sure = fx.math.absf(so_fast - p) > F(MARGIN) * _fmax(so_fast, p) + if const_expr(path == "mass"): + sure = F(0.0) > F(1.0) + b1 = _uni(fast.select((so_fast > p).select(I(1), I(0)), (so > p).select(I(1), I(0)))) + if fast & ~sure: + for t6 in range(TILES): + i6 = I(t6) + pr6, om6, ts6, bef6, tot6 = _outliers_exact(_load_tile(xr, i6, tx, -INF), ms, se, oprob, i6, + tx) + sw6 = _wsum(ts6) + if is_l0: + smem.wp2.ptr[i6 * I(8) + wave] = _bi(sw6) + fx.gpu.barrier() + so6 = F(0.0) + for t_ in range_constexpr(TILES): + so6 = so6 + _wtree8([_bf(I(smem.wp2.ptr[t_ * 8 + w_])) for w_ in range_constexpr(8)]) + b1 = _uni((so6 > p).select(I(1), I(0))) + + # ---- the pivot search + def search(b1, lo, div, smem=smem, buf=buf, lane=lane, wave=wave, is_l0=is_l0, nout=nout, hi0=hi0, + p=p, se=se, ps_base=ps_base, pm_base=pm_base, pc_base=pc_base): + """(ppiv, mlp, nml, psum) of the search over the compacted outliers (b1) or all probabilities; + ``div``: the buffer holds e. Lane q holds the maximum probability of this wave's group of chunk + q; with b1 the K groups of largest maximum stay in registers and a round loads only the active + groups past them.""" + if const_expr(b1): + nq = (nout + I(CHUNK - 1)) // I(CHUNK) + gml = _fmax(_bf(I(smem.bmax.ptr[lane * 16 + wave])), + _bf(I(smem.bmax.ptr[lane * 16 + 8 + wave]))) + else: + nq = I(TILES) + gml = _bf(I(smem.bmax.ptr[lane * 8 + wave])) + gml = (lane < nq).select(div.select(gml / se, gml), F(0.0)) + slot_q, slot_g, slot_v = [], [], [] + inslot = I(0) + if const_expr(b1): + avail = gml + for k_ in range_constexpr(K): + gk = _wmax(avail) + qk = I(fx.math.cttz(_ballot((avail == gk) & (gk > F(0.0))) | I64(-(1 << 63))).to(I)) + avail = (lane == qk).select(F(0.0), avail) + inslot = inslot | (lane == qk).select(I(1), I(0)) + slot_q.append(qk) + slot_g.append(gk) + slot_v = [_probs(_load_group(buf, slot_q[k_], wave, lane, True), se, div) + for k_ in range_constexpr(K)] + hi = hi0 + done = I(0) + conv = I64(0) + ppiv = F(1.0) + mlp = F(1.0) + nml = I(0) + psum = F(0.0) + for it in range(ITERS): + if done == I(0): + sr_it = I(it) + sr_piv = _uni((hi - lo) * F(0.5) + lo) + sr_par = (sr_it % I(2)) * I(PART) + sr_gact = (lane < nq) & (gml > sr_piv) + if (lane < nq) & ~sr_gact: + _write_part(smem, sr_par + lane * I(8) + wave, F(0.0), F(1.0), I(0)) + for kq_ in range_constexpr(K if b1 else 0): + if slot_g[kq_] > sr_piv: + sr_sw, sr_mw, sr_c = _group_eval(slot_v[kq_], sr_piv, b1) + if is_l0: + _write_part(smem, sr_par + slot_q[kq_] * I(8) + wave, sr_sw, sr_mw, sr_c) + sr_rem = _ballot(sr_gact & (inslot == I(0))) + while sr_rem != I64(0): + sr_qs, sr_rem = _take(sr_rem) + sr_raw = [_load_group(buf, q_, wave, lane, b1) for q_ in sr_qs] + for kb_ in range_constexpr(BATCH): + if sr_qs[kb_] < I(64): + # e -> probabilities once per group, written back (this wave's own group) + sr_pv = sr_raw[kb_] + if div & ~_bit(conv, sr_qs[kb_]): + sr_pv = [v / se for v in sr_raw[kb_]] + _store_group(buf, sr_qs[kb_], wave, lane, sr_pv, b1) + conv = conv | (I64(1) << I64(sr_qs[kb_])) + sr_sw2, sr_mw2, sr_c2 = _group_eval(sr_pv, sr_piv, b1) + if is_l0: + _write_part(smem, sr_par + sr_qs[kb_] * I(8) + wave, sr_sw2, sr_mw2, + sr_c2) + fx.gpu.barrier() + sr_o = (sr_par + lane * I(8)) * I(4) + sr_stb = (sr_it % I(2)) * I(16) + # wave 0: the chunk statistics, the chain over the active chunks and the decision + if wave == I(0): + sr_sq, sr_mq, sr_nq, sr_near = _chunk_stats(ps_base, pm_base, pc_base, sr_o, lane, + nq) + sr_s0, sr_m0, sr_n0 = _chain_start() + sr_act = _ballot(sr_nq > I(0)) + while sr_act != I64(0): + sr_q4 = I(fx.math.cttz(sr_act).to(I)) + sr_act = sr_act & (sr_act - I64(1)) + sr_s0, sr_m0, sr_n0 = _merge(sr_s0, sr_m0, sr_n0, _lane_val(sr_sq, sr_q4), + _lane_val(sr_mq, sr_q4), _lane_val(sr_nq, sr_q4)) + _decide(smem.st, sr_stb, sr_s0, sr_m0, sr_n0, sr_piv, p, lo, hi, ppiv, mlp, nml, psum, + sr_it) + _publish_near(smem, sr_stb, lane, sr_near, sr_mq) + fx.gpu.barrier() + sr_flags = _read_flags(smem.st, sr_stb) + if sr_flags != I64(0): + # a wave minimum within 1e-9 of its chunk minimum but not equal: recount those chunks + # against the chunk minimum (rare) + sr_recb = (sr_it % I(2)) * I(NCH * 8) + sr_rem3 = sr_flags + while sr_rem3 != I64(0): + sr_q3 = I(fx.math.cttz(sr_rem3).to(I)) + sr_rem3 = sr_rem3 & (sr_rem3 - I64(1)) + sr_c3 = _group_count( + _probs(_load_group(buf, sr_q3, wave, lane, b1), se, + div & ~_bit(conv, sr_q3)), sr_piv, _bf(I(smem.cmin.ptr[sr_q3]))) + if is_l0: + smem.rec.ptr[sr_recb + sr_q3 * I(8) + wave] = sr_c3 + fx.gpu.barrier() + if wave == I(0): + sr_sq5, sr_mq5, sr_nq5, sr_near5 = _chunk_stats(ps_base, pm_base, pc_base, sr_o, + lane, nq) + sr_nq5 = (sr_near5 != I(0)).select(_sum_rec(smem.rec, sr_recb + lane * I(8)), + sr_nq5) + sr_s5, sr_m5, sr_n5 = _chain_start() + sr_act5 = _ballot(sr_nq5 > I(0)) + while sr_act5 != I64(0): + sr_q5 = I(fx.math.cttz(sr_act5).to(I)) + sr_act5 = sr_act5 & (sr_act5 - I64(1)) + sr_s5, sr_m5, sr_n5 = _merge(sr_s5, sr_m5, sr_n5, _lane_val(sr_sq5, sr_q5), + _lane_val(sr_mq5, sr_q5), _lane_val(sr_nq5, sr_q5)) + _decide(smem.st, sr_stb, sr_s5, sr_m5, sr_n5, sr_piv, p, lo, hi, ppiv, mlp, nml, + psum, sr_it) + fx.gpu.barrier() + lo, hi, done, ppiv, mlp, nml, psum = _read_state(smem.st, sr_stb) + return ppiv, mlp, nml, psum + + ppiv = F(1.0) + mlp = F(1.0) + nml = I(0) + psum = F(0.0) + if b1 != I(0): + ppiv, mlp, nml, psum = search(True, oprob, fast) + else: + # Triton's probabilities of every column in column order and the (tile, wave) maxima + mn7 = F(INF) + for t7 in range(TILES): + i7 = I(t7) + pr7, gm7, tmn7 = _probs_tile(_load_tile(xr, i7, tx, -INF), ms, se, i7, tx) + _store_tile(buf, pr7, i7, tx) + gw7 = _wmax(gm7) + if is_l0: + smem.bmax.ptr[i7 * I(8) + wave] = _bi(gw7) + mn7 = _fmin(mn7, tmn7) + mnw7 = _wmin(mn7) + if is_l0: + smem.red.ptr[32 + wave] = _bi(mnw7) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + min_logit = F(INF) + for w_ in range_constexpr(8): + min_logit = _fmin(min_logit, _bf(I(smem.red.ptr[32 + w_]))) + min_logit = _uni(_fmin(min_logit, max_logit)) + ppiv, mlp, nml, psum = search(False, _uni(_exp(min_logit - ms) / se), F(0.0) > F(1.0)) + + # ---- the mask (Triton's sixth pass): keep x > fpiv; with ties the boundary duplicates past nkeep + # in column order go too. A tile whose maximum is neither kept nor a duplicate becomes -inf unread. + dup = _uni(_log_any(mlp * se) + ms) + nkeep = nml - ((psum - p) / mlp).to(fx.Uint32).to(I) + fpiv = _uni(_log_any(ppiv * se) + ms) + if (fpiv < max_logit) & (fpiv != F(-INF)): + ties = _ult(nkeep, nml) + clear = [F(-INF) for _ in range_constexpr(16)] + kept8 = I(0) + for t8 in range(TILES): + i8 = I(t8) + tm8 = _lane_val(tmaxl, i8) + far8 = ~((fx.math.absf(tm8 - dup) < F(EPS)) | (tm8 > dup)) + if (tm8 > fpiv) | (ties & ~far8): + xt8 = _load_raw(xr, i8, tx) + keep8 = _keep(xt8, fpiv) + if ties: + dm8, bef8, tot8 = _dup_flags(xt8, dup, i8, tx) + par8 = (i8 % I(2)) * I(32) + if is_l0: + _write_counts(smem.cnt, par8, wave, tot8) + fx.gpu.barrier() + base8, _own8, kept8n = _bases(smem.cnt, par8, kept8, lane, wave) + keep8 = _keep_ties(keep8, dm8, bef8, base8, nkeep) + kept8 = kept8n + _store_tile(xr, _masked(xt8, keep8), i8, tx) + else: + _store_tile(xr, clear, i8, tx) + fx.rocdl.s_waitcnt(vmcnt=0) + fx.gpu.barrier() + + @flyc.jit + def launch_top_p_mask_rows(arg_x: fx.Int64, i32_sx: fx.Int32, arg_p: fx.Int64, arg_tab: fx.Int64, + arg_buf: fx.Int64, i32_n: fx.Int32, i32_grid: fx.Int32, + stream: fx.Stream): + kernel(arg_x, i32_sx, arg_p, arg_tab, arg_buf, i32_n, i32_grid, + value_attrs={"rocdl.waves_per_eu": 2, "rocdl.flat_work_group_size": f"{NT},{NT}"}).launch( + grid=(fx.Int64(i32_grid), 1, 1), block=(NT, 1, 1), stream=stream) + + return launch_top_p_mask_rows + + +@launcher +def compile_sqrt_f32(): + """Elementwise v_sqrt_f32 (the reference's ``sqrt_f32``).""" + + @flyc.kernel(name="v_sqrt_f32", known_block_size=[256, 1, 1]) + def kernel(arg_x: fx.Int64, arg_y: fx.Int64, i32_n: fx.Int32): + i = I(fx.block_idx.x) * I(256) + I(fx.thread_idx.x) + xr = buffer_ops.create_buffer_resource_from_addr(arg_x) + yr = buffer_ops.create_buffer_resource_from_addr(arg_y) + if i < i32_n: + v = I(buffer_ops.buffer_load(xr, i, vec_width=1, dtype=I)).bitcast(F) + buffer_ops.buffer_store(_bi(_sqrt(v)), yr, i) + + @flyc.jit + def launch_sqrt_f32(arg_x: fx.Int64, arg_y: fx.Int64, i32_n: fx.Int32, i32_grid: fx.Int32, stream: fx.Stream): + kernel(arg_x, arg_y, i32_n).launch(grid=(fx.Int64(i32_grid), 1, 1), block=(256, 1, 1), stream=stream) + + return launch_sqrt_f32 diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/reference.py b/amd_kernels/sampling/top_p_mask_f32_v200064/reference.py index da7f219..65adcc8 100644 --- a/amd_kernels/sampling/top_p_mask_f32_v200064/reference.py +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/reference.py @@ -266,6 +266,8 @@ def top_p_mask_reference(logits: torch.Tensor, p: torch.Tensor, debug: dict | No set). ``debug`` gets the slice stats and every round's slice partials (Triton's STATS / PARTS layout) and the rounds each row ran.""" n = logits.shape[0] + if n > SPLIT_MAX_BATCH: + return top_p_mask_rows_reference(logits, p, debug) assert logits.dtype == torch.float32 and logits.shape[1] == V and 1 <= n <= SPLIT_MAX_BATCH S = splits_for(n) L = V // S @@ -325,3 +327,199 @@ def top_p_mask_reference(logits: torch.Tensor, p: torch.Tensor, debug: dict | No keep_dup = dmask & (rank <= col(st.numkeep)) keep = torch.where(ties, keep & (~dmask | keep_dup), keep) return torch.where(masked & ~keep, torch.full_like(x, float("-inf")), x) + + +# ---- more than 64 rows: vLLM's one-program-per-row kernel (``_topk_topp_kernel``, top-p only) + +TILE, CHUNK = 8192, 4096 +TILES = -(-V // TILE) # 25 +# a 4096-chunk of the compacted buffer: thread (c % 2048) // 4, register 4 (c // 2048) + c % 4 +CHUNK_OFFSETS = (2, 2048, 1, 32, 16, 8, 4, 64, 128, 1024, 512, 256) +ITERS = 18 +EPS = _f32(1e-9) + + +def fma32(a: torch.Tensor, b: torch.Tensor, c: torch.Tensor) -> torch.Tensor: + """fp32 ``a * b + c`` with one rounding, elementwise (the product is exact in fp64, the sum's error is kept by + TwoSum and decides a tie of the fp32 rounding).""" + a, b, c = torch.broadcast_tensors(a.double(), b.double(), c.double()) + pr = a * b + s = pr + c + bv = s - pr + err = (pr - (s - bv)) + (c - bv) + r = s.float() + up = torch.nextafter(r, torch.full_like(r, float("inf"))) + dn = torch.nextafter(r, torch.full_like(r, float("-inf"))) + r = torch.where((s == (r.double() + up.double()) / 2) & (err > 0), up, r) + r = torch.where((s == (r.double() + dn.double()) / 2) & (err < 0), dn, r) + return torch.where(torch.isfinite(s), r, s.float()) + + +def sqrt_f32(x: torch.Tensor) -> torch.Tensor: + """Triton's ``tl.sqrt`` on gfx950: the hardware ``v_sqrt_f32`` (not correctly rounded: it differs from torch's + sqrt by one ulp on about 15% of inputs), run elementwise on the GPU.""" + from .kernel_rows import compile_sqrt_f32 + + x = x.float().contiguous() + y = torch.empty_like(x) + if x.numel(): + exe = _sqrt_exe(compile_sqrt_f32) + _run(exe, x.data_ptr(), y.data_ptr(), x.numel(), -(-x.numel() // 256), torch.cuda.current_stream()) + return y + + +def _sqrt_exe(factory, _cache={}): + if "exe" not in _cache: + _cache["exe"] = factory() + return _cache["exe"] + + +def _run(exe, *args): + import flydsl.compiler as flyc + + cf = getattr(exe, "_cf", None) + if cf is None: + exe._cf = flyc.compile(exe, *args) + else: + cf(*args) + + +def log_any_f32(x: torch.Tensor) -> torch.Tensor: + """``tl.log`` for any x: below 2^-126 (zero and negatives too) the input is scaled by 2^32 and 32 ln2 is + subtracted; a non-finite ``v_log_f32`` passes through.""" + small = x < 2.0**-126 + xs = torch.where(small, x * 2.0**32, x) + y = torch.log2(xs) + fin = torch.isfinite(y) + yf = torch.where(fin, y, torch.ones_like(y)) + out = [] + for v in yf.flatten().tolist(): + v = Fraction(v) + r = Fraction(_round_f32(v * _LN2_HI)) + f0 = Fraction(_round_f32(v * _LN2_HI - r)) + f1 = Fraction(_round_f32(v * _LN2_LO + f0)) + out.append(_round_f32(v * _LN2_HI + f1)) + r = torch.tensor(out, dtype=torch.float32, device=x.device).view(x.shape) + r = torch.where(fin, r, y) + return r - torch.where(small, torch.full_like(r, _f32(32 * 0.6931471805599453)), torch.zeros_like(r)) + + +def _chain(parts: torch.Tensor) -> torch.Tensor: + """``acc = 0.0; acc += parts[:, i]`` in order.""" + acc = torch.zeros(parts.shape[0], device=parts.device) + for i in range(parts.shape[1]): + acc = acc + parts[:, i] + return acc + + +def _search(chunks, offsets, lo, hi, p): + """The pivot search on [n, nch, size] chunks (zeros past the data) from (lo, hi]: the pivot, the minimum above it, + that minimum's count and the mass above it, of the last round.""" + n = chunks.shape[0] + dev = chunks.device + done = torch.zeros(n, dtype=torch.bool, device=dev) + ppiv, mlp, psum = torch.ones(n, device=dev), torch.ones(n, device=dev), torch.zeros(n, device=dev) + nml = torch.zeros(n, dtype=torch.int64, device=dev) + for it in range(ITERS): + live = ~done + piv = (hi - lo) * 0.5 + lo + ab = chunks > piv[:, None, None] + cs = butterfly(torch.where(ab, chunks, torch.zeros_like(chunks)), offsets) + tmin = torch.where(ab, chunks, torch.ones_like(chunks)).amin(-1) + tcnt = (ab & ((chunks - tmin[..., None]).abs() < EPS)).sum(-1) + s0 = _chain(cs) + m0 = torch.ones(n, device=dev) + n0 = torch.zeros(n, dtype=torch.int64, device=dev) + for i in range(chunks.shape[1]): + tm, tc = tmin[:, i], tcnt[:, i] + n0 = torch.where(tm < m0, tc, n0 + tc * ((tm - m0).abs() < EPS)) + m0 = torch.minimum(m0, tm) + found = (s0 >= p) & (fma32(-m0, n0.float(), s0) < p) + take = live & found + ppiv, mlp, nml, psum = (torch.where(take, a, b) for a, b in ((piv, ppiv), (m0, mlp), (n0, nml), (s0, psum))) + lo = torch.where(live & (s0 > p), piv, lo) + hi = torch.where(live & ~(s0 > p) & (s0 < p), piv, hi) + stop = live & (((hi - lo) < EPS) | (it + 1 >= ITERS)) + ppiv = torch.where(stop, (hi + lo) / torch.full_like(hi, 2.0), ppiv) + mlp, nml, psum = (torch.where(stop, a, b) for a, b in ((m0, mlp), (n0, nml), (s0, psum))) + done = done | take | stop + return ppiv, mlp, nml, psum + + +def top_p_mask_rows_reference(logits: torch.Tensor, p: torch.Tensor, debug: dict | None = None) -> torch.Tensor: + """fp32 [n, 200064] logits, fp32 [n] top-p -> masked logits (a new tensor), as vLLM's ``_topk_topp_kernel`` + (top-k off, top-p on, one program per row) builds them on gfx950; any n.""" + from .sigma_table import NORMAL_CDF_TO_SIGMA + + n = logits.shape[0] + dev = logits.device + assert logits.dtype == torch.float32 and logits.shape[1] == V + x = logits.contiguous() + p = p.float().contiguous() + run = p < 1.0 + ninf = float("-inf") + pad = torch.full((n, TILES * TILE), ninf, device=dev) + pad[:, :V] = x + valid = torch.arange(TILES * TILE, device=dev) < V + b0 = pad[:, :TILE] + fin = b0 > ninf + nf = fin.sum(1).float() + fl = torch.where(fin, b0, torch.zeros_like(b0)) + s1 = butterfly(fl, STATS_OFFSETS) + s2 = butterfly(fl * fl, STATS_OFFSETS) + zero = torch.zeros(n, device=dev) + avg = torch.where(nf > 0, s1 / nf, zero) + sq = torch.where(nf > 0, s2 / nf, zero) + std = sqrt_f32(torch.clamp(fma32(-avg, avg, sq), min=0.0)) + ms = fma32(std, torch.full_like(std, 10.0), avg) + xm = pad - ms[:, None] + e = torch.where(valid, exp_f32(xm), torch.zeros_like(pad)) + se = _chain(butterfly(e.view(n, TILES, TILE), STATS_OFFSETS)) + max_logit = pad.amax(1) + min_logit = torch.minimum(torch.where(pad > ninf, pad, torch.full_like(pad, float("inf"))).amin(1), max_logit) + idx = (p * 200.0).to(torch.int32).clamp(0, 199).long() + sig = torch.tensor(NORMAL_CDF_TO_SIGMA, dtype=torch.float32, device=dev)[idx] + sig = sig + sig.abs() * -0.25 + oprob = exp_f32(fma32(std, sig, avg) - ms) / se + probs = exp_f32(xm) / se[:, None] + om = (probs > oprob[:, None]) & valid + so = _chain(butterfly(torch.where(om, probs, torch.zeros_like(probs)).view(n, TILES, TILE), STATS_OFFSETS)) + hi = exp_f32(max_logit - ms) / se + lo = exp_f32(min_logit - ms) / se + b1 = so > p + ppiv, mlp, psum = torch.ones(n, device=dev), torch.ones(n, device=dev), torch.zeros(n, device=dev) + nml = torch.zeros(n, dtype=torch.int64, device=dev) + r1 = (run & b1).nonzero().flatten() + if r1.numel(): + nout = om[r1].sum(1) + nch = int(-(-int(nout.max()) // CHUNK)) + buf = torch.zeros(r1.numel(), max(nch, 1) * CHUNK, device=dev) + for k, r in enumerate(r1.tolist()): + v = probs[r][om[r]] + buf[k, :v.numel()] = v + out = _search(buf.view(r1.numel(), -1, CHUNK), CHUNK_OFFSETS, oprob[r1], hi[r1], p[r1]) + for t, o in zip((ppiv, mlp, nml, psum), out): + t[r1] = o + r2 = (run & ~b1).nonzero().flatten() + if r2.numel(): + buf = torch.where(valid, probs[r2], torch.zeros_like(probs[r2])).view(r2.numel(), TILES, TILE) + out = _search(buf, STATS_OFFSETS, lo[r2], hi[r2], p[r2]) + for t, o in zip((ppiv, mlp, nml, psum), out): + t[r2] = o + dup = log_any_f32(mlp * se) + ms + q = (psum - p) / mlp + qi = torch.nan_to_num(q, nan=0.0).clamp(0, 4294967295.0).to(torch.int64) # v_cvt_u32_f32 saturates + nkeep = (nml - qi) % 2**32 + fpiv = log_any_f32(ppiv * se) + ms + apply = run & (fpiv < max_logit) & (fpiv != ninf) + keep = x > fpiv[:, None] + ties = apply & (nkeep < nml) + if bool(ties.any()): + dm = (x - dup[:, None]).abs() < EPS + rank = dm.long().cumsum(1) + keep = torch.where(ties[:, None], keep & ~(dm & (rank > nkeep[:, None])), keep) + if debug is not None: + debug.update(avg=avg, sq=sq, std=std, ms=ms, se=se, max=max_logit, min=min_logit, oprob=oprob, sout=so, + nout=om.sum(1), branch=torch.where(b1, 1, 2), ppiv=ppiv, mlp=mlp, nml=nml, psum=psum, dup=dup, + nkeep=nkeep, fpiv=fpiv) + return torch.where(apply[:, None] & ~keep, torch.full_like(x, ninf), x) diff --git a/amd_kernels/sampling/top_p_mask_f32_v200064/sigma_table.py b/amd_kernels/sampling/top_p_mask_f32_v200064/sigma_table.py new file mode 100644 index 0000000..c96931d --- /dev/null +++ b/amd_kernels/sampling/top_p_mask_f32_v200064/sigma_table.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# From vLLM v1/sample/ops/topk_topp_triton.py (_NORMAL_CDF_TO_SIGMA_TABLE), vLLM 0.33.0rc1.dev910+g4fab5a074. +"""The sigma of the outlier pivot per 1/200 step of top-p (``int(p * 200)``).""" + +NORMAL_CDF_TO_SIGMA = ( + 3.656, 3.650, 3.650, 3.650, 3.626, 3.626, 3.626, 3.514, 3.514, 3.503, + 3.503, 3.434, 3.434, 3.428, 3.428, 3.387, 3.380, 3.380, 3.376, 3.373, + 3.373, 3.356, 3.354, 3.354, 3.291, 3.249, 3.234, 3.214, 3.198, 3.198, + 3.185, 3.177, 3.177, 3.165, 3.164, 3.161, 3.138, 3.120, 3.115, 3.113, + 3.093, 3.066, 3.054, 3.043, 3.037, 3.023, 2.993, 2.991, 2.976, 2.970, + 2.952, 2.946, 2.932, 2.908, 2.902, 2.895, 2.886, 2.874, 2.861, 2.844, + 2.836, 2.810, 2.801, 2.790, 2.784, 2.779, 2.767, 2.757, 2.745, 2.733, + 2.723, 2.716, 2.693, 2.678, 2.671, 2.656, 2.649, 2.629, 2.611, 2.595, + 2.592, 2.585, 2.574, 2.550, 2.543, 2.534, 2.521, 2.518, 2.497, 2.485, + 2.468, 2.450, 2.441, 2.430, 2.412, 2.402, 2.389, 2.383, 2.377, 2.364, + 2.349, 2.338, 2.332, 2.319, 2.310, 2.301, 2.282, 2.274, 2.266, 2.250, + 2.242, 2.236, 2.226, 2.215, 2.207, 2.196, 2.179, 2.171, 2.162, 2.147, + 2.135, 2.121, 2.109, 2.095, 2.085, 2.073, 2.063, 2.045, 2.030, 2.016, + 2.003, 1.992, 1.983, 1.972, 1.960, 1.949, 1.940, 1.928, 1.912, 1.897, + 1.881, 1.869, 1.854, 1.838, 1.824, 1.807, 1.792, 1.779, 1.764, 1.751, + 1.739, 1.726, 1.711, 1.697, 1.685, 1.668, 1.652, 1.636, 1.622, 1.603, + 1.585, 1.568, 1.551, 1.534, 1.513, 1.499, 1.480, 1.464, 1.441, 1.422, + 1.394, 1.373, 1.347, 1.320, 1.296, 1.270, 1.246, 1.219, 1.190, 1.163, + 1.135, 1.104, 1.073, 1.041, 1.006, 0.969, 0.931, 0.894, 0.851, 0.806, + 0.757, 0.702, 0.643, 0.574, 0.498, 0.405, 0.288, 0.134, -0.110, -3.813, +) diff --git a/benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py b/benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py new file mode 100644 index 0000000..8ab89ec --- /dev/null +++ b/benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-replay timing of ``swiglu_oai_mxfp8`` (one launch) against what vLLM runs at TP2 without it: the draft's +aten SwiGLU-OAI chain (7 elementwise kernels, bf16 after each) or the target's Triton ``swiglu_oai_split``, each +followed by the MXFP8 quantizer in the down GEMM's layout. One graph per side holds 100 calls on 100 distinct +gate_up inputs (2.5 GB at 1024 rows, beyond the MALL); median of 5 rounds of 10 replays, the sides alternated +twice (the faster round printed), us per call. ``--sweep`` times every (pieces, threads) workgroup shape instead. + + python3 benchmarks/activation/swiglu_oai_mxfp8_i6144/benchmark_swiglu_oai_mxfp8.py [--form fp32] [--sweep] +""" + +import argparse +import statistics + +import torch + +from amd_kernels.activation.swiglu_oai_mxfp8_i6144 import swiglu_oai_mxfp8 +from amd_kernels.activation.swiglu_oai_mxfp8_i6144.reference import swiglu_bf16_chain +from amd_kernels.gemm.mxfp8_common import gemm as G + +I = 6144 +ALPHA, BETA, LIMIT = 1.702, 1.0, 7.0 +ROWS = [7, 8, 16, 56, 64, 112, 128, 224, 256, 448, 512, 896, 1024, 1792] +SHAPES = [(1, 768), (2, 384), (3, 256), (4, 192), (6, 128), (12, 64), (1, 384), (2, 192), (3, 128), (6, 64), (1, 256), (4, 64), (1, 192)] +CALLS = 100 +HBM = 6.5e12 + + +def _time(graph, rounds=5, replays=10): + for _ in range(3): + graph.replay() + torch.cuda.synchronize() + out = [] + for _ in range(rounds): + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(replays): + graph.replay() + e.record() + torch.cuda.synchronize() + out.append(s.elapsed_time(e) / (replays * CALLS) * 1e3) + return statistics.median(out) + + +def _graphs(fns, xs): + graphs = {} + for name, fn in fns.items(): + fn(xs[0]) + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for x in xs: + fn(x) + graphs[name] = g + t = {k: [] for k in graphs} + for _ in range(2): + for k, g in graphs.items(): + t[k].append(_time(g)) + return {k: min(v) for k, v in t.items()} + + +def _baseline(form, tile): + def quant(act): + return G.mxfp8_e4m3_quantize_shuffled(act, tile) if tile else G.mxfp8_e4m3_quantize_aiter(act) + + if form == "bf16_chain": + return lambda x: quant(swiglu_bf16_chain(x, ALPHA, BETA, LIMIT)) + from vllm.models.minimax_m3.amd.ops.swiglu_oai import swiglu_oai_split + + return lambda x: quant(swiglu_oai_split(x, ALPHA, BETA, LIMIT)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--rows", type=int, nargs="*", default=ROWS) + ap.add_argument("--form", default="bf16_chain", choices=("bf16_chain", "fp32")) + ap.add_argument("--sweep", action="store_true") + args = ap.parse_args() + for m in args.rows: + tile = G.prequant_code_tile_rows(m, 6144, I) + xs = [(torch.randn((m, 2 * I), device="cuda") * 3).bfloat16() for _ in range(CALLS)] + floor = m * (2 * I * 2 + I + I // 32) / HBM * 1e6 + if args.sweep: + fns = {s: (lambda x, s=s: swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, args.form, shape=s)) for s in SHAPES} + t = _graphs(fns, xs) + best = min(t, key=t.get) + cells = " ".join(f"{p}x{n}:{v:5.2f}" for (p, n), v in t.items()) + print(f"rows={m:5d} tile={tile:3d} {cells} | best {best}", flush=True) + continue + t = _graphs({"baseline": _baseline(args.form, tile), + "ours": lambda x: swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, args.form)}, xs) + print(f"rows={m:5d} tile={tile:3d} {args.form}: baseline {t['baseline']:6.2f} us | swiglu_oai_mxfp8 " + f"{t['ours']:6.2f} us | {t['baseline'] / t['ours']:.1f}x | HBM floor {floor:5.2f} us " + f"({floor / t['ours']:.0%})", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py b/benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py new file mode 100644 index 0000000..b4c33a2 --- /dev/null +++ b/benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py @@ -0,0 +1,175 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV chain (N 4992 x K 6144, 32 / 2 / 2 heads): the one-launch entry (``mxfp8_qkv_norm_rope_nvfp4_insert``) +vs the deferred chain (``gemm.mxfp8_gemm_shuffled(defer_reduce=True)`` -> split-K reduce-insert or bf16 GEMM -> +``norm_rope_nvfp4_insert``) vs vLLM's TP2 stock path (Triton ``tl.dot_scaled`` on row-major codes -> vLLM's FlyDSL +``minimax_m3_norm_rope_nvfp4_insert``), and the roofline. + + python3 benchmarks/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/benchmark_chain.py \ + --checkpoint --out /tmp/qkv_fused + +Layer 39 TP2 rank 0 weights in 14 cold copies (> 400 MB), up to 100 seeded inputs per graph (decode rows: requests +of 8 tokens at random positions, own pages); graph replay, 15 rounds of 10 replays (the amd_kernels arms alternate, +then the stock arm), median us per call. +Every arm's bytes are checked against the deferred chain before timing (q, index-q, pages; stock: reported only). +Past the shuffled-code tile (769+ rows) the entry arm is the insert package's dispatcher (row-major codes). +Roofline: 6.5 TB/s over the weight, the codes and the per-token insert bytes (q 8192, index-q 256, positions / slots +24, cos / sin 128, five NVFP4 rows 360), or 5 PF/s, whichever is longer. +""" + +import argparse +import json +import statistics +import sys +from pathlib import Path + +import numpy as np +import torch + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "gemm/mxfp8_n4992_k6144")) +import benchmark_n4992_k6144 as B # noqa: E402 (weights(), insert_inputs(): the TP2 GEMM bench's helpers) +from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 import ( # noqa: E402 + host, + mxfp8_qkv_norm_rope_nvfp4_insert, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( # noqa: E402 + norm_rope_nvfp4_insert, + splitk_reduce_norm_rope_nvfp4_insert, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import qkv as dispatch # noqa: E402 +from amd_kernels.gemm.mxfp8_common import gemm as g # noqa: E402 + +N, K = 4992, 6144 +HEADS = dict(num_heads=32, num_kv_heads=2, num_index_heads=2, eps=1e-6) +TOKEN_BYTES = 8192 + 256 + 24 + 128 + 360 + + +def roofline_us(m): + moved = (N + m) * K * (1 + 1 / 32) + m * TOKEN_BYTES + return max(moved / 6.5e12, 2 * m * N * K / 5e15) * 1e6 + + +def main(): + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import _mxfp8_dot_scaled_gemm + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import mxfp8_e4m3_quantize + from vllm.models.minimax_m3.amd.ops.norm_rope_nvfp4_insert.host import minimax_m3_norm_rope_nvfp4_insert + + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--checkpoint", required=True, type=Path) + ap.add_argument("--layer", type=int, default=39) + ap.add_argument("--ms", default="1,2,4,8,12,16,24,32,48,64,96,128,192,256,320,384,416,448,512,640,768,1024,1536") + ap.add_argument("--arms", default="entry,deferred,stock") + ap.add_argument("--out", type=Path) + args = ap.parse_args() + shuffled, raw = B.weights(args.checkpoint, args.layer) + copies = len(shuffled) + rows = [] + for m in map(int, args.ms.split(",")): + count = max(2, min(100, (400 << 20) // (m * K * 3))) + phases = (copies + count - 1) // count + tile = g.prequant_code_tile_rows(m, N, K) + akr_in, tri_in = [], [] + for i in range(count): + x = torch.from_numpy(np.random.default_rng(1000 * m + i).standard_normal( + (m, K), dtype=np.float32)).cuda().to(torch.bfloat16) + akr_in.append(g.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else g.mxfp8_e4m3_quantize_aiter(x)) + tri_in.append(mxfp8_e4m3_quantize(x)) + del x + ins = B.insert_inputs(args.checkpoint, args.layer, m) + q = torch.empty((m, 4096), dtype=torch.bfloat16, device="cuda") + iq = torch.empty((m, 256), dtype=torch.float8_e4m3fn, device="cuda") + + def entry(j, off): + w, ws = shuffled[(j + off) % copies] + if tile: + mxfp8_qkv_norm_rope_nvfp4_insert(*akr_in[j], w, ws, m, *ins, q, iq, **HEADS) + else: # rows past the shuffled-code tile: the dispatcher's GEMM + insert + dispatch.mxfp8_gemm_qkv_norm_rope_nvfp4_insert(*akr_in[j], w, ws, m, torch.bfloat16, *ins, **HEADS) + + def deferred(j, off): + w, ws = shuffled[(j + off) % copies] + if not tile: + out = g.mxfp8_gemm(*akr_in[j], w, ws, torch.bfloat16, activation_scale_preshuffled=True) + norm_rope_nvfp4_insert(out, *ins, q, iq, **HEADS) + return + out = g.mxfp8_gemm_shuffled(*akr_in[j], w, ws, torch.bfloat16, m, defer_reduce=True) + if isinstance(out, g.SkPartials): + splitk_reduce_norm_rope_nvfp4_insert(out.tmp, out.split_k, *ins, q, iq, **HEADS) + else: + norm_rope_nvfp4_insert(out, *ins, q, iq, **HEADS) + + def stock(j, off): + w, ws = raw[(j + off) % copies] + qkv = _mxfp8_dot_scaled_gemm(*tri_in[j], w, ws, torch.bfloat16) + minimax_m3_norm_rope_nvfp4_insert(qkv, *ins, q, iq, **HEADS) + + every = dict(entry=entry, deferred=deferred, stock=stock) + arms = {name: every[name] for name in args.arms.split(",")} + kv, idx = ins[8], ins[9] + + def snapshot(): + torch.cuda.synchronize() + return q.clone(), iq.view(torch.uint8).clone(), kv.clone(), idx.clone() + + checks = {} + for j in (0, count - 1): + kv.zero_(), idx.zero_() + deferred(j, 0) + want = snapshot() + for name, call in arms.items(): + kv.zero_(), idx.zero_() + q.fill_(7) + call(j, 0) + got = snapshot() + diff = [int((a.view(torch.uint8) != b.view(torch.uint8)).sum()) for a, b in zip(got, want)] + checks.setdefault(name, []).append(diff) + if name == "entry" and not tile: # the dispatcher allocates its own q / index-q + diff = diff[2:] + if name != "stock": + assert not any(diff), (name, m, diff) + graphs = {} + pool = torch.cuda.graph_pool_handle() + for name, call in arms.items(): + graphs[name] = [] + for phase in range(phases): + call(0, phase * count) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, pool=pool): + for j in range(count): + call(j, phase * count) + graphs[name].append(graph) + B.measure(graphs[name], count) + samples = {name: [] for name in arms} + # the amd_kernels arms alternate, order rotated every round; the stock arm runs alone afterwards (interleaved, + # the next arm after it reads 0.5-0.9 us slow at 256-768 rows) + names = [name for name in arms if name != "stock"] + for r in range(15): + for name in names[r % len(names):] + names[:r % len(names)]: + samples[name].append(B.measure(graphs[name], count)) + if "stock" in arms: + for _ in range(15): + samples["stock"].append(B.measure(graphs["stock"], count)) + row = dict(m=m, plan=str(host._plan(m)) if m <= host.FUSED_MAX_ROWS else "deferred chain", + roofline_us=roofline_us(m), checks=checks) + for name in arms: + row[name] = statistics.median(samples[name]) + row[name + "_range"] = [min(samples[name]), max(samples[name])] + rows.append(row) + print(json.dumps(row), flush=True) + del graphs, akr_in, tri_in + torch.cuda.empty_cache() + names = list(arms) + print("| M | " + " | ".join(f"{n} us" for n in names) + " | roofline us | entry % of roofline |") + print("|---|" + "---|" * (len(names) + 2)) + for r in rows: + cells = " | ".join(f"{r[n]:.2f}" for n in names) + frac = f"{100 * r['roofline_us'] / r['entry']:.0f}%" if "entry" in r else "-" + print(f"| {r['m']} | {cells} | {r['roofline_us']:.2f} | {frac} |") + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(rows, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/norm_rope_fp8_insert_d128_rot64/benchmark_norm_rope_insert.py b/benchmarks/attention/norm_rope_fp8_insert_d128_rot64/benchmark_norm_rope_insert.py index fdb6edd..25c4ff5 100644 --- a/benchmarks/attention/norm_rope_fp8_insert_d128_rot64/benchmark_norm_rope_insert.py +++ b/benchmarks/attention/norm_rope_fp8_insert_d128_rot64/benchmark_norm_rope_insert.py @@ -11,9 +11,9 @@ launch; GB/s counts the input read plus the cache bytes written. The first and the last launch are checked against ``reference.py`` before timing. -Cases (README): an M3.1 draft layer at TP4 (16 query / 1 KV heads, 5 layers, 128-token -pages); the qkv kernel at the draft's query block (R requests x 4 rows), the multi-layer -kernel at context-row counts from a decode step to a prefill chunk. +Cases (README): an M3.1 draft layer at TP4 (16 query / 1 KV heads) and TP2 (32 query / 2 KV heads), 5 layers, +128-token pages; the qkv kernel at the draft's query block (R requests x 4 rows), the multi-layer kernel at +context-row counts from a decode step to a prefill chunk. """ import argparse @@ -39,9 +39,11 @@ INPUT_BUDGET = 4 << 30 QKV_CASES = [ ("qkv_tp4_p128", 16, 1, 128, [4, 64, 256, 1024]), + ("qkv_tp2_p128", 32, 2, 128, [4, 64, 256, 1024, 2048]), ] ML_CASES = [ ("multilayer_tp4_p128", 1, 128, [4, 384, 4096]), + ("multilayer_tp2_p128", 2, 128, [4, 384, 4096, 8192]), ] @@ -161,16 +163,17 @@ def call(i): def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--copies", type=int, default=100, help="launches (inputs) per graph, at most") + parser.add_argument("--case", nargs="*", default=[c[0] for c in QKV_CASES + ML_CASES]) args = parser.parse_args() if not torch.cuda.is_available() or "gfx950" not in torch.cuda.get_device_properties(0).gcnArchName: raise RuntimeError("requires gfx950") scales = torch.tensor(1.0, device="cuda"), torch.tensor(1.3, device="cuda") with torch.inference_mode(): for name, nq, nkv, page, ns in QKV_CASES: - for n in ns: + for n in ns if name in args.case else (): bench_qkv(name, nq, nkv, page, n, args.copies, scales) for name, nkv, page, ns in ML_CASES: - for n in ns: + for n in ns if name in args.case else (): bench_multilayer(name, nkv, page, n, args.copies) diff --git a/benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py b/benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py index 82e0027..39894b5 100644 --- a/benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py +++ b/benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py @@ -7,15 +7,17 @@ of one large KV / index cache). The inputs rotate over more than 400 MB in all (above the 256 MB MALL), so every call reads and writes HBM-cold data; this is conservative for qkv, which serving gets warm from the QKV GEMM. Decode batches (up to 1024 tokens) -are requests of 4 tokens (the MTP / DSpark rows) in their own pages; larger batches +are requests of 4 tokens (``--decode-run 8``: DSpark 7) in their own pages; larger batches are 8 requests continuing 40K-token contexts. The first and last input are checked against ``norm_rope_nvfp4_insert_d128_rot64_p128/reference.py`` before timing. Median of 5 rounds of 10 replays of every graph, us per call. python3 benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py # the entry's kernel per size python3 benchmarks/attention/norm_rope_nvfp4_insert_d128_rot64_p128/benchmark_norm_rope_insert.py --reduce # the split-K reduce kernel + ... --heads 32 2 2 # TP2 heads (default q16 kv1 iq1, TP4) + ... --baseline # vLLM's own FlyDSL insert (ops/norm_rope_nvfp4_insert), the stock path at TP2 -Heads q16 kv1 iq1, rotary 64 (TP4). +Rotary 64. """ import argparse @@ -33,19 +35,22 @@ D, BLK, WIDTH = ref.D, ref.BLK, ref.WIDTH EPS, ROT = 1e-6, 64 HEADS = (16, 1, 1) +BASELINE = None # vLLM's minimax_m3_norm_rope_nvfp4_insert under --baseline +DECODE_RUN = 4 # tokens per decode request LLC_TARGET_BYTES = 400 * 1024**2 # above the 256 MB MALL CALLS = 100 # calls per graph, each on its own inputs VERBOSE = False MAX_POS = 65536 TOKENS = [1, 4, 64, 172, 256, 512, 1024, 4096, 16384] -REDUCE = [(32, 3), (128, 3), (200, 2), (288, 2)] # (tokens, split) of the QKV GEMM's deferred rows +# (tokens, split) of the QKV GEMM's deferred rows (N 2560 / N 4992) +REDUCE = {(16, 1, 1): [(32, 3), (128, 3), (200, 2), (288, 2)], (32, 2, 2): [(16, 2), (32, 2), (48, 3), (64, 2)]} def slot_plan(n, gen): - """(positions, slots, pages): decode batches as requests of up to 4 tokens, one + """(positions, slots, pages): decode batches as requests of up to DECODE_RUN tokens, one page each at a random offset; larger ones as 8 requests continuing 40K contexts.""" if n <= 1024: - run = min(4, n) + run = min(DECODE_RUN, n) reqs = -(-n // run) pos = [] for r in range(reqs): @@ -115,6 +120,8 @@ def call(self, i, kernel="main", op=None): if self.split_k: fn = op or splitk_reduce_norm_rope_nvfp4_insert fn(self.slab[i], self.split_k, *args, **kw) + elif op is None and BASELINE is not None: + BASELINE(self.qkv[i], *args, **kw) else: fn = op or norm_rope_nvfp4_insert fn(self.qkv[i], *args, **kw, prefill=kernel == "prefill") @@ -199,36 +206,39 @@ def bench(n, kernels, split_k=0, rounds=1): for _ in range(rounds): for k in kernels: times[k].append(time_graphs(graphs[k])) - for kernel, ts in times.items(): - name = "splitk" if split_k else kernel + for ts in times.values(): us = " / ".join(f"{t:.2f}" for t in ts) what = f"split {split_k}, " if split_k else "" - if split_k: - tp = host._reduce_token_pairs(n, split_k) - elif n >= host.PREFILL_TOKENS: - name, tp = "prefill", host._prefill_token_pairs(n) - else: - tp = host._main_token_pairs(n) - print(f"q16_kv1_iq1 n={n} {name}: {what}pairs {tp}, {inputs.copies} inputs; " + heads = (inputs.nq, inputs.nkv, inputs.niq) + name = "baseline" if BASELINE is not None else host._launcher(n, heads, split_k=split_k).kernel_name + print(f"q{heads[0]}_kv{heads[1]}_iq{heads[2]} n={n} {what}{name}, {inputs.copies} inputs; " f"{us} us ({inputs.bytes / statistics.median(ts) / 1e3:.0f} GB/s)", flush=True) # fmt: skip del graphs, inputs torch.cuda.empty_cache() def main() -> None: + global LLC_TARGET_BYTES, VERBOSE, HEADS, BASELINE, DECODE_RUN parser = argparse.ArgumentParser() parser.add_argument("--tokens", type=int, nargs="*", default=TOKENS) parser.add_argument("--reduce", action="store_true", help="the split-K reduce kernel") + parser.add_argument("--heads", type=int, nargs=3, default=list(HEADS), help="q, KV, index heads per rank") + parser.add_argument("--baseline", action="store_true", help="vLLM's FlyDSL insert (bf16 qkv only)") parser.add_argument("--llc-mb", type=int, default=400, help="footprint the inputs rotate over") + parser.add_argument("--decode-run", type=int, default=DECODE_RUN, help="tokens per decode request (<= 1024 rows)") parser.add_argument("-v", "--verbose", action="store_true") args = parser.parse_args() - global LLC_TARGET_BYTES, VERBOSE - LLC_TARGET_BYTES, VERBOSE = args.llc_mb * 1024**2, args.verbose + LLC_TARGET_BYTES, VERBOSE, HEADS = args.llc_mb * 1024**2, args.verbose, tuple(args.heads) + DECODE_RUN = args.decode_run + if args.baseline: + from vllm.models.minimax_m3.amd.ops.norm_rope_nvfp4_insert.host import minimax_m3_norm_rope_nvfp4_insert + + BASELINE = minimax_m3_norm_rope_nvfp4_insert if not torch.cuda.is_available() or "gfx950" not in torch.cuda.get_device_properties(0).gcnArchName: raise RuntimeError("requires gfx950") with torch.inference_mode(): if args.reduce: - for n, split_k in REDUCE: + for n, split_k in REDUCE[HEADS]: bench(n, ["main"], split_k) else: for n in args.tokens: diff --git a/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py new file mode 100644 index 0000000..4cf6c7a --- /dev/null +++ b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One-GPU timing of the decode block (index top-16 + sparse decode attention) on its two one-GPU routes, to place +``host.SELECT_MAX_ROWS``: ``fused`` (``nvfp4_index_score_rows``, then the attention with ``index_score``: the +in-kernel top-16) and ``chain`` (``nvfp4_index_score_top16``, then the attention), at 1 (TP4) or 2 (TP2) KV / index +heads, fp16 compute, the MXFP8 o_proj input. + +Request i of N at the (i + 0.5) / N quantile of a log-uniform 32K-192K context; every call has its own queries and +block tables onto random slots of a pool of index and KV pages (> 1 GB, beyond the MALL), graphs captured at +max_model_len 1M. Sides alternate per round; median us per call. The sides' ids and output bytes are compared first. + + python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode.py --hkv 2 \ + --ql 8 --reqs 1 4 16 32 48 +""" + +import argparse +import statistics + +import torch +from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import host +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import reference as iref +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.score_decode import ( + nvfp4_index_score_rows, + nvfp4_index_score_top16, +) + +BLK, D = 128, 128 +MAX_MODEL_LEN = 1 << 20 +POOL_SLOTS = 1 << 17 +INIT, LOCAL = 0, 1 + + +def lens_for(n, lo=32768, hi=196608): + return [round(lo * (hi / lo) ** ((i + 0.5) / n)) for i in range(n)] + + +def make_pool(hkv, gen): + keys = (torch.randn(4096 * BLK, D, device="cuda", generator=gen) * 2).bfloat16() + pages = iref.nvfp4_index_pages(keys) + icache = pages.repeat(POOL_SLOTS // pages.shape[0], 1, 1) + kv = torch.randint(0, 256, (POOL_SLOTS, 2 * hkv, BLK, 72), dtype=torch.uint8, device="cuda", generator=gen) & 0x77 + return icache, kv + + +def make_calls(hkv, ql, reqs, calls, gen): + lens = lens_for(reqs) + rows = reqs * ql + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + out = [] + for _ in range(calls): + bt = torch.zeros(reqs, MAX_MODEL_LEN // BLK, dtype=torch.int32, device="cuda") + for r, s in enumerate(lens): + n = -(-s // BLK) + bt[r, :n] = torch.randint(0, POOL_SLOTS, (n,), device="cuda", generator=gen, dtype=torch.int32) + out.append(dict( + bt=bt, + iq=(torch.randn(rows, hkv, D, device="cuda", generator=gen) * 0.5).to(torch.float8_e4m3fn), + q=torch.randn(rows, 16 * hkv, D, device="cuda", generator=gen).bfloat16(), + )) + return seq, out + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--hkv", type=int, default=2) + p.add_argument("--ql", type=int, nargs="*", default=[8]) + p.add_argument("--reqs", type=int, nargs="*", default=[1, 2, 4, 8, 16, 24, 32, 40, 48, 64]) + p.add_argument("--calls", type=int, default=50) + p.add_argument("--rounds", type=int, default=5) + p.add_argument("--replays", type=int, default=5) + p.add_argument("--select-max-rows", type=int, help="host.SELECT_MAX_ROWS[hkv] for the fused side (sweeps)") + a = p.parse_args() + if a.select_max_rows: + host.SELECT_MAX_ROWS[a.hkv] = a.select_max_rows + gen = torch.Generator(device="cuda").manual_seed(0) + icache, kv = make_pool(a.hkv, gen) + one = torch.ones((), dtype=torch.float32, device="cuda") + print("| KV heads | ql | requests | rows | fused us | chain us | fused vs chain |") + print("|---|---:|---:|---:|---:|---:|---:|") + with torch.inference_mode(): + for ql in a.ql: + for reqs in a.reqs: + rows = reqs * ql + seq, calls = make_calls(a.hkv, ql, reqs, a.calls, gen) + width = iref.score_width(MAX_MODEL_LEN) + score = torch.empty(a.hkv, rows, width, dtype=torch.float32, device="cuda") + ids = {k: torch.empty(a.hkv, rows, 16, dtype=torch.int32, device="cuda") for k in ("fused", "chain")} + out = {k: torch.empty(rows, 16 * a.hkv, D, dtype=torch.bfloat16, device="cuda") + for k in ("fused", "chain")} + quant = {k: host._quant_buffers(rows, 2048 * a.hkv, "cuda") for k in ("fused", "chain")} + + def side(name, c): + if name == "fused": + nvfp4_index_score_rows(c["iq"], icache, score, c["bt"], seq, INIT, LOCAL, query_len=ql, + max_seq_len=MAX_MODEL_LEN) + extra = dict(index_score=score, index_max_seq_len=MAX_MODEL_LEN) + else: + nvfp4_index_score_top16(c["iq"], icache, c["bt"], seq, INIT, LOCAL, ids[name], query_len=ql, + max_seq_len=MAX_MODEL_LEN, score=score) + extra = {} + host.nvfp4_sparse_decode(c["q"], kv, ids[name], c["bt"], seq, D**-0.5, out[name], k_scale=one, + v_scale=one, decode_query_len=ql, quant_out=quant[name], + compute_type=torch.float16, **extra) + + for name in ("fused", "chain"): + for t in quant[name][:2]: + t.zero_() + side(name, calls[0]) + torch.cuda.synchronize() + assert torch.equal(ids["fused"], ids["chain"]) + assert all(torch.equal(x, y) for x, y in zip(quant["fused"][:2], quant["chain"][:2], strict=True)) + graphs = {} + for name in ("fused", "chain"): + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for c in calls: + side(name, c) + graphs[name] = g + times = {k: [] for k in graphs} + for g in graphs.values(): + g.replay() + torch.cuda.synchronize() + for r in range(a.rounds): + for name in (("fused", "chain") if r % 2 == 0 else ("chain", "fused")): + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(a.replays): + graphs[name].replay() + e.record() + torch.cuda.synchronize() + times[name].append(s.elapsed_time(e) * 1e3 / (a.replays * len(calls))) + f, c = statistics.median(times["fused"]), statistics.median(times["chain"]) + print(f"| {a.hkv} | {ql} | {reqs} | {rows} | {f:.2f} | {c:.2f} | {100 * (f / c - 1):+.1f}% |", + flush=True) + del graphs, calls + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py new file mode 100644 index 0000000..386b53e --- /dev/null +++ b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-GPU timing of the TP2 decode block (``index_decode.nvfp4_index_sparse_decode``: index top-16 + sparse decode +attention, the served layer call) on each route, to place ``decode_top16.SPLIT2_MIN_REQS``. + +One process per GPU (gloo group); rank r = card r of ``HIP_VISIBLE_DEVICES``, index heads 2r, 2r + 1, KV heads 2 (32 +query heads), as served: ``IndexDecodeTop16`` with the two-rank group on the replicated index cache, fp16 compute, the +MXFP8 o_proj input (K 4096) from ``layer.QUANT_OUT_MIN_ROWS`` rows. Sides (the same inputs, buffers and cards): + +- ``one_gpu``: ``split=False``, the one-GPU route (``fused``: scorer + the attention's in-kernel top-16, or ``chain``); +- ``split2``: the two-GPU scorer + top-16, then the attention (``SPLIT2_MIN_REQS`` set to 1 at capture). + +Inputs: ``--reqs N,..``: N requests of the contexts ``F[str(N)]`` of ``--lens-json F``, a JSON file {"N": [context +length of request i of N, ...]} with an entry for every N of ``--reqs`` (for the route: the contexts requests have while +they decode, e.g. the (i + 0.5) / N quantiles of the contexts of the requests decoding at N in a workload, sampled over +time); without it, ``synthetic_lens``: request i of N at the (i + 0.5) / N quantile of a log-uniform 32K-192K +distribution. Row j of a request of context C sees C - ql + 1 + j tokens. Every rank holds the same pool of NVFP4 +index pages (Gaussian keys quantized as the model does) and KV pages; each call's requests sit on distinct random pages +of their own region of the pool, and a graph holds >= 100 calls and enough that a region comes back after >= 384 MB of +other pages (cold pages, graphs captured at max_model_len 1M). + +Timing: 2 warm-up replays, then ``--rounds`` rounds of ``--replays`` replays, the sides alternated (A B B A ...), +each graph after a host barrier and a small two-GPU call (device barrier); a sample = the max over the two ranks of +the per-call time. Checks: ids and attention output bytes of the sides equal on the first and last input (rows that +differ, max over ranks). + + HIP_VISIBLE_DEVICES=0,1 python3 \ + benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_index_decode_ranks2.py \ + --ql 8 --reqs 1,2,4,8,16,32,48 +""" + +import argparse +import json +import os +import socket +import statistics +import sys +import time +import traceback + +import numpy as np +import torch +import torch.distributed as dist + +BLK, D, H, TOPK = 128, 128, 32, 16 +PAGE_BYTES = BLK * 72 +HKV = 2 +KV_PAGE_BYTES = 2 * HKV * BLK * 72 +INIT, LOCAL = 0, 1 # MiniMax-M3.1 sparse_init_block / sparse_local_block +MAX_MODEL_LEN = 1 << 20 +MIN_POOL_BYTES = 768 << 20 +REUSE_BYTES = 384 << 20 +WORLD = 2 +OPROJ_N, OPROJ_K = 6144, 4096 # the o_proj GEMM the MXFP8 output feeds (TP2) +SYNTHETIC_RANGE = (32768, 196608) # the log-uniform context range without --lens-json + + +def synthetic_lens(n, lo=SYNTHETIC_RANGE[0], hi=SYNTHETIC_RANGE[1]): + """Request i of n at the (i + 0.5) / n quantile of a log-uniform [lo, hi] distribution.""" + return [round(lo * (hi / lo) ** ((i + 0.5) / n)) for i in range(n)] + + +def cdiv(a, b): + return -(-a // b) + + +def make_tables(lens, region, span, pool_slots, rng, cols): + """[reqs, cols] int32: each request's blocks on distinct random slots of this region of the pool.""" + nb = [cdiv(s, BLK) for s in lens] + perm = (region * span + rng.permutation(sum(nb))) % pool_slots + t = np.zeros((len(lens), cols), dtype=np.int32) + p = 0 + for r, n in enumerate(nb): + t[r, :n] = perm[p : p + n] + p += n + return torch.from_numpy(t).cuda() + + +def body(rank, port, args): + torch.accelerator.set_device_index(rank) + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=WORLD) + group = dist.group.WORLD + dev = torch.device("cuda", rank) + from amd_kernels.attention.nvfp4_sparse_layer_gqa16_d128_p128_top16 import layer + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import decode_top16 + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import nvfp4_index_pages, score_width + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.split2.host import IndexSplit2 + + ql = args.ql + reqs = [int(n) for n in args.reqs.split(",")] + if args.lens_json: + table = json.load(open(args.lens_json)) + lens_by_n = {n: table[str(n)] for n in reqs} + else: + lens_by_n = {n: synthetic_lens(n) for n in reqs} + if rank == 0: + print(f"# synthetic lengths (no --lens-json): log-uniform {SYNTHETIC_RANGE[0]}-{SYNTHETIC_RANGE[1]} " + "quantiles", flush=True) + max_reqs = max(64, max(reqs)) + t0 = time.time() + index = decode_top16.IndexDecodeTop16(dev, query_len=ql, max_reqs=max_reqs, max_seq_len=MAX_MODEL_LEN, + num_index_heads=2, rank=rank, group=group) + assert index.split2 is not None, index.split_off + # a second two-GPU object: the device barrier between timed graphs + bar_cp = IndexSplit2(rank, group, dev, ql, 8192, ql) + biggest = max(sum(cdiv(s, BLK) for s in v) for v in lens_by_n.values()) + slots = max(cdiv(MIN_POOL_BYTES, PAGE_BYTES), 3 * biggest + 8) + icache = torch.empty(slots, BLK, 72, dtype=torch.uint8, device=dev) + g = torch.Generator(device="cuda") + for c0 in range(0, slots, 512): # the same pages on every rank (the replicated cache) + n = min(512, slots - c0) + g.manual_seed(1_000_003 + c0 // 512) + icache[c0 : c0 + n] = nvfp4_index_pages((torch.randn(n * BLK, D, device=dev, generator=g) * 2).bfloat16()) + g.manual_seed(17) + kv = torch.randint(0, 256, (slots, 2 * HKV, BLK, 72), dtype=torch.uint8, device=dev, generator=g) & 0x77 + torch.cuda.synchronize() + if rank == 0: + print(f"# ql {ql}; pool {slots} slots ({slots * (PAGE_BYTES + KV_PAGE_BYTES) / 2**30:.2f} GB per rank); " + f"setup + warm-up {time.time() - t0:.1f} s", flush=True) + rng_b = np.random.default_rng(7) + b_iq = torch.from_numpy(rng_b.standard_normal((ql, 2, D), dtype=np.float32)).cuda().to(torch.float8_e4m3fn) + b_tab = torch.arange(4, dtype=torch.int32, device=dev).view(1, 4) + b_seq = torch.tensor([4 * BLK], dtype=torch.int32, device=dev) + b_ids = torch.empty(2, ql, TOPK, dtype=torch.int32, device=dev) + + def barrier(): + dist.barrier() + bar_cp.score_topk(b_iq, icache, b_tab, b_seq, INIT, LOCAL, b_ids, 8192 * BLK) + + one = torch.ones((), dtype=torch.float32, device=dev) + sm = D**-0.5 + + def call(inp, o, side): + saved = dict(decode_top16.SPLIT2_MIN_REQS) + if side == "split2": + decode_top16.SPLIT2_MIN_REQS[ql] = 1 + try: + r = layer.index_sparse_decode( + index, inp["iq"], icache, inp["table"], inp["seq"], INIT, LOCAL, inp["q"], kv, o["ids"], inp["table"], + inp["seq"], sm, o["out"], k_scale=one, v_scale=one, decode_query_len=ql, max_seq_len=MAX_MODEL_LEN, + quant_out=o["quant"], split=side != "one_gpu", + ) + finally: + decode_top16.SPLIT2_MIN_REQS.clear() + decode_top16.SPLIT2_MIN_REQS.update(saved) + want = "split2" if side == "split2" else ("fused", "chain") + assert r in want, (side, r) + + sides = ["one_gpu", "split2"] + for n in reqs: + t0 = time.time() + lens = lens_by_n[n] + rows = n * ql + span = sum(cdiv(s, BLK) for s in lens) + regions = max(1, slots // max(span, 1)) + assert regions >= 3, ("pool too small", slots, span) + per_call = span * PAGE_BYTES + calls = max(100, 1 + cdiv(REUSE_BYTES, per_call)) + if calls >= regions: + calls = cdiv(calls, regions) * regions + inputs = [] + for i in range(calls): + rng = np.random.default_rng(1000 * n + i) + q4 = rng.standard_normal((rows, 2 * WORLD, D), dtype=np.float32) * 0.5 + iq = torch.from_numpy(np.ascontiguousarray(q4[:, 2 * rank : 2 * rank + 2])).cuda().to(torch.float8_e4m3fn) + q = (torch.from_numpy(rng.standard_normal((rows, H, D), dtype=np.float32)).cuda()).bfloat16() + inputs.append(dict(iq=iq, q=q, table=make_tables(lens, i % regions, span, slots, rng, MAX_MODEL_LEN // BLK), + seq=torch.tensor(lens, dtype=torch.int32, device=dev))) + outs = {} + for s in sides: + outs[s] = dict(ids=torch.empty(HKV, rows, TOPK, dtype=torch.int32, device=dev), + out=torch.empty(rows, H, D, dtype=torch.bfloat16, device=dev), + quant=layer.quant_out_buffers(rows, OPROJ_N, OPROJ_K, dev)) + runs = {} + for s in sides: + for inp in inputs[:2]: + call(inp, outs[s], s) + torch.cuda.synchronize() + barrier() + graphs = [] + chunks = max(1, calls // 100) + for k in range(chunks): + gr = torch.cuda.CUDAGraph() + with torch.cuda.graph(gr): + for inp in inputs[k * calls // chunks : (k + 1) * calls // chunks]: + call(inp, outs[s], s) + graphs.append(gr) + torch.cuda.synchronize() + runs[s] = graphs + check = {} + for i in (0, calls - 1): + got = [] + for s in sides: + o = outs[s] + o["ids"].fill_(-7) + o["out"].fill_(7.0) + if o["quant"] is not None: # the GEMM tile's padding rows stay as filled + o["quant"][0].fill_(0) + o["quant"][1].fill_(0) + barrier() + call(inputs[i], o, s) + torch.cuda.synchronize() + got.append((o["ids"].clone(), o["out"].clone(), None if o["quant"] is None else o["quant"][0].clone())) + (ia, oa, qa), (ib, ob, qb) = got + bad = (ia != ib).any(-1).sum().item() + bad_out = (oa.view(torch.int16) != ob.view(torch.int16)).flatten(1).any(-1).sum().item() + if qa is not None: + bad_out += int(not torch.equal(qa, qb)) + t = torch.tensor([bad, bad_out], dtype=torch.float64) + dist.all_reduce(t, op=dist.ReduceOp.MAX) + check[i] = [int(x) for x in t.tolist()] + for s in sides: + barrier() + for _ in range(2): + for gr in runs[s]: + gr.replay() + torch.cuda.synchronize() + samples = {s: [] for s in sides} + for k in range(args.rounds): + order = sides if k % 2 == 0 else sides[::-1] + for s in order: + el = 0.0 + for _ in range(args.replays): + for gr in runs[s]: + st, en = torch.cuda.Event(True), torch.cuda.Event(True) + barrier() + st.record() + gr.replay() + en.record() + torch.cuda.synchronize() + el += st.elapsed_time(en) + us = el * 1e3 / (args.replays * calls) + t = torch.tensor([us], dtype=torch.float64) + dist.all_reduce(t, op=dist.ReduceOp.MAX) + samples[s].append(t.item()) + if rank == 0: + med = {s: statistics.median(v) for s, v in samples.items()} + rec = dict(ql=ql, reqs=n, rows=rows, lens_json=os.path.basename(args.lens_json or "synthetic"), + mean_ctx=int(np.mean(lens)), + calls=calls, samples=samples, check=check, secs=round(time.time() - t0, 1)) + if args.out: + with open(args.out, "a") as f: + f.write(json.dumps(rec) + "\n") + rng_s = " ".join(f"{s}={med[s]:.2f} [{min(v):.2f}-{max(v):.2f}]" for s, v in samples.items()) + ratio = med["split2"] / med["one_gpu"] + print(f"ql={ql} N={n:3d} ctx~{int(np.mean(lens)):7d} {rng_s} split2/one={ratio:.3f} diff(ids,out)={check}", + flush=True) + del inputs, runs + torch.cuda.empty_cache() + dist.barrier() + dist.destroy_process_group() + + +def worker(rank, port, args, errfile): + try: + with torch.inference_mode(): + body(rank, port, args) + except BaseException: + with open(f"{errfile}.{rank}", "w") as f: + f.write(traceback.format_exc()) + raise + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--ql", type=int, default=8, help="decode rows per request (1 + the draft length)") + ap.add_argument("--lens-json", default=None, help='JSON {"N": [context per request]}; default: synthetic_lens') + ap.add_argument("--reqs", required=True, help="request counts N to time") + ap.add_argument("--rounds", type=int, default=4) + ap.add_argument("--replays", type=int, default=5) + ap.add_argument("--out", default=None, help="append JSON lines here") + args = ap.parse_args() + assert torch.cuda.device_count() >= WORLD, "needs two GPUs" + import torch.multiprocessing as mp + + errfile = f"/tmp/bench_index_decode_ranks2_err_{os.getpid()}" + try: + mp.spawn(worker, args=(free_port(), args, errfile), nprocs=WORLD, join=True) + except Exception: + for r in range(WORLD): + if os.path.exists(f"{errfile}.{r}"): + print(open(f"{errfile}.{r}").read(), file=sys.stderr) + raise + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py new file mode 100644 index 0000000..5eda86c --- /dev/null +++ b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-replay A/B of decode attention -> o_proj with the attention's MXFP8 epilogue on or off, at the served +compute (fp16) and per-GPU heads: ``--hkv 1`` TP4 (o_proj N 6144 x K 2048), ``--hkv 2`` TP2 (K 4096); the crossover +behind the layer's ``QUANT_OUT_MIN_ROWS``. + +- ``bf16``: the attention writes bf16, ``gemm.mxfp8_linear`` quantizes and multiplies. +- ``mxfp8``: the attention writes the codes in ``gemm.prequant_code_tile_rows``' layout and the AITER-layout scales; + the GEMM reads them. + +Cases: ``benchmark_sparse_decode.make_calls`` (60K-100K contexts, the measured top-k overlap, one copy of the pages per +call); one graph per chain of ``--calls`` launches, each with its own inputs and one of ``--weights`` o_proj weights +(beyond the MALL). Rounds alternate the chains; median us per call. Before timing the MXFP8 bytes are checked against +the served quantizer of the bf16 output. + + python3 benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_oproj_input.py --hkv 2 \ + [--ql 8 4] [--reqs 1 2 4] +""" + +import argparse +import statistics + +import torch +from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import nvfp4_sparse_decode +from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16.reference import ( + aiter_scale_offsets, + tile_code_offsets, +) +from amd_kernels.gemm.mxfp8_common import gemm as G + +from benchmarks.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import benchmark_sparse_decode as sd + +N = 6144 +REQS = {8: [1, 2, 3, 4, 6, 8, 16, 32, 64], 4: [1, 2, 4, 8, 16, 32, 64], 1: [1, 2, 3, 4, 6, 8]} + + +def make_weights(count, k, seed=5): + gen = torch.Generator(device="cuda").manual_seed(seed) + out = [] + for _ in range(count): + w = (torch.randn((N, k), generator=gen, device="cuda") * 0.5).to(torch.float8_e4m3fn) + s = torch.randint(118, 124, (N, k // 32), generator=gen, device="cuda").to(torch.uint8) + out.append(G.preshuffle_mxfp8_weight(w, s)) + return out + + +def gemm_codes(codes, scales, tile, rows, w, ws): + c8 = codes.view(torch.float8_e4m3fn) + if tile: + return G.mxfp8_gemm_shuffled(c8, scales, w, ws, torch.bfloat16, rows) + return G.mxfp8_gemm(c8[:rows], scales, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + + +def mismatches(codes, scales, tile, out2d): + rows, k = out2d.shape + if tile: + want_c, want_s = G.mxfp8_e4m3_quantize_shuffled(out2d, tile) + offs = tile_code_offsets(rows, k, codes.device) + got_c, ref_c = codes.view(-1)[offs], want_c.view(torch.uint8).view(-1)[offs] + else: + want_c, want_s = G.mxfp8_e4m3_quantize_aiter(out2d) + got_c, ref_c = codes[:rows], want_c.view(torch.uint8)[:rows] + so = aiter_scale_offsets(rows, k, codes.device) + return int((got_c != ref_c).sum()) + int((scales.view(-1)[so] != want_s.view(torch.uint8).view(-1)[so]).sum()) + + +def capture(fns): + for f in fns: + f() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for f in fns: + f() + torch.cuda.synchronize() + return g + + +def time_graphs(graphs, calls, rounds, replays=10): + for g in graphs.values(): + g.replay() + torch.cuda.synchronize() + times = {k: [] for k in graphs} + for _ in range(rounds): + for name, g in graphs.items(): + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(replays): + g.replay() + e.record() + torch.cuda.synchronize() + times[name].append(s.elapsed_time(e) * 1e3 / (replays * calls)) + return {k: statistics.median(v) for k, v in times.items()} + + +def run_case(hkv, ql, reqs, calls, weights, rounds): + k = hkv * 2048 + rows = reqs * ql + shared, per_call = sd.make_calls(f"kv{hkv}_f16_ql{ql}", reqs, calls, seed=reqs * 7 + ql) + codes, scales, tile = G.prequant_buffers(rows, N, k, "cuda") + out = torch.empty((rows, 16 * hkv, 128), dtype=torch.bfloat16, device="cuda") + + def attn(i, quant): + c = per_call[i] + nvfp4_sparse_decode( + c["q"], shared["kv_cache"], shared["topk_idx"], c["block_table"], shared["seq_lens"], sd.SM_SCALE, out, + k_scale=shared["k_scale"], v_scale=shared["v_scale"], decode_query_len=ql, + quant_out=(codes, scales, tile) if quant else None, compute_type=torch.float16, + ) + + def chain_bf16(i): + attn(i, False) + w, ws = weights[i % len(weights)] + G.mxfp8_linear(out.view(rows, k), w, ws, torch.bfloat16) + + def chain_mxfp8(i): + attn(i, True) + w, ws = weights[i % len(weights)] + gemm_codes(codes, scales, tile, rows, w, ws) + + attn(0, False) + out2d = out.view(rows, k).clone() + attn(0, True) + torch.cuda.synchronize() + bad = mismatches(codes, scales, tile, out2d) + graphs = { + "bf16": capture([lambda i=i: chain_bf16(i) for i in range(calls)]), + "mxfp8": capture([lambda i=i: chain_mxfp8(i) for i in range(calls)]), + } + t = time_graphs(graphs, calls, rounds) + delta = 100 * (t["mxfp8"] / t["bf16"] - 1) + print(f"| {hkv} | {ql} | {reqs} | {rows} | {tile} | {t['bf16']:.2f} | {t['mxfp8']:.2f} | {delta:+.1f}% | {bad} |", + flush=True) + del graphs, shared, per_call + torch.cuda.empty_cache() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--hkv", type=int, default=2) + p.add_argument("--ql", type=int, nargs="*", default=[8, 4]) + p.add_argument("--reqs", type=int, nargs="*") + p.add_argument("--calls", type=int, default=100) + p.add_argument("--weights", type=int, default=24) + p.add_argument("--rounds", type=int, default=7) + a = p.parse_args() + props = torch.cuda.get_device_properties(0) + assert "gfx950" in props.gcnArchName, props.gcnArchName + print(f"{props.name}, o_proj N {N} x K {a.hkv * 2048}") + print("| KV heads | ql | requests | rows | code tile | bf16 chain us | mxfp8 chain us | mxfp8 vs bf16 " + "| byte mismatches |") + print("|---|---:|---:|---:|---:|---:|---:|---:|---:|") + weights = make_weights(a.weights, a.hkv * 2048) + with torch.inference_mode(): + for ql in a.ql: + for reqs in a.reqs or REQS[ql]: + run_case(a.hkv, ql, reqs, a.calls, weights, a.rounds) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_sparse_decode.py b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_sparse_decode.py index fc815ab..d1b8928 100644 --- a/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_sparse_decode.py +++ b/benchmarks/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/benchmark_sparse_decode.py @@ -48,6 +48,12 @@ "kv1_f16_ql1": [1, 16, 64, 128, 256], "kv1_f16_ql4": [1, 8, 16, 32, 64, 128], "kv1_f16_ql8": [1, 2, 4, 16, 32, 64, 128, 256], + "kv2_bf16_ql1": [1, 16, 64, 128], + "kv2_bf16_ql4": [1, 2, 4, 8, 16, 32, 64, 128], + "kv2_bf16_ql8": [1, 2, 4, 8, 16, 32, 64, 128], + "kv2_f16_ql1": [1, 16, 64, 128], + "kv2_f16_ql4": [1, 2, 4, 8, 16, 32, 64, 128], + "kv2_f16_ql8": [1, 2, 4, 8, 16, 24, 32, 40, 48, 52, 64, 128], } UNION_BLOCKS = 25 # measured: distinct blocks per 4 query rows PAGE_HEAD_BYTES = 2 * 9216 # one KV head's K and V of one page @@ -142,7 +148,7 @@ def make_calls(config, reqs, calls, seed): return shared, per_call -def launch(shared, call, quant, splits=None): +def launch(shared, call, quant, splits=None, tail_splits=None): nvfp4_sparse_decode( call["q"], shared["kv_cache"], @@ -157,6 +163,7 @@ def launch(shared, call, quant, splits=None): quant_out=(call["codes"], call["scales"], 16) if quant else None, compute_type=shared["compute"], splits=splits, + tail_splits=tail_splits, ) @@ -184,15 +191,15 @@ def check(shared, call, quant): assert rel < QUANT_REL, rel -def timeit_graph(shared, per_call, quant, splits=None, replays=10, rounds=5, warmup=5): +def timeit_graph(shared, per_call, quant, splits=None, replays=10, rounds=5, warmup=5, tail_splits=None): """Median over rounds of the mean time per call (us) of one graph of every call.""" for call in per_call: - launch(shared, call, quant, splits) + launch(shared, call, quant, splits, tail_splits) torch.cuda.synchronize() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): for call in per_call: - launch(shared, call, quant, splits) + launch(shared, call, quant, splits, tail_splits) torch.cuda.synchronize() for _ in range(warmup): graph.replay() @@ -218,6 +225,9 @@ def main() -> None: "--splits", type=int, nargs="*", help="time these explicit split counts too " "(the padded grid; the automatic schedule is the first row of each case)" ) + parser.add_argument( + "--tail-splits", type=int, nargs="*", help="time the tail grid with these leftover split counts too" + ) args = parser.parse_args() props = torch.cuda.get_device_properties(0) if "gfx950" not in props.gcnArchName: @@ -236,7 +246,7 @@ def main() -> None: torch.cuda.synchronize() check(shared, per_call[0], quant) us = timeit_graph(shared, per_call, quant) - splits, _, xcds, _, _, tail = plan(reqs, ql, hkv, compute, quant) + splits, _, xcds, _, _, tail = plan(reqs, ql, hkv, compute, quant)[:6] mb = shared["kv_bytes"] / 1e6 rows = [(f"{splits} / {xcds or '-'} / {'yes' if tail else 'no'}", us)] for forced in args.splits or []: @@ -245,9 +255,15 @@ def main() -> None: check(shared, per_call[0], quant) t = timeit_graph(shared, per_call, quant, forced) rows.append((f"{forced} (forced)", t)) + for forced in args.tail_splits or []: + launch(shared, per_call[0], quant, tail_splits=forced) + torch.cuda.synchronize() + check(shared, per_call[0], quant) + t = timeit_graph(shared, per_call, quant, tail_splits=forced) + rows.append((f"tail {forced} (forced)", t)) for key, t in rows: print( - f"| {config} | {reqs} | {reqs * ql} | {output} | {key} | {t:.1f} " + f"| {config} | {reqs} | {reqs * ql} | {output} | {key} | {t:.2f} " f"| {mb:.1f} | {shared['kv_bytes'] / t / 1e3:.0f} |", flush=True, ) diff --git a/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py b/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py index fde15b8..59f3b16 100644 --- a/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py +++ b/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/benchmark_sparse_prefill.py @@ -25,19 +25,25 @@ import torch from amd_kernels.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16 import nvfp4_sparse_prefill, reference -from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import BLOCK, GROUP, HEAD_DIM, SM_SCALE, TP4_HKV, shared_picks +from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import ( + BLOCK, GROUP, HEAD_DIM, SM_SCALE, TP2_HKV, TP4_HKV, shared_picks, +) PREFIX = 16384 N_INPUTS = 100 # Q + outputs of all inputs of a case; the largest steps get fewer than 100 inputs # (each of their calls already moves more than 256 MB) BUDGET_BYTES = 64 * 1024**3 -# (KV heads, prefill tokens per step): TP4 up to 65536 tokens +# (KV heads, prefill tokens per step): TP4 and TP2 up to 65536 tokens CASES = [ (TP4_HKV, 1024), (TP4_HKV, 4096), (TP4_HKV, 16384), (TP4_HKV, 65536), + (TP2_HKV, 256), + (TP2_HKV, 512), + (TP2_HKV, 2048), + (TP2_HKV, 32768), ] diff --git a/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/sweep_tiles.py b/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/sweep_tiles.py new file mode 100644 index 0000000..40e0bdc --- /dev/null +++ b/benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/sweep_tiles.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The small-tile crossovers (``host._local_tile``): every fp16 tile forced at one token count and KV head count, timed +as ``benchmark_sparse_prefill.py`` times (graph replay, 100 cold inputs, 16K prefix, plain bf16 output), plus the +dispatch's own choice (``auto``). The first input is checked against ``reference.py``. + + python3 benchmarks/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/sweep_tiles.py --hkv 2 --tokens 128,129,256 +""" + +import argparse +import statistics + +import numpy as np +import torch + +from amd_kernels.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16 import host +from benchmarks.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.benchmark_sparse_prefill import ( + check, + make_inputs, + n_inputs, + timeit_graph, +) +from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import SM_SCALE + +# tq1 (eager), tq2 / shared-Q tq2 / tq3 (lazy), the four-query kernel one round (eager) and two rounds (lazy) +TILES = { + "tq1": lambda: host._get_bf16_local(1, False, False, True), + "tq2": lambda: host._get_bf16_local(2, False, False, True, lazy=True), + "tq2_qlds": lambda: host._get_bf16_local(2, False, False, True, lazy=True, q_shared=True), + "tq3": lambda: host._get_bf16_local(3, False, False, True, lazy=True), + "tq4_r1": lambda: host._get_kernel("fp16", False, 1, False, sched=False), + "tq4_r2": lambda: host._get_kernel("fp16", False, 2, False, sched=False, lazy=True), +} + + +def forced(exe, inp, cache, req, pos, hkv, out): + """One call of ``exe`` with ``nvfp4_sparse_prefill``'s plain-output arguments.""" + q, unit = inp["q"], host._unit_scale(inp["q"].device) + host._run_compiled( + exe, q.data_ptr(), cache.data_ptr(), inp["topk"].data_ptr(), inp["table"].data_ptr(), req.data_ptr(), + pos.data_ptr(), out.data_ptr(), unit.data_ptr(), unit.data_ptr(), 0, q.shape[0], q.shape[1], hkv, + inp["topk"].stride(0), inp["table"].stride(0), cache.stride(0), + float(np.float32(SM_SCALE) * host.LOG2E_F32), torch.cuda.current_stream(), + ) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--hkv", type=int, default=2) + ap.add_argument("--tokens", default="64,128,129,256,257,384,385,512,513,768,769,1024") + ap.add_argument("--tiles", default=",".join(TILES) + ",auto") + ap.add_argument("--inputs", type=int, default=100) + ap.add_argument("--rounds", type=int, default=2, help="alternated rounds") + args = ap.parse_args() + hkv, names = args.hkv, args.tiles.split(",") + print("hkv | tokens | " + " | ".join(f"{v} us" for v in names) + " | fastest", flush=True) + for t in (int(x) for x in args.tokens.split(",")): + cache, pos, req, inputs = make_inputs(hkv, t, n_inputs(hkv, t, len(names), args.inputs)) + calls = {} + for v in names: + calls[v] = [] + for i, inp in enumerate(inputs): + out = torch.empty_like(inp["q"]) + if v == "auto": + f = (lambda inp=inp, out=out: host.nvfp4_sparse_prefill( + inp["q"], cache, inp["topk"], inp["table"], req, pos, hkv, SM_SCALE, out, + lazy_rescale=True)) + else: + exe = TILES[v]() + f = (lambda inp=inp, out=out, exe=exe: forced(exe, inp, cache, req, pos, hkv, out)) + if i == 0 and t >= 8: # the reference's sampled rows need a few tokens + f() + check(cache, pos, req, inp, hkv, "fp16", out) + calls[v].append(f) + times = {v: [] for v in names} + for r in range(args.rounds): + for v in (names if r % 2 == 0 else names[::-1]): + times[v].append(timeit_graph(calls[v]) * 1e3) + med = {v: statistics.median(times[v]) for v in names} + best = min((v for v in names if v != "auto"), key=med.get) + print(f"{hkv} | {t} | " + " | ".join(f"{med[v]:.2f}" for v in names) + f" | {best}", flush=True) + del calls, cache, inputs + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py b/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py index 11b51f4..902d0ce 100644 --- a/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py +++ b/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_decode.py @@ -12,7 +12,8 @@ Cases (README, "What it does"): TP4 spec3 (16 / 1 heads, 4 rows, bf16 output, 128-token pages); the TP4 DSpark draft's 7-row block, causal and non-causal (the -upstream vLLM draft). The MXFP8 epilogue with o_proj: ``benchmarks/attention/bench_attn_oproj_mxfp8.py``. +upstream vLLM draft); TP2 (32 / 2 heads) DSpark 7 and DSpark 3 blocks. The MXFP8 epilogue with o_proj: +``benchmarks/attention/bench_attn_oproj_mxfp8.py`` (TP4), ``benchmark_oproj_input.py`` (TP2). """ import argparse @@ -37,6 +38,9 @@ ("tp4_spec3_p128_bf16", 4, 1, 128, [1, 16, 64, 128, 256]), ("tp4_dspark7_p128_bf16", 7, 1, 128, [1, 2, 4, 16, 32, 64]), ("tp4_dspark7_noncausal_p128_bf16", 7, 1, 128, [1, 2, 4, 8, 11, 16, 24, 32, 48, 64], False), + ("tp2_dspark7_noncausal_p128_bf16", 7, 2, 128, [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 96, 128, 256], False), + ("tp2_dspark7_p128_bf16", 7, 2, 128, [1, 4, 16, 32, 48, 64, 128]), + ("tp2_dspark3_noncausal_p128_bf16", 3, 2, 128, [1, 4, 16, 32, 48, 64, 128], False), ] @@ -115,7 +119,7 @@ def call(i): check_close(out[sl], ref[sl], f"{name} R={reqs} launch {i} KV head {h}") us = timeit_graph([lambda i=i: call(i) for i in range(copies)]) * 1e3 kv_bytes = reqs * hkv * (W - 1 + ql) * TOK_B - splits = host.launch_splits(launches[0][3], pool, reqs) + splits = host.launch_splits(launches[0][3], pool, reqs, max_query_len=ql, window=W) print( f"{name:22s} R={reqs:3d} rows={rows:4d} splits={splits}: " f"{us:7.2f} us, {kv_bytes / us / 1e3:6.0f} GB/s KV" diff --git a/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py b/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py new file mode 100644 index 0000000..7ff8c4e --- /dev/null +++ b/benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-replay A/B of the TP2 draft attention -> o_proj chain (2 KV heads, 32 query heads, o_proj N 6144 x K 4096), +K / V and weights cold: where the MXFP8 epilogue (``oproj_input.MXFP8_MIN_ROWS``) pays. + +- ``bf16``: the attention writes bf16; ``gemm.mxfp8_linear`` quantizes it and runs the GEMM. +- ``mxfp8``: the attention's MXFP8 epilogue writes the codes in ``gemm.prequant_code_tile_rows``' layout and the + AITER-layout scales; the GEMM reads them. + +7-row (DSpark 7) and 3-row (DSpark 3) non-causal blocks, window 4096 (``benchmark_decode.make_launches``: a 1 GiB +pool, disjoint pages per call). One graph per chain holds ``--calls`` launches, each with its own inputs and one of +``--weights`` o_proj weights (40 x 25 MB > the 256 MB MALL); rounds alternate the chains, median us per call. Before +timing the codes / scales equal the served quantizer of the bf16 output. + + python3 benchmarks/attention/swa_decode_fp8kv_gqa16_d128/benchmark_oproj_input.py [--ql 7 3] [--reqs 1 2 4] +""" + +import argparse +import statistics + +import torch +from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16.reference import ( + aiter_scale_offsets, + tile_code_offsets, +) +from amd_kernels.attention.swa_decode_fp8kv_gqa16_d128 import swa_decode_fp8kv +from amd_kernels.gemm.mxfp8_common import gemm as G + +from benchmarks.attention.swa_decode_fp8kv_gqa16_d128 import benchmark_decode as swa + +N, K, HKV = 6144, 4096, 2 +SM_SCALE = 128**-0.5 +REQS = {7: [1, 2, 3, 4, 8, 16, 32, 48, 64], 3: [1, 2, 3, 4, 8, 16, 32, 64]} + + +def make_weights(count, seed): + gen = torch.Generator(device="cuda").manual_seed(seed) + out = [] + for _ in range(count): + w = (torch.randn((N, K), generator=gen, device="cuda") * 0.5).to(torch.float8_e4m3fn) + s = torch.randint(118, 124, (N, K // 32), generator=gen, device="cuda").to(torch.uint8) + out.append(G.preshuffle_mxfp8_weight(w, s)) + return out + + +def gemm_codes(codes, scales, tile, rows, w, ws): + c8 = codes.view(torch.float8_e4m3fn) + if tile: + return G.mxfp8_gemm_shuffled(c8, scales, w, ws, torch.bfloat16, rows) + return G.mxfp8_gemm(c8[:rows], scales, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + + +def capture(fns): + for f in fns: + f() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for f in fns: + f() + torch.cuda.synchronize() + return g + + +def run_case(ql, reqs, calls, weights, rounds): + rows = reqs * ql + tile = G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) + pad = max(tile, 16) + rows_p = (rows + pad - 1) // pad * pad + codes = torch.empty((rows_p, K), dtype=torch.uint8, device="cuda") + scales = torch.empty(((rows_p + 31) // 32 * 32, K // 32), dtype=torch.uint8, device="cuda") + out = torch.empty((rows, HKV * 16, 128), dtype=torch.bfloat16, device="cuda") + pool, launches = swa.make_launches(reqs, ql, HKV, 128, calls, seed=reqs * 100 + ql) + one = torch.ones((), dtype=torch.float32, device="cuda") + + def attn(i, quant): + bt, seq, qsl, q = launches[i] + swa_decode_fp8kv(q, pool, bt, seq, qsl, ql, SM_SCALE, 4096, False, one, one, None if quant else out, + quant_out=(codes, scales, tile) if quant else None) + + def chain_bf16(i): + attn(i, False) + w, ws = weights[i % len(weights)] + G.mxfp8_linear(out.view(rows, K), w, ws, torch.bfloat16) + + def chain_mxfp8(i): + attn(i, True) + w, ws = weights[i % len(weights)] + gemm_codes(codes, scales, tile, rows, w, ws) + + attn(0, False) + out2d = out.view(rows, K).clone() + codes.fill_(0xA5) + scales.fill_(0xA5) + attn(0, True) + torch.cuda.synchronize() + want_c, want_s = G.mxfp8_e4m3_quantize_shuffled(out2d, tile) if tile else G.mxfp8_e4m3_quantize_aiter(out2d) + if tile: + offs = tile_code_offsets(rows, K, codes.device) + got_c, ref_c = codes.view(-1)[offs], want_c.view(torch.uint8).view(-1)[offs] + else: + got_c, ref_c = codes[:rows], want_c.view(torch.uint8)[:rows] + so = aiter_scale_offsets(rows, K, codes.device) + assert torch.equal(got_c, ref_c) and torch.equal(scales.view(-1)[so], want_s.view(torch.uint8).view(-1)[so]) + graphs = { + "bf16": capture([lambda i=i: chain_bf16(i) for i in range(calls)]), + "mxfp8": capture([lambda i=i: chain_mxfp8(i) for i in range(calls)]), + } + for g in graphs.values(): + g.replay() + torch.cuda.synchronize() + times = {k: [] for k in graphs} + for _ in range(rounds): + for name, g in graphs.items(): + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(10): + g.replay() + e.record() + torch.cuda.synchronize() + times[name].append(s.elapsed_time(e) * 1e3 / (10 * calls)) + bf16, mx = statistics.median(times["bf16"]), statistics.median(times["mxfp8"]) + print(f"ql={ql} R={reqs:3d} rows={rows:4d} tile={tile:3d}: bf16 {bf16:7.2f} us, mxfp8 {mx:7.2f} us " + f"({100 * (mx / bf16 - 1):+.1f}%)", flush=True) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--ql", type=int, nargs="*", default=[7, 3]) + p.add_argument("--reqs", type=int, nargs="*") + p.add_argument("--calls", type=int, default=100) + p.add_argument("--weights", type=int, default=40) + p.add_argument("--rounds", type=int, default=7) + a = p.parse_args() + assert "gfx950" in torch.cuda.get_device_properties(0).gcnArchName + weights = make_weights(a.weights, 5) + with torch.inference_mode(): + for ql in a.ql: + for reqs in a.reqs or REQS[ql]: + run_case(ql, reqs, a.calls, weights, a.rounds) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/collectives/all_gather_i64_c7_ranks2/benchmark_gather.py b/benchmarks/collectives/all_gather_i64_c7_ranks2/benchmark_gather.py new file mode 100644 index 0000000..787abf3 --- /dev/null +++ b/benchmarks/collectives/all_gather_i64_c7_ranks2/benchmark_gather.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank graph replay of I64C7AllGather against the padded RCCL all-gather + index_select it replaces: 100 +distinct input sets per graph, five alternated rounds of ten replays, MAX rank, median us. +""" +import socket +import statistics + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from amd_kernels.collectives.all_gather_i64_c7_ranks2 import I64C7AllGather + +WORLD = 2 + + +def capture(calls): + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for c in calls: + c() + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for c in calls: + c() + g.replay() + torch.cuda.synchronize() + return g + + +def time_us(g, n): + dist.barrier() + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(10): + g.replay() + b.record() + b.synchronize() + t = torch.tensor([a.elapsed_time(b) * 1000 / (10 * n)], device="cuda") + dist.all_reduce(t, op=dist.ReduceOp.MAX) + return t.item() + + +def run(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", world_size=WORLD, rank=rank, device_id=dev) + gloo = dist.new_group(backend="gloo") + comm = I64C7AllGather(rank, gloo, dev) + comm.warmup() + gen = torch.Generator(device=dev).manual_seed(rank + 7200) + for R in (10, 16, 32, 40, 48, 52, 64, 128, 256): + counts = [R // WORLD + int(r < R % WORLD) for r in range(WORLD)] + n, stride = counts[rank], max(counts) + src = torch.tensor([i % WORLD * stride + i // WORLD for i in range(R)], device=dev, dtype=torch.int64) + sets = [(torch.randint(0, 200064, (stride, 7), generator=gen, device=dev, dtype=torch.int64), src, stride, + torch.empty(R, 7, device=dev, dtype=torch.int64)) for _ in range(100)] + gathered = torch.empty(WORLD * stride, 7, dtype=torch.int64, device=dev) + + def baseline(x, src, stride, out): + dist.all_gather_into_tensor(gathered, x) + torch.index_select(gathered, 0, src, out=out) + + graphs = [capture([lambda a=a: baseline(*a) for a in sets]), + capture([lambda a=a: comm(a[0][:n], *a[1:]) for a in sets])] + ts = [[], []] + for _ in range(5): + for i in (0, 1): + ts[i].append(time_us(graphs[i], len(sets))) + if rank == 0: + print(f"R {R}: baseline {statistics.median(ts[0]):.2f} us, this entry {statistics.median(ts[1]):.2f} " + f"[{min(ts[1]):.2f}-{max(ts[1]):.2f}] us", flush=True) + del graphs + comm.close() + dist.destroy_process_group() + + +if __name__ == "__main__": + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + mp.spawn(run, args=(port,), nprocs=WORLD, join=True) diff --git a/benchmarks/collectives/all_gather_sampler_i64_c8_ranks2/benchmark_gather.py b/benchmarks/collectives/all_gather_sampler_i64_c8_ranks2/benchmark_gather.py new file mode 100644 index 0000000..013545a --- /dev/null +++ b/benchmarks/collectives/all_gather_sampler_i64_c8_ranks2/benchmark_gather.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank graph replay of SamplerAllGather against vLLM's gather_sampler_output (its Triton pack, an RCCL +all-gather, its Triton unpack): 100 distinct input sets per graph, five alternated rounds of ten replays, MAX rank, +median us. +""" +import socket +import statistics + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2 import SamplerAllGather + +WORLD = 2 + + +def capture(calls): + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + for c in calls: + c() + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for c in calls: + c() + g.replay() + torch.cuda.synchronize() + return g + + +def time_us(g, n): + dist.barrier() + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(10): + g.replay() + b.record() + b.synchronize() + t = torch.tensor([a.elapsed_time(b) * 1000 / (10 * n)], device="cuda") + dist.all_reduce(t, op=dist.ReduceOp.MAX) + return t.item() + + +def run(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", world_size=WORLD, rank=rank, device_id=dev) + gloo = dist.new_group(backend="gloo") + from vllm.v1.worker.gpu.sample.batch_shard import _pack_sampler_output_kernel, _unpack_gathered_output_kernel + comm = SamplerAllGather(rank, gloo, dev) + comm.warmup() + gen = torch.Generator(device=dev).manual_seed(rank + 7200) + for R in (10, 16, 32, 40, 48, 52, 64, 128, 256): + counts = [R // WORLD + int(r < R % WORLD) for r in range(WORLD)] + n, stride = counts[rank], max(counts) + src = torch.tensor([i % WORLD * stride + i // WORLD for i in range(R)], device=dev, dtype=torch.int64) + cu = torch.arange(0, n * 8 + 1, 8, device=dev, dtype=torch.int32) + sets = [] + for _ in range(100): + tok = torch.randint(0, 200064, (n, 8), generator=gen, device=dev, dtype=torch.int64) + ns = torch.randint(1, 9, (n,), generator=gen, device=dev, dtype=torch.int32) + nan = torch.randint(0, 10, (n * 8,), generator=gen, device=dev, dtype=torch.int32) + out = (torch.empty(R, 8, device=dev, dtype=torch.int64), + *[torch.empty(R, device=dev, dtype=torch.int32) for _ in range(3)]) + sets.append((tok, ns, 8 - ns, nan, cu, src, stride, *out)) + packed = torch.empty(stride, 11, dtype=torch.int64, device=dev) + gathered = torch.empty(WORLD * stride, 11, dtype=torch.int64, device=dev) + + def baseline(tok, ns, nr, nan, cu, src, stride, ot, os_, or_, on): + if n: + _pack_sampler_output_kernel[(n,)](packed, packed.stride(0), tok, tok.stride(0), ns, nr, nan, cu, 8, 8, + BLOCK_SIZE=8) + dist.all_gather_into_tensor(gathered, packed) + _unpack_gathered_output_kernel[(R,)](gathered, gathered.stride(0), src, ot, os_, or_, on, 8, + BLOCK_SIZE=8) + + graphs = [capture([lambda a=a: baseline(*a) for a in sets]), capture([lambda a=a: comm(*a) for a in sets])] + ts = [[], []] + for _ in range(5): + for i in (0, 1): + ts[i].append(time_us(graphs[i], len(sets))) + if rank == 0: + print(f"R {R}: baseline {statistics.median(ts[0]):.2f} us, this entry {statistics.median(ts[1]):.2f} " + f"[{min(ts[1]):.2f}-{max(ts[1]):.2f}] us", flush=True) + del graphs + comm.close() + dist.destroy_process_group() + + +if __name__ == "__main__": + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + mp.spawn(run, args=(port,), nprocs=WORLD, join=True) diff --git a/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/benchmark_exchange.py b/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/benchmark_exchange.py new file mode 100644 index 0000000..edef71c --- /dev/null +++ b/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/benchmark_exchange.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: five alternated graph A/B rounds, MAX rank, dense and random request slots. + +Baseline: RCCL uint8 all_to_all_single + torch permute/reshape, exactly the +row transport of vLLM batch_shard.all_to_all_rows (no vLLM dependency). +Inputs rotate over >=256 MB. No allocations/fills enter the FlyDSL graph. Also prints the per-link rate: +the larger rank's received peer bytes (counts[r] x 100032 x 2 B over the one xGMI link) / time. +""" +import sys +import socket +import statistics +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll +from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.reference import N, WORLD, merge_received_reference + + +def _capture(calls): + for c in calls[:2]: + c() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for c in calls: + c() + torch.cuda.synchronize() + return graph + + +def _time(graph, calls): + dist.barrier() + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(10): + graph.replay() + b.record() + b.synchronize() + value = torch.tensor(a.elapsed_time(b) * 1000 / (10 * calls), device="cuda") + dist.all_reduce(value, op=dist.ReduceOp.MAX) + return value.item() + + +def _fmt(values): + return f"{statistics.median(values):.2f} [{min(values):.2f}-{max(values):.2f}]" + + +def _worker(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", world_size=WORLD, rank=rank, device_id=dev) + group = dist.new_group(backend="gloo") + comm = RowAllToAll(rank, group, dev) + comm.warmup() + generator = torch.Generator(device=dev).manual_seed(777 + rank) + for random_slots in (False, True): + for R in REQS: + slots = torch.randperm(256, generator=torch.Generator().manual_seed(1000 * R))[:R] if random_slots else torch.arange(R) + owners = torch.bincount(slots % WORLD, minlength=WORLD).tolist() + for steps in (8, 7): + counts = [o * steps for o in owners] + cs = torch.tensor(counts, dtype=torch.int32, device=dev) + n = counts[rank] + nsets = max(4, min(100, 1_000_000_000 // (R * steps * N * 2))) + xs = [torch.randint(-32768, 32768, (R * steps, N), device=dev, dtype=torch.int16, + generator=generator).view(torch.bfloat16) for _ in range(nsets)] + out = torch.empty(n, WORLD * N, dtype=torch.bfloat16, device=dev) + recv = torch.empty(WORLD * n * N * 2, dtype=torch.uint8, device=dev) + def baseline(x): + dist.all_to_all_single(recv, x.view(torch.uint8).flatten(), + output_split_sizes=[n * N * 2] * WORLD, + input_split_sizes=[c * N * 2 for c in counts]) + return merge_received_reference(recv.view(torch.bfloat16).view(WORLD, n, N)) + expected = baseline(xs[0]) + comm(xs[0], cs, out) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), expected.view(torch.int16)) + graphs = [_capture([lambda x=x: baseline(x) for x in xs]), + _capture([lambda x=x: comm(x, cs, out) for x in xs])] + timings = [[], []] + for _ in range(5): + for i in (0, 1): + timings[i].append(_time(graphs[i], nsets)) + if rank == 0: + faster = all(t > o for t, o in zip(*timings)) + link = max(counts) * N * 2 / statistics.median(timings[1]) / 1e3 + print(f"R={R} rows/request={steps} random={random_slots} owners={owners} " + f"baseline={_fmt(timings[0])} ours={_fmt(timings[1])} faster5/5={faster} " + f"link={link:.1f} GB/s", flush=True) + del graphs, xs, recv, out, expected + torch.cuda.empty_cache() + comm.close() + dist.destroy_process_group() + + +REQS = [int(v) for v in sys.argv[1:]] or [1, 4, 10, 16, 32, 40, 48, 52, 64, 128, 256] + +if __name__ == "__main__": + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + mp.spawn(_worker, args=(port,), nprocs=WORLD, join=True) diff --git a/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/sweep_pull.py b/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/sweep_pull.py new file mode 100644 index 0000000..acc2732 --- /dev/null +++ b/benchmarks/collectives/all_to_all_bf16_n100032_ranks2/sweep_pull.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: the pull grid (remote, local workgroups) of ``RowAllToAll`` per owned-row count; stage + pull, graph +replay, MAX over ranks, median us and the link rate, the five best grids per row count (``host.PULL``). + + python3 benchmarks/collectives/all_to_all_bf16_n100032_ranks2/sweep_pull.py [owned rows ...] +""" +import socket +import statistics +import sys + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll, kernel +from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.reference import N, WORLD + +GRIDS = [(4, 2), (8, 2), (8, 4), (12, 4), (16, 2), (16, 4), (16, 8), (24, 4), (24, 8), (32, 8), (48, 8), (64, 8), + (64, 16), (96, 16), (128, 16), (128, 32), (192, 32)] + + +def run(rank, port, owned): + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("gloo", init_method=f"tcp://127.0.0.1:{port}", world_size=WORLD, rank=rank) + comm = RowAllToAll(rank, dist.group.WORLD, dev, max_rows=2048) + comm.warmup() + pulls = {gr: kernel.compile_pull(*gr) for gr in GRIDS} + gen = torch.Generator(device=dev).manual_seed(rank) + for own in owned: + m = own * WORLD + cs = torch.tensor([own] * WORLD, dtype=torch.int32, device=dev) + nsets = max(4, min(50, 600_000_000 // (m * N * 2))) + xs = [torch.randint(-32768, 32768, (m, N), device=dev, dtype=torch.int16, generator=gen).view(torch.bfloat16) + for _ in range(nsets)] + out = torch.empty(own, WORLD * N, dtype=torch.bfloat16, device=dev) + res = [] + for gr in GRIDS: + comm.pull_exe = pulls[gr] + for x in xs[:2]: + comm(x, cs, out) + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for x in xs: + comm(x, cs, out) + g.replay() + torch.cuda.synchronize() + ts = [] + for _ in range(5): + dist.barrier() + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(5): + g.replay() + b.record() + b.synchronize() + t = torch.tensor([a.elapsed_time(b) * 1000 / (5 * nsets)]) + dist.all_reduce(t, op=dist.ReduceOp.MAX) + ts.append(t.item()) + res.append((statistics.median(ts), gr)) + del g + res.sort() + if rank == 0: + print(f"own {own}: " + " | ".join(f"{gr} {t:.1f} ({own * N * 2 / t / 1e3:.1f} GB/s)" for t, gr in res[:5]), + flush=True) + del xs, out + torch.cuda.empty_cache() + comm.close() + dist.destroy_process_group() + + +if __name__ == "__main__": + owned = [int(v) for v in sys.argv[1:]] or [4, 8, 16, 24, 32, 48, 64, 128, 256, 512] + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + mp.spawn(run, args=(port, owned), nprocs=WORLD, join=True) diff --git a/benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py b/benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py new file mode 100644 index 0000000..3e4cc04 --- /dev/null +++ b/benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py @@ -0,0 +1,299 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The SP-prefill collectives on two ranks vs RCCL at world 2, per served token count T (shard rows R = T / 2). + +Graph replay: ``CALLS`` distinct inputs per graph (rotating beyond the 256 MB MALL), 10 replays per round, 5 rounds in +alternating order, the max over ranks of each round, median over rounds; us per call. The unpacking all-to-all keeps +host sequence state (eager only), so it and its baseline are timed eagerly (back-to-back calls, same rounds). + +Entries (left) and their baseline (right), the same bytes on the link: +- ``moe_tail``: ``slots4_reduce_scatter_add_rmsnorm_mxfp8`` vs slot reduce + shared (one launch) + RCCL reduce-scatter + + the row-major norm; ``x3`` / ``bf16_link`` the one-launch tail with the coded / BF16 link; ``two_launch`` the same + tail as stage + pull and a norm launch; ``rccl_ar`` the non-SP path's RCCL all-reduce of the same [T, H] bf16. + Roofline: the default communicator's link bytes (9600 B per coded row, 12288 bf16). ``moe_tail_owner``: the same + tail with the owner-form shared term ([R, H], the own rows only) against its bf16-link form. +- ``rs``: ``reduce_scatter`` vs RCCL reduce-scatter of [T, H] bf16. +- ``compact_ag``: ``all_gather`` of the six compact MoE fields (7328 B/row) vs RCCL all-gather of the same bytes. +- ``routed_ag``: ``all_gather`` of the four routed MoE fields (3296 B/row) vs RCCL all-gather of the same bytes. +- ``codes_ag``: ``all_gather`` of MXFP8 codes + scales (6336 B/row) vs RCCL all-gather of the same bytes. +- ``norm_ag``: ``norm_all_gather`` vs the norm + RCCL all-gather of codes + scales. +- ``a2a``: ``all_to_all_unpack_shuffled`` of rows the producer wrote into the pool slot (4224 B/row each way) vs RCCL + all-to-all + torch unpack + the GEMM's scale shuffle (eager). + +Roofline: per-rank bytes over the link / ``LINK_GBS`` (the best measured rate of one link with both directions busy: +SDMA, ``bench_engines.py``); ``CU_PULL_GBS`` is the CU pull ceiling (``benchmark_link.py``). + + python3 benchmarks/collectives/pull_collectives_ranks2/benchmark_collectives.py [out.json] [entry ...] \ + [--tokens T ...] +""" +import json +import statistics +import sys + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +H = 6144 +D = 4096 +ROUNDS = 5 +REPLAYS = 10 +EPS = 1e-6 +LINK_GBS = 61.2 +CU_PULL_GBS = 57.4 +MALL = 256 << 20 +ENTRIES = ("moe_tail", "moe_tail_owner", "rs", "routed_ag", "compact_ag", "codes_ag", "norm_ag", "a2a") +COMPACT = (192, 16, 16, 6144, 192, 768) +ROUTED = (3072, 192, 16, 16) + + +def _calls(bytes_per_call): + return max(2, min(16, -(-2 * MALL // bytes_per_call))) + + +def _time_graph(g, calls): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + for _ in range(REPLAYS): + g.replay() + b.record() + torch.cuda.synchronize() + return a.elapsed_time(b) * 1000 / (REPLAYS * calls) + + +def _time_eager(fn, calls): + fn() + torch.cuda.synchronize() + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + for _ in range(2): + fn() + b.record() + torch.cuda.synchronize() + return a.elapsed_time(b) * 1000 / (2 * calls) + + +def _graph(fn): + fn() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + fn() + g.replay() + torch.cuda.synchronize() + return g + + +def _measure(arms, calls, eager=False): + """arms: name -> fn over all calls; -> name -> rank-max per round, median over rounds (us per call).""" + run = {k: (fn if eager else _graph(fn)) for k, fn in arms.items()} + times = {k: [] for k in arms} + for r in range(ROUNDS): + keys = list(arms) if r % 2 == 0 else list(reversed(arms)) + for k in keys: + t = _time_eager(run[k], calls) if eager else _time_graph(run[k], calls) + every = [None, None] + dist.all_gather_object(every, t) + times[k].append(max(every)) + return {k: statistics.median(v) for k, v in times.items()} + + +def _bf16(shape, seed, dev): + g = torch.Generator(device=dev).manual_seed(seed) + return torch.randn(shape, generator=g, device=dev, dtype=torch.float32).bfloat16() + + +def _u8(shape, seed, dev): + g = torch.Generator(device=dev).manual_seed(seed) + return torch.randint(0, 120, shape, generator=g, device=dev, dtype=torch.uint8) + + +def _worker(rank, errors, entries, tokens_list, out): + from amd_kernels.collectives.pull_collectives_ranks2 import ( + AllGatherComm, Collectives, FencedAllGatherComm, NormAllGatherComm, Pools, SlotsReduceScatterComm) + from amd_kernels.gemm.mxfp8_common.gemm import _preshuffle_activation_scale + from amd_kernels.moe.mxfp4_h6144_i768_e128_top4.common.reduce_slots_shared import reduce_slots_shared + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + + dev = torch.device("cuda", rank) + nccl = dist.new_group(backend="nccl") + max_t = max(tokens_list) + w = (_bf16((H,), 3, dev) * 0.2).bfloat16() + results = {} + comms = {} + if {"moe_tail", "moe_tail_owner", "rs"} & set(entries): + comms["rs"] = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, max_rows=max_t) + comms["rs"].warm_up() + if {"moe_tail", "moe_tail_owner"} & set(entries): + for coded in (True, False): + comms[coded] = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, max_rows=max_t, coded=coded) + comms[coded].warm_up() + if {"compact_ag", "routed_ag", "codes_ag"} & set(entries): + comms["ag"] = AllGatherComm(rank, dist.group.WORLD, dev, (max_t // 2) * 7328 + 6 * 256) + comms["ag"].warm_up() + if "norm_ag" in entries: + comms["norm"] = NormAllGatherComm(rank, dist.group.WORLD, dev, max_rows=max_t // 2) + comms["norm"].warm_up() + if "a2a" in entries: + pools = Pools(rank, 2, dist.group.WORLD, dev, max_t * (D + D // 32) + 4096, sets=1) + comms["a2a"] = FencedAllGatherComm(pools) + comms["a2a"].warm_up() + for T in tokens_list: + R = T // 2 + for entry in entries: + arms, eager = {}, False + if entry == "moe_tail": + calls = _calls(T * 5 * H * 2) + slots = [_bf16((T, 4, H), 100 * i + rank, dev) for i in range(calls)] + shared = [_bf16((T, H), 100 * i + 50 + rank, dev) for i in range(calls)] + res = [_bf16((R, H), 100 * i + 70 + rank, dev) for i in range(calls)] + rs_out = torch.empty((R, H), dtype=torch.bfloat16, device=dev) + ar_buf = [torch.empty((T, H), dtype=torch.bfloat16, device=dev) for _ in range(calls)] + + def fused(): + for s, h, r in zip(slots, shared, res): + comms["rs"].slots4_reduce_scatter_add_rmsnorm_mxfp8(s, h, r, w, EPS) + + def baseline(): + for s, h, r in zip(slots, shared, res): + x = reduce_slots_shared(s, h) + dist.reduce_scatter_tensor(rs_out, x, group=nccl) + an.gemma_add_rmsnorm_mxfp8_row_major(rs_out, r, w, EPS) + + def two_launch(): + for s, h, r in zip(slots, shared, res): + comms["rs"].slots4_reduce_scatter_add_rmsnorm_mxfp8(s, h, r, w, EPS, fused=False) + + def rccl_ar(): + for x in ar_buf: + dist.all_reduce(x, group=nccl) + + def link_form(coded): + def run(): + for s, h, r in zip(slots, shared, res): + comms[coded].slots4_reduce_scatter_add_rmsnorm_mxfp8(s, h, r, w, EPS) + return run + arms = {"ours": fused, "baseline": baseline, "x3": link_form(True), "bf16_link": link_form(False), + "two_launch": two_launch, "rccl_ar": rccl_ar} + link = R * (9600 if comms["rs"].coded else H * 2) + elif entry == "moe_tail_owner": + calls = _calls(T * 4 * H * 2 + R * H * 2) + slots = [_bf16((T, 4, H), 100 * i + rank, dev) for i in range(calls)] + shared = [_bf16((R, H), 100 * i + 50 + rank, dev) for i in range(calls)] + res = [_bf16((R, H), 100 * i + 70 + rank, dev) for i in range(calls)] + + def owner_form(coded): + def run(): + for s, h, r in zip(slots, shared, res): + comms[coded].slots4_reduce_scatter_add_rmsnorm_mxfp8(s, h, r, w, EPS) + return run + arms = {"ours": owner_form(True), "baseline": owner_form(False)} + link = R * 9600 + elif entry == "rs": + calls = _calls(T * H * 2) + xs = [_bf16((T, H), 100 * i + rank, dev) for i in range(calls)] + outs = torch.empty((R, H), dtype=torch.bfloat16, device=dev) + + def ours(): + for x in xs: + comms["rs"].reduce_scatter(x, outs) + + def baseline(): + for x in xs: + dist.reduce_scatter_tensor(outs, x, group=nccl) + arms = {"ours": ours, "baseline": baseline} + link = R * H * 2 + elif entry in ("compact_ag", "routed_ag", "codes_ag"): + cols = {"compact_ag": COMPACT, "routed_ag": ROUTED, "codes_ag": (H, H // 32)}[entry] + row_bytes = sum(cols) + calls = _calls(R * row_bytes * 3) + ins = [[_u8((R, c), 100 * i + j + rank, dev) for j, c in enumerate(cols)] for i in range(calls)] + outs = [[torch.empty((T, c), dtype=torch.uint8, device=dev) for c in cols] for _ in range(calls)] + flat_in = [_u8((R * row_bytes,), 100 * i + rank, dev) for i in range(calls)] + flat_out = [torch.empty((T * row_bytes,), dtype=torch.uint8, device=dev) for _ in range(calls)] + + def ours(): + for i, o in zip(ins, outs): + comms["ag"].all_gather(o, i) + + def baseline(): + for i, o in zip(flat_in, flat_out): + dist.all_gather_into_tensor(o, i, group=nccl) + arms = {"ours": ours, "baseline": baseline} + link = R * row_bytes + elif entry == "norm_ag": + calls = _calls(R * H * 2 * 4) + xs = [_bf16((R, H), 100 * i + rank, dev) for i in range(calls)] + rs_ = [_bf16((R, H), 100 * i + 50 + rank, dev) for i in range(calls)] + q = torch.empty((T, H), dtype=torch.float8_e4m3fn, device=dev) + sc = torch.empty((T, H // 32), dtype=torch.uint8, device=dev) + flat = torch.empty((T * (H + H // 32),), dtype=torch.uint8, device=dev) + + def ours(): + for x, r in zip(xs, rs_): + comms["norm"].norm_all_gather(x, r, w, EPS, q_out=q, scale_out=sc) + + def baseline(): + for x, r in zip(xs, rs_): + cq, cs, _ = an.gemma_add_rmsnorm_mxfp8_row_major(x, r, w, EPS) + local = torch.cat((cq.view(torch.uint8).reshape(-1), cs[:R].reshape(-1))) + dist.all_gather_into_tensor(flat, local, group=nccl) + arms = {"ours": ours, "baseline": baseline} + link = R * (H + H // 32) + elif entry == "a2a": + eager = True + calls = _calls(T * (D + D // 32) * 2) + packed = [_u8((T, D + D // 32), 100 * i + rank, dev) for i in range(calls)] + recv = torch.empty((T, D + D // 32), dtype=torch.uint8, device=dev) + a2a = comms["a2a"] + + def ours(): + # the producer (the pack) writes into the slot: no staging copy + for _ in packed: + c = Collectives(a2a.p, 0).bind(torch.cuda.current_stream()) + a2a.all_to_all_unpack_shuffled(c, c.alloc((T, D + D // 32), torch.uint8)) + + def baseline(): + for x in packed: + dist.all_to_all_single(recv, x, group=nccl) + blocks = recv.view(2, R, D + D // 32).transpose(0, 1) + codes = blocks[:, :, :D].reshape(R, 2 * D) + _preshuffle_activation_scale(blocks[:, :, D:].reshape(R, 2 * D // 32)) + codes.contiguous() + arms = {"ours": ours, "baseline": baseline} + link = R * (D + D // 32) + got = _measure(arms, calls, eager) + roof = link / LINK_GBS / 1e3 + row = {"tokens": T, "link_MB": link / 1e6, "roofline_us": roof, "cu_pull_us": link / CU_PULL_GBS / 1e3, + **got} + results.setdefault(entry, []).append(row) + if rank == 0: + extra = " ".join(f"{k} {v:.1f}" for k, v in got.items() if k not in ("ours", "baseline")) + print(f"{entry:10s} T={T:5d} ours {got['ours']:8.1f} us baseline {got['baseline']:8.1f} us " + f"roofline {roof:7.1f} us ({roof / got['ours'] * 100:5.1f}%) {extra}", flush=True) + del arms + torch.cuda.empty_cache() + if rank == 0 and out: + with open(out, "w") as f: + json.dump(results, f, indent=1) + + +def main(): + args = sys.argv[1:] + out = args.pop(0) if args and args[0].endswith(".json") else None + tokens = [4096, 8192, 16384, 32768] + if "--tokens" in args: + i = args.index("--tokens") + tokens = [int(a) for a in args[i + 1:]] + args = args[:i] + entries = tuple(args) or ENTRIES + run_on_2_gpus(_worker, entries, tokens, out, timeout=7200) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py b/benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py new file mode 100644 index 0000000..06ef5fe --- /dev/null +++ b/benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Bandwidth of the one xGMI link of a GPU pair with CU kernels: a persistent grid of ``wgs`` workgroups copies +``mb`` MB from the peer (pull: system-coherent loads of the peer's pool, local stores) or to the peer (push: local +loads, stores into the peer's pool), one direction (rank 0 only) or both at once. Each workgroup moves ``unroll`` x 16 B +per thread per step. Prints GB/s per direction (event-timed, best of 5 rounds of 10 copies, max over ranks). + + python3 benchmarks/collectives/pull_collectives_ranks2/benchmark_link.py [mb] [--policies] [out.json] + +``--policies``: the pull's load cache policy (``LOAD_POLICY``) instead of the workgroup / unroll sweep. +""" +import functools +import json +import sys + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +import torch.distributed as dist +from amd_kernels.common import buffer_ops +from flydsl.expr import range_constexpr +from flydsl.expr.typing import Vector as Vec + +from tests.multi_gpu import run_on_2_gpus + +NT = 256 +SYS = 0x11 +# gfx950 cache policy bits of a load: sc0 = 1, nt = 2, sc1 = 16 +LOAD_POLICY = {"pull": SYS, "pull_nt": SYS | 2, "pull_sc1": 16, "pull_plain": 0} + + +@functools.cache +def _copy_kernel(wgs: int, unroll: int, load_mod: int, store_mod: int): + step = NT * 4 * unroll + name = f"link_copy_w{wgs}_u{unroll}_l{load_mod}_s{store_mod}" + + @flyc.kernel(name=name, known_block_size=[NT, 1, 1]) + def kernel(arg_dst: fx.Int64, arg_src: fx.Int64, i32_bytes: fx.Int32): + n_dw = i32_bytes // 4 + src = buffer_ops.create_buffer_resource_from_addr(arg_src, num_records_bytes=i32_bytes) + dst = buffer_ops.create_buffer_resource_from_addr(arg_dst, num_records_bytes=i32_bytes) + off = fx.block_idx.x * step + fx.thread_idx.x * 4 + while off < n_dw: + vals = [] + for u in range_constexpr(unroll): + o = off + u * NT * 4 + vals.append(Vec(buffer_ops.buffer_load(src, o, vec_width=4, dtype=fx.Int32, mask=o < n_dw, + cache_modifier=load_mod))) + for u in range_constexpr(unroll): + o = off + u * NT * 4 + buffer_ops.buffer_store(vals[u], dst, o, mask=o < n_dw, cache_modifier=store_mod) + off = off + wgs * step + + @flyc.jit + def launch(arg_dst: fx.Int64, arg_src: fx.Int64, i32_bytes: fx.Int32, stream: fx.Stream): + kernel(arg_dst, arg_src, i32_bytes).launch(grid=(wgs, 1, 1), block=(NT, 1, 1), stream=stream) + + launch.kernel_name = name + return launch + + +def _worker(rank, errors, mb, configs, out): + from amd_kernels.collectives.pull_collectives_ranks2.host import _alloc_pool, _exchange, _open_pool, _run_compiled + + dev = torch.device("cuda", rank) + n = mb << 20 + ptr, handle, pool = _alloc_pool(2 * n, dev) + hs = _exchange(handle, rank, 2, dist.group.WORLD) + peer = 1 - rank + peer_ptr = _open_pool(hs[peer], dev) + local = torch.randint(0, 255, (n,), dtype=torch.uint8, device=dev) + pool[:n].copy_(local) + stream = torch.cuda.current_stream() + results = {} + for kind, wgs, unroll, both in configs: + if kind.startswith("pull"): + exe = _copy_kernel(wgs, unroll, LOAD_POLICY[kind], 0) + dst, src = local.data_ptr(), peer_ptr + elif kind == "push": + exe = _copy_kernel(wgs, unroll, 0, 0) + dst, src = peer_ptr + n, local.data_ptr() + else: # push with system-coherent stores + exe = _copy_kernel(wgs, unroll, 0, SYS) + dst, src = peer_ptr + n, local.data_ptr() + active = both or rank == 0 + _run_compiled(exe, dst, src, n, stream) + torch.cuda.synchronize() + best = 1e9 + for _ in range(5): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + if active: + for _ in range(10): + _run_compiled(exe, dst, src, n, stream) + b.record() + torch.cuda.synchronize() + best = min(best, a.elapsed_time(b) / 10) + times = [None, None] + dist.all_gather_object(times, best if active else 0.0) + ms = max(times) + key = f"{kind} wgs{wgs} u{unroll} {'bidir' if both else 'uni'}" + results[key] = n / ms / 1e6 + if rank == 0: + print(f"{key:32s} {n / ms / 1e6:7.1f} GB/s per direction ({ms * 1e3:.0f} us for {mb} MB)", flush=True) + if rank == 0 and out: + with open(out, "w") as f: + json.dump(results, f, indent=1) + + +def main(): + args = sys.argv[1:] + out = args.pop() if args and args[-1].endswith(".json") else None + policies = "--policies" in args + args = [a for a in args if a != "--policies"] + mb = int(args[0]) if args else 256 + configs = [] + for both in (False, True): + if policies: + for kind in LOAD_POLICY: + for wgs in (8, 16, 32): + configs.append((kind, wgs, 8, both)) + continue + for kind in ("pull", "push", "push_sys"): + for wgs in (4, 8, 16, 32, 64): + for unroll in (4, 8, 16): + configs.append((kind, wgs, unroll, both)) + run_on_2_gpus(_worker, mb, configs, out, timeout=3000) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/collectives/sdma_push_ranks2/bench_engines.py b/benchmarks/collectives/sdma_push_ranks2/bench_engines.py new file mode 100644 index 0000000..3e0c119 --- /dev/null +++ b/benchmarks/collectives/sdma_push_ranks2/bench_engines.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SDMA bandwidth over the one xGMI link of a GPU pair by engine assignment: rank 0 (one direction) or both ranks +(both directions) copy ``mb`` MB into the peer's IPC pool, split evenly over the listed engines. Prints the engines +``hsa_amd_memory_copy_engine_status`` reports free and GB/s per direction (host-timed from enqueue to the last +completion signal, best of 5, max over ranks). + + python3 benchmarks/collectives/sdma_push_ranks2/bench_engines.py [mb ...] [out.json] +""" +import ctypes +import json +import sys +import time + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +CONFIGS = ((2,), (3,), (8,), (2, 3), (2, 8), (2, 4), (2, 3, 8, 9), (2, 3, 4, 5, 6, 7, 8, 9)) + + +def _worker(rank, errors, sizes, out): + from amd_kernels.collectives.sdma_push_ranks2 import hsa_runtime as rt + from amd_kernels.collectives.sdma_push_ranks2.ipc import alloc_pool, exchange, open_pool + + dev = torch.device("cuda", rank) + peer = 1 - rank + n_max = max(sizes) << 20 + ptr, handle, pool = alloc_pool(n_max, dev) + hs = exchange((handle, rt.pci_key(dev)), rank, 2, dist.group.WORLD) + peer_ptr = open_pool(hs[peer][0], dev) + agents = rt.gpu_agents() + me, other = agents[hs[rank][1]], agents[hs[peer][1]] + lib = rt.hsa() + src = torch.randint(0, 255, (n_max,), dtype=torch.uint8, device=dev) + mask = ctypes.c_uint32() + lib.hsa_amd_memory_copy_engine_status(other, me, ctypes.byref(mask)) + free = [e for e in range(32) if mask.value >> e & 1] + print(f"rank {rank}: engines free to the peer {free}", flush=True) + torch.cuda.synchronize() + + def run(n, engines): + sigs = [] + part = n // len(engines) // 256 * 256 + t0 = time.perf_counter() + for j, e in enumerate(engines): + size = part if j < len(engines) - 1 else n - j * part + s = rt.signal(1) + sigs.append(s) + rc = lib.hsa_amd_memory_async_copy_on_engine(peer_ptr + j * part, other, src.data_ptr() + j * part, me, + size, 0, None, s, 1 << e, True) + assert rc == 0, rc + for s in sigs: + while lib.hsa_signal_wait_scacquire(s, rt.HSA_SIGNAL_CONDITION_LT, 1, 1 << 40, + rt.HSA_WAIT_STATE_ACTIVE) > 0: + pass + return time.perf_counter() - t0 + + results = {} + for mb in sizes: + n = mb << 20 + for both in (False, True): + for engines in CONFIGS: + active = both or rank == 0 + best = 1e9 + for _ in range(5): + dist.barrier() + if active: + best = min(best, run(n, engines)) + t = [None, None] + dist.all_gather_object(t, best if active else 0.0) + gbs = n / max(t) / 1e9 + key = f"{mb}MB {'bidir' if both else 'uni'} engines{list(engines)}" + results[key] = gbs + if rank == 0: + print(f"{key:44s} {gbs:6.1f} GB/s per direction ({max(t) * 1e6:.0f} us)", flush=True) + if rank == 0 and out: + with open(out, "w") as f: + json.dump(results, f, indent=1) + + +def main(): + args = sys.argv[1:] + out = args.pop() if args and args[-1].endswith(".json") else None + sizes = [int(a) for a in args] or [8, 64, 256] + run_on_2_gpus(_worker, sizes, out, timeout=1800) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/collectives/sdma_push_ranks2/benchmark_push.py b/benchmarks/collectives/sdma_push_ranks2/benchmark_push.py new file mode 100644 index 0000000..86f86ff --- /dev/null +++ b/benchmarks/collectives/sdma_push_ranks2/benchmark_push.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``SdmaPush`` on two ranks vs RCCL all-gather at world 2, the same bytes per rank (an SP layer's hand-offs at T +tokens: R = T / 2 shard rows of the a2a 4224 + compact 7328 + reduce-scatter 12288 + codes 6336 B). + +Both ranks push their block into the peer's region and wait for the peer's (an all-gather's link traffic); the pushes +alternate between two regions with a wait in between. Eager (stream memory operations, no graph capture): stream +events around 10 back-to-back push + wait pairs, 5 rounds, the max over ranks, median over rounds; us per push. Also a +bf16 GEMM loop as long as the pushes: its time alone and with the pushes running on another stream. + + python3 benchmarks/collectives/sdma_push_ranks2/benchmark_push.py [out.json] [--tokens T ...] +""" +import json +import statistics +import sys + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +ROW_BYTES = 4224 + 7328 + 12288 + 6336 +ROUNDS = 5 +N = 10 + + +def _round_time(fn): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + fn() + b.record() + torch.cuda.synchronize() + every = [None, None] + dist.all_gather_object(every, a.elapsed_time(b) * 1000 / N) + return max(every) + + +def _worker(rank, errors, tokens, out): + from amd_kernels.collectives.sdma_push_ranks2 import SdmaPush + + dev = torch.device("cuda", rank) + nccl = dist.new_group(backend="nccl") + peer = 1 - rank + max_n = max(tokens) // 2 * ROW_BYTES + t = SdmaPush(rank, dist.group.WORLD, dev, [2 * max_n, 2 * max_n]) + src = torch.randint(0, 255, (max_n,), dtype=torch.uint8, device=dev) + gathered = torch.empty((2 * max_n,), dtype=torch.uint8, device=dev) + a = torch.randn((8192, 6144), device=dev).bfloat16() + bmat = torch.randn((6144, 6144), device=dev).bfloat16() + side = torch.cuda.Stream() + results = {} + for T in tokens: + n = T // 2 * ROW_BYTES + + def sdma(): + for i in range(N): + seq = t.push(i % 2, [(peer, src.data_ptr(), rank * n, n)]) + t.wait(i % 2, seq) + + def rccl(): + for _ in range(N): + dist.all_gather_into_tensor(gathered[:2 * n], src[:n], group=nccl) + + arms = {"sdma": sdma, "rccl_ag": rccl} + for fn in arms.values(): + fn() + torch.cuda.synchronize() + times = {k: [] for k in arms} + for r in range(ROUNDS): + for k in (list(arms) if r % 2 == 0 else list(reversed(arms))): + times[k].append(_round_time(arms[k])) + got = {k: statistics.median(v) for k, v in times.items()} + # a GEMM loop as long as the pushes, alone and with the pushes on another stream (its own events) + g0, g1 = torch.cuda.Event(True), torch.cuda.Event(True) + a @ bmat + g0.record() + a @ bmat + g1.record() + torch.cuda.synchronize() + iters = max(1, int(got["sdma"] * N / (g0.elapsed_time(g1) * 1000))) + gemm_t = {"alone": [], "next_to_sdma": []} + for r in range(ROUNDS): + for kind in gemm_t: + dist.barrier() + torch.cuda.synchronize() + if kind == "next_to_sdma": + with torch.cuda.stream(side): + sdma() + g0.record() + for _ in range(iters): + a @ bmat + g1.record() + torch.cuda.synchronize() + gemm_t[kind].append(g0.elapsed_time(g1) * 1000) + got.update({f"gemm_{k}": statistics.median(v) for k, v in gemm_t.items()}) + got["GBs"] = n / got["sdma"] / 1e3 + got["MB"] = n / 1e6 + results[T] = got + if rank == 0: + print(f"T={T:5d} {n / 1e6:7.1f} MB/rank: sdma {got['sdma']:8.1f} us ({got['GBs']:.1f} GB/s) " + f"rccl all-gather {got['rccl_ag']:8.1f} us gemm loop alone {got['gemm_alone']:8.1f} us, " + f"next to the pushes {got['gemm_next_to_sdma']:8.1f} us", flush=True) + t.drain() + if rank == 0 and out: + with open(out, "w") as f: + json.dump(results, f, indent=1) + + +def main(): + args = sys.argv[1:] + out = args.pop(0) if args and args[0].endswith(".json") else None + tokens = [4096, 8192, 16384, 32768] + if "--tokens" in args: + tokens = [int(x) for x in args[args.index("--tokens") + 1:]] + run_on_2_gpus(_worker, tokens, out, timeout=3600) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py b/benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py new file mode 100644 index 0000000..4f51408 --- /dev/null +++ b/benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py @@ -0,0 +1,307 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""A MoE decoder layer of the TP2 prefill on two ranks, sequence-parallel vs the plain tensor-parallel path, per token +count T (shard rows R = T / 2). The layer from the attention output to the next layer's QKV output; the attention +core (identical in both paths) is not run. + +- ``sp`` (``SeqParallelHandoffs``, pull collectives) and ``sdma`` (``SdmaSeqParallelHandoffs``, one batch, no + overlap): ``attention_moe`` (pack, all-to-all unpack, o_proj_full [6144, 8192] on R rows, norm, MoE preparation, + the routed payload's gather, the full shared expert of the R own rows on the side stream), ``mxfp4_moe`` routed + partials on all T rows, ``ffn_norm_gather`` (join, MoE tail reduce-scatter with the owner's shared output + norm + + quant, codes gather), the QKV GEMM [4992, 6144] on all T rows. +- ``sdma_dbo``: DBO as vLLM runs it: two microbatches of T / 2 tokens (one ``SdmaSeqParallelHandoffs`` each, each + with its shared-expert side stream), two threads taking turns on one compute stream, each phase's ``between`` the + yield, so a microbatch's pushes run on the SDMA engine under the other microbatch's compute; ``DBO_LAYERS`` layers + per round, us per layer. +- ``*_serial``: the same with the full shared expert on the compute stream (``shared_stream=False``): the overlap's + reference. ``*_shard``: a shard copy of the shared expert (I 1536 on all T rows inside ``mxfp4_moe``, the compact + 7328 B/token gather; under DBO on the caller's stream, as ``set_dbo_query`` puts it in vLLM). +- ``tp_rccl`` / ``tp_aiter``: the attention output quantized + o_proj [6144, 4096] on all T rows, all-reduce (RCCL, or + AITER's custom all-reduce where vLLM takes it: at most 64 MiB), the bf16 add + norm, ``mxfp4_moe`` (reduced) on all + T rows, all-reduce, the add + norm + MXFP8 quant, the QKV GEMM. The same amd_kernels compute in both paths, so the + difference is the hand-offs. + +Eager (the SP phases keep host sequence state), ``ROUNDS`` rounds after a warm-up call, each bracketed by a barrier: +the max over ranks per round, the median over rounds, us. Stage columns: the SP hand-off phases alone. + +Roofline: link bytes per rank at ``LINK_GBS`` (SP: R x 23456 B: a2a 4224, routed gather 3296 (shard copy: compact +7328), coded tail 9600, codes gather 6336; TP: two all-reduces of [T, H] bf16, R x 49152 B) and the compute ideal (each GEMM / MoE at max(HBM +bytes / 6.5 TB/s, MXFP4 flops / 10 PF + MXFP8 flops / 5 PF), the norms at their bytes): their sum for one batch, +their maximum for DBO. + + python3 benchmarks/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/benchmark_sp_layer.py [out.json] \ + [--tokens T ...] [--paths sp sdma tp_rccl tp_aiter] [--bf16-tail] + +``--bf16-tail``: the SP hand-offs with the MoE tail's reduce-scatter in BF16 (12288 B per row) instead of the code. +""" +import json +import statistics +import sys +import threading + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +H, Q, WORLD, I, E = 6144, 4096, 2, 1536, 128 +QKV_N = 4992 +EPS = 1e-6 +ROUNDS = 7 +LINK_GBS = 61.2 +HBM_TBS = 6.5 +FP4_PF, FP8_PF = 10.0, 5.0 +AITER_MAX_BYTES = 64 << 20 # vLLM's AiterCustomAllreduce.effective_max_size +TOKENS = (4096, 8192, 16384, 32768) +PATHS = ("sp", "sp_serial", "sp_shard", "sdma", "sdma_dbo", "sdma_dbo_serial", "sdma_dbo_shard", "tp_rccl", + "tp_aiter") +DBO_LAYERS = 4 +SP_LINK_BYTES = 4224 + 3296 + 9600 + 6336 # per shard row (12288 for the tail with --bf16-tail) +SHARD_GATHER_EXTRA = 7328 - 3296 # the compact payload of a shard copy + + +def _compute_ideal_us(t: int) -> float: + """The layer's compute ideal at T tokens (module docstring).""" + def gemm(m, n, k, pf): + return max((n * k + m * k + 2 * m * n) / (HBM_TBS * 1e12), 2 * m * n * k / (pf * 1e15)) + + touched = E * (1 - (1 - 4 / E) ** t) + moe_bytes = touched * 15.04e6 + 58.4e6 + 3.1e6 + t * H * 4 + moe_flops = (t * 4 * 2 * (H * 2 * I + I * H)) / (FP4_PF * 1e15) + (t * 2 * (H * 2 * I + I * H)) / (FP8_PF * 1e15) + moe = max(moe_bytes / (HBM_TBS * 1e12), moe_flops) + o_proj = gemm(t // WORLD, H, WORLD * Q, FP8_PF) # = o_proj [6144, 4096] on T rows in flops + qkv = gemm(t, QKV_N, H, FP8_PF) + norms = 2 * (t // WORLD) * H * 2 * 3 / (HBM_TBS * 1e12) + return (moe + o_proj + qkv + norms) * 1e6 + + +def _weights(dev, rank): + from amd_kernels.gemm.mxfp8_common.gemm import preshuffle_mxfp8_weight + from tests.collectives.seq_parallel_handoffs_h6144_q4096_ranks2.workload import moe_weights, o_proj_weight + + def fp8(n, k, seed): + g = torch.Generator(device=dev).manual_seed(seed) + wq = (torch.randn((n, k), device=dev, generator=g) * 20).to(torch.float8_e4m3fn) + ws = torch.randint(116, 121, (n, k // 32), generator=g, device=dev, dtype=torch.uint8) + return preshuffle_mxfp8_weight(wq, ws) + + return o_proj_weight(dev), fp8(H, Q, 11), fp8(QKV_N, H, 12), moe_weights(dev, rank) + + +def _shard_copy(dev, mw): + """``mw`` with a shard copy of the shared expert (the same routed tensors).""" + from dataclasses import replace + + from tests.collectives.seq_parallel_handoffs_h6144_q4096_ranks2.workload import moe_weights + + return replace(mw, shared_weights=moe_weights(dev, full=False).shared_weights) + + +def _worker(rank, errors, tokens, paths, out_path, coded=True): + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import (SdmaSeqParallelHandoffs, + SeqParallelHandoffs, shard_rows) + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_e4m3_quantize_flydsl, mxfp8_gemm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe, set_dbo_query + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.host import gemma_add_rmsnorm + from tests.collectives.seq_parallel_handoffs_h6144_q4096_ranks2.workload import rand + + dev = torch.device("cuda", rank) + (o_w, o_s), (tp_w, tp_s), (qkv_w, qkv_s), mw_full = _weights(dev, rank) + mw_shard = _shard_copy(dev, mw_full) if any(p.endswith("_shard") or p.startswith("tp") for p in paths) else None + nw, nw2 = rand((H,), 3, dev, 0.2), rand((H,), 4, dev, 0.2) + nccl = dist.new_group(backend="nccl") + in_dbo = threading.local() + set_dbo_query(lambda: getattr(in_dbo, "on", False)) + hand, dbo = {}, {} + for path in ("sp", "sp_serial", "sp_shard"): + if path in paths: + hand[path] = SeqParallelHandoffs(rank, dist.group.WORLD, dev, max(tokens), coded_tail=coded, + shared_stream=path != "sp_serial") + if "sdma" in paths: + hand["sdma"] = SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, max(tokens), coded_tail=coded) + for path in ("sdma_dbo", "sdma_dbo_serial", "sdma_dbo_shard"): + if path in paths: + ub = max(tokens) // 2 + 1 + dbo[path] = [SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, ub, coded_tail=coded, + compact=path.endswith("_shard"), + shared_stream=not path.endswith("_serial")) for _ in range(2)] + aiter = None + if "tp_aiter" in paths: + from aiter.dist.device_communicators.custom_all_reduce import CustomAllreduce + + aiter = CustomAllreduce(dist.group.WORLD, dev, max_size=2 * AITER_MAX_BYTES) + + def qkv(codes, scales): + return mxfp8_gemm(codes, scales, qkv_w, qkv_s, torch.bfloat16, activation_scale_preshuffled=True) + + def moe(ph, attn, res, mw, **kw): + """attention_moe + the routed (and, with a shard copy, shared) partials -> (FFN output, residual).""" + moe_in, r, shared = ph.attention_moe(attn, o_w, o_s, res, nw, EPS, mw, **kw) + slots, sh = mxfp4_moe(moe_in, mw, return_partials=True) + return (slots, shared if shared is not None else sh), r + + def sp_layer(ph, attn, res, ev, mw): + ev[0].record() + moe_in, r, shared = ph.attention_moe(attn, o_w, o_s, res, nw, EPS, mw) + ev[1].record() + slots, sh = mxfp4_moe(moe_in, mw, return_partials=True) + del moe_in + ev[2].record() + codes, scales, r = ph.ffn_norm_gather((slots, shared if shared is not None else sh), r, nw2, EPS) + ev[3].record() + del slots, sh, shared + y = qkv(codes, scales) + ev[4].record() + return y + + def dbo_layers(dbo, attns, ress, ev, mw): + """Two microbatches through ``DBO_LAYERS`` layers, vLLM's DBO turn-taking (module docstring).""" + stream = torch.cuda.current_stream() + turn = [threading.Event(), threading.Event()] + done = [False, False] + errs = [] + + def body(i): + torch.cuda.set_device(dev) + torch.cuda.set_stream(stream) + + def yield_(): + if not done[1 - i]: + turn[1 - i].set() + turn[i].wait() + turn[i].clear() + + try: + in_dbo.on = True + turn[i].wait() + turn[i].clear() + ph, attn, res = dbo[i], attns[i], ress[i] + for _ in range(DBO_LAYERS): + ffn, r = moe(ph, attn, res, mw, between=yield_) + codes, scales, r = ph.ffn_norm_gather(ffn, r, nw2, EPS, between=yield_) + del ffn + qkv(codes, scales) + yield_() + except Exception as e: # noqa: BLE001 + errs.append(e) + finally: + done[i] = True + turn[1 - i].set() + + ev[0].record() + threads = [threading.Thread(target=body, args=(i,)) for i in range(2)] + for th in threads: + th.start() + turn[0].set() + for th in threads: + th.join() + if errs: + raise errs[0] + for e in ev[1:]: + e.record() + + def tp_layer(all_reduce, attn, res_full, ev): + ev[0].record() + xq, xs = mxfp8_e4m3_quantize_flydsl(attn) + h = mxfp8_gemm(xq, xs, tp_w, tp_s, torch.bfloat16, activation_scale_preshuffled=True) + h = all_reduce(h) + y, r = gemma_add_rmsnorm(h, res_full, nw, EPS) + ev[1].record() + out = mxfp4_moe(y, mw_shard) + ev[2].record() + out = all_reduce(out) + codes, scales, r = gemma_add_rmsnorm_mxfp8_row_major(out, r, nw2, EPS) + ev[3].record() + y = qkv(codes, scales) + ev[4].record() + return y + + def rccl(x): + dist.all_reduce(x, group=nccl) + return x + + def aiter_ar(x): + return aiter.custom_all_reduce(x) + + results = [] + for t in tokens: + rows = shard_rows(t) + attn = rand((t, Q), t + 10 + rank, dev, 0.5) + res = rand((rows, H), t + 20 + rank, dev) + res_full = rand((t, H), t + 30, dev) + for path in paths: + if path == "tp_aiter" and t * H * 2 > AITER_MAX_BYTES: + continue + mw = mw_shard if path.endswith("_shard") else mw_full + if path in dbo: + u = [t // 2, t - t // 2] + attns = [attn[: u[0]].contiguous(), attn[u[0] :].contiguous()] + ress = [rand((shard_rows(n), H), t + 40 + i + rank, dev) for i, n in enumerate(u)] + run = lambda ev, d=dbo[path], mw=mw: dbo_layers(d, attns, ress, ev, mw) # noqa: E731 + elif path in hand: + if not hand[path].supports(t): + continue + run = lambda ev, ph=hand[path], mw=mw: sp_layer(ph, attn, res, ev, mw) # noqa: E731 + else: + ar = rccl if path == "tp_rccl" else aiter_ar + run = lambda ev, ar=ar: tp_layer(ar, attn, res_full, ev) # noqa: E731 + ev = [torch.cuda.Event(enable_timing=True) for _ in range(5)] + run(ev) + torch.cuda.synchronize() + rounds = [] + for _ in range(ROUNDS): + dist.barrier() + torch.cuda.synchronize() + run(ev) + torch.cuda.synchronize() + rounds.append([ev[0].elapsed_time(e) * 1000 for e in ev[1:]]) + if path == "sdma": + hand[path].drain() + if path in dbo: + for ph in dbo[path]: + ph.drain() + rounds = [[x / DBO_LAYERS for x in r] for r in rounds] + every = [None] * WORLD + dist.all_gather_object(every, rounds) + per_round = [[max(every[p][i][j] for p in range(WORLD)) for j in range(4)] for i in range(ROUNDS)] + total = statistics.median(r[3] for r in per_round) + stage = [statistics.median(r[j] - (r[j - 1] if j else 0) for r in per_round) for j in range(4)] + link_bytes = rows * (4 * H * 2 if path.startswith("tp") else SP_LINK_BYTES + (0 if coded else 2688) + + (SHARD_GATHER_EXTRA if path.endswith("_shard") else 0)) + link_us = link_bytes / (LINK_GBS * 1e3) + comp_us = _compute_ideal_us(t) + # one batch serializes the link and the compute; DBO can overlap them + ideal = max(link_us, comp_us) if path in dbo else link_us + comp_us + if path in dbo: + stage = None + results.append(dict(tokens=t, path=path, total_us=total, stage_us=stage, link_bytes=link_bytes, + link_us=link_us, compute_ideal_us=comp_us, ideal_us=ideal)) + if rank == 0: + stages = " / ".join(f"{s:.1f}" for s in stage) if stage else "-" + print(f"T {t:6d} {path:9s} total {total:9.1f} us stages {stages} ideal {ideal:8.1f} " + f"(link {link_us:.1f}, compute {comp_us:.1f})", flush=True) + del attn, res, res_full + torch.cuda.empty_cache() + if rank == 0 and out_path: + with open(out_path, "w") as f: + json.dump(results, f, indent=1) + + +def main(): + args = sys.argv[1:] + out = args.pop(0) if args and args[0].endswith(".json") else None + tokens, paths, key = [], [], None + for a in args: + if a.startswith("--"): + key = a + elif key == "--tokens": + tokens.append(int(a)) + elif key == "--paths": + paths.append(a) + run_on_2_gpus(_worker, tuple(tokens) or TOKENS, tuple(paths) or PATHS, out, "--bf16-tail" not in args, + timeout=3000) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py b/benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py new file mode 100644 index 0000000..b0b623c --- /dev/null +++ b/benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The whole TP2 DSpark draft forward per step on 2 ranks: the input fc (5 taps) + the embedding + 5 layers (qkv, +q/k norm + RoPE + fp8 insert, SWA decode over a full 4096 window, o_proj, MLP, the all-reduce + norm tails) + the +final norm, 7 rows per request, against the TP2 stock path and the roofline. + +- ``block``: ``DenseDecoderRanks2`` (fc eager through ``ConcatLinearComm``; ``forward`` in one CUDA graph) around the + draft attention on its codes: qkv ``mxfp8_gemm_prequant`` (4608 x 6144), ``qkv_norm_rope_insert`` (32 q / 2 KV + heads), ``swa_decode_fp8kv_oproj_input`` (MXFP8 epilogue for the o_proj) and o_proj (6144 x 4096). +- ``stock``: what vLLM runs at TP2 without the block, in one CUDA graph: the replicated fc (``torch.cat`` + + ``F.linear``), ``VocabParallelEmbedding`` (mask, gather, AITER custom all-reduce), vLLM's Triton Gemma norms, + Triton ``tl.dot_scaled`` qkv / o_proj / gate_up (row-major weights, fused activation quant), the same draft + attention with bf16 output, the uncompiled SwiGLU-OAI aten chain, the amd_kernels down GEMM (6144 x 6144 is in + its table), AITER custom all-reduce after o_proj and down. + +Weights 5 x ~172 MB MXFP8 per path (> the 256 MB MALL), KV pools of disjoint pages per layer (cold), every request +one random token + six mask ids, context 8K-64K. Alternated rounds after a barrier, max over ranks, median us per +step. Roofline per rank: per op max(HBM bytes / 6.5 TB/s, flops / peak (MXFP8 5 PF, bf16 2.5 PF)), plus the link +bytes of the fc exchange and the ten all-reduce tails (owner form) at 56 GB/s per direction, serial. + + python3 benchmarks/decoder/dense_mxfp8_h6144_i6144_ranks2/benchmark_draft_forward.py [--reqs 1 8 32 52] [--parts] +""" +import argparse +import json +import statistics + +import torch +import torch.distributed as dist + +H, V2, I, QL, LAYERS = 6144, 100032, 6144, 7, 5 +NQ, NKV, D, PAGE, W = 32, 2, 128, 128, 4096 +QKV_N, O_K = (NQ + 2 * NKV) * D, NQ * D +EPS, SWIGLU = 1e-6, (1.702, 1.0, 7.0) +MASK_ID = 200058 +HBM, MXFP8_PF, BF16_PF, LINK = 6.5e12, 5e15, 2.5e15, 56e9 + + +def roofline_us(reqs): + """(total, of which link) us per rank of one draft step at ``reqs`` requests (module doc).""" + m = QL * reqs + w8 = 1 + 1 / 32 # MXFP8 codes + e8m0 scales + fc = max((3072 * 5 * H * 2 + m * 5 * H * 2 + m * H * 2) / HBM, 2 * m * 3072 * 5 * H / BF16_PF) + + def gemm(n, k): + return max((n * k * w8 + m * k + m * n * 2) / HBM, 2 * m * n * k / MXFP8_PF) + + kv = reqs * NKV * (W - 1 + QL) * 256 + attn = max((kv + m * QKV_N * 2 + m * O_K) / HBM, 4 * m * NQ * D * (W - 1 + QL) / BF16_PF) + mlp = max((2 * I * H * w8 + H * I * w8 + m * 2 * I * 2 * 2 + m * I * 2) / HBM, 2 * m * 3 * I * H / MXFP8_PF) + layer = gemm(QKV_N, H) + attn + gemm(H, O_K) + mlp + # per direction of the one link: the fc's 3072-column shard; nine MXFP8 tails and one bf16 tail (owner form from + # 24 rows: half the partial rows to their owner, the owner's codes + scales / bf16 rows back; else all rows) + quant_tail = m * (12288 + 6336) // 2 if m >= 24 else m * 12288 + link = (m * 3072 * 2 + 9 * quant_tail + m * 12288) / LINK + local = fc + LAYERS * layer + m * H * 2 * 3 / HBM + return (local + link) * 1e6, link * 1e6 + + +def _weights(n, k, g, row_major): + from amd_kernels.gemm.mxfp8_common import gemm as G + from amd_kernels.gemm.mxfp8_common.reference import quantize_mxfp8 + codes, exp = quantize_mxfp8((torch.randn((n, k), device=g.device, generator=g) * 0.02).bfloat16()) + pre = G.preshuffle_mxfp8_weight(codes.clone(), exp.clone()) + return pre, ((codes, exp) if row_major else None) + + +def _kv(reqs, g, cpu): + """A pool of disjoint pages per layer, the block tables, lengths and draft slots/positions (full windows).""" + per_req = (W + QL + PAGE - 1) // PAGE + 1 + pools, bts = [], [] + ctx = torch.randint(8192, 65536, (reqs,), generator=cpu) + seq = ctx + QL + nblk = int((seq.max() + PAGE - 1) // PAGE) + for _ in range(LAYERS): + n_pages = reqs * per_req + 2 + pool = (torch.randn((n_pages, NKV, PAGE, 256), device=g.device, generator=g) * 1.2).to( + torch.float8_e4m3fn).view(torch.uint8) + order = torch.randperm(n_pages, generator=cpu).int() + bt = torch.zeros((reqs, nblk), dtype=torch.int32) + cur = 0 + for r, (c, s_r) in enumerate(zip(ctx.tolist(), seq.tolist())): + nb = (s_r + PAGE - 1) // PAGE + lo = max((c - W + 1) // PAGE, 0) + bt[r, lo:nb] = order[cur:cur + nb - lo] + cur += nb - lo + pools.append(pool) + bts.append(bt.to(g.device)) + pos = (seq[:, None] - QL + torch.arange(QL)[None]).flatten() + slots = [(bt.cpu()[torch.arange(reqs).repeat_interleave(QL), pos // PAGE].long() * PAGE + pos % PAGE).to(g.device) + for bt in bts] + qsl = torch.arange(0, (reqs + 1) * QL, QL, dtype=torch.int32, device=g.device) + return pools, bts, seq.int().to(g.device), qsl, pos.to(g.device), slots + + +def _timed(rank, world, arms, rounds, reps): + """{name: median over rounds of the max over ranks, us per call} of ``arms`` {name: callable(reps)}.""" + ts = {k: [] for k in arms} + for r in range(rounds): + for k in (list(arms) if r % 2 == 0 else list(reversed(list(arms)))): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + arms[k](reps) + b.record() + b.synchronize() + ts[k].append(a.elapsed_time(b) * 1e3 / reps) + every = [None] * world + dist.all_gather_object(every, ts) + return {k: statistics.median(max(e[k][j] for e in every) for j in range(rounds)) for k in ts} + + +def worker(rank, errors, reqs_list, rounds, check, parts): + from aiter.dist.device_communicators.custom_all_reduce import CustomAllreduce + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import _mxfp8_dot_scaled_linear + from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import gemma_fused_add_rmsnorm, gemma_rmsnorm + + from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64.qkv_norm_rope_insert import qkv_norm_rope_insert + from amd_kernels.attention.swa_decode_fp8kv_gqa16_d128 import swa_decode_fp8kv_oproj_input + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import DenseDecoderRanks2, DenseLayer, mxfp8_swiglu_oai_mlp + from amd_kernels.gemm.mxfp8_common import gemm as G + from amd_kernels.gemm.mxfp8_common.reference import swiglu_bf16_reference + + torch.set_num_threads(8) + dev = torch.device("cuda", rank) + world = dist.get_world_size() + g = torch.Generator(device=dev).manual_seed(5) # replicated tensors + gr = torch.Generator(device=dev).manual_seed(50 + rank) # this rank's shards + fc_w = (torch.randn(H, 5 * H, device=dev, generator=g) * 0.01).bfloat16() + emb = (torch.randn(V2, H, device=dev, generator=gr) * 0.5).bfloat16() + blk = DenseDecoderRanks2(rank, dist.group.WORLD, dev, fc_weight=fc_w, embedding=emb) + ca = CustomAllreduce(group=dist.group.WORLD, device=dev, max_size=64 * 1024 * 1024) + + def vec(scale=0.1): + return (torch.randn(H, device=dev, generator=g) * scale).bfloat16() + + qkv = [_weights(QKV_N, H, gr, True) for _ in range(LAYERS)] + o = [_weights(H, O_K, gr, True) for _ in range(LAYERS)] + gu = [_weights(2 * I, H, gr, True) for _ in range(LAYERS)] + dn = [_weights(H, I, gr, False) for _ in range(LAYERS)] + layers = [DenseLayer(vec(), (QKV_N, H), vec(), *gu[i][0], *dn[i][0]) for i in range(LAYERS)] + final = vec() + qn = [(torch.randn(D, device=dev, generator=g) * 0.1).bfloat16() for _ in range(LAYERS)] + kn = [(torch.randn(D, device=dev, generator=g) * 0.1).bfloat16() for _ in range(LAYERS)] + cos_sin = (torch.rand(65536 + 64, 64, device=dev, generator=g) * 2 - 1).bfloat16() + one = torch.ones((), dtype=torch.float32, device=dev) + sm = D ** -0.5 + v_start = rank * V2 + + def stock_embed(ids): + local = ids - v_start + valid = (local >= 0) & (local < V2) + x = torch.nn.functional.embedding(torch.where(valid, local, 0), emb) + x.masked_fill_(~valid[:, None], 0) + return ca.custom_all_reduce(x) + + for reqs in reqs_list: + m = QL * reqs + cpu = torch.Generator().manual_seed(1000 + reqs) + pools, bts, seq, qsl, pos, slots = _kv(reqs, g, cpu) + ids = torch.full((reqs, QL), MASK_ID, dtype=torch.int64) + ids[:, 0] = torch.randint(0, 2 * V2, (reqs,), generator=cpu) + ids = ids.flatten().to(dev) + taps = [torch.randn(m, H, device=dev, generator=g).bfloat16() for _ in range(5)] + fc_out = torch.empty(m, H, dtype=torch.bfloat16, device=dev) + qbuf = torch.empty(m, NQ * D, dtype=torch.bfloat16, device=dev) + + def attention(i, x_qkv, mxfp8): + qkv_norm_rope_insert(x_qkv, pos, qn[i], kn[i], cos_sin, EPS, NQ, NKV, qbuf, pools[i], slots[i], one, one) + out = swa_decode_fp8kv_oproj_input(qbuf, pools[i], bts[i], seq, qsl, QL, sm, W, False, one, one, + oproj_n=H if mxfp8 else None) + if not mxfp8: + return out + if isinstance(out, torch.Tensor): + return G.mxfp8_linear(out, *o[i][0], torch.bfloat16) + codes, scales, tile = out + c8 = codes.view(torch.float8_e4m3fn) + return G.mxfp8_gemm_prequant(c8 if tile else c8[:m], scales, *o[i][0], torch.bfloat16, m).reshape(m, H) + + def block_attn(i, q, s): + x_qkv = G.mxfp8_gemm_prequant(q, s, *qkv[i][0], torch.bfloat16, m).reshape(m, QKV_N) + return attention(i, x_qkv, True) + + def block_stack(): + return blk.forward(ids, layers, block_attn, final, EPS, SWIGLU) + + def stock_fc(): + return torch.nn.functional.linear(torch.cat(taps, 1), fc_w) + + def stock_stack(): + x = stock_embed(ids) + residual, h = x, gemma_rmsnorm(x, layers[0].input_norm, EPS) + for i in range(LAYERS): + if i: + h, residual = gemma_fused_add_rmsnorm(h, residual, layers[i].input_norm, EPS) + x_qkv = _mxfp8_dot_scaled_linear(h, *qkv[i][1]) + a = attention(i, x_qkv, False) + h = ca.custom_all_reduce(_mxfp8_dot_scaled_linear(a, *o[i][1])) + h, residual = gemma_fused_add_rmsnorm(h, residual, layers[i].post_norm, EPS) + act = swiglu_bf16_reference(_mxfp8_dot_scaled_linear(h, *gu[i][1]), *SWIGLU, "bf16_chain") + h = ca.custom_all_reduce(G.mxfp8_linear(act, *dn[i][0], torch.bfloat16)) + return gemma_fused_add_rmsnorm(h, residual, final, EPS)[0] + + got, want = block_stack(), stock_stack() + blk.fc(taps, m, fc_out) + ref_fc = stock_fc() + torch.cuda.synchronize() + cos = torch.nn.functional.cosine_similarity(got.float().flatten(), want.float().flatten(), dim=0) + cos_fc = torch.nn.functional.cosine_similarity(fc_out.float().flatten(), ref_fc.float().flatten(), dim=0) + if check and (cos < 0.99 or cos_fc < 0.9999): + errors.append(f"rank {rank} R {reqs}: block vs stock cosine {float(cos):.5f}, fc {float(cos_fc):.6f}") + cases = [("block", block_stack, blk.capture), ("stock", stock_stack, ca.capture), + ("stock_fc", stock_fc, ca.capture)] + if parts: # one launch per layer's weights (cold), divided by the count below + q0, s0, res = blk.embed_norm_quant(ids, layers[0].input_norm, EPS, (QKV_N, H)) + part = (torch.randn(m, H, device=dev, generator=gr) * 0.1).bfloat16() + qg, sg, res = blk.ar_norm_quant(part, res, layers[0].post_norm, EPS, (2 * I, H)) + cases += [ + ("embed", lambda: blk.embed_norm_quant(ids, layers[0].input_norm, EPS, (QKV_N, H)), blk.capture), + ("attn", lambda: [block_attn(i, q0, s0) for i in range(LAYERS)], blk.capture), + ("tail_q", lambda: [blk.ar_norm_quant(part, res, layers[i].post_norm, EPS, (2 * I, H)) + for i in range(LAYERS)], blk.capture), + ("mlp", lambda: [mxfp8_swiglu_oai_mlp(qg, sg, m, *gu[i][0], *dn[i][0], *SWIGLU) for i in range(LAYERS)], + blk.capture), + ("tail_bf16", lambda: blk.ar_norm(part, res, final, EPS), blk.capture)] + graphs = {} + for name, fn, ctx in cases: + gph = torch.cuda.CUDAGraph() + with ctx(), torch.cuda.graph(gph): + fn() + graphs[name] = gph + for gph in graphs.values(): + gph.replay() + torch.cuda.synchronize() + + def replays(name): + def run(n): + for _ in range(n): + graphs[name].replay() + return run + + def block_fc(n): + for _ in range(n): + blk.fc(taps, m, fc_out) + + arms = {name: replays(name) for name in graphs} + arms["block_fc"] = block_fc + t = _timed(rank, world, arms, rounds, 10) + for name in ("attn", "tail_q", "mlp"): + if name in t: + t[name] /= LAYERS + if rank == 0: + roof, link = roofline_us(reqs) + new, old = t["block"] + t["block_fc"], t["stock"] + t["stock_fc"] + print(json.dumps(dict(reqs=reqs, rows=m, block_stack=round(t["block"], 1), block_fc=round(t["block_fc"], 1), + block=round(new, 1), stock_stack=round(t["stock"], 1), + stock_fc=round(t["stock_fc"], 1), stock=round(old, 1), speedup=round(old / new, 2), + roofline=round(roof, 1), roofline_link=round(link, 1), + frac=round(roof / new, 3), cos=round(float(cos), 5), + **{k: round(t[k], 1) for k in ("embed", "attn", "tail_q", "mlp", "tail_bf16") + if k in t})), flush=True) + del graphs, pools + torch.cuda.empty_cache() + blk.close() + ca.close() + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--reqs", nargs="+", type=int, default=[1, 2, 4, 8, 16, 24, 32, 40, 48, 52, 64, 96, 128]) + p.add_argument("--rounds", type=int, default=7) + p.add_argument("--no-check", action="store_true") + p.add_argument("--parts", action="store_true", help="also time the block's parts (per layer: attn, tails, MLP)") + a = p.parse_args() + from tests.multi_gpu import run_on_2_gpus + run_on_2_gpus(worker, a.reqs, a.rounds, not a.no_check, a.parts, timeout=3600) diff --git a/benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py b/benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py index aaf1f29..da1e5c9 100644 --- a/benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py +++ b/benchmarks/embedding/vocab_gather_gemma_mxfp8_h6144/benchmark_gather_norm.py @@ -1,20 +1,21 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Static peer embedding table, 100 changing ID batches, both capture orders. +"""Static peer embedding table on two or four ranks (``--world``), 100 changing ID batches, both capture orders. Six mask rows and one anchor per request; no cold-table serving claim. """ -import json,statistics as S,torch,torch.distributed as dist +import argparse,json,statistics as S,torch,torch.distributed as dist from amd_kernels.embedding.vocab_gather_h6144 import VocabGather from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import vocab_gather_gemma_mxfp8 from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144 import gemma_rmsnorm_mxfp8 -from tests.multi_gpu import run_on_4_gpus +from tests.multi_gpu import run_on_gpus def worker(rank,errors): torch.set_num_threads(8) dev=torch.device('cuda',rank) gen=torch.Generator(device=dev).manual_seed(913+rank) - shard=51200 + world=dist.get_world_size() + shard={4:51200,2:100032}[world] weight=torch.randn((shard,6144),device=dev,dtype=torch.bfloat16,generator=gen) lookup=VocabGather(weight,rank,dist.group.WORLD) norm=torch.randn(6144,device=dev,dtype=torch.bfloat16,generator=gen) @@ -24,7 +25,7 @@ def worker(rank,errors): inputs=[] for _ in range(100): ids=torch.full((requests,7),200058,device=dev,dtype=torch.int32) - ids[:,0]=torch.randint(0,4*shard,(requests,),device=dev,generator=gen,dtype=torch.int32) + ids[:,0]=torch.randint(0,world*shard,(requests,),device=dev,generator=gen,dtype=torch.int32) inputs.append(ids.flatten()) def fn(i,side): if side:return vocab_gather_gemma_mxfp8(inputs[i],lookup,norm) @@ -44,10 +45,12 @@ def fn(i,side): graphs[side].replay();st,en=torch.cuda.Event(True),torch.cuda.Event(True);st.record() for _ in range(10):graphs[side].replay() en.record();en.synchronize();times[side].append(st.elapsed_time(en)) - every=[None]*4;dist.all_gather_object(every,times) + every=[None]*world;dist.all_gather_object(every,times) if rank==0: vals=[[max(e[k][r] for e in every) for r in range(5)] for k in range(2)] print(json.dumps(dict(R=requests,order=order,old=vals[0],new=vals[1],unit='us')),flush=True) lookup.close() -if __name__=='__main__':run_on_4_gpus(worker,timeout=600) +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--world',type=int,default=4,choices=(2,4));a=p.parse_args() + run_on_gpus(worker,world=a.world,timeout=600) diff --git a/benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py b/benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py index 6e03a28..6a4cfc3 100644 --- a/benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py +++ b/benchmarks/embedding/vocab_gather_h6144/benchmark_gather.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Four-rank embedding unit latency; warm immutable weights, 100 distinct anchor batches. +"""Two- or four-rank embedding unit latency (``--world``); warm immutable weights, 100 distinct anchor batches. The full-draft A/B is the cold-serving evidence, since a small embedding-only graph cannot reproduce the 430 MB of intervening layer weights at R=1. @@ -10,13 +10,14 @@ import torch import torch.distributed as dist from amd_kernels.embedding.vocab_gather_h6144 import VocabGather -from tests.multi_gpu import run_on_4_gpus +from tests.multi_gpu import run_on_gpus def worker(rank,errors,requests): device=torch.device('cuda',rank) torch.manual_seed(31+rank) - weight=torch.randn((50016,6144),device=device,dtype=torch.bfloat16) + world=dist.get_world_size() + weight=torch.randn((200064//world,6144),device=device,dtype=torch.bfloat16) lookup=VocabGather(weight,rank,dist.group.WORLD) lookup.warm_up() for nr in requests: @@ -39,11 +40,12 @@ def worker(rank,errors,requests): for _ in range(10):graph.replay() end.record();end.synchronize() times.append(start.elapsed_time(end)) - every=[None]*4;dist.all_gather_object(every,times) + every=[None]*world;dist.all_gather_object(every,times) maximum=[max(values[i] for values in every) for i in range(5)] if rank==0:print(nr,statistics.median(maximum),[min(maximum),max(maximum)],'us/call',flush=True) lookup.close() if __name__=='__main__': - p=argparse.ArgumentParser();p.add_argument('--reqs',type=int,nargs='+',default=[1,10,16,32,64,128,256]);a=p.parse_args() - run_on_4_gpus(worker,a.reqs,timeout=300) + p=argparse.ArgumentParser();p.add_argument('--reqs',type=int,nargs='+',default=[1,10,16,32,64,128,256]) + p.add_argument('--world',type=int,default=4,choices=(2,4));a=p.parse_args() + run_on_gpus(worker,a.reqs,world=a.world,timeout=300) diff --git a/benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py b/benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py new file mode 100644 index 0000000..6164d99 --- /dev/null +++ b/benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two- or four-rank (``--world``) draft embedding against what vLLM runs without it: ``VocabParallelEmbedding``'s +mask / local gather / zero fill and AITER's custom all-reduce, then (``--norm``) the one-GPU Gemma norm + MXFP8 quant +against the fused ``vocab_gather_gemma_mxfp8``. Real shard size (200064 / world rows), warm weights, 100 distinct +batches of 7 rows per request (one random anchor + six mask ids) per graph, alternated rounds after a barrier, max +over ranks, median us per call. + + python3 benchmarks/embedding/vocab_gather_h6144/benchmark_stock.py --world 2 [--norm] +""" +import argparse +import statistics +import torch +import torch.distributed as dist +from amd_kernels.embedding.vocab_gather_h6144 import VocabGather +from tests.multi_gpu import run_on_gpus + + +def worker(rank, errors, requests, norm): + from aiter.dist.device_communicators.custom_all_reduce import CustomAllreduce + from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import vocab_gather_gemma_mxfp8 + from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144 import gemma_rmsnorm_mxfp8 + torch.set_num_threads(8) + dev = torch.device('cuda', rank) + world = dist.get_world_size() + v = 200064 // world + gen = torch.Generator(device=dev).manual_seed(31 + rank) + weight = torch.randn((v, 6144), device=dev, dtype=torch.bfloat16, generator=gen) + norm_w = torch.randn(6144, device=dev, dtype=torch.bfloat16, generator=gen) + lookup = VocabGather(weight, rank, dist.group.WORLD) + ca = CustomAllreduce(group=dist.group.WORLD, device=dev, max_size=64 * 1024 * 1024) + start = rank * v + + def stock_embed(ids): + local = ids - start + valid = (local >= 0) & (local < v) + x = torch.nn.functional.embedding(torch.where(valid, local, 0), weight) + x.masked_fill_(~valid[:, None], 0) + return ca.custom_all_reduce(x) + + for nr in requests: + inputs = [] + for i in range(100): + ids = torch.full((nr, 7), 200058, device=dev, dtype=torch.int64) + g = torch.Generator(device=dev).manual_seed(1000 * nr + i) + ids[:, 0] = torch.randint(0, 200064, (nr,), device=dev, generator=g) + inputs.append(ids.flatten()) + if norm: + fns = (lambda i: gemma_rmsnorm_mxfp8(stock_embed(inputs[i]), norm_w, 1e-6, 16), + lambda i: vocab_gather_gemma_mxfp8(inputs[i], lookup, norm_w, 1e-6, 16)) + else: + fns = (lambda i: stock_embed(inputs[i]), lambda i: lookup(inputs[i])) + for fn in fns: + fn(0) + torch.cuda.synchronize() + graphs = [] + for fn in fns: + gr = torch.cuda.CUDAGraph() + with ca.capture(), torch.cuda.graph(gr): + for i in range(100): + fn(i) + graphs.append(gr) + for gr in graphs: + gr.replay() + torch.cuda.synchronize() + ts = [[], []] + for r in range(6): + for k in ((0, 1) if r % 2 == 0 else (1, 0)): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + for _ in range(10): + graphs[k].replay() + b.record() + b.synchronize() + ts[k].append(a.elapsed_time(b)) # ms per 1000 calls = us per call + every = [None] * world + dist.all_gather_object(every, ts) + if rank == 0: + med = [statistics.median(max(e[k][r] for e in every) for r in range(6)) for k in range(2)] + print(f"requests {nr} (rows {7 * nr}): stock {med[0]:.2f} us | ours {med[1]:.2f} us | " + f"{med[0] / med[1]:.2f}x", flush=True) + del graphs + ca.close() + lookup.close() + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--reqs', type=int, nargs='+', default=[1, 10, 16, 32, 48, 64, 128, 256]) + p.add_argument('--world', type=int, default=2, choices=(2, 4)) + p.add_argument('--norm', action='store_true') + a = p.parse_args() + run_on_gpus(worker, a.reqs, a.norm, world=a.world, timeout=900) diff --git a/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py b/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py new file mode 100644 index 0000000..3fb0844 --- /dev/null +++ b/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks, the eager fc as the draft calls it: ``ConcatLinearComm`` (N-shard GEMM + slab reduce + shard exchange +into [rows, 6144]) on 3 rotated weight shards (566 MB) against what vLLM runs at TP2 (torch.cat + F.linear on the +replicated [6144, 30720] weight, 2 rotated copies, 755 MB). 20 distinct tap sets, 5 rounds of 20 eager calls +after a barrier, max over ranks, median us per call. ``--sweep`` times exchange forms (rows per group, push) instead. + + python3 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_comm.py [--rows ...] [--sweep] +""" +import argparse +import statistics +import torch +import torch.distributed as dist +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import pack_weight +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.comm import ConcatLinearComm, exchange_for +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.host import config_for +from tests.multi_gpu import run_on_2_gpus + +CALLS = 20 +XGMI = 64e9 + + +def _timed(fn, rank): + for i in range(3): + fn(i) + torch.cuda.synchronize() + times = [] + for _ in range(5): + dist.barrier() + torch.cuda.synchronize() + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for i in range(CALLS): + fn(i) + e.record() + e.synchronize() + times.append(s.elapsed_time(e) * 1e3 / CALLS) + every = [None] * 2 + dist.all_gather_object(every, times) + return statistics.median(max(v[i] for v in every) for i in range(5)) + + +def worker(rank, errors, rows, sweep): + torch.set_num_threads(8) + dev = torch.device('cuda', rank) + g = torch.Generator(device=dev).manual_seed(0) + full = [(torch.randn((6144, 30720), device=dev, generator=g) * 0.02).bfloat16() for _ in range(1 if sweep else 2)] + comms = [ConcatLinearComm(pack_weight(full[0][rank * 3072:(rank + 1) * 3072].clone()), rank, dist.group.WORLD) + for _ in range(3)] + for c in comms: + c.warm_up() + for m in rows: + xs = [[torch.randn((m, 6144), device=dev, generator=g).bfloat16() for _ in range(5)] for _ in range(CALLS)] + outs = [torch.empty((m, 6144), device=dev, dtype=torch.bfloat16) for _ in range(CALLS)] + if sweep: + res = {} + for push in (True, False): + for rpw in (1, 2, 4, 8): + for cpg in (1, 2, 3, 6, 12): + if (m + rpw - 1) // rpw * cpg > 1024 or (rpw > 1 and (m + rpw - 1) // rpw * cpg < 64): + continue + res[(rpw, cpg, push)] = _timed( + lambda i, ex=(rpw, cpg, push): comms[i % 3](xs[i], outs[i], exchange_config=ex), rank) + if rank == 0: + best = sorted(res.items(), key=lambda kv: kv[1])[:10] + print(f'rows {m}: ' + ' '.join(f"r{k[0]}p{k[1]}{'push' if k[2] else 'pull'}:{v:.1f}" for k, v in best), flush=True) + continue + ours = _timed(lambda i: comms[i % 3](xs[i], outs[i]), rank) + base = _timed(lambda i: torch.nn.functional.linear(torch.cat(xs[i], 1), full[i % 2]), rank) + hbm = (3072 * 30720 * 2 + m * 61440) / 6.5e12 + floor = (max(hbm, 2 * m * 3072 * 30720 / 2.5e15) + m * 3072 * 2 / XGMI) * 1e6 + if rank == 0: + print(f'rows {m}: replicated cat + F.linear {base:6.1f} us | ConcatLinearComm {ours:6.1f} us | ' + f'{base / ours:.2f}x | serial roofline {floor:5.1f} us ({floor / ours:.0%}) | tier {config_for(m)} ' + f'exchange {exchange_for(m)}', flush=True) + for c in comms: + c.close() + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--rows', type=int, nargs='+', default=[8, 56, 80, 128, 256, 448, 512, 1024, 2048]) + p.add_argument('--sweep', action='store_true') + a = p.parse_args() + run_on_2_gpus(worker, a.rows, a.sweep, timeout=3000) diff --git a/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py b/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py new file mode 100644 index 0000000..3b16f43 --- /dev/null +++ b/benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One GPU, cold weights: the N-shard GEMM + slab reduce (3 rotated packed 3072-column copies, 566 MB) vs torch.cat + +F.linear of the replicated weight (2 rotated copies, 755 MB; what vLLM runs at TP2), 20 distinct inputs per graph, 5 +alternated rounds x 10 replays, us per call (median [min-max]). The 2-rank gather is not included (see +benchmark_comm.py). ``--sweep`` times tier candidates (kernel, row tiles, column tiles per wave, K slabs) instead. + + python3 benchmarks/gemm/bf16_concat_linear_n3072_k30720/benchmark_concat_linear.py [--rows ...] [--sweep] +""" +import argparse +import statistics +import torch +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import bf16_concat_linear, pack_weight +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.host import config_for + +CANDIDATES = { + 'eightwave': [(8, nt, s) for nt in (2, 3, 4, 6) for s in (1, 2, 3, 4, 6, 8)], +} + + +def _lds_mt(m, mh): + tiles = (m + 128 * mh - 1) // (128 * mh) + return max(1, (m + 16 * mh * tiles - 1) // (16 * mh * tiles)) + + +def _candidates(kind, m): + if kind in ('lds', 'lds1', 'lds3'): + mh = 1 if kind == 'lds1' else 2 + return [(_lds_mt(m, mh), nr, s) for nr in (8, 12, 16, 24) for s in (5, 10, 15, 20, 30, 40)] + return CANDIDATES[kind] + + +def _valid(kind, mt, nt, s): + if kind in ('lds', 'lds1', 'lds3'): + mh = 1 if kind == 'lds1' else 2 + ring = 3 if kind == 'lds3' else 2 + nb = 480 // s + return 96 % nb == 0 and 192 % nt == 0 and nt % 4 == 0 and ring * (nt + mt * mh) * 2048 <= 160 * 1024 + nw = 8 if kind == 'eightwave' else 4 + steps = 960 // s + return 960 % s == 0 and steps % 2 == 0 and 3072 % (16 * nt * nw) == 0 + + +def _time(graphs, calls): + for gr in graphs: + gr.replay() + torch.cuda.synchronize() + ts = [[] for _ in graphs] + for r in range(5): + order = range(len(graphs)) if r % 2 == 0 else reversed(range(len(graphs))) + for k in order: + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(10): + graphs[k].replay() + e.record() + e.synchronize() + ts[k].append(s.elapsed_time(e) * 1e3 / (10 * calls)) + return ts + + +def _graph(fn, calls): + fn(0) + torch.cuda.synchronize() + gr = torch.cuda.CUDAGraph() + with torch.cuda.graph(gr): + for i in range(calls): + fn(i) + return gr + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--rows', type=int, nargs='+', default=[8, 56, 64, 80, 128, 256, 448, 512, 1024, 2048]) + p.add_argument('--sweep', action='store_true') + p.add_argument('--kinds', nargs='+', default=list(CANDIDATES) + ['lds', 'lds1', 'lds3']) + a = p.parse_args() + g = torch.Generator(device='cuda').manual_seed(0) + full = [(torch.randn((6144, 30720), device='cuda', generator=g) * 0.02).bfloat16() for _ in range(1 if a.sweep else 2)] + packed = [pack_weight(full[0][:3072].clone()) for _ in range(3)] + calls = 20 + for m in a.rows: + xs = [[torch.randn((m, 6144), device='cuda', generator=g).bfloat16() for _ in range(5)] for _ in range(calls)] + outs = [torch.empty((m, 3072), device='cuda', dtype=torch.bfloat16) for _ in range(calls)] + f = lambda v: '%.1f [%.1f-%.1f]' % (statistics.median(v), min(v), max(v)) # noqa: E731 + if a.sweep: + res = {} + for kind in a.kinds: + for mt, nt, s in _candidates(kind, m): + if not _valid(kind, mt, nt, s) or (mt * 16 > 2 * m and kind not in ('lds', 'lds1', 'lds3')): + continue + cfg = (kind, mt, nt, s) + try: + gr = _graph(lambda i: bf16_concat_linear(xs[i], packed[i % 3], out=outs[i], config=cfg), calls) + except Exception as ex: # noqa: BLE001 + print(f'rows {m} {cfg}: {type(ex).__name__}', flush=True) + continue + res[cfg] = statistics.median(_time([gr], calls)[0]) + del gr + best = sorted(res.items(), key=lambda kv: kv[1])[:8] + print(f'rows {m}: tier {config_for(m)} {res.get(config_for(m), float("nan")):.1f} | best ' + + ' '.join(f'{k}:{v:.1f}' for k, v in best), flush=True) + continue + + def old(i): + return torch.nn.functional.linear(torch.cat(xs[i], 1), full[i % 2]) + + def new(i): + return bf16_concat_linear(xs[i], packed[i % 3], out=outs[i]) + ts = _time([_graph(old, calls), _graph(new, calls)], calls) + flops = 2 * m * 3072 * 30720 + floor = max((3072 * 30720 * 2 + m * 61440) / 6.5e12, flops / 2.5e15) * 1e6 + print(f'rows {m}: cat + F.linear (replicated) {f(ts[0])} us, N-shard GEMM + reduce {f(ts[1])} us, ' + f'roofline {floor:.1f} us ({floor / statistics.median(ts[1]):.0%}), tier {config_for(m)}', flush=True) + del xs + + +if __name__ == '__main__': + main() diff --git a/benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py b/benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py new file mode 100644 index 0000000..ec733d1 --- /dev/null +++ b/benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph replay of bf16_gemm_n100032_k6144 (shuf layout, contiguous and the publish body) against vLLM's lm_head GEMM +(``rocm_unquantized_gemm_impl`` on the row-major shard, VLLM_ROCM_USE_AITER=1): one graph of 50 calls on 50 distinct +activation sets, one weight copy (1229 MB > the 256 MB MALL: HBM on every call, as in serving), 10 replays x 5 +alternated rounds, median us. +Run: python3 benchmarks/gemm/bf16_n100032_k6144/benchmark_gemm.py [M ...] +""" +import os +import statistics +import sys + +os.environ.setdefault("VLLM_ROCM_USE_AITER", "1") + +import torch + +from amd_kernels.gemm.bf16_n100032_k6144 import bf16_gemm_n100032_k6144, shuffle_weight, warmup_bf16_gemm_n100032_k6144 +from amd_kernels.gemm.bf16_n100032_k6144.reference import K, N + +CALLS = 50 + + +def capture(calls): + for f in calls[:2]: + f() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for f in calls: + f() + graph.replay() + torch.cuda.synchronize() + return graph + + +def replay_us(graph): + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(10): + graph.replay() + b.record() + b.synchronize() + return a.elapsed_time(b) * 1000 / (10 * CALLS) + + +def roofline_us(m): + hbm = (N * K + m * K + m * N) * 2 / 6.5e12 + return max(hbm, 2 * m * N * K / 2.5e15) * 1e6 + + +def vllm_gemm(): + try: + from vllm.model_executor.layers.utils import rocm_unquantized_gemm_impl + return rocm_unquantized_gemm_impl + except Exception as e: # noqa: BLE001 + print(f"no vLLM baseline: {e}") + return None + + +if __name__ == "__main__": + ms = [int(v) for v in sys.argv[1:]] or [1, 8, 16, 64, 80, 128, 192, 224, 256, 320, 384, 416, 448, 512, 832, 1024, + 2048] + dev = torch.device("cuda") + torch.manual_seed(441) + w = (torch.randn(N, K, device=dev) * 0.02).to(torch.bfloat16) + ws = shuffle_weight(w) + warmup_bf16_gemm_n100032_k6144(dev, True) + warmup_bf16_gemm_n100032_k6144(dev, True, "publish") + base = vllm_gemm() + print("M | vLLM us | contiguous us | publish body us | roofline us | contiguous / roofline", flush=True) + for m in ms: + xs = [torch.randn(m, K, device=dev).to(torch.bfloat16) for _ in range(CALLS)] + out = torch.empty(m, N, dtype=torch.bfloat16, device=dev) + idx = torch.randperm(m, device=dev) + pool = torch.empty(2 * m * N * 2, dtype=torch.uint8, device=dev) + counter = torch.zeros(1, dtype=torch.int32, device=dev) + flag = torch.zeros(16, dtype=torch.int32, device=dev) + publish = (pool.data_ptr(), m * N * 2, counter, flag.data_ptr()) + variants = {"contiguous": [lambda x=x: bf16_gemm_n100032_k6144(x, ws, out, shuffled=True) for x in xs], + "publish": [lambda x=x: bf16_gemm_n100032_k6144(x, ws, shuffled=True, row_indices=idx, + publish=publish) for x in xs]} + if base is not None: + variants["vllm"] = [lambda x=x: base(x, w) for x in xs] + graphs = {k: capture(v) for k, v in variants.items()} + times = {k: [] for k in graphs} + for _ in range(5): + for k, g in graphs.items(): + times[k].append(replay_us(g)) + med = {k: statistics.median(v) for k, v in times.items()} + roof = roofline_us(m) + print(f"{m} | {med.get('vllm', float('nan')):.1f} | {med['contiguous']:.1f} | {med['publish']:.1f} | " + f"{roof:.1f} | {roof / med['contiguous']:.2f}", flush=True) + del graphs, xs, pool + torch.cuda.empty_cache() diff --git a/benchmarks/gemm/bf16_n200064_k256/benchmark_gemm.py b/benchmarks/gemm/bf16_n200064_k256/benchmark_gemm.py index f873633..4f6d947 100644 --- a/benchmarks/gemm/bf16_n200064_k256/benchmark_gemm.py +++ b/benchmarks/gemm/bf16_n200064_k256/benchmark_gemm.py @@ -33,7 +33,7 @@ def timing(calls): warmup_bf16_gemm_packed_rows(dev) weight=torch.randn(N,K,dtype=torch.bfloat16,device=dev) weights=[weight,weight.clone(),weight.clone()] - for n in (3,4,8,16,32,64): + for n in (3,4,5,8,16,32,64,96,128): idx=torch.arange(n,device=dev) ps=[torch.empty(padded_rows(n),K,dtype=torch.bfloat16,device=dev) for _ in range(100)] for p in ps:pack_bf16_rows(torch.randn(n,K,dtype=torch.bfloat16,device=dev),idx,p) diff --git a/benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py b/benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py new file mode 100644 index 0000000..175b843 --- /dev/null +++ b/benchmarks/gemm/mxfp8_common/benchmark_decode_splitk.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Interleaved A/B of the table's row against explicit decode / split-K decode rows on (16, 16) codes, per M: graph +replay over rotating weight copies > 400 MB (cold HBM), median of interleaved rounds. Every candidate is checked +first: the fp64 contract of reference.py, bitwise repeats and weight copies, and on order-sensitive data (per-32 +scales over +-12 binades) whether it keeps the table row's bytes. + + python3 benchmark_decode_splitk.py --shape 4992,6144 --ms 96,176 \ + --cands "splitk:mb=1,waves=4,cols=1,loaders=4,ks=3,lw=2|decode:tm=48,tn=48,kw=3" + +splitk keys: mb (m-blocks), waves (MFMA waves), cols (16-column tiles per wave), loaders, ks, lw, ring (0: all the +LDS holds); decode keys: tm, tn, kw, ks (split-K over workgroups). +""" +import argparse +import json +import statistics + +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.reference import COS_MIN, REL_L2_MAX, compare, mxfp8_gemm_reference, quantize_mxfp8 + +BF16 = torch.bfloat16 +LLC_BYTES = 400 << 20 + + +def _data(rows, cols, scale, seed, binades, spikes=False): + g = torch.Generator(device="cuda").manual_seed(seed) + e = torch.randint(-binades, binades + 1, (rows, cols // 32), device="cuda", generator=g).float() + t = torch.randn(rows, cols, device="cuda", generator=g) * scale * torch.exp2(e).repeat_interleave(32, 1) + if spikes: + t[:, :: cols // 16] *= 8.0 + return t.to(BF16) + + +def config_of(spec, m): + kind, args = spec.split(":") + o = {key: int(v) for key, v in (kv.split("=") for kv in args.split(","))} + if kind == "decode": + return G._Config(o["tm"], o["tn"], 128, 0, 0, o.get("ks", 1), G._DECODE_KERNEL, o["kw"], 0) + tile_m = 16 * -(-(-(-m // o["mb"])) // 16) + return G._Config(tile_m, 16 * o["cols"] * o["waves"], 128, 0, 0, o["ks"], G._DECODE_SPLITK_KERNEL, o["waves"], + o["loaders"], o["lw"], o.get("ring", 0)) + + +def calls_of(config, m, x, weights): + q, s = G.mxfp8_e4m3_quantize_shuffled(x, max(16, config.tile_m)) + return [lambda w=w, ws=ws: G.mxfp8_gemm_shuffled(q, s, w, ws, BF16, m, config=config) for w, ws in weights], (q, s) + + +def build_graph(calls, inner=2): + for c in calls: + c() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for i in range(len(calls) * inner): + calls[i % len(calls)]() + torch.cuda.synchronize() + return g, len(calls) * inner + + +def time_graph(g, n_calls, replays): + g.replay() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + for _ in range(replays): + g.replay() + b.record() + b.synchronize() + return a.elapsed_time(b) * 1e3 / (replays * n_calls) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--shape", required=True) + p.add_argument("--ms", required=True) + p.add_argument("--cands", required=True) + p.add_argument("--rounds", type=int, default=9) + p.add_argument("--replays", type=int, default=4) + p.add_argument("--json") + a = p.parse_args() + n, k = map(int, a.shape.split(",")) + wq, we = quantize_mxfp8(_data(n, k, 0.05, n * 7 + k, 2)) + wwq, wwe = quantize_mxfp8(_data(n, k, 0.05, n * 3 + k, 12)) + wbytes = n * k * 33 // 32 + w0 = G.preshuffle_mxfp8_weight(wq.clone(), we.clone()) + weights = [w0] + [(w0[0].clone(), w0[1].clone()) for _ in range(max(2, -(-LLC_BYTES // wbytes)) - 1)] + wide = [G.preshuffle_mxfp8_weight(wwq.clone(), wwe.clone())] + res = {"N": n, "K": k, "m": {}} + for m in map(int, a.ms.split(",")): + x = _data(m, k, 1.0, 1000 + m, 2, spikes=True) + ref = mxfp8_gemm_reference(*quantize_mxfp8(x), wq, we, accumulate=torch.float64) + xw = _data(m, k, 1.0, 2000 + m, 12) + refw = mxfp8_gemm_reference(*quantize_mxfp8(xw), wwq, wwe, accumulate=torch.float64) + table = G._select_prequant_config(m, n, k, BF16) + configs = {"table": table, **{spec: config_of(spec, m) for spec in a.cands.split("|")}} + info, graphs, keep, table_wide = {}, {}, [], None + for name, config in configs.items(): + calls, buf = calls_of(config, m, x, weights) + wcalls, wbuf = calls_of(config, m, xw, wide) + o1, o2, o3 = calls[0]().clone(), calls[1]().clone(), calls[0]().clone() + ow = wcalls[0]().clone() + table_wide = ow if name == "table" else table_wide + std, wd = compare(o1, ref), compare(ow, refw) + ok = (std["cos"] >= COS_MIN and std["rel_l2"] <= REL_L2_MAX and wd["cos"] >= COS_MIN + and torch.equal(o1, o2) and torch.equal(o1, o3) and bool(torch.isfinite(o1.float()).all())) + info[name] = dict(config=str(config), std=std, wide=wd, ok=ok, + table_bytes=bool(torch.equal(ow, table_wide))) + keep += [calls, buf, wcalls, wbuf] + if ok: + graphs[name] = build_graph(calls) + samples = {key: [] for key in graphs} + for _ in range(a.rounds): + for key, (g, nc) in graphs.items(): + samples[key].append(time_graph(g, nc, a.replays)) + base = statistics.median(samples["table"]) + for key, v in samples.items(): + us = statistics.median(v) + info[key].update(us=us, us_min=min(v), us_max=max(v)) + tb = wbytes / us / 1e6 + print(f"N{n} K{k} M{m} {key:52s} {us:7.2f} us {tb:4.2f} TB/s ({tb / 6.5:4.0%} of 6.5, {tb / 8:4.0%} of 8)" + f" vs table {us / base - 1:+6.1%} ok {info[key]['ok']} table bytes {info[key]['table_bytes']}", + flush=True) + res["m"][m] = info + del graphs, keep + torch.cuda.empty_cache() + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1, default=str) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_common/benchmark_table.py b/benchmarks/gemm/mxfp8_common/benchmark_table.py new file mode 100644 index 0000000..c12a4b7 --- /dev/null +++ b/benchmarks/gemm/mxfp8_common/benchmark_table.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-replay timing of the table's rows of one (N, K) against vLLM's Triton ``tl.dot_scaled`` MXFP8 linear and +the roofline, weights HBM-cold (``sweep_rows.Bench``: rotating copies above the 256 MB MALL). + +Both sides start from the bf16 activation: ``mxfp8_linear`` (the row's quantizer + GEMM) and vLLM's +``rocm_native._mxfp8_dot_scaled_linear`` (its quantizer + GEMM with ``_M31_TILES`` / ``_select_cfg``, row-major +weights). Roofline: max(bytes / 6.5 TB/s, flops / 5 PF), bytes = weight codes + scales, bf16 activation, bf16 output. +Needs vLLM on the path (``PYTHONPATH=``). + + python3 benchmarks/gemm/mxfp8_common/benchmark_table.py --shape 3072,6144 --ms 1,8,64,256,1024,8192 +""" + +import argparse + +from benchmarks.gemm.mxfp8_common.sweep_rows import Bench, _key, roofline_us + +from amd_kernels.gemm.mxfp8_common import gemm as G + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--shape", required=True) + p.add_argument("--ms", default="1,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768") + p.add_argument("--no-baseline", action="store_true") + args = p.parse_args() + n, k = map(int, args.shape.split(",")) + bench = Bench(n, k, not args.no_baseline) + print(f"N {n} x K {k}, {bench.slots} weight copies") + print("| M | row | us | Triton dot_scaled us | vs Triton | roofline us | of roofline |") + print("|---|---|---|---|---|---|---|") + for m in map(int, args.ms.split(",")): + bench.set_m(m) + us = bench.time(None, "linear", fast=False) + base = None if args.no_baseline else bench.time("baseline", "linear", fast=False) + roof = roofline_us(m, n, k) + vs = "" if base is None else f"{(us / base - 1) * 100:+.0f}%" + print(f"| {m} | {_key(G._select_config(m, n, k))} | {us:.2f} | {'' if base is None else f'{base:.2f}'} | {vs} " + f"| {roof:.2f} | {roof / us * 100:.0f}% |", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_common/sweep_rows.py b/benchmarks/gemm/mxfp8_common/sweep_rows.py new file mode 100644 index 0000000..477b3a1 --- /dev/null +++ b/benchmarks/gemm/mxfp8_common/sweep_rows.py @@ -0,0 +1,310 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Graph-replay sweep of the table rows of one (N, K): every candidate configuration at every M, weights HBM-cold. + +A candidate is timed through the bf16 entry (``mxfp8_linear`` with an explicit config: the activation quantizer the +row needs plus the GEMM, or the in-kernel quantizer of ``decode_fq``), or with ``--entry codes`` on codes made +before timing (the producer path: (16, 16) codes for decode rows, row-major otherwise). Each candidate is checked +against ``reference.py`` on the first weight copy; the first pass times every candidate briefly, the second +re-times the best ``--refine`` per M. ``--table`` adds the table's own row, ``--baseline`` vLLM's Triton +``tl.dot_scaled`` linear (needs vLLM on the path). + + python3 benchmarks/gemm/mxfp8_common/sweep_rows.py --shape 3072,6144 --family decode --ms 1,8,64,256 +""" + +import argparse +import dataclasses +import functools +import json +import statistics + +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.gemm import _Config +from amd_kernels.gemm.mxfp8_common.reference import COS_MIN, REL_L2_MAX, compare, mxfp8_linear_reference, quantize_mxfp8 +from benchmarks.gemm.mxfp8_common.benchmark_gemm import LLC_TARGET_BYTES, MAX_SLOTS, timeit_graph + + +def _ks_kw_pairs(k): + steps = k // 128 + return [(ks, kw) for ks in (1, 2, 3, 4, 6, 8) for kw in (1, 2, 3, 4) + if steps % (ks * kw) == 0 and (steps // (ks * kw)) % 2 == 0] + + +def decode_candidates(m, n, k, w_cms=(None, 0)): + """Decode-kernel rows: the smallest row tile per m-block count (1-4 blocks; 5-8 with cached W, one or two + K-waves), column tiles 16..128, K-waves 1-4, split-K where one split leaves the grid under 256 workgroups.""" + out = [] + seen_blocks = set() + for rt in range(1, 13): + blocks = -(-m // (16 * rt)) + if blocks > 8 or blocks in seen_blocks: + continue + seen_blocks.add(blocks) + for ni in (1, 2, 3, 4, 5, 6, 8): + if n % (16 * ni) or rt * ni > 64: + continue + base = blocks * n // (16 * ni) + for ks, kw in _ks_kw_pairs(k): + if ks > 1 and base >= 256 or ks * base > 8192 or ks * base < 64: + continue + if (kw - 1) * rt * ni * 1024 > 160 * 1024: + continue + if blocks > 4 and (kw > 2 or ks > 1 or 0 not in w_cms): + continue + for w_cm in (w_cms if blocks <= 4 else (0,)) if blocks > 1 else (None,): + out.append(_Config(16 * rt, 16 * ni, 128, 0, 0, ks, "decode", kw, 0, w_cm=w_cm)) + return out + + +def decode_fq_candidates(m, n, k): + """In-kernel-quantizer rows: tile 4 / 8 (folded) or 16 / 32 rows, one m-block.""" + out = [] + tiles = [t for t in (4, 8, 16, 32, 64) if t >= m][:2] + for tile in tiles: + rt = max(1, tile // 16) + for ni in (1, 2, 3, 4, 6): + if n % (16 * ni): + continue + base = n // (16 * ni) + for ks, kw in _ks_kw_pairs(k): + sw = k // 128 // (ks * kw) + fold = 16 // tile if tile in (4, 8) else 1 + if sw % fold or (ks > 1 and base >= 256) or ks * base < 64: + continue + out.append(_Config(tile, 16 * ni, 128, 0, 0, ks, "decode_fq", kw, 0)) + return out + + +def aiter_candidates(m, n, k): + out = [] + for tm, tn, tk in ((32, 32, 256), (32, 64, 128), (32, 64, 256), (32, 128, 256), (64, 64, 256), (64, 128, 256), + (128, 64, 256), (128, 128, 128), (128, 128, 256), (128, 256, 128)): + if n % tn or (tm >= 128 and m <= 64): + continue + tiles = -(-m // tm) * (n // tn) + for wpe, xcd in ((0, 0), (2, 4)): + for sk in (1, 2, 3, 4, 6, 8): + if k % (256 * sk) or (sk > 1 and tiles >= 256) or sk * tiles > 8192: + continue + out.append(_Config(tm, tn, tk, wpe, xcd, sk)) + return out + + +def fourwave_candidates(m, n, k): + if n % 256: + return [] + out = [] + for tm, tn in ((256, 256), (128, 256), (256, 128), (128, 128)): + for kernel in ("4wave", "4wave_nt"): + for sk in ((0, 1) if (tm, tn) == (256, 256) else (1,)): + out.append(_Config(tm, tn, 128, 1, 1, sk, kernel, 2, 2)) + return out + + +def staged_candidates(m, n, k): + out = [] + for tm, tn, wn in ((128, 128, 4), (128, 256, 4), (256, 128, 4), (64, 128, 2), (128, 64, 2)): + if n % tn or tm > 64 and m < 512: + continue + for sk in (1, 2): + if k % (256 * sk): + continue + out.append(_Config(tm, tn, 128, 2, 4, sk, "staged_ab", 2, wn)) + return out + + +def tall_candidates(m, n, k): + """Decode rows of 13-16 row tiles (one m-block reads W once at M <= 256).""" + out = [] + for rt in range(13, 17): + for ni in (1, 2, 3, 4): + if n % (16 * ni) or -(-m // (16 * rt)) > 2: + continue + for ks, kw in _ks_kw_pairs(k): + if ks > 2 or (kw - 1) * rt * ni * 1024 > 160 * 1024: + continue + for w_cm in (None, 0): + out.append(_Config(16 * rt, 16 * ni, 128, 0, 0, ks, "decode", kw, 0, w_cm=w_cm)) + return out + + +def knob_candidates(m, n, k): + """The table's decode row at ``m`` with scheduling barriers on / off and shallower lookaheads.""" + row = G._select_config(m, n, k) + if row.kernel not in ("decode", "decode_fq"): + return [] + out = [] + for waves_n in (0, 1): + for la in ((0, 0), (1, 1), (1, 2), (2, 2), (2, 4), (4, 4), (4, 8), (6, 6), (8, 8), (8, 16)): + out.append(dataclasses.replace(row, waves_n=waves_n, la_w=la[0], la_a=la[1])) + return out + + +FAMILIES = { + "knobs": knob_candidates, + "tall": tall_candidates, + "decode": lambda m, n, k: decode_fq_candidates(m, n, k) * (m <= 64) + decode_candidates(m, n, k), + "decode_nocm": lambda m, n, k: decode_fq_candidates(m, n, k) * (m <= 64) + decode_candidates(m, n, k, (None,)), + "aiter": aiter_candidates, + "all": lambda m, n, k: (decode_fq_candidates(m, n, k) * (m <= 64) + decode_candidates(m, n, k) + + aiter_candidates(m, n, k) * (m >= 64) + + (fourwave_candidates(m, n, k) + staged_candidates(m, n, k)) * (m >= 256)), + "prefill": lambda m, n, k: fourwave_candidates(m, n, k) + aiter_candidates(m, n, k) + staged_candidates(m, n, k), +} + + +def _key(cfg): + if cfg is None: + return "table" + if isinstance(cfg, str): + return cfg + s = f"{cfg.kernel} {cfg.tile_m}x{cfg.tile_n}" + if cfg.kernel.startswith("decode"): + s += f" kw{cfg.waves_m} ks{cfg.split_k}" + ("" if cfg.w_cm is None else f" cm{cfg.w_cm}") + s += (f" la{cfg.la_w}x{cfg.la_a}" if cfg.la_w else "") + (" sb" if cfg.waves_n else "") + elif cfg.kernel.startswith("4wave"): + s += f" sk{cfg.split_k} xcd{cfg.xcd_swizzle}" + else: + s += f" k{cfg.tile_k} wpe{cfg.waves_per_eu} xcd{cfg.xcd_swizzle} sk{cfg.split_k}" + if cfg.kernel == "staged_ab": + s += f" wn{cfg.waves_n}" + return s + + +@functools.cache +def _decode_launcher_with(opts, n, k, config, a_bf16): + """``gemm._decode_launcher`` with kernel options the table rows do not carry (``GROUPED``, ...).""" + from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import compile_mxfp8_gemm_decode + + return compile_mxfp8_gemm_decode( + N=n, K=k, RT=max(1, config.tile_m // 16), NI=config.tile_n // 16, KW=config.waves_m, KS=config.split_k, + LA=(config.la_w, config.la_a) if config.la_w else None, sched_barriers=bool(config.waves_n), A_BF16=a_bf16, + FOLD=G._decode_fold(config) if a_bf16 else 1, W_CM=config.w_cm, **dict(opts), + ) + + +class Bench: + def __init__(self, n, k, baseline): + self.n, self.k = n, k + torch.manual_seed(n + k) + w = torch.randn(n, k, device="cuda", dtype=torch.bfloat16) * 0.05 + self.codes, self.exps = quantize_mxfp8(w) + sw, ss = G.preshuffle_mxfp8_weight(self.codes, self.exps) + wbytes = n * k * 33 // 32 + self.slots = min(MAX_SLOTS, max(2, -(-LLC_TARGET_BYTES // wbytes))) + self.weights = [(sw.clone(), ss.clone()) for _ in range(self.slots)] + self.rm = [(self.codes.clone(), self.exps.clone()) for _ in range(self.slots)] if baseline else None + self.x = None + self.ref = None + + def set_m(self, m): + torch.manual_seed(m) + self.x = torch.randn(m, self.k, device="cuda", dtype=torch.bfloat16) + self.ref = mxfp8_linear_reference(self.x, self.codes, self.exps) + + def calls(self, cfg, entry): + x, m = self.x, self.x.shape[0] + if cfg == "baseline": + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import _mxfp8_dot_scaled_linear + + return [lambda w=w, s=s: _mxfp8_dot_scaled_linear(x, w, s) for w, s in self.rm] + if entry == "linear": + return [lambda w=w, s=s: G.mxfp8_linear(x, w, s, torch.bfloat16, config=cfg) for w, s in self.weights] + if cfg is None: # the table's row on the codes its producer writes + tile = G.prequant_code_tile_rows(m, self.n, self.k) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else G.mxfp8_e4m3_quantize_aiter(x) + return [lambda w=w, ws=ws: G.mxfp8_gemm_prequant(q, s, w, ws, torch.bfloat16, m) for w, ws in self.weights] + if cfg.kernel.startswith("decode"): + c = dataclasses.replace(cfg, kernel="decode", tile_m=max(16, cfg.tile_m), min_m=0) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, c.tile_m) + return [lambda w=w, ws=ws: G.mxfp8_gemm_shuffled(q, s, w, ws, torch.bfloat16, m, config=c) + for w, ws in self.weights] + q, s = G.mxfp8_e4m3_quantize_aiter(x) + return [lambda w=w, ws=ws: G.mxfp8_gemm(q, s, w, ws, torch.bfloat16, True, config=cfg) + for w, ws in self.weights] + + def time(self, cfg, entry, fast, opts=()): + if opts: + original = G._decode_launcher + G._decode_launcher = functools.partial(_decode_launcher_with, opts) + try: + return self.time(cfg, entry, fast) + finally: + G._decode_launcher = original + calls = self.calls(cfg, entry) + got = calls[0]() + torch.cuda.synchronize() + met = compare(got, self.ref) + if not (met["cos"] >= COS_MIN and met["rel_l2"] <= REL_L2_MAX): + raise AssertionError(f"{_key(cfg)}: {met}") + m = self.x.shape[0] + flops = 2 * m * self.n * self.k + work = max(flops * len(calls), 1) + replays = max(2, min(20, int(2e10 // work))) if fast else max(5, min(50, int(1e11 // work))) + return timeit_graph(calls, inner=2, replays=replays, rounds=3 if fast else 7, warmup=2 if fast else 5) * 1e3 + + +def roofline_us(m, n, k, bw=6.5e12, fp8=5e15): + bytes_ = n * k * 33 / 32 + m * k * 2 + m * n * 2 + return max(bytes_ / bw, 2 * m * n * k / fp8) * 1e6 + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--shape", required=True) + p.add_argument("--ms", required=True) + p.add_argument("--family", default="decode", choices=[*FAMILIES, "none"]) + p.add_argument("--entry", default="linear", choices=("linear", "codes")) + p.add_argument("--table", action="store_true") + p.add_argument("--baseline", action="store_true") + p.add_argument("--refine", type=int, default=6) + p.add_argument("--repeat", type=int, default=1, help="interleaved repeats of the final timings") + p.add_argument("--grouped", action="store_true", help="also the best decode rows with the grouped XCD order") + p.add_argument("--extra", default="", help="';'-separated _Config(...) argument lists timed at every M") + p.add_argument("--json") + args = p.parse_args() + n, k = map(int, args.shape.split(",")) + ms = [int(v) for v in args.ms.split(",")] + bench = Bench(n, k, args.baseline) + extra = [eval(f"_Config({e})") for e in args.extra.split(";") if e.strip()] + results = {} + for m in ms: + bench.set_m(m) + cands = (FAMILIES[args.family](m, n, k) if args.family != "none" else []) + extra + times = {} + for cfg in cands: + try: + times[cfg] = bench.time(cfg, args.entry, fast=True) + except Exception as e: # noqa: BLE001 - a candidate the kernel rejects + print(f" M {m} {_key(cfg)}: {type(e).__name__} {str(e)[:120]}", flush=True) + best = sorted(times, key=times.get)[: args.refine] + entries = [(_key(cfg), cfg, ()) for cfg in best + ([None] if args.table else []) + + (["baseline"] if args.baseline else [])] + if args.grouped: + entries += [(_key(cfg) + " g4", cfg, (("GROUPED", True),)) for cfg in best + if cfg.kernel == "decode" and cfg.split_k == 1 and -(-m // cfg.tile_m) > 1] + samples = {} + for _ in range(args.repeat): # interleaved, the median per entry + for key, cfg, opts in entries: + try: + samples.setdefault(key, []).append(bench.time(cfg, args.entry, False, opts)) + except Exception as e: # noqa: BLE001 + print(f" M {m} {key}: {type(e).__name__} {str(e)[:200]}", flush=True) + cfgs = {key: (G._select_config(m, n, k) if cfg is None else cfg) for key, cfg, _ in entries} + final = {key: (statistics.median(v), cfgs[key]) for key, v in samples.items()} + order = sorted((v[0], key) for key, v in final.items() if v is not None) + roof = roofline_us(m, n, k) + line = " ".join(f"{key} {us:.2f}" for us, key in order) + print(f"N{n} K{k} M {m} ({len(cands)} cands, roof {roof:.2f}): {line}", flush=True) + results[m] = {key: {"us": v[0], "cfg": None if not isinstance(v[1], _Config) else v[1].__dict__} + for key, v in final.items() if v is not None} + results[m]["roofline_us"] = roof + if args.json: + with open(args.json, "w") as f: + json.dump({"n": n, "k": k, "entry": args.entry, "results": results}, f, indent=1) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py b/benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py new file mode 100644 index 0000000..652fb5d --- /dev/null +++ b/benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Whole dense FFN at 1-16 rows, TP2 (gate_up N 12288 K 6144, down N 6144 K 6144): the fused gate_up + SwiGLU kernel + +the down linear against what vLLM runs at TP2 without it (Triton ``tl.dot_scaled`` gate_up on row-major codes, the +form's SwiGLU -- Triton ``swiglu_oai_split`` for ``fp32``, the aten chain for ``bf16_chain`` -- and the same down +linear). Graph replay of 100 calls over 6 rotated weight copies (> 900 MB per side), both sides alternated over 5 +rounds, median us per FFN. ``--sweep`` times the fused kernel alone over K-wave counts and lookaheads (each checked +bit for bit against the decode GEMM of the same K-wave count first). + + python3 benchmarks/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/benchmark_gateup_swiglu_bf16.py [--rows 1 7 8] [--sweep] +""" +import argparse +import statistics + +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.reference import quantize_mxfp8, swiglu_bf16_reference +from amd_kernels.gemm.mxfp8_gateup_swiglu_n12288_k6144_i6144 import host as GUS + +N, K, I = GUS.N, GUS.K, GUS.I +ALPHA, BETA, LIMIT = 1.702, 1.0, 7.0 +COPIES, CALLS = 6, 100 +SWEEP = [(kw, (lw, la)) for kw in (1, 2, 3, 4) for lw in (4, 5, 6, 7, 8) for la in (4, 6, 7, 8)] +FINALISTS = [(1, (7, 7)), (1, (7, 8)), (1, (8, 6)), (2, (7, 7)), (2, (7, 8)), (3, (4, 8)), (3, (5, 8)), (3, (4, 7))] + + +def _time(graph, rounds): + graph.replay() + torch.cuda.synchronize() + samples = [] + for _ in range(rounds): + start, end = torch.cuda.Event(True), torch.cuda.Event(True) + start.record() + for _ in range(10): + graph.replay() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) / (10 * CALLS) * 1e3) + return statistics.median(samples) + + +def _capture(fn): + fn(0) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for i in range(CALLS): + fn(i) + return graph + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--rows', type=int, nargs='+', default=[1, 2, 4, 7, 8, 14, 16]) + parser.add_argument('--rounds', type=int, default=5) + parser.add_argument('--sweep', action='store_true') + parser.add_argument('--finalists', action='store_true', help='sweep only the shortlisted configs') + args = parser.parse_args() + torch.manual_seed(1973) + gq, gs = quantize_mxfp8((torch.randn((N, K), device='cuda') * .02).bfloat16()) + dq, ds = quantize_mxfp8((torch.randn((K, I), device='cuda') * .02).bfloat16()) + gate_up = [G.preshuffle_mxfp8_weight(gq, gs) for _ in range(COPIES)] + down = [G.preshuffle_mxfp8_weight(dq, ds) for _ in range(COPIES)] + row_major = [(gq.clone(), gs.clone()) for _ in range(COPIES)] + for m in args.rows: + assert 1 <= m <= 16 + xs = [(torch.randn((m, K), device='cuda') * 2).bfloat16() for _ in range(CALLS)] + tiled = [G.mxfp8_e4m3_quantize_shuffled(x, 16) for x in xs] + if args.sweep or args.finalists: + q, s = tiled[0] + cells = [] + for config in FINALISTS if args.finalists else SWEEP: + gu = torch.empty((m, N), device='cuda', dtype=torch.bfloat16) + G._launch_decode(G._Config(16, 32, 128, 0, 0, 1, 'decode', config[0], 0, 7, 7), q, s, *gate_up[0], gu) + forms = ('fp32',) if config[0] == 1 else ('fp32', 'bf16_chain') + for form in forms: + got = GUS.mxfp8_gateup_swiglu_bf16(q, s, *gate_up[0], m, form=form, config=config) + if not torch.equal(got.view(torch.int16), swiglu_bf16_reference(gu, form=form).view(torch.int16)): + cells.append(f"{form} kw{config[0]} la{config[1]}: WRONG") + continue + us = _time(_capture(lambda i, f=form, c=config: GUS.mxfp8_gateup_swiglu_bf16( + *tiled[i], *gate_up[i % COPIES], m, form=f, config=c)), args.rounds) + cells.append(f"{form} kw{config[0]} la{config[1][0]}x{config[1][1]}: {us:.2f}") + print(f"rows={m}: " + " | ".join(cells), flush=True) + continue + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import _mxfp8_dot_scaled_gemm + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import mxfp8_e4m3_quantize + from vllm.models.minimax_m3.amd.ops.swiglu_oai import swiglu_oai_split + rows = [mxfp8_e4m3_quantize(x) for x in xs] + for form in ('fp32', 'bf16_chain'): + act = (lambda h: swiglu_oai_split(h, ALPHA, BETA, LIMIT)) if form == 'fp32' else ( + lambda h: swiglu_bf16_reference(h, ALPHA, BETA, LIMIT, form)) + + def baseline(i): + h = _mxfp8_dot_scaled_gemm(*rows[i], *row_major[i % COPIES], torch.bfloat16) + return G.mxfp8_linear(act(h), *down[i % COPIES]) + + def fused(i): + a = GUS.mxfp8_gateup_swiglu_bf16(*tiled[i], *gate_up[i % COPIES], m, ALPHA, BETA, LIMIT, form) + return G.mxfp8_linear(a, *down[i % COPIES]) + + def fused_only(i): + return GUS.mxfp8_gateup_swiglu_bf16(*tiled[i], *gate_up[i % COPIES], m, ALPHA, BETA, LIMIT, form) + + graphs = {'baseline': _capture(baseline), 'fused': _capture(fused), 'kernel': _capture(fused_only)} + t = {k: [] for k in graphs} + for _ in range(2): + for k, g in graphs.items(): + t[k].append(_time(g, args.rounds)) + t = {k: min(v) for k, v in t.items()} + print(f"rows={m} {form}: baseline FFN {t['baseline']:6.2f} us | fused FFN {t['fused']:6.2f} us | " + f"{t['baseline'] / t['fused']:.2f}x | fused kernel alone {t['kernel']:6.2f} us", flush=True) + + +if __name__ == '__main__': + main() diff --git a/benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py b/benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py new file mode 100644 index 0000000..9414621 --- /dev/null +++ b/benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py @@ -0,0 +1,208 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV GEMM (N 4992 x K 6144): the dispatched row versus vLLM's TP2 fallback (Triton ``tl.dot_scaled``, +``_M31_TILES[(4992, 6144)]``), both from pre-quantized activations, and the roofline; ``chain`` adds the TP2 +norm / RoPE / NVFP4 insert (32 / 2 / 2 heads): ``mxfp8_gemm_qkv_norm_rope_nvfp4_insert`` (deferred split-K partials +where its row has them) versus the Triton GEMM + the insert. + +MiniMax-M3.1-Flash checkpoint, layer 39, TP2 rank 0 slice (q 4096 | k 256 | v 256 | index-q 256 | index-k 128 rows); +14 weight copies per arm (> 400 MB, cold); up to 100 seeded inputs per graph; 3 x 5 alternated rounds of 10 +replays, median us per call. Above 2048 rows (outside vLLM's graphs) the eager calls are timed too. +The akr arm gets the codes in the layout ``prequant_code_tile_rows`` names (what the fused norm + quant producer +writes) and calls ``mxfp8_gemm_prequant``; the Triton arm gets row-major codes and scales. + + python3 benchmarks/gemm/mxfp8_n4992_k6144/benchmark_n4992_k6144.py --checkpoint /path/to/model +""" + +import argparse +import json +import statistics +from pathlib import Path + +import numpy as np +import torch +from safetensors import safe_open + +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common.reference import compare + +N, K = 4992, 6144 +COPIES = 14 +ROWS = (("q_proj", 4096), ("k_proj", 256), ("v_proj", 256), ("index_q_proj", 256), ("index_k_proj", 128)) + + +def insert_inputs(root, layer, m): + """Norm weights of the layer, cos / sin, and NVFP4 pages for ``m`` tokens (8 per request up to 2048 rows).""" + index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"] + prefix = f"language_model.model.layers.{layer}.self_attn." + norms = [] + for name in ("q_norm", "k_norm", "index_q_norm", "index_k_norm"): + with safe_open(str(root / index[prefix + name + ".weight"]), framework="pt") as f: + norms.append(f.get_tensor(prefix + name + ".weight").cuda().contiguous()) + rng = np.random.default_rng(7 * m + 1) + pos, slots, total = [], [], 0 + for length in [8] * (m // 8) + ([m % 8] if m % 8 else []) if m <= 2048 else [m]: + start = int(rng.integers(0, 131072 - length)) + p = np.arange(start, start + length) + pos.extend(p) + slots.extend((total + p // 128 - start // 128) * 128 + p % 128) + total += (start + length - 1) // 128 - start // 128 + 1 + s = torch.tensor(slots, device="cuda") + perm = torch.tensor(rng.permutation(total), device="cuda") + s = perm[s // 128] * 128 + s % 128 + inv = 1.0 / (1e7 ** (torch.arange(0, 64, 2, dtype=torch.float64) / 64)) + f = torch.arange(131072, dtype=torch.float64)[:, None] * inv + one = torch.tensor(1.0, device="cuda") + return (*norms, torch.cat([f.cos(), f.sin()], 1).to(torch.bfloat16).cuda().contiguous(), + torch.tensor(pos, device="cuda"), s, s, + torch.zeros(total, 4, 128, 72, dtype=torch.uint8, device="cuda"), + torch.zeros(total, 128, 72, dtype=torch.uint8, device="cuda"), one, one) + + +def weights(root, layer): + index = json.loads((root / "model.safetensors.index.json").read_text())["weight_map"] + prefix = f"language_model.model.layers.{layer}.self_attn." + + def read(name): + with safe_open(str(root / index[prefix + name]), framework="pt") as f: + return f.get_tensor(prefix + name) + + codes = torch.cat([read(n + ".weight")[:r] for n, r in ROWS]).view(torch.float8_e4m3fn).cuda() + scales = torch.cat([read(n + ".weight_scale")[:r] for n, r in ROWS]).view(torch.uint8).cuda() + assert codes.shape == (N, K) and scales.shape == (N, K // 32) + w, ws = g.preshuffle_mxfp8_weight(codes, scales) + shuffled = [(w, ws)] + [(w.clone(), ws.clone()) for _ in range(COPIES - 1)] + raw = [(codes, scales)] + [(codes.clone(), scales.clone()) for _ in range(COPIES - 1)] + return shuffled, raw + + +def measure(graphs, calls): + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for replay in range(10): + graphs[replay % len(graphs)].replay() + b.record() + b.synchronize() + return a.elapsed_time(b) * 1000 / (10 * calls) + + +def eager(call, count, phases): + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for phase in range(phases): + for j in range(count): + call(j, phase * count) + b.record() + b.synchronize() + return a.elapsed_time(b) * 1000 / (phases * count) + + +def roofline_us(m): + moved = (N + m) * K * (1 + 1 / 32) + 2 * m * N + return max(moved / 6.5e12, 2 * m * N * K / 5e15) * 1e6 + + +def main(): + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import _mxfp8_dot_scaled_gemm + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import mxfp8_e4m3_quantize + + from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( + mxfp8_gemm_qkv_norm_rope_nvfp4_insert, qkv_norm_rope_nvfp4_insert) + + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--checkpoint", required=True, type=Path) + ap.add_argument("--layer", type=int, default=39) + ap.add_argument("--ms", default="1,8,16,32,64,128,192,256,320,384,448,512,640,768,1024,1536,2048," + "3072,4096,8192,16384,32768") + ap.add_argument("--units", default="gemm,chain") + ap.add_argument("--out", type=Path) + args = ap.parse_args() + units = args.units.split(",") + heads = dict(num_heads=32, num_kv_heads=2, num_index_heads=2, eps=1e-6) + shuffled, raw = weights(args.checkpoint, args.layer) + rows = [] + for m in map(int, args.ms.split(",")): + count = max(2, min(100, (400 << 20) // (m * K * 3))) + phases = (COPIES + count - 1) // count + tile = g.prequant_code_tile_rows(m, N, K) + akr_in, tri_in = [], [] + for i in range(count): + x = torch.from_numpy(np.random.default_rng(1000 * m + i).standard_normal( + (m, K), dtype=np.float32)).cuda().to(torch.bfloat16) + akr_in.append(g.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else g.mxfp8_e4m3_quantize_aiter(x)) + tri_in.append(mxfp8_e4m3_quantize(x)) + del x + + def akr(j, off): + w, ws = shuffled[(j + off) % COPIES] + return g.mxfp8_gemm_prequant(*akr_in[j], w, ws, torch.bfloat16, m) + + def triton(j, off): + w, ws = raw[(j + off) % COPIES] + return _mxfp8_dot_scaled_gemm(*tri_in[j], w, ws, torch.bfloat16) + + metrics = compare(akr(0, 0), triton(0, 0).float()) + assert metrics["cos"] >= 0.99999, metrics + arms = {} + if "gemm" in units: + arms.update(akr=akr, triton=triton) + if "chain" in units: + ins = insert_inputs(args.checkpoint, args.layer, m) + + def akr_chain(j, off): + w, ws = shuffled[(j + off) % COPIES] + return mxfp8_gemm_qkv_norm_rope_nvfp4_insert(*akr_in[j], w, ws, m, torch.bfloat16, *ins, **heads, + one_launch=False) + + def triton_chain(j, off): + return qkv_norm_rope_nvfp4_insert(triton(j, off), *ins, **heads) + + q_akr, q_tri = akr_chain(0, 0)[0], triton_chain(0, 0)[0] + assert compare(q_akr, q_tri.float())["cos"] >= 0.9999 + arms.update(akr_chain=akr_chain, triton_chain=triton_chain) + graphs = {} + pool = torch.cuda.graph_pool_handle() + for name, call in arms.items(): + graphs[name] = [] + for phase in range(phases): + call(0, phase * count) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, pool=pool): + for j in range(count): + call(j, phase * count) + graphs[name].append(graph) + measure(graphs[name], count) + samples = {name: [] for name in arms} + eager_samples = {name: [] for name in arms} + for _ in range(3): + for _ in range(5): + for name in arms: + samples[name].append(measure(graphs[name], count)) + if m > 2048: + for name, call in arms.items(): + eager_samples[name].append(eager(call, count, max(2, phases))) + row = dict(m=m, tile=tile, config=str(g._select_prequant_config(m, N, K, torch.bfloat16)), + roofline_us=roofline_us(m), cos=metrics["cos"]) + for name in arms: + row[name] = statistics.median(samples[name]) + row[name + "_range"] = [min(samples[name]), max(samples[name])] + if eager_samples[name]: + row[name + "_eager"] = statistics.median(eager_samples[name]) + rows.append(row) + print(json.dumps(row), flush=True) + del graphs, akr_in, tri_in + torch.cuda.empty_cache() + for unit in units: + a, b = ("akr", "triton") if unit == "gemm" else ("akr_chain", "triton_chain") + print(f"| M | {unit} us | Triton {unit} us | speedup | GEMM roofline us | GEMM fraction |") + print("|---|---|---|---|---|---|") + for r in rows: + print(f"| {r['m']} | {r[a]:.2f} | {r[b]:.2f} | {r[b] / r[a]:.2f}x | {r['roofline_us']:.2f} | " + f"{100 * r['roofline_us'] / r[a]:.0f}% |") + if args.out: + args.out.write_text(json.dumps(rows, indent=1) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py b/benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py new file mode 100644 index 0000000..6ba2c53 --- /dev/null +++ b/benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The sweeps behind ``gemm/mxfp8_n4992_k6144/host.py`` (random MXFP8 weights, 14 cold copies > 400 MB, graph +replay; every candidate is checked against the reference first, failures are listed and skipped). + + --decode M,.. decode rows: row tile x column tile x K-waves x K-split (GEMM only, (16, 16) codes) + --chain M,.. the same rows + the TP2 norm / RoPE / NVFP4 insert (split-K rows: partials + reduce-insert) + --prefill M,.. persistent tiles 256/192/128 x 192/128 (row-major codes), per-round us + --split M,.. host.persistent_plan's row split against its single-launch tile (bitwise equal) + + python3 benchmarks/gemm/mxfp8_n4992_k6144/sweep_n4992_k6144.py --decode 16,64,256 --out sweep.json +""" + +import argparse +import json +import statistics + +import numpy as np +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common.reference import compare, mxfp8_gemm_reference, quantize_mxfp8 +from amd_kernels.gemm.mxfp8_n4992_k6144 import host as H + +N, K = H.N, H.K +LLC_BYTES = 420 << 20 +TILES = ((256, 192), (256, 128), (192, 192), (192, 128), (128, 192), (128, 128)) + + +def timeit_graph(calls, inner=2, replays=10, rounds=5): + for c in calls: + c() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for i in range(len(calls) * inner): + calls[i % len(calls)]() + events = [(torch.cuda.Event(True), torch.cuda.Event(True)) for _ in range(rounds)] + for _ in range(3): + graph.replay() + for start, end in events: + start.record() + for _ in range(replays): + graph.replay() + end.record() + torch.cuda.synchronize() + return statistics.median(s.elapsed_time(e) for s, e in events) * 1e3 / (replays * len(calls) * inner) + + +class Bench: + def __init__(self): + torch.manual_seed(5) + self.weights = [] + for _ in range(-(-LLC_BYTES // (N * K * 33 // 32))): + codes, scales = quantize_mxfp8(torch.randn(N, K, device="cuda", dtype=torch.bfloat16) * 0.5) + self.weights.append(g.preshuffle_mxfp8_weight(codes, scales)) + if len(self.weights) == 1: + self.raw = (codes, scales) + self.m = None + + def inputs(self, m): + if self.m != m: + torch.manual_seed(m) + self.x = torch.randn(m, K, device="cuda", dtype=torch.bfloat16) + self.ref = mxfp8_gemm_reference(*quantize_mxfp8(self.x), *self.raw) + self.m = m + return self.x + + def gemm(self, m, config, chain=None): + x = self.inputs(m) + out = torch.empty((m, N), device="cuda", dtype=torch.bfloat16) + if config.kernel == g._DECODE_KERNEL: + a, asc = g.mxfp8_e4m3_quantize_shuffled(x, max(16, config.tile_m)) + + defer = chain is not None and config.split_k > 1 + + def call(w, s): + part = g._launch_decode(config, a, asc, w, s, out, defer_reduce=defer) + if chain is not None: + chain(out, part) + else: + a, asc = g.mxfp8_e4m3_quantize_aiter(x) + launch = H.LAUNCHERS[config.kernel] + + def call(w, s): + launch(config, a, asc, w, s, out) + if chain is None: + call(*self.weights[0]) + torch.cuda.synchronize() + metrics = compare(out, self.ref) + if not (metrics["cos"] >= 0.99999 and metrics["rel_l2"] <= 1e-3): + raise AssertionError(f"mismatch {metrics}") + return timeit_graph([lambda w=w, s=s: call(w, s) for w, s in self.weights]) + + +def insert_chain(m): + """The TP2 insert consumer (heads 32 / 2 / 2) of ``m`` decode tokens (8 per request).""" + from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( + norm_rope_nvfp4_insert, splitk_reduce_norm_rope_nvfp4_insert) + + rng = np.random.default_rng(m) + pos, slots, total = [], [], 0 + for length in [8] * (m // 8) + ([m % 8] if m % 8 else []): + start = int(rng.integers(0, 131072 - length)) + p = np.arange(start, start + length) + pos.extend(p) + slots.extend((total + p // 128 - start // 128) * 128 + p % 128) + total += (start + length - 1) // 128 - start // 128 + 1 + s = torch.tensor(slots, device="cuda") + perm = torch.tensor(rng.permutation(total), device="cuda") + s = perm[s // 128] * 128 + s % 128 + f = torch.arange(131072, dtype=torch.float64)[:, None] / (1e7 ** (torch.arange(0, 64, 2, dtype=torch.float64) / 64)) + one = torch.tensor(1.0, device="cuda") + ins = ([(torch.rand(128, device="cuda") + 0.5).bfloat16() for _ in range(4)] + + [torch.cat([f.cos(), f.sin()], 1).to(torch.bfloat16).cuda().contiguous(), + torch.tensor(pos, device="cuda"), s, s, + torch.zeros(total, 4, 128, 72, dtype=torch.uint8, device="cuda"), + torch.zeros(total, 128, 72, dtype=torch.uint8, device="cuda"), one, one, + torch.empty(m, 4096, dtype=torch.bfloat16, device="cuda"), + torch.empty(m, 256, dtype=torch.float8_e4m3fn, device="cuda")]) + kw = dict(num_heads=32, num_kv_heads=2, num_index_heads=2, eps=1e-6) + + def chain(out, part): + if isinstance(part, g.SkPartials): + splitk_reduce_norm_rope_nvfp4_insert(part.tmp, part.split_k, *ins, **kw) + else: + norm_rope_nvfp4_insert(out, *ins, **kw) + return chain + + +def decode_candidates(m, nis=(2, 3, 4, 6, 8), kws=(1, 2, 3, 4), kss=(1, 2, 3), workgroups=(150, 1400)): + for n_mb in range(1, 17): + rt = -(-m // (16 * n_mb)) + if rt > 16 or (n_mb > 1 and -(-m // (16 * rt)) != n_mb): + continue + for ni in nis: + for kw in kws: + for ks in kss: + wgs = ks * n_mb * (N // 16 // ni) + if 48 % (ks * kw) or (48 // (ks * kw)) % 2 or not workgroups[0] <= wgs <= workgroups[1]: + continue + if rt * ni * 48 // (ks * kw) > 640: # unrolled MFMAs per wave: larger ones compile for minutes + continue + yield g._Config(16 * rt, 16 * ni, 128, 0, 0, ks, g._DECODE_KERNEL, kw, 0) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--decode", default="") + ap.add_argument("--chain", default="") + ap.add_argument("--prefill", default="") + ap.add_argument("--split", default="") + ap.add_argument("--ni", default="2,3,4,6,8", help="16-column tiles per workgroup") + ap.add_argument("--kw", default="1,2,3,4", help="K-waves") + ap.add_argument("--ks", default="1,2,3", help="K splits") + ap.add_argument("--workgroups", default="150,1400") + ap.add_argument("--top", type=int, default=6) + ap.add_argument("--out") + args = ap.parse_args() + bench, result = Bench(), {} + jobs = [("decode", m) for m in args.decode.split(",") if m] + [("chain", m) for m in args.chain.split(",") if m] + jobs += [("prefill", m) for m in args.prefill.split(",") if m] + for m in (int(v) for v in args.split.split(",") if v): + plan = H.persistent_plan(m) + if len(plan) == 1: + print(f"split M {m}: one launch {plan[0][1]}", flush=True) + continue + single = bench.gemm(m, H._persistent(*H.persistent_tile(m))) + split = bench.gemm(m, H._persistent(*plan[0][1], H._PERSISTENT_SPLIT_KERNEL)) + print(f"split M {m}: {plan} {split:.1f} us vs one launch {H.persistent_tile(m)} {single:.1f} us " + f"({100 * (split / single - 1):+.1f}%)", flush=True) + result[f"split:{m}"] = [split, single] + for kind, m in jobs: + m = int(m) + if kind == "prefill": + configs = [H._persistent(*t) for t in TILES] + else: + ints = lambda v: tuple(int(x) for x in v.split(",")) # noqa: E731 + configs = list(decode_candidates(m, ints(args.ni), ints(args.kw), ints(args.ks), ints(args.workgroups))) + chain = insert_chain(m) if kind == "chain" else None + rows = [] + for config in configs: + try: + if chain is not None: + bench.gemm(m, config) # the reference check + us = bench.gemm(m, config, chain) + except Exception as e: # noqa: BLE001 + print(f" fail {config}: {str(e)[:120]}", flush=True) + continue + rounds = H._rounds(m, (config.tile_m, config.tile_n)) if kind == "prefill" else 0 + rows.append((us, config.tile_m, config.tile_n, config.waves_m, config.split_k, rounds)) + rows.sort() + print(f"{kind} M {m}", flush=True) + for us, tm, tn, kw, ks, rounds in rows[: args.top]: + extra = f"{rounds} rounds, {us / rounds:.1f} us/round" if rounds else f"kw{kw} ks{ks}" + print(f" {us:8.2f} us {tm}x{tn} {extra}", flush=True) + result[f"{kind}:{m}"] = rows + if args.out: + with open(args.out, "w") as f: + json.dump(result, f) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py b/benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py new file mode 100644 index 0000000..6799020 --- /dev/null +++ b/benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py @@ -0,0 +1,128 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 o_proj N6144 K4096: the public entries versus vLLM's Triton ``tl.dot_scaled`` linear (the TP2 fallback). + +Layer 39 o_proj, TP2 rank 0 (weight[:, :4096], scale[:, :128]), 16 rotating weight copies (> 400 MB), seeded +inputs, 15 alternated rounds of 10 graph replays, median us per call. ``--form codes``: the producer's MXFP8 +codes (ours: ``mxfp8_gemm_prequant`` in ``prequant_code_tile_rows``' layout; Triton: ``_mxfp8_dot_scaled_gemm`` on +row-major codes). ``--form bf16``: a bf16 activation (``mxfp8_linear`` / ``_mxfp8_dot_scaled_linear``, both +quantize). Roofline: max(bytes / 6.5 TB/s, flops / 5 PF). + + python3 benchmarks/gemm/mxfp8_n6144_k4096/benchmark_n6144_k4096.py --checkpoint /path/to/model --form codes +""" +import argparse +import json +import statistics +from pathlib import Path + +import numpy as np +import torch +from safetensors import safe_open +from amd_kernels.gemm.mxfp8_common import gemm as g, reference as ref + +N, K = 6144, 4096 +COPIES = 16 +DECODE_MS = "1,8,16,32,48,64,80,96,128,160,192,256,320,384,448,512" +PREFILL_MS = "640,768,1024,1280,1536,2048,2560,3072,4096,6144,8192,16384,32768" + + +def roofline_us(m): + nbytes = N * K * 33 / 32 + m * K * 33 / 32 + 2 * m * N + return max(nbytes / 6.5e6, 2 * m * N * K / 5e9) + + +def load_weights(checkpoint, layer): + index = json.loads((checkpoint / "model.safetensors.index.json").read_text())["weight_map"] + prefix = f"language_model.model.layers.{layer}.self_attn.o_proj." + raw = [] + for field, cols in (("weight", K), ("weight_scale", K // 32)): + with safe_open(str(checkpoint / index[prefix + field]), framework="pt") as f: + raw.append(f.get_tensor(prefix + field)[:, :cols].contiguous().cuda()) + codes, scales = raw[0].view(torch.float8_e4m3fn), raw[1].view(torch.uint8) + w, ws = g.preshuffle_mxfp8_weight(codes, scales) + ours = [(w, ws)] + [(w.clone(), ws.clone()) for _ in range(COPIES - 1)] + plain = [(codes, scales)] + [(codes.clone(), scales.clone()) for _ in range(COPIES - 1)] + return ours, plain + + +def capture(call, count): + for j in range(count): + call(j) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for j in range(count): + call(j) + graph.replay() + torch.cuda.synchronize() + return graph + + +def measure(graph, count): + start, stop = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(10): + graph.replay() + stop.record() + stop.synchronize() + return start.elapsed_time(stop) * 1000 / (10 * count) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--checkpoint", type=Path, required=True) + ap.add_argument("--layer", type=int, default=39) + ap.add_argument("--form", choices=("codes", "bf16"), default="codes") + ap.add_argument("--ms", default=None, help=f"default: codes {DECODE_MS}; bf16 {DECODE_MS},{PREFILL_MS}") + ap.add_argument("--out", type=Path) + args = ap.parse_args() + from vllm.model_executor.kernels.linear.mxfp8 import rocm_native as rn + from vllm.model_executor.layers.quantization.utils.mxfp8_utils import mxfp8_e4m3_quantize + + torch.set_num_threads(1) + ours_w, plain_w = load_weights(args.checkpoint, args.layer) + ms = args.ms or (DECODE_MS if args.form == "codes" else DECODE_MS + "," + PREFILL_MS) + results = {} + for m in map(int, ms.split(",")): + count = 32 if m <= 4096 else 16 + xs = [torch.from_numpy(np.random.default_rng(1000 * m + i).standard_normal((m, K), dtype=np.float32)) + .cuda().bfloat16() for i in range(count)] + if args.form == "codes": + tile = g.prequant_code_tile_rows(m, N, K) + prod = [g.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else g.mxfp8_e4m3_quantize_aiter(x) for x in xs] + plain = [mxfp8_e4m3_quantize(x) for x in xs] + arms = { + "ours": lambda j: g.mxfp8_gemm_prequant(*prod[j], *ours_w[j % COPIES], torch.bfloat16, m), + "triton": lambda j: rn._mxfp8_dot_scaled_gemm(*plain[j], *plain_w[j % COPIES], torch.bfloat16), + } + else: + arms = { + "ours": lambda j: g.mxfp8_linear(xs[j], *ours_w[j % COPIES]), + "triton": lambda j: rn._mxfp8_dot_scaled_linear(xs[j], *plain_w[j % COPIES]), + } + expected = ref.mxfp8_linear_reference(xs[0], *plain_w[0]) + for name, call in arms.items(): + metrics = ref.compare(call(0), expected) + assert metrics["cos"] >= ref.COS_MIN and metrics["rel_l2"] <= ref.REL_L2_MAX, (m, name, metrics) + graphs = {name: capture(call, count) for name, call in arms.items()} + rounds = {name: [] for name in arms} + for r in range(15): + for name in (list(arms) if r % 2 == 0 else list(arms)[::-1]): + rounds[name].append(measure(graphs[name], count)) + med = {name: statistics.median(v) for name, v in rounds.items()} + ideal = roofline_us(m) + config = g._select_prequant_config(m, N, K, torch.bfloat16) if args.form == "codes" else \ + g._select_config(m, N, K, torch.bfloat16) + results[m] = dict(us=med["ours"], triton_us=med["triton"], ideal_us=ideal, + fraction=ideal / med["ours"], config=repr(config), rounds=rounds) + print(f"M {m:6d} ours {med['ours']:8.2f} us triton {med['triton']:8.2f} us " + f"x{med['triton'] / med['ours']:.2f} roofline {ideal:7.2f} us ({100 * ideal / med['ours']:4.1f}%) " + f"{config.kernel} {config.tile_m}x{config.tile_n}", flush=True) + if args.out: + args.out.write_text(json.dumps(results, indent=2) + "\n") + del graphs, xs + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py b/benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py new file mode 100644 index 0000000..a1eba8c --- /dev/null +++ b/benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The row sweep behind ``gemm/mxfp8_n6144_k4096/host.py``: every candidate config of a family at each M, graph replay. + +16 rotating random-weight copies (415 MB > MALL), 4 seeded inputs, a graph of 32 calls, alternated rounds, median us +per call; every candidate is checked against the reference first. ``decode``: the decode kernel on (16, 16) codes +(tiles 16..160 x 32..96, K-waves 1/2/4, owner-read on the 96-column two-K-wave tiles); ``decode_fq``: the bf16 +activation quantized in the decode kernel; ``prefill``: row-major codes from a bf16 activation (quantizer timed) on +the LDS-staged, 4-wave, AITER and this folder's 256x192 / 128x192 tiles. ``--jobs`` compile-only processes build the +variants first (FlyDSL disk cache). + + python3 benchmarks/gemm/mxfp8_n6144_k4096/sweep_rows.py --family decode --ms 256,320,384 +""" +import argparse +import os +import statistics +import subprocess +import sys + +if os.environ.get("AKR_COMPILE_ONLY") == "1": + import tests # noqa: F401 (compile-only mode) +import torch +from amd_kernels.gemm.mxfp8_common import gemm as g, reference as ref +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import compile_mxfp8_gemm_decode +from amd_kernels.gemm.mxfp8_n6144_k4096 import host as H + +N, K = 6144, 4096 +C = g._Config +COPIES = 16 + + +def _decode_candidates(m): + out = [] + for tm in sorted({t for t in (16, 32, 48, 64, 80, 96, 112, 128, 144, 160) if -(-m // t) <= 8 and t <= 16 * -(-m // 16)}): + for ni in (2, 3, 4, 6): + for kw in (1, 2, 4): + out.append(("decode", C(tm, 16 * ni, 128, 0, 0, 1, g._DECODE_KERNEL, kw, 0), False)) + if ni == 6 and kw == 2: + out.append(("decode", C(tm, 96, 128, 0, 0, 1, g._DECODE_KERNEL, 2, 1), True)) + return out + + +def _decode_fq_candidates(m): + tiles = [t for t in (4, 8, 16, 32) if t >= min(m, 16) or t == 16 * -(-m // 16)] + return [("decode_fq", C(tm, 16 * ni, 128, 0, 0, 1, g._DECODE_FQ_KERNEL, kw, 0), False) + for tm in tiles for ni in (2, 3, 4) for kw in (1, 2, 4, 8) if tm >= 16 or (32 // kw) % (16 // tm) == 0] + + +def _prefill_candidates(m): + tile = lambda tm, tn, kernel, sk=1: C(tm, tn, 128, 1, 1, sk, kernel, 2, 2) # noqa: E731 + out = [tile(256, 256, g._FOURWAVE_NT_KERNEL), tile(256, 256, g._FOURWAVE_NT_KERNEL, 0), + tile(256, 128, g._FOURWAVE_NT_KERNEL), tile(128, 256, g._FOURWAVE_NT_KERNEL), + tile(256, 192, H._PERSISTENT_KERNEL), C(128, 256, 128, 2, 4, 1), C(128, 128, 128, 2, 4, 1)] + if m <= 2048: + out.append(tile(256, 192, H._ONE_TILE_KERNEL)) + if m <= 1280: + out += [tile(128, 192, H._TILE_128_KERNEL), C(128, 128, 128, 2, 4, 1, g._STAGED_KERNEL, 2, 4), + C(64, 128, 128, 2, 4, 1, g._STAGED_KERNEL, 2, 2)] + return [("prefill", c, False) for c in out] + + +FAMILIES = {"decode": _decode_candidates, "decode_fq": _decode_fq_candidates, "prefill": _prefill_candidates} + + +def _call(family, config, owner_read, xs, copies, m): + if family == "decode_fq" or family == "prefill": + return lambda j: g.mxfp8_linear(xs[j % 4], *copies[j % COPIES], config=config) + qs = [g.mxfp8_e4m3_quantize_shuffled(x, config.tile_m) for x in xs] + if not owner_read: + return lambda j: g.mxfp8_gemm_shuffled(*qs[j % 4], *copies[j % COPIES], torch.bfloat16, m, config=config) + from amd_kernels.common.tensor_shim import _run_compiled + exe = compile_mxfp8_gemm_decode(N=N, K=K, RT=config.tile_m // 16, NI=6, KW=2, OWNER_READ=True) + u8 = lambda t: t.view(torch.uint8).view(-1) # noqa: E731 + n_mb = -(-m // config.tile_m) + + def run(j): + (a, sa), (w, sw) = qs[j % 4], copies[j % COPIES] + out = torch.empty((m, N), dtype=torch.bfloat16, device="cuda") + _run_compiled(exe, u8(a), u8(w), u8(out), u8(sa), u8(sw), m, sa.numel(), n_mb, n_mb * (N // 96), + torch.cuda.current_stream()) + return out + return run + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--family", choices=FAMILIES, required=True) + ap.add_argument("--ms", required=True) + ap.add_argument("--rounds", type=int, default=9) + ap.add_argument("--jobs", type=int, default=8) + ap.add_argument("--shard", type=int, default=-1) + args = ap.parse_args() + compile_only = args.shard >= 0 + if not compile_only and args.jobs > 1: + env = dict(os.environ, AKR_COMPILE_ONLY="1") + for p in [subprocess.Popen([sys.executable] + sys.argv + ["--shard", str(i)], env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) for i in range(args.jobs)]: + p.wait() + torch.manual_seed(0) + raw = ref.quantize_mxfp8(torch.randn(N, K, device="cuda", dtype=torch.bfloat16) * 0.05) + copies = [g.preshuffle_mxfp8_weight(raw[0].clone(), raw[1].clone()) for _ in range(1 if compile_only else COPIES)] + copies = copies * (COPIES // len(copies)) + for m in map(int, args.ms.split(",")): + xs = [torch.randn(m, K, device="cuda", dtype=torch.bfloat16) for _ in range(4)] + expected = ref.mxfp8_linear_reference(xs[0], *raw) + calls = {} + for i, (family, config, owner_read) in enumerate(FAMILIES[args.family](m)): + if compile_only and i % args.jobs != args.shard: + continue + label = f"{config.kernel} {config.tile_m}x{config.tile_n} kw{config.waves_m} sk{config.split_k}" + ( + " sb owner-read" if owner_read else "") + try: + call = _call(args.family, config, owner_read, xs, copies, m) + got = call(0) + torch.cuda.synchronize() + except Exception as e: # an illegal combination (registers, LDS) + print(f"M {m} {label}: {type(e).__name__}", flush=True) + continue + if compile_only: + continue + metrics = ref.compare(got, expected) + if metrics["cos"] >= ref.COS_MIN and metrics["rel_l2"] <= ref.REL_L2_MAX: + calls[label] = call + if compile_only: + continue + graphs = {} + for label, call in calls.items(): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for j in range(2 * COPIES): + call(j) + graph.replay() + graphs[label] = graph + torch.cuda.synchronize() + times = {label: [] for label in graphs} + for r in range(args.rounds): + for label in (list(graphs) if r % 2 == 0 else list(graphs)[::-1]): + start, stop = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(10): + graphs[label].replay() + stop.record() + stop.synchronize() + times[label].append(start.elapsed_time(stop) * 1000 / (10 * 2 * COPIES)) + for label, us in sorted(((k, statistics.median(v)) for k, v in times.items()), key=lambda t: t[1])[:6]: + print(f"M {m:6d} {us:9.2f} us {label}", flush=True) + del graphs + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py new file mode 100644 index 0000000..076d109 --- /dev/null +++ b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One-GPU graph-replay timing of the decode index top-16 at two index heads (TP2: 2 index-q heads per GPU over the +one replicated index-K head), this checkout against frozen copies of the package (``--ref NAME=PATH``). + +Routes as the TP2 decode block runs them (``attention/.../index_decode.py``): ``chain`` = ``nvfp4_index_score_top16`` +(more than 256 rows), ``fused`` = ``nvfp4_index_score_rows`` then ``nvfp4_index_top16_from_scores`` (what the attention +runs at two KV heads, ``host._select_ids``). Every call has its own index query and block table into its own pages of +a pool above the 256 MB MALL; contexts per request count: ``--lens`` JSON {"N": [...]} or the (i + 0.5) / N quantiles +of a log-uniform 32K-192K distribution, plus uniform 32K / 128K. max_seq_len = 1M (what decode graphs capture). The +ids of the first and last input are checked against ``reference.topk_reference`` and between the sides. + + python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_index_decode_h2.py \ + --ref base=/path/to/frozen/indexer_nvfp4_d128_p128_top16 --ql 8 --reqs 1,2,4,8,16,32,64 +""" + +import argparse +import importlib +import json +import statistics + +import torch + +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_decode_topk import load_side, synthetic_lens +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_indexer import ( + PAGE_BYTES, + num_copies, + page_pool, + pool_pages_for, + tables, +) +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import BLK, cdiv, score_width, topk_reference + +MSL = 1 << 20 +INIT, LOCAL = 0, 1 + + +def capture(calls): + """One graph of ``calls`` (each run once eagerly first), warmed by three replays.""" + for c in calls: + c() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for c in calls: + c() + torch.cuda.synchronize() + for _ in range(3): + g.replay() + return g + + +def time_graph(g, n_calls, replays): + """Mean time per call (us) over ``replays`` replays of ``g``.""" + st, en = torch.cuda.Event(True), torch.cuda.Event(True) + st.record() + for _ in range(replays): + g.replay() + en.record() + torch.cuda.synchronize() + return st.elapsed_time(en) * 1e3 / (replays * n_calls) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--ref", default="", help="name=path,... frozen copies of the package") + ap.add_argument("--heads", type=int, default=2) + ap.add_argument("--ql", type=int, default=8) + ap.add_argument("--reqs", default="1,2,3,4,6,8,12,16,24,32,48,64") + ap.add_argument("--lens", default=None, help='JSON {"N": [context per request]}') + ap.add_argument("--kinds", default="mix,32K,128K") + ap.add_argument("--routes", default="fused,chain") + ap.add_argument("--rounds", type=int, default=6) + ap.add_argument("--replays", type=int, default=10) + args = ap.parse_args() + from amd_kernels import warm_up + + warm_up.set_auto(False) + sides = {"ours": load_side("ours", None)} + importlib.import_module(sides["ours"].__name__ + ".score_decode") + for x in filter(None, args.ref.split(",")): + name, path = x.split("=", 1) + registry = dict(warm_up.REGISTRY) + warm_up.REGISTRY.clear() # a frozen copy registers the same entry names + sides[name] = load_side("idx_" + name, path) + importlib.import_module(sides[name].__name__ + ".score_decode") + warm_up.REGISTRY.clear() + warm_up.REGISTRY.update(registry) + mods = {k: importlib.import_module(v.__name__ + ".score_decode") for k, v in sides.items()} + table = json.load(open(args.lens)) if args.lens else {} + heads, ql = args.heads, args.ql + print(f"| route | heads x rows | requests | contexts | " + " | ".join(f"{k} us" for k in sides) + + " | " + " | ".join(f"{k}/ours" for k in sides if k != "ours") + " | page GB/s (ours) |", flush=True) + for n in [int(x) for x in args.reqs.split(",")]: + for kind in args.kinds.split(","): + if kind == "mix": + lens = list(table[str(n)]) if str(n) in table else synthetic_lens(n) + else: + lens = [int(kind[:-1]) * 1024] * n + rows = n * ql + page_bytes = sum(cdiv(s, BLK) for s in lens) * PAGE_BYTES + copies = num_copies(page_bytes) + pool = pool_pages_for(lens) + cache = page_pool(pool) + tabs = tables(lens, copies, pool) + gen = torch.Generator(device="cuda").manual_seed(n) + qs = [(torch.randn(rows, heads, 128, device="cuda", generator=gen) * 0.5).to(torch.float8_e4m3fn) + for _ in range(copies)] + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + kv = (seq.long().repeat_interleave(ql) - ql + torch.arange(rows, device="cuda") % ql + 1).clamp(min=0) + nblk = (kv + BLK - 1) // BLK + for route in args.routes.split(","): + outs = {k: torch.full((heads, rows, 16), -7, dtype=torch.int32, device="cuda") for k in sides} + scores = {k: torch.empty(heads, rows, score_width(MSL), device="cuda") for k in sides} + + def call(k, i): + m = mods[k] + if route == "chain": + m.nvfp4_index_score_top16(qs[i], cache, tabs[i], seq, INIT, LOCAL, outs[k], query_len=ql, + max_seq_len=MSL, score=scores[k]) + else: + m.nvfp4_index_score_rows(qs[i], cache, scores[k], tabs[i], seq, INIT, LOCAL, query_len=ql, + max_seq_len=MSL) + m.nvfp4_index_top16_from_scores(scores[k], seq, outs[k], rows=rows, query_len=ql, + max_seq_len=MSL) + + for i in (0, copies - 1): + for k in sides: + call(k, i) + torch.cuda.synchronize() + want = topk_reference(scores["ours"], nblk, MSL) + for k in sides: + assert torch.equal(outs[k], want), (k, route, n, kind) + graphs = {k: capture([lambda k=k, i=i: call(k, i) for i in range(copies)]) for k in sides} + samples = {k: [] for k in sides} + names = list(sides) + for r in range(args.rounds): + for k in (names if r % 2 == 0 else names[::-1]): + samples[k].append(time_graph(graphs[k], copies, args.replays)) + med = {k: statistics.median(v) for k, v in samples.items()} + gbs = page_bytes / (med["ours"] * 1e-6) / 1e9 + print(f"| {route} | h{heads} x {ql} | {n} | {kind} | " + " | ".join(f"{med[k]:.2f}" for k in sides) + + " | " + " | ".join(f"{med[k] / med['ours']:.3f}" for k in sides if k != "ours") + + f" | {gbs:.0f} |", flush=True) + del graphs + del cache, tabs, qs + torch.cuda.empty_cache() + + +if __name__ == "__main__": + with torch.inference_mode(): + main() diff --git a/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py index 6b94566..b6a9467 100644 --- a/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py +++ b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/benchmark_indexer.py @@ -237,6 +237,7 @@ def topk_call(): def main(): ap = argparse.ArgumentParser() ap.add_argument("--prefill", action="store_true") + ap.add_argument("--heads", type=int, default=1, help="index heads per GPU (TP4 1, TP2 2)") args = ap.parse_args() dev = torch.cuda.get_device_properties(0) print( @@ -254,12 +255,12 @@ def main(): ([2048], [1 << 20], "2K / 1M"), ([3072, 3072, 3072], [203072, 3072, 207872], "3 x 3K / 200K, 3K, 208K"), ]: - bench_prefill(1, q_lens, seq_lens, label) + bench_prefill(args.heads, q_lens, seq_lens, label) return print("| Heads x rows | Requests | Lengths | Scorer us | Page GB/s | Top-k us | Chain us |") print("|---|---:|---|---:|---:|---:|---:|") rng = torch.Generator().manual_seed(0) - for heads, ql, counts in ((1, 4, (16, 64, 128, 256)),): + for heads, ql, counts in ((args.heads, 4, (16, 64, 128, 256)),): for n in counts: for label, lens in ( ("32K", [32768] * n), diff --git a/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_decode_h2.py b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_decode_h2.py new file mode 100644 index 0000000..7ac9c1e --- /dev/null +++ b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_decode_h2.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-head decode dispatch's crossovers (``score_decode.host``): every scorer pinned (``PIN_SCORER``; the run +scorer at ``runs:GRID:UNITS``), then both selectors (``PIN_PLAIN``) on the scorer's rows, timed as +``benchmark_index_decode_h2.py`` times (graph replay, a page pool above the MALL, max_seq_len 1M). The scores of the +first input are checked against ``reference.decode_scores_reference``, the ids against ``topk_reference``. + + python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_decode_h2.py --ql 8 --reqs 8,12,13,16,32 \ + --scorers first_page_xcd,runs:512:256,chunked +""" + +import argparse +import statistics + +import torch + +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import ( + BLK, + SCORE_ATOL, + SCORE_RTOL, + cdiv, + decode_scores_reference, + score_width, + topk_reference, +) +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.score_decode import host +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_decode_topk import synthetic_lens +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_index_decode_h2 import capture, time_graph +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_indexer import ( + PAGE_BYTES, + num_copies, + page_pool, + pool_pages_for, + tables, +) + +MSL = 1 << 20 + + +def pin(scorer): + kind, *grid = scorer.split(":") + host.PIN_SCORER = kind + host.PIN_RUNS_GRID, host.PIN_RUNS_UNITS = (int(x) for x in grid) if grid else (None, None) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--heads", type=int, default=2) + ap.add_argument("--ql", type=int, default=8) + ap.add_argument("--reqs", default="1,2,4,8,12,13,16,24,32,48,64") + ap.add_argument("--kinds", default="mix,32K,128K", help="mix: log-uniform 32K-192K quantiles; NK: N K tokens") + ap.add_argument("--scorers", default="page,first_page_xcd,runs:512:256,chunked") + ap.add_argument("--rounds", type=int, default=4) + args = ap.parse_args() + from amd_kernels import warm_up + + warm_up.set_auto(False) + heads, ql, scorers = args.heads, args.ql, args.scorers.split(",") + print("requests | contexts | " + " | ".join(scorers) + " | plain | one-wave | fastest scorer", flush=True) + for n in (int(x) for x in args.reqs.split(",")): + for kind in args.kinds.split(","): + lens = synthetic_lens(n) if kind == "mix" else [int(kind[:-1]) * 1024] * n + rows = n * ql + copies = num_copies(sum(cdiv(s, BLK) for s in lens) * PAGE_BYTES) + pool = pool_pages_for(lens) + cache = page_pool(pool) + tabs = tables(lens, copies, pool) + qs = [(torch.randn(rows, heads, 128, device="cuda") * 0.5).to(torch.float8_e4m3fn) for _ in range(copies)] + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + kv = (seq.long().repeat_interleave(ql) - ql + torch.arange(rows, device="cuda") % ql + 1).clamp(min=0) + nblk = (kv + BLK - 1) // BLK + score = torch.empty(heads, rows, score_width(MSL), device="cuda") + us = {} + for sc in scorers: + pin(sc) + try: + host._score_rows(qs[0], cache, score, tabs[0], seq, 0, 1, ql, MSL) + except AssertionError: # a scorer past its page-table range + us[sc] = float("nan") + continue + want = decode_scores_reference(qs[0], cache, tabs[0], lens, 0, 1, ql) + for r, s in enumerate(lens): + got = score[:, r * ql:(r + 1) * ql, :cdiv(s, BLK)].double() + torch.testing.assert_close(got, want[r], rtol=SCORE_RTOL, atol=SCORE_ATOL) + g = capture([lambda i=i: host._score_rows(qs[i], cache, score, tabs[i], seq, 0, 1, ql, MSL) + for i in range(copies)]) + us[sc] = statistics.median(time_graph(g, copies, 10) for _ in range(args.rounds)) + del g + pin("chunked") + host.PIN_SCORER = None + bufs = [torch.empty_like(score) for _ in range(16)] + for j, b in enumerate(bufs): + host._score_rows(qs[j], cache, b, tabs[j], seq, 0, 1, ql, MSL) + ids = torch.empty(heads, rows, 16, dtype=torch.int32, device="cuda") + for plain in (True, False): + host.PIN_PLAIN = plain + host.nvfp4_index_top16_from_scores(bufs[0], seq, ids, rows=rows, query_len=ql, max_seq_len=MSL) + assert torch.equal(ids, topk_reference(bufs[0], nblk, MSL)) + g = capture([lambda b=b: host.nvfp4_index_top16_from_scores(b, seq, ids, rows=rows, query_len=ql, + max_seq_len=MSL) for b in bufs] * 4) + us[plain] = statistics.median(time_graph(g, 64, 10) for _ in range(args.rounds)) + del g + host.PIN_PLAIN = None + best = min(scorers, key=lambda s: us[s] if us[s] == us[s] else float("inf")) + print(f"{n} | {kind} | " + " | ".join(f"{us[s]:.2f}" for s in scorers) + + f" | {us[True]:.2f} | {us[False]:.2f} | {best}", flush=True) + del cache, tabs, qs, bufs + torch.cuda.empty_cache() + + +if __name__ == "__main__": + with torch.inference_mode(): + main() diff --git a/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_prefill_plan.py b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_prefill_plan.py new file mode 100644 index 0000000..ad131f7 --- /dev/null +++ b/benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_prefill_plan.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The prefill scorer plan's crossovers (``score_prefill.host._score_plan``) at ``--heads``: the dispatch's choice and +the pinned kernel (``PIN_SHARED``: shared-dequant or the 512-row ``kernel.py``), tile rows (``PIN_MQ``) and workgroup +target (``PIN_TARGET``) of one request of M new rows after a prefix, scorer alone (graph replay, a page pool above the +MALL). Every pin's visible scores equal the first's bit for bit (asserted). + + python3 benchmarks/indexer/indexer_nvfp4_d128_p128_top16/sweep_prefill_plan.py --heads 2 --cases 2048:0,1024:32768 +""" + +import argparse +import statistics + +import torch + +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import score_width +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.score_prefill import host +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_index_decode_h2 import capture, time_graph +from benchmarks.indexer.indexer_nvfp4_d128_p128_top16.benchmark_indexer import page_pool, pool_pages_for, tables + +# name: (PIN_SHARED, PIN_MQ, PIN_TARGET) +PLANS = { + "auto": (None, None, None), + "shared_mq1": (True, 1, None), + "shared_mq2": (True, 2, None), + "tq512": (False, None, None), + "tq512_t1024": (False, None, 1024), + "tq512_t4096": (False, None, 4096), +} + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--heads", type=int, default=2) + ap.add_argument("--cases", default="1024:0,2048:0,4096:0,8192:0,16384:0,1024:32768,2048:32768,4096:32768," + "8192:65536,16384:131072,4096:262144", help="M:prefix,...") + ap.add_argument("--plans", default=",".join(PLANS)) + ap.add_argument("--rounds", type=int, default=3) + args = ap.parse_args() + from amd_kernels import warm_up + + warm_up.set_auto(False) + heads, names = args.heads, args.plans.split(",") + print("M | prefix | " + " | ".join(f"{p} us" for p in names) + " | fastest", flush=True) + for case in args.cases.split(","): + m, prefix = (int(x) for x in case.split(":")) + msl = m + prefix + copies = min(24, max(4, (4 << 30) // (m * heads * 128))) + pool = pool_pages_for([msl]) + cache = page_pool(pool) + tabs = tables([msl], copies, pool) + qs = [(torch.randn(m, heads, 128, device="cuda") * 0.5).to(torch.float8_e4m3fn) for _ in range(copies)] + seq = torch.tensor([msl], dtype=torch.int32, device="cuda") + cu = torch.tensor([0, m], dtype=torch.int32, device="cuda") + score = torch.empty(heads, m, score_width(msl), device="cuda") + nb = (msl + 127) // 128 + live = torch.arange(nb, device="cuda")[None, :] <= (torch.arange(prefix, msl, device="cuda") // 128)[:, None] + us, first = {}, None + for p in names: + host.PIN_SHARED, host.PIN_MQ, host.PIN_TARGET = PLANS[p] + host.nvfp4_index_score_prefill(qs[0], cache, score, tabs[0], cu, seq, 0, 1, m, msl) + got = score[:, :, :nb][:, live].view(torch.int32).clone() + first = got if first is None else first + assert torch.equal(got, first), p + g = capture([lambda i=i: host.nvfp4_index_score_prefill(qs[i], cache, score, tabs[i], cu, seq, 0, 1, m, + msl) for i in range(copies)]) + us[p] = statistics.median(time_graph(g, copies, 3) for _ in range(args.rounds)) + del g + host.PIN_SHARED = host.PIN_MQ = host.PIN_TARGET = None + print(f"{m} | {prefix} | " + " | ".join(f"{us[p]:.1f}" for p in names) + f" | {min(names, key=us.get)}", + flush=True) + del cache, tabs, qs, score + torch.cuda.empty_cache() + + +if __name__ == "__main__": + with torch.inference_mode(): + main() diff --git a/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_baseline_moe.py b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_baseline_moe.py new file mode 100644 index 0000000..fbb11d8 --- /dev/null +++ b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_baseline_moe.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MoE block vLLM runs at TP2 without this entry, timed like ``benchmark_shared_moe.py`` on the same synthetic +weights: ``GateLinear``'s ROCm router GEMM (FP32 kernel to 32 rows, BF16x3 from 2048), ``fused_topk_bias`` (AITER biased top-k), AITER +``fused_moe`` W4A4 (MXFP4, SwiGLU-OAI, separated gate / up, shuffled weights and e8m0), the shared expert on vLLM's +side stream (Triton ``dot_scaled`` MXFP8 gate_up, fused SwiGLU-OAI + MXFP8 quant, ``dot_scaled`` down), the routed +scale and the sum. Needs vLLM and AITER (``VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_MOE=1``).""" +import argparse +import json +import os +import statistics + +os.environ.setdefault("VLLM_ROCM_USE_AITER", "1") +os.environ.setdefault("VLLM_ROCM_USE_AITER_MOE", "1") + +import torch +from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import H, I, make_activation, make_experts + +SCALE = 2.0 + + +def weights(): + from aiter.utility.fp4_utils import e8m0_shuffle + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.router import bf16x3_router_gemm_rocm + + (w13, s13, w2, s2), _ = make_experts() + e = w13.shape[0] + fp4 = torch.float4_e2m1fn_x2 + sw13, sw2 = rocm_aiter_ops.shuffle_weights(w13.view(fp4), w2.view(fp4)) + s13 = e8m0_shuffle(s13.reshape(e * 2 * I, -1)).view(e, 2 * I, -1) + s2 = e8m0_shuffle(s2.reshape(e * H, -1)).view(e, H, -1) + g = torch.Generator(device="cuda").manual_seed(607) + + def fp8(n, k): + q = (torch.randn((n, k), device="cuda", generator=g) * 20).to(torch.float8_e4m3fn) + return q, torch.randint(118, 124, (n, k // 32), device="cuda", dtype=torch.uint8, generator=g) + + gate = torch.randn((128, H), device="cuda", generator=g) / H**.5 + bias = torch.randn((128,), device="cuda", generator=g) * .01 + return dict(gate=gate, gate_split=bf16x3_router_gemm_rocm.split_bf16x3(gate), bias=bias, w13=sw13, w2=sw2, s13=s13.view(torch.uint8), s2=s2.view(torch.uint8), + sh13=fp8(2 * I, H), sh2=fp8(H, I)) + + +def block(x, w, side): + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (_mxfp8_dot_scaled_gemm, + _mxfp8_dot_scaled_linear) + from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import fused_topk_bias + from vllm.model_executor.layers.fused_moe.router.gate_linear import rocm_bf16x3_router_gemm_dispatch_impl + from vllm.models.minimax_m3.amd.ops.swiglu_oai import swiglu_oai_quantize_mxfp8 + + current = torch.cuda.current_stream() + side.wait_stream(current) + with torch.cuda.stream(side): + gu = _mxfp8_dot_scaled_linear(x, *w["sh13"]) + q, s = swiglu_oai_quantize_mxfp8(gu, alpha=1.702, beta=1.0, limit=7.0, block_m=16) + shared = _mxfp8_dot_scaled_gemm(q, s, *w["sh2"], torch.bfloat16) + logits = rocm_bf16x3_router_gemm_dispatch_impl(x, w["gate"], w["gate_split"]) + wt, ids = fused_topk_bias(x, logits, "sigmoid", w["bias"], 4, True) + routed = rocm_aiter_ops.fused_moe( + x, w["w13"], w["w2"], wt, ids, quant_method=3, + activation_method=rocm_aiter_ops.get_aiter_activation_type("swiglu"), + w1_scale=w["s13"], w2_scale=w["s2"], output_dtype=torch.bfloat16, gate_mode="separated", + swiglu_limit=7.0) + current.wait_stream(side) + return routed * SCALE + shared + + +def measure(w, m, args, side): + copies = args.copies or (100 if m <= 2048 else 16) + xs = [make_activation(m, seed=1000 * m + i) for i in range(copies)] + block(xs[0], w, side) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for x in xs: + block(x, w, side) + for _ in range(20): + graph.replay() + torch.cuda.synchronize() + samples = [] + for _ in range(args.rounds): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.replays): + graph.replay() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) * 1000 / (args.replays * copies)) + return {"tokens": m, "copies": copies, "median_us": statistics.median(samples), + "range_us": [min(samples), max(samples)]} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokens", type=int, nargs="+", + default=[8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768]) + parser.add_argument("--copies", type=int) + parser.add_argument("--rounds", type=int, default=7) + parser.add_argument("--replays", type=int, default=10) + args = parser.parse_args() + props = torch.cuda.get_device_properties(0) + print(json.dumps({"gpu": props.name, "pci": props.pci_bus_id, "torch": torch.__version__, + "scope": "vLLM TP2 fallback MoE block: router + AITER W4A4 + Triton MXFP8 shared", + "mode": "graph"})) + w = weights() + side = torch.cuda.Stream() + for m in args.tokens: + print(json.dumps(measure(w, m, args, side)), flush=True) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_prepared.py b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_prepared.py new file mode 100644 index 0000000..4f2c0bf --- /dev/null +++ b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_prepared.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Local SP preparation from 1025 rows: fused router/FP4/FP8 versus the fused router/FP4 and a separate FP8 +quantization (below 1025 rows both quants share the top-k launch, with no separate counterpart). + +Synthetic weights and rotating BF16 inputs; no communication or expert timing. +Both sides produce identical row-major operands/routing for the public entry. +""" +import argparse +import json +import statistics + +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, prepare_mxfp4_moe +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import _prepare +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from benchmark_shared_moe import weights +from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_activation + + +def measure(w, n, args): + xs = [make_activation(n, seed=1000*n+i) for i in range(100 if n <= 2048 else 16)] + + def separate(x): + (q, s), wt, ids = _prepare(x, w) + q8, s8 = shared_quant(x) + return MoeInput(q, s.view(torch.uint8), wt, ids, q8, s8) + + arms = {'separate': separate, 'fused': lambda x: prepare_mxfp4_moe(x, w)} + order = list(arms)[::-1] if args.reverse else list(arms) + for x in xs: + assert all(torch.equal(a, b) for a, b in zip(separate(x).tensors(), arms['fused'](x).tensors())) + graphs, outputs = {}, {} + for name in order: + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs[name] = [arms[name](x).tensors() for x in xs] + graphs[name] = graph + for _ in range(20): + for name in order: + graphs[name].replay() + torch.cuda.synchronize() + samples = {name: [] for name in order} + for i in range(args.rounds): + for name in order if i % 2 == 0 else order[::-1]: + start, end = [torch.cuda.Event(enable_timing=True) for _ in range(2)] + start.record() + for _ in range(10): + graphs[name].replay() + end.record() + end.synchronize() + samples[name].append(start.elapsed_time(end)*1000/(10*len(xs))) + for a, b in zip(outputs['separate'], outputs['fused']): + assert all(torch.equal(t, u) for t, u in zip(a, b)) + return {'local_tokens': n, 'scope': 'local prep only, no collective or experts', + 'times': {name: {'median_us': statistics.median(t), 'range_us': [min(t), max(t)], 'rounds_us': t} + for name, t in samples.items()}} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--tokens', nargs='+', type=int, default=[1025, 2048, 4096, 8192]) + parser.add_argument('--rounds', type=int, default=7) + parser.add_argument('--reverse', action='store_true') + args = parser.parse_args() + torch.set_num_threads(8) + w = weights() + for n in args.tokens: + print(json.dumps(measure(w, n, args)), flush=True) + torch.cuda.empty_cache() + + +if __name__ == '__main__': + main() diff --git a/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_full.py b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_full.py new file mode 100644 index 0000000..fdb8cca --- /dev/null +++ b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_full.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The full shared expert (I 3072) on R own rows (``shared_expert_full``, sequence-parallel prefill) vs the TP2 +shard (I 1536) on all 2 R rows (``shared_dense`` on a shard copy); ``--sweep``: every gate/up schedule and tile of +the full expert. Graph replay of ``--copies`` inputs, median of ``--rounds``, us per call.""" +import argparse +import json +import statistics + +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import SharedWeights +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_dense import full_config_for, shared_dense, shared_dense_full +from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_full_shared, make_mxfp8 + + +def timed(fn, inputs, rounds, replays): + fn(*inputs[0]) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for x in inputs: + fn(*x) + for _ in range(5): + graph.replay() + torch.cuda.synchronize() + samples = [] + for _ in range(rounds): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(replays): + graph.replay() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) * 1000 / (replays * len(inputs))) + return statistics.median(samples) + + +def configs(r): + out = [] + for t1, sched in (((4, 4), "deferred"), ((2, 4), "deferred"), ((4, 4), "early"), ((4, 4), "single"), + ((2, 4), "single")): + if sched == "single" and ((r + 255) // 256) * 24 * (4 // t1[0]) > 256: + continue + for t2 in ((4, 4), (2, 4)): + for nt in (False, True): + out.append(((t1, sched), (t2, nt))) + return out + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rows", type=int, nargs="+", default=[1024, 2048, 3072, 4096, 6144, 8192, 12288, 16384]) + parser.add_argument("--sweep", action="store_true") + parser.add_argument("--copies", type=int, default=4) + parser.add_argument("--rounds", type=int, default=7) + parser.add_argument("--replays", type=int, default=10) + args = parser.parse_args() + raw, shards = make_full_shared() + full = SharedWeights.from_row_major(*raw, tp_rank=0) + shard = SharedWeights.from_row_major(*shards[0]) + for r in args.rows: + own = [make_mxfp8(r, seed=r + i) for i in range(args.copies)] + every = [make_mxfp8(2 * r, seed=r + i) for i in range(args.copies)] + rec = {"rows": r, "shard_on_2r_us": timed(lambda q, s: shared_dense(q, s, shard), every, args.rounds, + args.replays)} + rec["full_us"] = timed(lambda q, s: shared_dense_full(q, s, full), own, args.rounds, args.replays) + rec["full_config"] = str(full_config_for(r)) + if args.sweep: + rec["sweep"] = {str(c): timed(lambda q, s, c=c: shared_dense_full(q, s, full, config=c), own, + args.rounds, args.replays) for c in configs(r)} + rec["full_vs_shard"] = f"{rec['full_us'] / rec['shard_on_2r_us'] - 1:+.1%}" + print(json.dumps(rec), flush=True) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_moe.py b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_moe.py new file mode 100644 index 0000000..07e964b --- /dev/null +++ b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_shared_moe.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Public routed A4W4 + shared A8W8 entry, synthetic weights, no collective; ``--baseline``: the vLLM TP2 block on the +same weights and inputs in the same process. + +Each shape uses one live graph. An overlap claim also needs capture-order +controls; a checkpoint claim needs a same-buffer A/B on real weights. +""" +import argparse +import json +import os +import statistics +import sys + +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, SharedWeights, mxfp4_moe +from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_activation, make_experts + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + + +def weights(): + raw, _ = make_experts() + generator = torch.Generator(device="cuda").manual_seed(607) + + def fp8(n, k): + q = (torch.randn((n, k), device="cuda", generator=generator) * 20).to(torch.float8_e4m3fn) + s = torch.randint(118, 124, (n, k // 32), device="cuda", dtype=torch.uint8, generator=generator) + return q.view(torch.uint8), s + + shared = SharedWeights.from_row_major(*fp8(3072, 6144), *fp8(6144, 1536)) + gate = torch.randn((128, 6144), device="cuda", generator=generator) / 6144**.5 + bias = torch.randn((128,), device="cuda", generator=generator) * .01 + return MoeWeights.from_row_major(gate, bias, *raw, shared=shared) + + +def view_weights(): + """(MoeWeights over TP rank 1's shard copy, the same over the full copy read by view): one routed set.""" + from dataclasses import replace + + from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_full_shared + base = weights() + raw, shards = make_full_shared() + copy = replace(base, shared_weights=SharedWeights.from_row_major(*shards[1])) + return copy, replace(base, shared_weights=SharedWeights.from_row_major(*raw, tp_rank=1)) + + +def measure(w, m, args, block=None): + copies = args.copies or (100 if m <= 2048 else 16) + xs = [make_activation(m, seed=1000 * m + i) for i in range(copies)] + + def call(x): + return mxfp4_moe(x, w, return_partials=args.partials) if block is None else block(x) + + def flat(y): + return list(y) if isinstance(y, tuple) else [y] + + call(xs[0]) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs = [call(x) for x in xs] + for _ in range(20): + graph.replay() + torch.cuda.synchronize() + samples = [] + for _ in range(args.rounds): + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.replays): + if args.eager: + for x in xs: + call(x) + else: + graph.replay() + end.record() + end.synchronize() + samples.append(start.elapsed_time(end) * 1000 / (args.replays * copies)) + for x, captured in zip(xs, outputs if block is None else ()): + eager = call(x) + torch.cuda.synchronize() + assert all(torch.isfinite(t).all() and torch.equal(t, expected) + for t, expected in zip(flat(captured), flat(eager))) + return {"tokens": m, "copies": copies, "median_us": statistics.median(samples), + "range_us": [min(samples), max(samples)], "rounds_us": samples} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tokens", type=int, nargs="+", default=[8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768]) + parser.add_argument("--copies", type=int) + parser.add_argument("--rounds", type=int, default=7) + parser.add_argument("--replays", type=int, default=10) + parser.add_argument("--partials", action="store_true") + parser.add_argument("--eager", action="store_true",help="time real entry calls, including host issue, instead of graph replay") + parser.add_argument("--baseline", action="store_true", + help="also time the block vLLM runs at TP2 without this entry (benchmark_baseline_moe.py)") + parser.add_argument("--view", action="store_true", + help="the shard copy vs the full copy read by view (rank 1), alternating ``--ab`` times") + parser.add_argument("--ab", type=int, default=3) + args = parser.parse_args() + props = torch.cuda.get_device_properties(0) + print(json.dumps({"gpu": props.name, "pci": props.pci_bus_id, "torch": torch.__version__, + "scope": "synthetic routed + shared, no collective", "partials": args.partials, + "mode": "eager" if args.eager else "graph"})) + if args.view: + copy, view = view_weights() + for m in args.tokens: + runs = {"copy": [], "view": []} + for _ in range(args.ab): + for name, w in (("copy", copy), ("view", view)): + runs[name].append(measure(w, m, args)["median_us"]) + torch.cuda.empty_cache() + rec = {"tokens": m, **{f"{k}_us": statistics.median(v) for k, v in runs.items()}, "runs": runs} + rec["change"] = f"{rec['view_us'] / rec['copy_us'] - 1:+.2%}" + print(json.dumps(rec), flush=True) + return + w = weights() + if args.baseline: + import benchmark_baseline_moe as base + wb, side = base.weights(), torch.cuda.Stream() + for m in args.tokens: + rec = measure(w, m, args) + if args.baseline: + rec["baseline_us"] = measure(wb, m, args, block=lambda x: base.block(x, wb, side))["median_us"] + rec["change"] = f"{rec['median_us'] / rec['baseline_us'] - 1:+.1%}" + print(json.dumps(rec), flush=True) + torch.cuda.empty_cache() + + +if __name__ == "__main__": + main() diff --git a/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_sidecar.py b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_sidecar.py new file mode 100644 index 0000000..2500f6a --- /dev/null +++ b/benchmarks/moe/mxfp4_h6144_i1536_e128_top4/benchmark_sidecar.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Component cost of the optional lossless FP4 sidecar; no TP4 transport claim. + +Inputs rotate beyond MALL. Quantization and routing are outside timing. +Packing replaces 3072 B/token FP4 codes by a 768 B/token correction mask. +A layer claim requires packing + gather + public entry + fused collective. +""" +import argparse +import json +import statistics + +import numpy as np +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.sidecar_pack import pack_mxfp4_sidecar +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.sidecar_unpack import unpack_mxfp4_sidecar + + +def measure(m,args): + count=100 if m<=2048 else 16 + operands=[] + for i in range(count): + x=torch.from_numpy(np.random.default_rng(1000*m+i).standard_normal((m,6144),dtype=np.float32)).cuda().bfloat16() + q4,s4=mxfp4_quant(x);q8,s8=shared_quant(x) + mask=pack_mxfp4_sidecar(q4,s4,q8,s8) + operands.append((q4,s4,q8,s8,mask)) + graphs={};outputs={} + for name in ['pack','unpack']: + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + outputs[name]=[pack_mxfp4_sidecar(q4,s4,q8,s8) if name=='pack' + else unpack_mxfp4_sidecar(mask,s4,q8,s8) + for q4,s4,q8,s8,mask in operands] + graphs[name]=graph + for _ in range(20): + for graph in graphs.values():graph.replay() + torch.cuda.synchronize() + samples={name:[] for name in graphs} + for round in range(args.rounds): + for name in list(graphs) if round%2==0 else list(graphs)[::-1]: + start,end=torch.cuda.Event(enable_timing=True),torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(args.replays):graphs[name].replay() + end.record();end.synchronize() + samples[name].append(start.elapsed_time(end)*1000/(args.replays*count)) + for name,graph in graphs.items(): + for t in outputs[name]:t.fill_(173) + torch.cuda.synchronize();graph.replay();torch.cuda.synchronize() + expected=[row[4 if name=='pack' else 0] for row in operands] + assert all(torch.equal(t,e) for t,e in zip(outputs[name],expected)),name + return dict(m=m,inputs=count,input_bytes=count*m*9600, + explicit_bytes_per_token=9632,compact_bytes_per_token=7328, + samples={name:dict(median=statistics.median(v),min=min(v),max=max(v),rounds=v) + for name,v in samples.items()}) + + +def main(): + ap=argparse.ArgumentParser(description=__doc__) + ap.add_argument('--tokens',nargs='+',type=int,default=[1024,2048,4096,8192,16384,32768]) + ap.add_argument('--rounds',type=int,default=7) + ap.add_argument('--replays',type=int,default=10) + args=ap.parse_args() + print(json.dumps(dict(scope='codec components only; no collective',host=__import__('os').uname().nodename, + pci=torch.cuda.get_device_properties(0).pci_bus_id))) + for m in args.tokens: + print(json.dumps(measure(m,args)),flush=True) + torch.cuda.empty_cache() + + +if __name__=='__main__':main() diff --git a/benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py b/benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py new file mode 100644 index 0000000..00efa63 --- /dev/null +++ b/benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One row count per two-rank process set; distinct inputs beyond the 256 MB MALL, graphs of at most 256 calls, +alternated MAX-rank rounds. Baseline: AITER's custom all-reduce, the one-GPU bf16 norm, then (MXFP8) the FlyDSL +quantizer of its output (row-major codes). Arms: the all-row form and the owned-residual form (owner from +``host.OWNER_MIN_ROWS``); ``--sweep``: the (rows per batch, stagers, reducers, receivers) candidates of both forms. + + python3 benchmarks/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/benchmark_allreduce_ranks2.py [--rows ...] + [--tile 16] [--rounds 7] [--sweep] +""" +import argparse +import json +import statistics + +import torch +import torch.distributed as dist + +H = 6144 +LLC_BYTES = 400 << 20 +MAX_GRAPH_CALLS = 256 + + +def _candidates(rows, owner): + out = [] + q = (rows + 1) // 2 if owner else rows + for rpb in (1, 2, 4, 8): + nb = -(-q // rpb) * (2 if owner else 1) + for nstage in (32, 64, 128, 256): + for nred in (32, 64, 128, 256): + c = (rpb, min(nb, nstage), min(nb, nred), min(nb, nred) if owner else 0) + if c not in out and sum(c[1:]) <= 768: + out.append(c) + return out + + +def worker(rank, errors, m, tile, rounds, sweep, owner_min=None): + from aiter.dist.device_communicators.custom_all_reduce import CustomAllreduce + + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_e4m3_quantize_flydsl + from amd_kernels.norm.allreduce_gemma_bf16_mxfp8_h6144_ranks2 import Bf16ArNormComm, host + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + + if owner_min is not None: + host.OWNER_MIN_ROWS = owner_min + dev = torch.device('cuda', rank) + ca = CustomAllreduce(group=dist.group.WORLD, device=dev, max_size=64 * 1024 * 1024) + comm = Bf16ArNormComm(rank, dist.group.WORLD, dev) + w = (torch.randn(H, generator=torch.Generator().manual_seed(5)) * 0.2).bfloat16().to(dev) + comm.warm_up(w) + n = max(8, min(2048, -(-LLC_BYTES // (2 * m * H * 2)))) + if sweep: + n = min(n, 24) + g = torch.Generator(device=dev).manual_seed(12000 + rank) + xs = [torch.randn(m, H, generator=g, device=dev).bfloat16() for _ in range(n)] + rs = [torch.randn(m, H, generator=g, device=dev).bfloat16() for _ in range(n)] + + def old(i): + a = ca.custom_all_reduce(xs[i]) + y, res = an.gemma_add_rmsnorm(a, rs[i], w, 1e-6) + if tile < 0: + return y, res + return (*mxfp8_e4m3_quantize_flydsl(y), res) + + def new(owner): + def fn(i): + if tile < 0: + return comm.ar_add_rmsnorm(xs[i], rs[i], w, owned_residual=owner) + return comm.ar_add_rmsnorm_mxfp8(xs[i], rs[i], w, 1e-6, tile, owned_residual=owner) + return fn + + arms = {'base': old, 'all_row': new(False)} + if m >= host.OWNER_MIN_ROWS: + arms['owner'] = new(True) + if sweep: + arms = {} + for owner in (False, True): + for cfg in _candidates(m, owner): + def fn(i, cfg=cfg, owner=owner): + comm.config = lambda *a, cfg=cfg: cfg + saved = host.OWNER_MIN_ROWS + host.OWNER_MIN_ROWS = 1 + try: + return new(owner)(i) + finally: + comm.config = None + host.OWNER_MIN_ROWS = saved + arms[('owner' if owner else 'all_row',) + cfg] = fn + for fn in arms.values(): + fn(0) + torch.cuda.synchronize() + graphs = {} + for k, fn in arms.items(): + gs = [] + for lo in range(0, n, MAX_GRAPH_CALLS): + gr = torch.cuda.CUDAGraph() + with ca.capture(), comm.capture(), torch.cuda.graph(gr): + for i in range(lo, min(n, lo + MAX_GRAPH_CALLS)): + fn(i) + gs.append(gr) + graphs[k] = gs + for gs in graphs.values(): + for gr in gs: + gr.replay() + torch.cuda.synchronize() + ts = {k: [] for k in graphs} + for r in range(rounds): + for k in (list(graphs) if r % 2 == 0 else list(reversed(list(graphs)))): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(True), torch.cuda.Event(True) + a.record() + for _ in range(2): + for gr in graphs[k]: + gr.replay() + b.record() + b.synchronize() + ts[k].append(a.elapsed_time(b) * 1e3 / (2 * n)) + every = [None, None] + dist.all_gather_object(every, ts) + if rank == 0: + vals = {k: [max(v[k][j] for v in every) for j in range(rounds)] for k in ts} + med = {str(k): statistics.median(v) for k, v in vals.items()} + if sweep: + best = sorted(med.items(), key=lambda kv: kv[1]) + print(json.dumps(dict(rows=m, tile=tile, best=best[:4] + [kv for kv in best if 'all_row' in kv[0]][:3])), + flush=True) + else: + print(json.dumps(dict(rows=m, tile=tile, sets=n, medians=med)), flush=True) + comm.close() + ca.close() + + +if __name__ == '__main__': + p = argparse.ArgumentParser() + p.add_argument('--rows', nargs='+', type=int, default=[7, 56, 70, 112, 224, 448, 896, 1792]) + p.add_argument('--tile', type=int, default=16, help='code tile rows; 0 row-major; -1 bf16 out') + p.add_argument('--rounds', type=int, default=7) + p.add_argument('--sweep', action='store_true') + p.add_argument('--owner-min', type=int, help='override host.OWNER_MIN_ROWS (the owner arm from this row count)') + a = p.parse_args() + from tests.multi_gpu import run_on_2_gpus + for m in a.rows: + run_on_2_gpus(worker, m, a.tile, a.rounds, a.sweep, a.owner_min, timeout=3600) diff --git a/benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py b/benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py new file mode 100644 index 0000000..57953df --- /dev/null +++ b/benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py @@ -0,0 +1,200 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Both decoder-layer tails at two ranks (``allreduce_add_rmsnorm_ranks2``) against the sequence vLLM runs at TP2 +without the entry, on 2 gfx950 GPUs: + +- ``bf16`` (attention tail of a MoE layer): rows <= 80 AITER's one-stage fused all-reduce + Gemma norm (vLLM's + ``use_1stage_fused_ar_rms`` at world 2), above AITER's custom all-reduce + the one-GPU ``gemma_add_rmsnorm``; +- ``mxfp8`` (attention tail of a dense layer): AITER's custom all-reduce + the one-GPU MXFP8 norm (codes tiled to 16 + rows up to 512 rows, row-major above); +- ``slots`` (MoE tail): the entry sums the rank's 4 weighted slots + shared expert itself; the baseline all-reduces one + bf16 partial (its MoE combine is not counted) + the one-GPU MXFP8 norm. + +Graph replay: every arm cycles through distinct inputs (and residuals) whose total exceeds 400 MB (beyond the 256 MB +MALL), captured in graphs of at most 256 calls; rounds alternate the arm order, each sample is the max over the 2 +ranks of the per-call time, printed median [min-max] over rounds. ``--sweep`` times the entry's +(rows per batch, stagers, reducers, coded) candidates instead. + + python3 benchmarks/norm/gemma_add_rmsnorm_h6144/benchmark_ar_add_rmsnorm_ranks2.py [--kinds bf16,mxfp8,slots] + [--rounds 7] [--json out.json] [--sweep] [rows ...] +""" + +import argparse +import json +import statistics + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +EPS = 1e-6 +H = 6144 +ROWS = [8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4095] +LLC_BYTES = 400 << 20 +MAX_GRAPH_CALLS = 256 +LINK_GBPS = 56.0 # measured per direction with both directions busy + + +def _sets(rows: int, kind: str) -> int: + per_call = rows * H * 2 * (2 + (5 if kind == "slots" else 1)) + return max(8, min(2048, -(-LLC_BYTES // per_call))) + + +def _sweep_configs(rows): + out = [] + for coded in (False, True): + for rpb in (1, 2, 4, 8): + nb = -(-rows // rpb) + for nstage in (16, 32, 64, 128): + for nred in (32, 64, 128, 256): + c = (rpb, min(nb, nstage), min(nb, nred), coded) + if c not in out and c[1] + c[2] <= 512: + out.append(c) + return out + + +def _worker(rank, errors, rows_list, kinds, rounds, out_json, sweep): + from aiter.dist.device_communicators.custom_all_reduce import CustomAllreduce + + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + from amd_kernels.norm.gemma_add_rmsnorm_h6144.allreduce_add_rmsnorm_ranks2 import host as h2 + + dev = torch.device("cuda", rank) + group = dist.group.WORLD + ca = CustomAllreduce(group=group, device=dev, max_size=8192 * 1024 * 8 * 2) # vLLM's MAX_SIZE + assert not ca.disabled + buffers = h2.SharedBuffers(rank, group, dev) + staged = arn.staged_comm(rank, group, dev, buffers) + slots_comm = arn.slots4_comm(rank, group, dev, buffers) + w = (torch.randn(H, generator=torch.Generator().manual_seed(39)) * 0.5).bfloat16().to(dev) + staged.warm_up(w, EPS) + slots_comm.warm_up(w, EPS) + results = {} + for rows in rows_list: + for kind in kinds: + n = min(_sets(rows, kind), 24) if sweep else _sets(rows, kind) + xs, rs, sl, sh = [], [], [], [] + for i in range(n): + g = torch.Generator(device=dev).manual_seed(1000 * rows + 10 * i + rank) + rs.append(torch.randn(rows, H, generator=g, device=dev).bfloat16()) + xs.append(torch.randn(rows, H, generator=g, device=dev).bfloat16()) + if kind == "slots": + sl.append(torch.randn(rows, 4, H, generator=g, device=dev).bfloat16()) + sh.append(torch.randn(rows, H, generator=g, device=dev).bfloat16()) + tile = 16 if rows <= 512 else 0 + + def norm8(y, r): + if tile: + return an.gemma_add_rmsnorm_mxfp8(y, r, w, EPS, tile) + return an.gemma_add_rmsnorm_mxfp8_row_major(y, r, w, EPS) + + if kind == "bf16": + if rows <= 80: + def base(i): + return ca.custom_fused_ar_rms(xs[i], rs[i], w, EPS, use_1stage=True, gemma_norm=True) + else: + def base(i): + return an.gemma_add_rmsnorm(ca.custom_all_reduce(xs[i]), rs[i], w, EPS) + + def entry(i): + return staged.ar_add_rmsnorm(xs[i], rs[i], w, EPS) + elif kind == "mxfp8": + def base(i): + return norm8(ca.custom_all_reduce(xs[i]), rs[i]) + + def entry(i): + return staged.ar_add_rmsnorm_mxfp8(xs[i], rs[i], w, EPS, tile) + else: + def base(i): + return norm8(ca.custom_all_reduce(xs[i]), rs[i]) + + def entry(i): + return slots_comm.slots4_ar_add_rmsnorm_mxfp8(sl[i], sh[i], rs[i], w, EPS, tile) + + arms = {"base": base, "entry": entry} + if sweep: + arms = {} + for cfg in _sweep_configs(rows): + def run(i, cfg=cfg): + buffers.config = lambda *a, cfg=cfg: cfg + try: + return entry(i) + finally: + buffers.config = None + arms[cfg] = run + for fn in arms.values(): # each candidate compiles collectively + fn(0) + base(0) + entry(0) + torch.cuda.synchronize() + graphs = {} + for name, fn in arms.items(): + gs = [] + for lo in range(0, n, MAX_GRAPH_CALLS): + g = torch.cuda.CUDAGraph() + with ca.capture(), torch.cuda.graph(g): + for i in range(lo, min(n, lo + MAX_GRAPH_CALLS)): + fn(i) + gs.append(g) + graphs[name] = gs + for gs in graphs.values(): + for g in gs: + g.replay() + torch.cuda.synchronize() + times = {k: [] for k in graphs} + for rnd in range(rounds): + order = list(graphs) if rnd % 2 == 0 else list(reversed(list(graphs))) + for k in order: + dist.barrier() + torch.cuda.synchronize() + s, e = torch.cuda.Event(True), torch.cuda.Event(True) + s.record() + for _ in range(2): + for g in graphs[k]: + g.replay() + e.record() + torch.cuda.synchronize() + times[k].append(s.elapsed_time(e) * 1e3 / (2 * n)) + every = [None, None] + dist.all_gather_object(every, times) + if rank == 0: + samples = {k: [max(ev[k][j] for ev in every) for j in range(rounds)] for k in times} + med = {k: statistics.median(v) for k, v in samples.items()} + roof = rows * H * 2 / (LINK_GBPS * 1e3) + if sweep: + best = sorted(med.items(), key=lambda kv: kv[1]) + raw = [kv for kv in best if not kv[0][3]][:2] + print(f"rows={rows} {kind} sweep best: " + ", ".join(f"{k}: {v:.2f}" for k, v in best[:4] + raw), + flush=True) + else: + parts = [f"rows={rows} {kind} sets={n}"] + for k, v in samples.items(): + parts.append(f"{k} {med[k]:.2f} [{min(v):.2f}-{max(v):.2f}]") + parts.append(f"link roofline {roof:.2f} us; entry/base {med['entry'] / med['base']:.3f}") + print(" | ".join(parts), flush=True) + results[f"{rows}_{kind}"] = {"samples": {str(k): v for k, v in samples.items()}, + "link_roofline_us": roof} + del graphs, xs, rs, sl, sh + torch.cuda.synchronize() + torch.cuda.empty_cache() + if rank == 0 and out_json: + with open(out_json, "w") as fh: + json.dump(results, fh, indent=1) + ca.close() + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("rows", nargs="*", type=int) + p.add_argument("--json") + p.add_argument("--rounds", type=int, default=7) + p.add_argument("--kinds", default="bf16,mxfp8,slots") + p.add_argument("--sweep", action="store_true") + a = p.parse_args() + run_on_2_gpus(_worker, a.rows or ROWS, a.kinds.split(","), a.rounds, a.json, a.sweep, timeout=7200) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py b/benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py new file mode 100644 index 0000000..c8db5b8 --- /dev/null +++ b/benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: graph replay of the verify logits of R requests (8 rows each, dense owners) and of the seven-step +Markov draft, MAX rank, five alternated rounds, median us. + +Target logits: +- vllm: vLLM's TP2 path: its lm_head GEMM (``rocm_unquantized_gemm_impl``, VLLM_ROCM_USE_AITER=1) over + all rows on the row-major shard, then the RCCL all-gather of the shards into [M, 200064] on every rank; +- vllm a2a: upstream batch-sharded sampling: the same GEMM, then RCCL all_to_all_single + permute (this rank's rows); +- exchange / local: ``BatchShardedSampling.target_logits`` through the logits exchange / with the peer shard copy. +Draft: ``markov_draft`` of R requests (draft lm_head of 7 R rows, seven Markov steps over the owned requests, the id +gather) through the exchange / locally; vllm = only the draft logits of vLLM's replicated draft (its GEMM over 7 R +rows + the all-gather), before its Markov loop over all R requests. +Run: python3 benchmarks/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/benchmark_block.py [R ...] +""" +import os +import socket +import statistics +import sys + +os.environ.setdefault("VLLM_ROCM_USE_AITER", "1") + +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +WORLD, V, H, N, RANK, STEPS = 2, 200064, 6144, 100032, 256, 7 +SETS = 12 + + +def capture(calls): + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for c in calls[:2]: + c() + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for c in calls: + c() + g.replay() + torch.cuda.synchronize() + return g + + +def time_us(g, n): + dist.barrier() + torch.cuda.synchronize() + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + for _ in range(5): + g.replay() + b.record() + b.synchronize() + t = torch.tensor([a.elapsed_time(b) * 1000 / (5 * n)], device="cuda") + dist.all_reduce(t, op=dist.ReduceOp.MAX) + return t.item() + + +def run(rank, port, reqs): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", world_size=WORLD, rank=rank, device_id=dev) + gloo = dist.new_group(backend="gloo") + from vllm.model_executor.layers.utils import rocm_unquantized_gemm_impl + from amd_kernels.gemm.bf16_n100032_k6144 import shuffle_weight + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2 import BatchShardedSampling, host + g = torch.Generator(device=dev).manual_seed(500 + rank) + w = (torch.randn(N, H, device=dev, generator=g) * 0.02).to(torch.bfloat16) + lm = shuffle_weight(w) + max_reqs = max(reqs) + s_x = BatchShardedSampling(rank, gloo, dev, max_reqs=max_reqs, target_rows=max_reqs * 8) + s_l = BatchShardedSampling(rank, gloo, dev, max_reqs=max_reqs, target_rows=max_reqs * 8, lm_weight=lm) + dg = torch.Generator(device=dev).manual_seed(77) + q = STEPS + 1 + t = max_reqs * q + slots = max_reqs + 8 + req = torch.arange(max_reqs, device=dev).repeat_interleave(STEPS) + step = torch.arange(STEPS, device=dev).repeat(max_reqs) + sample_indices = (req * q + step + 1).to(torch.int64) + sample_idx_mapping = torch.randperm(slots, device=dev, generator=dg)[:max_reqs][req].to(torch.int32) + sample_pos = torch.randint(1, 1 << 20, (max_reqs,), device=dev, generator=dg)[req] + step + input_ids = torch.randint(0, V, (t,), device=dev, generator=dg).to(torch.int32) + anchor = torch.arange(max_reqs, device=dev) * q + temperature = torch.ones(slots, device=dev) + seeds = torch.randint(0, 1 << 62, (slots,), device=dev, generator=dg) + w1 = (torch.randn(V, RANK, device=dev, generator=dg) * 0.05).to(torch.bfloat16) + w2 = (torch.randn(V, RANK, device=dev, generator=dg) * 0.05).to(torch.bfloat16) + cache = torch.zeros(slots, STEPS, V, dtype=torch.bfloat16, device=dev) + ids = torch.zeros(max_reqs, STEPS, dtype=torch.int64, device=dev) + for s in (s_x, s_l): + s.setup_markov(STEPS, max_reqs, anchor, w1, w2) + hg = torch.Generator(device=dev).manual_seed(9) + hidden = [torch.randn(max_reqs * 8, H, device=dev, generator=hg).to(torch.bfloat16) for _ in range(SETS)] + if rank == 0: + print("target: R | M | vllm us | vllm a2a us | exchange us | local us", flush=True) + for R in reqs: + m = 8 * R + owners = torch.arange(R) % WORLD + counts = [int((owners == r).sum()) * 8 for r in range(WORLD)] + req_sorted = torch.cat([torch.nonzero(owners == r).flatten() for r in range(WORLD)]) + rows = (req_sorted[:, None] * 8 + torch.arange(8)).flatten().to(dev) + n = counts[rank] + gathered = torch.empty(WORLD * m, N, dtype=torch.bfloat16, device=dev) + recv = torch.empty(WORLD * n, N, dtype=torch.bfloat16, device=dev) + + def vllm_ag(x): + y = rocm_unquantized_gemm_impl(x[:m], w) + dist.all_gather_into_tensor(gathered, y) + return gathered.view(WORLD, m, N).permute(1, 0, 2).reshape(m, V) + + def vllm_a2a(x): + y = rocm_unquantized_gemm_impl(x[rows], w) + dist.all_to_all_single(recv, y, [n] * WORLD, counts) + return recv.view(WORLD, n, N).permute(1, 0, 2).reshape(n, V) + + variants = { + "vllm": [lambda x=x: vllm_ag(x) for x in hidden], + "vllm_a2a": [lambda x=x: vllm_a2a(x) for x in hidden], + "exchange": [lambda x=x: s_x.target_logits(x, lm, rows, counts, n) for x in hidden], + "local": [lambda x=x: s_l.target_logits(x, lm, rows, counts, n) for x in hidden], + } + host.LOCAL_MIN_ROWS = 0 # the local variant at every M + a = s_x.target_logits(hidden[0], lm, rows, counts, n) + b = s_l.target_logits(hidden[0], lm, rows, counts, n) + c = vllm_a2a(hidden[0]) + torch.cuda.synchronize() + assert torch.equal(a.view(torch.int16), b.view(torch.int16)), (rank, R) + if n: + cos = torch.nn.functional.cosine_similarity(a.float().flatten(), c.float().flatten(), dim=0) + assert cos > 0.9999, float(cos) + graphs = {k: capture(v) for k, v in variants.items()} + ts = {k: [] for k in graphs} + for _ in range(5): + for k, gr in graphs.items(): + ts[k].append(time_us(gr, SETS)) + if rank == 0: + print(f"{R} | {m} | " + " | ".join(f"{statistics.median(ts[k]):.1f}" for k in variants), flush=True) + del graphs + torch.cuda.empty_cache() + heads = [torch.randn(t, H, device=dev, generator=dg).to(torch.bfloat16) for _ in range(SETS)] + if rank == 0: + print("draft: R | rows | vllm logits us | exchange us | local us", flush=True) + for R in reqs: + m = R * STEPS + gathered = torch.empty(WORLD * m, N, dtype=torch.bfloat16, device=dev) + + def vllm_draft(x): + y = rocm_unquantized_gemm_impl(x[:m], w) + dist.all_gather_into_tensor(gathered, y) + return gathered.view(WORLD, m, N).permute(1, 0, 2).reshape(m, V) + + def block(s, x): + s.markov_draft(R, x, lm, sample_indices, sample_idx_mapping, sample_pos, input_ids, temperature, seeds, + cache, ids) + + variants = {"vllm": [lambda x=x: vllm_draft(x) for x in heads], + "exchange": [lambda x=x: block(s_x, x) for x in heads], + "local": [lambda x=x: block(s_l, x) for x in heads]} + graphs = {k: capture(v) for k, v in variants.items()} + ts = {k: [] for k in graphs} + for _ in range(5): + for k, gr in graphs.items(): + ts[k].append(time_us(gr, SETS)) + if rank == 0: + print(f"{R} | {m} | " + " | ".join(f"{statistics.median(ts[k]):.1f}" for k in variants), flush=True) + del graphs + torch.cuda.empty_cache() + s_x.close() + s_l.close() + dist.destroy_process_group() + + +if __name__ == "__main__": + reqs = [int(v) for v in sys.argv[1:]] or [1, 4, 8, 10, 12, 16, 20, 24, 32, 40, 48, 52, 64, 96, 128, 256] + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.close() + mp.spawn(run, args=(port, reqs), nprocs=WORLD, join=True) diff --git a/benchmarks/sampling/top_p_mask_f32_v200064/benchmark_top_p_mask.py b/benchmarks/sampling/top_p_mask_f32_v200064/benchmark_top_p_mask.py index 7222f22..3c6b2b1 100644 --- a/benchmarks/sampling/top_p_mask_f32_v200064/benchmark_top_p_mask.py +++ b/benchmarks/sampling/top_p_mask_f32_v200064/benchmark_top_p_mask.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Graph-replay time of ``top_p_mask`` per row count (the sampled rows of one TP4 rank: requests x (1 + draft -tokens) / 4): 100 distinct inputs per graph, each a bf16 -> fp32 copy of LM-like logits and the mask (the served +"""Graph-replay time of ``top_p_mask`` per row count (the sampled rows of one rank: requests x (1 + draft tokens) / +ranks; above 64 rows the one-program-per-row kernel, with a caller's buffer as vLLM passes it): 100 distinct inputs +per graph, each a bf16 -> fp32 copy of LM-like logits and the mask (the served step: the mask is in place, so every replay restarts from the bf16 rows); the copy alone is timed as well and subtracted. Median of 5 rounds x 5 replays, us per call. ``--rows ``: a saved [R, 200064] tensor of real logits rows; each input then takes random rows of it at a temperature in 0.85-1.15 (the README's table); without it @@ -13,7 +14,7 @@ import statistics import torch -from amd_kernels.sampling.top_p_mask_f32_v200064 import V, splits_for, top_p_mask, warmup_top_p_mask +from amd_kernels.sampling.top_p_mask_f32_v200064 import V, rows_grid, splits_for, top_p_mask, warmup_top_p_mask NIN = 100 @@ -57,7 +58,7 @@ def main(): real = torch.load(args.rows, weights_only=True).float() assert real.dim() == 2 and real.shape[1] == V, real.shape print("rows | slices | copy us | copy + mask us | mask us") - for n in (1, 4, 8, 16, 32, 64): + for n in (1, 4, 8, 16, 32, 64, 80, 96, 128, 160, 208, 256): src = [] for _ in range(NIN): if real is not None: @@ -69,6 +70,7 @@ def main(): src.append(x.to(torch.bfloat16).to(dev)) bufs = [torch.empty(n, V, device=dev) for _ in range(NIN)] p = torch.full((n,), 0.95, device=dev) + work = torch.empty(rows_grid(n, 0), V, device=dev) if n > 64 else None def copy(): for b, s in zip(bufs, src): @@ -77,7 +79,7 @@ def copy(): def step(): for b, s in zip(bufs, src): b.copy_(s) - top_p_mask(b, p) + top_p_mask(b, p, buffer=work) gc, gs = graph(copy), graph(step) tc, ts = [], [] @@ -85,7 +87,7 @@ def step(): tc.append(timed(gc)) ts.append(timed(gs)) c, t = statistics.median(tc), statistics.median(ts) - print(f"{n} | {splits_for(n)} | {c:.1f} | {t:.1f} | {t - c:.1f}", flush=True) + print(f"{n} | {splits_for(n) if n <= 64 else 'rows'} | {c:.1f} | {t:.1f} | {t - c:.1f}", flush=True) del gc, gs, bufs, src torch.cuda.empty_cache() diff --git a/tests/activation/swiglu_oai_mxfp8_i6144/__init__.py b/tests/activation/swiglu_oai_mxfp8_i6144/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/activation/swiglu_oai_mxfp8_i6144/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/activation/swiglu_oai_mxfp8_i6144/test_swiglu_oai_mxfp8.py b/tests/activation/swiglu_oai_mxfp8_i6144/test_swiglu_oai_mxfp8.py new file mode 100644 index 0000000..f8c3698 --- /dev/null +++ b/tests/activation/swiglu_oai_mxfp8_i6144/test_swiglu_oai_mxfp8.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``swiglu_oai_mxfp8`` against its contract: the aten chain on the same bf16 gate_up, then the +MXFP8 quantizer (``amd_kernels.gemm.mxfp8_common``'s row-major and (16, 16) quantizers), bit for bit; the codes also +against ``reference.quantize_mxfp8``. Rows: the draft's 7R and 8R (R = 1, 10, 16, 32, 64, 128, 256) and odd +counts; rows scaled over +-5 binades, values at and beyond the clamp limit; graph replay; determinism.""" + +import pytest +import torch + +from amd_kernels.activation.swiglu_oai_mxfp8_i6144 import swiglu_oai_mxfp8 +from amd_kernels.activation.swiglu_oai_mxfp8_i6144.reference import quantize_mxfp8, swiglu_bf16_chain, swiglu_fp32 + +I = 6144 +ALPHA, BETA, LIMIT = 1.702, 1.0, 7.0 +ROWS = [1, 7, 8, 13, 16, 17, 63, 65, 127, 129, 70, 80, 112, 128, 224, 256, 448, 512, 704, 705, 896, 1024, 1792, 2048] +TILES = [16, 32, 64, 96, 128] + + +def _gate_up(m, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn((m, 2 * I), generator=g, device="cuda") * 3.0 + x *= torch.exp2(torch.randint(-5, 6, (m, 1), generator=g, device="cuda").float()) + x[:, :8] = torch.tensor([7.0, 7.5, -7.0, -9.0, 0.0, -0.0, 1e-30, 30.0], device="cuda") + return x.bfloat16() + + +def _today(x, tile, form="bf16_chain"): + from amd_kernels.gemm.mxfp8_common import gemm as G + + act = (swiglu_bf16_chain if form == "bf16_chain" else swiglu_fp32)(x, ALPHA, BETA, LIMIT) + if tile: + return G.mxfp8_e4m3_quantize_shuffled(act, tile) + return G.mxfp8_e4m3_quantize_aiter(act) + + +@pytest.mark.parametrize("m", ROWS) +def test_rowmajor_bitwise(m): + x = _gate_up(m, m) + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, 0) + ref = _today(x, 0) + assert torch.equal(got[0][:m].view(torch.uint8), ref[0][:m].view(torch.uint8)), "codes" + _scales_eq(got[1], ref[1].view(torch.uint8), m) + q_ref, s_ref = quantize_mxfp8(swiglu_bf16_chain(x, ALPHA, BETA, LIMIT)) + assert torch.equal(got[0][:m].view(torch.uint8), q_ref.view(torch.uint8)) + + +@pytest.mark.parametrize("m", ROWS) +def test_fp32_form_bitwise(m): + """The target's form: swiglu_oai_split's fp32 math + one rounding, then the quantizer.""" + x = _gate_up(m, 11 * m) + for tile in (0, 16, 64): + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, "fp32") + ref = _today(x, tile, "fp32") + _codes_eq(got[0], ref[0], m, tile) + _scales_eq(got[1], ref[1].view(torch.uint8), m) + + +@pytest.mark.parametrize("m", ROWS) +@pytest.mark.parametrize("tile", TILES) +def test_tiled_bitwise(m, tile): + x = _gate_up(m, 7 * m + tile) + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile) + ref = _today(x, tile) + _codes_eq(got[0], ref[0], m, tile) + _scales_eq(got[1], ref[1].view(torch.uint8), m) + + +def _codes_eq(got, ref, m, tile): + gq, rq = got.view(torch.uint8), ref.view(torch.uint8) + if not tile: + assert torch.equal(gq[:m], rq[:m]) + return + full = m // 16 * 16 + assert torch.equal(gq[:full], rq[:full]), "codes (full 16-row groups)" + if m % 16: + blk = gq[full:full + 16].reshape(I // 64, 4, 16, 16)[:, :, : m % 16] + rblk = rq[full:full + 16].reshape(I // 64, 4, 16, 16)[:, :, : m % 16] + assert torch.equal(blk, rblk), "codes (partial group)" + + +def _scales_eq(gs, rs, m): + """Shuffled scales: compare the bytes that belong to real rows (unshuffle by permuting an index table).""" + from aiter.ops.shuffle import shuffle_scale_a16w4 + + rows = gs.shape[0] + idx = torch.arange(rows, device=gs.device, dtype=torch.int32)[:, None].expand(rows, I // 32).contiguous() + row_of = shuffle_scale_a16w4(idx, 1, False).reshape(-1) + real = row_of < m + assert torch.equal(gs.reshape(-1)[real], rs[:rows].reshape(-1)[real]), "scales" + + +def test_rowmajor_scales_partial(): + for m in (1, 7, 13, 70): + x = _gate_up(m, 99 + m) + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, 0) + ref = _today(x, 0) + _scales_eq(got[1], ref[1].view(torch.uint8), m) + + +@pytest.mark.parametrize("m", (7,17,70,112,224)) +def test_graph_and_determinism(m): + tile = 32 + xs = [_gate_up(m, 500 + i) for i in range(4)] + eager = [swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile) for x in xs] + q = torch.empty_like(eager[0][0]) + s = torch.empty_like(eager[0][1]) + graph = torch.cuda.CUDAGraph() + src = xs[0].clone() + with torch.cuda.graph(graph): + swiglu_oai_mxfp8(src, ALPHA, BETA, LIMIT, tile, q_out=q, scale_out=s) + for x, (eq, es) in zip(xs, eager): + src.copy_(x) + q.view(torch.uint8).fill_(0x7f) + s.fill_(0xff) + graph.replay() + torch.cuda.synchronize() + _codes_eq(q, eq, m, tile) + _scales_eq(s, es, m) + again = swiglu_oai_mxfp8(xs[1], ALPHA, BETA, LIMIT, tile) + _codes_eq(again[0], eager[1][0], m, tile) + + +def test_no_compile_after_warmup(): + """README rule 4: after the warm-up, every row count of the draft batch (7R, 8R) and both code layouts + run on the compiled variants (the launcher's cached CompiledFunction; no new compile).""" + from amd_kernels.activation.swiglu_oai_mxfp8_i6144 import host, warmup_swiglu_oai_mxfp8 + + host._get_kernel.cache_clear() + warmup_swiglu_oai_mxfp8() + before = host._get_kernel.cache_info().currsize + keys = [(f, t, p, n) for f in ("bf16_chain", "fp32") for t in (False, True) for _, p, n in host.SHAPES] + assert before == len(keys) + compiled = {k: getattr(host._get_kernel(*k), "_cf", None) for k in keys} + assert all(v is not None for v in compiled.values()) + for m in ROWS + [k * 7 for k in range(1, 300, 13)]: + x = _gate_up(m, 3 * m) + for tile in (0, 16, 32, 64, 96, 128): + for form in ("bf16_chain", "fp32"): + swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, form) + assert host._get_kernel.cache_info().currsize == before + assert all(getattr(host._get_kernel(*k), "_cf", None) is v for k, v in compiled.items()) + + +@pytest.mark.parametrize("shape", ((1, 768), (1, 384), (2, 384), (3, 256), (6, 128), (12, 64), (4, 192))) +@pytest.mark.parametrize("form", ("bf16_chain", "fp32")) +def test_every_shape_bitwise(shape, form): + """Every workgroup shape the dispatch table can name gives the same bits.""" + for m, tile in ((7, 16), (70, 0), (129, 64)): + x = _gate_up(m, 41 * m) + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, form, shape=shape) + ref = _today(x, tile, form) + _codes_eq(got[0], ref[0], m, tile) + _scales_eq(got[1], ref[1].view(torch.uint8), m) + + +@pytest.mark.parametrize('form', ('bf16_chain', 'fp32')) +@pytest.mark.parametrize('tile', (0, 64)) +def test_partitioned_next_to_mfma(form, tile): + from tests.mfma_spinner import mfma_spinner + m = 70 + x = _gate_up(m, 987) + ref = _today(x, tile, form) + spinner = mfma_spinner(256) + side = torch.cuda.Stream(priority=-1) + for _ in range(3): + spinner(100_000, side) + got = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile, form) + torch.cuda.synchronize() + _codes_eq(got[0], ref[0], m, tile) + _scales_eq(got[1], ref[1].view(torch.uint8), m) + + +def test_zero_rows(): + x = torch.empty((0, 2 * I), device='cuda', dtype=torch.bfloat16) + for tile in (0, 16): + q, scales = swiglu_oai_mxfp8(x, ALPHA, BETA, LIMIT, tile) + assert q.numel() == scales.numel() == 0 diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_compile.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_compile.py new file mode 100644 index 0000000..6b11214 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_compile.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Nothing compiles after the entry's first-call warm-up (root README, rule 4): in a fresh process the first call +warms every variant up (``host._sizes``: every token count 1-768, compile-only), then calls at 1-64 rows and at the +deferred chain's tier edges to 768 compile nothing.""" + +import json +import os +import subprocess +import sys + +import pytest + +SIZES = list(range(1, 65)) + [65, 96, 97, 128, 129, 192, 193, 256, 257, 320, 321, 384, 385, 416, 449, 512, 513, 577, + 625, 673, 721, 768] + + +def scenario(): + import torch + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels import warm_up + from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 import ( + mxfp8_qkv_norm_rope_nvfp4_insert) + + from .test_qkv_insert import IQW, KW, make, outputs, weights + + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + warm_up.set_workers(1) + count = [] + original = MlirCompiler.compile + + def spy(self, *args, **kw): + if warm_up._in_warm_up.get() or warm_up._current.get() is not None: + count.append(1) + return original(self, *args, **kw) + + MlirCompiler.compile = spy + _, (w, ws), _, _ = weights() + + def call(m): + sa, _, ins = make(m, m) + q, iq_rows = outputs(m) + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, *ins, q, iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + + call(8) # the first call: the warm-up + torch.cuda.synchronize() + warm = len(count) + late = [] + for m in SIZES: + before = len(count) + call(m) + if len(count) > before: + late.append(m) + torch.cuda.synchronize() + return {"warm_compiles": warm, "late": late, "late_reported": warm_up.late_compiles()} + + +def test_no_compile_after_warm_up(tmp_path): + import torch + + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + # an empty FlyDSL artifact cache: the warm-up itself compiles every variant + proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=3000, + env={**os.environ, "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + print(result) + assert result["warm_compiles"] > 0 + assert not result["late"] + assert result["late_reported"] == 0 + + +if __name__ == "__main__": + print(json.dumps(scenario())) diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_counters.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_counters.py new file mode 100644 index 0000000..33c1d20 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_counters.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Host-thread counter reservation: distinct streams/captures must never alias a workspace.""" + +import concurrent.futures +import threading +import time + +import pytest +import torch + +from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 import host + + +def _isolate(monkeypatch, *, shared_key): + device = torch.device("cpu") + monkeypatch.setattr(host, "_POOL", {}) + monkeypatch.setattr(host, "_COUNTERS", {}) + monkeypatch.setattr(host, "_COUNTER_SETS", 16) + monkeypatch.setattr(host, "_counter_key", lambda d: (d, ("thread", 0 if shared_key else threading.get_ident()))) + return device + + +def _parallel_reserve(device): + start = threading.Barrier(8) + + def reserve(_): + start.wait(timeout=10) + return host._counters(device) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: + return list(executor.map(reserve, range(8))) + + +@pytest.mark.parametrize("shared_key", [False, True]) +def test_counter_slices_reserved_atomically(monkeypatch, shared_key): + """Yield the GIL during tensor slicing, as real torch tensor operations can do.""" + device = _isolate(monkeypatch, shared_key=shared_key) + pool = torch.zeros(host._COUNTER_SETS * host._COUNTER_WORDS, dtype=torch.int32) + slices = [] + + class SlowPool: + def __getitem__(self, section): + time.sleep(0.02) + slices.append(section) + return pool[section] + + host._POOL[device] = SlowPool() + results = _parallel_reserve(device) + pointers = [t.data_ptr() for t in results] + assert len(set(pointers)) == (1 if shared_key else 8) + assert len(slices) == (1 if shared_key else 8) + assert len(host._COUNTERS) == len(slices) + if not shared_key: + assert sorted(s.start for s in slices) == [i * host._COUNTER_WORDS for i in range(8)] + for tensor in results: + assert tensor.numel() == host._COUNTER_WORDS and not tensor.count_nonzero() + + +@pytest.mark.parametrize("shared_key", [False, True]) +def test_counter_pool_initialized_once(monkeypatch, shared_key): + """Concurrent first calls share one fully initialized pool; no unacknowledged slice is published.""" + device = _isolate(monkeypatch, shared_key=shared_key) + real_zeros = torch.zeros + allocations, acknowledgements = [], [] + + def slow_zeros(*args, **kwargs): + time.sleep(0.02) + allocations.append(1) + return real_zeros(*args, **kwargs) + + class Ready: + def record(self, stream): + pass + + def synchronize(self): + time.sleep(0.02) + assert not host._COUNTERS + acknowledgements.append(1) + + monkeypatch.setattr(torch, "zeros", slow_zeros) + monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False) + monkeypatch.setattr(torch.cuda, "current_stream", lambda device: None) + monkeypatch.setattr(torch.cuda, "Event", Ready) + results = _parallel_reserve(device) + assert len(allocations) == len(acknowledgements) == 1 + assert len({t.data_ptr() for t in results}) == (1 if shared_key else 8) + assert len(host._COUNTERS) == (1 if shared_key else 8) + assert all(t.untyped_storage().data_ptr() == host._POOL[device].untyped_storage().data_ptr() for t in results) + + +@pytest.mark.parametrize("ms", [(8, 16), (1, 13)]) +def test_two_host_threads_enqueue_independent_streams(ms): + """Exercise actual thread-local stream keys and concurrent entry calls, after serial warm-up.""" + from . import test_qkv_insert as checks + + checks.require_gfx950() + _, (weight, scale), _, _ = checks.weights() + jobs = [] + for m in ms: + codes, row_major, inputs = checks.make(m, 7200 + m) + expected = checks.run("chain", m, codes, row_major, inputs) + checks.run("entry", m, codes, row_major, inputs) + q, iq = checks.outputs(m) + jobs.append((m, codes, inputs, expected, (q, iq, inputs[8].clone(), inputs[9].clone()))) + streams = [torch.cuda.Stream(), torch.cuda.Stream()] + torch.cuda.synchronize() + start = threading.Barrier(2) + pointers = [] + + def enqueue(i): + m, codes, inputs, expected, (q, iq, kv, index) = jobs[i] + torch.cuda.set_device(0) + with torch.cuda.stream(streams[i]): + start.wait(timeout=30) + pointers.append(host._counters(codes[0].device).data_ptr()) + start.wait(timeout=30) + assert len(set(pointers)) == 2 # fail before launching kernels with aliased counters + for _ in range(20): + kv.copy_(inputs[8]) + index.copy_(inputs[9]) + q.fill_(7) + iq[:, :checks.IQW].fill_(0x77) + host.mxfp8_qkv_norm_rope_nvfp4_insert( + *codes, weight, scale, m, *inputs[:8], kv, index, *inputs[10:], q, + iq.view(torch.float8_e4m3fn)[:, :checks.IQW], **checks.KW, + ) + streams[i].synchronize() + assert checks.nbytes_differ((q, iq, kv, index), expected) == [0] * 4 + + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: + list(executor.map(enqueue, range(2))) diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_isa.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_isa.py new file mode 100644 index 0000000..d173db2 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_isa.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Every GEMM this entry compiles itself (the one-launch kernel at 1-16 rows, the non-temporal-store decode rows at +257-448) keeps its MFMA operands in VGPRs: a parked operand's copy back races the AGPR write (``gemm/mxfp8_common`` +``mxfp8_gemm_decode.VGPR_BUDGET``), so the final ISA holds no ``v_accvgpr_write``. Compiled in a subprocess, +compile-only, cache off, ISA dumped.""" + +import glob +import os +import subprocess +import sys +from pathlib import Path + +import torch + +from .test_qkv_insert import require_gfx950 + +ROWS = list(range(1, 17)) + list(range(257, 449)) + + +def _launchers(): + from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 import host + + fused = {host._plan(m) for m in ROWS if m <= host.FUSED_MAX_ROWS} + nt = {host._nt_config(m) for m in ROWS if m > host.FUSED_MAX_ROWS} - {None} + return [host._fused_launcher(p) for p in sorted(fused, key=str)], [host._nt_gemm_launcher(c) + for c in sorted(nt, key=str)] + + +def test_operands_stay_in_vgprs(tmp_path): + require_gfx950() + root = Path(__file__).resolve().parents[3] + env = dict(os.environ, AKR_COMPILE_ONLY="1", FLYDSL_DEBUG_DUMP_ASM="1", FLYDSL_DUMP_DIR=str(tmp_path), + FLYDSL_RUNTIME_ENABLE_CACHE="0") + module = "tests.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128.test_isa" + subprocess.run([sys.executable, "-m", module], cwd=root, env=env, check=True) + fused, nt = _launchers() + assert fused and len(nt) == 3 + for exe in fused + nt: + isa = glob.glob(f"{tmp_path}/{exe.kernel_name}/*.s") + assert isa, exe.kernel_name + assert "v_accvgpr_write" not in Path(isa[0]).read_text(), exe.kernel_name + + +if __name__ == "__main__": + from amd_kernels.common.tensor_shim import _run_compiled + + d = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") + p = d.data_ptr() + fused, nt = _launchers() + for exe in fused: + _run_compiled(exe, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, p, 8, 1024, 256, 1000, 1000, 1, 0, + torch.cuda.current_stream()) + for exe in nt: + _run_compiled(exe, d, d, d, d, d, 256, 1024, 1, 104, torch.cuda.current_stream()) diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_packing.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_packing.py new file mode 100644 index 0000000..26215e8 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_packing.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``packing._v_expanded_words``: every input basis bit plus random four/eight-row packets equal the row-by-row +nibble insert.""" + +import functools + +import pytest +import torch +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import range_constexpr + +from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128.packing import ( + _v_expand, + _v_expanded_words, +) + + +@functools.cache +def packing_launcher(rows): + name = f"test_n4992_v_nibble_transpose_rows{rows}" + + @flyc.kernel(name=name, known_block_size=[64, 1, 1]) + def pack(x: fx.Tensor, out: fx.Tensor): + idx = fx.block_idx.x * 64 + fx.thread_idx.x + values = [fx.Int32(x[idx * rows + r]) for r in range_constexpr(rows)] + words = _v_expanded_words([_v_expand(v) for v in values]) + for i in range_constexpr(4): + out[idx * 4 + i] = words[i] + + @flyc.jit + def launch_n4992_v_nibble_transpose_test(x: fx.Tensor, out: fx.Tensor, blocks: fx.Int32, stream: fx.Stream): + pack(x, out).launch(grid=(blocks, 1, 1), block=(64, 1, 1), stream=stream) + + return launch_n4992_v_nibble_transpose_test + + +def expected(x, rows): + """Row r's nibble of dimension d goes to bits 4r of word d (the run-by-row insert).""" + out = torch.zeros(x.shape[0], 4, dtype=torch.int64) + for r in range(rows): + for d in range(4): + out[:, d] |= ((x[:, r] >> (4 * d)) & 15) << (4 * r) + return out.to(torch.int32) + + +@pytest.mark.parametrize("rows", [4, 8]) +def test_expanded_words(rows): + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + basis = torch.tensor([[1 << b if r == rr else 0 for r in range(rows)] for rr in range(rows) for b in range(16)]) + rand = torch.randint(0, 1 << 16, (4096 - basis.shape[0], rows)) + x = torch.cat([basis, rand]).to(torch.int32) + out = torch.empty(x.shape[0] * 4, dtype=torch.int32, device="cuda") + packing_launcher(rows)(x.flatten().cuda(), out, x.shape[0] // 64, torch.cuda.current_stream()) + torch.cuda.synchronize() + assert torch.equal(out.view(-1, 4).cpu(), expected(x.to(torch.int64), rows)) diff --git a/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_qkv_insert.py b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_qkv_insert.py new file mode 100644 index 0000000..ee8bee0 --- /dev/null +++ b/tests/attention/mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128/test_qkv_insert.py @@ -0,0 +1,409 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``mxfp8_qkv_norm_rope_nvfp4_insert`` at TP2 (N 4992, heads 32 / 2 / 2): bytes of the deferred chain at every tier, +the reference's tolerances, graph == eager, determinism, epoch counters back at a multiple of 16 after every call, +two streams at once, the routing of ``qkv.mxfp8_gemm_qkv_norm_rope_nvfp4_insert``. + +Inputs: a synthetic MXFP8 weight (normal values, per-32 scales over +-5 binades), activations with per-32 scales over ++-5 binades and one zero row, serving slot runs (requests of up to 8 tokens from a random page offset, or one request +across a page boundary), the last token's slot -1, one index slot -1, pages full of stale random bytes, index-q a +strided view (row stride 320 bytes). +""" + +import functools + +import numpy as np +import pytest +import torch + +from amd_kernels.attention.mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128 import ( + host, + mxfp8_qkv_norm_rope_nvfp4_insert, + reference, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( + norm_rope_nvfp4_insert, + splitk_reduce_norm_rope_nvfp4_insert, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import qkv as chain_entry +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import reference as insert_ref +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common.reference import compare, quantize_mxfp8 + +N, K = 4992, 6144 +KW = dict(num_heads=32, num_kv_heads=2, num_index_heads=2, eps=1e-6) +QW, IQW, IQ_ROW = 4096, 256, 320 +# every one-launch row and the deferred chain's tier edges above it (split-K reduce-insert to 64, bf16 GEMM above) +EDGE = list(range(1, 17)) + [17, 32, 33, 48, 49, 64, 65, 96, 97, 128, 129, 192, 193, 256, 257, 320, 321, 384, 385, + 416, 448, 449, 512, 513, 576, 577, 624, 625, 672, 673, 720, 721, 768] + + +def require_gfx950(): + if not torch.cuda.is_available() or "gfx950" not in torch.cuda.get_device_properties(0).gcnArchName: + pytest.skip("requires gfx950") + + +def _spread(rng, shape): + x = rng.standard_normal(shape, dtype=np.float32) + e = rng.integers(-5, 6, size=(shape[0], shape[1] // 32)).astype(np.float32) + return x * np.repeat(2.0 ** e, 32, axis=1) + + +@functools.cache +def weights(): + rng = np.random.default_rng(39) + w = torch.from_numpy(_spread(rng, (N, K)) / 64).cuda().to(torch.bfloat16) + wc, we = quantize_mxfp8(w) + shuffled = g.preshuffle_mxfp8_weight(wc, we) + norm = [torch.from_numpy(rng.standard_normal(128, dtype=np.float32) * 0.3).cuda().to(torch.bfloat16) + for _ in range(4)] + inv = 1.0 / (1e7 ** (torch.arange(0, 64, 2, dtype=torch.float64) / 64)) + f = torch.arange(131072, dtype=torch.float64)[:, None] * inv + cs = torch.cat([f.cos(), f.sin()], 1).to(torch.bfloat16).cuda().contiguous() + return (wc, we), shuffled, norm, cs + + +def make(m, seed): + """One call's inputs: (shuffled codes, scales), (row-major codes, exponents), insert arguments.""" + (_, _), _, norm, cs = weights() + rng = np.random.default_rng(seed) + x = _spread(rng, (m, K)) + if m > 2: + x[m // 2] = 0 + x = torch.from_numpy(x).cuda().to(torch.bfloat16) + tile = g.prequant_code_tile_rows(m, N, K) + codes, scales = g.mxfp8_e4m3_quantize_shuffled(x, tile) + rc, re = quantize_mxfp8(x) + blocks = max(m // 128 + 4, m // 8 + 1) + sl = [] + if seed % 2: + start = min(128 - (m % 128 or 128) // 2, 128 * 4 - m) + sl = list(range(start, start + m)) + else: + pages = rng.permutation(blocks) + for i, length in enumerate([8] * (m // 8) + ([m % 8] if m % 8 else [])): + start = int(rng.integers(0, 128 - length + 1)) + sl += [int(pages[i]) * 128 + start + j for j in range(length)] + slots = torch.tensor(sl, device="cuda", dtype=torch.int64) + islots = slots.clone() + if m > 3: + slots[m - 1] = -1 # at a run's end (inside a run is the insert's strict V-octet xfail) + if m > 4: + islots[3] = -1 + pos = torch.from_numpy(rng.integers(0, 131000, size=m)).cuda() + kv = torch.from_numpy(rng.integers(0, 256, size=(blocks, 4, 128, 72), dtype=np.uint8)).cuda() + idx = torch.from_numpy(rng.integers(0, 256, size=(blocks, 128, 72), dtype=np.uint8)).cuda() + ks = torch.tensor(0.37, device="cuda") + vs = torch.tensor(0.61, device="cuda") + return (codes, scales), (rc, re), (*norm, cs, pos, slots, islots, kv, idx, ks, vs) + + +def outputs(m): + q = torch.full((m, QW), 7, dtype=torch.bfloat16, device="cuda") + iq_rows = torch.full((m, IQ_ROW), 5, dtype=torch.uint8, device="cuda") + return q, iq_rows + + +def run(kind, m, shuffled_act, row_act, ins): + """(q, index-q row bytes, kv, idx) of one call on fresh page copies.""" + _, (w, ws), _, _ = weights() + q, iq_rows = outputs(m) + iq = iq_rows.view(torch.float8_e4m3fn)[:, :IQW] + args = (*ins[:8], ins[8].clone(), ins[9].clone(), *ins[10:]) + if kind == "entry": + mxfp8_qkv_norm_rope_nvfp4_insert(*shuffled_act, w, ws, m, *args, q, iq, **KW) + elif kind == "chain": # the deferred chain + qkv = g.mxfp8_gemm_shuffled(*shuffled_act, w, ws, torch.bfloat16, m, defer_reduce=True) + if isinstance(qkv, g.SkPartials): + splitk_reduce_norm_rope_nvfp4_insert(qkv.tmp, qkv.split_k, *args, q, iq, **KW) + else: + norm_rope_nvfp4_insert(qkv, *args, q, iq, **KW) + else: + (wc, we), _, _, _ = weights() + reference.mxfp8_qkv_norm_rope_nvfp4_insert(*row_act, wc, we, *args, q, iq, **KW) + torch.cuda.synchronize() + return q, iq_rows, args[8], args[9] + + +def nbytes_differ(a, b): + return [int((x.view(torch.uint8) != y.view(torch.uint8)).sum()) for x, y in zip(a, b)] + + +@pytest.mark.parametrize("m", EDGE) +@pytest.mark.parametrize("seed", [1, 2]) +def test_bytes_equal_deferred_chain(m, seed): + require_gfx950() + sa, ra, ins = make(m, 1000 * m + seed) + assert nbytes_differ(run("entry", m, sa, ra, ins), run("chain", m, sa, ra, ins)) == [0, 0, 0, 0] + + +@pytest.mark.parametrize("m", range(1, 17)) +def test_v_octet_alignment_and_stale_bits(m): + """Every run length and starting nibble, including full octets and page crossings; random stale page bytes expose + masks that erase an unwritten nibble or scale byte. The final main slot stays -1; index slots skip on their own.""" + require_gfx950() + sa, ra, ins = make(m, 6000 + m) + for offset in range(8): + slots = torch.arange(m, device="cuda", dtype=torch.int64) + 120 + offset + islots = slots.clone() + if m > 1: + slots[-1] = -1 + islots[0] = -1 + args = (*ins[:6], slots, islots, *ins[8:]) + assert nbytes_differ(run("entry", m, sa, ra, args), run("chain", m, sa, ra, args)) == [0, 0, 0, 0] + + +@pytest.mark.parametrize("m", [1, 5, 8, 13, 16]) +def test_reference_tolerance(m): + """The one-launch rows against the contract (the deferred chain's rows hold their own packages' contracts).""" + require_gfx950() + sa, ra, ins = make(m, 77 * m + 1) + got = run("entry", m, sa, ra, ins) + ref = run("reference", m, sa, ra, ins) + for name, a, b in (("q", got[0], ref[0]), ("index_q", got[1][:, :IQW].view(torch.float8_e4m3fn), + ref[1][:, :IQW].view(torch.float8_e4m3fn))): + c = compare(a.float(), b.float()) + bound = reference.REL_L2 if name == "q" else reference.INDEX_Q_REL_L2 + assert c["cos"] >= reference.COS_MIN and c["rel_l2"] <= bound, (name, c) + slots, islots = ins[6], ins[7] + kv_mask, idx_mask = insert_ref.written_bits(ins[8], ins[9], slots, islots) + for page, stale, mask in ((got[2], ins[8], kv_mask), (got[3], ins[9], idx_mask)): + keep = mask.view(-1) == 0 + assert torch.equal(page.view(-1)[keep], stale.view(-1)[keep]) + gk, gks, gv, gvs = insert_ref.read_main_pages(got[2], slots) + rk, rks, rv, rvs = insert_ref.read_main_pages(ref[2], slots) + gi, gis = insert_ref.read_index_pages(got[3], islots) + ri, ris = insert_ref.read_index_pages(ref[3], islots) + for name, a, b in (("K", gk, rk), ("V", gv, rv), ("index_K", gi, ri)): + assert (a != b).float().mean().item() <= reference.CODE_MISMATCH or a.numel() < 1 / reference.CODE_MISMATCH, name + for a, b in ((gks, rks), (gvs, rvs), (gis, ris)): + if a.numel(): + c = compare(a.view(torch.float8_e4m3fn).float(), b.view(torch.float8_e4m3fn).float()) + assert c["cos"] >= reference.COS_MIN and c["rel_l2"] <= reference.PAGE_REL_L2, c + + +def _all_counters_rearmed(): + return all(int((cnt & (host._kernel.EPOCH - 1)).count_nonzero()) == 0 for cnt in host._COUNTERS.values()) + + +@pytest.mark.parametrize("m", [1, 4, 8, 9, 16, 17, 64, 256]) +def test_graph_equals_eager_and_counters_rearm(m): + """Outputs poisoned before every replay; after it the bytes equal the eager call's and every counter is a multiple + of the epoch.""" + require_gfx950() + _, (w, ws), _, _ = weights() + calls = [make(m, 3000 * m + i) for i in range(4)] + bufs = [] + for sa, ra, ins in calls: + q, iq_rows = outputs(m) + bufs.append((q, iq_rows, ins[8].clone(), ins[9].clone())) + eager = [run("entry", m, sa, ra, ins) for sa, ra, ins in calls] + gr = torch.cuda.CUDAGraph() + with torch.cuda.graph(gr): + for (sa, _, ins), (q, iq_rows, kv, idx) in zip(calls, bufs): + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, *ins[:8], kv, idx, *ins[10:], q, + iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + for _ in range(3): + for (_, _, ins), (q, iq_rows, kv, idx) in zip(calls, bufs): + q.fill_(1.5) + iq_rows.fill_(0x33) + kv.copy_(ins[8]) + idx.copy_(ins[9]) + gr.replay() + torch.cuda.synchronize() + for e, b in zip(eager, bufs): + got = (b[0], torch.where(torch.arange(IQ_ROW, device="cuda") < IQW, b[1], e[1]), b[2], b[3]) + assert nbytes_differ(e, got) == [0, 0, 0, 0] + assert _all_counters_rearmed() + + +@pytest.mark.parametrize("m", [1, 8, 13, 16]) +def test_deterministic(m): + require_gfx950() + sa, ra, ins = make(m, 5 * m) + first = run("entry", m, sa, ra, ins) + for _ in range(5): + assert nbytes_differ(first, run("entry", m, sa, ra, ins)) == [0, 0, 0, 0] + + +@pytest.mark.parametrize("ms", [(8, 5), (16, 1), (16, 64)]) +def test_two_streams_at_once(ms): + """Two streams (DBO) call the entry concurrently on their own inputs: each gets its own counters and bytes.""" + require_gfx950() + _, (w, ws), _, _ = weights() + jobs = [make(m, 900 + m) for m in ms] + want = [run("entry", m, *job) for m, job in zip(ms, jobs)] + streams = [torch.cuda.Stream(), torch.cuda.Stream()] + bufs = [] + for m, (sa, ra, ins) in zip(ms, jobs): + q, iq_rows = outputs(m) + bufs.append((q, iq_rows, ins[8].clone(), ins[9].clone())) + torch.cuda.synchronize() + for _ in range(20): + for st, m, (sa, _, ins), (q, iq_rows, kv, idx) in zip(streams, ms, jobs, bufs): + with torch.cuda.stream(st): + kv.copy_(ins[8]) + idx.copy_(ins[9]) + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, *ins[:8], kv, idx, *ins[10:], q, + iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + torch.cuda.synchronize() + for e, b in zip(want, bufs): + assert nbytes_differ(e, b) == [0, 0, 0, 0] + + +def test_zero_rows(): + require_gfx950() + sa, ra, ins = make(4, 11) + _, (w, ws), _, _ = weights() + q = torch.empty((0, QW), dtype=torch.bfloat16, device="cuda") + iq = torch.empty((0, IQW), dtype=torch.float8_e4m3fn, device="cuda") + kv = ins[8].clone() + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, 0, *ins[:5], ins[5][:0], ins[6][:0], ins[7][:0], kv, ins[9], + *ins[10:], q, iq, **KW) + torch.cuda.synchronize() + assert torch.equal(kv, ins[8]) + + +def test_rows_past_the_shuffled_tile_raise(): + sa, ra, ins = make(8, 12) + _, (w, ws), _, _ = weights() + q, iq_rows = outputs(769) + with pytest.raises(ValueError): + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, 769, *ins, q, iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + + +@pytest.mark.parametrize("m", [8, 16]) +def test_next_to_mfma(m): + """The hand-off and its packed arithmetic stay correct while another stream issues MFMAs.""" + require_gfx950() + from tests.mfma_spinner import mfma_spinner + + sa, ra, ins = make(m, 7100 + m) + want = run("chain", m, sa, ra, ins) + spin = mfma_spinner(64) + high = torch.cuda.Stream(priority=-1) + torch.cuda.synchronize() + for _ in range(5): + spin(200000, high) # 2 ms, enough to overlap the entry on another stream + assert nbytes_differ(want, run("entry", m, sa, ra, ins)) == [0, 0, 0, 0] + assert _all_counters_rearmed() + + +def test_graphs_captured_on_one_stream_replay_concurrently(): + """A graph's capture stream is not necessarily its replay stream; capture IDs keep their counters separate.""" + require_gfx950() + _, (w, ws), _, _ = weights() + ms = (8, 16) + jobs = [make(m, 8100 + m) for m in ms] + want = [run("chain", m, *job) for m, job in zip(ms, jobs)] + for m, job in zip(ms, jobs): + run("entry", m, *job) # pool initialization and compilation happen outside capture + capture_stream = torch.cuda.Stream() + replay_streams = [torch.cuda.Stream(), torch.cuda.Stream()] + graphs, bufs, pointers = [], [], [] + for m, (sa, _, ins) in zip(ms, jobs): + q, iq_rows = outputs(m) + kv, idx = ins[8].clone(), ins[9].clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + pointers.append(host._counters(sa[0].device).data_ptr()) + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, *ins[:8], kv, idx, *ins[10:], q, + iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + graphs.append(graph) + bufs.append((q, iq_rows, kv, idx)) + assert pointers[0] != pointers[1] # fail before launching unsafe shared counters + for _ in range(20): + for (_, _, ins), (q, iq_rows, kv, idx) in zip(jobs, bufs): + q.fill_(7) + iq_rows.fill_(5) + iq_rows[:, :IQW].fill_(0x77) + kv.copy_(ins[8]) + idx.copy_(ins[9]) + torch.cuda.synchronize() + for _ in range(3): + for stream, graph in zip(replay_streams, graphs): + with torch.cuda.stream(stream): + graph.replay() + torch.cuda.synchronize() + assert all(nbytes_differ(a, b) == [0, 0, 0, 0] for a, b in zip(want, bufs)) + assert _all_counters_rearmed() + + +@pytest.mark.parametrize("m", [1, 8, 16]) +@pytest.mark.parametrize("initial", [0x7FFFFFF0, -16]) +def test_epoch_counter_wrap(m, initial): + """Cross the signed and unsigned i32 boundaries without losing an arrival or changing output bytes; every head's + counter (one per 128-byte line) takes exactly one epoch per call.""" + require_gfx950() + sa, ra, ins = make(m, 9800 + m) + want = run("chain", m, sa, ra, ins) + run("entry", m, sa, ra, ins) + cnt = host._counters(sa[0].device) + cnt.fill_(initial) + before = cnt.clone() + epoch = host._kernel.EPOCH + lines = torch.arange(cnt.numel(), device="cuda") % host._kernel.CNT_STRIDE == 0 + for iteration in range(2): + got = run("entry", m, sa, ra, ins) + assert nbytes_differ(want, got) == [0, 0, 0, 0] + delta = (cnt.to(torch.int64) - before.to(torch.int64)) & 0xFFFFFFFF + assert torch.equal(delta[lines], torch.full_like(delta[lines], epoch * (iteration + 1))) + assert int(delta[~lines].count_nonzero()) == 0 + assert int((cnt & (epoch - 1)).count_nonzero()) == 0 + + +@pytest.mark.parametrize("which", [10, 11]) # k_scale, v_scale +@pytest.mark.parametrize("m", [8, 16]) +def test_graph_reads_changed_scale(m, which): + """A captured call rebuilds its exact reciprocal tables from the current K / V scale.""" + require_gfx950() + sa, ra, ins = make(m, 19500 + m) + _, (w, ws), _, _ = weights() + run("entry", m, sa, ra, ins) + q, iq_rows = outputs(m) + kv, idx = ins[8].clone(), ins[9].clone() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + mxfp8_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, *ins[:8], kv, idx, *ins[10:], q, + iq_rows.view(torch.float8_e4m3fn)[:, :IQW], **KW) + for scale in (0.61, 1.5, 2.0**-8, 2.0**8, 0.37): + ins[which].fill_(scale) + want = run("chain", m, sa, ra, ins) + q.fill_(-7) + iq_rows.fill_(5) + iq_rows[:, :IQW].fill_(0x77) + kv.copy_(ins[8]) + idx.copy_(ins[9]) + graph.replay() + torch.cuda.synchronize() + assert nbytes_differ((q, iq_rows, kv, idx), want) == [0, 0, 0, 0] + + +@pytest.mark.parametrize("m", [1, 8, 16, 17, 64, 65, 256, 416, 768]) +@pytest.mark.parametrize("one_launch", [True, False]) +def test_qkv_entry_routes_tp2_heads_here(m, one_launch): + """``qkv.mxfp8_gemm_qkv_norm_rope_nvfp4_insert`` at N 4992: this entry where ``one_launch`` allows it, else the + GEMM and the insert; the same bytes either way.""" + require_gfx950() + sa, ra, ins = make(m, 4400 + m) + _, (w, ws), _, _ = weights() + assert chain_entry._one_launch_entry(N, K) is host + want = run("chain", m, sa, ra, ins) + kv, idx = ins[8].clone(), ins[9].clone() + q, iq = chain_entry.mxfp8_gemm_qkv_norm_rope_nvfp4_insert(*sa, w, ws, m, torch.bfloat16, *ins[:8], kv, idx, + *ins[10:], **KW, one_launch=one_launch) + torch.cuda.synchronize() + assert nbytes_differ((q, iq, kv, idx), (want[0], want[1][:, :IQW], want[2], want[3])) == [0, 0, 0, 0] + + +@pytest.mark.parametrize("m", [17, 24, 32, 33, 40, 48]) +def test_one_launch_past_its_window(m): + """The one-launch kernel on the deferred split rows the host leaves to two launches (17-48): the same bytes.""" + require_gfx950() + sa, ra, ins = make(m, 5100 + m) + _, (w, ws), _, _ = weights() + q, iq_rows = outputs(m) + args = (*ins[:8], ins[8].clone(), ins[9].clone(), *ins[10:], q, iq_rows.view(torch.float8_e4m3fn)[:, :IQW]) + host._fused(*sa, w, ws, m, *args, KW["eps"]) + torch.cuda.synchronize() + assert nbytes_differ((q, iq_rows, args[8], args[9]), run("chain", m, sa, ra, ins)) == [0, 0, 0, 0] diff --git a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_compile.py b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_compile.py index ee81710..7fd8612 100644 --- a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_compile.py +++ b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_compile.py @@ -25,10 +25,12 @@ # (id, query heads, KV heads, page, served token counts) QKV_SERVED = [ ("tp4_p128", 16, 1, 128, [4, 12, 100, 400, 1024]), + ("tp2_p128", 32, 2, 128, [4, 12, 100, 400, 1024, 2048]), ] # (id, KV heads, page, served context-row counts) ML_SERVED = [ ("tp4_p128", 1, 128, [4, 64, 384, 4096]), + ("tp2_p128", 2, 128, [4, 64, 384, 4096, 8192]), ] diff --git a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_entry_warm_up.py b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_entry_warm_up.py index 557f7cd..726bbba 100644 --- a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_entry_warm_up.py +++ b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_entry_warm_up.py @@ -11,7 +11,7 @@ import torch -def scenario(): +def scenario(nq, nkv): from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64.kv_norm_rope_insert_multilayer import ( @@ -28,7 +28,7 @@ def spy(*a, **k): MlirCompiler.compile = spy dev, bf16 = torch.device("cuda"), torch.bfloat16 - nq, nkv, nl = 16, 1, 5 + nl = 5 caches = [torch.zeros((8, nkv, 128, 256), dtype=torch.uint8, device=dev) for _ in range(nl)] one = torch.ones((), dtype=torch.float32, device=dev) w = torch.ones(128, dtype=bf16, device=dev) @@ -51,10 +51,12 @@ def spy(*a, **k): return {"late": bad, "reported": warm_up.late_compiles(), "warmed": len(warm_up.warmed())} -def test_first_call_warm_up_compiles_every_variant(tmp_path): +@pytest.mark.parametrize("nq,nkv", [(16, 1), (32, 2)], ids=["tp4", "tp2"]) +def test_first_call_warm_up_compiles_every_variant(tmp_path, nq, nkv): if not torch.cuda.is_available(): pytest.skip("needs a GPU") - proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=600, + proc = subprocess.run([sys.executable, "-m", __name__, str(nq), str(nkv)], capture_output=True, text=True, + timeout=600, env={**os.environ, "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:] result = json.loads(proc.stdout.strip().splitlines()[-1]) @@ -62,4 +64,4 @@ def test_first_call_warm_up_compiles_every_variant(tmp_path): if __name__ == "__main__": - print(json.dumps(scenario())) + print(json.dumps(scenario(int(sys.argv[1]), int(sys.argv[2])))) diff --git a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_norm_rope_insert.py b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_norm_rope_insert.py index ddb1d4b..57c7074 100644 --- a/tests/attention/norm_rope_fp8_insert_d128_rot64/test_norm_rope_insert.py +++ b/tests/attention/norm_rope_fp8_insert_d128_rot64/test_norm_rope_insert.py @@ -2,8 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """Both kernels against the contract in ``norm_rope_fp8_insert_d128_rot64/reference.py``. -Served shapes (README): an M3.1 draft layer at TP4 (16 query heads, 1 KV head, 5 -layers, 128-token pages). The qkv kernel sees the draft's query block (R requests x 4 +Served shapes (README): an M3.1 draft layer at TP4 (16 query heads, 1 KV head) and TP2 +(32 query heads, 2 KV heads), 5 layers, 128-token pages. The qkv kernel sees the draft's query block (R requests x 4 rows); the multi-layer kernel the context rows of a step (the accepted tokens in decode, a prefill chunk). Every case: q and every cache byte against the reference, the bytes the kernel must not write unchanged (skipped slots, other tokens), three @@ -75,6 +75,12 @@ def graph_replay(fn, refill): ("tp4_t1000_p128", 1000, 16, 1, 128, 0.37, None, True), ("tp4_t384_p128_fewer_slots", 384, 16, 1, 128, 1.0, 300, True), ("tp4_t384_noinsert", 384, 16, 1, 128, 1.0, None, False), + ("tp2_t1_p128", 1, 32, 2, 128, 1.0, None, True), + ("tp2_t4_p128", 4, 32, 2, 128, 1.0, None, True), + ("tp2_t1000_p128", 1000, 32, 2, 128, 0.37, None, True), + ("tp2_t2048_p128", 2048, 32, 2, 128, 1.0, None, True), + ("tp2_t384_p128_fewer_slots", 384, 32, 2, 128, 1.0, 300, True), + ("tp2_t384_noinsert", 384, 32, 2, 128, 1.0, None, False), ] @@ -139,6 +145,11 @@ def refill(): ("tp4_n384_p128", 384, 1, 128, False, None), ("tp4_n4096_p128", 4096, 1, 128, True, 4000), ("tp4_n100_p128_dummy", 100, 1, 128, False, 0), + ("tp2_n7_p128", 7, 2, 128, False, None), + ("tp2_n384_p128", 384, 2, 128, False, None), + ("tp2_n4096_p128", 4096, 2, 128, True, 4000), + ("tp2_n8192_p128", 8192, 2, 128, False, None), + ("tp2_n100_p128_dummy", 100, 2, 128, False, 0), ] diff --git a/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_compile.py b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_compile.py index f6faca3..c7bfc81 100644 --- a/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_compile.py +++ b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_compile.py @@ -10,6 +10,8 @@ 2. TP4: the warm-up is what CUDA graph capture launches (1 .. 1024 tokens), eager steps up to 65536 (DBO micro-batches of >= 4096 above 8192); covered with the init call ``warmup_prefill_kernels``, not without it (strict xfail). +3. TP2: with the init call ``warmup_prefill_kernels(32, 2, 2)`` and the capture sizes (up to 2048) nothing compiles + on eager steps up to 65536. """ import flydsl.compiler as flyc @@ -17,10 +19,11 @@ import torch from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import host -from .test_norm_rope_insert import EPS, ROT, TP4, require_gfx950 +from .test_norm_rope_insert import EPS, ROT, TP2, TP4, require_gfx950 PROBES = 6 -GETTERS = ("_get_kernel", "_get_prefill_kernel", "_get_reduce_kernel", "_get_streaming_q16_kernel") +GETTERS = ("_get_kernel", "_get_prefill_kernel", "_get_reduce_kernel", "_get_streaming_q16_kernel", + "_get_vsplit_q32_kernel") def pairs(n): @@ -33,8 +36,20 @@ def reduce_pairs(n, split): return 1 if n <= 32 else (2 if n <= 512 else 4) -def key(n, prefill=False, split=0): +def tp2_key(n, prefill=False, split=0): + if split: + return "_get_reduce_kernel", (*TP2, ROT, 1 if n <= 160 else 2, split) + if n < 8 and not prefill: + return "_get_kernel", (*TP2, ROT, 1) + if n <= 512 and not prefill: + return "_get_vsplit_q32_kernel", () + return "_get_prefill_kernel", (*TP2, ROT, 4) + + +def key(n, prefill=False, split=0, heads=TP4): """The getter and key the host should ask for.""" + if heads == TP2: + return tp2_key(n, prefill, split) if split: return "_get_reduce_kernel", (*TP4, ROT, reduce_pairs(n, split), split) base_pairs = 4 if 489 <= n <= 512 else pairs(n) @@ -50,8 +65,8 @@ class Data: """One batch of the largest size (a single request in contiguous slots); calls run on its first n tokens.""" - def __init__(self, n_max, splits=()): - nq, nkv, niq = self.heads = TP4 + def __init__(self, n_max, splits=(), heads=TP4): + nq, nkv, niq = self.heads = heads dev = "cuda" row = (nq + 2 * nkv + niq + 1) * 128 self.qkv = torch.randn(n_max, row, device=dev).to(torch.bfloat16) @@ -109,12 +124,16 @@ def call(self, data, n, want, **kw): assert self.keys == [want], f"n={n}: the host asked {self.keys}, the mirror says {want}" -# (prefill, split) -> token range of the variant set +# (heads, prefill, split) -> token range of the variant set RANGES = { - "q16_kv1_iq1": (False, 0, range(1, 65537)), + "q16_kv1_iq1": (TP4, False, 0, range(1, 65537)), # the QKV's deferred rows (split 3 up to 128 tokens, split 2 at 129-288) - "q16_kv1_iq1_split2": (False, 2, range(1, 513)), - "q16_kv1_iq1_split3": (False, 3, range(1, 513)), + "q16_kv1_iq1_split2": (TP4, False, 2, range(1, 513)), + "q16_kv1_iq1_split3": (TP4, False, 3, range(1, 513)), + "q32_kv2_iq2": (TP2, False, 0, range(1, 65537)), + "q32_kv2_iq2_prefill": (TP2, True, 0, range(1, 4097)), + "q32_kv2_iq2_split2": (TP2, False, 2, range(1, 1025)), + "q32_kv2_iq2_split3": (TP2, False, 3, range(1, 1025)), } @@ -130,17 +149,17 @@ def warm_and_probe(served, variant): @torch.inference_mode() def test_token_count_not_in_key(name, monkeypatch): require_gfx950() - prefill, split, served = RANGES[name] + heads, prefill, split, served = RANGES[name] kw = dict(prefill=prefill, split=split) - data = Data(served[-1], (split,) if split else ()) + data = Data(served[-1], (split,) if split else (), heads) spy = Spy(monkeypatch) - warm, probe = warm_and_probe(served, lambda n: key(n, **kw)) + warm, probe = warm_and_probe(served, lambda n: key(n, **kw, heads=heads)) for n in warm.values(): - spy.call(data, n, key(n, **kw), **kw) + spy.call(data, n, key(n, **kw, heads=heads), **kw) torch.cuda.synchronize() spy.compiled.clear() for n in probe: - spy.call(data, n, key(n, **kw), **kw) + spy.call(data, n, key(n, **kw, heads=heads), **kw) assert not spy.compiled, f"{name}: n={n} compiled {spy.compiled} after warm-up on {sorted(warm.values())}" @@ -193,22 +212,48 @@ def test_tp4_capture_alone_covers_serving(monkeypatch): def test_variants_are_few(): """The served ranges reach a small set of variants (printed for the README).""" - variants = sorted({key(n, *RANGES[k][:2]) for k in RANGES for n in RANGES[k][2]}) + variants = sorted({key(n, p, sk, h) for h, p, sk, rows in RANGES.values() for n in rows}) print(f"{len(variants)} variants: {variants}") - # main 1 / 4 pairs, the streaming one-pair tile, prefill 4 / 8 pairs; the deferred QKV rows 1 / 2 token pairs x - # split 2 / 3 - assert len(variants) == 9 + # TP4: main 1 / 4 pairs, the streaming one-pair tile, prefill 4 / 8 pairs; the deferred QKV rows 1 / 2 token pairs + # x split 2 / 3. TP2: main 1 pair, the V-split one-pair tile, prefill 4 pairs; reduce 1 / 2 pairs x split 2 / 3 + assert len(variants) == 9 + 7 + + +# vLLM's capture sizes up to the recipe's max_cudagraph_capture_size 2048 +TP2_CAPTURE = [1, 2, 4, *range(8, 256, 8), *range(256, 2049, 16)] +TP2_EAGER = [1, 3, 7, 100, 333, 369, 401, 512, 513, 777, 1024, 1500, 2049, 4096, 8191, 12345, 16383, 32768, 65536] + + +@torch.inference_mode() +def test_tp2_capture_and_warmup_cover_serving(monkeypatch): + """TP2 with the init call ``warmup_prefill_kernels(32, 2, 2)`` and the capture sizes up to 2048.""" + require_gfx950() + data = Data(65536, heads=TP2) + spy = Spy(monkeypatch) + host.warmup_prefill_kernels(*TP2, ROT) + for n in TP2_CAPTURE: + spy.call(data, n, key(n, heads=TP2)) + torch.cuda.synchronize() + spy.compiled.clear() + late = {} + for n in TP2_EAGER: + spy.call(data, n, key(n, heads=TP2)) + if spy.compiled: + late[n] = list(spy.compiled) + spy.compiled.clear() + assert not late, f"tp2: compiled after warm-up: {late}" +@pytest.mark.parametrize("heads", [TP4, TP2], ids=["q16", "q32"]) @torch.inference_mode() -def test_prefill_init_warms_both_pair_counts(monkeypatch): - """One init call covers both prefill pair counts.""" +def test_prefill_init_warms_both_pair_counts(heads, monkeypatch): + """One init call covers every prefill pair count.""" require_gfx950() spy = Spy(monkeypatch) - host.warmup_prefill_kernels(*TP4) - data = Data(32768) + host.warmup_prefill_kernels(*heads) + data = Data(32768, heads=heads) spy.compiled.clear() - for n in (8192, 24576, 32768): + for n in (600, 8192, 24576, 32768) if heads == TP2 else (8192, 24576, 32768): data.call(n) torch.cuda.synchronize() assert not spy.compiled, spy.compiled diff --git a/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_norm_rope_insert.py b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_norm_rope_insert.py index a988a1a..87357ab 100644 --- a/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_norm_rope_insert.py +++ b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_norm_rope_insert.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""The norm / RoPE / NVFP4 insert kernels against ``reference.py`` on the served heads, q16 kv1 iq1 (TP4): per path -(main kernel, prefill variant, split-K form) the contract, CUDA graph replay on new inputs, three bitwise-equal -repeats, and the variants' bytes equal to the main kernel's.""" +"""The norm / RoPE / NVFP4 insert kernels against ``reference.py`` on the served heads, q16 kv1 iq1 (TP4) and q32 kv2 +iq2 (TP2): per path (main kernel, prefill variant, split-K form) the contract, CUDA graph replay on new inputs, three +bitwise-equal repeats, and the variants' bytes equal to the main kernel's.""" import pytest import torch @@ -16,8 +16,8 @@ D, BLK, WIDTH = ref.D, ref.BLK, ref.WIDTH EPS = 1e-6 ROT = 64 -HEADS = {"q16_kv1_iq1": (16, 1, 1)} -TP4 = HEADS["q16_kv1_iq1"] +HEADS = {"q16_kv1_iq1": (16, 1, 1), "q32_kv2_iq2": (32, 2, 2)} +TP4, TP2 = HEADS["q16_kv1_iq1"], HEADS["q32_kv2_iq2"] DSPARK = ([3 + 9 * i for i in range(43)], [4] * 43) # 172 decode rows (4 per request) # (starts, lens, pad) CASES = { @@ -208,7 +208,7 @@ def test_main_kernel_contract(heads, case, scales): """The main kernel (1 token pair per wave up to 512 tokens, 4 above).""" require_gfx950() batch = Batch(*CASES[case], HEADS[heads], scales) - check(batch, batch.run(), batch.reference(), f"{heads} {case} pairs {host._token_pairs_per_wave(batch.n)}") + check(batch, batch.run(), batch.reference(), f"{heads} {case} {host._launcher(batch.n, HEADS[heads]).kernel_name}") @pytest.mark.parametrize("m", [512, 513, 1024, 2048, 4095]) @@ -224,16 +224,20 @@ def test_streaming_q_store_contract(m): def test_insert_blocker(): - """The served layer fits; another head set, head dim, RoPE, index-q dtype or cos / sin cache gets a reason, the - warm-up compiles nothing for it and the entries assert it (host only).""" - assert host.insert_blocker(*TP4, ROT) is None + """The served layers (TP4, TP2) fit; another head set, head dim, RoPE, index-q dtype or cos / sin cache gets a + reason, the warm-up compiles nothing for it and the entries assert it (host only).""" cache = torch.zeros(8, ROT, dtype=torch.bfloat16) - assert host.insert_blocker(*TP4, ROT, head_dim=D, index_head_dim=D, neox=True, - index_q_dtype=torch.float8_e4m3fn, cos_sin_cache=cache) is None + for heads in (TP4, TP2): + assert host.insert_blocker(*heads, ROT) is None + assert host.insert_blocker(*heads, ROT, head_dim=D, index_head_dim=D, neox=True, + index_q_dtype=torch.float8_e4m3fn, cos_sin_cache=cache) is None for args, kw, word in [ ((64, 4, 4, ROT), {}, "heads"), - ((32, 2, 2, ROT), {}, "heads"), + ((32, 2, 4, ROT), {}, "heads"), # TP2 with the context-parallel indexer + ((32, 1, 1, ROT), {}, "heads"), ((16, 1, 4, ROT), {}, "heads"), + ((*TP2, 32), {}, "RoPE"), + ((*TP2, ROT), dict(index_q_dtype=torch.bfloat16), "index-q"), ((*TP4, 32), {}, "RoPE"), ((*TP4, ROT), dict(neox=False), "NeoX"), ((*TP4, ROT), dict(head_dim=64), "head dim"), @@ -261,10 +265,15 @@ def test_insert_blocker(): def test_streaming_q_store_scope(): - """Only the four-pair main kernel uses streaming stores.""" + """TP4: only the four-pair main kernel uses streaming stores. TP2: every tile from 8 rows does, and no row count + reaches the q16 tiles.""" for pairs, want in [(4, True), (1, False)]: exe = host._get_kernel(*TP4, ROT, pairs) assert exe.kernel_name.endswith("_qnt") == want + for n in (1, 289, 300, 384, 512, 513, 1537, 24576, 32768): + for prefill in (False, True): + name = host._launcher(n, TP2, prefill).kernel_name + assert "_q32_kv2_iq2_" in name and (name.endswith("_qnt") or n < 8), (n, prefill, name) @pytest.mark.parametrize("case", list(PREFILL_CASES)) @@ -348,34 +357,98 @@ def test_tp4_streaming_onepair_bytes(m, monkeypatch): assert_bitwise(got, batch.run(), f"streaming one-pair {m}") +def tp2_main_one_pair(monkeypatch): + """Every TP2 call on the plain one-pair main kernel (no nt stores).""" + monkeypatch.setattr(host, "_tp2_launcher", lambda n, prefill: host._get_kernel(*TP2, ROT, 1)) + + +# every TP2 tier and its edges (the one-pair main kernel to 7 rows, the V-split one-pair tile to 512, the prefill +# variant above) +TP2_ROWS = [2, 7, 8, 9, 64, 172, 289, 384, 450, 511, 512, 513, 1024, 1537, 4097] + + +@pytest.mark.parametrize("m", TP2_ROWS) +@torch.inference_mode() +def test_tp2_tier_bytes(m, monkeypatch): + """TP2 heads at every tier: the contract, and (as the prefill hint) the one-pair main kernel's bytes; odd rows, a + zero row, stale pages, disabled index slots and a padding tail.""" + require_gfx950() + batch = Batch([125], [m - 1], 1, TP2, (0.37, 0.21), seed=m, index_skip=0.01) + batch.qkv[0].zero_() + got = batch.run() + check(batch, got, batch.reference(), f"tp2 {host._launcher(m, TP2).kernel_name}") + assert_bitwise(got, batch.run(prefill=True), f"tp2 {m} prefill hint") + tp2_main_one_pair(monkeypatch) + assert_bitwise(got, batch.run(), f"tp2 {m}") + + +def dspark7_case(n_req, lead, pad): + """DSpark-7 decode rows: a ``lead``-token request, then ``n_req`` requests of 8 tokens whose first position + walks every octet offset (runs of 8, 7 + 1, ..., 1 + 7 tokens).""" + starts = [3001 + 128 * r + r % 8 for r in range(n_req)] + return [1000, *starts] if lead else starts, ([lead] if lead else []) + [8] * n_req, pad + + +@pytest.mark.parametrize("n_req,lead,pad", [(1, 0, 0), (8, 0, 0), (8, 3, 5), (31, 1, 0), (32, 0, 0), (40, 3, 0), + (47, 5, 3), (52, 0, 0), (60, 7, 9)]) +@torch.inference_mode() +def test_tp2_dspark7_bytes(n_req, lead, pad, monkeypatch): + """TP2 decode as served (8 rows per request, every octet offset, odd leading requests that flip which wave half + starts a run, graph padding): the contract, and the one-pair main kernel's bytes.""" + require_gfx950() + batch = Batch(*dspark7_case(n_req, lead, pad), TP2, (0.37, 0.21), seed=n_req + lead, index_skip=0.01) + got = batch.run() + check(batch, got, batch.reference(), f"tp2 dspark7 {host._launcher(batch.n, TP2).kernel_name}") + tp2_main_one_pair(monkeypatch) + assert_bitwise(got, batch.run(), f"tp2 dspark7 {n_req} {lead} {pad}") + + +@pytest.mark.parametrize("case", [*PREFILL_CASES, "n32768_req8"]) +@torch.inference_mode() +def test_tp2_prefill_variant_bytes(case, monkeypatch): + """TP2 heads at prefill sizes (up to a 32768-token chunk): the contract and the one-pair main kernel's bytes.""" + require_gfx950() + batch = Batch(*(PREFILL_CASES.get(case) or prefill_case(32768, 8)), TP2, (0.37, 0.21), seed=2, index_skip=0.01) + got = batch.run() + check(batch, got, batch.reference(), f"tp2 prefill variant {case}") + tp2_main_one_pair(monkeypatch) + assert_bitwise(got, batch.run(), case) + + # (tokens, split count): the QKV GEMM's deferred rows (split 3 at 1-128 tokens, 2 at 193-288) and other split counts REDUCE_CASES = [(1, 3), (7, 2), (8, 3), (16, 3), (32, 4), (64, 4), (68, 4), (132, 2), (172, 8), (256, 3), (600, 3)] +# TP2: both tiers (one pair to 160 rows, two above) at splits 1-8 +TP2_REDUCE_CASES = [(1, 3), (8, 2), (32, 3), (64, 4), (160, 2), (161, 3), (256, 2), (384, 1), (449, 3), (600, 8)] +REDUCE = [(TP4, m, sk) for m, sk in REDUCE_CASES] + [(TP2, m, sk) for m, sk in TP2_REDUCE_CASES] +REDUCE_IDS = [f"q{h[0]}-m{m}-s{sk}" for h, m, sk in REDUCE] -@pytest.mark.parametrize("m,split_k", REDUCE_CASES) +@pytest.mark.parametrize("heads,m,split_k", REDUCE, ids=REDUCE_IDS) @torch.inference_mode() -def test_splitk_reduce_contract(m, split_k): +def test_splitk_reduce_contract(heads, m, split_k): """The split-K reduce kernel, the last token graph padding.""" require_gfx950() - batch = Batch([3], [m - 1], 1, TP4, (0.5, 2.0), seed=m) + batch = Batch([3], [m - 1], 1, heads, (0.5, 2.0), seed=m) slab = make_slab(batch, split_k, m) check(batch, batch.run_reduce(slab), batch.reference(slab=slab), f"reduce m{m} s{split_k}") @pytest.mark.parametrize("iq_mode", ["rank", "cols"]) @pytest.mark.parametrize("case", ["dspark", "prefill600"]) +@pytest.mark.parametrize("heads", list(HEADS)) @torch.inference_mode() -def test_index_q_out_view(iq_mode, case): - """index_q_out as a slice of a bigger buffer (the context-parallel indexer's - [4, n, 128] buffer, or a column slice); bytes outside it stay untouched.""" +def test_index_q_out_view(heads, iq_mode, case): + """index_q_out as a slice of a bigger buffer (a per-rank [ranks, n, niq * 128] buffer, or a column slice); bytes + outside it stay untouched.""" require_gfx950() starts, lens = DSPARK if case == "dspark" else ([7], [600]) - batch = Batch(starts, lens, 5, TP4) + batch = Batch(starts, lens, 5, HEADS[heads]) n, w = batch.n, batch.niq * D if iq_mode == "rank": - buf = torch.full((4, n, w), 0x7F, dtype=torch.uint8, device="cuda") - iq = buf[2] + ranks = 64 // batch.nq + buf = torch.full((ranks, n, w), 0x7F, dtype=torch.uint8, device="cuda") + iq = buf[ranks - 2] else: buf = torch.full((n, 3 * w), 0x7F, dtype=torch.uint8, device="cuda") iq = buf[:, w : 2 * w] @@ -386,10 +459,11 @@ def test_index_q_out_view(iq_mode, case): assert bool((buf == 0x7F).all()), "bytes outside index_q_out written" +@pytest.mark.parametrize("heads", list(HEADS)) @torch.inference_mode() -def test_64k_tokens(): +def test_64k_tokens(heads): require_gfx950() - batch = Batch([0, 777], [32768, 32768], 0, TP4) + batch = Batch([0, 777], [32768, 32768], 0, HEADS[heads]) check(batch, batch.run(), batch.reference(), "64K") @@ -405,15 +479,17 @@ def test_all_padding(heads): @pytest.mark.parametrize("variant", ["main", "prefill"]) +@pytest.mark.parametrize("heads", list(HEADS)) @torch.inference_mode() -def test_block_outer_pages(variant, monkeypatch): +def test_block_outer_pages(heads, variant, monkeypatch): """Each block holds the layers' main and index pages in layer order ([M0 I0 M1 I1]): writing layer 0 through its strided views stores the bytes of dense caches and leaves layer 1's pages untouched.""" require_gfx950() - if variant == "main": - monkeypatch.setattr(host, "PREFILL_TOKENS", 1 << 30) # the main kernel at this size - batch = Batch(*PREFILL_CASES["n4097_req3"], TP4, (0.37, 0.21), seed=1, index_skip=0.01) + if variant == "main": # the main kernel at this size + monkeypatch.setattr(host, "PREFILL_TOKENS", 1 << 30) + tp2_main_one_pair(monkeypatch) + batch = Batch(*PREFILL_CASES["n4097_req3"], HEADS[heads], (0.37, 0.21), seed=1, index_skip=0.01) nb = batch.kv_full.shape[0] kv_page, idx_page = batch.main, BLK * WIDTH block = torch.randint(0, 256, (nb, 2 * (kv_page + idx_page)), dtype=torch.uint8, device="cuda") @@ -446,6 +522,13 @@ def test_block_outer_pages(variant, monkeypatch): "reduce_q16_pairs2_s3": ("reduce3", TP4, ([3], [127], 1)), "reduce_q16_pairs2_s2": ("reduce2", TP4, ([125], [150], 1)), "reduce_q16_pairs1_s2": ("reduce2", TP4, ([125, 254], [123, 132], 1)), + "vsplit_q32": ("main", TP2, CASES["dspark172_pad20"]), + "vsplit_q32_dspark7": ("main", TP2, dspark7_case(47, 5, 3)), + "prefill_pairs4_q32": ("main", TP2, PREFILL_CASES["n4097_req3"]), + "prefill_hint_q32": ("prefill", TP2, ([0, 300], [200, 250], 0)), + "reduce_q32_pairs1_s3": ("reduce3", TP2, ([3], [31], 1)), + "reduce_q32_pairs2_s2": ("reduce2", TP2, ([125, 254], [123, 132], 1)), + "reduce_q32_pairs2_s3": ("reduce3", TP2, ([125], [599], 1)), } @@ -523,13 +606,13 @@ def test_deterministic(name): # --- the variants write the main kernel's bytes ----------------------------------------- -@pytest.mark.parametrize("m,split_k", REDUCE_CASES) +@pytest.mark.parametrize("heads,m,split_k", REDUCE, ids=REDUCE_IDS) @torch.inference_mode() -def test_splitk_reduce_bitwise(m, split_k): +def test_splitk_reduce_bitwise(heads, m, split_k): """The reduce kernel equals the reduce (``((0 + s0) + s1) ...`` in fp32, bf16 store) followed by the main kernel, bit for bit.""" require_gfx950() - batch = Batch([3], [m - 1], 1, TP4, (0.5, 2.0), seed=m) + batch = Batch([3], [m - 1], 1, heads, (0.5, 2.0), seed=m) slab = make_slab(batch, split_k, m) fused = batch.run_reduce(slab) batch.qkv = ref.splitk_reduce(slab) diff --git a/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_qkv_chain_tp2.py b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_qkv_chain_tp2.py new file mode 100644 index 0000000..32766b3 --- /dev/null +++ b/tests/attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_qkv_chain_tp2.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``qkv.mxfp8_gemm_qkv_norm_rope_nvfp4_insert`` at the TP2 QKV shape (N 4992 = (32 + 2 * 2 + 2 + 1) * 128, K 6144) +over 1-4096 rows, shuffled-code and row-major: bit for bit the explicit calls (the GEMM's deferred split-K slab into +``splitk_reduce_norm_rope_nvfp4_insert``, or its bf16 rows into ``norm_rope_nvfp4_insert``), the rows the one-launch +entry ``mxfp8_qkv_norm_rope_nvfp4_insert_n4992_k6144_rot64_p128`` takes included.""" + +import functools + +import pytest +import torch +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import ( + norm_rope_nvfp4_insert, + splitk_reduce_norm_rope_nvfp4_insert, +) +from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import qkv as chain +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common.reference import quantize_mxfp8 + +from .test_norm_rope_insert import TP2, Batch, assert_bitwise, prefill_case, require_gfx950 + +N, K = 4992, 6144 +KW = dict(num_heads=32, num_kv_heads=2, num_index_heads=2, eps=1e-6) +ROWS = [1, 8, 16, 17, 32, 64, 65, 128, 129, 192, 256, 288, 289, 369, 401, 512, 513, 1024, 1537, 2048, 4096] + + +@functools.cache +def weights(): + gen = torch.Generator(device="cuda").manual_seed(39) + w = (torch.randn(N, K, device="cuda", generator=gen) / 64).to(torch.bfloat16) + return g.preshuffle_mxfp8_weight(*quantize_mxfp8(w)) + + +def codes_for(m, seed): + """The producer's codes for ``m`` rows: the GEMM row's tile, or row-major codes where it is 0.""" + gen = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(m, K, device="cuda", generator=gen).to(torch.bfloat16) + tile = g.prequant_code_tile_rows(m, N, K, torch.bfloat16) + codes, scales = g.mxfp8_e4m3_quantize_shuffled(x, tile or 16) + if not tile: + codes = quantize_mxfp8(x)[0].contiguous() + return codes, scales + + +def insert_args(m, seed): + """Decode rows as requests of 8 tokens (DSpark), 512 and more as three prefill requests.""" + if m >= 512: + case = prefill_case(m, 3) + else: + lens = [8] * (m // 8) + ([m % 8] if m % 8 else []) + case = ([1000 + 211 * r for r in range(len(lens))], lens, 0) + batch = Batch(*case, TP2, (0.37, 0.61), seed=seed, index_skip=0.01) + return batch, (*batch.w, batch.cs, batch.pos, batch.slots, batch.islots) + + +def explicit(codes, scales, m, batch, ins): + w, ws = weights() + q, iq, kv, idx = batch.outputs() + args = (*ins, batch.kv(kv), batch.idx(idx), batch.ks, batch.vs) + if g.prequant_code_tile_rows(m, N, K, torch.bfloat16): + out = g.mxfp8_gemm_shuffled(codes, scales, w, ws, torch.bfloat16, m, defer_reduce=True) + else: + out = g.mxfp8_gemm(codes, scales, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + if isinstance(out, g.SkPartials): + rows = out.tmp.shape[1] + q, iq = q.new_empty(rows, q.shape[1]), iq.new_empty(rows, iq.shape[1]) + splitk_reduce_norm_rope_nvfp4_insert(out.tmp, out.split_k, *args, q, iq, **KW) + else: + norm_rope_nvfp4_insert(out.reshape(m, N), *args, q, iq, **KW) + return q, iq, kv, idx + + +def entry(codes, scales, m, batch, ins): + w, ws = weights() + _, _, kv, idx = batch.outputs() + args = (*ins, batch.kv(kv), batch.idx(idx), batch.ks, batch.vs) + q, iq = chain.mxfp8_gemm_qkv_norm_rope_nvfp4_insert(codes, scales, w, ws, m, torch.bfloat16, *args, **KW) + return q, iq, kv, idx + + +@pytest.mark.parametrize("m", ROWS) +@torch.inference_mode() +def test_tp2_chain_bitwise_explicit_calls(m): + require_gfx950() + codes, scales = codes_for(m, 7 * m + 1) + batch, ins = insert_args(m, m) + a = explicit(codes, scales, m, batch, ins) + b = entry(codes, scales, m, batch, ins) + torch.cuda.synchronize() + assert a[0].shape == b[0].shape == (a[0].shape[0], 4096) and b[1].shape[1] == 256 + assert_bitwise(a, b, f"tp2 chain m{m}") + + +@torch.inference_mode() +def test_tp2_zero_rows(): + require_gfx950() + codes, scales = codes_for(8, 5) + batch, ins = insert_args(8, 5) + w, ws = weights() + q, iq = chain.mxfp8_gemm_qkv_norm_rope_nvfp4_insert( + codes[:0], scales, w, ws, 0, torch.bfloat16, *ins, batch.kv(batch.kv_full.clone()), + batch.idx(batch.idx_full.clone()), batch.ks, batch.vs, **KW) # fmt: skip + assert q.shape == (0, 4096) and iq.shape == (0, 256) diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/plan.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/plan.py index 315e05b..aea912e 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/plan.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/plan.py @@ -11,8 +11,8 @@ XCDS = 8 # MI355X XCDs (``device.initialize_xcd_topology``) # The served configurations per GPU: (KV heads, compute type, decode_query_len, requests). 1 KV head = 16 query heads -# (TP4) computes in bf16 or exact fp16. decode_query_len 1 = no draft, 4 = DSpark with 3 drafts, 8 = DSpark with 7 -# drafts (up to 256 requests = the 2048-token FULL decode graphs). +# (TP4), 2 KV heads = 32 query heads (TP2); each computes in bf16 or exact fp16. decode_query_len 1 = no draft, 4 = +# DSpark with 3 drafts, 8 = DSpark with 7 drafts (TP4 up to 256 requests = the 2048-token FULL decode graphs). SERVED = { "kv1_bf16_ql1": (1, torch.bfloat16, 1, 256), "kv1_bf16_ql4": (1, torch.bfloat16, 4, 128), @@ -20,38 +20,54 @@ "kv1_f16_ql1": (1, torch.float16, 1, 256), "kv1_f16_ql4": (1, torch.float16, 4, 128), "kv1_f16_ql8": (1, torch.float16, 8, 256), + "kv2_bf16_ql1": (2, torch.bfloat16, 1, 128), + "kv2_bf16_ql4": (2, torch.bfloat16, 4, 128), + "kv2_bf16_ql8": (2, torch.bfloat16, 8, 128), + "kv2_f16_ql1": (2, torch.float16, 1, 128), + "kv2_f16_ql4": (2, torch.float16, 4, 128), + "kv2_f16_ql8": (2, torch.float16, 8, 128), } def plan(reqs, ql, hkv, compute, quant, xcd_count=XCDS, topk=16, cus=CUS): """``host._get_kernel``'s key ``(splits, topk, xcds, quant, exact_fp16, tail)`` for an automatic launch - (``splits`` and ``tail_splits`` None).""" + (``splits`` and ``tail_splits`` None); the q8 variants append their flags (and hkv 2 at 2 KV heads).""" exact_fp16 = compute == torch.float16 rows = reqs * ql xcds = xcd_count if xcd_count and xcd_count > 1 else 0 - tail = host._use_tail(exact_fp16, quant, ql, hkv, topk, xcds, reqs) - if not tail and exact_fp16 and hkv > 1 and ql > 1 and rows <= 4 * ql: + tail = host._use_tail(exact_fp16, quant, ql, hkv, topk, xcds, reqs, cus) + q8 = hkv in host.Q8_KV_HEADS and ql == 8 and topk == 16 + row_xcd = (not tail and q8 and hkv in host.ROW_XCD_MAX_ROWS and xcds == 8 + and rows <= host.ROW_XCD_MAX_ROWS[hkv]) + if not tail and exact_fp16 and hkv > host.PLAIN_GRID_MIN_KV and ql > 1 and rows <= 4 * ql: xcds = 0 + row_spread = (not tail and not row_xcd and xcds > 0 and 1 < ql < 8 + and rows <= host.ROW_SPREAD_MAX_ROWS.get(hkv, 0)) if tail: one = 1 if reqs in host._TAIL_ONE_SPLIT.get(ql, ()) else None splits = host.tail_schedule(rows * hkv, topk, cus, xcds, one)[2] else: - grid_x = -(-rows // (xcds * ql)) * xcds * ql if xcds else rows + group = xcds if row_xcd or row_spread else xcds * ql + grid_x = -(-rows // group) * group if xcds else rows wg_rows = grid_x * hkv if wg_rows > host.MAX_SPLIT_ROWS: splits = 1 + elif row_xcd: + splits = 4 else: tuned = host._tuned_splits(exact_fp16, quant, ql, hkv, topk, reqs, xcds) splits = tuned if tuned is not None else host.decode_splits(rows * hkv, topk, cus) key = (splits, topk, xcds, quant, exact_fp16, tail) - if not tail and hkv == 1 and ql == 8 and topk == 16 and xcds == 8 and rows <= 32: - return (*key, False, True) - if (not tail and not exact_fp16 and hkv == 1 and ql == 8 and topk == 16 - and xcds == 8 and splits in (1, 2, 4) and grid_x * splits <= cus): - return (*key, False, False, True) - if (not tail and not exact_fp16 and hkv == 1 and ql == 8 and topk == 16 - and xcds == 8 and splits in (2, 4) and grid_x <= 192): - return (*key, False, False, False, True) + q8 = q8 and xcds == 8 + hk = (2,) if q8 and hkv == 2 else () + if row_xcd: + return (*key, False, True, *((False, False) + hk if hk else ())) + if not tail and not exact_fp16 and q8 and splits in (1, 2, 4) and grid_x * hkv * splits <= cus: + return (*key, False, False, True, *((False,) + hk if hk else ())) + if not tail and not exact_fp16 and q8 and splits in (2, 4) and grid_x * hkv <= 192: + return (*key, False, False, False, True, *hk) + if row_spread: + return (*key, False, False, False, False, 1, True) return key diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_fp8_decode.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_fp8_decode.py index 988db2b..aa135cc 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_fp8_decode.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_fp8_decode.py @@ -46,7 +46,7 @@ def unit_scales(case): @pytest.mark.parametrize( "hkv,ql,reqs,splits,xcd", [(1, 8, 1, 4, 8), (1, 8, 5, 4, 8), (1, 8, 9, 2, 8), (1, 8, 24, 1, 8), (1, 8, 40, 2, 8), (1, 8, 4, 8, 0), - (1, 4, 6, 3, 8)], + (1, 4, 6, 3, 8), (2, 8, 1, 4, 8), (2, 8, 3, 1, 8), (2, 8, 9, 2, 8), (2, 8, 20, 1, 8), (2, 4, 6, 3, 8)], ) def test_contract_fp8(hkv, ql, reqs, splits, xcd): require_gfx950() diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode.py index 2605553..537b002 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode.py @@ -3,8 +3,10 @@ """``index_decode.nvfp4_index_sparse_decode``, the decode block (index top-16 + sparse decode): the route choice, and on one GPU both routes (``fused``: the scorer + the attention's in-kernel top-16; ``chain``: scorer + selector, then the attention) bit for bit the three-launch chain (``nvfp4_index_score_top16`` + ``nvfp4_sparse_decode``), ids and -output, bf16 and MXFP8 outputs, bf16 and fp16 compute, at the served TP4 shapes (one KV head x 16 heads, 4 / 8 rows per request, 1-64 -requests, 32K-192K). The four-GPU route: ``test_index_decode_ranks4.py``. Warm-up: ``test_index_decode_warm_up.py``.""" +output, bf16 and MXFP8 outputs, bf16 and fp16 compute, at the served shapes (TP4: one KV / index head x 16 heads, TP2: +two, 4 / 8 rows per request, 1-64 requests, 32K-192K). The four-GPU route: ``test_index_decode_ranks4.py``; the +two-GPU one: ``test_index_decode_ranks2.py``. Warm-up: +``test_index_decode_warm_up.py``.""" from types import SimpleNamespace import pytest @@ -22,14 +24,22 @@ def test_route(): one = IndexDecodeTop16("cpu", query_len=8, max_reqs=64, max_seq_len=MAX, warm_up=False) assert blk.route(None, 32, 8, 1, 1) == "fused" and blk.route(one, 32, 8, 1, 1) == "fused" - assert blk.route(None, 33, 8, 1, 1) == "chain" # past FUSED_MAX_ROWS + assert blk.route(None, 33, 8, 1, 1) == "chain" # past SELECT_MAX_ROWS assert blk.route(None, 4, 8, 1, 1, torch.float16) == "fused" assert blk.route(None, 4, 8, 1, 1, torch.float8_e4m3fn) == "chain" # fp8 selects in no attention kernel assert blk.route(None, 4, 8, 4, 1) == "chain" # index heads != KV heads + assert blk.route(None, 128, 8, 2, 2) == "fused" and blk.route(None, 129, 8, 2, 2) == "chain" # TP2 one.split = SimpleNamespace(query_len=8, max_rows=256 * 8) n = decode_top16.SPLIT_MIN_REQS[8] # split4 from n requests on (decode_top16.route) assert blk.route(one, n - 1, 8, 1, 1) == "fused" and blk.route(one, n, 8, 1, 1) == "split4" assert blk.route(one, n, 8, 1, 1, split=False) == "fused" + two = IndexDecodeTop16("cpu", query_len=8, max_reqs=64, max_seq_len=MAX, num_index_heads=2, warm_up=False) + two.split2 = SimpleNamespace(query_len=8, max_rows=64 * 8) + n2 = decode_top16.SPLIT2_MIN_REQS[8] # split2 from n2 requests on (TP2: two index heads per rank) + assert blk.route(two, n2 - 1, 8, 2, 2, torch.float16) == "fused" + assert blk.route(two, n2, 8, 2, 2, torch.float16) == "split2" + assert blk.route(two, n2, 8, 2, 2, torch.float16, split=False) == "fused" + assert blk.route(two, 65, 8, 2, 2, torch.float16) == "fused" and blk.route(two, n2, 8, 1, 1) == "fused" def three_launch_chain(c, init, local, ids, out, quant, compute=torch.bfloat16): @@ -40,25 +50,26 @@ def three_launch_chain(c, init, local, ids, out, quant, compute=torch.bfloat16): decode_query_len=c["ql"], quant_out=quant, compute_type=compute) +@pytest.mark.parametrize("hkv", [1, 2], ids=["kv1", "kv2"]) @pytest.mark.parametrize("compute", [torch.bfloat16, torch.float16], ids=["bf16", "f16"]) @pytest.mark.parametrize("ql", [8, 4]) @pytest.mark.parametrize("reqs", [1, 2, 3, 4, 5, 8, 17, 32, 33, 64]) @pytest.mark.parametrize("ctx", [32768, 196608]) @pytest.mark.parametrize("quant", [False, True]) -def test_one_gpu_block_matches_three_launch_chain(ql, reqs, ctx, quant, compute): +def test_one_gpu_block_matches_three_launch_chain(ql, reqs, ctx, quant, compute, hkv): require_gfx950() - c = build([max(1, ctx - 4093 * r) for r in range(reqs)], ql, seed=reqs * 7 + ql + ctx % 97) + c = build([max(1, ctx - 4093 * r) for r in range(reqs)], ql, seed=reqs * 7 + ql + ctx % 97, hkv=hkv) rows = c["rows"] res = [] for new in (False, True): - ids = torch.full((1, rows + 3, 16), -7, dtype=torch.int32, device="cuda") - out = torch.full((rows, 16, 128), 7.0, dtype=torch.bfloat16, device="cuda") - q = (*quant_buffers(rows, 2048), 16) if quant else None + ids = torch.full((hkv, rows + 3, 16), -7, dtype=torch.int32, device="cuda") + out = torch.full((rows, 16 * hkv, 128), 7.0, dtype=torch.bfloat16, device="cuda") + q = (*quant_buffers(rows, 2048 * hkv), 16) if quant else None if new: r = blk.nvfp4_index_sparse_decode(None, c["iq"], c["icache"], c["bt"], c["seq"], 2, 3, c["q"], c["kv"], ids, c["bt"], c["seq"], 128**-0.5, out, k_scale=None, v_scale=None, decode_query_len=ql, max_seq_len=MAX, quant_out=q, compute_type=compute) - assert r == ("fused" if rows <= blk.FUSED_MAX_ROWS else "chain") + assert r == ("fused" if rows <= ah.SELECT_MAX_ROWS[hkv] else "chain") else: three_launch_chain(c, 2, 3, ids[:, :rows], out, q, compute) torch.cuda.synchronize() diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks2.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks2.py new file mode 100644 index 0000000..669b7bb --- /dev/null +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks2.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The decode block on two GPUs (``IndexDecodeTop16`` with a two-rank group, the replicated index cache, TP2: 2 index / +KV heads per rank): each rank's ids and attention output (bf16 or MXFP8) bit for bit the one-GPU three-launch chain +on its own heads, on both sides of the request-count threshold (one-GPU ``fused`` below, ``split2`` from +``SPLIT2_MIN_REQS``), in graph replay at other lengths too.""" +import torch + +from ...multi_gpu import run_on_2_gpus +from .test_index_decode import three_launch_chain +from .test_index_select import MAX, build +from .test_sparse_decode import quant_buffers, require_gfx950 + + +def _body(rank, errors): + import torch.distributed as dist + from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import index_decode as blk + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import decode_top16 + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.decode_top16 import IndexDecodeTop16 + + dev = torch.device("cuda", rank) + for compute, ql in ((torch.float16, 8), (torch.float16, 4), (torch.bfloat16, 8)): + index = IndexDecodeTop16(dev, query_len=ql, max_reqs=64, max_seq_len=MAX, num_index_heads=2, rank=rank, + group=dist.group.WORLD, warm_up=False) + m = decode_top16.SPLIT2_MIN_REQS[ql] + for reqs in (1, m - 1, m, 17, 64): + for quant in (False, True): + c = build([max(1, 131072 - 3001 * r) for r in range(reqs)], ql, seed=reqs * 13 + ql, hkv=2) + g = torch.Generator(device="cuda").manual_seed(reqs + 100 * ql) + iq4 = (torch.randn(c["rows"], 4, 128, device="cuda", generator=g) * 0.5).to(torch.float8_e4m3fn) + c["iq"] = iq4[:, 2 * rank : 2 * rank + 2].contiguous() + rows = c["rows"] + want_ids = torch.full((2, rows, 16), -7, dtype=torch.int32, device="cuda") + want = torch.full((rows, 32, 128), 7.0, dtype=torch.bfloat16, device="cuda") + wq = (*quant_buffers(rows, 4096), 16) if quant else None + three_launch_chain(c, 2, 3, want_ids, want, wq, compute) + ids = torch.full_like(want_ids, -7) + out = torch.full_like(want, 7.0) + q = (*quant_buffers(rows, 4096), 16) if quant else None + + def step(): + return blk.nvfp4_index_sparse_decode( + index, c["iq"], c["icache"], c["bt"], c["seq"], 2, 3, c["q"], c["kv"], ids, c["bt"], + c["seq"], 128**-0.5, out, k_scale=None, v_scale=None, decode_query_len=ql, max_seq_len=MAX, + quant_out=q, compute_type=compute) + + def same(): + ok = torch.equal(ids, want_ids) + if quant: + return ok and torch.equal(q[0], wq[0]) and torch.equal(q[1], wq[1]) + return ok and torch.equal(out.view(torch.int16), want.view(torch.int16)) + + r = step() + torch.cuda.synchronize() + tag = (rank, compute, ql, reqs, quant, r) + if r != ("split2" if reqs >= m else "fused"): + errors.append(f"{tag}: route") + if not same(): + errors.append(f"{tag}: ids / output differ") + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + step() + c["seq"].copy_(torch.tensor([max(1, 70000 - 999 * i) for i in range(reqs)], dtype=torch.int32)) + graph.replay() + torch.cuda.synchronize() + three_launch_chain(c, 2, 3, want_ids, want, wq, compute) + torch.cuda.synchronize() + if not same(): + errors.append(f"{tag}: replay ids / output differ") + + +def test_index_decode_two_gpus(): + require_gfx950() + run_on_2_gpus(_body) diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_warm_up.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_warm_up.py index 38b0d59..3f96824 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_warm_up.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_warm_up.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """The decode block's warm-up (``IndexDecodeTop16`` built + ``index_decode.warm_up``) compiles every variant the -block launches on one GPU: afterwards no step of 1-64 requests (and 65-256 for the attention) at 32K / 192K / 1M, bf16 -and MXFP8 output, bf16 and fp16 compute, compiles.""" +block launches on one GPU, 1 (TP4) or 2 (TP2) KV / index heads: afterwards no step of 1-64 requests (and up to 256 / +128 for the attention) at 32K / 192K / 1M, bf16 and MXFP8 output, bf16 and fp16 compute, compiles.""" import json import os import subprocess @@ -12,7 +12,7 @@ import torch -def scenario(ql, compute): +def scenario(ql, compute, hkv): from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import host @@ -30,43 +30,44 @@ def spy(*a, **kw): return original(*a, **kw) MlirCompiler.compile = spy - dev, hmax, max_reqs = torch.device("cuda"), 1 << 20, 256 - index = IndexDecodeTop16(dev, query_len=ql, max_reqs=max_reqs, max_seq_len=hmax) - blk.warm_up(dev, num_heads=16, num_kv_heads=1, query_len=ql, max_reqs=max_reqs, quant_min_rows=8, + dev, hmax, max_reqs = torch.device("cuda"), 1 << 20, 256 // hkv + index = IndexDecodeTop16(dev, query_len=ql, max_reqs=max_reqs, max_seq_len=hmax, num_index_heads=hkv) + blk.warm_up(dev, num_heads=16 * hkv, num_kv_heads=hkv, query_len=ql, max_reqs=max_reqs, quant_min_rows=8, compute_type=compute) warm = len(count) icache = torch.zeros(2, 128, 72, device=dev, dtype=torch.uint8) - kv = torch.zeros(2, 2, 128, 72, device=dev, dtype=torch.uint8) + kv = torch.zeros(2, 2 * hkv, 128, 72, device=dev, dtype=torch.uint8) late = [] - for n in list(range(1, 65)) + [65, 100, 128, 129, 200, 256]: + for n in [n for n in list(range(1, 65)) + [65, 100, 128, 129, 200, 256] if n <= max_reqs]: rows = n * ql for ctx in ((32768, 196608, hmax) if n in (1, 2, 3, 4, 5, 8, 16, 32, 33, 64, 128, 256) else (hmax,)): for quant in (False, True): if quant and rows < 8: continue - iq = torch.zeros(rows, 1, 128, device=dev, dtype=torch.float8_e4m3fn) - q = torch.zeros(rows, 16, 128, device=dev, dtype=torch.bfloat16) + iq = torch.zeros(rows, hkv, 128, device=dev, dtype=torch.float8_e4m3fn) + q = torch.zeros(rows, 16 * hkv, 128, device=dev, dtype=torch.bfloat16) table = torch.zeros(n, ctx // 128, device=dev, dtype=torch.int32) seq = torch.full((n,), ctx, device=dev, dtype=torch.int32) - ids = torch.empty(1, rows, 16, device=dev, dtype=torch.int32) + ids = torch.empty(hkv, rows, 16, device=dev, dtype=torch.int32) before = len(count) blk.nvfp4_index_sparse_decode(index, iq, icache, table, seq, 0, 1, q, kv, ids, table, seq, 128**-0.5, torch.empty_like(q), k_scale=None, v_scale=None, decode_query_len=ql, max_seq_len=ctx, compute_type=compute, - quant_out=host._quant_buffers(rows, 2048, dev) if quant else None) + quant_out=host._quant_buffers(rows, 2048 * hkv, dev) if quant else None) torch.cuda.synchronize() if len(count) != before: late.append((n, ctx, quant)) return dict(warm=warm, late=late, reported=warm_up.late_compiles()) +@pytest.mark.parametrize("hkv", [1, 2], ids=["kv1", "kv2"]) @pytest.mark.parametrize("compute", ["bfloat16", "float16"]) @pytest.mark.parametrize("ql", [8, 4]) -def test_block_warm_up_then_no_compile(ql, compute, tmp_path): +def test_block_warm_up_then_no_compile(ql, compute, hkv, tmp_path): from .test_sparse_decode import require_gfx950 require_gfx950() - proc = subprocess.run([sys.executable, "-m", __name__, str(ql), compute], capture_output=True, text=True, + proc = subprocess.run([sys.executable, "-m", __name__, str(ql), compute, str(hkv)], capture_output=True, text=True, timeout=2400, env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) @@ -76,4 +77,4 @@ def test_block_warm_up_then_no_compile(ql, compute, tmp_path): if __name__ == "__main__": - print(json.dumps(scenario(int(sys.argv[1]), getattr(torch, sys.argv[2])))) + print(json.dumps(scenario(int(sys.argv[1]), getattr(torch, sys.argv[2]), int(sys.argv[3])))) diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_select.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_select.py index c00abad..deba249 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_select.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_select.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """``nvfp4_sparse_decode(..., index_score=, index_max_seq_len=)``: the attention takes the indexer's scores (``nvfp4_index_score_rows``) and writes ``topk_idx`` itself. Against the two-launch chain (``nvfp4_index_score_top16`` -then the attention on its ids): the ids bit for bit and the attention output bytes equal, on the fused routes (one KV -head, top-16, bf16 or fp16, <= 256 rows: the rowxcd / prefetch / fixed kernels at ql 8 (fp16: rowxcd, else the -generic kernel), the generic kernel at ql 4; the selector body runs in every attention workgroup) and on the route that launches the selector first (> 256 rows). +then the attention on its ids): the ids bit for bit and the attention output bytes equal, on the fused routes (1 KV +head <= 256 rows, 2 KV heads (index head h feeds KV head h) <= 1024 rows, top-16, bf16 or fp16: the rowxcd / prefetch / +fixed kernels at ql 8 (fp16: rowxcd, else the generic kernel), the generic kernel at ql 1 / 4; the selector body runs in +every attention workgroup) and on the route that launches the selector first (more rows). Short rows (< 16 blocks: the exact fallback, ids from memory), rows past 1792 blocks (the window loop), padding requests, 1-token rows, init / local blocks, a NaN index-q element, equal keys (ties), graph replay at other lengths, determinism.""" @@ -24,7 +25,7 @@ BLK, D, H = 128, 128, 16 -def build(lengths, ql, seed, nan_q=False, equal_keys=False): +def build(lengths, ql, seed, nan_q=False, equal_keys=False, hkv=1): g = torch.Generator(device="cuda") g.manual_seed(seed) reqs = len(lengths) @@ -35,18 +36,18 @@ def build(lengths, ql, seed, nan_q=False, equal_keys=False): if equal_keys: keys = keys[:1].expand(pages * BLK, D).contiguous() icache = iref.nvfp4_index_pages(keys) - kv = torch.randint(0, 256, (pages, 2, BLK, 72), dtype=torch.uint8, device="cuda", generator=g) & 0x77 + kv = torch.randint(0, 256, (pages, 2 * hkv, BLK, 72), dtype=torch.uint8, device="cuda", generator=g) & 0x77 bt = torch.zeros(reqs, -(-MAX // BLK), dtype=torch.int32, device="cuda") p = 1 for r, n in enumerate(nb): bt[r, :n] = torch.randperm(n, device="cuda", generator=g).to(torch.int32) + p p += n seq = torch.tensor(lengths, dtype=torch.int32, device="cuda") - iq = (torch.randn(rows, 1, D, device="cuda", generator=g) * 0.5).to(torch.float8_e4m3fn) + iq = (torch.randn(rows, hkv, D, device="cuda", generator=g) * 0.5).to(torch.float8_e4m3fn) if nan_q: iq.view(torch.uint8)[rows // 2, 0, 7] = 0x7F - q = torch.randn(rows, H, D, device="cuda", generator=g).bfloat16() - return dict(iq=iq, icache=icache, kv=kv, bt=bt, seq=seq, q=q, ql=ql, rows=rows) + q = torch.randn(rows, H * hkv, D, device="cuda", generator=g).bfloat16() + return dict(iq=iq, icache=icache, kv=kv, bt=bt, seq=seq, q=q, ql=ql, rows=rows, hkv=hkv) def chain(c, init, local, fused, ids, out, score, compute=torch.bfloat16): @@ -66,10 +67,10 @@ def chain(c, init, local, fused, ids, out, score, compute=torch.bfloat16): def buffers(c): - rows = c["rows"] - return (torch.full((1, rows, 16), -7, dtype=torch.int32, device="cuda"), - torch.full((rows, H, D), 7.0, dtype=torch.bfloat16, device="cuda"), - torch.full((1, rows, iref.score_width(MAX)), float("nan"), dtype=torch.float32, device="cuda")) + rows, hkv = c["rows"], c["hkv"] + return (torch.full((hkv, rows, 16), -7, dtype=torch.int32, device="cuda"), + torch.full((rows, H * hkv, D), 7.0, dtype=torch.bfloat16, device="cuda"), + torch.full((hkv, rows, iref.score_width(MAX)), float("nan"), dtype=torch.float32, device="cuda")) def same(c, init, local, monkeypatch=None, compute=torch.bfloat16): @@ -88,7 +89,7 @@ def spy(exe, *args): chain(c, init, local, True, ib, ob, sb, compute) if monkeypatch is not None: monkeypatch.undo() - fused = c["rows"] <= 256 # rowxcd / prefetch / fixed (ql 8) or generic (ql 4) + fused = c["rows"] <= ah.SELECT_MAX_ROWS[c["hkv"]] # rowxcd / prefetch / fixed (ql 8) or generic (ql 4) assert any(n.endswith("_select16") for n in names) == fused, names assert torch.equal(ia, ib), (ia - ib).abs().max() assert torch.equal(oa.view(torch.int16), ob.view(torch.int16)) @@ -124,6 +125,37 @@ def test_ids_and_output_match_the_two_launch_chain(ql, lengths, init, local, com assert int((ids == -7).sum()) == 0 +CASES_KV2 = [ + (8, [32768]), # 8 rows: the rowxcd kernels at 2 KV heads + (8, [131072 + 37, 900]), + (8, [0, 300, 1]), + (8, [240000, 0, 1000, 196608]), # 1875 blocks: the window loop + (8, [1500, 16 * 128 + 9, 15 * 128 + 8, 8]), # around 16 blocks: fallback / fast + (8, [40000] * 5), # 40 rows + (8, [3000 + 997 * i for i in range(17)]), # 136 rows + (8, [2000 + 501 * i for i in range(31)] + [0]), # 256 rows + (8, [1500 + 37 * i for i in range(33)]), # 264 rows: in the attention at 2 KV heads + (8, [700 + 97 * i for i in range(129)]), # 1032 rows: the selector launch + (4, [65536, 9]), # ql 4: the generic kernel + (4, [0, 3, 1900 * 128 + 3, 16 * 128]), + (4, [9000 + 1001 * i for i in range(16)]), + (1, [70000, 33, 0, 4096 + 5]), +] + + +@pytest.mark.parametrize("compute", [torch.bfloat16, torch.float16], ids=["bf16", "f16"]) +@pytest.mark.parametrize("ql,lengths", CASES_KV2) +@pytest.mark.parametrize("init,local", [(0, 1), (2, 3)]) +def test_two_kv_heads_ids_and_output_match_the_two_launch_chain(ql, lengths, init, local, compute, monkeypatch): + """2 KV heads = 2 index heads (TP2): KV head h's workgroups select from index head h's scores.""" + require_gfx950() + c = build(lengths, ql, seed=len(lengths) * 71 + lengths[0] % 977 + init, hkv=2) + ids, _ = same(c, init, local, monkeypatch, compute) + assert int((ids == -7).sum()) == 0 + if max(lengths) > 16 * BLK: + assert not torch.equal(ids[0], ids[1]) + + @pytest.mark.parametrize("kind", ["nan_q", "equal_keys"]) def test_exceptional_scores(kind): require_gfx950() diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_oproj_input.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_oproj_input.py index 8ad0038..66be11e 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_oproj_input.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_oproj_input.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""The MXFP8 output as the TP4 o_proj's input: at the served decode shapes (1 KV head, 8 rows a request = DSpark 7, -4 = DSpark 3; 1-64 requests; contexts 32K-192K) the epilogue writes, in the code layout +"""The MXFP8 output as the o_proj's input (TP4: 1 KV head, N 6144 x K 2048; TP2: 2 KV heads, K 4096): at the served +decode shapes (8 rows a request = DSpark 7, 4 = DSpark 3; 1-64 requests; contexts 32K-192K) the epilogue writes, in the +code layout ``gemm.prequant_code_tile_rows`` names for the row count, the bytes the served quantizer makes of the bf16 output (``gemm.mxfp8_linear``'s: the shuffled quantizer on the decode rows, the row-major one elsewhere), so the o_proj GEMM on those codes equals the bf16 chain's GEMM bit for bit wherever both take the same table row (above 32 rows); graph @@ -21,10 +22,10 @@ from .test_sparse_decode import make_case, require_gfx950, run -N, K = 6144, 2048 # the TP4 o_proj -SERVED = [(8, r) for r in (1, 2, 3, 4, 5, 6, 8, 13, 16, 32, 48, 64)] + [ - (4, r) for r in (1, 4, 8, 9, 16, 32, 64) -] +N = 6144 # the o_proj: K = 2048 per KV head +SERVED = [(1, 8, r) for r in (1, 2, 3, 4, 5, 6, 8, 13, 16, 32, 48, 64)] + [ + (1, 4, r) for r in (1, 4, 8, 9, 16, 32, 64) +] + [(2, 8, r) for r in (1, 2, 3, 4, 6, 8, 16, 32, 64)] + [(2, 4, r) for r in (1, 8, 16, 64)] def long_lengths(reqs, seed): @@ -32,14 +33,14 @@ def long_lengths(reqs, seed): return torch.randint(32768, 196609, (reqs,), generator=gen).tolist() -def oproj_weight(seed=3): +def oproj_weight(K, seed=3): gen = torch.Generator(device="cuda").manual_seed(seed) w = (torch.randn((N, K), generator=gen, device="cuda") * 0.5).to(torch.float8_e4m3fn) s = torch.randint(118, 124, (N, K // 32), generator=gen, device="cuda").to(torch.uint8) return G.preshuffle_mxfp8_weight(w, s) -def code_buffers(rows, tile, fill=0xA5): +def code_buffers(rows, tile, K, fill=0xA5): pad = max(tile, 16) rows_p = (rows + pad - 1) // pad * pad codes = torch.full((rows_p, K), fill, dtype=torch.uint8, device="cuda") @@ -55,6 +56,7 @@ def gemm_codes(codes, scales, tile, rows, w, ws): def real_bytes(codes, scales, rows, tile): + K = codes.shape[1] if tile: c = codes.view(-1)[tile_code_offsets(rows, K, codes.device)] else: @@ -62,17 +64,19 @@ def real_bytes(codes, scales, rows, tile): return c, scales.view(-1)[aiter_scale_offsets(rows, K, scales.device)] -@pytest.mark.parametrize("ql,reqs", SERVED, ids=[f"ql{a}-r{b}" for a, b in SERVED]) +@pytest.mark.parametrize("hkv,ql,reqs", SERVED, ids=[f"kv{h}-ql{a}-r{b}" for h, a, b in SERVED]) @torch.inference_mode() -def test_served_quantizer_bytes_and_oproj(ql, reqs): +def test_served_quantizer_bytes_and_oproj(hkv, ql, reqs): require_gfx950() - case = make_case(1, ql, long_lengths(reqs, reqs * 10 + ql), seed=reqs + ql) + case = make_case(hkv, ql, long_lengths(reqs, reqs * 10 + ql), seed=reqs + ql) rows = reqs * ql - assert case["q"].shape[0] == rows and GROUP * HEAD_DIM == K + K = hkv * GROUP * HEAD_DIM + assert case["q"].shape[0] == rows tile = G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) - out = run(case, torch.bfloat16) - codes, scales = code_buffers(rows, tile) - run(case, torch.bfloat16, out=torch.empty_like(out), quant=(codes, scales, tile)) + compute = torch.bfloat16 if hkv == 1 else torch.float16 + out = run(case, compute) + codes, scales = code_buffers(rows, tile, K) + run(case, compute, out=torch.empty_like(out), quant=(codes, scales, tile)) torch.cuda.synchronize() out2d = out.view(rows, K) if tile: @@ -87,7 +91,7 @@ def test_served_quantizer_bytes_and_oproj(ql, reqs): if tile and rows % max(tile, 16): pad = tile_code_offsets(codes.shape[0], K, codes.device)[rows:] assert (codes.view(-1)[pad] == 0xA5).all() - w, ws = oproj_weight() + w, ws = oproj_weight(K) got = gemm_codes(codes, scales, tile, rows, w, ws) ref = G.mxfp8_linear(out2d, w, ws, torch.bfloat16) torch.cuda.synchronize() @@ -99,39 +103,42 @@ def test_served_quantizer_bytes_and_oproj(ql, reqs): assert rel < 1e-2, float(rel) -GRAPH = [(8, 1), (8, 4), (8, 6), (8, 16), (8, 64), (4, 32), (4, 64)] +GRAPH = [(1, 8, 1), (1, 8, 4), (1, 8, 6), (1, 8, 16), (1, 8, 64), (1, 4, 32), (1, 4, 64), (2, 8, 1), (2, 8, 4), + (2, 8, 16), (2, 4, 32)] -@pytest.mark.parametrize("ql,reqs", GRAPH, ids=[f"ql{a}-r{b}" for a, b in GRAPH]) +@pytest.mark.parametrize("hkv,ql,reqs", GRAPH, ids=[f"kv{h}-ql{a}-r{b}" for h, a, b in GRAPH]) @torch.inference_mode() -def test_graph_replay_attention_and_oproj(ql, reqs): +def test_graph_replay_attention_and_oproj(hkv, ql, reqs): """attention (MXFP8 out) + o_proj captured: every replay writes the eager codes, scales and o_proj output, also after new q / cache values are copied in.""" require_gfx950() lengths = long_lengths(reqs, reqs * 3 + ql) - a, b = make_case(1, ql, lengths, seed=5), make_case(1, ql, lengths, seed=5) + a, b = make_case(hkv, ql, lengths, seed=5), make_case(hkv, ql, lengths, seed=5) b["q"] = (torch.randn_like(a["q"].float()) * 1.5).bfloat16() b["kv_cache"] = torch.randint_like(a["kv_cache"], 0, 256) for side in range(2): # random codes, finite block scales (test_graph_replay's) - start = side * 9216 + 8192 - sc_b = b["kv_cache"].view(b["kv_cache"].shape[0], -1)[:, start : start + 1024] + start = side * hkv * 9216 + hkv * 8192 + sc_b = b["kv_cache"].view(b["kv_cache"].shape[0], -1)[:, start : start + hkv * 1024] sc_b.masked_fill_((sc_b & 0x7F) == 0x7F, 0x30) rows = reqs * ql + K = hkv * GROUP * HEAD_DIM tile = G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) - w, ws = oproj_weight() + w, ws = oproj_weight(K) + compute = torch.bfloat16 if hkv == 1 else torch.float16 def chain(case, codes, scales): - run(case, torch.bfloat16, out=torch.empty_like(case["q"]), quant=(codes, scales, tile)) + run(case, compute, out=torch.empty_like(case["q"]), quant=(codes, scales, tile)) return gemm_codes(codes, scales, tile, rows, w, ws) def eager(case): - codes, scales = code_buffers(rows, tile) + codes, scales = code_buffers(rows, tile, K) o = chain(case, codes, scales) torch.cuda.synchronize() return real_bytes(codes, scales, rows, tile), o.clone() want_a, want_b = eager(a), eager(b) - g_codes, g_scales = code_buffers(rows, tile) + g_codes, g_scales = code_buffers(rows, tile, K) chain(a, g_codes, g_scales) torch.cuda.synchronize() graph = torch.cuda.CUDAGraph() diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_sparse_decode.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_sparse_decode.py index 73576bb..b04a34f 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_sparse_decode.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_sparse_decode.py @@ -232,7 +232,7 @@ def same_plan_kwargs(key): CONTRACT = [] for _config in SERVED: for _quant in (False, True): - _extra = {2, 3, 4} if _config in ("kv1_bf16_ql8", "kv1_f16_ql8") else set() + _extra = {2, 3, 4} if _config.endswith("_ql8") else set() _reqs = sorted(set(variants(_config, _quant).values()) | {SERVED[_config][3]} | _extra) CONTRACT += [(_config, _quant, r) for r in _reqs] @@ -312,10 +312,39 @@ def test_explicit_schedules_one_kv_head(compute, ql, reqs, splits, random_bytes, assert torch.equal(outs[0], outs[1]) +EXPLICIT_KV2 = [ + (compute, ql, reqs, splits, random_bytes, xcd) + for compute in (torch.bfloat16, torch.float16) + for ql, reqs in ((1, 6), (4, 6), (8, 6), (8, 2), (8, 4)) + for splits in (None, 1, 2, 4, 8) + for random_bytes in (False, True) + for xcd in (0, XCDS) +] + + +@pytest.mark.parametrize( + "compute,ql,reqs,splits,random_bytes,xcd", + EXPLICIT_KV2, + ids=[f"{str(a)[6:]}-ql{b}-r{c}-s{d}-{'random' if e else 'quantized'}-xcd{f}" for a, b, c, d, e, f in EXPLICIT_KV2], +) +@torch.inference_mode() +def test_explicit_schedules_two_kv_heads(compute, ql, reqs, splits, random_bytes, xcd): + """2 KV heads (32 query heads, TP2), bf16 and fp16 compute: every split count with and without the XCD grid (2 + and 4 requests of 8 rows: the rowxcd kernels at 2 KV heads), realistic caches and caches of random bytes.""" + require_gfx950() + lengths = [9, 300, 2100, 20000, 131, 4096][:reqs] + case = make_case(2, ql, lengths, seed=4, random_bytes=random_bytes) + outs = [run(case, compute, splits=splits, xcd_count=xcd) for _ in range(2)] + ref = reference(case, compute) + torch.cuda.synchronize() + check_bf16(f"kv2 {compute} ql{ql} s{splits} xcd{xcd}", outs[0], ref, live_rows(case)) + assert torch.equal(outs[0], outs[1]) + + OTHER = [ (hkv, compute, ql, splits, xcd) - for hkv, compute in ((1, torch.float16),) - for ql, splits, xcd in ((1, 1, 0), (4, 4, XCDS), (8, 4, 0)) + for hkv, compute in ((1, torch.float16), (2, torch.float16)) + for ql, splits, xcd in ((1, 1, 0), (4, 4, XCDS), (8, 4, 0), (8, 4, XCDS)) ] @@ -326,7 +355,7 @@ def test_explicit_schedules_one_kv_head(compute, ql, reqs, splits, random_bytes, ) @torch.inference_mode() def test_fp16_explicit_splits_with_padding(hkv, compute, ql, splits, xcd): - """fp16 on 1 KV head at explicit splits, padded and plain grids, with a graph padding request.""" + """fp16 on 1 and 2 KV heads at explicit splits, padded and plain grids, with a graph padding request.""" require_gfx950() lengths = [131, 4097, 2100, 777, 65, 0] case = make_case(hkv, ql, lengths, seed=42) @@ -343,6 +372,10 @@ def test_fp16_explicit_splits_with_padding(hkv, compute, ql, splits, xcd): for hkv, ql, splits, xcd, tile in ( (1, 1, 1, 0, 0), (1, 4, 4, XCDS, 16), + (2, 1, 1, 0, 0), + (2, 4, 4, XCDS, 16), + (2, 8, 4, XCDS, 16), + (2, 8, 1, XCDS, 0), ) for compute in (torch.bfloat16, torch.float16) ] @@ -447,8 +480,8 @@ def poison_tail(case, hkv): POISON = [ (hkv, compute, ql, reqs) - for hkv, compute in ((1, torch.bfloat16), (1, torch.float16)) - for ql, reqs in ((1, 6), (4, 6), (8, 6), (8, 4)) + for hkv, compute in ((1, torch.bfloat16), (1, torch.float16), (2, torch.bfloat16), (2, torch.float16)) + for ql, reqs in ((1, 6), (4, 6), (8, 6), (8, 4), (8, 2)) ] @@ -458,7 +491,8 @@ def poison_tail(case, hkv): @torch.inference_mode() def test_bytes_past_the_length_are_ignored(hkv, compute, ql, reqs): """NaN scales and random codes past each request's length (its last block, a row's own block) leave the output - finite and bit for bit the clean cache's (4 requests of 8 rows: the rowxcd kernels at 1 KV head).""" + finite and bit for bit the clean cache's (4 requests of 8 rows: the rowxcd kernels at 1 KV head, 2 requests at + 2 KV heads).""" require_gfx950() lengths = [131, 4097, 2100, 777, 65, 300][:reqs] case = make_case(hkv, ql, lengths, seed=3) @@ -471,15 +505,18 @@ def test_bytes_past_the_length_are_ignored(hkv, compute, ql, reqs): @pytest.mark.parametrize("compute", [torch.bfloat16, torch.float16], ids=["bf16", "f16"]) -@pytest.mark.parametrize("reqs", [1, 2, 4, 8, 16, 20, 32, 64]) +@pytest.mark.parametrize( + "hkv,reqs", [(1, r) for r in (1, 2, 4, 8, 16, 20, 32, 64)] + [(2, r) for r in (1, 2, 4, 8, 13, 32)] +) @torch.inference_mode() -def test_bitwise_next_to_mfma_waves(reqs, compute): - """The TP4 paths (1 KV head, 8 rows per request) with a high-priority MFMA kernel (``tests/mfma_spinner.py``) on - the same CUs: the output stays bit for bit the same (gfx950 has given wrong VALU products next to MFMA waves).""" +def test_bitwise_next_to_mfma_waves(hkv, reqs, compute): + """The TP4 / TP2 paths (1 / 2 KV heads, 8 rows per request) with a high-priority MFMA kernel + (``tests/mfma_spinner.py``) on the same CUs: the output stays bit for bit the same (gfx950 has given wrong VALU + products next to MFMA waves).""" from tests.mfma_spinner import mfma_spinner require_gfx950() - case = make_case(1, 8, lengths_for(reqs, seed=31, pad=1 if reqs > 1 else 0), seed=31) + case = make_case(hkv, 8, lengths_for(reqs, seed=31, pad=1 if reqs > 1 else 0), seed=31) clean = run(case, compute) main, side = torch.cuda.Stream(), torch.cuda.Stream(priority=-1) spin = mfma_spinner(torch.cuda.get_device_properties(0).multi_processor_count) @@ -495,6 +532,7 @@ def test_bitwise_next_to_mfma_waves(reqs, compute): (1, 1, 1, torch.bfloat16), (1, 4, 1, torch.bfloat16), (1, 8, 2, torch.bfloat16), (1, 8, 5, torch.bfloat16), (1, 4, 1, torch.float16), (1, 8, 1, torch.float16), (1, 8, 5, torch.float16), + (2, 8, 1, torch.float16), (2, 8, 4, torch.float16), (2, 8, 1, torch.bfloat16), (2, 4, 2, torch.float16), ]) @torch.inference_mode() def test_strided_pages(hkv, ql, repeats, compute): @@ -514,15 +552,16 @@ def test_strided_pages(hkv, ql, repeats, compute): assert torch.isfinite(dense).all() and torch.equal(dense, strided) +@pytest.mark.parametrize("hkv", [1, 2]) @pytest.mark.parametrize("ql", [1, 4, 8]) @pytest.mark.parametrize("splits", [1, 4]) @torch.inference_mode() -def test_padding_requests(ql, splits): +def test_padding_requests(ql, splits, hkv): """Graph padding requests (length 0) between real ones and an all-padding batch: the real rows match the reference, the padding rows are zero.""" require_gfx950() lengths = [700, 0, 3000, 0, 0, 129] - case = make_case(1, ql, lengths, seed=1) + case = make_case(hkv, ql, lengths, seed=1) compute = torch.bfloat16 out = run(case, compute, splits=splits) ref = reference(case, compute) diff --git a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_warm_up_entry.py b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_warm_up_entry.py index 9ee8a9e..2c9beba 100644 --- a/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_warm_up_entry.py +++ b/tests/attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_warm_up_entry.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """``host.warm_up_decode`` (and the entry's first-call warm-up, ``amd_kernels.warm_up``) compiles every variant a TP4 -server launches, bf16 and fp16 compute: afterwards no request count of 1-64 (bf16 output, and the MXFP8 output from 8 -rows) compiles. Also the first-call warm-up alone: one call, then every count to ``WARM_MAX_REQS`` without a -compile.""" +(1 KV head) or TP2 (2 KV heads) server launches, bf16 and fp16 compute: afterwards no request count of 1-64 (bf16 +output, and the MXFP8 output from 8 rows) compiles. Also the first-call warm-up alone: one call, then every count to +``WARM_MAX_REQS`` without a compile.""" import json import os import subprocess @@ -13,7 +13,7 @@ import torch -def scenario(ql, explicit, compute): +def scenario(ql, explicit, compute, hkv): from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16 import host @@ -30,7 +30,7 @@ def spy(*a, **kw): MlirCompiler.compile = spy dev = torch.device("cuda") - heads, hkv, max_reqs = 16, 1, 64 + heads, max_reqs = 16 * hkv, 64 walk = range(1, max_reqs + 1) if explicit else (1, 2, 7, 33, 64, 100, 255, 256) kv = torch.zeros((2, 2 * hkv, 128, 72), dtype=torch.uint8, device=dev) @@ -66,14 +66,16 @@ def call(reqs, quant): return dict(warm=warm, late=late, reported=warm_up.late_compiles()) +@pytest.mark.parametrize("hkv", [1, 2], ids=["kv1", "kv2"]) @pytest.mark.parametrize("compute", ["bfloat16", "float16"]) @pytest.mark.parametrize("ql", [8, 4]) @pytest.mark.parametrize("explicit", [True, False], ids=["warm_up_decode", "first_call"]) -def test_warm_up_then_no_compile(ql, explicit, compute, tmp_path): +def test_warm_up_then_no_compile(ql, explicit, compute, hkv, tmp_path): from .test_sparse_decode import require_gfx950 require_gfx950() - proc = subprocess.run([sys.executable, "-m", __name__, str(ql), str(int(explicit)), compute], capture_output=True, + proc = subprocess.run([sys.executable, "-m", __name__, str(ql), str(int(explicit)), compute, str(hkv)], + capture_output=True, text=True, timeout=1500, env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) @@ -83,4 +85,4 @@ def test_warm_up_then_no_compile(ql, explicit, compute, tmp_path): if __name__ == "__main__": - print(json.dumps(scenario(int(sys.argv[1]), bool(int(sys.argv[2])), getattr(torch, sys.argv[3])))) + print(json.dumps(scenario(int(sys.argv[1]), bool(int(sys.argv[2])), getattr(torch, sys.argv[3]), int(sys.argv[4])))) diff --git a/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_init_warm_up.py b/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_init_warm_up.py index d83989f..a6d7bc9 100644 --- a/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_init_warm_up.py +++ b/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_init_warm_up.py @@ -1,27 +1,30 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""``init`` compiles every variant a TP4 server (16 q heads, 1 KV / index head, 8 rows per request, 1-256 requests, -1M context) launches through the layer: afterwards no step compiles: the index top-16 + decode block and the decode -alone at 1-64 and up to 256 requests (32K / 1M context, bf16 and MXFP8 output from ``QUANT_OUT_MIN_ROWS`` rows), the -prefill and the prefill indexer at 1-32768 tokens (indexer: up to 1M context) and the insert's prefill tiles at -1537-40000 tokens.""" +"""``init`` compiles every variant a server launches through the layer, TP4 (16 q heads, 1 KV / index head, 1-256 +requests) and TP2 (32 q heads, 2 KV / index heads, 1-128 requests), 8 rows per request, 1M context: afterwards no step +compiles: the index top-16 + decode block and the decode alone at 1-64 and up to the request limit (32K / 1M context, +bf16 and MXFP8 output from ``quant_out_buffers``' rows), the prefill and the prefill indexer at 1-32768 tokens (indexer: +up to 1M context) and the insert's prefill tiles at 1537-40000 tokens.""" import json import os import subprocess import sys +import pytest import torch from tests.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16.test_sparse_decode import require_gfx950 -QL, HMAX, MAX_REQS = 8, 1 << 20, 256 +QL, HMAX = 8, 1 << 20 +MAX_REQS = {1: 256, 2: 128} # per KV heads (TP4, TP2) -def scenario(): +def scenario(hkv): from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up from amd_kernels.attention import nvfp4_sparse_layer_gqa16_d128_p128_top16 as layer from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128 import norm_rope_nvfp4_insert + from amd_kernels.attention.norm_rope_nvfp4_insert_d128_rot64_p128.host import insert_blocker from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import prefill_top16 torch.set_num_threads(8) @@ -35,7 +38,8 @@ def spy(*a, **kw): MlirCompiler.compile = spy dev = torch.device("cuda") - index = layer.init(dev, num_heads=16, num_kv_heads=1, num_index_heads=1, query_len=QL, max_reqs=MAX_REQS, + heads, max_reqs, k = 16 * hkv, MAX_REQS[hkv], 2048 * hkv + index = layer.init(dev, num_heads=heads, num_kv_heads=hkv, num_index_heads=hkv, query_len=QL, max_reqs=max_reqs, max_seq_len=HMAX) warm, late = len(count), [] @@ -49,19 +53,19 @@ def probe(tag, fn): z = dict(device=dev) one = torch.ones((), dtype=torch.float32, **z) icache = torch.zeros(2, 128, 72, dtype=torch.uint8, **z) - kv = torch.zeros(2, 2, 128, 72, dtype=torch.uint8, **z) - for n in list(range(1, 65)) + [65, 100, 128, 129, 200, 256]: + kv = torch.zeros(2, 2 * hkv, 128, 72, dtype=torch.uint8, **z) + for n in [n for n in list(range(1, 65)) + [65, 100, 128, 129, 200, 256] if n <= max_reqs]: rows = n * QL - for ctx in ((32768, HMAX) if n in (1, 8, 32, 33, 64, 256) else (HMAX,)): + for ctx in ((32768, HMAX) if n in (1, 8, 32, 33, 64, 128, 256) else (HMAX,)): for quant in (False, True): - qo = layer.quant_out_buffers(rows, 6144, 2048, dev) if quant else None + qo = layer.quant_out_buffers(rows, 6144, k, dev) if quant else None if quant and qo is None: continue - iq = torch.zeros(rows, 1, 128, dtype=torch.float8_e4m3fn, **z) - q = torch.zeros(rows, 16, 128, dtype=torch.bfloat16, **z) + iq = torch.zeros(rows, hkv, 128, dtype=torch.float8_e4m3fn, **z) + q = torch.zeros(rows, heads, 128, dtype=torch.bfloat16, **z) table = torch.zeros(n, ctx // 128, dtype=torch.int32, **z) seq = torch.full((n,), ctx, dtype=torch.int32, **z) - ids = torch.full((1, rows, 16), -1, dtype=torch.int32, **z) + ids = torch.full((hkv, rows, 16), -1, dtype=torch.int32, **z) ids[..., 0] = 0 probe(("decode", n, ctx, quant), lambda: layer.sparse_decode( q, kv, ids, table, seq, 128**-0.5, torch.empty_like(q), k_scale=one, v_scale=one, @@ -70,38 +74,40 @@ def probe(tag, fn): index, iq, icache, table, seq, 0, 1, q, kv, ids, table, seq, 128**-0.5, torch.empty_like(q), k_scale=one, v_scale=one, decode_query_len=QL, max_seq_len=ctx, quant_out=qo)) for t in (1, 8, 100, 512, 513, 1500, 2048, 2049, 4096, 16384, 32768): - q = torch.zeros(t, 16, 128, dtype=torch.bfloat16, **z) - topk = torch.full((1, t, 16), -1, dtype=torch.int32, **z) + q = torch.zeros(t, heads, 128, dtype=torch.bfloat16, **z) + topk = torch.full((hkv, t, 16), -1, dtype=torch.int32, **z) topk[..., 0] = 0 zeros = torch.zeros(t, dtype=torch.int32, **z) probe(("prefill", t), lambda: layer.sparse_prefill( - q, kv, topk, torch.zeros(1, 1, dtype=torch.int32, **z), zeros, zeros, 1, 128**-0.5, torch.empty_like(q), + q, kv, topk, torch.zeros(1, 1, dtype=torch.int32, **z), zeros, zeros, hkv, 128**-0.5, torch.empty_like(q), one, one)) for ctx in sorted({t, 65536 + t, 212992 + t, HMAX} if t <= 16384 else {t, 262144}): - iq = torch.zeros(t, 1, 128, dtype=torch.float8_e4m3fn, **z) + iq = torch.zeros(t, hkv, 128, dtype=torch.float8_e4m3fn, **z) table = torch.zeros(1, -(-ctx // 128), dtype=torch.int32, **z) cu = torch.tensor([0, t], dtype=torch.int32, **z) seq = torch.tensor([ctx], dtype=torch.int32, **z) nvp = torch.ones(t, dtype=torch.int32, **z) probe(("index prefill", t, ctx), lambda: prefill_top16.index_prefill_top16( - iq, icache, table, cu, seq, 0, 1, torch.empty(1, t, 16, dtype=torch.int32, **z), max_query_len=t, + iq, icache, table, cu, seq, 0, 1, torch.empty(hkv, t, 16, dtype=torch.int32, **z), max_query_len=t, max_seq_len=ctx, num_valid_pages=nvp)) w = torch.zeros(128, dtype=torch.bfloat16, **z) - for t in (1537, 1600, 4096, 9000, 24576, 32768, 40000): + blocked = insert_blocker(heads, hkv, hkv, 64) + for t in (1537, 1600, 4096, 9000, 24576, 32768, 40000) if blocked is None else (): slots = torch.full((t,), -1, dtype=torch.int64, **z) probe(("insert", t), lambda: norm_rope_nvfp4_insert( - torch.zeros(t, 20 * 128, dtype=torch.bfloat16, **z), w, w, w, w, + torch.zeros(t, (heads + 3 * hkv + 1) * 128, dtype=torch.bfloat16, **z), w, w, w, w, torch.zeros(t, 64, dtype=torch.bfloat16, **z), torch.arange(t, **z), slots, slots, - torch.zeros(1, 2, 128, 72, dtype=torch.uint8, **z), torch.zeros(1, 128, 72, dtype=torch.uint8, **z), one, - one, torch.empty(t, 16 * 128, dtype=torch.bfloat16, **z), - torch.empty(t, 128, dtype=torch.float8_e4m3fn, **z), num_heads=16, num_kv_heads=1, num_index_heads=1, - eps=1e-6)) - return dict(warm=warm, late=late, reported=warm_up.late_compiles()) + torch.zeros(1, 2 * hkv, 128, 72, dtype=torch.uint8, **z), torch.zeros(1, 128, 72, dtype=torch.uint8, **z), + one, one, torch.empty(t, heads * 128, dtype=torch.bfloat16, **z), + torch.empty(t, hkv * 128, dtype=torch.float8_e4m3fn, **z), num_heads=heads, num_kv_heads=hkv, + num_index_heads=hkv, eps=1e-6)) + return dict(warm=warm, late=late, reported=warm_up.late_compiles(), insert=blocked) -def test_init_then_no_compile(tmp_path): +@pytest.mark.parametrize("hkv", [1, 2], ids=["tp4", "tp2"]) +def test_init_then_no_compile(tmp_path, hkv): require_gfx950() - proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=3000, + proc = subprocess.run([sys.executable, "-m", __name__, str(hkv)], capture_output=True, text=True, timeout=3000, env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-6000:] @@ -110,4 +116,4 @@ def test_init_then_no_compile(tmp_path): if __name__ == "__main__": - print(json.dumps(scenario())) + print(json.dumps(scenario(int(sys.argv[1])))) diff --git a/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_layer.py b/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_layer.py index 40ebaf5..a709caf 100644 --- a/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_layer.py +++ b/tests/attention/nvfp4_sparse_layer_gqa16_d128_p128_top16/test_layer.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """The layer calls are the per-kernel calls at the served compute dtypes, bit for bit: the decode, the index top-16 + decode block (ids and output) and the prefill, bf16 and MXFP8 outputs; ``quant_out_buffers`` declines below -``QUANT_OUT_MIN_ROWS`` rows and is ``gemm.prequant_buffers`` from there. The warm-up: ``test_init_warm_up.py``.""" +``QUANT_OUT_MIN_ROWS[k]`` rows and is ``gemm.prequant_buffers`` from there. The warm-up: ``test_init_warm_up.py``.""" import pytest import torch @@ -20,7 +20,7 @@ quant_buffers, require_gfx950, ) -from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import TP4_HKV +from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import TP2_HKV, TP4_HKV from tests.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.cases import make_case as prefill_case @@ -33,15 +33,15 @@ def _same(a, b): @pytest.mark.parametrize("quant", [False, True]) -@pytest.mark.parametrize("reqs", [1, 4, 33]) -def test_sparse_decode_is_the_fp16_decode(reqs, quant): +@pytest.mark.parametrize("hkv,reqs", [(1, 1), (1, 4), (1, 33), (2, 1), (2, 4), (2, 17)]) +def test_sparse_decode_is_the_fp16_decode(hkv, reqs, quant): require_gfx950() - c = make_case(1, 8, lengths_for(reqs, reqs), reqs) + c = make_case(hkv, 8, lengths_for(reqs, reqs), reqs) rows = c["q"].shape[0] res = [] for fn, kw in ((layer.sparse_decode, {}), (dh.nvfp4_sparse_decode, {"compute_type": torch.float16})): out = torch.full_like(c["q"], float("nan")) - q = (*quant_buffers(rows, 2048), 16) if quant else None + q = (*quant_buffers(rows, hkv * 2048), 16) if quant else None fn(c["q"], c["kv_cache"], c["topk_idx"], c["block_table"], c["seq_lens"], SM_SCALE, out, k_scale=c["k_scale"], v_scale=c["v_scale"], decode_query_len=c["ql"], quant_out=q, **kw) res.append((out, *(q[:2] if quant else ()))) @@ -50,16 +50,16 @@ def test_sparse_decode_is_the_fp16_decode(reqs, quant): @pytest.mark.parametrize("quant", [False, True]) -@pytest.mark.parametrize("reqs", [1, 5, 32, 33]) -def test_index_sparse_decode_is_the_fp16_block(reqs, quant): +@pytest.mark.parametrize("hkv,reqs", [(1, 1), (1, 5), (1, 32), (1, 33), (2, 1), (2, 5), (2, 32), (2, 33)]) +def test_index_sparse_decode_is_the_fp16_block(hkv, reqs, quant): require_gfx950() - c = build([max(1, 65536 - 4093 * r) for r in range(reqs)], 8, seed=reqs) + c = build([max(1, 65536 - 4093 * r) for r in range(reqs)], 8, seed=reqs, hkv=hkv) rows = c["rows"] res, routes = [], [] for fn, kw in ((layer.index_sparse_decode, {}), (blk.nvfp4_index_sparse_decode, {"compute_type": torch.float16})): - ids = torch.full((1, rows, 16), -7, dtype=torch.int32, device="cuda") - out = torch.full((rows, 16, 128), 7.0, dtype=torch.bfloat16, device="cuda") - q = (*quant_buffers(rows, 2048), 16) if quant else None + ids = torch.full((hkv, rows, 16), -7, dtype=torch.int32, device="cuda") + out = torch.full((rows, 16 * hkv, 128), 7.0, dtype=torch.bfloat16, device="cuda") + q = (*quant_buffers(rows, hkv * 2048), 16) if quant else None routes.append(fn(None, c["iq"], c["icache"], c["bt"], c["seq"], 2, 3, c["q"], c["kv"], ids, c["bt"], c["seq"], 128**-0.5, out, k_scale=None, v_scale=None, decode_query_len=8, max_seq_len=MAX, quant_out=q, **kw)) @@ -68,10 +68,11 @@ def test_index_sparse_decode_is_the_fp16_block(reqs, quant): assert routes[0] == routes[1] and _same(*res) +@pytest.mark.parametrize("hkv", [TP4_HKV, TP2_HKV]) @pytest.mark.parametrize("q_lens", [[7], [600, 40], [3000]]) -def test_sparse_prefill_is_the_fp16_lazy_prefill(q_lens): +def test_sparse_prefill_is_the_fp16_lazy_prefill(q_lens, hkv): require_gfx950() - c = prefill_case(q_lens, [q + 5000 for q in q_lens], TP4_HKV, seed=len(q_lens)) + c = prefill_case(q_lens, [q + 5000 for q in q_lens], hkv, seed=len(q_lens)) res = [] for fn, kw in ((layer.sparse_prefill, {}), (nvfp4_sparse_prefill, {"compute_dtype": "fp16", "lazy_rescale": True})): out = torch.full(c.q.shape, float("nan"), dtype=torch.bfloat16, device="cuda") @@ -81,17 +82,19 @@ def test_sparse_prefill_is_the_fp16_lazy_prefill(q_lens): assert _same(*res) -def test_quant_out_buffers(): - n, k = 6144, 2048 - for rows in range(1, layer.QUANT_OUT_MIN_ROWS): +@pytest.mark.parametrize("k", [2048, 4096]) +def test_quant_out_buffers(k): + n = 6144 + for rows in range(1, layer.QUANT_OUT_MIN_ROWS[k]): assert layer.quant_out_buffers(rows, n, k, "cuda") is None - for rows in (layer.QUANT_OUT_MIN_ROWS, 9, 64, 129, 256, 2048): + for rows in (layer.QUANT_OUT_MIN_ROWS[k], 9, 64, 129, 256, 2048): got, want = layer.quant_out_buffers(rows, n, k, "cuda"), gemm.prequant_buffers(rows, n, k, "cuda") assert got[2] == want[2] for a, b in zip(got[:2], want[:2], strict=True): assert (a.shape, a.dtype, a.device) == (b.shape, b.dtype, b.device) -def test_init_without_routes_builds_nothing(): - assert layer.init("cuda", num_heads=16, num_kv_heads=1, num_index_heads=1, query_len=8, max_reqs=4, +@pytest.mark.parametrize("hkv", [1, 2]) +def test_init_without_routes_builds_nothing(hkv): + assert layer.init("cuda", num_heads=16 * hkv, num_kv_heads=hkv, num_index_heads=hkv, query_len=8, max_reqs=4, max_seq_len=MAX, insert=False, attention=False, index=False) is None diff --git a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/cases.py b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/cases.py index c96cfc7..4817875 100644 --- a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/cases.py +++ b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/cases.py @@ -17,8 +17,9 @@ BLOCK, HEAD_DIM, GROUP, TOPK = ref.BLK, ref.D, ref.HG, ref.TOPK SM_SCALE = HEAD_DIM**-0.5 -# served KV heads per GPU: TP4 (16 query heads) +# served KV heads per GPU: TP4 (16 query heads), TP2 (32 query heads) TP4_HKV = 1 +TP2_HKV = 2 def require_gfx950(): diff --git a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_compile.py b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_compile.py index 1aef4ab..66c1dcd 100644 --- a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_compile.py +++ b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_compile.py @@ -16,14 +16,17 @@ import torch from amd_kernels.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16.query import pack_query -from .cases import TP4_HKV, make_case, require_gfx950 +from .cases import TP2_HKV, TP4_HKV, make_case, require_gfx950 from .test_sparse_prefill import case_slice, run, run_pack, run_quant -# served prefill tokens per step: max-num-batched-tokens 65536 (TP4) -RANGES = {TP4_HKV: range(1, 65537)} +# served prefill tokens per step: max-num-batched-tokens 65536 (TP4, TP2) +RANGES = {TP4_HKV: range(1, 65537), TP2_HKV: range(1, 65537)} PROBES = 6 # (compute dtype, KV heads, output mode): pack is the sequence-parallel TP4 prefill's -CONFIGS = [(variant, TP4_HKV, mode) for variant in ("bf16", "fp16") for mode in ("plain", "quant", "pack")] +CONFIGS = [(variant, TP4_HKV, mode) for variant in ("bf16", "fp16") for mode in ("plain", "quant", "pack")] + [ + (variant, TP2_HKV, mode) for variant in ("bf16", "fp16") for mode in ("plain", "quant")] +# Hkv 2 small tiles: (most tokens, queries per workgroup, shared Q) +TP2_TILES = ((128, 1, False), (256, 2, True), (384, 3, False), (512, 2, True), (768, 3, False)) def variant_key(variant, hkv, mode, t): @@ -32,6 +35,9 @@ def variant_key(variant, hkv, mode, t): rounds = 2 if (t + 3) // 4 * hkv > cus else 1 if variant in ("bf16", "fp16") and hkv == TP4_HKV and t <= 1536: return (variant, mode == "pack", rounds, mode == "quant", False, False, 1 if t <= 256 else 2 if t <= 1025 else 3, 512 < t <= 1025) + if variant in ("bf16", "fp16") and hkv == TP2_HKV and t <= TP2_TILES[-1][0]: + tile = next((tq, shared) for most, tq, shared in TP2_TILES if t <= most) + return (variant, mode == "pack", 1, mode == "quant", False, False, *tile) return (variant, mode == "pack", rounds, mode == "quant", False, False) diff --git a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_sparse_prefill.py b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_sparse_prefill.py index 91ea017..8a9b081 100644 --- a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_sparse_prefill.py +++ b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_sparse_prefill.py @@ -1,8 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """The FlyDSL NVFP4 sparse prefills (bf16 and fp16 compute; fp8 below) against the -torch contract in ``reference.py``, on the served shape: Hkv = 1 (TP4, 16 query heads), -1 to 65536 prefill tokens.""" +torch contract in ``reference.py``, on the served shapes: Hkv = 1 (TP4, 16 query heads) and Hkv = 2 (TP2, 32 query +heads), 1 to 65536 prefill tokens.""" import dataclasses @@ -16,13 +16,14 @@ BLOCK, HEAD_DIM, TOPK, + TP2_HKV, TP4_HKV, make_case, require_gfx950, ) VARIANTS = ["bf16", "fp16", "fp8"] # fp16: the default compute dtype -HKVS = [TP4_HKV] +HKVS = [TP4_HKV, TP2_HKV] FILL = 0xA5 @@ -156,9 +157,11 @@ def test_one_and_two_round_combine_consistent(variant, hkv): case = make_case([n], [n], hkv, seed=5, picks="shared") full = run(case, variant) check_reference(case, full, variant) - # BF16 / FP16 at one KV head: TQ1 (256), TQ2 (256, and 1536 = its largest), TQ4 (2048) - small = variant in ("bf16", "fp16") and hkv == TP4_HKV - cuts = [0, 256, 512, 2048, n] if small else [0, edge, 3 * edge, n] + # BF16 / FP16 at one KV head: TQ1 (256), TQ2 (256, and 1536 = its largest), TQ4 (2048); at two: TQ1 (128), + # shared-Q TQ2 (256, 512), TQ3 (384, 768) + small = variant in ("bf16", "fp16") + small_cuts = [0, 256, 512, 2048, n] if hkv == TP4_HKV else [0, 128, 384, 768, 1280, n] + cuts = small_cuts if small else [0, edge, 3 * edge, n] parts = torch.cat([run(case_slice(case, a, b), variant) for a, b in zip(cuts, cuts[1:])]) if variant == "fp8" or small: # Small query tiles walk their local pages; the four-query layout @@ -235,10 +238,10 @@ def test_pack_epilogue_bytes(variant, hkv, q_lens): assert bool((pack[n_tok:] == FILL).all()) -@pytest.mark.parametrize("hkv,n_tok", [(TP4_HKV, 65536)], ids=["tp4-64k"]) +@pytest.mark.parametrize("hkv,n_tok", [(TP4_HKV, 65536), (TP2_HKV, 65536)], ids=["tp4-64k", "tp2-64k"]) @pytest.mark.parametrize("variant", VARIANTS) def test_largest_step(variant, hkv, n_tok): - """The largest served step (max-num-batched-tokens 65536 at TP4): the reference on + """The largest served step (max-num-batched-tokens 65536 at TP4 and TP2): the reference on sampled rows and the quant epilogue's bytes.""" require_gfx950() case = make_case([n_tok], [n_tok + 4096], hkv, seed=0, picks="shared") @@ -295,7 +298,8 @@ def test_graph_replay(variant, hkv, mode): # --- determinism next to other kernels ----------------------------------------------- -@pytest.mark.parametrize("hkv,n_tok", [(TP4_HKV, 1024), (TP4_HKV, 4096)]) +@pytest.mark.parametrize("hkv,n_tok", [(TP4_HKV, 1024), (TP4_HKV, 4096), (TP2_HKV, 384), (TP2_HKV, 512), + (TP2_HKV, 2048)]) @pytest.mark.parametrize("variant", VARIANTS) def test_bitwise_next_to_mfma_waves(variant, hkv, n_tok): """With another kernel's MFMA waves on every CU the output equals an undisturbed diff --git a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_tp2_tiles.py b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_tp2_tiles.py new file mode 100644 index 0000000..cf8bb09 --- /dev/null +++ b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_tp2_tiles.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The small tiles at two KV heads (TP2, 32 query heads; ``host.HKV2_TILES``) at every tile boundary, and the +one-KV-head (TP4) tile choice unchanged for every token count.""" + +import dataclasses + +import pytest +import torch + +from amd_kernels.attention.nvfp4_sparse_prefill_gqa16_d128_p128_top16 import host, reference +from .cases import TP2_HKV, make_case, require_gfx950 +from .test_sparse_prefill import check_quant_bytes, check_reference, run, run_quant + +LAZY = dict(lazy_rescale=True) +# both sides of every Hkv 2 tile boundary, and the four-query kernel past the last +EDGES = [1, 128, 129, 256, 257, 384, 385, 512, 513, 768, 769] + + +def tp4_tile(t): + """The one-KV-head small tile of the TP4 dispatch: (queries per workgroup, shared Q), or None.""" + if t > 1536: + return None + return (1 if t <= 256 else 2 if t <= 1025 else 3), 512 < t <= 1025 + + +def test_tp4_tiles_unchanged(): + for t in range(1, 65537): + assert host._local_tile(t, 1) == tp4_tile(t), t + assert host._local_tile(t, 4) is None + + +def test_tp2_tiles(): + want = {1: (1, False), 128: (1, False), 129: (2, True), 256: (2, True), 257: (3, False), 384: (3, False), + 385: (2, True), 512: (2, True), 513: (3, False), 768: (3, False), 769: None, 65536: None} + for t, tile in want.items(): + assert host._local_tile(t, TP2_HKV) == tile, t + + +@pytest.mark.parametrize("variant,lazy", [("fp16", True), ("fp16", False), ("bf16", False)], + ids=["fp16lazy", "fp16", "bf16"]) +@pytest.mark.parametrize("tokens", EDGES) +@torch.inference_mode() +def test_tp2_tile_contract(tokens, variant, lazy): + """Two requests (one of 3 tokens) across the tiles' token groups: the reference, deterministic, graph replay bit for + bit, the MXFP8 output = the reference quant of the bf16 output.""" + require_gfx950() + kw = LAZY if lazy else {} + q_lens = [tokens] if tokens < 8 else [tokens - 3, 3] + case = make_case(q_lens, [tokens + 5000, 700][: len(q_lens)], TP2_HKV, scales=(0.5, 2.0), seed=tokens, + picks="shared") + out = run(case, variant, **kw) + check_reference(case, out, variant, rows=case.rows(24)) + for _ in range(2): + assert torch.equal(out, run(case, variant, **kw)) + target = torch.full_like(out, float("nan")) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run(case, variant, out=target, **kw) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, target) + codes, scales = run_quant(case, variant, **kw) + check_quant_bytes(out, codes, scales, tokens) + assert (codes[tokens:] == 0xA5).all() + assert (reference.aiter_unshuffle_scales(scales)[tokens:] == 0xA5).all() + + +@pytest.mark.parametrize("tokens", [129, 300, 400, 600]) +@torch.inference_mode() +def test_tp2_lazy_runs_its_own_kernel(tokens): + """The two- / three-query tiles take the lazy rescale at Hkv 2; the one-query tile keeps the eager one.""" + require_gfx950() + case = make_case([tokens], [tokens + 3000], TP2_HKV, seed=330 + tokens, picks="random") + lazy = run(case, "fp16", **LAZY) + check_reference(case, lazy, "fp16") + assert not torch.equal(lazy, run(case, "fp16")) + small = make_case([100], [3000], TP2_HKV, seed=331, picks="random") + assert torch.equal(run(small, "fp16", **LAZY), run(small, "fp16")) + + +@pytest.mark.parametrize("tokens", [128, 256, 384, 512, 768]) +@torch.inference_mode() +def test_tp2_nan_scales_past_length(tokens): + """NaN scale bytes past the sequence end (unwritten tokens, both KV heads) leave the output finite and unchanged.""" + require_gfx950() + case = make_case([tokens], [tokens + 17], TP2_HKV, seed=61, picks="shared") + clean = run(case, "fp16", **LAZY) + kc, ks, vc, vs = reference.unpack_kv_pages(case.cache, TP2_HKV) + end = int(case.pos[-1]) + 1 + page = int(case.table[0, end // 128]) + ks[page, :, end % 128:] = 0x7F + vs[page, :, end % 128:] = 0x7F + dirty = run(dataclasses.replace(case, cache=reference.pack_kv_pages(kc, ks, vc, vs)), "fp16", **LAZY) + assert torch.isfinite(dirty).all() + assert torch.equal(clean, dirty) + + +@pytest.mark.parametrize("tokens", [128, 256, 384, 512, 768, 2048]) +def test_tp2_lazy_bitwise_next_to_mfma_waves(tokens): + """With another kernel's MFMA waves on every CU the served output (fp16 lazy) equals an undisturbed run's.""" + require_gfx950() + from tests.mfma_spinner import mfma_spinner + + case = make_case([tokens], [tokens + 30000], TP2_HKV, seed=7) + main, side = torch.cuda.Stream(), torch.cuda.Stream(priority=-1) + + def on(stream): + with torch.cuda.stream(stream): + return run(case, "fp16", **LAZY) + + ref = on(main) + torch.cuda.synchronize() + spin = mfma_spinner(torch.cuda.get_device_properties(0).multi_processor_count) + for _ in range(8): + spin(200_000, side) + out = on(main) + torch.cuda.synchronize() + assert torch.equal(out, ref) diff --git a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_warm_up_entry.py b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_warm_up_entry.py index a2087d0..6e9640d 100644 --- a/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_warm_up_entry.py +++ b/tests/attention/nvfp4_sparse_prefill_gqa16_d128_p128_top16/test_warm_up_entry.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """``host.warm_up_prefill`` compiles every prefill variant of a compute precision at -one KV head (TP4): afterwards no token count across the tiers compiles.""" +one KV head (TP4) and two (TP2): afterwards no token count across the tiers compiles.""" import json import os import subprocess @@ -11,7 +11,7 @@ import torch -def scenario(compute): +def scenario(compute, hkv=1): lazy = compute == "fp16-lazy" compute = "fp16" if lazy else compute from flydsl.compiler.jit_function import MlirCompiler @@ -30,12 +30,14 @@ def spy(*a, **kw): MlirCompiler.compile = spy dev = torch.device("cuda") - heads, hkv = 16, 1 + heads = 16 * hkv host.warm_up_prefill(dev, num_heads=heads, num_kv_heads=hkv, compute_dtype=compute, lazy_rescale=lazy) warm = len(count) kv = torch.zeros((2, 2 * hkv, 128, 72), dtype=torch.uint8, device=dev) late = [] - for n in (1, 255, 256, 257, 511, 512, 513, 1024, 1025, 1026, 1027, 1087, 1088, 1536, 1537, 4096, 16384): + probes = (1, 255, 256, 257, 511, 512, 513, 1024, 1025, 1026, 1027, 1087, 1088, 1536, 1537, 4096, 16384) if hkv == 1 \ + else (1, 128, 129, 256, 257, 384, 385, 512, 513, 768, 769, 1024, 1025, 4096, 16384) + for n in probes: q = torch.zeros((n, heads, 128), dtype=torch.bfloat16, device=dev) idx = torch.full((hkv, n, 16), -1, dtype=torch.int32, device=dev) idx[..., 0] = 0 @@ -49,11 +51,12 @@ def spy(*a, **kw): return dict(warm=warm, late=late, reported=warm_up.late_compiles()) +@pytest.mark.parametrize("hkv", [1, 2]) @pytest.mark.parametrize("compute", ["fp16", "bf16", "fp8", "fp16-lazy"]) -def test_warm_up_then_no_compile(compute, tmp_path): +def test_warm_up_then_no_compile(compute, hkv, tmp_path): if not torch.cuda.is_available() or "gfx950" not in torch.cuda.get_device_properties(0).gcnArchName: pytest.skip("requires gfx950") - proc = subprocess.run([sys.executable, "-m", __name__, compute], capture_output=True, text=True, timeout=1500, + proc = subprocess.run([sys.executable, "-m", __name__, compute, str(hkv)], capture_output=True, text=True, timeout=1500, env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-6000:] @@ -62,4 +65,4 @@ def test_warm_up_then_no_compile(compute, tmp_path): if __name__ == "__main__": - print(json.dumps(scenario(sys.argv[1]))) + print(json.dumps(scenario(sys.argv[1], int(sys.argv[2])))) diff --git a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_compile.py b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_compile.py index c688cb9..ead701e 100644 --- a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_compile.py +++ b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_compile.py @@ -25,6 +25,10 @@ # upstream vLLM's DSpark draft block (non-causal), TP4 up to 256 requests ("tp4_spec7_noncausal_p128", 7, 1, 128, False, range(1, 257)), ("tp4_spec3_p128", 4, 1, 128, True, range(1, 257)), + # TP2 (32 / 2 heads): the DSpark 7 and DSpark 3 blocks (host._TP2_PLANS) + ("tp2_spec7_p128", 7, 2, 128, True, range(1, 257)), + ("tp2_spec7_noncausal_p128", 7, 2, 128, False, range(1, 257)), + ("tp2_spec3_noncausal_p128", 3, 2, 128, False, range(1, 257)), ] @@ -57,6 +61,8 @@ def test_no_compile_after_warmup(name, ql, hkv, page, causal, served, monkeypatc out = torch.empty_like(q) def variant(reqs): + if hkv == 2: + return host._plan(reqs, hkv, page, ql, 4096, causal, 8, host._cu_count(q.device), 4, None) tile = host._tp4_q7_rows(reqs) if ql == 7 and hkv == 1 and page == 128 else 0 return host.launch_splits(q, full[0], reqs), tile diff --git a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_decode.py b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_decode.py index 7738547..fdcf103 100644 --- a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_decode.py +++ b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_decode.py @@ -4,7 +4,9 @@ Served shapes (README, "What it does"): TP4 (16 query heads, 1 KV head, 128-token pages): spec3 = 4 rows per request, bf16 output; the DSpark draft's 7-row block, causal -and non-causal, with the MXFP8 epilogue for o_proj. Every case: each request and KV head +and non-causal, with the MXFP8 epilogue for o_proj. TP2 (32 query heads, 2 KV heads): the +7-row and 3-row blocks at every tier of ``host._TP2_PLANS`` (query tiles, LDS-ring kernel, +request-wide kernel), with the MXFP8 epilogue for o_proj (K 4096). Every case: each request and KV head against the fp64 reference, three launches bit for bit equal, and a CUDA graph replayed into NaN-filled outputs. The batches carry e4m3 NaN bytes in every slot past a sequence end (the kernel must not read them) and, in the ragged cases, padding requests @@ -72,6 +74,28 @@ # ragged rows (a combining wave reused, a partial last row group) + padding requests ("ragged_781_p128_r9", 9, (7, 8, 1), 1, 128, False, 3), ] +# TP2 (32 / 2 heads): every boundary of host._TP2_PLANS, ragged blocks and padding requests +CASES += [ + (f"tp2_r7_p128_noncausal_r{r}", r, 7, 2, 128, False, 0) + for r in (1, 2, 3, 4, 5, 6, 7, 10, 11, 16, 17, 32, 33, 40, 41, 64, 65, 100) +] + [ + (f"tp2_r7_p128_r{r}", r, 7, 2, 128, True, 0) for r in (1, 5, 11, 17, 33, 41, 65) +] + [ + (f"tp2_r7_ragged_p128{'' if c else '_noncausal'}_r{r}", r, (7, 1, 3, 2), 2, 128, c, pad) + for r, pad in ((4, 1), (7, 3), (20, 3), (45, 3), (70, 3)) + for c in (True, False) +] + [ + ("tp2_r7_ragged_p128_noncausal_r28", 28, (7, 1, 3, 2, 4, 5, 6), 2, 128, False, 3), + ("tp2_r7_ragged_p128_r100", 100, (7, 1, 3, 2, 4, 5, 6), 2, 128, True, 5), +] + [ + (f"tp2_r3_p128_noncausal_r{r}", r, 3, 2, 128, False, 0) + for r in (1, 4, 5, 8, 9, 16, 17, 24, 25, 32, 33, 64, 65, 128, 129) +] + [ + (f"tp2_r3_p128_r{r}", r, 3, 2, 128, True, 0) for r in (1, 17, 65) +] + [ + ("tp2_r3_ragged_p128_noncausal_r10", 10, (3, 1, 2), 2, 128, False, 2), + ("tp2_ragged_781_p128_r9", 9, (7, 8, 1), 2, 128, False, 3), +] OPROJ_N = 6144 # the TP4 o_proj (N 6144, K 2048) @@ -141,16 +165,19 @@ def launch(batch, max_ql, causal, out=None, quant_out=None, **kw): def spy_names(monkeypatch): - """The kernel names ``host._get_kernel`` hands out.""" + """The kernel names ``host._get_kernel`` / ``host._get_ring_kernel`` hand out.""" names = [] - real = host._get_kernel - def spy(*args): - exe = real(*args) - names.append(exe.kernel_name) - return exe + def spy(real): + def get(*args): + exe = real(*args) + names.append(exe.kernel_name) + return exe + + return get - monkeypatch.setattr(host, "_get_kernel", spy) + monkeypatch.setattr(host, "_get_kernel", spy(host._get_kernel)) + monkeypatch.setattr(host, "_get_ring_kernel", spy(host._get_ring_kernel)) return names @@ -202,7 +229,7 @@ def test_decode_contract(name, reqs, ql, hkv, page, causal, pad, monkeypatch): torch.cuda.synchronize() # the dispatched kernel: rows, page, causal, the planned split count (kname,) = set(names) - splits = host.launch_splits(q, batch[0], len(batch[2])) + splits = host.launch_splits(q, batch[0], len(batch[2]), max_query_len=max_ql) assert f"_r{max_ql}_" in kname and f"_s{splits}_w{W}_p{page}_" in kname, kname assert ("_c" in kname.split("_n", 1)[1]) == causal, kname assert same_bits(outs[0], outs[1]) and same_bits(outs[0], outs[2]), name @@ -213,8 +240,8 @@ def test_decode_contract(name, reqs, ql, hkv, page, causal, pad, monkeypatch): assert same_bits(out, outs[0]), name -# the MXFP8 epilogue's served query block: 7 rows a request, 1 KV head, 128-token pages -QUANT_CASES = [c for c in CASES if c[2] == 7 and c[3] == 1] +# the MXFP8 epilogue's served query blocks: TP4 7 rows a request, 1 KV head; TP2 7 and 3 rows, 2 KV heads +QUANT_CASES = [c for c in CASES if (c[2] == 7 and c[3] == 1) or (c[2] in (3, 7) and c[3] == 2)] @pytest.mark.parametrize("name,reqs,ql,hkv,page,causal,pad", QUANT_CASES, ids=[c[0] for c in QUANT_CASES]) @@ -271,6 +298,10 @@ def test_supports_bounds(): assert not supports_swa_decode_fp8kv(q, cache, MAX_QL + 1, W) assert not supports_swa_decode_fp8kv(q[:, :12], cache, 8, W) # 12 query heads per KV head assert not supports_swa_decode_fp8kv(q, cache[:, :, :80], 8, W) # page not a multiple of 32 + q2 = torch.empty((8, 32, D), dtype=torch.bfloat16, device="cuda") + cache2 = torch.empty((4, 2, 128, 2 * D), dtype=torch.uint8, device="cuda") + assert supports_swa_decode_fp8kv(q2, cache2, 7, W) and supports_swa_decode_fp8kv(q2, cache2, 3, W) + assert not supports_swa_decode_fp8kv(q2, cache, 7, W) @pytest.mark.parametrize("causal", [True, False], ids=["causal", "noncausal"]) @@ -295,6 +326,44 @@ def test_tp4_q7_batch_combine_next_to_mfma(reqs, causal): assert torch.equal(out, expected) +@pytest.mark.parametrize("causal", [True, False], ids=["causal", "noncausal"]) +@pytest.mark.parametrize( + "ql,reqs", [(7, r) for r in (1, 6, 16, 17, 33, 41, 64, 65)] + [(3, r) for r in (1, 17, 64, 129)] +) +@torch.inference_mode() +def test_tp2_combine_next_to_mfma(ql, reqs, causal): + """TP2 plans (query tiles, LDS-ring splits, ticket combine) give the same bits with an MFMA spinner on every CU.""" + from tests.mfma_spinner import mfma_spinner + + require_gfx950() + gen = torch.Generator().manual_seed(127 + reqs) + batch = make_batch(reqs, ql, 0, gen, 128, 2) + expected = torch.empty_like(batch[4]) + launch(batch, ql, causal, expected) + main, side = torch.cuda.Stream(), torch.cuda.Stream(priority=-1) + spin = mfma_spinner(torch.cuda.get_device_properties(0).multi_processor_count) + for _ in range(8): + spin(200_000, side) + with torch.cuda.stream(main): + out = torch.empty_like(expected) + launch(batch, ql, causal, out) + torch.cuda.synchronize() + assert torch.equal(out, expected) + + +def test_tp2_plans_fit_the_scratch(): + """Every TP2 plan (1-256 requests) fits the tickets and split partials, and the tables are ordered.""" + for ql, plan in host._TP2_PLANS.items(): + lasts = [last for last, _, _ in plan] + assert lasts[-1] is None and lasts[:-1] == sorted(lasts[:-1]) + for reqs in range(1, 257): + splits, ring_rows, batch_rows, group_rows = host._plan(reqs, 2, 128, ql, W, False, 8, 256, 4, None) + assert not batch_rows + work = -(-reqs * ({1: 7, 2: 4, 3: 3}[group_rows] if group_rows else 1) // 8) * 8 * 2 + rows = group_rows or ql + assert work <= host.MAX_SPLIT_WGS and work * splits * rows <= host.MAX_SPLIT_WGS * MAX_QL, (ql, reqs) + + @pytest.mark.parametrize("causal", [True, False], ids=["causal", "noncausal"]) @pytest.mark.parametrize("reqs", [1, 8, 16, 32, 64]) @torch.inference_mode() @@ -338,7 +407,7 @@ def test_tp4_q7_group_cross_batch_cosine(): def test_layer_blocker(): from amd_kernels.attention.swa_decode_fp8kv_gqa16_d128 import layer_blocker - for heads, kv_heads, window in ((16, 1, 4096), (16, 1, max_window(MAX_QL))): + for heads, kv_heads, window in ((16, 1, 4096), (16, 1, max_window(MAX_QL)), (32, 2, 4096)): assert layer_blocker(128, heads, kv_heads, window) is None for args in ((64, 16, 1, 4096), (128, 8, 1, 4096), (128, 16, 2, 4096), (128, 16, 1, None), (128, 16, 1, 0), (128, 16, 1, max_window(MAX_QL) + 1)): diff --git a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input.py b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input.py index aa06571..a8f58ba 100644 --- a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input.py +++ b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""The MXFP8 output as the TP4 DSpark draft's o_proj input (vLLM with the library's draft -path and GEMMs): at the served draft shapes (16 / 1 heads, 128-token pages, -window 4096, 7 rows a request = DSpark 7, 3 = DSpark 3, non-causal as the served draft -and causal; 1-64 requests) the epilogue writes, in the code layout +"""The MXFP8 output as the DSpark draft's o_proj input (vLLM with the library's draft +path and GEMMs): at the served draft shapes (TP4 16 / 1 heads, o_proj K 2048; TP2 32 / 2 +heads, K 4096; 128-token pages, window 4096, 7 rows a request = DSpark 7, 3 = DSpark 3, +non-causal as the served draft and causal) the epilogue writes, in the code layout ``gemm.prequant_code_tile_rows`` names for the row count, the bytes the served quantizer (``gemm.mxfp8_linear``'s activation quantizer) makes of the bf16 output, so the o_proj GEMM on those codes equals the bf16 chain's GEMM bit for bit wherever both take the @@ -20,32 +20,37 @@ from .test_decode import launch, make_batch, require_gfx950 -N, K = 6144, 2048 # the TP4 o_proj +N, K = 6144, 2048 # the TP4 o_proj (TP2: K 4096) # 3 rows (DSpark 3) take the generic kernel (kernel.py): 6 splits at 1-40 requests, 5 at # 41-48, 4 at 49-64. Its non-causal MXFP8 code objects with 3, 5 or 6 splits read row 0's # P right after the v_exp_f32 writing it; an inline-asm bf16 pack there gets no wait state # (NaN codes for row 0, heads 0-3 and 8-11), the native fptrunc does. # test_split_counts_ql3 walks every split count. SERVED = [ - pytest.param(ql, r, causal, id=f"ql{ql}-r{r}-{'c' if causal else 'nc'}") + pytest.param(ql, r, causal, 1, id=f"ql{ql}-r{r}-{'c' if causal else 'nc'}") for ql, reqs in ((7, (1, 2, 4, 5, 6, 7, 8, 11, 16, 32, 48, 49, 64)), (3, (1, 11, 16, 32, 41, 48, 64))) for r in reqs for causal in (False, True) +] + [ + pytest.param(ql, r, causal, 2, id=f"tp2-ql{ql}-r{r}-{'c' if causal else 'nc'}") + for ql, reqs in ((7, (1, 3, 5, 6, 11, 16, 17, 33, 41, 64, 65)), (3, (1, 5, 9, 17, 25, 33, 65, 129))) + for r in reqs + for causal in (False, True) ] -def oproj_weight(seed=3): +def oproj_weight(seed=3, k=K): gen = torch.Generator(device="cuda").manual_seed(seed) - w = (torch.randn((N, K), generator=gen, device="cuda") * 0.5).to(torch.float8_e4m3fn) - s = torch.randint(118, 124, (N, K // 32), generator=gen, device="cuda").to(torch.uint8) + w = (torch.randn((N, k), generator=gen, device="cuda") * 0.5).to(torch.float8_e4m3fn) + s = torch.randint(118, 124, (N, k // 32), generator=gen, device="cuda").to(torch.uint8) return G.preshuffle_mxfp8_weight(w, s) -def code_buffers(rows, tile, fill=0xA5): +def code_buffers(rows, tile, fill=0xA5, k=K): pad = max(tile, 16) rows_p = (rows + pad - 1) // pad * pad - codes = torch.full((rows_p, K), fill, dtype=torch.uint8, device="cuda") - scales = torch.full(((rows_p + 31) // 32 * 32, K // 32), fill, dtype=torch.uint8, device="cuda") + codes = torch.full((rows_p, k), fill, dtype=torch.uint8, device="cuda") + scales = torch.full(((rows_p + 31) // 32 * 32, k // 32), fill, dtype=torch.uint8, device="cuda") return codes, scales @@ -57,25 +62,26 @@ def gemm_codes(codes, scales, tile, rows, w, ws): def real_bytes(codes, scales, rows, tile): + k = codes.shape[1] if tile: - c = codes.view(-1)[tile_code_offsets(rows, K, codes.device)] + c = codes.view(-1)[tile_code_offsets(rows, k, codes.device)] else: c = codes[:rows] - return c, scales.view(-1)[aiter_scale_offsets(rows, K, scales.device)] + return c, scales.view(-1)[aiter_scale_offsets(rows, k, scales.device)] def check_served_bytes(batch, ql, causal, **kw): """The epilogue's codes / scales equal the served quantizer of the bf16 output of the same launch configuration (``kw``: explicit splits / XCD grid).""" q = batch[4] - rows = q.shape[0] - tile = G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) + rows, k = q.shape[0], q.shape[1] * q.shape[2] + tile = G.prequant_code_tile_rows(rows, N, k, torch.bfloat16) out = torch.full_like(q, float("nan")) launch(batch, ql, causal, out, **kw) - codes, scales = code_buffers(rows, tile) + codes, scales = code_buffers(rows, tile, k=k) launch(batch, ql, causal, None, (codes, scales, tile), **kw) torch.cuda.synchronize() - out2d = out.view(rows, K) + out2d = out.view(rows, k) if tile: want_c, want_s = G.mxfp8_e4m3_quantize_shuffled(out2d, tile) else: @@ -100,40 +106,41 @@ def test_split_counts_ql3(splits, xcd, causal): check_served_bytes(batch, 3, causal, splits=splits, xcd_count=xcd) -@pytest.mark.parametrize("ql,reqs,causal", SERVED) +@pytest.mark.parametrize("ql,reqs,causal,hkv", SERVED) @torch.inference_mode() -def test_served_quantizer_bytes_and_oproj(ql, reqs, causal): +def test_served_quantizer_bytes_and_oproj(ql, reqs, causal, hkv): require_gfx950() gen = torch.Generator().manual_seed(reqs * 10 + ql) - batch = make_batch(reqs, ql, 0, gen, 128, 1) - rows = batch[4].shape[0] + batch = make_batch(reqs, ql, 0, gen, 128, hkv) + rows, k = batch[4].shape[0], hkv * 16 * 128 out2d, codes, scales, tile = check_served_bytes(batch, ql, causal) - w, ws = oproj_weight() + w, ws = oproj_weight(k=k) got = gemm_codes(codes, scales, tile, rows, w, ws) ref = G.mxfp8_linear(out2d, w, ws, torch.bfloat16) torch.cuda.synchronize() - if G._select_config(rows, N, K, torch.bfloat16).kernel != G._DECODE_FQ_KERNEL: + if G._select_config(rows, N, k, torch.bfloat16).kernel != G._DECODE_FQ_KERNEL: assert torch.equal(got, ref) else: # the bf16 chain's fused-quant row sums K in another order rel = (got.float() - ref.float()).norm() / ref.float().norm() assert rel < 1e-2, float(rel) -GRAPH = [(7, 1), (7, 6), (7, 7), (7, 16), (7, 49), (7, 64), (3, 16), (3, 48), (3, 64)] +GRAPH = [(7, 1, 1), (7, 6, 1), (7, 7, 1), (7, 16, 1), (7, 49, 1), (7, 64, 1), (3, 16, 1), (3, 48, 1), (3, 64, 1), + (7, 1, 2), (7, 6, 2), (7, 17, 2), (7, 41, 2), (7, 65, 2), (3, 1, 2), (3, 33, 2), (3, 129, 2)] -@pytest.mark.parametrize("ql,reqs", GRAPH, ids=[f"ql{a}-r{b}" for a, b in GRAPH]) +@pytest.mark.parametrize("ql,reqs,hkv", GRAPH, ids=[f"{'tp2-' if c == 2 else ''}ql{a}-r{b}" for a, b, c in GRAPH]) @torch.inference_mode() -def test_graph_replay_attention_and_oproj(ql, reqs): +def test_graph_replay_attention_and_oproj(ql, reqs, hkv): """attention (MXFP8 out, non-causal) + o_proj captured: every replay writes the eager codes, scales and o_proj output, also after new q values are copied in.""" require_gfx950() gen = torch.Generator().manual_seed(reqs * 3 + ql) - batch = make_batch(reqs, ql, 0, gen, 128, 1) + batch = make_batch(reqs, ql, 0, gen, 128, hkv) q = batch[4] - rows = q.shape[0] - tile = G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) - w, ws = oproj_weight() + rows, k = q.shape[0], hkv * 16 * 128 + tile = G.prequant_code_tile_rows(rows, N, k, torch.bfloat16) + w, ws = oproj_weight(k=k) q_a, q_b = q.clone(), (torch.randn_like(q.float()) * 1.5).bfloat16() def chain(codes, scales): @@ -143,12 +150,12 @@ def chain(codes, scales): wants = [] for src in (q_a, q_b): q.copy_(src) - codes, scales = code_buffers(rows, tile) + codes, scales = code_buffers(rows, tile, k=k) o = chain(codes, scales) torch.cuda.synchronize() wants.append((real_bytes(codes, scales, rows, tile), o.clone())) q.copy_(q_a) - g_codes, g_scales = code_buffers(rows, tile) + g_codes, g_scales = code_buffers(rows, tile, k=k) chain(g_codes, g_scales) torch.cuda.synchronize() graph = torch.cuda.CUDAGraph() diff --git a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input_entry.py b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input_entry.py index 6484a49..2545043 100644 --- a/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input_entry.py +++ b/tests/attention/swa_decode_fp8kv_gqa16_d128/test_oproj_input_entry.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """``swa_decode_fp8kv_oproj_input`` (the attention as its o_proj's input, the MXFP8 / bf16 choice and the codes' buffers in the library): bit for bit what the kernel entry writes when the caller makes the same choice itself -(the TP4 DSpark 3 / 7 shapes, 1-64 requests); its first call of a mode compiles every request count's variant.""" +(the TP4 and TP2 DSpark 3 / 7 shapes); its first call of a mode compiles every request count's variant.""" import json import os import subprocess @@ -15,19 +15,20 @@ from amd_kernels.gemm.mxfp8_common import gemm as G from .test_decode import SM_SCALE, W, make_batch, require_gfx950, scales -from .test_oproj_input import K, N, code_buffers, real_bytes +from .test_oproj_input import N, code_buffers, real_bytes +@pytest.mark.parametrize("hkv", [1, 2], ids=["tp4", "tp2"]) @pytest.mark.parametrize("ql", [3, 7]) @pytest.mark.parametrize("reqs", [1, 2, 5, 11, 16, 48, 64]) @pytest.mark.parametrize("causal", [False, True]) -def test_entry_equals_kernel_calls(ql, reqs, causal): +def test_entry_equals_kernel_calls(ql, reqs, causal, hkv): require_gfx950() gen = torch.Generator().manual_seed(reqs * 10 + ql) pool, bt, sl, qsl, q, _ = [t.cuda() if isinstance(t, torch.Tensor) else t - for t in make_batch(reqs, ql, 0, gen, 128, 1)] + for t in make_batch(reqs, ql, 0, gen, 128, hkv)] ks, vs = scales() - rows = q.shape[0] + rows, k = q.shape[0], hkv * 16 * 128 args = (pool, bt, sl, qsl, ql, SM_SCALE, W, causal, ks, vs) # bf16: the entry without an o_proj (3-D and 2-D q) want = torch.full_like(q, float("nan")) @@ -41,15 +42,15 @@ def test_entry_equals_kernel_calls(ql, reqs, causal): assert isinstance(got, torch.Tensor) and torch.equal(got.view(torch.int16), want.view(rows, -1).view(torch.int16)) return codes, sc, tile = got - assert tile == G.prequant_code_tile_rows(rows, N, K, torch.bfloat16) - want_c, want_s = code_buffers(rows, tile) + assert tile == G.prequant_code_tile_rows(rows, N, k, torch.bfloat16) + want_c, want_s = code_buffers(rows, tile, k=k) swa_decode_fp8kv(q, *args, None, quant_out=(want_c, want_s, tile)) torch.cuda.synchronize() for a, b in zip(real_bytes(codes, sc, rows, tile), real_bytes(want_c, want_s, rows, tile)): assert torch.equal(a, b) -def scenario(): +def scenario(hkv): from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up warm_up.set_auto(True) @@ -67,7 +68,7 @@ def spy(*a, **k): for ql in (3, 7): gen = torch.Generator().manual_seed(reqs) pool, bt, sl, qsl, q, _ = [t.cuda() if isinstance(t, torch.Tensor) else t - for t in make_batch(reqs, ql, 0, gen, 128, 1, ctx_hi=600)] + for t in make_batch(reqs, ql, 0, gen, 128, hkv, ctx_hi=600)] before = len(late) for oproj in (None, N): swa_decode_fp8kv_oproj_input(q.view(q.shape[0], -1), pool, bt, sl, qsl, ql, SM_SCALE, W, False, ks, @@ -78,9 +79,10 @@ def spy(*a, **k): return {"late": bad, "reported": warm_up.late_compiles(), "warmed": len(warm_up.warmed())} -def test_first_call_warm_up_compiles_every_variant(tmp_path): +@pytest.mark.parametrize("hkv", [1, 2], ids=["tp4", "tp2"]) +def test_first_call_warm_up_compiles_every_variant(tmp_path, hkv): require_gfx950() - proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=1500, + proc = subprocess.run([sys.executable, "-m", __name__, str(hkv)], capture_output=True, text=True, timeout=1500, env={**os.environ, "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:] result = json.loads(proc.stdout.strip().splitlines()[-1]) @@ -88,4 +90,4 @@ def test_first_call_warm_up_compiles_every_variant(tmp_path): if __name__ == "__main__": - print(json.dumps(scenario())) + print(json.dumps(scenario(int(sys.argv[1])))) diff --git a/tests/collectives/all_gather_i64_c7_ranks2/__init__.py b/tests/collectives/all_gather_i64_c7_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/all_gather_i64_c7_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/all_gather_i64_c7_ranks2/test_i64_c7_gather.py b/tests/collectives/all_gather_i64_c7_ranks2/test_i64_c7_gather.py new file mode 100644 index 0000000..af38e14 --- /dev/null +++ b/tests/collectives/all_gather_i64_c7_ranks2/test_i64_c7_gather.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +import socket +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from amd_kernels.collectives.all_gather_i64_c7_ranks2.reference import reference + +def run(rank,port): + torch.set_num_threads(8);torch.cuda.set_device(rank) + dist.init_process_group('gloo',init_method=f'tcp://127.0.0.1:{port}',world_size=2,rank=rank) + dev=torch.device('cuda',rank) + from amd_kernels.collectives.all_gather_i64_c7_ranks2 import I64C7AllGather + comm=I64C7AllGather(rank,dist.group.WORLD,dev);comm.warmup() + from unittest.mock import patch + import flydsl.compiler.jit_function as jf + with patch.object(jf.MlirCompiler,'compile',side_effect=AssertionError('compile after warmup')): + for case,counts in enumerate(([1,0],[3,2],[4]*2,[8]*2,[16]*2,[32]*2,[64]*2,[128]*2,[0,256],[256,0],[5,0],[0,8],[0]*2)): + stride=max(max(counts),1);R=sum(counts) + src=torch.tensor([r*stride+i for r,n in enumerate(counts) for i in range(n)],dtype=torch.int64) + src=src[torch.randperm(R,generator=torch.Generator().manual_seed(case))] + # A larger source with stride14 exercises non-contiguous row views and source padding. + x=torch.empty(counts[rank],14,device=dev,dtype=torch.int64)[:,:7] + out=torch.empty(R,7,device=dev,dtype=torch.int64);smap=src.to(dev) + stream=torch.cuda.Stream();stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream):comm(x,smap,stride,out) + torch.cuda.synchronize();dist.barrier();g=torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for _ in range(10):comm(x,smap,stride,out) + for rep in range(5): + gen=torch.Generator().manual_seed(1000+case*10+rep) + local=[torch.randint(-(1<<60),1<<60,(n,7),generator=gen,dtype=torch.int64) for n in counts] + x.copy_(local[rank].to(dev));now=src.flip(0) if rep%2 else src + smap.copy_(now.to(dev));out.fill_(-123);dist.barrier();g.replay();torch.cuda.synchronize() + assert torch.equal(out.cpu(),reference(local,now,stride)),(rank,case,rep) + del g + comm.close();dist.destroy_process_group() + +def test_two_rank_graph_bytes_and_padding(): + if torch.cuda.device_count()<2:pytest.skip('requires two MI355X GPUs') + sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1];sock.close() + mp.spawn(run,args=(port,),nprocs=2,join=True) diff --git a/tests/collectives/all_gather_sampler_i64_c8_ranks2/__init__.py b/tests/collectives/all_gather_sampler_i64_c8_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/all_gather_sampler_i64_c8_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/all_gather_sampler_i64_c8_ranks2/test_sampler_gather.py b/tests/collectives/all_gather_sampler_i64_c8_ranks2/test_sampler_gather.py new file mode 100644 index 0000000..0024e02 --- /dev/null +++ b/tests/collectives/all_gather_sampler_i64_c8_ranks2/test_sampler_gather.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +import os,socket +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp +from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2.reference import reference + + +def fixtures(counts,seed,has=True,ncols=8,fixed_cu=None): + g=torch.Generator().manual_seed(seed) + local=[] + for r,n in enumerate(counts): + tok=torch.randint(-(1<<60),1<<60,(n,ncols),generator=g,dtype=torch.int64) + ns=torch.randint(1,ncols+1,(n,),generator=g,dtype=torch.int32) + nr=torch.randint(0,8,(n,),generator=g,dtype=torch.int32) + lens=torch.randint(1,9,(n,),generator=g,dtype=torch.int32) + cu=torch.cat([torch.zeros(1,dtype=torch.int32),lens.cumsum(0).int()]) + if fixed_cu is not None:cu=fixed_cu[r] + nan=torch.randint(0,200065,(int(cu[-1]),),generator=g,dtype=torch.int32) if has else None + local.append((tok,ns,nr,nan,cu)) + return local + + +def run(rank,port): + torch.set_num_threads(8);torch.cuda.set_device(rank) + dist.init_process_group('gloo',init_method=f'tcp://127.0.0.1:{port}',world_size=2,rank=rank) + dev=torch.device('cuda',rank) + from amd_kernels.collectives.all_gather_sampler_i64_c8_ranks2 import SamplerAllGather + comm=SamplerAllGather(rank,dist.group.WORLD,dev);comm.warmup() + # All variants share a single launcher; shape, NaN presence and columns are runtime values. + import flydsl.compiler.jit_function as jf + from unittest.mock import patch + with patch.object(jf.MlirCompiler,'compile',side_effect=AssertionError('compile after warmup')): + for ic,counts in enumerate(([1,0],[3,2],[4]*2,[8]*2,[16]*2,[32]*2,[64]*2,[128]*2,[0,256],[256,0],[5,0],[0,8],[0]*2)): + stride=max(max(counts),1);R=sum(counts) + src=torch.tensor([r*stride+i for r,n in enumerate(counts) for i in range(n)],dtype=torch.int64) + if R:src=src[torch.randperm(R,generator=torch.Generator().manual_seed(ic))] + for has in (False,True): + for ncols in (1,7,8): + local=fixtures(counts,100+ic,has,ncols) + inp=[a.to(dev) if a is not None else None for a in local[rank]] + # Counts/prefixes may be views with only natural four-byte alignment. + for k in (1,2,3,4): + if inp[k] is not None: + buf=torch.empty(inp[k].numel()+1,device=dev,dtype=torch.int32) + buf[1:].copy_(inp[k]);inp[k]=buf[1:] + if ncols==7: + strided=torch.empty(counts[rank],12,device=dev,dtype=torch.int64)[:,:7] + strided.copy_(inp[0]);inp[0]=strided + smap=src.to(dev) + out=(torch.empty(R,8,device=dev,dtype=torch.int64),torch.empty(R+1,device=dev,dtype=torch.int32)[1:], + torch.empty(R+1,device=dev,dtype=torch.int32)[1:],torch.empty(R+1,device=dev,dtype=torch.int32)[1:] if has else None) + def call():return comm(*inp,smap,stride,*out) + call();torch.cuda.synchronize();want=reference(local,src,stride) + for a,b in zip(out,want): + if a is not None:assert torch.equal(a.cpu(),b),(rank,counts,has,ncols) + stream=torch.cuda.Stream();stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream):call() + torch.cuda.synchronize();dist.barrier() + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(10):call() + for rep in range(3): + # Same data/map pointers, fresh payload and reordered output every replay. + new=fixtures(counts,1000+rep+ic,has,ncols,[a[4] for a in local]) + for dst,ss in zip(inp,new[rank]): + if dst is not None:dst.copy_(ss) + now=src.flip(0) if rep%2 else src + smap.copy_(now.to(dev)) + for a in out: + if a is not None:a.fill_(-123) + dist.barrier();graph.replay();torch.cuda.synchronize() + want=reference(new,now,stride) + for a,b in zip(out,want): + if a is not None:assert torch.equal(a.cpu(),b),(rank,ic,has,ncols,rep) + del graph + # A thousand double-slot uses, sequence wrap, duplicate map entries. + counts=[128]*2;local=fixtures(counts,9);inp=[a.to(dev) for a in local[rank]] + src=torch.tensor([0,127,128,255,1,126,129,254]*32,dtype=torch.int64);smap=src.to(dev) + out=(torch.empty(256,8,device=dev,dtype=torch.int64),*[torch.empty(256,device=dev,dtype=torch.int32) for _ in range(3)]) + comm.counter.fill_(2147483645) + # All ranks' uncached flags must be synchronized to the reset sequence first. + from amd_kernels.common.hip_runtime import load_hip_runtime + import ctypes + lib=load_hip_runtime();lib.hipMemcpy.argtypes=[ctypes.c_void_p,ctypes.c_void_p,ctypes.c_size_t,ctypes.c_int] + v=ctypes.c_int32(2147483645) + assert lib.hipMemcpy(ctypes.c_void_p(comm.flags[rank]),ctypes.byref(v),4,1)==0 + dist.barrier() + for _ in range(1000):comm(*inp,smap,128,*out) + torch.cuda.synchronize();want=reference(local,src,128) + for a,b in zip(out,want):assert torch.equal(a.cpu(),b) + comm.close();dist.destroy_process_group() + + +def test_two_rank_graph_bytes_and_wrap(): + if torch.cuda.device_count()<2:pytest.skip('requires two MI355X GPUs') + sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1];sock.close() + mp.spawn(run,args=(port,),nprocs=2,join=True) diff --git a/tests/collectives/all_to_all_bf16_n100032_ranks2/__init__.py b/tests/collectives/all_to_all_bf16_n100032_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/all_to_all_bf16_n100032_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/all_to_all_bf16_n100032_ranks2/test_exchange.py b/tests/collectives/all_to_all_bf16_n100032_ranks2/test_exchange.py new file mode 100644 index 0000000..4619909 --- /dev/null +++ b/tests/collectives/all_to_all_bf16_n100032_ranks2/test_exchange.py @@ -0,0 +1,134 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two spawned ranks: raw bits, dynamic GPU counts, changing slots, graphs. + +No sampling arithmetic occurs here. Exact byte exchange proves equality for +all processor parameters and seeds, including NaNs and structured masks. +""" +import os +import socket +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.reference import N, WORLD, merge_received_reference + + +def _rccl_reference(x, counts, rank): + n = counts[rank] + recv = torch.empty(WORLD * n * N * 2, dtype=torch.uint8, device=x.device) + dist.all_to_all_single(recv, x.view(torch.uint8).flatten(), + output_split_sizes=[n * N * 2] * WORLD, + input_split_sizes=[c * N * 2 for c in counts]) + return merge_received_reference(recv.view(torch.bfloat16).view(WORLD, n, N)) + + +def _worker(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=WORLD, + timeout=timedelta(minutes=3), device_id=dev) + group = dist.new_group(backend="gloo") + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll + comm = RowAllToAll(rank, group, dev) + generator = torch.Generator(device=dev).manual_seed(17001 + rank) + # Dense slots, odd counts, unequal owners, empty receivers, all on one + # owner. 7 and 8 rows per request exercise both draft and target paths. + patterns = [] + for reqs in (1, 3, 10, 16, 31, 32, 64, 128, 256): + dense = torch.bincount(torch.arange(reqs) % WORLD, minlength=WORLD).tolist() + slots = torch.randperm(256, generator=torch.Generator().manual_seed(reqs))[:reqs] + random = torch.bincount(slots % WORLD, minlength=WORLD).tolist() + for requests in (dense, random): + for steps in (7, 8): + patterns.append([v * steps for v in requests]) + patterns += [[0, 0], [0, 512], [8, 0], [0, 8], [2048, 0], [0, 2048], [1023, 1], [7, 1017]] + comm.warmup() + # Start close to signed wrap. All flags must match the counter before + # the first exchange: otherwise a peer could appear far ahead. + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import kernel + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.host import _run + import flydsl.compiler as flyc + import flydsl.expr as fx + from amd_kernels.common import buffer_ops + @flyc.kernel(name="test_row_exchange_init_wrap", known_block_size=[64, 1, 1]) + def init_flag(arg_flag: fx.Int64): + fr = buffer_ops.create_buffer_resource_from_addr(arg_flag, num_records_bytes=4) + buffer_ops.buffer_store(fx.Int32(2147483645), fr, 0, mask=fx.thread_idx.x == 0, cache_modifier=0x11) + @flyc.jit + def launch_test_row_exchange_init_wrap(arg_flag: fx.Int64, stream: fx.Stream): + init_flag(arg_flag).launch(grid=(1,1,1), block=(64,1,1), stream=stream) + _run(launch_test_row_exchange_init_wrap, comm.flags[rank], torch.cuda.current_stream()) + comm.counter.fill_(2147483645) + torch.cuda.synchronize() + dist.barrier() + from flydsl.compiler.jit_function import MlirCompiler + compile_calls = [] + original = MlirCompiler.compile + def count_compile(*args, **kwargs): + compile_calls.append(1) + return original(*args, **kwargs) + MlirCompiler.compile = count_compile + try: + for k, counts in enumerate(patterns): + cs = torch.tensor(counts, device=dev, dtype=torch.int32) + n = counts[rank] + # All bf16 bit patterns: signed zero, subnormals, +/-inf, all + # NaN payloads. Integer generation avoids canonicalizing NaNs. + sets = [torch.randint(-32768, 32768, (sum(counts), N), dtype=torch.int16, + device=dev, generator=generator).view(torch.bfloat16) for _ in range(3)] + outs = [torch.empty(n, WORLD * N, device=dev, dtype=torch.bfloat16) for _ in sets] + refs = [_rccl_reference(x, counts, rank) for x in sets] + for x, out, ref in zip(sets, outs, refs): + comm(x, cs, out) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), ref.view(torch.int16)), (rank, k, "eager") + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for x, out in zip(sets, outs): + comm(x, cs, out) + for repeat in range(3): + for out in outs: + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + for out, ref in zip(outs, refs): + assert torch.equal(out.view(torch.int16), ref.view(torch.int16)), (rank, k, repeat) + del graph, sets, outs, refs + # Same captured addresses, new counts and inputs on every replay. + # This detects a host plan or sequence frozen into the graph. + x = torch.empty(80, N, dtype=torch.bfloat16, device=dev) + out = torch.empty(80, WORLD * N, dtype=torch.bfloat16, device=dev) + cs = torch.tensor([20] * WORLD, dtype=torch.int32, device=dev) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + comm(x, cs, out) + for k in range(16): + counts = ([80 if j == k % WORLD else 0 for j in range(WORLD)] if k % 2 + else [24, 56]) + cs.copy_(torch.tensor(counts, dtype=torch.int32)) + x.view(torch.int16).random_(-32768, 32768, generator=generator) + ref = _rccl_reference(x, counts, rank) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out[:counts[rank]].view(torch.int16), ref.view(torch.int16)), (rank, "dynamic", k) + assert compile_calls == [], compile_calls + finally: + MlirCompiler.compile = original + del graph + comm.close() + dist.barrier() + dist.destroy_process_group() + + +def test_two_rank_exchange(): + if torch.cuda.device_count() < WORLD: + pytest.skip("requires two gfx950 GPUs") + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + mp.spawn(_worker, args=(port,), nprocs=WORLD, join=True) diff --git a/tests/collectives/pull_collectives_ranks2/__init__.py b/tests/collectives/pull_collectives_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/pull_collectives_ranks2/served.py b/tests/collectives/pull_collectives_ranks2/served.py new file mode 100644 index 0000000..4e06ef3 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/served.py @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The served shapes of the pull collectives: the sequence-parallel prefill on 2 ranks, hidden 6144, the o_proj input +4096 columns per rank (32 heads x 128), up to ``SP_COLL_MAX_TOKENS`` tokens per forward (a DBO microbatch of a +65536-token step), shard rows a multiple of 32.""" + +WORLD = 2 +HIDDEN = 6144 +Q_SIZE = 4096 # this rank's heads x head dim: the all-to-all's d +SP_COLL_MAX_TOKENS = 65536 // 2 +ROUTING_INTS = 64 # the gathered routing per row (int32) + + +def collectives_slot_bytes(hidden: int = HIDDEN, q_size: int = Q_SIZE) -> int: + """The inputs of the largest phase at ``SP_COLL_MAX_TOKENS`` (the reduce-scatter's bf16 partial of every row, the + shard's gathered codes + scales + routing, two bf16 gathers of the shard, alignment slack).""" + rows = SP_COLL_MAX_TOKENS // WORLD + padded = rows * WORLD + assert padded * (q_size + q_size // 32) <= padded * hidden * 2 + return ( + padded * hidden * 2 + + rows * (hidden + hidden // 32 + ROUTING_INTS * 4) + + 2 * rows * hidden * 2 + + 8 * 256 + ) diff --git a/tests/collectives/pull_collectives_ranks2/test_all_gather.py b/tests/collectives/pull_collectives_ranks2/test_all_gather.py new file mode 100644 index 0000000..9927685 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/test_all_gather.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: graph/eager reuse, changing payloads and guarded writes for lossless gather; the compact MoE payload +through the i1536 MoE's gather.""" +import torch +import pytest +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus +from amd_kernels.collectives.pull_collectives_ranks2.reference import all_gather_reference + + +def _worker(rank, errors, num_fields): + from amd_kernels.collectives.pull_collectives_ranks2 import AllGatherComm + dev = torch.device('cuda', rank) + comm = AllGatherComm(rank, dist.group.WORLD, dev, 4*2**20) + comm.warm_up() + compiled = comm.kernel._cf + expected_count = int(comm.counter.item()) + per_call = -(-num_fields // 6) # one launch per six fields + for rows in (1, 3, 17, 129, 352): + inputs = [torch.empty((rows, 192), dtype=torch.uint8, device=dev), + torch.empty((rows, 4), dtype=torch.float32, device=dev), + torch.empty((rows, 4), dtype=torch.int32, device=dev), + torch.empty((rows,6144),dtype=torch.uint8,device=dev), + torch.empty((rows,192),dtype=torch.uint8,device=dev), + torch.empty((rows,768),dtype=torch.uint8,device=dev), + torch.empty((rows,16),dtype=torch.bfloat16,device=dev)][:num_fields] + bases = [torch.empty(t.numel()*2 + 32, dtype=t.dtype, device=dev) for t in inputs] + outputs = [b[16:-16].view(2*rows, *t.shape[1:]) for b, t in zip(bases, inputs)] + comm.all_gather(outputs, inputs) + expected_count += per_call + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(3): + comm.all_gather(outputs, inputs) + for iteration in range(5): + for j, t in enumerate(inputs): + t.copy_((torch.arange(t.numel(),device=dev).view(t.shape) + rank*19 + iteration*7 + j).to(t.dtype)) + bases[j].fill_(91) + if iteration % 2: + comm.all_gather(outputs, inputs) + expected_count += per_call + else: + graph.replay() + expected_count += 3 * per_call + torch.cuda.synchronize() + for t, o, b in zip(inputs, outputs, bases): + other = [None]*2 + dist.all_gather_object(other, t.cpu()) + if not torch.equal(o.cpu(), all_gather_reference(other)): + errors.append(f'gather mismatch r{rank} rows{rows} iteration{iteration}') + if not (torch.all(b[:16] == 91) and torch.all(b[-16:] == 91)): + errors.append(f'guard mismatch r{rank} rows{rows}') + if int(comm.counter.item()) != expected_count: + errors.append(f'counter mismatch rank{rank}') + assert compiled is comm.kernel._cf + + +@pytest.mark.parametrize('num_fields',[3,6,7]) +def test_graph_eager_gather_changed_payloads(num_fields): + run_on_2_gpus(_worker, num_fields, timeout=300) + + +def _compact_worker(rank, errors, spin=False): + from amd_kernels.collectives.pull_collectives_ranks2 import AllGatherComm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, gather_mxfp4_moe + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant + from flydsl.compiler.jit_function import MlirCompiler + dev = torch.device('cuda',rank) + comm = AllGatherComm(rank,dist.group.WORLD,dev,16384*9632) + comm.warm_up() + if spin: + from tests.mfma_spinner import mfma_spinner + spinner = mfma_spinner(32) + side = torch.cuda.Stream(priority=-1) + compiles = [] + original_compile = MlirCompiler.compile + for rows in (1,3,17,129,513,1025,2048,4096,4097,8192,16384): + x = torch.randn((rows,6144),device=dev,dtype=torch.bfloat16) + bits = torch.arange(65536,device=dev,dtype=torch.int32).to(torch.int16).view(torch.bfloat16) + bits = bits[torch.isfinite(bits)] + x.view(-1)[:min(x.numel(),bits.numel())].copy_(bits[:min(x.numel(),bits.numel())]) + x[-1].zero_() + q,s = mxfp4_quant(x) + q8,s8 = shared_quant(x) + ids = torch.arange(rows*4,device=dev,dtype=torch.int32).view(rows,4)%128 + weights = torch.randn((rows,4),device=dev) + source = MoeInput(q,s,weights,ids,q8,s8) + wants = [torch.empty((2*rows,*t.shape[1:]),dtype=t.dtype,device=dev) for t in source.tensors()] + comm.all_gather(wants,source.tensors()) + eager = gather_mxfp4_moe(source,comm,world=2).expand() + torch.cuda.synchronize() + for a,b in zip(eager.tensors(),wants): + assert torch.equal(a.view(torch.uint8),b.view(torch.uint8)),(rank,rows,'eager') + def counting(*a,**kw): + compiles.append(1) + return original_compile(*a,**kw) + MlirCompiler.compile = counting + try: + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = gather_mxfp4_moe(source,comm,world=2).expand() + for iteration in range(3): + for t in source.tensors(): + t.copy_(t.roll(1,0)) + source.topk_weights.add_(.125) + comm.all_gather(wants,source.tensors()) + for t in got.tensors(): + t.view(torch.uint8).fill_(255) + if spin: + with torch.cuda.stream(side): + spinner(100000,side) + graph.replay() + if spin: + torch.cuda.current_stream().wait_stream(side) + torch.cuda.synchronize() + for i,(a,b) in enumerate(zip(got.tensors(),wants)): + assert torch.equal(a.view(torch.uint8),b.view(torch.uint8)),(rank,rows,i,'changed replay') + finally: + MlirCompiler.compile = original_compile + assert not compiles,compiles + + +def test_compact_gather_matches_direct_operands(): + run_on_2_gpus(_compact_worker,timeout=300) + + +def test_compact_gather_with_mfma_work(): + run_on_2_gpus(_compact_worker,True,timeout=300) + + +def _compact_bytes_worker(rank, errors, rows_list): + """The six compact MoE fields (7328 B/row) as raw bytes at the served shard rows, every byte checked, both the + joined six-field stage + publish and the separate stages, eager and replayed.""" + from amd_kernels.collectives.pull_collectives_ranks2 import AllGatherComm + from flydsl.compiler.jit_function import MlirCompiler + dev = torch.device('cuda', rank) + comm = AllGatherComm(rank, dist.group.WORLD, dev, 16384*9632) + comm.warm_up() + real = MlirCompiler.compile + compiles = [] + def counting(*a, **kw): + compiles.append(1) + return real(*a, **kw) + cols = (192, 16, 16, 6144, 192, 768) + for rows in rows_list: + every = [] + for p in range(2): + g = torch.Generator(device=dev).manual_seed(rows * 7 + p) + every.append([torch.randint(0, 256, (rows, c), generator=g, dtype=torch.uint8, device=dev) for c in cols]) + outs = [torch.empty((2*rows, c), dtype=torch.uint8, device=dev) for c in cols] + MlirCompiler.compile = counting + try: + comm.all_gather(outs, every[rank]) + torch.cuda.synchronize() + for i, o in enumerate(outs): + if not torch.equal(o, all_gather_reference([e[i] for e in every])): + errors.append(f'rank {rank} rows {rows} field {i} eager') + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + comm.all_gather(outs, every[rank]) + for t in every[rank]: + t.copy_(t.roll(1, 0)) + for p in range(2): + if p != rank: + for t in every[p]: + t.copy_(t.roll(1, 0)) + for o in outs: + o.fill_(7) + graph.replay() + torch.cuda.synchronize() + for i, o in enumerate(outs): + if not torch.equal(o, all_gather_reference([e[i] for e in every])): + errors.append(f'rank {rank} rows {rows} field {i} replay') + finally: + MlirCompiler.compile = real + del graph + if compiles: + errors.append(f'{rank}: {len(compiles)} compiles after warm-up') + + +def test_compact_fields_served_rows(): + # the part count per field follows its bytes: sizes rise and fall on one communicator + run_on_2_gpus(_compact_bytes_worker, [32, 1024, 2048, 4096, 4128, 8192, 16384, 1024, 16384, 32, 2048], timeout=600) diff --git a/tests/collectives/pull_collectives_ranks2/test_gather_odd_rows.py b/tests/collectives/pull_collectives_ranks2/test_gather_odd_rows.py new file mode 100644 index 0000000..c3e7158 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/test_gather_odd_rows.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks, one fresh ``AllGatherComm`` each: row counts that are not multiples of 32 (or of anything) up to 16384 +per rank, rising and falling, through ``gather_mxfp4_moe`` of the i1536 MoE and through direct gathers whose field +count and part counts change from call to call (a field's part count follows its bytes; a call with more parts than +an earlier one must not wait on tickets the earlier one left behind). Every byte vs ``all_gather_reference``, eager +and replayed.""" +import torch + +from tests.multi_gpu import run_on_2_gpus +from amd_kernels.collectives.pull_collectives_ranks2.reference import all_gather_reference + +# 1 -> 129 -> 2049: the sequence that hung (the first call with a 24-part field after calls of one part per field) +MOE_ROWS = (1, 129, 2049, 3071, 4097, 6143, 8191, 12289, 16383, 16384, 2049, 33, 10001, 1) +BF16_ROWS = (2049, 1, 5003, 16383, 77) + + +def _x(rows, rank, seed): + g = torch.Generator(device="cuda").manual_seed(seed * 2 + rank) + return torch.randn((rows, 6144), generator=g, device="cuda", dtype=torch.bfloat16) + + +def _moe_input(x): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant + + rows = x.shape[0] + q, s = mxfp4_quant(x) + q8, s8 = shared_quant(x) + ids = (torch.arange(rows * 4, device=x.device, dtype=torch.int32).view(rows, 4) * 7 + rows) % 128 + weights = x[:, :4].float().contiguous() + return MoeInput(q, s, weights, ids, q8, s8) + + +def _worker(rank, errors): + from amd_kernels.collectives.pull_collectives_ranks2 import AllGatherComm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import gather_mxfp4_moe + + dev = torch.device("cuda", rank) + comm = AllGatherComm(rank, torch.distributed.group.WORLD, dev, (16384 * 2 * 6144 + 6 * 256 + 255) // 256 * 256) + comm.warm_up() + + def check(got, every, what): + torch.cuda.synchronize() + for i, o in enumerate(got): + want = all_gather_reference([e[i] for e in every]) + if not torch.equal(o.view(torch.uint8), want.view(torch.uint8)): + errors.append(f"rank {rank} {what} field {i}") + + for step, rows in enumerate(MOE_ROWS): + every = [_moe_input(_x(rows, p, step)).compact().tensors() for p in range(2)] + local = _moe_input(_x(rows, rank, step)) + check(gather_mxfp4_moe(local, comm, world=2).tensors(), every, f"moe eager rows {rows}") + if rows in (2049, 16383): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = gather_mxfp4_moe(local, comm, world=2) + for t in got.tensors(): + t.view(torch.uint8).fill_(255) + graph.replay() + check(got.tensors(), every, f"moe replay rows {rows}") + del graph + + # one wide field (up to 64 parts), seven fields (two launches), then the compact payload again + for step, rows in enumerate(BF16_ROWS): + every = [[_x(rows, p, 100 + step)] for p in range(2)] + out = torch.empty((2 * rows, 6144), dtype=torch.bfloat16, device=dev) + comm.all_gather([out], every[rank]) + check([out], every, f"bf16 rows {rows}") + every7 = [[t[:, :c].contiguous() for c in (8, 16, 2048, 24, 3072, 8, 64)] for t in (e[0] for e in every)] + outs = [torch.empty((2 * rows, t.shape[1]), dtype=t.dtype, device=dev) for t in every7[rank]] + comm.all_gather(outs, every7[rank]) + check(outs, every7, f"7 fields rows {rows}") + every = [_moe_input(_x(2049, p, 999)).compact().tensors() for p in range(2)] + check(gather_mxfp4_moe(_moe_input(_x(2049, rank, 999)), comm, world=2).tensors(), every, "moe after bf16") + + +def test_gather_odd_rows(): + run_on_2_gpus(_worker, timeout=600) diff --git a/tests/collectives/pull_collectives_ranks2/test_norm_gather_h6144.py b/tests/collectives/pull_collectives_ranks2/test_norm_gather_h6144.py new file mode 100644 index 0000000..c566f78 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/test_norm_gather_h6144.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: chunk-ready dense norm/all-gather against the dense tiled norm's exact bits.""" +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + + +def _worker(rank,errors,rows_list): + from amd_kernels.collectives.pull_collectives_ranks2.norm_gather_h6144 import NormAllGatherComm + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.host import gemma_add_rmsnorm_mxfp8 + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import untile_codes + dev=torch.device('cuda',rank) + comm=NormAllGatherComm(rank,dist.group.WORLD,dev) + comm.warm_up() + assert comm.supports(0) and not comm.supports(1) and not comm.supports(-32) + torch.manual_seed(100+rank) + w=torch.randn(6144,device=dev).bfloat16() + for rows in rows_list: + before = comm.counter.item() if rows == 0 else None + x=torch.randn((rows,6144),device=dev).bfloat16() + r=torch.randn_like(x) + refq,refs,refr=gemma_add_rmsnorm_mxfp8(x,r,w,1e-6,32) + refq=untile_codes(refq,rows).contiguous() + q,s,rr=comm.norm_all_gather(x,r,w) + expectq_cpu=[None]*2; expects_cpu=[None]*2 + dist.all_gather_object(expectq_cpu,refq.cpu()) + dist.all_gather_object(expects_cpu,refs.cpu()) + torch.cuda.synchronize() + for what,a,b in [('q',q.cpu(),torch.cat(expectq_cpu)),('s',s.cpu(),torch.cat(expects_cpu)),('r',rr,refr)]: + if not torch.equal(a.view(torch.uint8),b.view(torch.uint8)): + errors.append(f'{rank=} {rows=} {what}: {(a.view(torch.uint8)!=b.view(torch.uint8)).sum().item()} bytes') + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + comm.norm_all_gather(x,r,w,q_out=q,scale_out=s,res_out=rr) + for rep in range(3): + x.neg_();r.neg_() + # Include an eager call between changed-input replays. + eager=comm.norm_all_gather(x,r,w) + for o in (q,s,rr):o.view(torch.uint8).fill_(0xFF) + torch.cuda.synchronize() + graph.replay() + torch.cuda.synchronize() + for a,b in zip((q,s,rr),eager): + if not torch.equal(a.view(torch.uint8),b.view(torch.uint8)): + errors.append(f'{rank=} {rows=} {rep=} replay differs') + if rows == 0 and comm.counter.item() != before: + errors.append('zero row advanced generation') + + +def test_norm_gather_bits_and_changed_graphs(): + run_on_2_gpus(_worker,[0,32,96,1024,1056,2048,4096,8192,16384]) + + +def _guarded_worker(rank, errors, joined): + from flydsl.compiler.jit_function import MlirCompiler + from amd_kernels.collectives.pull_collectives_ranks2.norm_gather_h6144 import NormAllGatherComm + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + from tests.mfma_spinner import mfma_spinner + dev = torch.device('cuda',rank) + comm = NormAllGatherComm(rank,dist.group.WORLD,dev) + comm.warm_up() + compute = torch.cuda.Stream(priority=-1) + spinner = mfma_spinner(128) + torch.manual_seed(234+rank) + w = torch.randn(6144,device=dev).bfloat16() + real = MlirCompiler.compile + count = [] + def compile_count(*a,**kw): + count.append(1) + return real(*a,**kw) + for rows in (32,64,96,256,480,992,1024,1056,2048,4096,8192,16384): + x = torch.randn((rows,6152),device=dev).bfloat16()[:,:6144] + r = torch.randn((rows,6160),device=dev).bfloat16()[:,:6144] + x[:,:32]=0.;r[:,:32]=2.**-133 + x[:,32:64]=-0.;r[:,32:64]=-0. + if rows==64: + x[0]=2.**80 # Finite BF16 input with overflowing SSQ. + refs = gemma_add_rmsnorm_mxfp8_row_major(x,r,w,1e-6) + guards=[];outs=[] + for shape,dtype in (((2*rows,6144),torch.float8_e4m3fn),((2*rows,192),torch.uint8),((rows,6144),torch.bfloat16)): + size=shape[0]*shape[1]*dtype.itemsize + g=torch.full((size+64,),0x5A,dtype=torch.uint8,device=dev) + guards.append(g);outs.append(g[32:-32].view(dtype).view(shape)) + if joined: + compute.wait_stream(torch.cuda.current_stream()) + spinner(500000,compute) + if rank==(rows//32)%2:torch.cuda._sleep(20000) + MlirCompiler.compile=compile_count + try: + comm.norm_all_gather(x,r,w,q_out=outs[0],scale_out=outs[1],res_out=outs[2]) + torch.cuda.synchronize() + finally: + MlirCompiler.compile=real + if count:errors.append('compile after warm-up');count.clear() + for i,ref in enumerate(refs): + expected=[None]*2 + dist.all_gather_object(expected,ref.cpu().view(torch.uint8)) + want=torch.cat(expected) if i<2 else ref.cpu().view(torch.uint8) + if not torch.equal(outs[i].cpu().view(torch.uint8),want.view(torch.uint8)): + errors.append(f'guarded {rank=} {rows=} field {i}') + for g in guards: + if not bool((g[:32]==0x5A).all() & (g[-32:]==0x5A).all()):errors.append('guard overwritten') + + +def test_norm_gather_guards_compile_and_mfma(): + run_on_2_gpus(_guarded_worker,True) + + +def _reuse_worker(rank,errors): + from amd_kernels.collectives.pull_collectives_ranks2.norm_gather_h6144 import NormAllGatherComm + from tests.mfma_spinner import mfma_spinner + dev=torch.device('cuda',rank) + comm=NormAllGatherComm(rank,dist.group.WORLD,dev) + comm.warm_up() + rows=1024;calls=24 + torch.manual_seed(712+rank) + sources=[torch.randn(rows,6144,device=dev).bfloat16() for _ in range(calls)] + r=torch.randn_like(sources[0]);w=torch.randn(6144,device=dev).bfloat16() + x=torch.empty_like(r) + spinner=mfma_spinner(64) + compute=torch.cuda.Stream(priority=-1) + eager=[] + for src in sources: + x.copy_(src) + eager.append(comm.norm_all_gather(x,r,w)) + graph=torch.cuda.CUDAGraph() + captured=[] + with torch.cuda.graph(graph): + for src in sources: + x.copy_(src) + if rank==1:torch.cuda._sleep(10000) + captured.append(comm.norm_all_gather(x,r,w)) + x.fill_(float('nan')) # Immediate producer reuse after every call. + for rep in range(3): + if rep==1: + for src in sources:src.neg_() + eager=[] + for src in sources: + x.copy_(src);eager.append(comm.norm_all_gather(x,r,w)) + for group in captured: + for t in group:t.view(torch.uint8).fill_(0xFF) + torch.cuda.synchronize() + spinner(3000000,compute) + graph.replay();torch.cuda.synchronize() + for got,want in zip(captured,eager): + for a,b in zip(got,want): + if not torch.equal(a.view(torch.uint8),b.view(torch.uint8)): + errors.append(f'reuse graph {rank=} {rep=}') + + +def test_norm_gather_input_lifetime_long_graph(): + run_on_2_gpus(_reuse_worker) diff --git a/tests/collectives/pull_collectives_ranks2/test_slots4_reduce_scatter.py b/tests/collectives/pull_collectives_ranks2/test_slots4_reduce_scatter.py new file mode 100644 index 0000000..6a7f9fe --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/test_slots4_reduce_scatter.py @@ -0,0 +1,356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: contract, changed-input graph replay, double-buffer reuse and warm-up. + +Every rank generates all rank operands deterministically; no reference +communication or sum-order ambiguity. Total rows include non-power-of-two +shards, the SP boundary and the served maximum. Every written byte is checked. +""" +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + +H = 6144 + + +def _bits(t): + return t.contiguous().view(torch.uint8) + + +def _expect(errors, tag, got, want): + if not torch.equal(_bits(got), _bits(want)): + n = int((_bits(got) != _bits(want)).sum()) + errors.append(f"{tag}: {n} bytes differ") + + +def _inputs(rows, p, dev, step, owner=False): + """(slots, shared) of rank p; ``owner``: shared [rows / 2, H] (rank p's whole shared expert on its own rows).""" + g = torch.Generator(device=dev).manual_seed(rows * 19 + p + step * 1000) + s = torch.randn((rows, 4, H), generator=g, device=dev).bfloat16() + h = torch.randn((rows // 2 if owner else rows, H), generator=g, device=dev).bfloat16() + # Signed zeros and retained BF16 denormals, including cancellation. + s[:, :, :2] = -0.0 + s[:, p, 2:4] = 2.0**-133 + h[:, :4] = 0.0 + return s, h + + +def _worker(rank, errors, rows_list, owner=False): + from amd_kernels.collectives.pull_collectives_ranks2 import SlotsReduceScatterComm + from amd_kernels.collectives.pull_collectives_ranks2.reference import slots4_reduce_scatter_reference + from flydsl.compiler.jit_function import MlirCompiler + + dev = torch.device('cuda', rank) + comm = SlotsReduceScatterComm(rank, dist.group.WORLD, dev) + comm.warm_up() + compiles = [] + real = MlirCompiler.compile + def counting(*a, **kw): + compiles.append(1) + return real(*a, **kw) + MlirCompiler.compile = counting + try: + for rows in rows_list: + xs = [_inputs(rows, p, dev, 0, owner) for p in range(2)] + s, h = xs[rank] + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + guard = torch.full((rows//2*H+16,), 3.25, dtype=torch.bfloat16, device=dev) + out = guard[8:-8].view(rows//2,H) + out.fill_(float('nan')) + comm.slots4_reduce_scatter(s, h, out) + torch.cuda.synchronize() + _expect(errors, f'{rank} rows{rows} eager', out, want) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + comm.slots4_reduce_scatter(s, h, out) + for step in range(3): + if step == 1: + xs = [_inputs(rows, p, dev, 1, owner) for p in range(2)] + s.copy_(xs[rank][0]); h.copy_(xs[rank][1]) + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + out.fill_(float('nan')) + graph.replay() + torch.cuda.synchronize() + _expect(errors, f'{rank} rows{rows} replay{step}', out, want) + if not bool((guard[:8] == 3.25).all() & (guard[-8:] == 3.25).all()): + errors.append(f'{rank} rows{rows}: output guard overwritten') + del graph, xs, s, h, out, want + torch.cuda.empty_cache() + if compiles: + errors.append(f'{rank}: {len(compiles)} compiles after warmup') + finally: + MlirCompiler.compile = real + + +def test_slots4_reduce_scatter_contract_graph_warmup(): + # Uneven chunk sizes exercise the last vector of the narrower pull stripes; 4096..32768 are the served tokens. + run_on_2_gpus(_worker, [2, 4, 20, 66, 132, 4096, 4100, 8192, 16384, 32766, 32768]) + + +def test_slots4_reduce_scatter_owner_form(): + run_on_2_gpus(_worker, [2, 4, 20, 66, 132, 4096, 4100, 8192, 16384, 32766, 32768], True) + + +def _norm_worker(rank, errors, row_counts, spin=False, max_rows=32768, coded=True, owner=False): + from amd_kernels.collectives.pull_collectives_ranks2 import SlotsReduceScatterComm + from amd_kernels.collectives.pull_collectives_ranks2.reference import ( + slots4_reduce_scatter_reference, reduce_scatter_reference, + ) + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import ( + CODE_MISMATCH, MXFP8_COS, SCALE_MISMATCH, unshuffle_scales, add_rmsnorm_reference, compare_mxfp8, + ) + from flydsl.compiler.jit_function import MlirCompiler + dev = torch.device('cuda', rank) + comm = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, max_rows=max_rows, coded=coded) + if spin: + from tests.mfma_spinner import mfma_spinner + spinner = mfma_spinner(32 if spin is True else int(spin)) + side = torch.cuda.Stream(priority=-1) + comm.warm_up() + compiles = [] + real = MlirCompiler.compile + def counting(*a, **kw): + compiles.append(1) + return real(*a, **kw) + MlirCompiler.compile = counting + try: + for rows in row_counts: + shard = rows // 2 + xs = [_inputs(rows, p, dev, 0, owner) for p in range(2)] + s, h = xs[rank] + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + if not owner: + dense = comm.reduce_scatter(h) + torch.cuda.synchronize() + _expect(errors, f'{rank} dense{rows}', dense, + reduce_scatter_reference([x[1] for x in xs], rank)) + r = torch.randn((shard,H), device=dev).bfloat16() + w = (torch.randn(H, device=dev)*0.2).bfloat16() + high, res_ref = add_rmsnorm_reference(want, r, w, 1e-6) + def valid(got): + q, sc, res = got + return q[:shard].view(torch.uint8), unshuffle_scales(sc, shard), res + eager = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(s,h,r,w,1e-6) + torch.cuda.synchronize() + _expect(errors, f'{rank} norm{rows} residual', eager[2],res_ref) + q,sc,_ = valid(eager) + stats = compare_mxfp8(q,sc,high) + if (stats['scale_max_diff'] > 1 or stats['scale_mismatch'] > SCALE_MISMATCH + or stats['code_mismatch'] > CODE_MISMATCH or stats['code_far'] != 0 + or stats['cos'] < MXFP8_COS): + errors.append(f'{rank} norm{rows}: {stats}') + ref = [t.clone() for t in eager] + two = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(s,h,r,w,1e-6,fused=False) + torch.cuda.synchronize() + for k,(x,y) in enumerate(zip(valid(two),valid(ref))): + _expect(errors,f'{rank} norm{rows} two launches vs one, field {k}',x,y) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + got = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(s,h,r,w,1e-6) + for _ in range(3): + got[0].view(torch.uint8).fill_(255); got[1].fill_(255); got[2].fill_(float('nan')) + if spin: + side.wait_stream(torch.cuda.current_stream()) + spinner(1000000,side) + g.replay() + torch.cuda.synchronize() + for k,(a,b) in enumerate(zip(valid(got),valid(ref))): + _expect(errors,f'{rank} norm{rows} replay{k}',a,b) + del g,got,eager,ref + del xs,s,h,want,r,w,high,res_ref + torch.cuda.empty_cache() + if compiles: + errors.append(f'{rank}: {len(compiles)} compiles after warmup') + finally: + MlirCompiler.compile = real + + +def test_slots4_norm_contract_graph_warmup(): + run_on_2_gpus(_norm_worker, [2, 20, 66, 132, 4096, 4100, 8192, 16384, 32766, 32768]) + + +def test_slots4_norm_bf16_link_contract(): + run_on_2_gpus(_norm_worker, [2, 66, 4100, 16384, 32768], False, 32768, False) + + +def test_slots4_norm_owner_form(): + """Owner-form shared [m, H]: coded and BF16-link tails, one and two launches, graph replay.""" + run_on_2_gpus(_norm_worker, [2, 20, 66, 132, 4096, 4100, 8192, 16384, 32766, 32768], False, 32768, True, True) + run_on_2_gpus(_norm_worker, [2, 66, 4100, 32768], False, 32768, False, True) + + +def test_slots4_norm_owner_form_next_to_mfma(): + run_on_2_gpus(_norm_worker, [32768], 256, 32768, True, True) + + +def test_slots4_norm_next_to_mfma(): + run_on_2_gpus(_norm_worker, [4096], True) + + +def test_slots4_norm_next_to_mfma_at_max_rows(): + # The largest SP payload and a resident MFMA workgroup on every CU. + # Unlike the short 4K test, this keeps the chunk handoffs busy throughout the transfer. + run_on_2_gpus(_norm_worker, [32768], 256) + + +def _late_worker(rank, errors, rows_list, coded=True, owner=False): + """One rank starts each call late (a 300 us kernel ahead of it on its stream), so the + peer's pullers wait on its per-chunk tickets while their own stagers have finished.""" + from amd_kernels.collectives.pull_collectives_ranks2 import SlotsReduceScatterComm + from amd_kernels.collectives.pull_collectives_ranks2.reference import ( + reduce_scatter_reference, slots4_reduce_scatter_reference, + ) + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import unshuffle_scales + from tests.mfma_spinner import mfma_spinner + dev = torch.device('cuda', rank) + comm = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, coded=coded) + comm.warm_up() + spinner = mfma_spinner(32) + stream = torch.cuda.current_stream() + for rows in rows_list: + for step in range(8): + xs = [_inputs(rows, p, dev, step, owner) for p in range(2)] + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + dist.barrier() + if rank == step % 2: + spinner(30000, stream) + out = comm.slots4_reduce_scatter(*xs[rank]) + torch.cuda.synchronize() + _expect(errors, f'{rank} rows{rows} late rank {step % 2}', out, want) + # the one-launch tail and the dense reduce-scatter, the other rank late + r = torch.zeros((rows // 2, H), dtype=torch.bfloat16, device=dev) + w = torch.ones(H, dtype=torch.bfloat16, device=dev) + two = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(*xs[rank], r, w, 1e-6, fused=False) + torch.cuda.synchronize() + dist.barrier() + if rank != step % 2: + spinner(30000, stream) + one = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(*xs[rank], r, w, 1e-6) + dense = comm.reduce_scatter(xs[rank][1]) if not owner else None + torch.cuda.synchronize() + shard = rows // 2 + def valid(got): + return got[0][:shard].view(torch.uint8), unshuffle_scales(got[1], shard), got[2] + for k, (a, b) in enumerate(zip(valid(one), valid(two))): + _expect(errors, f'{rank} rows{rows} late tail field {k}', a, b) + if not owner: + _expect(errors, f'{rank} rows{rows} late dense', dense, + reduce_scatter_reference([x[1] for x in xs], rank)) + del xs, want, out, one, two, dense + torch.cuda.empty_cache() + + +def test_slots4_reduce_scatter_late_rank(): + run_on_2_gpus(_late_worker, [132, 4096, 32768]) + + +def test_slots4_owner_form_late_rank(): + run_on_2_gpus(_late_worker, [132, 4096, 32768], True, True) + + +def _mixed_worker(rank, errors, rows_list): + """Legacy and owner-form calls interleaved on one communicator (they share its tickets and pool slots); the + owner form equals the legacy form on a shared tensor that is zero outside this rank's block.""" + from amd_kernels.collectives.pull_collectives_ranks2 import SlotsReduceScatterComm + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import unshuffle_scales + dev = torch.device('cuda', rank) + comm = SlotsReduceScatterComm(rank, dist.group.WORLD, dev) + comm.warm_up() + for rows in rows_list: + shard = rows // 2 + s, h = _inputs(rows, rank, dev, 3, True) + full = torch.zeros((rows, H), dtype=torch.bfloat16, device=dev) + full[rank * shard:(rank + 1) * shard] = h + r = torch.randn((shard, H), device=dev).bfloat16() + w = (torch.randn(H, device=dev) * 0.2).bfloat16() + def valid(got): + return got[0][:shard].view(torch.uint8), unshuffle_scales(got[1], shard), got[2] + for fused in (True, False): + a = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(s, h, r, w, 1e-6, fused) + b = comm.slots4_reduce_scatter_add_rmsnorm_mxfp8(s, full, r, w, 1e-6, fused) + torch.cuda.synchronize() + for k, (x, y) in enumerate(zip(valid(a), valid(b))): + _expect(errors, f'{rank} rows{rows} fused{fused} owner vs zero-padded legacy field {k}', x, y) + _expect(errors, f'{rank} rows{rows} rs owner vs legacy', comm.slots4_reduce_scatter(s, h), + comm.slots4_reduce_scatter(s, full)) + torch.cuda.empty_cache() + + +def test_slots4_owner_form_equals_zero_padded_legacy(): + run_on_2_gpus(_mixed_worker, [2, 66, 4100, 16384]) + + +def test_slots4_norm_small_pool_warmup(): + run_on_2_gpus(_norm_worker, [2], False, 2) + + +def _codec_rows(rows, p, dev, kind, seed=0): + """Slot 0 carries the values (slots 1-3 and shared zero: the slot sum is slot 0 with -0 -> +0), shaped to hit + every path of the code: signed zeros, one exception per lane (<= 6 per virtual wave), several per lane and more + than six per wave (raw virtual waves), NaN / Inf / denormals, wide exponent spreads.""" + g = torch.Generator(device=dev).manual_seed(rows * 7 + p * 3 + kind + 100 * seed) + s = torch.zeros((rows, 4, H), dtype=torch.bfloat16, device=dev) + x = torch.randn((rows, H), generator=g, device=dev) + if kind == 1: # one tiny value in lanes 0..5 of every virtual wave: exceptions, coded + x.view(rows, 12, 64, 8)[:, :, :6, 3] = 2.0**-30 + elif kind == 2: # two tiny values in lane 7: a raw virtual wave; every third row also all of wave 5 + x.view(rows, 12, 64, 8)[:, :, 7, 1:3] = 2.0**-40 + x.view(rows, 12, 64, 8)[::3, 5, :, 0] *= 2.0**-60 + elif kind == 3: # seven lanes with one exception each in wave 9: raw + x.view(rows, 12, 64, 8)[:, 9, 10:17, 6] = 2.0**-25 + elif kind == 4: # specials, zeros of both signs, denormals, exponent extremes + v = x.view(rows, 12, 64, 8) + v[:, 0, 0, 0] = float("nan") + v[:, 1, 3, 2] = float("inf") + v[:, 2, 5, 7] = -float("inf") + v[:, 3, :, :] = 0.0 + v[:, 4, 1::2, :] = -0.0 + v[:, 6, 2, 4] = 2.0**-133 + v[:, 7, 4, 0] = 3.0e38 + v[:, 8, :, :] *= 2.0**100 + v[:, 10, ::5, 1] = 2.0**-14 + s[:, 0] = x.bfloat16() + return s, torch.zeros((rows, H), dtype=torch.bfloat16, device=dev) + + +def _codec_worker(rank, errors, rows_list): + """The coded tail's three outputs equal the BF16-link tail's byte for byte (and the residual the reference) on + rows that take every branch of the code, eager and on graph replay.""" + from amd_kernels.collectives.pull_collectives_ranks2 import SlotsReduceScatterComm + from amd_kernels.collectives.pull_collectives_ranks2.reference import slots4_reduce_scatter_reference + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import add_rmsnorm_reference, unshuffle_scales + dev = torch.device('cuda', rank) + coded = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, coded=True) + plain = SlotsReduceScatterComm(rank, dist.group.WORLD, dev, coded=False) + coded.warm_up() + plain.warm_up() + for rows in rows_list: + shard = rows // 2 + r = torch.randn((shard, H), device=dev).bfloat16() + w = (torch.randn(H, device=dev) * 0.2).bfloat16() + def valid(got): + return got[0][:shard].view(torch.uint8), unshuffle_scales(got[1], shard), got[2] + for kind in range(5): + xs = [_codec_rows(rows, p, dev, kind) for p in range(2)] + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + a = coded.slots4_reduce_scatter_add_rmsnorm_mxfp8(*xs[rank], r, w, 1e-6) + b = plain.slots4_reduce_scatter_add_rmsnorm_mxfp8(*xs[rank], r, w, 1e-6) + torch.cuda.synchronize() + _expect(errors, f'{rank} rows{rows} kind{kind} residual', a[2], add_rmsnorm_reference(want, r, w, 1e-6)[1]) + for k, (x, y) in enumerate(zip(valid(a), valid(b))): + _expect(errors, f'{rank} rows{rows} kind{kind} coded vs bf16 field {k}', x, y) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + got = coded.slots4_reduce_scatter_add_rmsnorm_mxfp8(*xs[rank], r, w, 1e-6) + for rep in range(2): + got[0].view(torch.uint8).fill_(255); got[1].fill_(255); got[2].fill_(float('nan')) + g.replay() + torch.cuda.synchronize() + for k, (x, y) in enumerate(zip(valid(got), valid(b))): + _expect(errors, f'{rank} rows{rows} kind{kind} replay{rep} field {k}', x, y) + del g, got, a, b, xs, want + torch.cuda.empty_cache() + + +def test_slots4_coded_tail_codec_paths(): + run_on_2_gpus(_codec_worker, [2, 66, 4100, 16384]) diff --git a/tests/collectives/pull_collectives_ranks2/test_unpack_q4096_shuffled.py b/tests/collectives/pull_collectives_ranks2/test_unpack_q4096_shuffled.py new file mode 100644 index 0000000..2760888 --- /dev/null +++ b/tests/collectives/pull_collectives_ranks2/test_unpack_q4096_shuffled.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Unpack with o_proj_full's scale layout on two ranks: codes as the reference unpack, scales bit-equal to the GEMM's +own shuffle of the reference scales (padding rows 127), the GEMM on them bit-equal to the row-major call; both +publication forms, both pool sets on the same flags, late ranks, MFMA work, no compile after warm-up.""" +import torch +import torch.distributed as dist +from amd_kernels.collectives.pull_collectives_ranks2.reference import WORLD, all_to_all_reference, unpack_reference +from tests.multi_gpu import run_on_2_gpus +from .served import collectives_slot_bytes + +D = 4096 # code bytes per packed row + + +def _u8(shape, seed, dev): + g = torch.Generator(device=dev).manual_seed(seed) + return torch.randint(0, 256, shape, generator=g, dtype=torch.uint8, device=dev) + + +def _packed(shape, seed, dev): + """Random finite e4m3 codes, e8m0 scales 2^-12..2^-3 (the attention epilogue's range).""" + x = _u8(shape, seed, dev) + g = torch.Generator(device=dev).manual_seed(seed + 7) + x[:, :D] = torch.where((x[:, :D] & 0x7F) == 0x7F, x[:, :D] ^ 1, x[:, :D]) + x[:, D:] = torch.randint(115, 125, (shape[0], D // 32), generator=g, device=dev, dtype=torch.uint8) + return x + + +def _worker(rank, errors, row_counts, wgs, joined=False, gemm_rows=(), fence=None): + from flydsl.compiler.jit_function import MlirCompiler + from amd_kernels.collectives.pull_collectives_ranks2 import Collectives, FencedAllGatherComm, Pools + from amd_kernels.collectives.pull_collectives_ranks2.pull_unpack_q4096_shuffled import ( + all_to_all_unpack_q4096_shuffled) + from amd_kernels.gemm.mxfp8_common.gemm import _preshuffle_activation_scale, mxfp8_gemm, preshuffle_mxfp8_weight + dev = torch.device('cuda', rank) + main = torch.cuda.current_stream() + pools = Pools(rank, WORLD, dist.group.WORLD, dev, collectives_slot_bytes(), sets=2) + if wgs is None: + FencedAllGatherComm(pools).warm_up() + else: + for f in (True, False): + c = Collectives(pools, 0).bind(main) + x = c.alloc((WORLD * 32, D + D // 32), torch.uint8) + x.zero_() + all_to_all_unpack_q4096_shuffled(c, x, wgs, f) + torch.cuda.synchronize() + if joined: + from tests.mfma_spinner import mfma_spinner + spinner = mfma_spinner(128) + side = torch.cuda.Stream(priority=-1) + if gemm_rows: + g = torch.Generator(device=dev).manual_seed(5) + wq = torch.randint(0, 120, (6144, WORLD * D), generator=g, device=dev, dtype=torch.uint8) + wq = wq.view(torch.float8_e4m3fn) + wsc = torch.randint(118, 124, (6144, WORLD * D // 32), generator=g, device=dev, dtype=torch.uint8) + w, ws = preshuffle_mxfp8_weight(wq, wsc) + real = MlirCompiler.compile + compiles = [] + + def count(*a, **kw): + compiles.append(1) + return real(*a, **kw) + for rows in row_counts: + for rep in range(4): + every = [_packed((WORLD * rows, D + D // 32), rows * 1000 + rep * 10 + p, dev) for p in range(WORLD)] + c = Collectives(pools, rep % 2).bind(main) + x = c.alloc(every[rank].shape, torch.uint8) + x.copy_(every[rank]) + if rank == rep % WORLD: + torch.cuda._sleep(30000) + if joined: + side.wait_stream(main) + spinner(100000, side) + MlirCompiler.compile = count + try: + codes, scales = all_to_all_unpack_q4096_shuffled(c, x, wgs, fence) + torch.cuda.synchronize() + finally: + MlirCompiler.compile = real + if compiles: + errors.append(f'rows {rows} rep {rep}: compile after warm-up') + compiles.clear() + want_c, want_s = unpack_reference(all_to_all_reference(every, rank), rows, D) + if not torch.equal(codes.view(torch.uint8), want_c.view(torch.uint8)): + errors.append(f'rank {rank} rows {rows} rep {rep}: codes') + if not torch.equal(scales, _preshuffle_activation_scale(want_s)): + errors.append(f'rank {rank} rows {rows} rep {rep}: shuffled scales') + if rows in gemm_rows and rep == 0: + a = mxfp8_gemm(codes, scales, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + b = mxfp8_gemm(want_c.view(torch.float8_e4m3fn), want_s, w, ws, torch.bfloat16) + if not torch.equal(a.view(torch.int16), b.view(torch.int16)): + errors.append(f'rank {rank} rows {rows}: o_proj_full on the shuffled scales differs') + dist.barrier() + + +def test_shuffled_unpack_uneven_rows_and_slot_reuse(): + # served shard rows 2048..16384 (4096..32768 tokens) and uneven rows around the 32-row groups and the fence split + rows = (1, 7, 31, 32, 33, 96, 255, 256, 257, 1023, 1024, 1025, 2047, 2048, 2049, 4095, 4096, 4097, 8192, 16384, 1) + run_on_2_gpus(_worker, rows, None, False, (33, 2048, 16384)) + + +def test_shuffled_unpack_workgroups_next_to_mfma(): + for wgs in (16, 64): + for fence in (True, False): + run_on_2_gpus(_worker, (32, 1031, 2048, 4128), wgs, True, (), fence) diff --git a/tests/collectives/sdma_push_ranks2/__init__.py b/tests/collectives/sdma_push_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/sdma_push_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/sdma_push_ranks2/test_transport.py b/tests/collectives/sdma_push_ranks2/test_transport.py new file mode 100644 index 0000000..fc40df0 --- /dev/null +++ b/tests/collectives/sdma_push_ranks2/test_transport.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SdmaPush on 2 ranks as a chain of phases (the use the transport documents): per round an all-gather push into +region 0, its check, then a reduce-scatter-shaped push into region 1, its check. Distinct bytes per round, the +producer slowed down so the copies would read stale bytes if they started early, more rounds than the signal ring +holds, the sources dropped by the caller right after the push (the transport holds them), the receivers check on +their stream right after each wait. Item sizes below and above the striping threshold and uneven ones, on the default +engine and striped over two engines.""" + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + +WORLD = 2 + + +def _fill(r, rnd, k): # rank r's bytes of round rnd, kind k + return (r * 37 + rnd * 11 + k * 5 + 1) % 251 + + +def _worker(rank, errors, nbytes, rounds, engines): + from amd_kernels.collectives.sdma_push_ranks2 import RING, SdmaPush + + dev = torch.device("cuda", rank) + t = SdmaPush(rank, dist.group.WORLD, dev, [WORLD * nbytes, WORLD * nbytes], engines=engines) + assert rounds > RING + bad = torch.zeros(1, dtype=torch.int64, device=dev) + peer = 1 - rank + for rnd in range(rounds): + # all-gather: my bytes -> slot `rank` of the peer's region 0 + src = torch.empty(nbytes, dtype=torch.uint8, device=dev) + torch.cuda._sleep(200_000) # a slow producer: the copies must not start before it is done + src.fill_(_fill(rank, rnd, 0)) + src[::4093].fill_(_fill(rank, rnd, 9)) # bytes that differ inside a piece + seq = t.push(0, [(peer, src.data_ptr(), rank * nbytes, nbytes)], keep=(src,)) + del src + t.wait(0, seq) + g = t.region(0).view(WORLD, nbytes) + want = torch.full((nbytes,), _fill(peer, rnd, 0), dtype=torch.uint8, device=dev) + want[::4093].fill_(_fill(peer, rnd, 9)) + bad += (g[peer] != want).sum() + # reduce-scatter shape: block p of my buffer -> slot `rank` of peer p's region 1, two items + rs = torch.tensor([_fill(rank, rnd, 1 + p) for p in range(WORLD)], dtype=torch.uint8, device=dev) + rs = rs[:, None].expand(WORLD, nbytes).contiguous() + torch.cuda._sleep(200_000) + rs.add_(0) + half = nbytes // 2 // 16 * 16 + seq = t.push(1, [(peer, rs[peer].data_ptr(), rank * nbytes, half), + (peer, rs[peer].data_ptr() + half, rank * nbytes + half, nbytes - half)], keep=(rs,)) + del rs + t.wait(1, seq) + r1 = t.region(1).view(WORLD, nbytes) + bad += (r1[peer] != _fill(peer, rnd, 1 + rank)).sum() + torch.cuda.synchronize(dev) + t.drain() + if bad.item(): + errors.append(f"rank {rank}: {bad.item()} stale or wrong bytes over {rounds} rounds") + + +@pytest.mark.parametrize("engines", [None, (2, 3)]) +@pytest.mark.parametrize("nbytes", [1 << 16, (1 << 20) + 4096 + 16, 3 << 21]) +def test_push_chain_ordered_and_reused(nbytes, engines): + run_on_2_gpus(_worker, nbytes, 40, engines, timeout=600) diff --git a/tests/collectives/sdma_push_ranks4/test_agents.py b/tests/collectives/sdma_push_ranks4/test_agents.py new file mode 100644 index 0000000..645e3f8 --- /dev/null +++ b/tests/collectives/sdma_push_ranks4/test_agents.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The HSA agent of every rank is that rank's GPU: keys unique over every GPU agent of the node (two PCI domains can +share bus / device numbers), checked against the agent UUID, which torch reports as the ASCII bytes of its suffix.""" + +import ctypes + +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_4_gpus + +HSA_AMD_AGENT_INFO_UUID = 0xA011 + + +def _agent_uuid(agent) -> str: + from amd_kernels.collectives.sdma_push_ranks4 import hsa_runtime as rt + + buf = ctypes.create_string_buffer(64) + assert rt.hsa().hsa_agent_get_info(agent, HSA_AMD_AGENT_INFO_UUID, buf) == 0 + return buf.value.decode() + + +def _device_uuid(device: torch.device) -> str: + return "GPU-" + bytes.fromhex(str(torch.cuda.get_device_properties(device).uuid).replace("-", "")).decode() + + +def _all_gpu_agent_uuids() -> list[str]: + from amd_kernels.collectives.sdma_push_ranks4 import hsa_runtime as rt + + lib, out = rt.hsa(), [] + + def visit(agent, _): + kind = ctypes.c_uint32() + lib.hsa_agent_get_info(agent, rt.HSA_AGENT_INFO_DEVICE, ctypes.byref(kind)) + if kind.value == rt.HSA_DEVICE_TYPE_GPU: + out.append(_agent_uuid(rt.Agent(agent.handle))) + return 0 + + assert lib.hsa_iterate_agents(lib._visit_type(visit), None) == 0 + return out + + +def test_agent_keys_unique_and_match_devices(): + from amd_kernels.collectives.sdma_push_ranks4 import hsa_runtime as rt + + agents = rt.gpu_agents() + assert sorted(_agent_uuid(a) for a in agents.values()) == sorted(_all_gpu_agent_uuids()) + devices = [torch.device("cuda", i) for i in range(torch.cuda.device_count())] + keys = [rt.pci_key(d) for d in devices] + assert len(set(keys)) == len(keys) + for d, k in zip(devices, keys): + assert _agent_uuid(agents[k]) == _device_uuid(d), (d, hex(k)) + + +def _worker(rank, errors): + from amd_kernels.collectives.sdma_push_ranks4 import WORLD, SdmaPush + + dev = torch.device("cuda", rank) + t = SdmaPush(rank, dist.group.WORLD, dev, [4096]) + want = [None] * WORLD + dist.all_gather_object(want, _device_uuid(dev)) + got = [_agent_uuid(a) for a in t.agents] + if got != want: + errors.append(f"rank {rank}: agents {got}, devices {want}") + + +def test_transport_agents_are_the_ranks_gpus(): + run_on_4_gpus(_worker, timeout=300) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_coded_tail.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_coded_tail.py new file mode 100644 index 0000000..f3e0c37 --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_coded_tail.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MoE tail's coded link (``lossless_x3``) in both forms, bit for bit: the SDMA phase (``slots4_stage_rows_x3`` + +push + ``sum2_x3_add_rmsnorm_mxfp8``) against the pull phase on rows that take every branch of the code (zeros, +exceptions, raw virtual waves, NaN / Inf / denormals), the pull phase's rank sum against the reference, on 2 ranks, +twice in a row per size (the raw stores of the second call land in a region the first one used).""" + +import torch +import torch.distributed as dist +from tests.collectives.pull_collectives_ranks2.test_slots4_reduce_scatter import _codec_rows +from tests.multi_gpu import run_on_2_gpus + +from .workload import H, WORLD, rand + +EPS = 1e-6 + + +def _expect(errors, what, got, want): + if not torch.equal(got.contiguous().view(torch.uint8), want.contiguous().view(torch.uint8)): + n = int((got.contiguous().view(torch.uint8) != want.contiguous().view(torch.uint8)).sum()) + errors.append(f"{what}: {n} bytes differ") + + +def _worker(rank, errors, token_counts): + from amd_kernels.collectives.pull_collectives_ranks2.reference import slots4_reduce_scatter_reference + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import ( + SdmaSeqParallelHandoffs, SeqParallelHandoffs, shard_rows) + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import add_rmsnorm_reference + + dev = torch.device("cuda", rank) + pull = SeqParallelHandoffs(rank, dist.group.WORLD, dev, max(token_counts)) + sdma = SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, max(token_counts)) + nw = rand((H,), 3, dev, 0.2) + for t in token_counts: + rows = shard_rows(t) + res = rand((rows, H), t + rank, dev) + for kind in range(5): + for rep in range(2): + xs = [_codec_rows(WORLD * rows, p, dev, kind, rep) for p in range(WORLD)] + tag = f"rank {rank} t {t} kind {kind} call {rep}" + a = pull.ffn_norm_gather(xs[rank], res, nw, EPS) + a = [x.clone() for x in a] + b = sdma.ffn_norm_gather(xs[rank], res, nw, EPS, between=lambda: torch.cuda._sleep(20_000 * (1 + rank))) + torch.cuda.synchronize() + for k, name in enumerate(("codes", "scales", "residual")): + _expect(errors, f"{tag} sdma vs pull {name}", b[k], a[k]) + want = slots4_reduce_scatter_reference([x[0] for x in xs], [x[1] for x in xs], rank) + _expect(errors, f"{tag} residual vs reference", a[2], add_rmsnorm_reference(want, res, nw, EPS)[1]) + del a, b, xs + torch.cuda.empty_cache() + sdma.drain() + + +def test_coded_moe_tail_sdma_vs_pull(): + run_on_2_gpus(_worker, [4096, 4097, 8192, 16385], timeout=1500) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_dbo.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_dbo.py new file mode 100644 index 0000000..7da342a --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_dbo.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""DBO as vLLM runs it on 2 ranks: two microbatches, one ``SdmaSeqParallelHandoffs`` each, two threads taking turns on +one compute stream (a phase's ``between`` is the yield, so one microbatch's pushes are in flight while the other's +kernels are queued; each microbatch's full shared expert runs on its instance's side stream across the yield), two +MoE layers then the final reduce-scatter and gathers; every output bit for bit the pull form's +(``SeqParallelHandoffs``) on the same microbatch.""" + +import threading + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + +from .workload import H, Q, moe_weights, o_proj_weight, rand + +EPS = 1e-6 + + +def _layers(ph, attn, res, o_w, o_s, nw, mw, between=None): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + + kw = {} if between is None else {"between": between} + outs = [] + for _ in range(2): + moe_in, res, shared = ph.attention_moe(attn, o_w, o_s, res, nw, EPS, mw, **kw) + slots, _ = mxfp4_moe(moe_in, mw, return_partials=True) + outs += [t.clone() for t in moe_in.tensors()[:4]] + codes, scales, res = ph.ffn_norm_gather((slots, shared), res, nw, EPS, **kw) + outs += [codes.clone(), scales.clone(), res] + h = ph.ffn_reduce_scatter((slots, shared), **kw) + outs += [h, ph.gather(h, **kw), ph.gather(res, **kw)] + return outs + + +def _worker(rank, errors, tokens): + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import (SdmaSeqParallelHandoffs, + SeqParallelHandoffs, shard_rows) + + dev = torch.device("cuda", rank) + o_w, o_s = o_proj_weight(dev) + mw = moe_weights(dev, rank) + nw = rand((H,), 3, dev, 0.2) + u = [tokens // 2, tokens - tokens // 2] + attns = [rand((n, Q), 10 * i + rank, dev, 0.5) for i, n in enumerate(u)] + ress = [rand((shard_rows(n), H), 10 * i + 5 + rank, dev) for i, n in enumerate(u)] + pull = SeqParallelHandoffs(rank, dist.group.WORLD, dev, max(u)) + want = [_layers(pull, attns[i], ress[i], o_w, o_s, nw, mw) for i in range(2)] + torch.cuda.synchronize() + + sdma = [SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, max(u)) for _ in range(2)] + stream = torch.cuda.Stream(device=dev) + turn = [threading.Event(), threading.Event()] + done = [False, False] + got, errs = [None, None], [] + + def body(i): + torch.cuda.set_device(dev) + torch.cuda.set_stream(stream) + + def yield_(): + if not done[1 - i]: + turn[1 - i].set() + turn[i].wait() + turn[i].clear() + + try: + turn[i].wait() + turn[i].clear() + got[i] = _layers(sdma[i], attns[i], ress[i], o_w, o_s, nw, mw, between=yield_) + except Exception as e: # noqa: BLE001 + errs.append(repr(e)) + finally: + done[i] = True + turn[1 - i].set() + + stream.wait_stream(torch.cuda.current_stream()) + threads = [threading.Thread(target=body, args=(i,)) for i in range(2)] + for th in threads: + th.start() + turn[0].set() + for th in threads: + th.join() + torch.cuda.synchronize() + for ph in sdma: + ph.drain() + errors.extend(f"rank {rank}: {e}" for e in errs) + for i in range(2): + if got[i] is None: + continue + for k, (a, b) in enumerate(zip(got[i], want[i])): + if a.shape != b.shape or not torch.equal(a.contiguous().view(torch.uint8), b.contiguous().view(torch.uint8)): + errors.append(f"rank {rank} microbatch {i} output {k} differs") + + +@pytest.mark.parametrize("tokens", [8192, 32768 + 1]) +def test_dbo_sdma_equals_pull(tokens): + run_on_2_gpus(_worker, tokens, timeout=900) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_handoffs.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_handoffs.py new file mode 100644 index 0000000..65c229d --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_handoffs.py @@ -0,0 +1,371 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The phases of SeqParallelHandoffs on 2 ranks, bit for bit against their kernels run on one GPU per shard (every rank +rebuilds every rank's seeded inputs): the input quant + gather, the attention all-to-all + o_proj + norm (+ MoE +preparation, the routed payload's gather, the full shared expert of the own rows) + gather of a MoE and a dense layer, +the FFN reduce-scatter + next norm / quant + gather of both (MoE: owner-form and every-row shared terms), the final +reduce-scatter and the bf16 gather; at the served token counts (tails off the 32-row grid included). A MoE layer's +chain (hand-off, routed experts, owner-form tail) against the one-GPU composition, with the shared expert's side +stream delayed; with its join skipped the comparison must fail. Layers enqueued far ahead of the GPU must not grow the +allocator pool by the side stream's buffers (recorded on the side stream instead, they must). The MoE FFN's norm + quant has no one-GPU kernel: its +codes and scales meet ``norm/gemma_add_rmsnorm_h6144/reference.py``'s tolerances, its residual bit for bit.""" + +import contextlib + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + +from .workload import H, Q, SERVED_TOKENS, WORLD, moe_weights, o_proj_weight, rand + +EPS = 1e-6 + + +def _expect(errors, what, got, want): + if got.shape != want.shape: + errors.append(f"{what}: shape {tuple(got.shape)} != {tuple(want.shape)}") + elif not torch.equal(got.contiguous().view(torch.uint8), want.contiguous().view(torch.uint8)): + bad = (got.contiguous().view(torch.uint8) != want.contiguous().view(torch.uint8)).reshape(got.shape[0], -1) + errors.append(f"{what}: rows {bad.any(dim=1).nonzero().flatten()[:6].tolist()} differ") + + +def _expect_mxfp8(errors, what, codes, scales, y64): + """Row-major codes and AITER-shuffled scales of a shard against the fp64 norm (the reference's tolerances).""" + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import reference as R + + st = R.compare_mxfp8(codes.contiguous().view(torch.uint8), R.unshuffle_scales(scales, codes.shape[0]), y64) + if (st["scale_max_diff"] > 1 or st["scale_mismatch"] > R.SCALE_MISMATCH or st["code_mismatch"] > R.CODE_MISMATCH + or st["code_far"] or st["cos"] < R.MXFP8_COS): # fmt: skip + errors.append(f"{what}: {st}") + + +def _unpacked(attn, t, rows, p, dev): + """Rank p's o_proj input from every rank's attention output: codes [rows, 2 Q], scales in the GEMM's layout.""" + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import mxfp8_pack_q4096 + from amd_kernels.gemm.mxfp8_common.gemm import _preshuffle_activation_scale + + codes, scales = [], [] + for a in attn: + packed = torch.zeros((WORLD * rows, Q + Q // 32), dtype=torch.uint8, device=dev) + mxfp8_pack_q4096(a, out=packed[:t]) + block = packed[p * rows : (p + 1) * rows] + codes.append(block[:, :Q]) + scales.append(block[:, Q:]) + return torch.cat(codes, 1).contiguous().view(torch.float8_e4m3fn), _preshuffle_activation_scale( + torch.cat(scales, 1).contiguous() + ) + + +class _Delayed: + """The SDMA phases with a ``between`` that delays this rank's stream by a rank-dependent time, as the other + microbatch's compute would: the pushes and waits must order the bytes, not the timing.""" + + def __init__(self, ph, rank): + self.ph, self.cycles = ph, 50_000 * (1 + rank) + + def __getattr__(self, name): + fn = getattr(self.ph, name) + if name not in ("input_gather", "attention_moe", "attention_dense", "ffn_norm_gather", "ffn_reduce_scatter", + "gather"): # fmt: skip + return fn + return lambda *a, **k: fn(*a, between=lambda: torch.cuda._sleep(self.cycles), **k) + + +def _worker(rank, errors, token_counts, sdma=False, full=True): + from amd_kernels.collectives.pull_collectives_ranks2.reference import ( + reduce_scatter_reference, + slots4_reduce_scatter_reference, + ) + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import SeqParallelHandoffs, shard_rows + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_e4m3_quantize_flydsl, mxfp8_gemm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import prepare_mxfp4_moe, shared_expert_full + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm import gemma_add_rmsnorm_mxfp8_row_major + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.host import gemma_add_rmsnorm + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import add_rmsnorm_reference + + dev = torch.device("cuda", rank) + if sdma: + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import SdmaSeqParallelHandoffs + + ph = _Delayed(SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, max(token_counts), compact=not full), rank) + else: + ph = SeqParallelHandoffs(rank, dist.group.WORLD, dev, max(token_counts)) + o_w, o_s = o_proj_weight(dev) + mw = moe_weights(dev, rank, full) + nw = rand((H,), 3, dev, 0.2) + + def o_proj(xq, xs): + return mxfp8_gemm(xq, xs, o_w, o_s, torch.bfloat16, activation_scale_preshuffled=True) + + for t in token_counts: + rows = shard_rows(t) + padded = WORLD * rows + tag = f"rank {rank} t {t}" + seed = t * 100 + # 1. the first layer's input + xs = [rand((rows, H), seed + p, dev) for p in range(WORLD)] + codes, scales = ph.input_gather(xs[rank]) + want = [mxfp8_e4m3_quantize_flydsl(x) for x in xs] + torch.cuda.synchronize() + _expect(errors, f"{tag} input codes", codes, torch.cat([w[0] for w in want])) + _expect(errors, f"{tag} input scales", scales, torch.cat([w[1] for w in want])) + # 2. the attention output of a MoE layer, then of a dense layer + attn = [rand((t, Q), seed + 10 + p, dev, 0.5) for p in range(WORLD)] + res = [rand((rows, H), seed + 20 + p, dev) for p in range(WORLD)] + moe_in, r, pending = ph.attention_moe(attn[rank], o_w, o_s, res[rank], nw, EPS, mw) + if full: + pending.wait() + local, rs = [], [] + for p in range(WORLD): + y, rp = gemma_add_rmsnorm(o_proj(*_unpacked(attn, t, rows, p, dev)), res[p], nw, EPS) + local.append(prepare_mxfp4_moe(y, mw)) + rs.append(rp) + _expect(errors, f"{tag} moe residual", r, rs[rank]) + if full: + own = shared_expert_full(local[rank].shared_codes, local[rank].shared_scales, mw.shared_weights) + torch.cuda.synchronize() + if not moe_in.routed_only: + errors.append(f"{tag}: the gathered MoE input is not the routed-only payload") + for k, field in enumerate(moe_in.tensors()[:4]): + _expect(errors, f"{tag} moe input field {k}", field, torch.cat([c.tensors()[k] for c in local])) + _expect(errors, f"{tag} shared expert of the own rows", pending.out, own) + del own + else: + compact = [c.compact() for c in local] + torch.cuda.synchronize() + if pending is not None: + errors.append(f"{tag}: a shard copy returned a pending shared expert") + for k, field in enumerate(moe_in.tensors()): + _expect(errors, f"{tag} compact moe input field {k}", field, torch.cat([c.tensors()[k] for c in compact])) + del compact + del moe_in, pending, local + codes, scales, r = ph.attention_dense(attn[rank], o_w, o_s, res[rank], nw, EPS) + want = [gemma_add_rmsnorm_mxfp8_row_major(o_proj(*_unpacked(attn, t, rows, p, dev)), res[p], nw, EPS) + for p in range(WORLD)] # fmt: skip + torch.cuda.synchronize() + _expect(errors, f"{tag} dense codes", codes, torch.cat([w[0] for w in want])) + _expect(errors, f"{tag} dense scales", scales, torch.cat([w[1] for w in want])) + _expect(errors, f"{tag} dense residual", r, want[rank][2]) + # 3. the FFN output: MoE partials, then a dense partial; the last layer's reduce-scatter; the gather + slots = [rand((padded, 4, H), seed + 30 + p, dev, 0.25) for p in range(WORLD)] + shared = [rand((padded, H), seed + 40 + p, dev, 0.25) for p in range(WORLD)] + codes, scales, r = ph.ffn_norm_gather((slots[rank], shared[rank]), res[rank], nw, EPS) + sums = [slots4_reduce_scatter_reference(slots, shared, p) for p in range(WORLD)] + want = [add_rmsnorm_reference(s, res[p], nw, EPS) for p, s in enumerate(sums)] + torch.cuda.synchronize() + for p, (y64, _) in enumerate(want): + blk = slice(p * rows, (p + 1) * rows) + _expect_mxfp8(errors, f"{tag} slots codes / scales of rank {p}", codes[blk], scales[blk], y64) + _expect(errors, f"{tag} slots residual", r, want[rank][1]) + h = ph.ffn_reduce_scatter((slots[rank], shared[rank])) + torch.cuda.synchronize() + _expect(errors, f"{tag} slots reduce-scatter", h, sums[rank]) + # the owner form: each rank's shared term of its own rows only + owned = [rand((rows, H), seed + 60 + p, dev, 0.25) for p in range(WORLD)] + codes, scales, r = ph.ffn_norm_gather((slots[rank], owned[rank]), res[rank], nw, EPS) + sums = [slots4_reduce_scatter_reference(slots, owned, p) for p in range(WORLD)] + want = [add_rmsnorm_reference(s, res[p], nw, EPS) for p, s in enumerate(sums)] + torch.cuda.synchronize() + for p, (y64, _) in enumerate(want): + blk = slice(p * rows, (p + 1) * rows) + _expect_mxfp8(errors, f"{tag} owner slots codes / scales of rank {p}", codes[blk], scales[blk], y64) + _expect(errors, f"{tag} owner slots residual", r, want[rank][1]) + h = ph.ffn_reduce_scatter((slots[rank], owned[rank])) + torch.cuda.synchronize() + _expect(errors, f"{tag} owner slots reduce-scatter", h, sums[rank]) + del slots, shared, owned + dense = [rand((padded, H), seed + 50 + p, dev, 0.25) for p in range(WORLD)] + codes, scales, r = ph.ffn_norm_gather(dense[rank], res[rank], nw, EPS) + sums = [reduce_scatter_reference(dense, p) for p in range(WORLD)] + want = [gemma_add_rmsnorm_mxfp8_row_major(s, res[p], nw, EPS) for p, s in enumerate(sums)] + torch.cuda.synchronize() + _expect(errors, f"{tag} dense ffn codes", codes, torch.cat([w[0] for w in want])) + _expect(errors, f"{tag} dense ffn scales", scales, torch.cat([w[1] for w in want])) + _expect(errors, f"{tag} dense ffn residual", r, want[rank][2]) + h = ph.ffn_reduce_scatter(dense[rank]) + out = ph.gather(res[rank]) + torch.cuda.synchronize() + _expect(errors, f"{tag} dense reduce-scatter", h, sums[rank]) + _expect(errors, f"{tag} gather", out, torch.cat(res)) + del dense + torch.cuda.empty_cache() + + +def _chain_worker(rank, errors, token_counts, sdma=False, delay=0, skip_join=False): + """A MoE layer through the hand-offs (attention_moe, the routed experts on the gathered payload, the owner-form + tail and the final reduce-scatter) against the one-GPU composition: every rank's preparation, the routed partials + of the concatenated payload, each owner's ``shared_expert_full``, the reference sum. ``delay``: cycles the side + stream sleeps before the shared expert; ``skip_join``: ``SharedPending.wait`` does nothing (negative control).""" + from unittest import mock + + from amd_kernels.collectives.pull_collectives_ranks2.reference import slots4_reduce_scatter_reference + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import (SdmaSeqParallelHandoffs, + SeqParallelHandoffs, handoffs, + shard_rows) + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_gemm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, mxfp4_moe, prepare_mxfp4_moe, shared_expert_full + from amd_kernels.norm.gemma_add_rmsnorm_h6144.add_rmsnorm.host import gemma_add_rmsnorm + from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import add_rmsnorm_reference + + dev = torch.device("cuda", rank) + cls = SdmaSeqParallelHandoffs if sdma else SeqParallelHandoffs + ph = cls(rank, dist.group.WORLD, dev, max(token_counts)) + o_w, o_s = o_proj_weight(dev) + mw = moe_weights(dev, rank) + nw = rand((H,), 3, dev, 0.2) + fork = handoffs._fork_shared + + def delayed_fork(side, *a): + with torch.cuda.stream(side): + torch.cuda._sleep(delay) + pending = fork(side, *a) + if skip_join: # the output's block may still hold an earlier call's identical bits: poison it first + pending.out.fill_(float("nan")) + return pending + + patches = [mock.patch.object(handoffs, "_fork_shared", delayed_fork)] if delay else [] + if sdma: + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import handoffs_sdma + + patches = [mock.patch.object(handoffs_sdma, "_fork_shared", delayed_fork)] if delay else [] + if skip_join: + patches.append(mock.patch.object(handoffs.SharedPending, "wait", lambda self: None)) + for t in token_counts: + rows = shard_rows(t) + tag = f"rank {rank} t {t} delay {delay} skip_join {skip_join}" + seed = t * 100 + 7 + attn = [rand((t, Q), seed + 10 + p, dev, 0.5) for p in range(WORLD)] + res = [rand((rows, H), seed + 20 + p, dev) for p in range(WORLD)] + with contextlib.ExitStack() as stack: + for p_ in patches: + stack.enter_context(p_) + moe_in, r, pending = ph.attention_moe(attn[rank], o_w, o_s, res[rank], nw, EPS, mw) + slots, none = mxfp4_moe(moe_in, mw, return_partials=True) + codes, scales, r2 = ph.ffn_norm_gather((slots, pending), r, nw, EPS) + moe_in, r3, pending = ph.attention_moe(attn[rank], o_w, o_s, res[rank], nw, EPS, mw) + slots, _ = mxfp4_moe(moe_in, mw, return_partials=True) + h = ph.ffn_reduce_scatter((slots, pending)) + local, rs = [], [] + for p in range(WORLD): + xq, xs = _unpacked(attn, t, rows, p, dev) + y, rp = gemma_add_rmsnorm(mxfp8_gemm(xq, xs, o_w, o_s, torch.bfloat16, activation_scale_preshuffled=True), + res[p], nw, EPS) + local.append(prepare_mxfp4_moe(y, mw)) + rs.append(rp) + gathered = MoeInput(*[torch.cat([c.tensors()[k] for c in local]) for k in range(4)]) + ref_slots, _ = mxfp4_moe(gathered, mw, return_partials=True) + owned = [shared_expert_full(c.shared_codes, c.shared_scales, mw.shared_weights) for c in local] + sums = [slots4_reduce_scatter_reference([ref_slots, ref_slots], owned, p) for p in range(WORLD)] + y64, res_ref = add_rmsnorm_reference(sums[rank], rs[rank], nw, EPS) + torch.cuda.synchronize() + if none is not None: + errors.append(f"{tag}: the routed-only partials returned a shared output") + _expect(errors, f"{tag} routed slots", slots, ref_slots) + _expect(errors, f"{tag} residual after the MoE tail", r2, res_ref) + _expect_mxfp8(errors, f"{tag} next codes / scales", codes[rank * rows:(rank + 1) * rows], + scales[rank * rows:(rank + 1) * rows], y64) + _expect(errors, f"{tag} final reduce-scatter", h, sums[rank]) + _expect(errors, f"{tag} second residual", r3, rs[rank]) + del moe_in, pending, slots, local, gathered, ref_slots, owned + torch.cuda.empty_cache() + if sdma: + ph.drain() + + +def _negative_worker(rank, errors, token_counts, sdma=False): + """The chain with the side stream delayed and the join skipped: it must report a mismatch.""" + found = [] + _chain_worker(rank, found, token_counts, sdma, 200_000_000, True) + if not found: + errors.append(f"rank {rank}: the skipped join went unnoticed") + + +@pytest.mark.parametrize("sdma", [False, True]) +def test_moe_chain_delayed_side_stream(sdma): + run_on_2_gpus(_chain_worker, [2048, 4097, 15745], sdma, 2_000_000, timeout=1500) + + +@pytest.mark.parametrize("sdma", [False, True]) +def test_moe_chain_skipped_join_fails(sdma): + run_on_2_gpus(_negative_worker, [4097], sdma, timeout=900) + + +@pytest.mark.parametrize("tokens", [SERVED_TOKENS]) +def test_handoffs_bitwise(tokens): + run_on_2_gpus(_worker, tokens, timeout=1500) + + +@pytest.mark.parametrize("tokens", [SERVED_TOKENS]) +def test_handoffs_sdma_bitwise(tokens): + run_on_2_gpus(_worker, tokens, True, timeout=1500) + + +@pytest.mark.parametrize("sdma", [False, True]) +def test_handoffs_shard_copy_compact(sdma): + """A shard copy of the shared expert keeps the compact payload (no pending shared expert).""" + run_on_2_gpus(_worker, (2048, 8885), sdma, False, timeout=1500) + + +def _pool_worker(rank, errors, t, layers, sdma=False, recorded=False): + """``layers`` MoE layers enqueued behind a long sleep (the host far ahead of the GPU): the pool may grow by about + one layer's working set, not by a layer's side-stream buffers per layer. ``recorded``: the side stream's buffers + are recorded on it instead of held by the ``SharedPending`` (negative control).""" + from unittest import mock + + import amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 as P + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import (SdmaSeqParallelHandoffs, + SeqParallelHandoffs, shard_rows) + + dev = torch.device("cuda", rank) + cls = SdmaSeqParallelHandoffs if sdma else SeqParallelHandoffs + ph = cls(rank, dist.group.WORLD, dev, t) + o_w, o_s = o_proj_weight(dev) + mw = moe_weights(dev, rank) + nw = rand((H,), 3, dev, 0.2) + x = rand((shard_rows(t), H), t + rank, dev) + attn = rand((t, Q), t + 10 + rank, dev, 0.5) + full = P.shared_expert_full + patch = mock.patch.object(P, "shared_expert_full", lambda *a, keep=None, **k: full(*a, **k)) + + def run(n): + r = x + for _ in range(n): + moe_in, r, shared = ph.attention_moe(attn, o_w, o_s, r, nw, EPS, mw) + slots, _ = P.mxfp4_moe(moe_in, mw, return_partials=True) + del moe_in + _, _, r = ph.ffn_norm_gather((slots, shared), r, nw, EPS) + del slots, shared + + with patch if recorded else contextlib.nullcontext(): + run(1) + torch.cuda.synchronize() + dist.barrier() + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + r0, a0 = torch.cuda.memory_reserved(), torch.cuda.memory_allocated() + torch.cuda._sleep(4_000_000_000) + run(layers) + torch.cuda.synchronize() + grown, used = torch.cuda.max_memory_reserved() - r0, torch.cuda.max_memory_allocated() - a0 + if grown > used + 2**29: + errors.append(f"rank {rank}: the pool grew by {grown / 2**30:.2f} GiB, {used / 2**30:.2f} GiB allocated") + if sdma: + ph.drain() + + +def _pool_negative_worker(rank, errors, t, layers, sdma): + found = [] + _pool_worker(rank, found, t, layers, sdma, True) + if not found: + errors.append(f"rank {rank}: recorded side-stream buffers did not grow the pool") + + +@pytest.mark.parametrize("sdma", [False, True]) +def test_side_stream_pool_bounded(sdma): + run_on_2_gpus(_pool_worker, 16384, 24, sdma, timeout=900) + + +@pytest.mark.parametrize("sdma", [False, True]) +def test_side_stream_pool_recorded_grows(sdma): + run_on_2_gpus(_pool_negative_worker, 16384, 24, sdma, timeout=900) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_pack.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_pack.py new file mode 100644 index 0000000..2b2af44 --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_pack.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MXFP8 pack of the attention output vs reference.pack_reference, bit for bit: row counts on and off the 32-row +tiles (the served 2048-32768-token range, tails), an all-zero row, tiny and large blocks, a block below 2^-119 (the +documented conversion rule), into a pool-slot view (rows beyond M untouched), eager vs graph replay; and the codes +equal the library quantizer's on the same rows.""" + +import pytest +import torch +from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import mxfp8_pack_q4096 +from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2.reference import pack_reference + +D, ROW_BYTES = 4096, 4224 + + +def _require_gfx950(): + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + if arch != "gfx950": + pytest.skip(f"gfx950 only, found {arch}") + + +def _rows(m: int, seed: int) -> torch.Tensor: + g = torch.Generator(device="cuda").manual_seed(seed) + x = torch.randn(m, D, device="cuda", generator=g) + x = x * torch.rand(m, 1, device="cuda", generator=g) * 40 + if m > 5: + x[1] = 0 # an all-zero row: the scale floor + x[2, :32] = 2.0**-36 # a tiny block + x[3, :64] = 3e4 # a large block + x[4, 64:96] = -(2.0**-124) # below 2^-119: byte 0, converted with 2^-126 + x[5, 96:128] = 448.0 * 2.0**3 # exactly 448 * 2^k: the scale boundary + return x.to(torch.bfloat16) + + +@pytest.mark.parametrize("m", [1, 6, 7, 8, 9, 31, 32, 33, 1000, 2048, 4097, 8192, 8885, 15745, 32768]) +def test_pack_reference(m): + _require_gfx950() + x = _rows(m, m) + got = mxfp8_pack_q4096(x) + torch.cuda.synchronize() + ref = pack_reference(x) + bad = (got != ref).any(dim=1).nonzero().flatten()[:8].tolist() + assert torch.equal(got, ref), f"rows {bad} differ" + + +def test_pack_codes_equal_library_quantizer(): + """Codes as gemm/mxfp8_common's quantizer gives them (scales: that one shuffles them).""" + _require_gfx950() + from amd_kernels.gemm.mxfp8_common.gemm import mxfp8_e4m3_quantize_aiter, mxfp8_e4m3_quantize_flydsl + + x = _rows(4096, 7) + packed = mxfp8_pack_q4096(x) + for quant in (mxfp8_e4m3_quantize_aiter, mxfp8_e4m3_quantize_flydsl): + q, _ = quant(x) + same = packed[:, :D] == q.view(torch.uint8) + same[4, 64:96] = True # the byte-0 block: the Triton quantizer multiplies by 2^127 there + assert bool(same.all()), quant.__name__ + + +def test_pack_into_slot_view(): + _require_gfx950() + m, total = 1000, 4128 + x = _rows(m, 3) + slot = torch.full((total + 7, ROW_BYTES), 0xA5, dtype=torch.uint8, device="cuda") + view = slot[5 : 5 + total] + mxfp8_pack_q4096(x, out=view[:m]) + torch.cuda.synchronize() + assert torch.equal(view[:m], pack_reference(x)) + assert bool((slot[:5] == 0xA5).all()) and bool((view[m:] == 0xA5).all()) and bool((slot[5 + total :] == 0xA5).all()) + + +def test_pack_graph_replay(): + _require_gfx950() + x = _rows(4097, 11) + out = torch.empty((4097, ROW_BYTES), dtype=torch.uint8, device="cuda") + mxfp8_pack_q4096(x, out=out) # loads the kernel before capture + torch.cuda.synchronize() + eager = out.clone() + out.fill_(0) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + mxfp8_pack_q4096(x, out=out) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager) + x.copy_(_rows(4097, 12)) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, pack_reference(x)) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_shard.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_shard.py new file mode 100644 index 0000000..0a311d1 --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_shard.py @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``shard``: rank r's ``shard_rows`` rows of every token, zero padded, every token on one rank in rank order; +``SeqParallelHandoffs`` sizes itself for a batch-token limit clamped to ``MIN_TOKENS``..``MAX_TOKENS`` (2 GPUs).""" + +import pytest +import torch +import torch.distributed as dist +from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import ( + MAX_TOKENS, + MIN_TOKENS, + WORLD, + SeqParallelHandoffs, + shard, + shard_rows, +) +from tests.multi_gpu import run_on_2_gpus + + +@pytest.mark.parametrize("t", [1, 31, 4096, 4097, 8885, 15745, 32768]) +def test_shard(t): + x = torch.randn(t, 8, generator=torch.Generator().manual_seed(t)).bfloat16() + parts = [shard(x, r) for r in range(WORLD)] + rows = shard_rows(t) + assert all(p.shape == (rows, 8) and p.is_contiguous() for p in parts) + whole = torch.cat(parts) + assert torch.equal(whole[:t], x) and not whole[t:].any() + + +def _clamp_worker(rank, errors): + dev = torch.device("cuda", rank) + for asked, want in ((100, MIN_TOKENS), (12000, 12000), (10**6, MAX_TOKENS)): + ph = SeqParallelHandoffs(rank, dist.group.WORLD, dev, asked) + if (ph.max_tokens, ph.rows) != (want, shard_rows(want)): + errors.append(f"rank {rank}: max_tokens {asked} -> {ph.max_tokens}, rows {ph.rows}; want {want}") + del ph + torch.cuda.empty_cache() + + +def test_max_tokens_clamped(): + run_on_2_gpus(_clamp_worker, timeout=900) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_warm_up.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_warm_up.py new file mode 100644 index 0000000..315802a --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_warm_up.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Rule 4 on 2 ranks, fresh rank processes (this file's first call): the communicators' construction and ``warm_up`` +compile every variant, then a decoder layer's phases at the served token counts (2048-32768, off the 32-row grid +too) compile nothing: the input gather, the MoE layer (attention hand-off with the full shared expert, ``mxfp4_moe`` +on the gathered routed-only input, the FFN hand-off on its partials), the dense layer's hand-offs, the final reduce-scatter and the gathers; the +same for the SDMA form after its construction. No FlyDSL launch happens inside a warm-up.""" + +import torch +import torch.distributed as dist +from tests.multi_gpu import run_on_2_gpus + +from .workload import H, Q, WORLD, moe_weights, o_proj_weight, rand + +WALK_TOKENS = (2048, 2049, 3000, 4096, 4097, 6000, 8192, 8193, 8885, 12289, 15745, 16384, 16385, 24577, 31425, 32768) +SDMA_TOKENS = (2048, 4097, 8885, 12289, 16385) + + +def _worker(rank, errors): + from amd_kernels import warm_up + from amd_kernels.collectives.seq_parallel_handoffs_h6144_q4096_ranks2 import (SdmaSeqParallelHandoffs, + SeqParallelHandoffs, shard_rows) + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + from flydsl.compiler.jit_function import CompiledFunction, MlirCompiler + + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + warm_up.set_workers(8) # 2 ranks share the lease's CPUs + dev = torch.device("cuda", rank) + in_warm_up = [] + launch = CompiledFunction.__call__ + + def spy(self, *args): + if warm_up._in_warm_up.get(): + in_warm_up.append(1) + return launch(self, *args) + + CompiledFunction.__call__ = spy + try: + ph = SeqParallelHandoffs(rank, dist.group.WORLD, dev) + o_w, o_s = o_proj_weight(dev) + mw = moe_weights(dev, rank) + nw = rand((H,), 3, dev, 0.2) + ph.warm_up(o_w, o_s, mw) + finally: + CompiledFunction.__call__ = launch + if in_warm_up: + errors.append(f"rank {rank}: {len(in_warm_up)} launches inside a warm-up") + # the SDMA form (one instance per DBO microbatch of a 32768-token step) compiles its local kernels when built + sdma = SdmaSeqParallelHandoffs(rank, dist.group.WORLD, dev, SDMA_TOKENS[-1]) + late0 = warm_up.late_compiles() + real = MlirCompiler.compile + compiles = [] + + def counting(*a, **kw): + compiles.append(1) + return real(*a, **kw) + + MlirCompiler.compile = counting + try: + for hand, tokens in ((ph, WALK_TOKENS), (sdma, SDMA_TOKENS)): + for t in tokens: + rows = shard_rows(t) + x = rand((rows, H), t + rank, dev) + codes, scales = hand.input_gather(x) + attn = rand((t, Q), t + 10 + rank, dev, 0.5) + moe_in, r, shared = hand.attention_moe(attn, o_w, o_s, x, nw, 1e-6, mw) + slots, _ = mxfp4_moe(moe_in, mw, return_partials=True) + codes, scales, r = hand.ffn_norm_gather((slots, shared), r, nw, 1e-6) + h = hand.ffn_reduce_scatter((slots, shared)) + del slots, shared, moe_in + codes, scales, r = hand.attention_dense(attn, o_w, o_s, r, nw, 1e-6) + partial = rand((WORLD * rows, H), t + 20 + rank, dev, 0.25) + codes, scales, r = hand.ffn_norm_gather(partial, r, nw, 1e-6) + h = hand.ffn_reduce_scatter(partial) + hand.gather(h) + torch.cuda.synchronize() + if compiles: + errors.append(f"rank {rank}: {len(compiles)} compiles at {t} tokens ({type(hand).__name__})") + compiles.clear() + if not bool(torch.isfinite(h).all()): + errors.append(f"rank {rank}: non-finite output at {t} tokens ({type(hand).__name__})") + finally: + MlirCompiler.compile = real + if warm_up.late_compiles() != late0: + errors.append(f"rank {rank}: {warm_up.late_compiles() - late0} late compiles under an entry") + if ("collectives/seq_parallel_handoffs_h6144_q4096_ranks2", ("moe", True, True)) not in warm_up.warmed(): + errors.append(f"rank {rank}: the MoE hand-off mode was not warmed: {sorted(warm_up.warmed())}") + + +def _worker_then_reset(rank, errors): + """``_worker``, then the first-call warm-up off again: later tests reuse these rank processes.""" + from amd_kernels import warm_up + + try: + _worker(rank, errors) + finally: + warm_up.set_auto(False) + warm_up.set_workers(None) + + +def test_warm_up_then_no_compile(): + run_on_2_gpus(_worker_then_reset, timeout=2400) diff --git a/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/workload.py b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/workload.py new file mode 100644 index 0000000..fac17e9 --- /dev/null +++ b/tests/collectives/seq_parallel_handoffs_h6144_q4096_ranks2/workload.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared inputs of the two-rank hand-off tests: the served token counts, the weights (o_proj_full, norms, the MoE +block with its full shared expert) and seeded per-rank inputs (every rank can rebuild every rank's).""" + +import torch + +H, Q, WORLD, I = 6144, 4096, 2, 1536 +# the SP range's first count, counts off the 32-row grid (4097, 8885, 15745), 32768: the cap +SERVED_TOKENS = (2048, 4096, 4097, 8885, 15745, 32768) + + +def rand(shape, seed, dev, scale=1.0, dtype=torch.bfloat16): + g = torch.Generator(device=dev).manual_seed(seed) + return (torch.randn(shape, device=dev, generator=g) * scale).to(dtype) + + +def o_proj_weight(dev): + """A preshuffled MXFP8 o_proj_full (N 6144, K 8192): codes and their scales.""" + from amd_kernels.gemm.mxfp8_common.gemm import preshuffle_mxfp8_weight + + g = torch.Generator(device=dev).manual_seed(5) + wq = torch.randint(0, 120, (H, WORLD * Q), generator=g, device=dev, dtype=torch.uint8) + wq = torch.where(torch.rand(wq.shape, generator=g, device=dev) < 0.5, wq, wq | 0x80) # both signs + wsc = torch.randint(116, 121, (H, WORLD * Q // 32), generator=g, device=dev, dtype=torch.uint8) + return preshuffle_mxfp8_weight(wq.view(torch.float8_e4m3fn), wsc) + + +def moe_weights(dev, rank=0, full=True): + """The TP2 MoE block (router, biased top-4, MXFP4 routed experts at I 1536, the MXFP8 shared expert) as + MoeWeights: ``full``, the whole shared expert (I 3072) with TP rank ``rank``; else rank 0's shard copy.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, SharedWeights + from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_experts + + raw, _ = make_experts() + g = torch.Generator(device=dev).manual_seed(607) + + def weight(n, k): + q = (torch.randn((n, k), device=dev, generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + return q, torch.randint(118, 124, (n, k // 32), device=dev, dtype=torch.uint8, generator=g) + + if full: + shared = SharedWeights.from_row_major(*weight(4 * I, H), *weight(H, 2 * I), tp_rank=rank) + else: + shared = SharedWeights.from_row_major(*weight(2 * I, H), *weight(H, I)) + g2 = torch.Generator(device=dev).manual_seed(5) + router = torch.randn(128, H, device=dev, generator=g2) / H**0.5 + bias = torch.randn(128, device=dev, generator=g2) * 1e-2 + return MoeWeights.from_row_major(router, bias, *raw, shared=shared) diff --git a/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_decoder_ranks2.py b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_decoder_ranks2.py new file mode 100644 index 0000000..f93fc8a --- /dev/null +++ b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_decoder_ranks2.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: ``DenseDecoderRanks2``'s calls equal the per-kernel calls they wrap with the code layout of the +consumer GEMM (bit for bit), the fc in 2048-row pieces equals the torch reference (cosine) and the per-piece calls; +``forward`` (eager and graph replay) and ``store_context_kv`` equal the per-call loop and chain bit for bit at +1-2048 rows; after the block's construction and the entries' first calls nothing compiles.""" +import os + +import pytest +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +WORLD, H, V2, I = 2, 6144, 100032, 6144 +QKV, CTX_N, EPS, SWIGLU = (4608, H), 2560, 1e-6, (1.702, 1.0, 7.0) +KV_HEADS = 2 + + +def _real(q, s, m, tile): + """The written bytes of codes / scales of m rows (padding rows stay unwritten).""" + from amd_kernels.attention.nvfp4_sparse_decode_gqa16_d128_p128_top16.reference import ( + aiter_scale_offsets, + tile_code_offsets, + ) + c = q.view(torch.uint8) + c = c.reshape(-1)[tile_code_offsets(m, H, q.device)] if tile else c[:m] + return c, s.reshape(-1)[aiter_scale_offsets(m, H, s.device)] + + +def _check(errors, ok, what): + if not ok: + errors.append(what) + + +def _calls_worker(rank, errors): + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import DenseDecoderRanks2 + from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import vocab_gather_gemma_mxfp8 + from amd_kernels.gemm.mxfp8_common import gemm as G + dev = torch.device("cuda", rank) + g = torch.Generator(device=dev).manual_seed(11) # the same replicated tensors on both ranks + fc_w = (torch.randn(H, 5 * H, device=dev, generator=g) * 0.01).bfloat16() + emb = (torch.randn(V2, H, device=dev, generator=torch.Generator(device=dev).manual_seed(20 + rank))).bfloat16() + blk = DenseDecoderRanks2(rank, dist.group.WORLD, dev, fc_weight=fc_w, embedding=emb) + nw = (torch.randn(H, device=dev, generator=g) * 0.1).bfloat16() + for m in (1, 7, 8, 21, 56, 224, 364, 448, 1792, 2048, 2049, 4100): + taps = [torch.randn(m + 3, H, device=dev, generator=g).bfloat16() for _ in range(5)] + out = torch.empty(m + 3, H, dtype=torch.bfloat16, device=dev) + blk.fc(taps, m, out) + want = torch.empty(m, H, dtype=torch.bfloat16, device=dev) + for i in range(0, m, 2048): + j = min(i + 2048, m) + blk.fc_comm([t[i:j] for t in taps], want[i:j]) + torch.cuda.synchronize() + _check(errors, torch.equal(out[:m].view(torch.int16), want.view(torch.int16)), f"rank {rank} m {m}: fc pieces") + ref = (torch.cat([t[:m] for t in taps], 1).double() @ fc_w.double().T) + cos = torch.nn.functional.cosine_similarity(out[:m].double().flatten(), ref.flatten(), dim=0) + _check(errors, cos > 0.9999, f"rank {rank} m {m}: fc cosine {float(cos)}") + if m > 2048: + continue + _check(errors, torch.equal(blk.fc_cat(torch.cat([t[:m] for t in taps], 1)).view(torch.int16), + out[:m].view(torch.int16)), f"rank {rank} m {m}: fc_cat") + # embedding + norm + quant, all-reduce + norm (+ quant) for a consumer of (N, K) = qkv / gate_up shapes + ids = torch.randint(0, WORLD * V2, (m,), device=dev, generator=g) + for shape in (QKV, (2 * I, H)): + tile = G.prequant_code_tile_rows(m, *shape, torch.bfloat16) + got = blk.embed_norm_quant(ids, nw, EPS, shape) + want = vocab_gather_gemma_mxfp8(ids, blk.vocab, nw, EPS, tile) + x = torch.randn(m, H, device=dev, generator=torch.Generator(device=dev).manual_seed(m + rank)).bfloat16() + res = torch.randn(m, H, device=dev, generator=g).bfloat16() + got2 = blk.ar_norm_quant(x, res, nw, EPS, shape) + want2 = blk.ar.ar_add_rmsnorm_mxfp8(x, res, nw, EPS, tile, owned_residual=True) + torch.cuda.synchronize() + lo, hi = blk.ar.owned_rows(m) if m >= 24 else (0, m) # the owner form writes its residual rows only + for name, g_, w_, rows in (("embed", got, want, slice(0, m)), ("ar", got2, want2, slice(lo, hi))): + same = g_[0].shape == w_[0].shape and g_[1].shape == w_[1].shape + same = same and all(torch.equal(a, b) for a, b in zip(_real(g_[0], g_[1], m, tile), + _real(w_[0], w_[1], m, tile))) + same = same and torch.equal(g_[2][rows].view(torch.int16), w_[2][rows].view(torch.int16)) + _check(errors, same, f"rank {rank} m {m} {shape}: {name}") + got3 = blk.ar_norm(x, res, nw, EPS) + want3 = blk.ar.ar_add_rmsnorm(x, res, nw, EPS, owned_residual=True) + torch.cuda.synchronize() + _check(errors, torch.equal(got3[0].view(torch.int16), want3[0].view(torch.int16)) + and torch.equal(got3[1][lo:hi].view(torch.int16), want3[1][lo:hi].view(torch.int16)), + f"rank {rank} m {m}: ar_norm") + blk.close() + + +ROWS = (1, 7, 8, 9, 16, 17, 56, 224, 280, 364, 448, 896, 1792, 2048) + + +def _mxfp8_weight(n, k, g): + from amd_kernels.gemm.mxfp8_common import gemm as G + codes = (torch.randn(n, k, device=g.device, generator=g) * 0.6).to(torch.float8_e4m3fn) + exp = torch.randint(117, 122, (n, k // 32), device=g.device, generator=g).to(torch.uint8) + return G.preshuffle_mxfp8_weight(codes, exp) + + +def _loop(blk, ids, layers, attn, final, form): + """The per-call sequence ``forward`` replaces.""" + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import mxfp8_swiglu_oai_mlp + m = ids.numel() + q, s, res = blk.embed_norm_quant(ids, layers[0].input_norm, EPS, layers[0].qkv_shape) + for i, layer in enumerate(layers): + o = attn(i, q, s) + q, s, res = blk.ar_norm_quant(o, res, layer.post_norm, EPS, layer.gate_up_w.shape) + d = mxfp8_swiglu_oai_mlp(q, s, m, layer.gate_up_w, layer.gate_up_ws, layer.down_w, layer.down_ws, *SWIGLU, + form=form) + if i + 1 < len(layers): + q, s, res = blk.ar_norm_quant(d, res, layers[i + 1].input_norm, EPS, layers[i + 1].qkv_shape) + return blk.ar_norm(d, res, final, EPS)[0] + + +def _setup(rank, dev): + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import DenseDecoderRanks2, DenseLayer + g = torch.Generator(device=dev).manual_seed(11) + fc_w = (torch.randn(H, 5 * H, device=dev, generator=g) * 0.01).bfloat16() + gr = torch.Generator(device=dev).manual_seed(30 + rank) # this rank's shards + emb = torch.randn(V2, H, device=dev, generator=gr).bfloat16() + blk = DenseDecoderRanks2(rank, dist.group.WORLD, dev, fc_weight=fc_w, embedding=emb) + + def norm(): + return (torch.randn(H, device=dev, generator=g) * 0.1).bfloat16() + + layers = [DenseLayer(norm(), QKV, norm(), *_mxfp8_weight(2 * I, H, gr), *_mxfp8_weight(H, I, gr)) + for _ in range(3)] + return blk, layers, norm, g, gr + + +def _forward_worker(rank, errors): + from amd_kernels.attention.norm_rope_fp8_insert_d128_rot64.kv_norm_rope_insert_multilayer import ( + kv_norm_rope_insert_multilayer, + ) + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import gemma_rmsnorm_mxfp8_linear + from amd_kernels.gemm.mxfp8_common import gemm as G + dev = torch.device("cuda", rank) + blk, layers, norm, g, gr = _setup(rank, dev) + taps = [torch.randn(16, H, device=dev, generator=g).bfloat16() for _ in range(5)] + out = torch.empty(16, H, dtype=torch.bfloat16, device=dev) + for bad in (taps[:4], taps[:4] + [torch.randn(16, H + 1, device=dev).bfloat16()[:, :H]]): + try: + blk.fc(bad, 16, out) + errors.append(f"rank {rank}: fc accepted bad taps") + except AssertionError: + pass + final = norm() + ctx_w, ctx_ws = _mxfp8_weight(CTX_N, H, g) + k_norm = (torch.randn(5, 128, device=dev, generator=g) * 0.1).bfloat16() + cos_sin = (torch.rand(4096, 64, device=dev, generator=g) * 2 - 1).bfloat16() + ks = [torch.tensor([0.5 + 0.1 * i], device=dev) for i in range(5)] + vs = [torch.tensor([0.4 + 0.1 * i], device=dev) for i in range(5)] + for m in ROWS: + ids = torch.randint(0, WORLD * V2, (m,), device=dev, generator=g) + outs = [(torch.randn(m, H, device=dev, generator=gr) * 0.1).bfloat16() for _ in layers] + tile = G.prequant_code_tile_rows(m, *QKV, torch.bfloat16) + seen = {"forward": [], "loop": [], "graph": []} + + def attn(key): + return lambda i, q, s: (seen[key].append(_real(q, s, m, tile)), outs[i])[1] + + got = blk.forward(ids, layers, attn("forward"), final, EPS, SWIGLU) + want = _loop(blk, ids, layers, attn("loop"), final, "bf16_chain") + torch.cuda.synchronize() + _check(errors, torch.equal(got.view(torch.int16), want.view(torch.int16)), f"rank {rank} m {m}: forward") + for a, b in zip(seen["forward"], seen["loop"], strict=True): + _check(errors, all(torch.equal(x, y) for x, y in zip(a, b)), f"rank {rank} m {m}: qkv codes") + if m in (7, 56, 364, 2048): + graph = torch.cuda.CUDAGraph() + with blk.capture(), torch.cuda.graph(graph): + captured = blk.forward(ids, layers, lambda i, q, s: outs[i], final, EPS, SWIGLU) + graph.replay() + torch.cuda.synchronize() + _check(errors, torch.equal(captured.view(torch.int16), want.view(torch.int16)), + f"rank {rank} m {m}: graph replay") + del graph + # the context K / V chain: norm + GEMM, then the multi-layer insert into the 2-KV-head caches + states = torch.randn(m, H, device=dev, generator=g).bfloat16() + hn = norm() + pos = torch.randint(0, 4096, (m,), device=dev, generator=g) + slots = torch.randperm(24 * 128, device=dev, generator=g)[:m] + slots[1::5] = -1 + caches = [torch.zeros(24, KV_HEADS, 128, 256, dtype=torch.uint8, device=dev) for _ in range(5)] + ref = [c.clone() for c in caches] + blk.store_context_kv(states, hn, EPS, ctx_w, ctx_ws, pos, k_norm, EPS, cos_sin, caches, slots, ks, vs) + kv = gemma_rmsnorm_mxfp8_linear(states, hn, EPS, ctx_w, ctx_ws) + kv_norm_rope_insert_multilayer(kv, pos, k_norm, cos_sin, ref, [slots] * 5, ks, vs, EPS) + torch.cuda.synchronize() + _check(errors, all(torch.equal(a, b) for a, b in zip(caches, ref)), f"rank {rank} m {m}: context K/V") + _check(errors, all(bool(c[:, h].any()) for c in caches for h in range(KV_HEADS)), + f"rank {rank} m {m}: a KV head not written") + blk.close() + + +def _compile_worker(rank, errors): + """The block built (its comms warm up), the entries' first calls made: forward and store_context_kv compile + nothing at the draft's row counts (fresh FlyDSL cache, so a variant missed reaches the compiler).""" + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels import warm_up + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import gemma_rmsnorm_mxfp8_linear, mxfp8_swiglu_oai_mlp + from amd_kernels.gemm.mxfp8_common import gemm as G + warm_up.set_auto(True) + try: + dev = torch.device("cuda", rank) + blk, layers, norm, g, gr = _setup(rank, dev) + ctx_w, ctx_ws = _mxfp8_weight(CTX_N, H, g) + k_norm = (torch.randn(5, 128, device=dev, generator=g) * 0.1).bfloat16() + cos_sin = (torch.rand(4096, 64, device=dev, generator=g) * 2 - 1).bfloat16() + ks = [torch.tensor([1.0], device=dev) for _ in range(5)] + caches = [torch.zeros(24, KV_HEADS, 128, 256, dtype=torch.uint8, device=dev) for _ in range(5)] + # the caller's warm-ups: the GEMM weights' (vLLM's linear integration) and the entries' first calls + for w, ws in ((layers[0].gate_up_w, layers[0].gate_up_ws), (layers[0].down_w, layers[0].down_ws), + (ctx_w, ctx_ws)): + G.warmup_mxfp8_gemm_weight(w, ws, 2048, torch.bfloat16) + x = torch.zeros(1, H, dtype=torch.bfloat16, device=dev) + q, s, _ = blk.embed_norm_quant(torch.zeros(1, dtype=torch.int64, device=dev), layers[0].post_norm, EPS, + layers[0].gate_up_w.shape) + mxfp8_swiglu_oai_mlp(q, s, 1, layers[0].gate_up_w, layers[0].gate_up_ws, layers[0].down_w, + layers[0].down_ws, *SWIGLU, form="bf16_chain") + gemma_rmsnorm_mxfp8_linear(x, norm(), EPS, ctx_w, ctx_ws) + blk.store_context_kv(x, norm(), EPS, ctx_w, ctx_ws, torch.zeros(1, dtype=torch.int64, device=dev), k_norm, + EPS, cos_sin, caches, torch.full((1,), -1, dtype=torch.int64, device=dev), ks, ks) + torch.cuda.synchronize() + late, original = [], MlirCompiler.compile + + def spy(*a, **k): + if not warm_up._in_warm_up.get(): + late.append(m) + return original(*a, **k) + + MlirCompiler.compile = spy + try: + final = norm() + for m in (1, 2, 3, 7, 8, 14, 15, 16, 17, 21, 28, 35, 49, 56, 63, 70, 112, 140, 175, 224, 231, 280, 287, + 336, 343, 364, 371, 448, 512, 700, 896, 1024, 1400, 1792, 2047, 2048): + ids = torch.zeros(m, dtype=torch.int64, device=dev) + o = torch.zeros(m, H, dtype=torch.bfloat16, device=dev) + blk.forward(ids, layers, lambda i, q, s: o, final, EPS, SWIGLU) + blk.store_context_kv(o, final, EPS, ctx_w, ctx_ws, torch.zeros(m, dtype=torch.int64, device=dev), + k_norm, EPS, cos_sin, caches, torch.full((m,), -1, dtype=torch.int64, device=dev), + ks, ks) + torch.cuda.synchronize() + finally: + MlirCompiler.compile = original + _check(errors, not late, f"rank {rank}: compiles after the warm-ups at rows {sorted(set(late))}") + blk.close() + finally: + warm_up.set_auto(False) + + +def test_decoder_ranks2_calls(): + run_on_2_gpus(_calls_worker, timeout=1800) + + +def test_decoder_ranks2_forward_context_kv(): + run_on_2_gpus(_forward_worker, timeout=1800) + + +def test_decoder_ranks2_no_compile_after_warm_up(tmp_path): + saved = os.environ.get("FLYDSL_RUNTIME_CACHE_DIR") + os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = str(tmp_path / "flydsl") # new rank processes with a fresh cache + try: + run_on_2_gpus(_compile_worker, timeout=3600) + finally: + if saved is None: + os.environ.pop("FLYDSL_RUNTIME_CACHE_DIR") + else: + os.environ["FLYDSL_RUNTIME_CACHE_DIR"] = saved diff --git a/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_mlp_norm_linear.py b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_mlp_norm_linear.py new file mode 100644 index 0000000..df8e117 --- /dev/null +++ b/tests/decoder/dense_mxfp8_h6144_i6144_ranks2/test_mlp_norm_linear.py @@ -0,0 +1,195 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MLP block and the norm + GEMM at TP2 (I 6144, context K/V N 2560): bit for bit the per-kernel chains they +replace (the GEMM calls the MXFP8 linear integration makes for codes / bf16 input), cosine ~1 against the torch +references, the first call compiles every variant of 1-2048 rows.""" +import json +import os +import subprocess +import sys +import time + +import pytest +import torch + +from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import gemma_rmsnorm_mxfp8_linear, mxfp8_swiglu_oai_mlp +from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2.reference import mlp_reference, norm_linear_reference +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_gateup_swiglu_n12288_k6144_i6144 import host as GUS +from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144 import gemma_rmsnorm_mxfp8 + +H, I, CTX_N, EPS = 6144, 6144, 2560, 1e-6 +ALPHA, BETA, LIMIT = 1.702, 1.0, 7.0 +# graph sizes of 7-row draft blocks (7R, R = 1..64 incl. the c32-c52 band), the fused / tiled / row-major edges +ROWS = [1, 2, 7, 8, 9, 14, 16, 17, 21, 32, 33, 56, 64, 100, 112, 128, 129, 160, 161, 224, 225, 256, 257, 280, 288, + 300, 320, 336, 364, 384, 385, 416, 448, 449, 512, 513, 640, 768, 769, 896, 1024, 1792, 2048] + + +def _weight(n, k, seed): + g = torch.Generator(device="cuda").manual_seed(seed) + codes = (torch.randn(n, k, device="cuda", generator=g) * 0.6).to(torch.float8_e4m3fn) + exp = torch.randint(117, 122, (n, k // 32), device="cuda", generator=g).to(torch.uint8) + w, ws = G.preshuffle_mxfp8_weight(codes.clone(), exp.clone()) + return codes, exp, w, ws + + +@pytest.fixture(scope="module") +def weights(): + return _weight(2 * I, H, 1), _weight(H, I, 2), _weight(CTX_N, H, 3), torch.randn(H, device="cuda").bfloat16() * 0.1 + + +def _codes(x, nw, m, n, k): + """The producer's codes (the plain norm in the consumer's layout).""" + tile = G.prequant_code_tile_rows(m, n, k, torch.bfloat16) + return gemma_rmsnorm_mxfp8(x, nw, EPS, tile) + + +def _per_kernel_mlp(q, s, m, gu, gus, dw, dws, form): + """The per-kernel chain (the MXFP8 GEMM picks of the linear integration).""" + from amd_kernels.activation.swiglu_oai_mxfp8_i6144 import swiglu_oai_mxfp8 + + def gemm(q, s, w, ws): + n, k = w.shape + if G.prequant_code_tile_rows(m, n, k, torch.bfloat16): + return G.mxfp8_gemm_shuffled(q, s, w, ws, torch.bfloat16, m).reshape(m, n) + return G.mxfp8_gemm(q, s, w, ws, torch.bfloat16, activation_scale_preshuffled=True).reshape(m, n) + + if m <= 16 and q.shape[0] == 16: + act = GUS.mxfp8_gateup_swiglu_bf16(q, s, gu, gus, m, ALPHA, BETA, LIMIT, form=form) + return G.mxfp8_linear(act.reshape(-1, I), dw, dws, torch.bfloat16).reshape(m, H) + h = gemm(q, s, gu, gus) + tile = G.prequant_code_tile_rows(m, H, I, torch.bfloat16) + qd, sd = swiglu_oai_mxfp8(h, ALPHA, BETA, LIMIT, tile, form=form) + return gemm(qd, sd, dw, dws) + + +@pytest.mark.parametrize("m", ROWS) +@pytest.mark.parametrize("form", ["bf16_chain", "fp32"]) +def test_mlp(weights, m, form): + (guc, gue, gu, gus), (dc, de, dw, dws), _, nw = weights + x = torch.randn(m, H, device="cuda").bfloat16() + q, s = _codes(x, nw, m, 2 * I, H) + if m <= 16: + assert q.shape[0] == 16 # the fused gate_up path's code block + got = mxfp8_swiglu_oai_mlp(q, s, m, gu, gus, dw, dws, ALPHA, BETA, LIMIT, form=form) + want = _per_kernel_mlp(q, s, m, gu, gus, dw, dws, form) + torch.cuda.synchronize() + assert got.shape == (m, H) and torch.equal(got.view(torch.int16), want.view(torch.int16)) + from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144.reference import gemma_rmsnorm_reference + ref = mlp_reference(gemma_rmsnorm_reference(x, nw, EPS), guc, gue, dc, de, ALPHA, BETA, LIMIT, form) + cos = torch.nn.functional.cosine_similarity(got.float().flatten(), ref.float().flatten(), dim=0) + assert cos > 0.999, float(cos) + + +@pytest.mark.parametrize("m", [0] + ROWS + [1536, 1537, 3072, 3073, 4096, 4801, 8192]) +def test_norm_linear(weights, m): + _, _, (cc, ce, cw, cws), nw = weights + x = torch.randn(m, H, device="cuda").bfloat16() + got = gemma_rmsnorm_mxfp8_linear(x, nw, EPS, cw, cws) + assert got.shape == (m, CTX_N) + if not m: + return + q, s = _codes(x, nw, m, CTX_N, H) + if G.prequant_code_tile_rows(m, CTX_N, H, torch.bfloat16): + want = G.mxfp8_gemm_shuffled(q, s, cw, cws, torch.bfloat16, m) + else: + want = G.mxfp8_gemm(q, s, cw, cws, torch.bfloat16, activation_scale_preshuffled=True) + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), want.reshape(m, CTX_N).view(torch.int16)) + ref = norm_linear_reference(x, nw, EPS, cc, ce) + cos = torch.nn.functional.cosine_similarity(got.float().flatten(), ref.float().flatten(), dim=0) + assert cos > 0.999, float(cos) + + +def test_mlp_graph_replay(weights): + """Captured calls replay the eager bits (fused, tiled and row-major gate_up rows).""" + (_, _, gu, gus), (_, _, dw, dws), _, nw = weights + for m in (7, 16, 112, 364, 1024): + x = torch.randn(m, H, device="cuda").bfloat16() + q, s = _codes(x, nw, m, 2 * I, H) + want = mxfp8_swiglu_oai_mlp(q, s, m, gu, gus, dw, dws, ALPHA, BETA, LIMIT, form="bf16_chain") + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = mxfp8_swiglu_oai_mlp(q, s, m, gu, gus, dw, dws, ALPHA, BETA, LIMIT, form="bf16_chain") + graph.replay() + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), want.view(torch.int16)), m + + +def scenario(): + from flydsl.compiler.jit_function import MlirCompiler + from amd_kernels import warm_up + warm_up.set_auto(True) + (_, _, gu, gus), (_, _, dw, dws), (_, _, cw, cws), nw = _weight(2 * I, H, 1), _weight(H, I, 2), \ + _weight(CTX_N, H, 3), torch.zeros(H, device="cuda").bfloat16() + t0 = time.perf_counter() + for w, ws in ((gu, gus), (dw, dws), (cw, cws)): # the linear integration's warm-up of each weight at load + G.warmup_mxfp8_gemm_weight(w, ws, 32768, torch.bfloat16) + t1 = time.perf_counter() + late, original = [], MlirCompiler.compile + + def spy(*a, **k): + if warm_up._current.get() is not None and not warm_up._in_warm_up.get(): + late.append(1) + return original(*a, **k) + + MlirCompiler.compile = spy + bad = [] + for m in ROWS + [3, 5, 11, 15, 40, 63, 72, 96, 113, 192, 200, 231, 287, 343, 350, 357, 371, 399, 640, 1000, 1280, + 1536, 2047]: + before = len(late) + x = torch.zeros(m, H, device="cuda").bfloat16() + q, s = _codes(x, nw, m, 2 * I, H) + for form in ("bf16_chain", "fp32"): + mxfp8_swiglu_oai_mlp(q, s, m, gu, gus, dw, dws, ALPHA, BETA, LIMIT, form=form) + gemma_rmsnorm_mxfp8_linear(x, nw, EPS, cw, cws) + if len(late) > before: + bad.append(m) + torch.cuda.synchronize() + return {"late": bad, "reported": warm_up.late_compiles(), "warmed": len(warm_up.warmed()), + "gemm_warm_up_s": round(t1 - t0), "entries_s": round(time.perf_counter() - t1)} + + +def test_first_call_warm_up_compiles_every_variant(tmp_path): + proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=2400, + env={**os.environ, "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) + assert proc.returncode == 0, proc.stdout[-3000:] + proc.stderr[-3000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + assert result["late"] == [] and result["reported"] == 0 and result["warmed"] == 3, result + print(result) + + +def test_stack_and_context_kv_blockers(): + from amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2 import DenseLayer, context_kv_blocker, stack_blocker + + bf, f8 = torch.bfloat16, torch.float8_e4m3fn + norm = torch.empty(H, dtype=bf) + gu, gus = torch.empty(2 * I, H, dtype=f8), torch.empty(2 * I, H // 32, dtype=torch.uint8) + dw, dws = torch.empty(H, I, dtype=f8), torch.empty(H, I // 32, dtype=torch.uint8) + layer = DenseLayer(norm, (4608, H), norm, gu, gus, dw, dws) + fc, emb = torch.empty(H, 5 * H, dtype=bf), torch.empty(64, H, dtype=bf) + ok = dict(world=2, max_rows=2048, on_library=[True] * 20, fc_weight=fc, embedding=emb, layers=[layer] * 5) + assert stack_blocker(**ok) is None + tp4 = DenseLayer(norm, (2304, H), norm, gu[:6144], gus[:6144], dw[:, :3072], dws[:, :96]) + for key, bad in (("world", 4), ("world", 1), ("max_rows", 2049), ("max_rows", 0), + ("on_library", [True] * 19 + [False]), ("fc_weight", fc[:, :H]), ("embedding", emb.float()), + ("embedding", emb.T), ("layers", []), ("layers", [tp4]), + ("layers", [DenseLayer(norm, (4608, H), norm, gu.float(), gus, dw, dws)]), + ("layers", [DenseLayer(norm, (4608, 4096), norm, gu, gus, dw, dws)]), + ("layers", [DenseLayer(norm.float(), (4608, H), norm, gu, gus, dw, dws)])): + assert stack_blocker(**{**ok, key: bad}) is not None, key + caches = [torch.empty(4, 2, 128, 256, dtype=torch.uint8) for _ in range(5)] + scales = [torch.ones(1) for _ in range(5)] + cs = torch.empty(4096, 64, dtype=bf) + proj = torch.empty(5 * 2 * 2 * 128, H, dtype=f8) + ctx = dict(proj_on_library=True, proj_w=proj, rotary_dim=64, neox=True, cos_sin_cache=cs, kv_caches=caches, + k_scales=scales, v_scales=scales) + assert context_kv_blocker(**ctx) is None + for key, bad in (("proj_on_library", False), ("proj_w", proj[:1280]), ("rotary_dim", 32), ("neox", False), + ("cos_sin_cache", cs.float()), ("kv_caches", [c.bfloat16() for c in caches]), + ("k_scales", [s.half() for s in scales])): + assert context_kv_blocker(**{**ctx, key: bad}) is not None, key + + +if __name__ == "__main__": + print(json.dumps(scenario())) diff --git a/tests/embedding/vocab_gather_gemma_mxfp8_h6144/test_gather_norm.py b/tests/embedding/vocab_gather_gemma_mxfp8_h6144/test_gather_norm.py index ac19332..0eeb306 100644 --- a/tests/embedding/vocab_gather_gemma_mxfp8_h6144/test_gather_norm.py +++ b/tests/embedding/vocab_gather_gemma_mxfp8_h6144/test_gather_norm.py @@ -7,7 +7,7 @@ from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144 import vocab_gather_gemma_mxfp8,warmup_vocab_gather_gemma_mxfp8 from amd_kernels.embedding.vocab_gather_gemma_mxfp8_h6144.reference import lookup_reference from amd_kernels.norm.gemma_rmsnorm_bf16_mxfp8_h6144 import gemma_rmsnorm_mxfp8 -from tests.multi_gpu import run_on_4_gpus +from tests.multi_gpu import run_on_2_gpus, run_on_4_gpus def written(q,s,m,tile): @@ -25,8 +25,10 @@ def worker(rank,errors): from tests.mfma_spinner import mfma_spinner torch.set_num_threads(8) dev=torch.device('cuda',rank) + world=dist.get_world_size() + vocab=world*128 gen=torch.Generator().manual_seed(963) - shards=[(torch.randn((128,6144),generator=gen)*torch.exp2(torch.randint(-5,6,(128,192,1),generator=gen).expand(128,192,32).reshape(128,6144).float())).bfloat16() for _ in range(4)] + shards=[(torch.randn((128,6144),generator=gen)*torch.exp2(torch.randint(-5,6,(128,192,1),generator=gen).expand(128,192,32).reshape(128,6144).float())).bfloat16() for _ in range(world)] for shard in shards: shard[0].zero_() shard[1,:8]=torch.tensor([0.,-0.,2.**-133,-2.**-133,2.**-126,-2.**-126,1.,-1.],dtype=torch.bfloat16) @@ -48,9 +50,9 @@ def unexpected(*a,**kw):raise AssertionError('compile after warmup') for dtype in (torch.int32,torch.int64): for tile in (0,16): for m in (0,1,7,33,70,112,224,448,512,513,896,1792,2048): - ids=torch.randint(-2,514,(m,),generator=gen,dtype=dtype) + ids=torch.randint(-2,vocab+2,(m,),generator=gen,dtype=dtype) if m:ids[0]=0 - if m>7:ids[:7]=torch.tensor([0,1,128,256,384,-1,512]) + if m>7:ids[:7]=torch.tensor([0,1,128,vocab-128,vocab-1,-1,vocab]) inputs=ids.to(dev) raw=lookup(inputs) q,s=gemma_rmsnorm_mxfp8(raw,norm,1e-6,tile) @@ -87,7 +89,7 @@ def unexpected(*a,**kw):raise AssertionError('compile after warmup') if base is None:base=(aq[0].clone(),bs[0].clone()) assert torch.equal(aq,base[0].expand_as(aq)) and torch.equal(bs,base[1].expand_as(bs)) for m in (7,896): - ids=torch.arange(m,device=dev,dtype=torch.int32)%512 + ids=torch.arange(m,device=dev,dtype=torch.int32)%vocab ref=vocab_gather_gemma_mxfp8(ids,lookup,norm) torch.cuda.synchronize() spinner(100_000,side) @@ -100,3 +102,6 @@ def unexpected(*a,**kw):raise AssertionError('compile after warmup') def test_peer_lookup_norm_quant():run_on_4_gpus(worker,timeout=300) + + +def test_peer_lookup_norm_quant_ranks2():run_on_2_gpus(worker,timeout=300) diff --git a/tests/embedding/vocab_gather_h6144/test_gather.py b/tests/embedding/vocab_gather_h6144/test_gather.py index 27f9242..4c478eb 100644 --- a/tests/embedding/vocab_gather_h6144/test_gather.py +++ b/tests/embedding/vocab_gather_h6144/test_gather.py @@ -4,14 +4,15 @@ import torch.distributed as dist from amd_kernels.embedding.vocab_gather_h6144 import VocabGather from amd_kernels.embedding.vocab_gather_h6144.reference import vocab_gather_reference -from tests.multi_gpu import run_on_4_gpus +from tests.multi_gpu import run_on_2_gpus, run_on_4_gpus def worker(rank,errors): from flydsl.compiler.jit_function import MlirCompiler dev=torch.device('cuda',rank) + world=dist.get_world_size() torch.manual_seed(37) - shards=[torch.randn((128,6144),dtype=torch.bfloat16) for _ in range(4)] + shards=[torch.randn((128,6144),dtype=torch.bfloat16) for _ in range(world)] for shard in shards: shard[0,:8]=torch.tensor([0.,-0.,2.**-133,-2.**-133,2.**-126,-2.**-126,1.,-1.],dtype=torch.bfloat16) weight=shards[rank].to(dev) @@ -24,9 +25,9 @@ def unexpected(*a,**kw):raise AssertionError('compile after warmup') for test_case in range(18): m=(1,7,70,112,224,448,896,1792,2048)[test_case%9] dtype=torch.int32 if test_case<9 else torch.int64 - ids=torch.randint(-2,514,(m,),dtype=dtype) + ids=torch.randint(-2,world*128+2,(m,),dtype=dtype) ids[0]=0 - if m>7:ids[:7]=torch.tensor([0,128,256,384,-1,512,511]) + if m>7:ids[:7]=torch.tensor([0,128,(world-1)*128,world*128-129,-1,world*128,world*128-1]) inputs=ids.to(dev) expected=vocab_gather_reference(shards,ids).to(dev) eager=c(inputs) @@ -45,3 +46,6 @@ def unexpected(*a,**kw):raise AssertionError('compile after warmup') def test_peer_embedding():run_on_4_gpus(worker,timeout=180) + + +def test_peer_embedding_ranks2():run_on_2_gpus(worker,timeout=180) diff --git a/tests/gemm/bf16_concat_linear_n3072_k30720/__init__.py b/tests/gemm/bf16_concat_linear_n3072_k30720/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/gemm/bf16_concat_linear_n3072_k30720/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/gemm/bf16_concat_linear_n3072_k30720/test_compile.py b/tests/gemm/bf16_concat_linear_n3072_k30720/test_compile.py new file mode 100644 index 0000000..6c1d964 --- /dev/null +++ b/tests/gemm/bf16_concat_linear_n3072_k30720/test_compile.py @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The warm-up compiles every row tier of the GEMM and of the exchange: no compile afterwards.""" +import torch +import torch.distributed as dist +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import pack_weight +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.comm import ConcatLinearComm +from tests.multi_gpu import run_on_2_gpus + + +def worker(rank, errors): + from flydsl.compiler.jit_function import MlirCompiler + dev = torch.device('cuda', rank) + w = torch.zeros((3072, 30720), device=dev, dtype=torch.bfloat16) + c = ConcatLinearComm(pack_weight(w), rank, dist.group.WORLD) + c.warm_up() + orig = MlirCompiler.compile + + def unexpected(*a, **kw): + raise AssertionError('compile after warm-up') + MlirCompiler.compile = unexpected + try: + for m in (1, 7, 8, 16, 17, 56, 63, 64, 65, 70, 80, 81, 112, 128, 129, 224, 256, 257, 448, 512, 513, 896, + 1024, 1025, 1792, 2048): + taps = [torch.zeros((m, 6144), device=dev, dtype=torch.bfloat16) for _ in range(5)] + out = torch.empty((m, 6144), device=dev, dtype=torch.bfloat16) + c(taps, out) + torch.cuda.synchronize() + finally: + MlirCompiler.compile = orig + c.close() + + +def test_warmup(): + run_on_2_gpus(worker, timeout=600) diff --git a/tests/gemm/bf16_concat_linear_n3072_k30720/test_concat_linear.py b/tests/gemm/bf16_concat_linear_n3072_k30720/test_concat_linear.py new file mode 100644 index 0000000..32d0c64 --- /dev/null +++ b/tests/gemm/bf16_concat_linear_n3072_k30720/test_concat_linear.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""One-GPU contract of the N-shard concat linear: fp64 reference, every row tier, odd and zero rows, strided taps, +exponents +-5 binades, determinism, graph replay with poisoned outputs.""" +import pytest +import torch +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import bf16_concat_linear, pack_weight, warmup_concat_linear +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.host import config_for +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.reference import concat_linear_reference + +ROWS = (1, 7, 8, 16, 17, 33, 56, 63, 64, 65, 80, 81, 127, 128, 129, 255, 256, 257, 300, 448, 512, 513, 1000, 1024, 1025, 2048) + + +@pytest.fixture(scope='module') +def weights(): + g = torch.Generator(device='cuda').manual_seed(5) + w = (torch.randn((3072, 30720), device='cuda', generator=g) * 0.02).bfloat16() + return w, pack_weight(w) + + +def taps_for(m, seed, stride=6152): + g = torch.Generator(device='cuda').manual_seed(seed) + out = [] + for _ in range(5): + base = torch.randn((m, stride), device='cuda', generator=g) + ex = torch.randint(-5, 6, (m, 192, 1), device='cuda', generator=g).expand(m, 192, 32).reshape(m, 6144) + x = base[:, :6144] * torch.exp2(ex.float()) + base[:, :6144] = x + t = base.bfloat16()[:, :6144] + t[0].zero_() + out.append(t) + return out + + +@pytest.mark.parametrize('m', ROWS) +def test_contract(weights, m): + w, packed = weights + taps = taps_for(m, m) + ref = concat_linear_reference(taps, w) + got = bf16_concat_linear(taps, packed) + torch.cuda.synchronize() + assert not got.isnan().any() and not got.isinf().any() + assert torch.equal(got[0], torch.zeros_like(got[0])) # zero row + live = slice(1, m) + if m > 1: + cos = torch.nn.functional.cosine_similarity(got[live].double(), ref[live].double(), dim=1) + assert float(cos.min()) >= 0.99999, (m, config_for(m), float(cos.min())) + rel = float((got[live].double() - ref[live].double()).norm() / ref[live].double().norm()) + assert rel <= 1e-3, (m, rel) + per_row_rel = (got[live].double() - ref[live].double()).norm(dim=1) / ref[live].double().norm(dim=1) + assert float(per_row_rel.max()) <= 1e-3, (m, float(per_row_rel.max())) + again = bf16_concat_linear(taps, packed) + torch.cuda.synchronize() + assert torch.equal(got, again) + + +@pytest.mark.parametrize('m', (8, 80, 128, 256, 1024)) +def test_graph_replay(weights, m): + w, packed = weights + taps = taps_for(m, 100 + m) + warmup_concat_linear(taps, packed, max_rows=m) + eager = bf16_concat_linear(taps, packed) + out = torch.empty_like(eager) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + bf16_concat_linear(taps, packed, out=out) + for _ in range(3): + out.fill_(float('nan')) + g.replay() + torch.cuda.synchronize() + assert torch.equal(out, eager) + # changed inputs (in place) must be consumed by the replay: equal to a fresh eager call, not the old result + for t in taps: + t.mul_(0.5).add_(0.25) + fresh = bf16_concat_linear(taps, packed) + g.replay() + torch.cuda.synchronize() + assert not torch.equal(fresh, eager) + assert torch.equal(out, fresh) + + +@pytest.mark.parametrize('m',(80,256)) +def test_distinct_tap_strides(weights,m): + w,packed=weights + gen=torch.Generator(device='cuda').manual_seed(901+m) + taps=[torch.randn((m,6144+8*j),device='cuda',generator=gen,dtype=torch.bfloat16)[:,:6144] for j in range(5)] + got=bf16_concat_linear(taps,packed) + ref=concat_linear_reference(taps,w) + cos=torch.nn.functional.cosine_similarity(got.double(),ref.double(),dim=1) + assert cos.min()>=.99999 + assert (got.double()-ref.double()).norm()/ref.double().norm()<=1e-3 + + +@pytest.mark.parametrize('m', (8, 17, 80, 256, 512, 1024, 2048)) +def test_mfma_co_residency(weights, m): + from tests.mfma_spinner import mfma_spinner + _, packed = weights + taps = taps_for(m, 2000 + m) + ref = bf16_concat_linear(taps, packed) + torch.cuda.synchronize() + spinner = mfma_spinner(256) + side = torch.cuda.Stream(priority=-1) + for _ in range(3): + spinner(100_000, side) + got = bf16_concat_linear(taps, packed) + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), ref.view(torch.int16)) diff --git a/tests/gemm/bf16_concat_linear_n3072_k30720/test_gather.py b/tests/gemm/bf16_concat_linear_n3072_k30720/test_gather.py new file mode 100644 index 0000000..3d3cf43 --- /dev/null +++ b/tests/gemm/bf16_concat_linear_n3072_k30720/test_gather.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks: each projects its 3072-column shard and the exchange writes [rows, 6144] on every rank, equal bit for +bit to the two local shard products; queued eager calls with changing rows, a delayed rank (slot reuse), and both +exchange forms (push / pull) at several rows per group.""" +import time +import pytest +import torch +import torch.distributed as dist +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720 import bf16_concat_linear, pack_weight +from amd_kernels.gemm.bf16_concat_linear_n3072_k30720.comm import ConcatLinearComm +from tests.multi_gpu import run_on_2_gpus + +ROWS = (8, 56, 70, 80, 128, 256, 448, 512, 513, 1024, 1025, 2048) + + +def worker(rank, errors, exchanges): + dev = torch.device('cuda', rank) + g = torch.Generator(device=dev).manual_seed(11) + full = (torch.randn((6144, 30720), device=dev, generator=g) * 0.02).bfloat16() # same on every rank + packed = [pack_weight(full[r * 3072:(r + 1) * 3072]) for r in range(2)] + del full + c = ConcatLinearComm(packed[rank], rank, dist.group.WORLD) + c.warm_up() + history = [] + for call in range(2 * len(ROWS)): + m = ROWS[call % len(ROWS)] + gi = torch.Generator(device=dev).manual_seed(1000 + call) + taps = [torch.randn((m, 6144), device=dev, generator=gi).bfloat16() for _ in range(5)] + if rank == call % 2: + time.sleep(0.01) + out = torch.full((m, 6144), float('nan'), device=dev, dtype=torch.bfloat16) + ex = exchanges[call % len(exchanges)] if exchanges else None + c(taps, out, exchange_config=ex) + history.append((taps, out)) + torch.cuda.synchronize() + for taps, out in history: + want = torch.cat([bf16_concat_linear(taps, p) for p in packed], dim=1) + if not torch.equal(out, want): + errors.append(f'rank {rank}: rows {out.shape[0]} differ in {int((out != want).sum())} values') + c.close() + + +@pytest.mark.parametrize('exchanges', (None, ((1, 1, True), (2, 3, False), (4, 6, True), (1, 12, False), (8, 2, False), (2, 1, True)))) +def test_exchange_rank_skew_and_slot_reuse(exchanges): + run_on_2_gpus(worker, exchanges, timeout=600) diff --git a/tests/gemm/bf16_n100032_k6144/__init__.py b/tests/gemm/bf16_n100032_k6144/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/gemm/bf16_n100032_k6144/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/gemm/bf16_n100032_k6144/test_gemm.py b/tests/gemm/bf16_n100032_k6144/test_gemm.py new file mode 100644 index 0000000..a3887e7 --- /dev/null +++ b/tests/gemm/bf16_n100032_k6144/test_gemm.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""amd_kernels.gemm.bf16_n100032_k6144 against its fp64 reference, both weight layouts.""" +import pytest +import torch + +from amd_kernels.gemm.bf16_n100032_k6144 import (bf16_gemm_n100032_k6144, shuffle_weight, unshuffle_weight, + warmup_bf16_gemm_n100032_k6144) +from amd_kernels.gemm.bf16_n100032_k6144 import host +from amd_kernels.gemm.bf16_n100032_k6144.reference import K, N, bf16_gemm_n100032_k6144_reference + +MS = [1, 2, 7, 15, 16, 17, 33, 64, 70, 80, 96, 97, 112, 127, 128, 129, 160, 161, 192, 193, 200, 224, 225, 256, 257, + 300, 384, 385, 448, 513, 1024, 2100] + + +@pytest.fixture(scope="module") +def weights(): + g = torch.Generator(device="cuda").manual_seed(0) + w = (torch.randn(N, K, device="cuda", generator=g) * 0.02).to(torch.bfloat16) + return {False: w, True: shuffle_weight(w)} + + +def check_numeric(y, ref): + yf, rf = y.double(), ref.double() + cos = torch.nn.functional.cosine_similarity(yf, rf, dim=1) + l2 = (yf - rf).norm(dim=1) / rf.norm(dim=1) + assert bool(torch.all(cos >= 0.99999)), cos.min().item() + assert bool(torch.all(l2 <= 1e-3)), l2.max().item() + + +def test_shuffle_is_a_permutation(weights): + w, ws = weights[False], weights[True] + assert torch.equal(unshuffle_weight(ws).view(torch.int16), w.view(torch.int16)) + flat = ws.view(-1) + for n, k in [(0, 0), (15, 63), (16, 64), (50015, 6143), (50016, 0), (75000, 4321), (100031, 6143)]: + idx = (((n // 16) * (K // 64) + k // 64) * 16 + n % 16) * 64 + k % 64 + assert flat[idx].view(torch.int16) == w[n, k].view(torch.int16) + + +@pytest.mark.parametrize("shuffled", [False, True]) +@pytest.mark.parametrize("m", MS) +def test_reference(weights, shuffled, m): + w = weights[shuffled] + x = torch.randn(m, K, device="cuda").to(torch.bfloat16) + y = torch.full((m, N), float("nan"), device="cuda", dtype=torch.bfloat16) + bf16_gemm_n100032_k6144(x, w, y, shuffled=shuffled) + check_numeric(y, bf16_gemm_n100032_k6144_reference(x, weights[False])) + old = y.clone() + for _ in range(2): + y.fill_(float("nan")) + bf16_gemm_n100032_k6144(x, w, y, shuffled=shuffled) + assert torch.equal(y.view(torch.int16), old.view(torch.int16)) + + +def test_layouts_agree(weights): + x = torch.randn(80, K, device="cuda").to(torch.bfloat16) + a = bf16_gemm_n100032_k6144(x, weights[False], shuffled=False) + b = bf16_gemm_n100032_k6144(x, weights[True], shuffled=True) + assert torch.equal(a.view(torch.int16), b.view(torch.int16)) + + +def test_zero_rows(weights): + x = torch.empty(0, K, device="cuda", dtype=torch.bfloat16) + assert bf16_gemm_n100032_k6144(x, weights[False]).shape == (0, N) + + +@pytest.mark.parametrize("shuffled", [False, True]) +def test_nonfinite_propagation(weights, shuffled): + w = weights[False].clone() + w[0, 2] = float("nan") + w[100031, 8] = float("inf") + w[50016, 5] = float("-inf") + w[16].zero_() + x = torch.randn(3, K, device="cuda").to(torch.bfloat16) + x[0, 1] = float("nan") + x[1, 3] = float("inf") + x[2].zero_() + y = bf16_gemm_n100032_k6144(x, shuffle_weight(w) if shuffled else w, shuffled=shuffled) + ref = bf16_gemm_n100032_k6144_reference(x, w) + assert torch.equal(torch.isnan(y), torch.isnan(ref)) + assert torch.equal(torch.isposinf(y), torch.isposinf(ref)) + assert torch.equal(torch.isneginf(y), torch.isneginf(ref)) + fin = torch.isfinite(ref[2]) # a zero row stays 0 except at the NaN / inf weight columns (0 x inf = NaN) + assert bool(torch.all(y[2][fin] == 0)) and int((~fin).sum()) == 3 + + +def warm_up_one_mode(monkeypatch, shuffled, mode): + """Warm up one call mode; check it asks for exactly that mode's tile variants (one per WARMUP_ROWS entry), no + variant of another mode (each mode is its own kernel: the caller warms the modes it runs).""" + host._kernel.cache_clear() + keys, real = set(), host._kernel + + def spy(*key): + keys.add(key) # (tile config, shuffled, form) + return real(*key) + + monkeypatch.setattr(host, "_kernel", spy) + warmup_bf16_gemm_n100032_k6144(torch.device("cuda"), shuffled, mode) + monkeypatch.setattr(host, "_kernel", real) + assert len(keys) == len(host.WARMUP_ROWS) == len({host.tile_config(m) for m in range(1, 2049)}), sorted(keys) + assert {k[2] for k in keys} == {mode}, sorted(keys) + + +@pytest.mark.parametrize("shuffled", [False, True]) +def test_graph_no_compile_after_warmup(weights, shuffled, monkeypatch): + from flydsl.compiler.jit_function import MlirCompiler + warm_up_one_mode(monkeypatch, shuffled, "contiguous") + compiled = [] + original = MlirCompiler.compile + + def track(*args, **kwargs): + compiled.append(1) + return original(*args, **kwargs) + + monkeypatch.setattr(MlirCompiler, "compile", track) + w = weights[shuffled] + for m in (1, 17, 80, 96, 97, 128, 129, 161, 192, 193, 224, 225, 256, 257, 300, 384, 385, 513, 700): + x = torch.randn(m, K, device="cuda").to(torch.bfloat16) + y = torch.empty(m, N, device="cuda", dtype=torch.bfloat16) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + bf16_gemm_n100032_k6144(x, w, y, shuffled=shuffled) + for _ in range(2): + x.normal_() + y.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + check_numeric(y, bf16_gemm_n100032_k6144_reference(x, weights[False])) + assert compiled == [] + + + + +@pytest.mark.parametrize("shuffled", [False, True]) +@pytest.mark.parametrize("m", [1, 7, 17, 70, 80, 96, 97, 112, 128, 129, 161, 192, 193, 224, 225, 256, 257, 384, 385, + 448, 512, 513, 896, 1024, 1792, 2048]) +def test_publish_rows(weights, shuffled, m): + """One GPU: the row-indexed publish kernels into a private slot pair equal the contiguous GEMM of x[row_indices] + bit for bit (duplicates, the first / last source rows), and the last workgroup publishes slot 0.""" + x = torch.randn(m + 19, K, device="cuda").to(torch.bfloat16) + indices = torch.randint(0, len(x), (m,), device="cuda", dtype=torch.int64) + indices[0] = len(x) - 1 + if m > 1: + indices[1] = 0 + slot_bytes = m * N * 2 + pool = torch.full((2 * slot_bytes,), 0xFF, dtype=torch.uint8, device="cuda") + counter = torch.zeros(1, dtype=torch.int32, device="cuda") + flag = torch.zeros(16, dtype=torch.int32, device="cuda") + publish = (pool.data_ptr(), slot_bytes, counter, flag.data_ptr()) + assert bf16_gemm_n100032_k6144(x, weights[shuffled], shuffled=shuffled, row_indices=indices, publish=publish) is None + expected = bf16_gemm_n100032_k6144(x[indices], weights[shuffled], shuffled=shuffled) + torch.cuda.synchronize() + y = pool[:slot_bytes].view(torch.bfloat16).view(m, N) + assert torch.equal(y.view(torch.int16), expected.view(torch.int16)) + check_numeric(y, bf16_gemm_n100032_k6144_reference(x, weights[False], indices)) + assert int(counter[0]) == 1 and int(flag[0]) == 1 and int(flag[1]) == 0 + + +@pytest.mark.parametrize("mode", ["rows", "publish"]) +@pytest.mark.parametrize("shuffled", [False, True]) +def test_indexed_warm_up(shuffled, mode, monkeypatch): + warm_up_one_mode(monkeypatch, shuffled, mode) + + +@pytest.mark.parametrize("m", [1, 7, 64, 129, 192, 193, 256, 448, 1024, 2048]) +def test_rows_into_column_slices(weights, m): + """The mapped rows (and the contiguous call) written into either half of a [m, 200064] tensor equal the + contiguous GEMM of x[row_indices] bit for bit; the other half is untouched.""" + w = weights[True] + x = torch.randn(m + 5, K, device="cuda").to(torch.bfloat16) + indices = torch.randint(0, len(x), (m,), device="cuda", dtype=torch.int64) + expected = bf16_gemm_n100032_k6144(x[indices], w, shuffled=True).view(torch.int16) + for half in (0, 1): + full = torch.full((m, 2 * N), float("nan"), device="cuda", dtype=torch.bfloat16) + out = full[:, half * N:(half + 1) * N] + assert bf16_gemm_n100032_k6144(x, w, out, shuffled=True, row_indices=indices) is out + assert torch.equal(out.view(torch.int16), expected) + assert bool(torch.all(torch.isnan(full[:, (1 - half) * N:(2 - half) * N]))) + full.fill_(float("nan")) + bf16_gemm_n100032_k6144(x[indices], w, out, shuffled=True) + assert torch.equal(out.view(torch.int16), expected) + assert bool(torch.all(torch.isnan(full[:, (1 - half) * N:(2 - half) * N]))) + + +def test_supports(weights): + from amd_kernels.gemm.bf16_n100032_k6144 import supports + + w = weights[False] + assert supports(w) and supports(weights[True]) + assert not supports(w[:, :K - 64]) + assert not supports(w.T) + assert not supports(torch.empty(N, K, dtype=torch.float16)) + assert not supports(torch.empty(N - 16, K, dtype=torch.bfloat16)) + assert not supports(w, torch.zeros(N, dtype=torch.bfloat16, device=w.device)) diff --git a/tests/gemm/bf16_n100032_k6144/test_publish.py b/tests/gemm/bf16_n100032_k6144/test_publish.py new file mode 100644 index 0000000..eb1c481 --- /dev/null +++ b/tests/gemm/bf16_n100032_k6144/test_publish.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two spawned ranks: the lm_head GEMM's publish epilogue into all_to_all_bf16_n100032_ranks2's slot, then its pull. + +The received bytes must equal the contiguous GEMM of the mapped rows followed by an RCCL all_to_all of them, at the +target and draft row counts of every batch, dense and random owners, a rank owning 0 rows, all rows on one owner; +slots alternate through graph replays with new inputs, row maps and counts; no compile after the warm-up. +""" +import socket +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +N, K, WORLD = 100032, 6144, 2 + + +def _rccl_reference(y, counts, rank): + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2.reference import merge_received_reference + n = counts[rank] + recv = torch.empty(WORLD * n * N * 2, dtype=torch.uint8, device=y.device) + dist.all_to_all_single(recv, y.view(torch.uint8).flatten(), + output_split_sizes=[n * N * 2] * WORLD, + input_split_sizes=[c * N * 2 for c in counts]) + return merge_received_reference(recv.view(torch.bfloat16).view(WORLD, n, N)) + + +def _worker(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=WORLD, + timeout=timedelta(minutes=10), device_id=dev) + group = dist.new_group(backend="gloo") + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll + from amd_kernels.gemm.bf16_n100032_k6144 import (bf16_gemm_n100032_k6144, shuffle_weight, + warmup_bf16_gemm_n100032_k6144) + gen = torch.Generator(device=dev).manual_seed(4100 + rank) + w = shuffle_weight((torch.randn(N, K, device=dev, generator=gen) * 0.02).to(torch.bfloat16)) + comm = RowAllToAll(rank, group, dev) + comm.warmup() + warmup_bf16_gemm_n100032_k6144(dev, True, "contiguous") # the references (the same bits on x[row_indices]) + warmup_bf16_gemm_n100032_k6144(dev, True, "publish") + torch.cuda.synchronize() + dist.barrier() + # the same hidden rows on every rank (as the model's), with NaN / inf rows + xg = torch.Generator(device=dev).manual_seed(99) + x = (torch.randn(2048, K, device=dev, generator=xg)).to(torch.bfloat16) + x[5, 17] = float("nan") + x[9, 100] = float("inf") + patterns = [] + for reqs in (1, 3, 10, 16, 32, 64, 128, 256): + dense = torch.bincount(torch.arange(reqs) % WORLD, minlength=WORLD).tolist() + slots = torch.randperm(256, generator=torch.Generator().manual_seed(reqs))[:reqs] + random = torch.bincount(slots % WORLD, minlength=WORLD).tolist() + for requests in (dense, random): + for steps in (7, 8): + patterns.append([v * steps for v in requests]) + patterns += [[0, 24], [40, 0], [7, 1], [96, 97], [129, 0], [1024, 1024]] + from flydsl.compiler.jit_function import MlirCompiler + compile_calls = [] + original = MlirCompiler.compile + + def count_compile(*args, **kwargs): + compile_calls.append(1) + return original(*args, **kwargs) + + MlirCompiler.compile = count_compile + try: + for k, counts in enumerate(patterns): + m = sum(counts) + cs = torch.tensor(counts, dtype=torch.int32, device=dev) + n = counts[rank] + maps = [torch.randperm(2048, generator=torch.Generator().manual_seed(1000 * k + j))[:m].to(dev) + for j in range(3)] + outs = [torch.empty(n, WORLD * N, dtype=torch.bfloat16, device=dev) for _ in maps] + refs = [_rccl_reference(bf16_gemm_n100032_k6144(x[rm], w, shuffled=True), counts, rank) for rm in maps] + for rm, out, ref in zip(maps, outs, refs): + bf16_gemm_n100032_k6144(x, w, shuffled=True, row_indices=rm, publish=comm.slot()) + comm.pull(m, cs, out) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), ref.view(torch.int16)), (rank, k, counts, "eager") + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for rm, out in zip(maps, outs): + bf16_gemm_n100032_k6144(x, w, shuffled=True, row_indices=rm, publish=comm.slot()) + comm.pull(m, cs, out) + for repeat in range(2): + for out in outs: + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + for out, ref in zip(outs, refs): + assert torch.equal(out.view(torch.int16), ref.view(torch.int16)), (rank, k, counts, repeat) + del graph, outs, refs + # One captured call, new row maps, counts and hidden rows on every replay; the stage path in between + # (both producers share the slots and the counter). + m = 256 + rm = torch.arange(m, device=dev) + cs = torch.tensor([128] * WORLD, dtype=torch.int32, device=dev) + out = torch.empty(m, WORLD * N, dtype=torch.bfloat16, device=dev) + xs = torch.empty(m, N, dtype=torch.bfloat16, device=dev) + out2 = torch.empty_like(out) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + bf16_gemm_n100032_k6144(x, w, shuffled=True, row_indices=rm, publish=comm.slot()) + comm.pull(m, cs, out) + comm(xs, cs, out2) + for k in range(8): + counts = [m if j == k % WORLD else 0 for j in range(WORLD)] if k % 2 else [64, 192] + cs.copy_(torch.tensor(counts, dtype=torch.int32)) + rm.copy_(torch.randperm(2048, generator=torch.Generator().manual_seed(77 + k))[:m].to(dev)) + x[:64].copy_(torch.randn(64, K, device=dev, generator=xg).to(torch.bfloat16)) + xs.view(torch.int16).random_(-32768, 32768, generator=gen) + ref = _rccl_reference(bf16_gemm_n100032_k6144(x[rm], w, shuffled=True), counts, rank) + ref2 = _rccl_reference(xs.clone(), counts, rank) + graph.replay() + torch.cuda.synchronize() + n = counts[rank] + assert torch.equal(out[:n].view(torch.int16), ref.view(torch.int16)), (rank, "dynamic", k) + assert torch.equal(out2[:n].view(torch.int16), ref2.view(torch.int16)), (rank, "dynamic stage", k) + assert compile_calls == [], len(compile_calls) + finally: + MlirCompiler.compile = original + del graph + comm.close() + dist.barrier() + dist.destroy_process_group() + + +def test_two_rank_publish_pull(): + if torch.cuda.device_count() < WORLD: + pytest.skip("requires two gfx950 GPUs") + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + mp.spawn(_worker, args=(port,), nprocs=WORLD, join=True) diff --git a/tests/gemm/bf16_n200064_k256/test_gemm.py b/tests/gemm/bf16_n200064_k256/test_gemm.py index 7bebfb5..cca15dc 100644 --- a/tests/gemm/bf16_n200064_k256/test_gemm.py +++ b/tests/gemm/bf16_n200064_k256/test_gemm.py @@ -32,7 +32,7 @@ def check_numeric(y,ref): assert (yf-rf).norm()/rf.norm()<=1e-3 -@pytest.mark.parametrize("n",[0,1,2,3,4,8,15,16,17,24,31,32,33,48,63,64]) +@pytest.mark.parametrize("n",[0,1,2,3,4,8,15,16,17,24,31,32,33,48,63,64,65,80,96,127,128]) def test_reference(weight,n): x,idx,packed,y=operands(n) pack_bf16_rows(x,idx,packed) @@ -69,7 +69,7 @@ def test_graph_no_compile_after_warmup(weight,monkeypatch): def track(*args,**kwargs): compiled.append(1);return original(*args,**kwargs) monkeypatch.setattr(MlirCompiler,"compile",track) - for n in (1,3,16,17,31,32,33,48,64): + for n in (1,3,16,17,31,32,33,48,64,65,100,128): x,idx,p,y=operands(n) graph=torch.cuda.CUDAGraph() with torch.cuda.graph(graph): @@ -81,13 +81,24 @@ def track(*args,**kwargs): assert compiled==[] -def test_next_to_mfma(weight): +@pytest.mark.parametrize("n",[64,128]) +def test_next_to_mfma(weight,n): from tests.mfma_spinner import mfma_spinner spin=mfma_spinner(256);stream=torch.cuda.Stream(priority=-1) - x,idx,p,y=operands(64);pack_bf16_rows(x,idx,p) - bf16_gemm_packed_rows(p,weight,64,y);torch.cuda.synchronize();expected=y.clone() + x,idx,p,y=operands(n);pack_bf16_rows(x,idx,p) + bf16_gemm_packed_rows(p,weight,n,y);torch.cuda.synchronize();expected=y.clone() for _ in range(20): spin(100000,stream) - bf16_gemm_packed_rows(p,weight,64,y) + bf16_gemm_packed_rows(p,weight,n,y) torch.cuda.synchronize() assert torch.equal(y.view(torch.int16),expected.view(torch.int16)) + + +def test_halves_match_two_64_row_calls(weight): + """The 128-row tier equals the 64-row kernel run on each half, bit for bit.""" + x,idx,p,y=operands(128);pack_bf16_rows(x,idx,p) + bf16_gemm_packed_rows(p,weight,128,y) + for h in range(2): + q=torch.empty(64,K,device="cuda",dtype=torch.bfloat16);pack_bf16_rows(x,idx[h*64:(h+1)*64],q) + z=torch.empty(64,N,device="cuda",dtype=torch.bfloat16);bf16_gemm_packed_rows(q,weight,64,z) + assert torch.equal(y[h*64:(h+1)*64].view(torch.int16),z.view(torch.int16)),h diff --git a/tests/gemm/mxfp8_common/test_compile.py b/tests/gemm/mxfp8_common/test_compile.py index 59c4f83..ab2930e 100644 --- a/tests/gemm/mxfp8_common/test_compile.py +++ b/tests/gemm/mxfp8_common/test_compile.py @@ -31,6 +31,11 @@ "staged": range(769, 1025), "fourwave": range(769, 1281), "fourwave_nt": range(1281, 16385), + "decode_fq_n12288": range(5, 9), + "decode_cm0_n4608": range(289, 385), + "decode_k1536": range(449, 513), + "staged_n3072": range(897, 1025), + "fourwave_nt_n12288": range(769, 16385), } PROBES = 6 @@ -46,7 +51,7 @@ def variant(name, m, n, k, config): """The part of a launch that picks a compiled variant.""" if config.kernel == "decode_fq": return None # the kernel quantizes the bf16 activation itself - if config.kernel == "decode": + if config.kernel in ("decode", "decode_splitk"): return quantizer(-(-m // config.tile_m) * config.tile_m, shuffled=True) quant = quantizer(m, shuffled=False) if config.kernel in ("4wave", "4wave_nt"): @@ -94,8 +99,8 @@ def test_no_compile_after_warmup(name, m, n, k, config, monkeypatch): ) -# inside a DBO microbatch (TP4: prefill steps of >= 8192 tokens, so >= 4096 rows per -# microbatch) the 4-wave rows launch the plain grid, other kernels than outside one +# inside a DBO microbatch (prefill steps from the DBO threshold on, e.g. >= 8192 tokens: >= 4096 +# rows per microbatch) the 4-wave rows launch the plain grid, other kernels than outside one DBO_RANGES = {"fourwave_nt": range(4096, 16385)} diff --git a/tests/gemm/mxfp8_common/test_ctx_kv_rows_n2560.py b/tests/gemm/mxfp8_common/test_ctx_kv_rows_n2560.py new file mode 100644 index 0000000..525840b --- /dev/null +++ b/tests/gemm/mxfp8_common/test_ctx_kv_rows_n2560.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""N 2560 x K 6144 serves two layers: the QKV projection (N 2560 = 16 q + 1 k + 1 v + 2 index heads) and the draft +context K / V projection (N 2560 = 5 layers x K, V x 2 kv heads x 128). The QKV-only hooks of its shape host +(``select_prequant`` at 9-16 rows, ``select_deferred`` for the norm / RoPE / insert chain) must hold for the context +K / V too: the producer's code tile is the one the GEMM reads, the codes entry and the deferred split-K partials meet +the fp64 contract per output row, and a deferred row reads a prefix of the producer's codes.""" +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.reference import mxfp8_gemm_reference, quantize_mxfp8 + +from .test_gemm import require_gfx950 +from .test_shared_expert_rows import _spread + +N, K = 2560, 6144 +MS = [1, 7, 8, 9, 14, 16, 17, 56, 64, 80, 112, 128, 144, 192, 224, 256, 288, 416, 512, 1024, 2048, 3072, 4096] + + +@pytest.fixture(scope="module") +def weights(): + gen = torch.Generator(device="cuda").manual_seed(N + K) + q, s = quantize_mxfp8((_spread((N, K), gen) * 0.02).bfloat16()) + return q, s, *G.preshuffle_mxfp8_weight(q, s) + + +def _check(out, exact, tag): + lo = 1 if out.shape[0] > 1 else 0 # row 0 is the zero row + got = out[lo:].double() + cos = torch.nn.functional.cosine_similarity(got, exact[lo:], dim=1) + rel = (got - exact[lo:]).norm(dim=1) / exact[lo:].norm(dim=1) + assert cos.min() >= 0.99999 and rel.max() <= 1e-3, (tag, float(cos.min()), float(rel.max())) + + +def test_code_tile_is_the_rows_tile(): + for m in range(1, 4097): + tile = G.prequant_code_tile_rows(m, N, K) + cfg = G._select_prequant_config(m, N, K, torch.bfloat16) + if cfg.kernel in ("decode", "decode_fq"): + assert tile == max(16, cfg.tile_m), m + deferred = G._deferred_qkv_config(m, N, K, cfg) + rows = -(-m // deferred.tile_m) * deferred.tile_m + assert rows <= -(-m // tile) * tile, m # a deferred row reads a prefix of the producer's codes + else: + assert tile == 0, m + + +@pytest.mark.parametrize("m", MS) +@torch.inference_mode() +def test_codes_and_deferred_entries(weights, m): + require_gfx950() + qw, sw, w, ws = weights + gen = torch.Generator(device="cuda").manual_seed(m) + x = _spread((m, K), gen).bfloat16() + if m > 1: + x[0].zero_() + exact = mxfp8_gemm_reference(*quantize_mxfp8(x), qw, sw, accumulate=torch.float64).double() + linear = G.mxfp8_linear(x, w, ws, torch.bfloat16) + _check(linear, exact, ("linear", m)) + tile = G.prequant_code_tile_rows(m, N, K) + if not tile: + a, s = G.mxfp8_e4m3_quantize_aiter(x) + _check(G.mxfp8_gemm_prequant(a, s, w, ws, torch.bfloat16, m), exact, ("codes", m)) + return + q, s = G.mxfp8_e4m3_quantize_shuffled(x, tile) + codes = G.mxfp8_gemm_prequant(q, s, w, ws, torch.bfloat16, m) + _check(codes, exact, ("codes", m)) + deferred = G.mxfp8_gemm_shuffled(q, s, w, ws, torch.bfloat16, m, defer_reduce=True) + if isinstance(deferred, G.SkPartials): + deferred = deferred.materialize() + _check(deferred, exact, ("deferred", m)) + if m > 1: + assert not codes[0].any() and not deferred[0].any() diff --git a/tests/gemm/mxfp8_common/test_decode_splitk.py b/tests/gemm/mxfp8_common/test_decode_splitk.py new file mode 100644 index 0000000..38bdab8 --- /dev/null +++ b/tests/gemm/mxfp8_common/test_decode_splitk.py @@ -0,0 +1,339 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The split-K decode kernel (``mxfp8_gemm_decode_splitk``) behind the ``decode_splitk`` rows: every row the +dispatcher serves fits one resident grid at every row count of its window, keeps its MFMA operands in VGPRs (final +ISA: no spill, no ``v_accvgpr_write``), meets the fp64 contract at its window edges through the bf16 and codes +entries (same bytes, bitwise repeats, graph replay) and compiles nothing after the warm-up; K split 3 gives the bytes +of the decode kernel's three K-waves on order-sensitive data (the QKV kw3 rows, so the QKV chain keeps its bytes); +the workgroups of one b % 8 run on one XCD (the partials stay in that XCD's L2), alone and next to a concurrent +kernel; calls on two streams at once and next to a concurrent kernel give the isolated bytes; the epoch counters stay +multiples of ``EPOCH``.""" + +import glob +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common import mxfp8_gemm_decode_splitk as SK +from amd_kernels.gemm.mxfp8_common.reference import mxfp8_gemm_reference, quantize_mxfp8 + +from .test_gemm import require_gfx950 + +BF16 = torch.bfloat16 +MAX_M = 512 + + +def _shapes(): + hosts = set(G._SHAPE_HOSTS) + return sorted(set(G.tuned_shapes(BF16)) | hosts) + + +def _rows(): + """{(n, k, config): [first M, last M]} of every split-K row the bf16 / codes entries select.""" + rows = {} + for n, k in _shapes(): + for m in range(1, MAX_M + 1): + for config in (G._select_config(m, n, k, BF16), G._select_prequant_config(m, n, k, BF16)): + if config.kernel == G._DECODE_SPLITK_KERNEL: + lo, hi = rows.setdefault((n, k, config), [m, m]) + rows[(n, k, config)] = [min(lo, m), max(hi, m)] + return rows + + +ROWS = _rows() +SHAPES = sorted({(n, k) for n, k, _ in ROWS}) + + +def test_rows_fit_one_resident_grid(): + assert {(4992, 6144), (6144, 4096), (4608, 6144), (12288, 6144), (6144, 6144), (3072, 6144)} <= set(SHAPES) + for (n, k, config), (lo, hi) in ROWS.items(): + assert G._splitk_config_ok(n, k, config), config + for m in range(lo, hi + 1): + grid, n_mb, counters, _ = SK.geometry(n, m, config.tile_m // 16, config.tile_n // (16 * config.waves_m), + config.waves_m, config.split_k) + assert grid <= SK.GRID_MAX and n_mb == -(-m // config.tile_m), (n, k, m, config) + assert counters <= SK.GRID_MAX // 2 * 8 * SK.CNT_STRIDE + assert G.prequant_code_tile_rows(m, n, k) == config.tile_m + + +def _launchers(shape): + return [(cfg, G._splitk_launcher(n, k, cfg)) for (n, k, cfg) in sorted(ROWS, key=str) if (n, k) == shape] + + +@pytest.mark.parametrize("shape", SHAPES, ids=[f"n{n}_k{k}" for n, k in SHAPES]) +def test_rows_hold_operands_in_vgprs(shape, tmp_path): + """Compiled in a subprocess (compile-only, cache off, ISA dumped).""" + require_gfx950() + root = Path(__file__).resolve().parents[3] + env = dict(os.environ, AKR_COMPILE_ONLY="1", FLYDSL_DEBUG_DUMP_ASM="1", FLYDSL_DUMP_DIR=str(tmp_path), + FLYDSL_RUNTIME_ENABLE_CACHE="0") + subprocess.run([sys.executable, "-m", "tests.gemm.mxfp8_common.test_decode_splitk", *map(str, shape)], cwd=root, + env=env, check=True) + names = [exe.kernel_name for _, exe in _launchers(shape)] + assert names + for name in names: + isa = glob.glob(f"{tmp_path}/{name}/*.s") + assert isa, name + text = Path(isa[0]).read_text() + assert "v_accvgpr_write" not in text, name + assert "scratch_" not in text and ".vgpr_spill_count: 0" in text, name + + +def _spread(shape, gen): + """normal values with per-32 scales spread over +-5 binades""" + rows, cols = shape + e = torch.randint(-5, 6, (rows, cols // 32, 1), generator=gen, device="cuda") + x = torch.randn(shape, generator=gen, device="cuda") + return x * torch.exp2(e.expand(rows, cols // 32, 32).reshape(rows, cols).float()) + + +_WEIGHTS = {} + + +def _spread_weight(n, k): + if (n, k) not in _WEIGHTS: + gen = torch.Generator(device="cuda").manual_seed(n + k) + q, s = quantize_mxfp8((_spread((n, k), gen) * 0.02).bfloat16()) + _WEIGHTS[(n, k)] = (q, s, *G.preshuffle_mxfp8_weight(q, s)) + return _WEIGHTS[(n, k)] + + +EDGES = sorted({(n, k, m) for (n, k, _), (lo, hi) in ROWS.items() for m in (lo, hi)}) + + +@pytest.mark.parametrize("n,k,m", EDGES, ids=[f"n{n}_k{k}_m{m}" for n, k, m in EDGES]) +@torch.inference_mode() +def test_rows_contract(n, k, m): + """Each row at its window edges: the fp64 contract per output row (a zero row), the bf16 and codes entries' same + bytes, bitwise repeats, graph replay over a NaN-poisoned output and changed inputs.""" + require_gfx950() + qw, sw, w, ws = _spread_weight(n, k) + gen = torch.Generator(device="cuda").manual_seed(m + n) + x = _spread((m, k), gen).bfloat16() + x[m // 2].zero_() + out = G.mxfp8_linear(x, w, ws, BF16) + exact = mxfp8_gemm_reference(*quantize_mxfp8(x), qw, sw, accumulate=torch.float64).double() + live = exact.norm(dim=1) > 0 + got = out.double() + cos = torch.nn.functional.cosine_similarity(got[live], exact[live], dim=1) + rel = (got[live] - exact[live]).norm(dim=1) / exact[live].norm(dim=1) + assert cos.min() >= 0.99999 and rel.max() <= 1e-3 and not out[m // 2].any(), (float(cos.min()), float(rel.max())) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, G.prequant_code_tile_rows(m, n, k)) + call = lambda: G.mxfp8_gemm_prequant(q, s, w, ws, BF16, m) # noqa: E731 + for _ in range(3): + assert torch.equal(call().view(torch.int16), out.view(torch.int16)) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + captured = call() + captured.fill_(float("nan")) + g.replay() + torch.cuda.synchronize() + assert torch.equal(captured.view(torch.int16), out.view(torch.int16)) + q.copy_(G.mxfp8_e4m3_quantize_shuffled(_spread((m, k), gen).bfloat16(), q.shape[0])[0]) + g.replay() + eager = call() + torch.cuda.synchronize() + assert torch.equal(captured.view(torch.int16), eager.view(torch.int16)) + + +@pytest.mark.parametrize("shape", SHAPES, ids=[f"n{n}_k{k}" for n, k in SHAPES]) +@torch.inference_mode() +def test_no_compile_after_warmup(shape, monkeypatch): + """After ``warmup_mxfp8_gemm_weight`` every row count of the split-K windows compiles nothing (bf16 and codes).""" + from flydsl.compiler.jit_function import MlirCompiler + + require_gfx950() + n, k = shape + _, _, w, ws = _spread_weight(n, k) + G.warmup_mxfp8_gemm_weight(w, ws, MAX_M, BF16) + torch.cuda.synchronize() + compiled = [] + real = MlirCompiler.compile + + def counting(self, *a, **kw): + compiled.append(1) + return real(self, *a, **kw) + + monkeypatch.setattr(MlirCompiler, "compile", counting) + x = torch.zeros((MAX_M, k), device="cuda", dtype=BF16) + ms = sorted({m for (n_, k_, _), (lo, hi) in ROWS.items() if (n_, k_) == shape for m in range(lo, hi + 1)}) + for m in ms: + G.mxfp8_linear(x[:m], w, ws, BF16) + q, s = G.mxfp8_e4m3_quantize_shuffled(x[:m], G.prequant_code_tile_rows(m, n, k)) + G.mxfp8_gemm_prequant(q, s, w, ws, BF16, m) + torch.cuda.synchronize() + assert not compiled, len(compiled) + + +def _wide(shape, gen): + """order-sensitive data: per-32 scales over +-12 binades (fp32 sums depend on the K order)""" + rows, cols = shape + e = torch.randint(-12, 13, (rows, cols // 32, 1), generator=gen, device="cuda") + x = torch.randn(shape, generator=gen, device="cuda") + return (x * torch.exp2(e.expand(rows, cols // 32, 32).reshape(rows, cols).float())).bfloat16() + + +def _weight(n, k, seed): + gen = torch.Generator(device="cuda").manual_seed(seed) + q, s = quantize_mxfp8(_wide((n, k), gen) * 0.01) + return q, s, *G.preshuffle_mxfp8_weight(q, s) + + +@pytest.mark.parametrize("m", [65, 96, 112, 113, 176, 256]) +@torch.inference_mode() +def test_split3_is_the_kw3_decode_order(m): + require_gfx950() + n, k = 4992, 6144 + qw, sw, w, ws = _weight(n, k, 7) + gen = torch.Generator(device="cuda").manual_seed(m) + x = _wide((m, k), gen) + row = G._select_prequant_config(m, n, k, BF16) + assert row.kernel == G._DECODE_SPLITK_KERNEL and row.split_k == 3 + q, s = G.mxfp8_e4m3_quantize_shuffled(x, row.tile_m) + got = G.mxfp8_gemm_shuffled(q, s, w, ws, BF16, m) + kw3 = G._Config(48, 48, 128, 0, 0, 1, G._DECODE_KERNEL, 3, 0) + q3, s3 = G.mxfp8_e4m3_quantize_shuffled(x, 48) + ref = G.mxfp8_gemm_shuffled(q3, s3, w, ws, BF16, m, config=kw3) + assert torch.equal(got.view(torch.int16), ref.view(torch.int16)) + exact = mxfp8_gemm_reference(*quantize_mxfp8(x), qw, sw, accumulate=torch.float64).double() + assert not torch.equal(exact.bfloat16().view(torch.int16), got.view(torch.int16)) # the data sees the K order + + +def _probe_launcher(): + import flydsl.compiler as flyc + import flydsl.expr as fx + from flydsl._mlir.dialects import llvm + from flydsl.expr.typing import T + + from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import _addr + + @fx.struct + class Shared: + pad: fx.Array[fx.Uint8, 150 * 1024, 16] + + @flyc.kernel(name="xcc_probe_512", known_block_size=[512, 1, 1]) + def kernel(arg_out: fx.Int64): + smem = fx.SharedAllocator().allocate(Shared).peek() + tx, bx = fx.thread_idx.x, fx.block_idx.x + xcc = fx.Int32(llvm.inline_asm(T.i32, [], "s_getreg_b32 $0, hwreg(HW_REG_XCC_ID)", "=s", + has_side_effects=True)) + if tx == 0: + smem.pad.ptr[0] = fx.Uint8(1) + p = fx.inttoptr(fx.PointerType.get(T.i32, address_space=fx.AddressSpace.Global, alignment=4), + arg_out + fx.Int64(bx * 4)) + p[0] = xcc + + @flyc.jit + def launch(OUT: fx.Tensor, grid: fx.Int32, stream: fx.Stream): + kernel(_addr(OUT)).launch(grid=(fx.Int64(grid), 1, 1), block=(512, 1, 1), stream=stream) + + return launch + + +def _background(stream, n=4): + a = torch.randn(4096, 4096, device="cuda", dtype=BF16) + small = torch.randn(333, 517, device="cuda", dtype=BF16) + + def run(): + with torch.cuda.stream(stream): + for _ in range(n): + a @ a + for _ in range(8): + small @ small.T + return run + + +@torch.inference_mode() +def test_one_xcd_per_residue(): + """XCC_ID of every workgroup of a 256 x 512-thread grid (one per CU, the kernel's shape): the workgroups of one + b % 8 share an XCD in every launch, alone and next to a concurrent stream.""" + from amd_kernels.common.tensor_shim import _run_compiled + + require_gfx950() + exe = _probe_launcher() + side = torch.cuda.Stream() + busy = _background(side) + outs = [] + for i in range(400): + if i % 20 == 0: + busy() + out = torch.full((256,), -1, dtype=torch.int32, device="cuda") + _run_compiled(exe, out, 256, torch.cuda.current_stream()) + outs.append(out) + torch.cuda.synchronize() + b = torch.arange(256, device="cuda", dtype=torch.int32) + for out in outs: + assert (out >= 0).all() + groups = out.view(32, 8) # [b // 8, b % 8] + assert (groups == groups[:1]).all(), out.tolist() + assert len(set((out - b).remainder(8).tolist())) == 1 + + +def _calls(n, k, m, seed): + _, _, w, ws = _weight(n, k, seed) + gen = torch.Generator(device="cuda").manual_seed(seed + m) + x = _wide((m, k), gen) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, G.prequant_code_tile_rows(m, n, k)) + return lambda: G.mxfp8_gemm_shuffled(q, s, w, ws, BF16, m) + + +CASES = [(4992, 6144, 96), (4992, 6144, 176), (6144, 4096, 96), (3072, 6144, 96), (12288, 6144, 160)] + + +@torch.inference_mode() +def test_two_streams_and_a_concurrent_kernel(): + require_gfx950() + calls = [_calls(n, k, m, i) for i, (n, k, m) in enumerate(CASES)] + ref = [c().clone() for c in calls] + torch.cuda.synchronize() + streams = [torch.cuda.Stream(), torch.cuda.Stream()] + side = torch.cuda.Stream() + busy = _background(side, 2) + got = [] + for it in range(60): + busy() + for j, st in enumerate(streams): + with torch.cuda.stream(st): + for i, c in enumerate(calls): + if (i + j + it) % 2 == 0: + got.append((i, c().clone())) + torch.cuda.synchronize() + assert len(got) >= 300 + for i, out in got: + assert torch.equal(out.view(torch.int16), ref[i].view(torch.int16)), CASES[i] + for (_, stream), cnt in G._SPLITK_COUNTERS.items(): + assert (cnt % SK.EPOCH == 0).all(), stream + + +@torch.inference_mode() +def test_graph_replay_and_counters(): + require_gfx950() + calls = [_calls(n, k, m, 10 + i) for i, (n, k, m) in enumerate(CASES)] + ref = [c().clone() for c in calls] + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + outs = [c() for c in calls for _ in range(3)][2::3] + for o in outs: + o.fill_(float("nan")) + for _ in range(5): + g.replay() + torch.cuda.synchronize() + for o, r in zip(outs, ref): + assert torch.equal(o.view(torch.int16), r.view(torch.int16)) + for cnt in G._SPLITK_COUNTERS.values(): + assert (cnt % SK.EPOCH == 0).all() and cnt.any() + + +if __name__ == "__main__": + from amd_kernels.common.tensor_shim import _run_compiled + + shape = (int(sys.argv[1]), int(sys.argv[2])) + d = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") + for config, exe in _launchers(shape): + _run_compiled(exe, d, d, d, d, d, d, d, config.tile_m, 1024, 1, 8 * config.split_k, + torch.cuda.current_stream()) diff --git a/tests/gemm/mxfp8_common/test_dispatch.py b/tests/gemm/mxfp8_common/test_dispatch.py index 9750b0e..6f55242 100644 --- a/tests/gemm/mxfp8_common/test_dispatch.py +++ b/tests/gemm/mxfp8_common/test_dispatch.py @@ -88,7 +88,7 @@ def test_tp4_prefill_rows(): """The TP4 rows (graph replay A/B on layer 39 rank 0's weights, bitwise-equal outputs): QKV (2560, 6144) at 385-1536 (but 513-576) decode rows with 80-column tiles (eight row tiles x 32 column tiles = one workgroup per CU); 1537-2048 uses the rectangular - 128x160 tile, 2049-3072 4-wave NT 128x256, 3073-10240 4-wave NT 256x256; o_proj + 128x160 tile, 2049-3072 4-wave NT 256x128, 3073-10240 4-wave NT 256x256; o_proj (6144, 2048) the rectangular 128x192 tile at 769-1024, 4-wave NT 256x128 at 1025-1280, 4-wave NT 256x256 at 1281-3328, the persistent 256x192 tile at 3329-3968, the AITER row kept at 3969-4096 (no stable eager gain there), 4-wave NT 256x256 at 4097-12288; o_proj_full (6144, 8192, the SP o_proj) the @@ -122,8 +122,8 @@ def tile(m, n, k, allow_decode=True): for m in (1537, 1600, 2048): assert tile(m, 2560, 6144) == ("rect_n2560_k6144", 128, 160), m assert g.prequant_code_tile_rows(m, 2560, 6144, BF16) == 0 - assert tile(2049, 2560, 6144) == ("4wave_nt", 128, 256) - assert tile(3072, 2560, 6144) == ("4wave_nt", 128, 256) + assert tile(2049, 2560, 6144) == ("4wave_nt", 256, 128) + assert tile(3072, 2560, 6144) == ("4wave_nt", 256, 128) for m in (3073, 3584, 4095, 4096, 4097, 4608, 4799, 4800): assert tile(m, 2560, 6144) == ("prefill_n2560_k6144_192x256", 192, 256), m assert tile(m, 2560, 6144, False) == ("prefill_n2560_k6144_192x256", 192, 256), m diff --git a/tests/gemm/mxfp8_common/test_ffn_qkv_decode_rows.py b/tests/gemm/mxfp8_common/test_ffn_qkv_decode_rows.py index 97817be..2c679d1 100644 --- a/tests/gemm/mxfp8_common/test_ffn_qkv_decode_rows.py +++ b/tests/gemm/mxfp8_common/test_ffn_qkv_decode_rows.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""TP4 decode rows of N 2304 x K 6144 (QKV), N 6144 x K 6144 (gate_up) and N 6144 x K 3072 (down): QKV 97-112 -(32x32, K-waves 2, cached W), 193-224 (80x32), 385-448 (cached W), gate_up 193-224 (112x48), down 65-80 (48x48) and -193-224 (112x48); each keeps the K-wave count of a baseline tile (``ROWS``' last column), so its K order is the +"""TP4 decode rows of N 2304 x K 6144 (QKV) and N 6144 x K 3072 (down): QKV 97-112 (32x32, K-waves 2, cached W), +193-224 (80x32), 385-448 (cached W), down 65-80 (48x48) and 193-224 (112x48); each keeps the K-wave count of a baseline tile (``ROWS``' last column), so its K order is the baseline's: bitwise equal to the baseline tile, the fp64 contract of reference.py, graph replay, the bf16 entry, no compile after the warm-up, MFMA co-residency. Neighbouring rows are pinned.""" import dataclasses @@ -18,7 +17,6 @@ (2304, 6144, 97, 112, (32, 32, 2, 0), (32, 48, None)), (2304, 6144, 193, 224, (80, 32, 3, None), (64, 48, None)), (2304, 6144, 385, 448, (64, 64, 3, 0), (64, 64, None)), - (6144, 6144, 193, 224, (112, 48, 3, None), (128, 48, None)), (6144, 3072, 65, 80, (48, 48, 3, None), (64, 48, None)), (6144, 3072, 193, 224, (112, 48, 3, None), (128, 48, None)), ] @@ -27,7 +25,6 @@ (2304, 6144, 96, (32, 32, 1, None)), (2304, 6144, 113, (32, 48, 2, None)), (2304, 6144, 160, (32, 48, 2, None)), (2304, 6144, 192, (64, 32, 4, None)), (2304, 6144, 225, (64, 48, 3, None)), (2304, 6144, 320, (64, 48, 3, None)), (2304, 6144, 384, (128, 32, 3, None)), (2304, 6144, 449, (128, 48, 3, None)), - (6144, 6144, 192, (96, 48, 4, None)), (6144, 6144, 225, (128, 48, 3, None)), (6144, 6144, 256, (128, 48, 3, None)), (6144, 3072, 64, (64, 32, 1, None)), (6144, 3072, 81, (64, 48, 3, None)), (6144, 3072, 128, (64, 48, 3, None)), (6144, 3072, 192, (96, 48, 3, None)), (6144, 3072, 225, (128, 48, 3, None)), (6144, 3072, 256, (128, 48, 3, None)), ] diff --git a/tests/gemm/mxfp8_common/test_gateup_decode_rows.py b/tests/gemm/mxfp8_common/test_gateup_decode_rows.py deleted file mode 100644 index 61c6ec0..0000000 --- a/tests/gemm/mxfp8_common/test_gateup_decode_rows.py +++ /dev/null @@ -1,96 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""The 65-80-row gate/up tile: same K order, 48-row producer padding, unchanged neighbouring rows.""" -import dataclasses -import pytest -import torch -from amd_kernels.gemm.mxfp8_common import gemm as G -from amd_kernels.gemm.mxfp8_common.reference import quantize_mxfp8,mxfp8_gemm_reference - - -def test_dispatch_edges(): - for m in (65,69,70,79,80): - cfg=G._select_config(m,6144,6144) - assert (cfg.kernel,cfg.tile_m,cfg.tile_n,cfg.waves_m,cfg.split_k)==('decode',48,48,3,1) - assert G.prequant_code_tile_rows(m,6144,6144)==48 - for m,expected in ((64,(64,32,3)),(81,(96,32,3)),(96,(96,32,3)),(97,(64,48,3))): - cfg=G._select_config(m,6144,6144) - assert (cfg.tile_m,cfg.tile_n,cfg.waves_m)==expected - - -@pytest.fixture(scope='module') -def weights(): - gen=torch.Generator(device='cuda').manual_seed(777) - w=torch.randn((6144,6144),generator=gen,device='cuda')*.02 - w*=torch.exp2(torch.randint(-5,6,(6144,192,1),generator=gen,device='cuda').expand(6144,192,32).reshape(6144,6144).float()) - q,s=quantize_mxfp8(w.bfloat16()) - return q,s,G.preshuffle_mxfp8_weight(q,s) - - -@pytest.mark.parametrize('m',(65,69,70,79,80)) -def test_old_order_and_fp64(weights,m): - qw,sw,(w,ws)=weights - gen=torch.Generator(device='cuda').manual_seed(m) - x=torch.randn((m,6144),generator=gen,device='cuda') - x*=torch.exp2(torch.randint(-5,6,(m,192,1),generator=gen,device='cuda').expand(m,192,32).reshape(m,6144).float()) - x=x.bfloat16();x[0].zero_() - q,s=G.mxfp8_e4m3_quantize_shuffled(x,48) - cfg=G._select_config(m,6144,6144) - old=dataclasses.replace(cfg,tile_m=96,tile_n=32) - out=torch.empty((m,6144),device='cuda',dtype=torch.bfloat16);ref=torch.empty_like(out) - G._launch_decode(old,q,s,w,ws,ref) - G._launch_decode(cfg,q,s,w,ws,out) - torch.cuda.synchronize() - assert torch.equal(out,ref) - qx,sx=quantize_mxfp8(x) - exact=mxfp8_gemm_reference(qx,sx,qw,sw) - cos=torch.nn.functional.cosine_similarity(out[1:].double(),exact[1:].double(),dim=1) - rel=(out.double()-exact.double()).norm()/exact.double().norm() - assert cos.min()>=.99999 and rel<=1e-3,(cos.min(),rel) - per_row_rel=(out[1:].double()-exact[1:].double()).norm(dim=1)/exact[1:].double().norm(dim=1) - assert per_row_rel.max()<=1e-3,per_row_rel.max() - gr=torch.cuda.CUDAGraph() - with torch.cuda.graph(gr):G._launch_decode(cfg,q,s,w,ws,out) - out.fill_(float('nan'));gr.replay();torch.cuda.synchronize() - assert torch.equal(out,ref) - - -def test_warmup(weights): - from flydsl.compiler.jit_function import MlirCompiler - _,_,(w,ws)=weights - x=torch.randn((80,6144),device='cuda',dtype=torch.bfloat16) - q,s=G.mxfp8_e4m3_quantize_shuffled(x,48) - out=torch.empty((80,6144),device='cuda',dtype=torch.bfloat16) - cfg=G._select_config(80,6144,6144) - G._launch_decode(cfg,q,s,w,ws,out);torch.cuda.synchronize() - orig=MlirCompiler.compile - def unexpected(*a,**kw):raise AssertionError('compile after warm-up') - MlirCompiler.compile=unexpected - try: - for m in (65,69,70,79,80): - o=torch.empty((m,6144),device='cuda',dtype=torch.bfloat16) - G._launch_decode(G._select_config(m,6144,6144),q,s,w,ws,o) - torch.cuda.synchronize() - finally:MlirCompiler.compile=orig - - -@pytest.mark.parametrize('m', (70, 80)) -def test_mfma_co_residency(weights, m): - from tests.mfma_spinner import mfma_spinner - _, _, (w, ws) = weights - x = torch.randn((m, 6144), device='cuda', dtype=torch.bfloat16) - tile = G.prequant_code_tile_rows(m, 6144, 6144) - q, sc = G.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else G.mxfp8_e4m3_quantize_aiter(x) - def call(): - if tile: - return G.mxfp8_gemm_shuffled(q, sc, w, ws, torch.bfloat16, m) - return G.mxfp8_gemm(q[:m], sc, w, ws, torch.bfloat16, activation_scale_preshuffled=True) - ref = call() - torch.cuda.synchronize() - spinner = mfma_spinner(256) - side = torch.cuda.Stream(priority=-1) - for _ in range(3): - spinner(100_000, side) - got = call() - torch.cuda.synchronize() - assert torch.equal(got.view(torch.int16), ref.view(torch.int16)) diff --git a/tests/gemm/mxfp8_common/test_gemm.py b/tests/gemm/mxfp8_common/test_gemm.py index 0e34af4..1c4f1e4 100644 --- a/tests/gemm/mxfp8_common/test_gemm.py +++ b/tests/gemm/mxfp8_common/test_gemm.py @@ -32,6 +32,12 @@ ("staged", 1000, 2560, 6144, _Config(128, 256, 128, 2, 4, 3, "staged_ab", 2, 4)), ("fourwave", 1200, 6144, 3072, _Config(256, 128, 128, 1, 1, 1, "4wave", 2, 2)), ("fourwave_nt", 5000, 6144, 3072, _Config(256, 256, 128, 1, 1, 1, "4wave_nt", 2, 2)), + # rows of the 12288 / 4608 / 3072 x 6144 and 6144 x 1536 shapes + ("decode_fq_n12288", 7, 12288, 6144, _Config(8, 48, 128, 0, 0, 1, "decode_fq", 4, 0)), + ("decode_cm0_n4608", 300, 4608, 6144, _Config(64, 64, 128, 0, 0, 1, "decode", 1, 0, w_cm=0)), + ("decode_k1536", 500, 6144, 1536, _Config(128, 96, 128, 0, 0, 1, "decode", 2, 0)), + ("staged_n3072", 1000, 3072, 6144, _Config(128, 256, 128, 2, 4, 2, "staged_ab", 2, 4)), + ("fourwave_nt_n12288", 4096, 12288, 6144, _Config(256, 256, 128, 1, 1, 1, "4wave_nt", 2, 2)), ] IDS = [case[0] for case in CASES] diff --git a/tests/gemm/mxfp8_common/test_oproj_out_k4096.py b/tests/gemm/mxfp8_common/test_oproj_out_k4096.py new file mode 100644 index 0000000..c43ba0d --- /dev/null +++ b/tests/gemm/mxfp8_common/test_oproj_out_k4096.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Caller-owned output arena (``out=``) of the o_proj N 6144 x K 4096: arena views at decode and prefill row counts, +the same bytes as a fresh output, graph replay, and rejection before any write (other shapes, bad arenas).""" +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as g + +from .test_gemm import make_inputs, require_gfx950 + +N, K = 6144, 4096 + + +@pytest.mark.parametrize("m", [1, 8, 64, 200, 512, 1024, 1025, 2048, 4095]) +@torch.inference_mode() +def test_oproj_out(m): + require_gfx950() + x, _, _, (w, ws) = make_inputs(m, N, K, seed=m) + a, s = g.mxfp8_e4m3_quantize_aiter(x) + arena = torch.full((m + 2, N), -13, device="cuda", dtype=torch.bfloat16) + out = arena[1:-1] + for run in ( + lambda dst: g.mxfp8_linear(x, w, ws, out=dst), + lambda dst: g.mxfp8_gemm(a, s, w, ws, torch.bfloat16, activation_scale_preshuffled=True, out=dst), + ): + expected = run(None) + for _ in range(2): + out.fill_(float("nan")) + assert run(out) is out + assert torch.equal(out, expected) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + got = run(out) + assert got is out + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, expected) + assert (arena[0] == -13).all() and (arena[-1] == -13).all() + + +@pytest.mark.parametrize("bad", ["shape", "dtype", "stride", "overlap", "other_shape"]) +@torch.inference_mode() +def test_out_rejected_before_write(bad): + require_gfx950() + m, n = 256, N + if bad == "other_shape": + n = 4608 + x, _, _, (w, ws) = make_inputs(m, n, K if bad != "other_shape" else 6144) + a, s = g.mxfp8_e4m3_quantize_aiter(x) + out = torch.full((m, n), -13, device="cuda", dtype=torch.bfloat16) + if bad == "shape": + out = out[: m - 1] + if bad == "dtype": + out = out.float() + if bad == "stride": + out = out.t().contiguous().t() + if bad == "overlap": + a = out.view(torch.float8_e4m3fn).view(-1)[: m * K].view(m, K) + with pytest.raises(ValueError, match="out="): + g.mxfp8_gemm(a, s, w, ws, torch.bfloat16, activation_scale_preshuffled=True, out=out) + assert (out == -13).all() + if bad != "overlap": + with pytest.raises(ValueError, match="out="): + g.mxfp8_linear(x, w, ws, out=out) + assert (out == -13).all() diff --git a/tests/gemm/mxfp8_common/test_shape_rows.py b/tests/gemm/mxfp8_common/test_shape_rows.py new file mode 100644 index 0000000..250eeab --- /dev/null +++ b/tests/gemm/mxfp8_common/test_shape_rows.py @@ -0,0 +1,285 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The bf16 rows of shared-expert gate_up N 3072 x K 6144 and down 6144 x 1536, draft QKV 4608 x 6144 and dense / +draft gate_up 12288 x 6144 (TP2 shards, the split-K decode rows included): the pinned row per M window over 1-65536 and the producer code tile; each +decode row against the fp64 contract of reference.py per output row (a zero row, per-32 scales over +-5 binades) +through the bf16 entry (``mxfp8_linear``) and the producer entry (``mxfp8_gemm_prequant``, same bytes as the bf16 +entry for plain decode rows), repeats bitwise, graph replay with a poisoned output and changed +inputs, MFMA co-residency; each prefill row against the contract at its window edges; no compile after +``warmup_mxfp8_gemm_weight``.""" +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.reference import COS_MIN, REL_L2_MAX, compare, mxfp8_gemm_reference, quantize_mxfp8 + +from .test_gemm import require_gfx950 +from .test_shared_expert_rows import _key, _spread + +BF16 = torch.bfloat16 +# (N, K, first M, last M, (kernel, tile_m, tile_n, K-waves, split-K, w_cm)) +ROWS = [ + (3072, 6144, 1, 4, ("decode_fq", 4, 16, 2, 1, None)), + (3072, 6144, 5, 8, ("decode_fq", 8, 16, 3, 1, None)), + (3072, 6144, 9, 16, ("decode_fq", 16, 48, 1, 8, None)), + (3072, 6144, 17, 32, ("decode_fq", 32, 48, 2, 4, None)), + (3072, 6144, 33, 48, ("decode_splitk", 48, 48, 3, 4, None)), + (3072, 6144, 49, 64, ("decode_splitk", 64, 48, 3, 4, None)), + (3072, 6144, 65, 80, ("decode_splitk", 80, 48, 3, 4, None)), + (3072, 6144, 81, 96, ("decode_splitk", 96, 48, 3, 4, None)), + (3072, 6144, 97, 112, ("decode_splitk", 112, 48, 3, 4, None)), + (3072, 6144, 113, 128, ("decode_splitk", 128, 48, 3, 4, None)), + (3072, 6144, 129, 144, ("decode_splitk", 144, 48, 3, 4, None)), + (3072, 6144, 145, 160, ("decode_splitk", 160, 48, 3, 4, None)), + (3072, 6144, 161, 176, ("decode_splitk", 176, 48, 3, 4, None)), + (3072, 6144, 177, 192, ("decode_splitk", 192, 48, 3, 4, None)), + (3072, 6144, 193, 208, ("decode_splitk", 208, 48, 3, 4, None)), + (3072, 6144, 209, 224, ("decode_splitk", 112, 48, 3, 2, None)), + (3072, 6144, 225, 256, ("decode_splitk", 128, 48, 3, 2, None)), + (3072, 6144, 257, 320, ("decode", 80, 48, 3, 1, None)), + (3072, 6144, 321, 384, ("decode", 96, 48, 3, 1, None)), + (3072, 6144, 385, 448, ("decode", 112, 48, 3, 1, 0)), + (3072, 6144, 449, 512, ("decode", 128, 48, 3, 1, 0)), + (3072, 6144, 513, 640, ("decode", 160, 48, 3, 1, 0)), + (3072, 6144, 641, 768, ("decode", 192, 48, 3, 1, 0)), + (3072, 6144, 769, 896, ("aiter", 128, 64, 0, 1, None)), + (3072, 6144, 897, 1024, ("staged_ab", 128, 256, 2, 2, None)), + (3072, 6144, 1025, 1280, ("staged_ab", 128, 128, 2, 1, None)), + (3072, 6144, 1281, 2560, ("4wave_nt", 256, 128, 2, 1, None)), + (3072, 6144, 2561, 65536, ("4wave_nt", 256, 256, 2, 1, None)), + (4608, 6144, 1, 4, ("decode_fq", 4, 32, 2, 1, None)), + (4608, 6144, 5, 8, ("decode_fq", 8, 32, 2, 1, None)), + (4608, 6144, 9, 16, ("decode_fq", 16, 64, 2, 3, None)), + (4608, 6144, 17, 32, ("decode", 16, 32, 1, 1, 0)), + (4608, 6144, 33, 48, ("decode_splitk", 48, 64, 4, 3, None)), + (4608, 6144, 49, 64, ("decode_splitk", 64, 64, 4, 3, None)), + (4608, 6144, 65, 80, ("decode_splitk", 80, 64, 4, 3, None)), + (4608, 6144, 81, 96, ("decode_splitk", 96, 64, 4, 3, None)), + (4608, 6144, 97, 112, ("decode_splitk", 112, 64, 4, 3, None)), + (4608, 6144, 113, 128, ("decode_splitk", 64, 128, 4, 3, None)), + (4608, 6144, 129, 160, ("decode_splitk", 80, 128, 4, 3, None)), + (4608, 6144, 161, 192, ("decode_splitk", 96, 128, 4, 3, None)), + (4608, 6144, 193, 224, ("decode_splitk", 112, 128, 4, 3, None)), + (4608, 6144, 225, 256, ("decode_splitk", 128, 128, 4, 3, None)), + (4608, 6144, 257, 288, ("decode", 48, 64, 1, 1, 0)), + (4608, 6144, 289, 384, ("decode", 64, 64, 1, 1, 0)), + (4608, 6144, 385, 448, ("decode", 80, 64, 2, 1, 0)), + (4608, 6144, 449, 512, ("decode", 64, 96, 2, 1, 0)), + (4608, 6144, 513, 640, ("decode", 128, 96, 2, 1, 0)), + (4608, 6144, 641, 768, ("decode", 128, 64, 2, 1, 0)), + (4608, 6144, 769, 896, ("decode", 112, 96, 2, 1, 0)), + (4608, 6144, 897, 1024, ("decode", 128, 96, 2, 1, 0)), + (4608, 6144, 1025, 1536, ("4wave", 256, 128, 2, 1, None)), + (4608, 6144, 1537, 65536, ("4wave_nt", 256, 256, 2, 1, None)), + (6144, 1536, 1, 8, ("decode_fq", 8, 32, 3, 1, None)), + (6144, 1536, 9, 16, ("decode_fq", 16, 32, 3, 1, None)), + (6144, 1536, 17, 32, ("decode_fq", 32, 32, 3, 1, None)), + (6144, 1536, 33, 48, ("decode", 48, 32, 1, 1, None)), + (6144, 1536, 49, 64, ("decode", 32, 48, 1, 1, None)), + (6144, 1536, 65, 96, ("decode", 48, 48, 3, 1, None)), + (6144, 1536, 97, 128, ("decode", 32, 64, 1, 1, 0)), + (6144, 1536, 129, 160, ("decode", 80, 48, 3, 1, None)), + (6144, 1536, 161, 192, ("decode", 48, 64, 1, 1, 0)), + (6144, 1536, 193, 256, ("decode", 64, 64, 1, 1, 0)), + (6144, 1536, 257, 288, ("decode", 144, 48, 3, 1, None)), + (6144, 1536, 289, 320, ("decode", 80, 96, 2, 1, None)), + (6144, 1536, 321, 384, ("decode", 96, 96, 3, 1, None)), + (6144, 1536, 385, 448, ("decode", 64, 96, 2, 1, 0)), + (6144, 1536, 449, 512, ("decode", 128, 96, 2, 1, None)), + (6144, 1536, 513, 640, ("decode", 160, 96, 2, 1, None)), + (6144, 1536, 641, 896, ("decode", 112, 96, 2, 1, 0)), + (6144, 1536, 897, 1024, ("4wave_nt", 256, 128, 2, 1, None)), + (6144, 1536, 1025, 1280, ("4wave_nt", 128, 256, 2, 1, None)), + (6144, 1536, 1281, 65536, ("4wave_nt", 256, 256, 2, 1, None)), + (12288, 6144, 1, 4, ("decode_fq", 8, 32, 1, 1, None)), + (12288, 6144, 5, 8, ("decode_fq", 8, 48, 4, 1, None)), + (12288, 6144, 9, 16, ("decode_fq", 16, 48, 3, 1, None)), + (12288, 6144, 17, 32, ("decode", 32, 48, 1, 1, None)), + (12288, 6144, 33, 48, ("decode", 48, 48, 1, 1, None)), + (12288, 6144, 49, 64, ("decode_splitk", 64, 96, 6, 2, None)), + (12288, 6144, 65, 80, ("decode_splitk", 80, 96, 6, 2, None)), + (12288, 6144, 81, 96, ("decode_splitk", 96, 96, 6, 2, None)), + (12288, 6144, 97, 112, ("decode_splitk", 112, 96, 6, 2, None)), + (12288, 6144, 113, 128, ("decode_splitk", 128, 96, 6, 2, None)), + (12288, 6144, 129, 144, ("decode_splitk", 144, 96, 6, 2, None)), + (12288, 6144, 145, 160, ("decode_splitk", 160, 96, 6, 2, None)), + (12288, 6144, 161, 176, ("decode_splitk", 176, 96, 6, 2, None)), + (12288, 6144, 177, 192, ("decode_splitk", 192, 96, 6, 2, None)), + (12288, 6144, 193, 208, ("decode_splitk", 208, 96, 6, 2, None)), + (12288, 6144, 209, 224, ("decode_splitk", 224, 96, 6, 2, None)), + (12288, 6144, 225, 256, ("decode", 128, 96, 2, 1, None)), + (12288, 6144, 257, 288, ("decode", 96, 96, 2, 1, 0)), + (12288, 6144, 289, 320, ("decode", 112, 96, 2, 1, 0)), + (12288, 6144, 321, 384, ("decode", 128, 96, 2, 1, 0)), + (12288, 6144, 385, 448, ("decode", 160, 96, 2, 1, 0)), + (12288, 6144, 449, 512, ("4wave_nt", 256, 128, 2, 1, None)), + (12288, 6144, 513, 640, ("decode", 160, 96, 2, 1, 0)), + (12288, 6144, 641, 768, ("decode", 128, 96, 1, 1, 0)), + (12288, 6144, 769, 65536, ("4wave_nt", 256, 256, 2, 1, None)), +] +SHAPES = sorted({(r[0], r[1]) for r in ROWS}) +DECODE_KINDS = ("decode", "decode_fq", "decode_splitk") +DECODE = [r for r in ROWS if r[4][0] in DECODE_KINDS] +PREFILL = [r for r in ROWS if r[4][0] not in DECODE_KINDS] + + +def _edges(lo, hi): + return sorted({lo, lo + 1, (lo + hi) // 2, hi - 1, hi} & set(range(lo, hi + 1))) + + +def test_tuned_shapes(): + assert set(SHAPES) <= G.tuned_shapes(BF16) + + +def test_dispatch_edges(): + for n, k, lo, hi, row in ROWS: + for m in _edges(lo, hi): + cfg = G._select_config(m, n, k) + assert _key(cfg) == row, (n, k, m, cfg) + decode = row[0] in DECODE_KINDS + assert G.prequant_code_tile_rows(m, n, k) == (max(16, row[1]) if decode else 0), (n, k, m) + if not decode: + assert G._select_config(m, n, k, allow_decode=False) == cfg + for n, k in SHAPES: + rows = sorted((r[2], r[3]) for r in ROWS if (r[0], r[1]) == (n, k)) + assert rows[0][0] == 1 and rows[-1][1] == 65536, (n, k) + assert all(a[1] + 1 == b[0] for a, b in zip(rows, rows[1:])), (n, k) # contiguous, no overlap + + +_WEIGHTS = {} + + +def _weights(n, k): + if (n, k) not in _WEIGHTS: + gen = torch.Generator(device="cuda").manual_seed(n + k) + q, s = quantize_mxfp8((_spread((n, k), gen) * 0.02).bfloat16()) + _WEIGHTS[(n, k)] = (q, s, *G.preshuffle_mxfp8_weight(q, s)) + return _WEIGHTS[(n, k)] + + +def _check_rows(out, exact, lo, tag): + got = out[lo:].double() + cos = torch.nn.functional.cosine_similarity(got, exact[lo:], dim=1) + rel = (got - exact[lo:]).norm(dim=1) / exact[lo:].norm(dim=1) + assert cos.min() >= 0.99999 and rel.max() <= 1e-3, (tag, float(cos.min()), float(rel.max())) + + +DECODE_CASES = [] +for _n, _k, _lo, _hi, _row in DECODE: + for _m in sorted({_lo, _hi, (_lo + _hi) // 2}): + DECODE_CASES.append((_n, _k, _m, _row)) + + +@pytest.mark.parametrize("n,k,m,row", DECODE_CASES, ids=[f"n{c[0]}_k{c[1]}_m{c[2]}" for c in DECODE_CASES]) +@torch.inference_mode() +def test_decode_rows(n, k, m, row): + require_gfx950() + qw, sw, w, ws = _weights(n, k) + gen = torch.Generator(device="cuda").manual_seed(m + n) + x = _spread((m, k), gen).bfloat16() + if m > 1: + x[0].zero_() + out = G.mxfp8_linear(x, w, ws, BF16) + # the table's row is the pinned one (test_dispatch_edges): its explicit config gives the same bytes + explicit = G.mxfp8_linear(x, w, ws, BF16, config=G._select_config(m, n, k)) + assert torch.equal(explicit.view(torch.int16), out.view(torch.int16)) + qx, sx = quantize_mxfp8(x) + exact = mxfp8_gemm_reference(qx, sx, qw, sw, accumulate=torch.float64).double() + lo = 1 if m > 1 else 0 + if m > 1: + assert not out[0].any() + _check_rows(out, exact, lo, (n, k, m)) + for _ in range(3): + assert torch.equal(G.mxfp8_linear(x, w, ws, BF16).view(torch.int16), out.view(torch.int16)) + tile = G.prequant_code_tile_rows(m, n, k) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, tile) + shuffled = G.mxfp8_gemm_prequant(q, s, w, ws, BF16, m) + _check_rows(shuffled, exact, lo, (n, k, m, "codes")) + if row[0] in ("decode", "decode_splitk"): + assert torch.equal(shuffled.view(torch.int16), out.view(torch.int16)) + xs = x.clone() + gr = torch.cuda.CUDAGraph() + with torch.cuda.graph(gr): + gout = G.mxfp8_linear(xs, w, ws, BF16) + gout.fill_(float("nan")) + gr.replay() + torch.cuda.synchronize() + assert torch.equal(gout.view(torch.int16), out.view(torch.int16)) + xs.copy_(_spread((m, k), gen).bfloat16()) + gr.replay() + eager = G.mxfp8_linear(xs, w, ws, BF16) + torch.cuda.synchronize() + assert torch.equal(gout.view(torch.int16), eager.view(torch.int16)) + + +PREFILL_CASES = [] +for _n, _k, _lo, _hi, _row in PREFILL: + for _m in sorted({_lo, min(_hi, 8192), *([_lo + 255] if _lo + 255 < min(_hi, 8192) else [])}): + PREFILL_CASES.append((_n, _k, _m, _row)) + + +@pytest.mark.parametrize("n,k,m,row", PREFILL_CASES, ids=[f"n{c[0]}_k{c[1]}_m{c[2]}" for c in PREFILL_CASES]) +@torch.inference_mode() +def test_prefill_rows(n, k, m, row): + require_gfx950() + qw, sw, w, ws = _weights(n, k) + torch.manual_seed(m) + x = torch.randn((m, k), device="cuda", dtype=BF16) + out = G.mxfp8_linear(x, w, ws, BF16) + ref = mxfp8_gemm_reference(*quantize_mxfp8(x), qw, sw) + metrics = compare(out, ref) + assert metrics["cos"] >= COS_MIN and metrics["rel_l2"] <= REL_L2_MAX, (n, k, m, metrics) + assert torch.equal(G.mxfp8_linear(x, w, ws, BF16), out) + a, s = G.mxfp8_e4m3_quantize_aiter(x) + assert torch.equal(G.mxfp8_gemm_prequant(a, s, w, ws, BF16, m), out) + + +@pytest.mark.parametrize("n,k", SHAPES) +@torch.inference_mode() +def test_no_compile_after_warmup(n, k, monkeypatch): + """After ``warmup_mxfp8_gemm_weight`` up to 32768 rows nothing compiles: every M of 1-1024 and every window edge + above, through the bf16, codes and producer entries.""" + from flydsl.compiler.jit_function import MlirCompiler + + require_gfx950() + _, _, w, ws = _weights(n, k) + G.warmup_mxfp8_gemm_weight(w, ws, 32768, BF16) + torch.cuda.synchronize() + compiled = [] + real = MlirCompiler.compile + + def counting(self, *a, **kw): + compiled.append(1) + return real(self, *a, **kw) + + monkeypatch.setattr(MlirCompiler, "compile", counting) + edges = {e for r in ROWS if (r[0], r[1]) == (n, k) for e in (r[2], r[3]) if 1024 < e <= 32768} + x = torch.zeros((32768, k), device="cuda", dtype=BF16) + for m in [*range(1, 1025), *sorted(edges)]: + G.mxfp8_linear(x[:m], w, ws, BF16) + tile = G.prequant_code_tile_rows(m, n, k) + if tile: + q, s = G.mxfp8_e4m3_quantize_shuffled(x[:m], tile) + else: + q, s = G.mxfp8_e4m3_quantize_aiter(x[:m]) + G.mxfp8_gemm_prequant(q, s, w, ws, BF16, m) + torch.cuda.synchronize() + assert not compiled, len(compiled) + + +@pytest.mark.parametrize("n,k,m", [(r[0], r[1], r[3]) for r in DECODE]) +def test_mfma_co_residency(n, k, m): + from tests.mfma_spinner import mfma_spinner + + require_gfx950() + _, _, w, ws = _weights(n, k) + x = torch.randn((m, k), device="cuda", dtype=BF16) + ref = G.mxfp8_linear(x, w, ws, BF16) + torch.cuda.synchronize() + spinner = mfma_spinner(256) + side = torch.cuda.Stream(priority=-1) + for _ in range(3): + spinner(100_000, side) + got = G.mxfp8_linear(x, w, ws, BF16) + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), ref.view(torch.int16)) diff --git a/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py b/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/test_gateup_swiglu_bf16.py b/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/test_gateup_swiglu_bf16.py new file mode 100644 index 0000000..0f24e3e --- /dev/null +++ b/tests/gemm/mxfp8_gateup_swiglu_n12288_k6144_i6144/test_gateup_swiglu_bf16.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Paired gate/up epilogue (N 12288, K 6144) preserves materialized BF16 dots and both activations.""" +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as G +from amd_kernels.gemm.mxfp8_common.reference import (mxfp8_gateup_swiglu_bf16_reference, quantize_mxfp8, + swiglu_bf16_reference) +from amd_kernels.gemm.mxfp8_gateup_swiglu_n12288_k6144_i6144 import host as GUS + +N, K = GUS.N, GUS.K +SWEPT = [(1, (7, 7)), (2, (7, 7)), (2, (5, 5)), (3, (5, 8)), (4, (6, 6))] + + +def materialized_gu(q, s, pw, ps, m, kw): + """The decode GEMM with the fused kernel's K-wave count: the same K partition and sum order.""" + cfg = G._Config(16, 32, 128, 0, 0, 1, 'decode', kw, 0, 7, 7) + out = torch.empty((m, N), device=q.device, dtype=torch.bfloat16) + G._launch_decode(cfg, q, s, pw, ps, out) + return out + + +@pytest.fixture(scope='module') +def weights(): + torch.manual_seed(1973) + w = (torch.randn((N, K), device='cuda') * .02).bfloat16() + q, s = quantize_mxfp8(w) + return q, s, G.preshuffle_mxfp8_weight(q, s) + + +def _x(m, seed): + torch.manual_seed(seed) + scale = torch.exp2(torch.randint(-5, 6, (m, K // 32, 1), device='cuda').expand(m, K // 32, 32).reshape(m, K).float()) + x = (torch.randn((m, K), device='cuda') * scale).bfloat16() + if m > 1: + x[0].zero_() + return x + + +@pytest.mark.parametrize('form', ('fp32', 'bf16_chain')) +@pytest.mark.parametrize('m', (1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, 16)) +def test_fp64_and_graph(weights, form, m): + w, s, (pw, ps) = weights + x = _x(m, m) + q, sc = G.mxfp8_e4m3_quantize_shuffled(x, 16) + got = GUS.mxfp8_gateup_swiglu_bf16(q, sc, pw, ps, m, form=form) + expected = swiglu_bf16_reference(materialized_gu(q, sc, pw, ps, m, GUS.CONFIG[form][0]), form=form) + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), expected.view(torch.int16)), (m, form, int((got != expected).sum())) + qx, sx = quantize_mxfp8(x) + ref = mxfp8_gateup_swiglu_bf16_reference(qx, sx, w, s, form=form) + cos = torch.nn.functional.cosine_similarity(got.double(), ref.double(), dim=1) + cos = torch.where(ref.double().norm(dim=1) == 0, (got == 0).all(dim=1).double(), cos) + assert cos.min() >= .9999, (m, form, cos.min()) + gr = torch.cuda.CUDAGraph() + with torch.cuda.graph(gr): + out = GUS.mxfp8_gateup_swiglu_bf16(q, sc, pw, ps, m, form=form) + for _ in range(3): + out.fill_(float('nan')) + gr.replay() + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), got.view(torch.int16)) + q.view(torch.uint8).zero_() + gr.replay() + torch.cuda.synchronize() + assert not out.count_nonzero() + + +@pytest.mark.parametrize('form', ('fp32', 'bf16_chain')) +def test_many_fusion_inputs(weights, form): + _, _, (pw, ps) = weights + for seed in range(64): + torch.manual_seed(seed + 771) + x = (torch.randn((8, K), device='cuda') * 2 ** (seed % 11 - 5)).bfloat16() + q, sc = G.mxfp8_e4m3_quantize_shuffled(x, 16) + ref = swiglu_bf16_reference(materialized_gu(q, sc, pw, ps, 8, GUS.CONFIG[form][0]), form=form) + got = GUS.mxfp8_gateup_swiglu_bf16(q, sc, pw, ps, 8, form=form) + torch.cuda.synchronize() + assert torch.equal(got.view(torch.int16), ref.view(torch.int16)), (form, seed, int((got != ref).sum())) + + +@pytest.mark.parametrize('config', SWEPT) +def test_swept_configs_bitwise(weights, config): + """Every K-wave count / lookahead the sweep times equals the decode GEMM of the same K-wave count.""" + _, _, (pw, ps) = weights + x = _x(7, 99) + q, sc = G.mxfp8_e4m3_quantize_shuffled(x, 16) + gu = materialized_gu(q, sc, pw, ps, 7, config[0]) + for form in ('fp32',) if config[0] == 1 else ('fp32', 'bf16_chain'): + got = GUS.mxfp8_gateup_swiglu_bf16(q, sc, pw, ps, 7, form=form, config=config) + assert torch.equal(got.view(torch.int16), swiglu_bf16_reference(gu, form=form).view(torch.int16)), form + + +def test_warmup(weights): + from flydsl.compiler.jit_function import MlirCompiler + _, _, (pw, ps) = weights + x = torch.zeros((8, K), device='cuda', dtype=torch.bfloat16) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, 16) + GUS._launcher.cache_clear() + GUS.warmup_mxfp8_gateup_swiglu_bf16(q, s, pw, ps) + orig = MlirCompiler.compile + + def unexpected(*a, **kw): + raise AssertionError('compile after warm-up') + MlirCompiler.compile = unexpected + try: + for m in range(1, 17): + for form in ('fp32', 'bf16_chain'): + GUS.mxfp8_gateup_swiglu_bf16(q, s, pw, ps, m, form=form) + torch.cuda.synchronize() + finally: + MlirCompiler.compile = orig + + +def test_mfma_co_residency(weights): + from tests.mfma_spinner import mfma_spinner + _, _, (pw, ps) = weights + x = torch.randn((8, K), device='cuda', dtype=torch.bfloat16) + q, s = G.mxfp8_e4m3_quantize_shuffled(x, 16) + spinner = mfma_spinner(256) + side = torch.cuda.Stream(priority=-1) + for form in ('fp32', 'bf16_chain'): + ref = GUS.mxfp8_gateup_swiglu_bf16(q, s, pw, ps, 8, form=form) + torch.cuda.synchronize() + for _ in range(3): + spinner(100_000, side) + got = GUS.mxfp8_gateup_swiglu_bf16(q, s, pw, ps, 8, form=form) + torch.cuda.synchronize() + assert torch.equal(ref.view(torch.int16), got.view(torch.int16)) diff --git a/tests/gemm/mxfp8_n4992_k6144/__init__.py b/tests/gemm/mxfp8_n4992_k6144/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/gemm/mxfp8_n4992_k6144/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/gemm/mxfp8_n4992_k6144/test_isa.py b/tests/gemm/mxfp8_n4992_k6144/test_isa.py new file mode 100644 index 0000000..dc4d0c0 --- /dev/null +++ b/tests/gemm/mxfp8_n4992_k6144/test_isa.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Every decode row the N 4992 host serves keeps its operands in VGPRs: the decode kernel parks an operand in an +AGPR when its registers run out and the copy back races the AGPR write (``mxfp8_gemm_decode.VGPR_BUDGET``: wrong +sums now and then). At N 4992 the deepest lookahead parks on several tiles (e.g. 192x48 three or four K-waves), so +the final ISA of each served row must contain no ``v_accvgpr_write``. Compiled in a subprocess, compile-only, cache +off, ISA dumped.""" + +import glob +import os +import subprocess +import sys +from pathlib import Path + +import torch + +from ..mxfp8_common.test_gemm import require_gfx950 + + +def _rows(): + """The decode kernel's rows (the split-K decode rows: tests/gemm/mxfp8_common/test_decode_splitk.py).""" + from amd_kernels.gemm.mxfp8_common.gemm import _DECODE_KERNEL + from amd_kernels.gemm.mxfp8_n4992_k6144 import host as H + + return sorted({c for _, c in H.DECODE_ROWS + H.DEFERRED_ROWS if c.kernel == _DECODE_KERNEL}, key=str) + + +def test_decode_rows_hold_operands_in_vgprs(tmp_path): + require_gfx950() + root = Path(__file__).resolve().parents[3] + env = dict(os.environ, AKR_COMPILE_ONLY="1", FLYDSL_DEBUG_DUMP_ASM="1", FLYDSL_DUMP_DIR=str(tmp_path), + FLYDSL_RUNTIME_ENABLE_CACHE="0") + subprocess.run([sys.executable, "-m", "tests.gemm.mxfp8_n4992_k6144.test_isa"], cwd=root, env=env, check=True) + names = [g.kernel_name for g in _launchers()] + assert len(names) == len(_rows()) + for name in names: + isa = glob.glob(f"{tmp_path}/{name}/*.s") + assert isa, name + assert "v_accvgpr_write" not in Path(isa[0]).read_text(), name + + +def _launchers(): + from amd_kernels.gemm.mxfp8_common import gemm as g + + return [g._decode_launcher(4992, 6144, config, False) for config in _rows()] + + +if __name__ == "__main__": + from amd_kernels.common.tensor_shim import _run_compiled + + d = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") + for config, exe in zip(_rows(), _launchers()): + _run_compiled(exe, d, d, d, d, d, config.tile_m, 1024, 1, config.split_k * (4992 // config.tile_n), + torch.cuda.current_stream()) diff --git a/tests/gemm/mxfp8_n4992_k6144/test_n4992_k6144.py b/tests/gemm/mxfp8_n4992_k6144/test_n4992_k6144.py new file mode 100644 index 0000000..26c2572 --- /dev/null +++ b/tests/gemm/mxfp8_n4992_k6144/test_n4992_k6144.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""TP2 QKV (N 4992, K 6144): every row ``host.select`` / ``host.select_deferred`` dispatches, at its window edges +(window ends are multiples of 16, as vLLM's graph sizes), against the fp64 product of the same quantized operands (a +zero row, spread scales); bitwise repeats and entry agreement (``mxfp8_gemm_prequant`` from the producer's layout, +``mxfp8_linear``, ``mxfp8_gemm``); graph replay over a NaN-poisoned output; deferred partials; every persistent tile, +the row split and the DBO plain grid bitwise equal; guard rows; next to an MFMA co-runner.""" + +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common import reference as ref +from amd_kernels.gemm.mxfp8_n4992_k6144 import host as H + +from ..mxfp8_common.test_gemm import require_gfx950 + +N, K = H.N, H.K + + +def _edges(): + """The first and last row count of every decode and deferred window.""" + tops = sorted({top for top, _ in H.DECODE_ROWS} | {top for top, _ in H.DEFERRED_ROWS}) + ms = {1, 2, 7, 8} + for top in tops: + ms |= {top, top + 1} + return sorted(ms) + + +def _persistent_edges(): + """Row counts where ``persistent_plan`` changes, both sides, up to 12288.""" + ms, prev = [], None + for m in range(H.DECODE_MAX + 1, 12289): + plan = H.persistent_plan(m) + if plan != prev: + ms += [m - 1, m] if prev is not None else [m] + prev = plan + return ms + + +DECODE_MS = _edges() +PERSISTENT_MS = _persistent_edges() + + +@pytest.fixture(scope="module") +def weights(): + torch.manual_seed(4992) + raw = torch.randn((N, K), device="cuda") * 0.02 + raw *= torch.exp2(torch.randint(-5, 6, (N, K // 32, 1), device="cuda").float()).expand(N, K // 32, 32).reshape(N, K) + codes, scales = ref.quantize_mxfp8(raw.bfloat16()) + return codes, scales, g.preshuffle_mxfp8_weight(codes, scales) + + +def _activation(m, seed): + torch.manual_seed(seed) + x = torch.randn((m, K), device="cuda") * torch.exp2( + torch.randint(-5, 6, (m, K // 32, 1), device="cuda").float()).expand(m, K // 32, 32).reshape(m, K) + x = x.bfloat16() + if m > 1: + x[m // 2] = 0 + return x + + +def _check(got, x, codes, scales): + expected = ref.mxfp8_gemm_reference(*ref.quantize_mxfp8(x), codes, scales, accumulate=torch.float64) + metrics = ref.compare(got, expected) + assert metrics["cos"] >= ref.COS_MIN and metrics["rel_l2"] <= ref.REL_L2_MAX, metrics + active = expected.double().norm(dim=1) > 0 + row_l2 = (got.double() - expected.double()).norm(dim=1) / expected.double().norm(dim=1).clamp_min(1e-30) + assert row_l2[active].max() <= ref.REL_L2_MAX + if x.shape[0] > 1: + assert torch.count_nonzero(got[x.shape[0] // 2]) == 0 + + +def _producer_codes(x): + m = x.shape[0] + tile = g.prequant_code_tile_rows(m, N, K) + return (g.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else g.mxfp8_e4m3_quantize_aiter(x)), tile + + +def _replays(call, want): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = call() + for _ in range(3): + captured.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(captured, want) + + +@pytest.mark.parametrize("m", DECODE_MS + PERSISTENT_MS[::3]) +@torch.inference_mode() +def test_rows(weights, m): + require_gfx950() + codes, scales, (w, ws) = weights + x = _activation(m, m) + (a, asc), tile = _producer_codes(x) + config = g._select_prequant_config(m, N, K, torch.bfloat16) + assert (tile != 0) == (config.kernel in g._SHUFFLED_A_KERNELS) + call = lambda: g.mxfp8_gemm_prequant(a, asc, w, ws, torch.bfloat16, m) # noqa: E731 + got = call() + _check(got, x, codes, scales) + for _ in range(3): + assert torch.equal(call(), got) + assert torch.equal(g.mxfp8_linear(x, w, ws), got) + if not tile: + assert torch.equal(g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True), got) + _replays(call, got) + + +@pytest.mark.parametrize("m", PERSISTENT_MS) +@torch.inference_mode() +def test_persistent_edges(weights, m): + require_gfx950() + codes, scales, (w, ws) = weights + x = _activation(m, m + 1) + a, asc = g.mxfp8_e4m3_quantize_aiter(x) + config = g._select_config(m, N, K, torch.bfloat16, allow_decode=False) + plan = H.persistent_plan(m) + assert (config.tile_m, config.tile_n) == plan[0][1] + assert config.kernel == (H._PERSISTENT_SPLIT_KERNEL if len(plan) > 1 else H._PERSISTENT_KERNEL) + got = g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + _check(got, x, codes, scales) + assert torch.equal(got, g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True)) + if config.kernel == H._PERSISTENT_SPLIT_KERNEL: # the split's launches sum like the one-launch tile + assert torch.equal(got, g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True, + config=H._persistent(*H.persistent_tile(m)))) + + +@pytest.mark.parametrize("m", [1, 16, 100, H.ROW_MAJOR_FALLBACK_MAX, H.ROW_MAJOR_FALLBACK_MAX + 1, H.DECODE_MAX]) +@torch.inference_mode() +def test_row_major_codes(weights, m): + """Row-major codes below the decode rows' end (``mxfp8_gemm``): the common fallback, then persistent tiles.""" + require_gfx950() + codes, scales, (w, ws) = weights + x = _activation(m, 5 * m) + a, asc = g.mxfp8_e4m3_quantize_aiter(x) + got = g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + _check(got, x, codes, scales) + assert torch.equal(got, g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True)) + + +@pytest.mark.parametrize("m", [m for m in DECODE_MS if m <= H.DEFERRED_ROWS[-1][0] + 1]) +@torch.inference_mode() +def test_deferred(weights, m): + """The chain's split-K partials sum (split order) to the standalone row's output within the contract, read a + prefix of the producer's codes, and repeat bitwise.""" + require_gfx950() + codes, scales, (w, ws) = weights + x = _activation(m, 3 * m) + (a, asc), tile = _producer_codes(x) + assert tile + out = g.mxfp8_gemm_shuffled(a, asc, w, ws, torch.bfloat16, m, defer_reduce=True) + if not isinstance(out, g.SkPartials): + assert torch.equal(out, g.mxfp8_gemm_prequant(a, asc, w, ws, torch.bfloat16, m)) + return + again = g.mxfp8_gemm_shuffled(a, asc, w, ws, torch.bfloat16, m, defer_reduce=True) + assert torch.equal(out.tmp, again.tmp) + _check(out.materialize(), x, codes, scales) + + +@pytest.mark.parametrize("m", [1000, 2049]) +@torch.inference_mode() +def test_persistent_tiles_bitwise(weights, m): + """Every persistent tile sums K in the same order: the tile a row count takes never changes a byte.""" + require_gfx950() + _, _, (w, ws) = weights + a, asc = g.mxfp8_e4m3_quantize_aiter(_activation(m, 13)) + outs = [] + for tile in H.PERSISTENT_ROUND_US: + out = torch.empty((m, N), dtype=torch.bfloat16, device="cuda") + H.LAUNCHERS[H._PERSISTENT_KERNEL](H._persistent(*tile), a, asc, w, ws, out) + outs.append(out) + assert all(torch.equal(outs[0], out) for out in outs[1:]) + + +@pytest.mark.parametrize("m", [4097, 8192, 12288]) +@torch.inference_mode() +def test_dbo_plain_grid_bitwise(weights, m, monkeypatch): + require_gfx950() + _, _, (w, ws) = weights + a, asc = g.mxfp8_e4m3_quantize_aiter(_activation(m, 7)) + run = lambda: g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True) # noqa: E731 + persistent = run() + monkeypatch.setattr(g, "DBO", True) + assert torch.equal(run(), persistent) + + +@pytest.mark.parametrize("m", [721, 1025, 2048, 4097, 5632]) +@torch.inference_mode() +def test_persistent_output_guards(weights, m): + require_gfx950() + _, _, (w, ws) = weights + a, asc = g.mxfp8_e4m3_quantize_aiter(_activation(m, 11)) + want = g.mxfp8_gemm(a, asc, w, ws, torch.bfloat16, activation_scale_preshuffled=True) + arena = torch.empty((m + 2, N), dtype=torch.bfloat16, device="cuda") + arena.view(torch.int16).fill_(0x3555) + guard0, guard1 = arena[0].clone(), arena[-1].clone() + out = arena[1:-1] + config = g._select_config(m, N, K, torch.bfloat16, allow_decode=False) + H.LAUNCHERS[config.kernel](config, a, asc, w, ws, out) + torch.cuda.synchronize() + assert torch.equal(out, want) + assert torch.equal(arena[0], guard0) and torch.equal(arena[-1], guard1) + + +@pytest.mark.parametrize("m", [16, 256, 700, 1024, 4096, 5632]) +@torch.inference_mode() +def test_next_to_mfma(weights, m): + from tests.mfma_spinner import mfma_spinner + + require_gfx950() + _, _, (w, ws) = weights + (a, asc), _ = _producer_codes(_activation(m, 5)) + run = lambda: g.mxfp8_gemm_prequant(a, asc, w, ws, torch.bfloat16, m) # noqa: E731 + want = run() + spin = mfma_spinner(32) + side = torch.cuda.Stream(priority=-1) + torch.cuda.synchronize() + spin(1_000_000, side) + results = [run() for _ in range(16)] + torch.cuda.synchronize() + assert all(torch.equal(want, got) for got in results) + diff --git a/tests/gemm/mxfp8_n4992_k6144/test_warmup.py b/tests/gemm/mxfp8_n4992_k6144/test_warmup.py new file mode 100644 index 0000000..345bbed --- /dev/null +++ b/tests/gemm/mxfp8_n4992_k6144/test_warmup.py @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""After ``warmup_mxfp8_gemm_weight`` of the N 4992 weight for 32768 tokens nothing compiles: every entry at every +window edge and at random row counts, the deferred partials, and the DBO plain grid.""" + +import random + +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_n4992_k6144 import host as H + +from ..mxfp8_common.test_gemm import make_inputs, require_gfx950 +from .test_n4992_k6144 import DECODE_MS, PERSISTENT_MS + +N, K = H.N, H.K +MAX_TOKENS = 32768 + + +@torch.inference_mode() +def test_nothing_compiles_after_warmup(monkeypatch): + import flydsl.compiler as flyc + from flydsl.compiler.jit_function import MlirCompiler + + require_gfx950() + _, _, _, (w, ws) = make_inputs(16, N, K) + g.warmup_mxfp8_gemm_weight(w, ws, MAX_TOKENS, torch.bfloat16) + rng = random.Random(4992) + ms = sorted(set(DECODE_MS + PERSISTENT_MS + [rng.randint(1, MAX_TOKENS) for _ in range(24)] + [MAX_TOKENS])) + x = torch.zeros((MAX_TOKENS, K), dtype=torch.bfloat16, device="cuda") + compiled = [] + real = MlirCompiler.compile + real_jit = flyc.compile + # a variant's first launch in a process goes through flyc.compile even when the disk cache has it + monkeypatch.setattr(MlirCompiler, "compile", lambda self, *a, **kw: compiled.append(1) or real(self, *a, **kw)) + monkeypatch.setattr(flyc, "compile", lambda *a, **kw: compiled.append(2) or real_jit(*a, **kw)) + for dbo in (False, True): + monkeypatch.setattr(g, "DBO", dbo) + for m in ms: + tile = g.prequant_code_tile_rows(m, N, K) + a, asc = g.mxfp8_e4m3_quantize_shuffled(x[:m], tile) if tile else g.mxfp8_e4m3_quantize_aiter(x[:m]) + g.mxfp8_gemm_prequant(a, asc, w, ws, torch.bfloat16, m) + if tile: + g.mxfp8_gemm_shuffled(a, asc, w, ws, torch.bfloat16, m, defer_reduce=True) + g.mxfp8_linear(x[:m], w, ws) + assert not compiled, (dbo, m) + torch.cuda.synchronize() diff --git a/tests/gemm/mxfp8_n6144_k4096/__init__.py b/tests/gemm/mxfp8_n6144_k4096/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/gemm/mxfp8_n6144_k4096/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/gemm/mxfp8_n6144_k4096/test_kernels_k4096.py b/tests/gemm/mxfp8_n6144_k4096/test_kernels_k4096.py new file mode 100644 index 0000000..0b2bf21 --- /dev/null +++ b/tests/gemm/mxfp8_n6144_k4096/test_kernels_k4096.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""This shape's kernel variants: the owner-read decode rows equal the common decode kernel bit for bit, the persistent +256x192 tile's DBO plain grid equals its persistent grid, and the LDS-ring tiles keep their bits next to resident +MFMA work.""" + +import pytest +import torch +from amd_kernels.gemm.mxfp8_common import gemm as g +from amd_kernels.gemm.mxfp8_common.mxfp8_gemm_decode import compile_mxfp8_gemm_decode +from amd_kernels.gemm.mxfp8_n6144_k4096 import host as H + +from ..mxfp8_common.test_gemm import make_inputs, require_gfx950 + +N, K = 6144, 4096 + + +@pytest.mark.parametrize("m", [225, 256, 320, 384, 448, 513, 640, 641, 768]) +@torch.inference_mode() +def test_owner_read_rows_bitwise(m, monkeypatch): + require_gfx950() + config = g._select_prequant_config(m, N, K, torch.bfloat16) + assert H.decode_launcher(config, False) is not None + x, _, _, (w, s) = make_inputs(m, N, K, seed=m) + a, sa = g.mxfp8_e4m3_quantize_shuffled(x, config.tile_m) + got = g.mxfp8_gemm_shuffled(a, sa, w, s, torch.bfloat16, m) + plain = compile_mxfp8_gemm_decode(N=N, K=K, RT=config.tile_m // 16, NI=config.tile_n // 16, KW=2, + OWNER_READ=False) + monkeypatch.setattr(H, "decode_launcher", lambda config, a_bf16: plain) + g._decode_launcher.cache_clear() + try: + expected = g.mxfp8_gemm_shuffled(a, sa, w, s, torch.bfloat16, m) + finally: + g._decode_launcher.cache_clear() + assert torch.equal(got, expected) + + +@pytest.mark.parametrize("m", [3329, 4096, 5121, 6144]) +@torch.inference_mode() +def test_persistent_256x192_dbo_grid_bitwise(m, monkeypatch): + require_gfx950() + assert g._select_config(m, N, K, torch.bfloat16).kernel == H._PERSISTENT_KERNEL + x, _, _, (w, s) = make_inputs(m, N, K, seed=m) + a, sa = g.mxfp8_e4m3_quantize_aiter(x) + persistent = g.mxfp8_gemm(a, sa, w, s, torch.bfloat16, activation_scale_preshuffled=True) + monkeypatch.setattr(g, "DBO", True) + plain = g.mxfp8_gemm(a, sa, w, s, torch.bfloat16, activation_scale_preshuffled=True) + assert torch.equal(persistent, plain) + + +@pytest.mark.parametrize("m", [769, 1024, 1281, 2048]) +@torch.inference_mode() +def test_lds_ring_tiles_next_to_mfma(m): + from tests.mfma_spinner import mfma_spinner + + require_gfx950() + assert g._select_config(m, N, K, torch.bfloat16).kernel in (H._TILE_128_KERNEL, H._ONE_TILE_KERNEL) + x, _, _, (w, s) = make_inputs(m, N, K, seed=m) + a, sa = g.mxfp8_e4m3_quantize_aiter(x) + run = lambda: g.mxfp8_gemm(a, sa, w, s, torch.bfloat16, activation_scale_preshuffled=True) # noqa: E731 + expected = run() + spin = mfma_spinner(32) + side = torch.cuda.Stream(priority=-1) + torch.cuda.synchronize() + spin(1_000_000, side) + results = [run() for _ in range(16)] + torch.cuda.synchronize() + assert all(torch.equal(expected, got) for got in results) diff --git a/tests/gemm/mxfp8_n6144_k4096/test_oproj_k4096.py b/tests/gemm/mxfp8_n6144_k4096/test_oproj_k4096.py new file mode 100644 index 0000000..750d10b --- /dev/null +++ b/tests/gemm/mxfp8_n6144_k4096/test_oproj_k4096.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The TP2 o_proj N 6144 x K 4096: every row of ``host`` at its window edges and at the served CUDA-graph sizes, through +the three entries (bf16 ``mxfp8_linear``, producer codes ``mxfp8_gemm_prequant``, row-major codes ``mxfp8_gemm``), +against the FP64 reference; determinism, graph replay and the warm-up.""" + +import pytest +import torch +from amd_kernels.gemm.mxfp8_common import gemm as g, reference as ref +from amd_kernels.gemm.mxfp8_n6144_k4096 import host as H + +from ..mxfp8_common.test_gemm import make_inputs, require_gfx950 + +N, K = 6144, 4096 +MAX_TOKENS = 32768 +# vLLM's capture sizes up to max_cudagraph_capture_size 2048 (decode: 8 rows per request with DSpark 7) +CAPTURE = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 2049, 16)) + + +def _configs(m): + return (g._select_config(m, N, K, torch.bfloat16), g._select_config(m, N, K, torch.bfloat16, False), + g._select_prequant_config(m, N, K, torch.bfloat16)) + + +def _edges(): + """The first and last row of every window of the three entries' rows.""" + edges, previous = {1, MAX_TOKENS}, None + for m in range(1, MAX_TOKENS + 1): + configs = _configs(m) + if configs != previous and previous is not None: + edges |= {m - 1, m} + previous = configs + return sorted(edges) + + +EDGES = _edges() + + +def test_host_owns_every_row(): + for m in range(1, MAX_TOKENS + 1): + linear, plain, prequant = _configs(m) + assert H.select(m, torch.bfloat16, True) == linear and H.select(m, torch.bfloat16, False) == plain + assert plain.kernel not in g._SHUFFLED_A_KERNELS + if linear.kernel not in g._SHUFFLED_A_KERNELS: + assert linear == plain + tile = g.prequant_code_tile_rows(m, N, K) + assert tile == (max(16, prequant.tile_m) if prequant.kernel in g._SHUFFLED_A_KERNELS else 0) + + +def _spread(x): + m = x.shape[0] + exponents = torch.randint(-5, 6, (m, K // 32), device="cuda") + return (x.float().view(m, -1, 32) * torch.exp2(exponents.float())[..., None]).view_as(x).bfloat16() + + +def _check(got, expected): + metrics = ref.compare(got, expected) + assert metrics["cos"] >= ref.COS_MIN and metrics["rel_l2"] <= ref.REL_L2_MAX, metrics + row_g, row_r = got.float(), expected.float() + valid = row_r.norm(dim=1) > 0 + assert torch.nn.functional.cosine_similarity(row_g[valid], row_r[valid]).min() >= ref.COS_MIN + assert ((row_g - row_r).norm(dim=1) / row_r.norm(dim=1).clamp_min(1e-30)).max() <= ref.REL_L2_MAX + + +def _entries(x, w, s, m): + tile = g.prequant_code_tile_rows(m, N, K) + codes = g.mxfp8_e4m3_quantize_shuffled(x, tile) if tile else g.mxfp8_e4m3_quantize_aiter(x) + plain = g.mxfp8_e4m3_quantize_aiter(x) + return { + "linear": lambda: g.mxfp8_linear(x, w, s), + "prequant": lambda: g.mxfp8_gemm_prequant(*codes, w, s, torch.bfloat16, m), + "gemm": lambda: g.mxfp8_gemm(*plain, w, s, torch.bfloat16, activation_scale_preshuffled=True), + } + + +def _run_entries(m, spread): + x, wc, ws, (w, s) = make_inputs(m, N, K, seed=m) + if spread: + x = _spread(x) + if m > 1: + x[m // 2] = 0 + expected = ref.mxfp8_gemm_reference(*ref.quantize_mxfp8(x), wc, ws, accumulate=torch.float64) + entries = _entries(x, w, s, m) + outs = {} + for name, call in entries.items(): + got = call() + _check(got, expected) + assert m == 1 or torch.count_nonzero(got[m // 2]) == 0, name + for _ in range(3): + assert torch.equal(got, call()), name + outs[name] = got + linear, plain, prequant = _configs(m) + if linear == plain: + assert torch.equal(outs["linear"], outs["gemm"]) + if linear == prequant: + assert torch.equal(outs["linear"], outs["prequant"]) + if prequant.kernel not in g._SHUFFLED_A_KERNELS: + assert torch.equal(outs["prequant"], outs["gemm"]) + return entries, outs + + +@pytest.mark.parametrize("m", EDGES) +@pytest.mark.parametrize("spread", [False, True]) +@torch.inference_mode() +def test_window_edges(m, spread): + require_gfx950() + _run_entries(m, spread) + + +@pytest.mark.parametrize("m", [m for m in CAPTURE if m not in EDGES]) +@torch.inference_mode() +def test_capture_sizes(m): + require_gfx950() + entries, outs = _run_entries(m, False) + for name, call in entries.items(): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = call() + for _ in range(2): + captured.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(outs[name], captured), name + + +@torch.inference_mode() +def test_warmup_compiles_every_row(monkeypatch): + import flydsl.compiler as flyc + + require_gfx950() + _, _, _, (w, s) = make_inputs(16, N, K) + g._decode_launcher.cache_clear() + g._warmup_weight(w, s, MAX_TOKENS, torch.bfloat16) + calls = [(m, name, call) for m in sorted(set(CAPTURE) | set(EDGES)) + for name, call in _entries(torch.zeros((m, K), dtype=torch.bfloat16, device="cuda"), w, s, m).items()] + compiled = [] + real = flyc.compile + + def counting(*args, **kwargs): + compiled.append(args[0]) + return real(*args, **kwargs) + + monkeypatch.setattr(flyc, "compile", counting) + for m, name, call in calls: + call() + assert not compiled, (m, name, compiled) + torch.cuda.synchronize() diff --git a/tests/gemm/mxfp8_n6144_k4096/test_out_k4096.py b/tests/gemm/mxfp8_n6144_k4096/test_out_k4096.py new file mode 100644 index 0000000..95fc855 --- /dev/null +++ b/tests/gemm/mxfp8_n6144_k4096/test_out_k4096.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Caller-owned o_proj output at K 4096: every row family writes only its [M, 6144] arena view (guard rows around +it), the same bits as an allocated output, eager and in graph replay.""" +import pytest +import torch + +from amd_kernels.gemm.mxfp8_common import gemm as g +from ..mxfp8_common.test_gemm import make_inputs, require_gfx950 + + +@pytest.mark.parametrize("m", [8, 32, 33, 256, 768, 769, 1024, 1025, 1281, 2048, 2049, 3329, 4096, 5121, 6145]) +@torch.inference_mode() +def test_oproj_out_k4096(m): + require_gfx950() + if (6144, 4096) not in getattr(g, "_OPROJ_OUT_SHAPES", ()): + pytest.skip("the dispatcher takes out= only at K 2048") + x, _, _, (w, ws) = make_inputs(m, 6144, 4096, seed=m) + a, s = g.mxfp8_e4m3_quantize_aiter(x) + arena = torch.full((m + 2, 6144), -13, device="cuda", dtype=torch.bfloat16) + out = arena[1:-1] + for run in ( + lambda dst: g.mxfp8_linear(x, w, ws, out=dst), + lambda dst: g.mxfp8_gemm(a, s, w, ws, torch.bfloat16, activation_scale_preshuffled=True, out=dst), + ): + expected = run(None) + out.fill_(float("nan")) + assert run(out) is out + assert torch.equal(out, expected) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run(out) + for _ in range(2): + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(out, expected) + assert (arena[0] == -13).all() and (arena[-1] == -13).all() diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile.py index adb6f62..355bea8 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile.py @@ -42,7 +42,7 @@ # the TP4 one-GPU route (one head per GPU), 8 rows: 1..64 requests ("h1q8", 1, 8, list(range(1, 65)), 64), ] -PREFILL_HEADS = [1] +PREFILL_HEADS = [1, 2] PREFILL_WARMUP_GAP = pytest.mark.xfail( strict=True, @@ -312,7 +312,8 @@ def test_variants_are_few(): for shape in PREFILL_GRID: kernels.update(launched(lambda s=shape: chain.call(*s))) print(f"prefill h{heads} {'table' if table_topk else 'indices'}: {len(kernels)}") - assert len(kernels) <= (34 if table_topk else 7) + # two heads: also the shared-dequant scorer's 256-row tiles + assert len(kernels) <= (34 if table_topk else 7) + (heads == 2) # ---------------------------------------------------------------- rank-owned index cache diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_entry.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_entry.py index 9c844dc..1716472 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_entry.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_entry.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""First-call worker warm-up covers every one-GPU top-16 variant.""" +"""First-call worker warm-up covers every one-GPU top-16 variant at one index head (TP4) and two (TP2).""" import json import os import subprocess @@ -8,7 +8,7 @@ import pytest -def scenario(ql, workers): +def scenario(ql, workers, heads=1): import torch from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up @@ -24,16 +24,16 @@ def spy(*a, **kw): MlirCompiler.compile = spy cache = torch.zeros(2, 128, 72, device='cuda', dtype=torch.uint8) def call(n, ctx): - q = torch.zeros(n * ql, 1, 128, device='cuda', dtype=torch.float8_e4m3fn) + q = torch.zeros(n * ql, heads, 128, device='cuda', dtype=torch.float8_e4m3fn) table = torch.zeros(n, max(1, (ctx + 127) // 128), device='cuda', dtype=torch.int32) seq = torch.full((n,), ql, device='cuda', dtype=torch.int32) - ids = torch.empty(1, n * ql, 16, device='cuda', dtype=torch.int32) + ids = torch.empty(heads, n * ql, 16, device='cuda', dtype=torch.int32) nvfp4_index_score_top16(q, cache, table, seq, 0, 1, ids, query_len=ql, max_seq_len=ctx) call(1, 1024) warm = len(count) late = [] for ctx in (8192, 65536, 196608, 524288, 1048576): - for n in (1, 2, 3, 4, 5, 8, 9, 15, 16, 17, 31, 32, 33, 64, 65, 128, 129, 256): + for n in (1, 2, 3, 4, 5, 8, 9, 12, 13, 15, 16, 17, 31, 32, 33, 64, 65, 128, 129, 256): before = len(count) call(n, ctx) if len(count) != before: @@ -42,12 +42,13 @@ def call(n, ctx): return dict(warm=warm, late=late, reported=warm_up.late_compiles(), warmed=len(warm_up.warmed())) +@pytest.mark.parametrize('heads', [1, 2], ids=['h1', 'h2']) @pytest.mark.parametrize('ql', [4, 8]) @pytest.mark.parametrize('workers', [1, 2], ids=['in_process', 'workers']) -def test_first_call_warmup(ql, workers, tmp_path): +def test_first_call_warmup(ql, workers, heads, tmp_path): from .inputs import require_gfx950 require_gfx950() - proc = subprocess.run([sys.executable, '-m', __name__, str(ql), str(workers)], + proc = subprocess.run([sys.executable, '-m', __name__, str(ql), str(workers), str(heads)], capture_output=True, text=True, timeout=900, env={**os.environ, 'FLYDSL_RUNTIME_ENABLE_CACHE': '1', 'FLYDSL_RUNTIME_CACHE_DIR': str(tmp_path / 'flydsl')}) @@ -60,4 +61,4 @@ def test_first_call_warmup(ql, workers, tmp_path): if __name__ == '__main__': - print(json.dumps(scenario(int(sys.argv[1]), int(sys.argv[2])))) + print(json.dumps(scenario(int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3])))) diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_prefill_entry.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_prefill_entry.py index 43fcea2..b5fef21 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_prefill_entry.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_prefill_entry.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project """The prefill top-16 entry's warm-up (``prefill_top16.warm_up``, as the layer's ``init`` calls it) covers every -variant the served steps reach: no FlyDSL compile afterwards at any row count up to 32768 and context up to 1M.""" +variant the served steps reach at one index head (TP4) and two (TP2): no FlyDSL compile afterwards at any row count up +to 32768 and context up to 1M.""" import json import os import subprocess @@ -9,7 +10,7 @@ import pytest -def scenario(workers): +def scenario(workers, heads=1): import torch from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up @@ -26,20 +27,20 @@ def spy(*a, **kw): cache = torch.zeros(2, 128, 72, device='cuda', dtype=torch.uint8) def call(q_lens, ctx): rows = sum(q_lens) - q = torch.zeros(rows, 1, 128, device='cuda', dtype=torch.float8_e4m3fn) + q = torch.zeros(rows, heads, 128, device='cuda', dtype=torch.float8_e4m3fn) table = torch.zeros(len(q_lens), (ctx + 127) // 128, device='cuda', dtype=torch.int32) cu = torch.tensor([0] + q_lens, device='cuda', dtype=torch.int32).cumsum(0).to(torch.int32) seq = torch.full((len(q_lens),), ctx, device='cuda', dtype=torch.int32) nvp = torch.full((rows,), (ctx + 127) // 128, device='cuda', dtype=torch.int32) - ids = torch.empty(1, rows, 16, device='cuda', dtype=torch.int32) + ids = torch.empty(heads, rows, 16, device='cuda', dtype=torch.int32) prefill_top16.index_prefill_top16(q, cache, table, cu, seq, 0, 1, ids, max_query_len=max(q_lens), max_seq_len=ctx, num_valid_pages=nvp) with torch.inference_mode(): - prefill_top16.warm_up(torch.device('cuda', 0), 1) + prefill_top16.warm_up(torch.device('cuda', 0), heads) warm = len(count) late = [] for ctx in (4096, 8192, 32769, 65537, 131073, 163841, 212993, 262144, 1048576): - for q_lens in ([1], [100], [512], [1000, 3096], [4096], [8192], [16384], [32768]): + for q_lens in ([1], [64], [65], [100], [512], [1000, 3096], [4096], [8192], [16384], [32768]): if sum(q_lens) > ctx or (ctx > 262144 and sum(q_lens) > 16384): continue before = len(count) @@ -50,11 +51,12 @@ def call(q_lens, ctx): return dict(warm=warm, late=late, reported=warm_up.late_compiles(), warmed=len(warm_up.warmed())) +@pytest.mark.parametrize('heads', [1, 2], ids=['h1', 'h2']) @pytest.mark.parametrize('workers', [1, 2], ids=['in_process', 'workers']) -def test_warm_up_covers_served_sizes(workers, tmp_path): +def test_warm_up_covers_served_sizes(workers, heads, tmp_path): from .inputs import require_gfx950 require_gfx950() - proc = subprocess.run([sys.executable, '-m', __name__, str(workers)], + proc = subprocess.run([sys.executable, '-m', __name__, str(workers), str(heads)], capture_output=True, text=True, timeout=900, env={**os.environ, 'FLYDSL_RUNTIME_ENABLE_CACHE': '1', 'FLYDSL_RUNTIME_CACHE_DIR': str(tmp_path / 'flydsl')}) @@ -67,4 +69,4 @@ def test_warm_up_covers_served_sizes(workers, tmp_path): if __name__ == '__main__': - print(json.dumps(scenario(int(sys.argv[1])))) + print(json.dumps(scenario(int(sys.argv[1]), int(sys.argv[2])))) diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_rows_entry.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_rows_entry.py index 27cf99d..1f05aff 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_rows_entry.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_compile_rows_entry.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""First-call warm-up of the score-only entry (``nvfp4_index_score_rows``) covers every one-head scorer variant.""" +"""First-call warm-up of the score-only entry (``nvfp4_index_score_rows``) covers every scorer variant at one index +head (TP4) and two (TP2).""" import json import os import subprocess @@ -8,7 +9,7 @@ import pytest -def scenario(ql): +def scenario(ql, heads=1): import torch from flydsl.compiler.jit_function import MlirCompiler from amd_kernels import warm_up @@ -25,16 +26,16 @@ def spy(*a, **kw): MlirCompiler.compile = spy cache = torch.zeros(2, 128, 72, device='cuda', dtype=torch.uint8) def call(n, ctx): - q = torch.zeros(n * ql, 1, 128, device='cuda', dtype=torch.float8_e4m3fn) + q = torch.zeros(n * ql, heads, 128, device='cuda', dtype=torch.float8_e4m3fn) table = torch.zeros(n, max(1, (ctx + 127) // 128), device='cuda', dtype=torch.int32) seq = torch.full((n,), ql, device='cuda', dtype=torch.int32) - score = torch.empty(1, n * ql, score_width(ctx), device='cuda') + score = torch.empty(heads, n * ql, score_width(ctx), device='cuda') nvfp4_index_score_rows(q, cache, score, table, seq, 0, 1, query_len=ql, max_seq_len=ctx) call(1, 1024) warm = len(count) late = [] for ctx in (8192, 196608, 1048576): - for n in (1, 2, 3, 4, 5, 16, 17, 32, 33, 64, 128, 256): + for n in (1, 2, 3, 4, 5, 12, 13, 16, 17, 32, 33, 64, 128, 256): before = len(count) call(n, ctx) if len(count) != before: @@ -43,11 +44,12 @@ def call(n, ctx): return dict(warm=warm, late=late, reported=warm_up.late_compiles(), warmed=len(warm_up.warmed())) +@pytest.mark.parametrize('heads', [1, 2], ids=['h1', 'h2']) @pytest.mark.parametrize('ql', [4, 8]) -def test_first_call_warmup(ql, tmp_path): +def test_first_call_warmup(ql, heads, tmp_path): from .inputs import require_gfx950 require_gfx950() - proc = subprocess.run([sys.executable, '-m', __name__, str(ql)], + proc = subprocess.run([sys.executable, '-m', __name__, str(ql), str(heads)], capture_output=True, text=True, timeout=900, env={**os.environ, 'FLYDSL_RUNTIME_ENABLE_CACHE': '1', 'FLYDSL_RUNTIME_CACHE_DIR': str(tmp_path / 'flydsl')}) @@ -60,4 +62,4 @@ def test_first_call_warmup(ql, tmp_path): if __name__ == '__main__': - print(json.dumps(scenario(int(sys.argv[1])))) + print(json.dumps(scenario(int(sys.argv[1]), int(sys.argv[2])))) diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_prefill_top16.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_prefill_top16.py index 2f92753..da84c18 100644 --- a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_prefill_top16.py +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_prefill_top16.py @@ -91,11 +91,21 @@ def check_reference(out, q, cache, table, q_lens, seq_lens, pos, nvp, init=INIT, ] -@pytest.mark.parametrize("q_lens,seq_lens", SERVED, ids=[f"{a}-{b}".replace(" ", "") for a, b in SERVED]) +# two heads (TP2): the scorer plans of ``score_prefill.host.H2_MQ1_WORK`` (128-row and 256-row shared tiles, 512-row) +SERVED_H2 = [ + ([64], [64]), + ([2048], [2048]), + ([1024], [33792]), + ([3000, 700], [20000, 30000]), + ([8192], [73728]), +] +CASES = [(1, *c) for c in SERVED] + [(2, *c) for c in SERVED + SERVED_H2] + + +@pytest.mark.parametrize("heads,q_lens,seq_lens", CASES, ids=[f"h{h}-{a}-{b}".replace(" ", "") for h, a, b in CASES]) @torch.inference_mode() -def test_reference(q_lens, seq_lens): +def test_reference(heads, q_lens, seq_lens): require_gfx950() - heads = 1 torch.manual_seed(sum(q_lens) + heads) q, cache, table, cu, seq, pos, nvp = make_case(q_lens, seq_lens, heads) out = entry(q, cache, table, cu, seq, q_lens, seq_lens, nvp) diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_score_top16_h2.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_score_top16_h2.py new file mode 100644 index 0000000..15a736a --- /dev/null +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_score_top16_h2.py @@ -0,0 +1,193 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The decode top-16 at two index heads (TP2: 2 index-q heads per GPU over the replicated index-K head): every +two-head scorer (pinned) writes ``nvfp4_index_score_decode``'s bits below each row's block count, the ids equal +``reference.topk_reference`` on both selectors, graph replay at other lengths (CUDA-graph padding requests of length +0), and the one-head dispatch is the TP4 one.""" + +import pytest +import torch +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import BLK, decode_kv_lens, score_width, topk_reference +from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.score_decode import ( + host, + nvfp4_index_score_decode, + nvfp4_index_score_rows, + nvfp4_index_score_top16, + nvfp4_index_top16_from_scores, +) + +from .inputs import index_cache, queries, require_gfx950 + +MAX_MODEL_LEN = 1 << 20 +HEADS = 2 +SCORERS = ["page", "first_page_xcd", "runs", "chunked", None] + + +def tp4_fast_scorer(reqs, ql, heads): + """The one-head scorer choice of the TP4 dispatch.""" + if heads != 1 or ql not in (4, 8) or not 1 <= reqs <= 16: + return None + if reqs in (1, 3): + return "page" + if reqs == 2: + return "first_page_xcd" if ql == 8 else "first_page" + return "sector" if reqs == 4 and ql == 8 else "scan" + + +def test_one_head_dispatch_unchanged(): + for ql in (1, 2, 4, 8): + for reqs in range(0, 300): + assert host._fast_scorer(reqs, ql, 1) == tp4_fast_scorer(reqs, ql, 1), (reqs, ql) + rows = reqs * ql + assert host._plain_selector(rows, ql, 1) == (ql in (4, 8) and 0 < rows <= 128) + + +def test_two_head_dispatch(): + want = {1: "page", 2: "first_page_xcd", 12: "first_page_xcd", 13: "runs", 64: "runs", 256: "runs", 257: None} + for ql in (4, 8): + for reqs, kind in want.items(): + assert host._fast_scorer(reqs, ql, HEADS) == kind, (reqs, ql) + assert host._fast_scorer(8, 7, HEADS) is None + assert host._plain_selector(256, 8, HEADS) and not host._plain_selector(257, 8, HEADS) + + +def chunked(iq, cache, bt, seq, init, local, ql): + rows = iq.shape[0] + score = torch.full((HEADS, rows, score_width(MAX_MODEL_LEN)), float("nan"), device="cuda") + nvfp4_index_score_decode(iq, cache, score, bt, seq, init, local, query_len=ql, max_seq_len=MAX_MODEL_LEN) + return score + + +def want_ids(score, lens, ql): + kv = torch.cat([decode_kv_lens(L, ql, "cuda") for L in lens]) + return topk_reference(score, (kv.long() + BLK - 1) // BLK, MAX_MODEL_LEN) + + +def check_bits(got, want, lens, ql): + for r, length in enumerate(lens): + nb = (length + BLK - 1) // BLK + a = got[:, r * ql:(r + 1) * ql, :nb].view(torch.int32) + b = want[:, r * ql:(r + 1) * ql, :nb].view(torch.int32) + assert torch.equal(a, b), (r, length) + + +@pytest.fixture +def pin(monkeypatch): + def set_pin(scorer=None, plain=None): + monkeypatch.setattr(host, "PIN_SCORER", scorer) + monkeypatch.setattr(host, "PIN_PLAIN", plain) + + return set_pin + + +@pytest.mark.parametrize("scorer", SCORERS, ids=lambda s: s or "auto") +@pytest.mark.parametrize("ql", [8, 4]) +@pytest.mark.parametrize("reqs", [1, 2, 3, 4, 5, 8, 12, 13, 16, 17, 33, 64, 128]) +def test_scorer_bits_and_ids(scorer, ql, reqs, pin): + require_gfx950() + if scorer == "page" and reqs > 12 or scorer == "first_page_xcd" and reqs > 24: + pytest.skip("a lane holds one strided page-table entry: at 1M, 12 / 24 requests at most") + torch.manual_seed(reqs * 13 + ql) + pages = 600 + cache = index_cache(pages) + bt = torch.randint(0, pages, (reqs, MAX_MODEL_LEN // BLK), dtype=torch.int32, device="cuda") + iq = queries(reqs * ql, HEADS) + for lengths in ([131077, 1, 0, 33000], [7, 129, 212993, 2048], [0, 0, 0, 0], [1048576, 3, 128, 300001]): + lens = (lengths * ((reqs + 3) // 4))[:reqs] + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + for init, local in ((0, 1), (2, 3)): + old = chunked(iq, cache, bt, seq, init, local, ql) + want = want_ids(old, lens, ql) + for plain in (True, False): + pin(scorer, plain) + sc = torch.full_like(old, float("nan")) + ids = torch.full_like(want, -7) + nvfp4_index_score_top16(iq, cache, bt, seq, init, local, ids, query_len=ql, + max_seq_len=MAX_MODEL_LEN, score=sc) + assert torch.equal(ids, want), (lens, init, local, plain) + check_bits(sc, old, lens, ql) + again = torch.full_like(want, -7) + nvfp4_index_score_rows(iq, cache, sc, bt, seq, init, local, query_len=ql, max_seq_len=MAX_MODEL_LEN) + nvfp4_index_top16_from_scores(sc, seq, again, rows=reqs * ql, query_len=ql, max_seq_len=MAX_MODEL_LEN) + assert torch.equal(again, want) + + +@pytest.mark.parametrize("ql", [8, 4]) +@pytest.mark.parametrize("reqs", [1, 2, 3, 4, 8, 16, 24, 48]) +def test_graph_replay_other_lengths(ql, reqs): + require_gfx950() + torch.manual_seed(5) + pages = 2600 + cache = index_cache(pages) + bt = torch.randint(0, pages, (reqs, MAX_MODEL_LEN // BLK), dtype=torch.int32, device="cuda") + iq = queries(reqs * ql, HEADS) + seq = torch.tensor(([40000, 9000, 1, 120000] * ((reqs + 3) // 4))[:reqs], dtype=torch.int32, device="cuda") + score = torch.empty(HEADS, reqs * ql, score_width(MAX_MODEL_LEN), device="cuda") + ids = torch.empty(HEADS, reqs * ql, 16, dtype=torch.int32, device="cuda") + run = lambda: nvfp4_index_score_top16( # noqa: E731 + iq, cache, bt, seq, 0, 1, ids, query_len=ql, max_seq_len=MAX_MODEL_LEN, score=score + ) + run() + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + run() + for lens in ([100000, 50000, 200000, 1000], [0, 0, 0, 0], [7, 129, 1000, 1], [250000, 0, 3, 212993]): + lens = (lens * ((reqs + len(lens) - 1) // len(lens)))[:reqs] + seq.copy_(torch.tensor(lens, dtype=torch.int32)) + iq.view(torch.uint8).copy_(queries(reqs * ql, HEADS).view(torch.uint8)) + cache.random_(0, 256) + score.fill_(float("nan")) + ids.fill_(-7) + g.replay() + torch.cuda.synchronize() + assert torch.equal(ids, want_ids(chunked(iq, cache, bt, seq, 0, 1, ql), lens, ql)), lens + + +@pytest.mark.parametrize("reqs", [2, 24]) +def test_strided_query_takes_the_chunked_scorer(reqs): + """A non-contiguous [rows, 2, 128] query (or score) is scored by ``nvfp4_index_score_decode``: same ids.""" + require_gfx950() + torch.manual_seed(7) + ql = 8 + cache = index_cache(300) + bt = torch.randint(0, 300, (reqs, MAX_MODEL_LEN // BLK), dtype=torch.int32, device="cuda") + wide = queries(reqs * ql, 3) + iq = wide[:, :2] + lens = ([33000, 7, 131077] * reqs)[:reqs] + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + ids = torch.full((HEADS, reqs * ql, 16), -7, dtype=torch.int32, device="cuda") + nvfp4_index_score_top16(iq, cache, bt, seq, 0, 1, ids, query_len=ql, max_seq_len=MAX_MODEL_LEN) + assert torch.equal(ids, want_ids(chunked(iq.contiguous(), cache, bt, seq, 0, 1, ql), lens, ql)) + + +@pytest.mark.parametrize("reqs", [1, 4, 32]) +def test_with_mfma_corunner(reqs): + from tests.mfma_spinner import mfma_spinner + + require_gfx950() + torch.manual_seed(292) + ql = 8 + iq = queries(reqs * ql, HEADS) + cache = index_cache(64) + bt = torch.randint(0, 64, (reqs, 8192), device="cuda", dtype=torch.int32) + lens = ([131077, 7, 0, 300001] * reqs)[:reqs] + seq = torch.tensor(lens, device="cuda", dtype=torch.int32) + old = chunked(iq, cache, bt, seq, 2, 3, ql) + want = want_ids(old, lens, ql) + sc = torch.full_like(old, float("nan")) + ids = torch.full_like(want, -7) + + def run(): + nvfp4_index_score_top16(iq, cache, bt, seq, 2, 3, ids, query_len=ql, max_seq_len=MAX_MODEL_LEN, score=sc) + + run() + spin = mfma_spinner(256) + high = torch.cuda.Stream(priority=-1) + with torch.cuda.stream(high): + spin(100000, high) + for _ in range(20): + run() + torch.cuda.synchronize() + assert torch.equal(ids, want) + check_bits(sc, old, lens, ql) diff --git a/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split2_ranks2.py b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split2_ranks2.py new file mode 100644 index 0000000..6562ec4 --- /dev/null +++ b/tests/indexer/indexer_nvfp4_d128_p128_top16/test_split2_ranks2.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The decode indexer split over two GPUs (``split2.host.IndexSplit2``, TP2): each rank's two heads' top-16 ids bit for +bit the one-GPU ``nvfp4_index_score_top16`` (itself ``reference.topk_reference``), over short / long / mixed contexts, +the init / local sentinels, saturated keys, a NaN query, shared prefixes, repeated calls, graph replay with changed +lengths and the device call counts; ``IndexDecodeTop16`` at two ranks (route, warm-up, no compile after it).""" +import torch + +from ...multi_gpu import run_on_2_gpus +from .inputs import block_table, index_cache, require_gfx950 + +MAX = 1 << 20 + + +def _case(lens, ql, rank, *, saturate=False, nan_q=False, share=False, seed=0): + torch.manual_seed(seed) + table, pages = block_table(lens, share_prefixes=share) + cache = index_cache(pages, saturate=saturate) + wide = torch.zeros(len(lens), MAX // 128, dtype=torch.int32, device="cuda") + wide[:, : table.shape[1]] = table + rows = len(lens) * ql + q4 = (torch.randn(rows, 4, 128, device="cuda") * 0.5).to(torch.float8_e4m3fn) + if nan_q: + q4.view(torch.uint8)[rows // 2, :, 5] = 0x7F + iq = q4[:, 2 * rank : 2 * rank + 2].contiguous() + seq = torch.tensor(lens, dtype=torch.int32, device="cuda") + return iq, cache, wide, seq + + +def _one_gpu(iq, cache, table, seq, init, local, ql): + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.score_decode import nvfp4_index_score_top16 + + want = torch.full((2, iq.shape[0], 16), -7, dtype=torch.int32, device="cuda") + nvfp4_index_score_top16(iq, cache, table, seq, init, local, want, query_len=ql, max_seq_len=MAX) + return want + + +CASES = [ + # (lengths, init, local, options) + ([1], 2, 3, {}), + ([129, 3, 2047], 2, 3, {}), + ([5000], 0, 1, {}), + ([196608 - 977 * r for r in range(5)], 2, 3, {}), + ([100000 - 3001 * r for r in range(17)], 0, 1, {"saturate": True}), + ([65536 + 129 * r for r in range(64)], 2, 3, {}), + ([40000 - 1111 * r for r in range(12)], 2, 3, {"nan_q": True}), + ([30000 - 977 * r for r in range(9)], 2, 3, {"share": True}), + ([1048576, 1, 300001], 1, 1, {}), +] + + +def _ids_body(rank, errors): + import torch.distributed as dist + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import score_width + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.split2 import host + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.split2.host import IndexSplit2 + + dev = torch.device("cuda", rank) + # the served bound, then runs and page ranges forced on every case + for ql, pin in ((8, None), (4, None), (8, 0), (8, 1 << 20)): + host.PIN_PAGES_RUNS = pin + cp = IndexSplit2(rank, dist.group.WORLD, dev, 64 * ql, score_width(MAX), ql) + calls = 0 + for i, (lens, init, local, opt) in enumerate(CASES): + iq, cache, table, seq = _case(lens, ql, rank, seed=100 * ql + i, **opt) + want = _one_gpu(iq, cache, table, seq, init, local, ql) + for it in range(2): + got = torch.full_like(want, -9) + cp.score_topk(iq, cache, table, seq, init, local, got, MAX) + calls += 1 + torch.cuda.synchronize() + if not torch.equal(got, want): + bad = (got != want).any(-1) + errors.append(f"rank {rank} ql {ql} pin {pin} case {i} call {it}: {int(bad.sum())} rows differ, " + f"first {bad.nonzero()[:2].tolist()}") + counts = cp.counts[:2].tolist() + if counts != [calls, calls]: + errors.append(f"rank {rank} ql {ql}: device counts {counts}, {calls} calls") + host.PIN_PAGES_RUNS = None + + +def _replay_body(rank, errors): + import torch.distributed as dist + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.reference import score_width + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.split2.host import IndexSplit2 + + dev = torch.device("cuda", rank) + ql, n = 8, 24 + cp = IndexSplit2(rank, dist.group.WORLD, dev, 32 * ql, score_width(MAX), ql) + iq, cache, table, seq = _case([150000 - 2999 * r for r in range(n)], ql, rank, seed=7) + got = torch.full((2, n * ql, 16), -9, dtype=torch.int32, device="cuda") + cp.score_topk(iq, cache, table, seq, 2, 3, got, MAX) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(3): + cp.score_topk(iq, cache, table, seq, 2, 3, got, MAX) + for pattern in ([32768, 65543, 150001, 1, 0], [0], [129, 3, 1, 0, 140000]): + lens = (pattern * ((n + len(pattern) - 1) // len(pattern)))[:n] + seq.copy_(torch.tensor(lens, dtype=torch.int32)) + got.fill_(-9) + graph.replay() + torch.cuda.synchronize() + want = _one_gpu(iq, cache, table, seq, 2, 3, ql) + if not torch.equal(got, want): + errors.append(f"rank {rank} replay {pattern}: {int((got != want).any(-1).sum())} rows differ") + if cp.counts[:2].tolist() != [10, 10]: + errors.append(f"rank {rank}: device counts {cp.counts[:2].tolist()} after 1 + 3 x 3 calls") + + +def _route_body(rank, errors): + import torch.distributed as dist + from amd_kernels import warm_up + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16 import decode_top16 + from amd_kernels.indexer.indexer_nvfp4_d128_p128_top16.decode_top16 import IndexDecodeTop16 + + dev = torch.device("cuda", rank) + warm_up.set_auto(True) + try: + for ql in (8, 4): + index = IndexDecodeTop16(dev, query_len=ql, max_reqs=64, max_seq_len=MAX, num_index_heads=2, rank=rank, + group=dist.group.WORLD) + if index.split2 is None or index.split is not None: + errors.append(f"rank {rank} ql {ql}: split2 {index.split2} split4 {index.split} ({index.split_off})") + continue + m = decode_top16.SPLIT2_MIN_REQS[ql] + routes = [index.route(m - 1, ql, 2), index.route(m, ql, 2), index.route(m, ql, 2, split=False), + index.route(64, ql, 2), index.route(65, ql, 2)] + if routes != ["one_gpu", "split2", "one_gpu", "split2", "one_gpu"]: + errors.append(f"rank {rank} ql {ql}: routes {routes}") + for n in (m, 33, 64): + lens = [90000 - 1301 * r for r in range(n)] + iq, cache, table, seq = _case(lens, ql, rank, seed=n) + got = torch.full((2, n * ql, 16), -9, dtype=torch.int32, device="cuda") + index(iq, cache, table, seq, 2, 3, got, query_len=ql, max_seq_len=MAX) + torch.cuda.synchronize() + want = _one_gpu(iq, cache, table, seq, 2, 3, ql) + if not torch.equal(got, want): + errors.append(f"rank {rank} ql {ql} n {n}: route ids differ") + if warm_up.late_compiles() != 0: + errors.append(f"rank {rank}: {warm_up.late_compiles()} compiles after the warm-up") + finally: + warm_up.set_auto(False) + + +def test_split2_ids_two_gpus(): + require_gfx950() + run_on_2_gpus(_ids_body) + + +def test_split2_graph_replay_two_gpus(): + require_gfx950() + run_on_2_gpus(_replay_body) + + +def test_index_decode_top16_two_ranks(): + require_gfx950() + run_on_2_gpus(_route_body) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/__init__.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_small.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_small.py new file mode 100644 index 0000000..04ae9e0 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_small.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The small-tile host (shared tiles folded in, and the routed launches alone): its bits next to an MFMA co-runner, and +inline routing equal to the sorted rows.""" +import pytest +import torch +from .test_entry import weights # noqa: F401 +from .workload import make_routing, make_activation, replay_check, small_slots + + +@pytest.mark.parametrize("m", [1, 8, 16, 257, 385, 512]) +def test_next_to_mfma(weights, m): + from tests.mfma_spinner import mfma_spinner + _, w = weights + x = make_activation(m, seed=200 + m) + wt, ids = make_routing(m, seed=200 + m) + fn = lambda: small_slots(x, w, wt, ids) # noqa: E731 + expected = [t.clone() for t in fn()] + torch.cuda.synchronize() + spinner = mfma_spinner(256) + stream = torch.cuda.Stream(priority=-1) + spinner(5_000_000, stream) # 50 ms; the contraction must retain its own bits + out = fn() + torch.cuda.synchronize() + assert all(torch.equal(a, b) for a, b in zip(out, expected)) + + +@pytest.mark.parametrize("m", [96, 257, 321, 512]) +def test_routed_alone_next_to_mfma(weights, m): + """From 96 rows the routed launches alone (the shared expert on the side stream): bits next to an MFMA co-runner, + eager equal to replay.""" + from tests.mfma_spinner import mfma_spinner + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + _, w = weights + xq = mxfp4_quant(make_activation(m, seed=200 + m)) + wt, ids = make_routing(m, seed=200 + m, skewed=True) + eager, replay = replay_check(lambda: [mxfp4_moe_experts(xq, wt, ids, w)]) + assert torch.equal(eager[0], replay[0]) and torch.isfinite(eager[0]).all() + torch.cuda.synchronize() + spinner = mfma_spinner(256) + stream = torch.cuda.Stream(priority=-1) + spinner(5_000_000, stream) + out = mxfp4_moe_experts(xq, wt, ids, w) + torch.cuda.synchronize() + assert torch.equal(out, eager[0]) + + +@pytest.mark.parametrize("m", [1, 2, 3, 5, 8, 11, 12, 13, 15, 16]) +@pytest.mark.parametrize("kind", ["uniform", "hot", "four", "one"]) +def test_inline_sort(weights, m, kind): + _, w = weights + x = make_activation(m, seed=m) + wt, ids = make_routing(m, seed=m, skewed=kind == 'hot') + if kind == 'four': + ids = torch.arange(4, device='cuda', dtype=torch.int32).expand(m, 4).contiguous() + if kind == 'one': + ids.zero_() + expect = small_slots(x, w, wt, ids, id_rows=False, inline=False) + eager, replay = replay_check(lambda: list(small_slots(x, w, wt, ids, id_rows=False, inline=True))) + assert all(torch.equal(a, b) for a, b in zip(eager, replay)) + assert all(torch.equal(a, b) for a, b in zip(eager, expect)) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_variants.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_variants.py new file mode 100644 index 0000000..8d18c60 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_a4w4_variants.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The BM128 variants equal BM256 slot by slot on odd and skewed routing.""" +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4 import a4w4_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_bm128 import a4w4_experts as candidate +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_routing, make_activation, two_launch_slots + + +@pytest.mark.parametrize('m',[513,1024,1537,1792,2048,2560,3072,3583,4096]) +@pytest.mark.parametrize('skewed',[False,True]) +def test_bm128(weights,m,skewed): + _,w=weights + wt,ids=make_routing(m,seed=m,skewed=skewed) + xq=mxfp4_quant(make_activation(m,seed=m)) + expect=a4w4_experts(xq,w.w13,w.w2,w.a4w4_scales,wt,ids) + out=candidate(xq,w.w13,w.w2,w.a4w4_scales,wt,ids) + assert torch.equal(out,expect) + assert torch.equal(out,candidate(xq,w.w13,w.w2,w.a4w4_scales,wt,ids)) + + +@pytest.mark.parametrize("m", [4097, 32768]) +def test_bm256_four_experts_scale_ring(weights, m): + """Stress repeated scale-slot reuse with long expert runs.""" + from .workload import replay_check + + _, w = weights + wt, _ = make_routing(m, seed=m) + ids = torch.tensor([7, 63, 81, 127], device="cuda", dtype=torch.int32).expand(m, 4).contiguous() + xq = mxfp4_quant(make_activation(m, seed=m)) + expect = two_launch_slots(xq, w, wt, ids) + + def run(): + return a4w4_experts(xq, w.w13, w.w2, w.a4w4_scales, wt, ids) + + assert torch.equal(run(), expect) + assert torch.equal(run(), expect) + eager, replayed = replay_check(lambda: [run()]) + assert torch.equal(eager[0], expect) + assert torch.equal(replayed[0], expect) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_compile_entry.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_compile_entry.py new file mode 100644 index 0000000..6df8d19 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_compile_entry.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Count every compiler invocation after the first-call warm-up (amd_kernels.warm_up) in a fresh process.""" +import json +import os +import subprocess +import sys + +import pytest + + +def scenario(workers=None): + """The first call of each mode warms that mode up (amd_kernels.warm_up; ``workers`` 1: compiles in this process, + the BF16 reduced mode only; None: in worker processes, every mode and the preparation); every later call compiles + nothing.""" + import torch + from flydsl.compiler.jit_function import MlirCompiler + from amd_kernels import warm_up + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, mxfp4_moe, prepare_mxfp4_moe + from .workload import H, make_experts, make_activation, make_shared + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + warm_up.set_workers(workers) + torch.set_num_threads(8) + raw, _ = make_experts() + gate = torch.randn(128, H, device='cuda') / H**.5 + bias = torch.zeros(128, device='cuda') + w = MoeWeights.from_row_major(gate, bias, *raw, shared=make_shared()[1]) + count = [] + original = MlirCompiler.compile + + def spy(self, *args, **kw): + # an entry's compiles only (its warm-up or its call); the caller's own ops (p.compact()) are not entries + if warm_up._in_warm_up.get() or warm_up._current.get() is not None: + count.append(1) + return original(self, *args, **kw) + + MlirCompiler.compile = spy + calls = [lambda x, p: mxfp4_moe(x, w), lambda x, p: mxfp4_moe(x, w, return_partials=True), + lambda x, p: mxfp4_moe(p, w), lambda x, p: mxfp4_moe(p, w, return_partials=True), + lambda x, p: mxfp4_moe(p.compact(), w), lambda x, p: mxfp4_moe(p.compact(), w, return_partials=True)] + prepare = prepare_mxfp4_moe + if workers == 1: # one mode compiles in this process in minutes; the worker path takes them all + calls, prepare = calls[:1], lambda x, w: None + x = make_activation(1, seed=1) + p = prepare(x, w) # the preparation's first call + for c in calls: # each mode's first call + c(x, p) + torch.cuda.synchronize() + warm = len(count) + late = [] + for m in [1, 9, 15, 16, 18, 26, 27, 32, 40, 42, 63, 66, 80, 90, 127, 130, 159, 160, 191, 192, 194, 230, 255, 258, 384, 385, + 511, 1024, 1536, 2047, 2049, 2560, 3072, 3583, 3584, 3585, 3840, 3841, 4095, 4096, 4097, 7425, 7681, 8191, 8192, 8193, 16384, 24577, 32769, + 2689, 5377, 10881, 16385, 21761]: + before = len(count) + x = make_activation(m, seed=m) + p = prepare(x, w) + for c in calls: + c(x, p) + if len(count) > before: + late.append(m) + torch.cuda.synchronize() + return {'warm_compiles': warm, 'late': late, 'warmed': len(warm_up.warmed()), + 'late_reported': warm_up.late_compiles()} + + +# both compile paths: in this process, in worker processes +@pytest.mark.parametrize("workers", [1, None], ids=["in_process", "workers"]) +def test_first_call_warm_up_compiles_every_variant(workers, tmp_path): + import torch + if not torch.cuda.is_available(): + pytest.skip('needs a GPU') + # the serving default: FlyDSL's artifact cache on (the warm-up compiles into it), here in an empty directory + proc = subprocess.run([sys.executable, '-m', __name__, str(workers or 0)], capture_output=True, + text=True, timeout=900, env={**os.environ, 'FLYDSL_RUNTIME_ENABLE_CACHE': '1', + 'FLYDSL_RUNTIME_CACHE_DIR': str(tmp_path / 'flydsl')}) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + print(result) + # with workers every compile replays in a worker: one left here is a launcher without @launcher + assert result['warm_compiles'] == 0 if workers is None else result['warm_compiles'] > 0 + assert any((tmp_path / 'flydsl').iterdir()) # the warm-up compiled into the cache + assert not result['late'] + assert result['late_reported'] == 0 + assert result['warmed'] == (1 if workers == 1 else 7) # the preparation + one key per mxfp4_moe mode + + +if __name__ == '__main__': + print(json.dumps(scenario(int(sys.argv[1]) or None))) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_entry.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_entry.py new file mode 100644 index 0000000..c8f356f --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_entry.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The entry at every M: MXFP4 x MXFP4 (one activation quant rule), per-slot stores, the shared expert's slots.""" + +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, mxfp4_moe +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import _prepare, _route_quant_small, route, variant_for +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import a16w4_moe_ref, w4a4_moe_ref + +from .workload import (H, make_activation, make_experts, make_routing, make_shared, replay_check, require_gfx950, + small_slots, two_launch_slots) + + +@pytest.fixture(scope="module") +def weights(): + require_gfx950() + raw, _ = make_experts() + gen = torch.Generator(device="cuda").manual_seed(76) + gate = torch.randn((128, H), generator=gen, device="cuda") / H**0.5 + bias = torch.randn((128,), generator=gen, device="cuda") * 0.01 + return raw, MoeWeights.from_row_major(gate, bias, *raw, shared=make_shared()[1]) + + +def _routing(m, routing): + wt, ids = make_routing(m, seed=m, skewed=routing == "hot") + if routing == "four": + ids = torch.arange(4, device="cuda", dtype=torch.int32).expand(m, 4).contiguous() + if routing == "one-hot": + # A single expert receives every token; the remaining slots stay distinct. + ids = ids % 127 + 1 + ids[:, 0] = 0 + return wt, ids + + +def _w4a4_ref(x, raw, wt, ids): + q, s = mxfp4_quant(x) + return w4a4_moe_ref((q, s.view(torch.uint8)), *raw, wt, ids) + + +@pytest.mark.parametrize("m", [1, 2, 5, 8, 15, 16, 17, 32, 63, 128, 129, 191, 192, 207, 255, 256, 257, 300, 384, 385, 448, 511, 512]) +@pytest.mark.parametrize("routing", ["uniform", "hot", "four", "one-hot"]) +def test_small_m_w4a4_contract(weights, m, routing): + """To 512 tokens the folded routed slots (the entry's below 96 rows) against the FP32 reference on the same MXFP4 + operands, deterministic and finite.""" + raw, w = weights + x = make_activation(m, seed=m) + wt, ids = _routing(m, routing) + parts, _ = small_slots(x, w, wt, ids) + expect = _w4a4_ref(x, raw, wt, ids) + assert torch.isfinite(parts).all() + for _ in range(2): + assert torch.equal(parts, small_slots(x, w, wt, ids)[0]) + got = sum(parts[:, k].float() for k in range(4)) + cos = torch.nn.functional.cosine_similarity(got, expect, dim=1) + assert cos.min() >= 0.999, cos.min().item() + + +@pytest.mark.parametrize("m", [96, 97, 128, 191, 256, 257, 300, 336, 337, 384, 448, 511, 512]) +@pytest.mark.parametrize("routing", ["uniform", "hot", "one-hot"]) +def test_small_routed_alone(weights, m, routing): + """From 96 rows the small-tile host's routed launches run without the shared tiles: their slots equal the folded + launches' bit for bit (the same routed tile body), deterministic, against the FP32 reference.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_small import small_config_for + raw, w = weights + x = make_activation(m, seed=m) + wt, ids = _routing(m, routing) + xq = mxfp4_quant(x) + parts = mxfp4_moe_experts(xq, wt, ids, w) + assert torch.equal(parts, small_slots(x, w, wt, ids, bm=small_config_for(m)[1], id_rows=False)[0]) + assert torch.equal(parts, mxfp4_moe_experts(xq, wt, ids, w)) + got = sum(parts[:, k].float() for k in range(4)) + cos = torch.nn.functional.cosine_similarity(got, _w4a4_ref(x, raw, wt, ids), dim=1) + assert cos.min() >= 0.999, cos.min().item() + + +@pytest.mark.parametrize("m", [1, 8, 32, 128, 256, 512]) +def test_small_m_against_bf16_activations(weights, m): + """Against BF16 activations the entry is as close as the ideal W4A4 emulation of its quant rule (minus 0.001).""" + raw, w = weights + x = make_activation(m, seed=m) + wt, ids = _routing(m, "uniform") + hi = a16w4_moe_ref(x, *raw, wt, ids).float() + parts, _ = small_slots(x, w, wt, ids) + got = sum(parts[:, k].float() for k in range(4)) + ideal = _w4a4_ref(x, raw, wt, ids) + cos = torch.nn.functional.cosine_similarity(got.flatten(), hi.flatten(), dim=0) + cos_ideal = torch.nn.functional.cosine_similarity(ideal.flatten(), hi.flatten(), dim=0) + assert cos >= cos_ideal - 0.001, (cos.item(), cos_ideal.item()) + + +def test_one_quant_rule_at_every_m(weights): + """The entry's MXFP4 codes and scales of the same rows are bit-equal whichever M (and so router / quant kernel) + they arrive in; a token's routed slots agree across M to cos >= 0.9999.""" + _, w = weights + full = make_activation(4097, seed=4097) + q_ref, s_ref = mxfp4_quant(full) + parts = {} + for m in [1, 8, 64, 65, 512, 513, 1025, 2048, 2049, 4097]: + x = full[:m].contiguous() + q, s = _prepare(x, w)[0] if m >= 1025 else _route_quant_small(x, w)[0] + assert torch.equal(q, q_ref[:m]), m + assert torch.equal(s.view(torch.uint8), s_ref[:m].view(torch.uint8)), m + parts[m] = mxfp4_moe(x, w, return_partials=True)[0][:1].float() + for m, p in parts.items(): + cos = torch.nn.functional.cosine_similarity(p.flatten(), parts[1].flatten(), dim=0) + assert cos >= 0.9999, (m, cos.item()) + + +@pytest.mark.parametrize("m", [8, 16, 17, 128, 256, 257, 385, 448, 512, 640, 1024, 2048, 4096, 8192, 16384, 32768]) +def test_entry_routing_and_replay(weights, m): + _, w = weights + x = make_activation(m, seed=m, scale=1.0) + wt, ids = route(x, w) + scores = (x.double() @ w.router_weight.double().t()).sigmoid() + expect_ids = (scores + w.correction_bias.double()).topk(4, dim=1).indices + assert torch.equal(ids, expect_ids.int()) + expect_wt = scores.gather(1, expect_ids) + expect_wt = expect_wt / expect_wt.sum(1, keepdim=True) * 2 + torch.testing.assert_close(wt.double(), expect_wt, atol=1e-3, rtol=0) + for partial in [False, True]: + eager, replayed = replay_check(lambda: list(mxfp4_moe(x, w, return_partials=True)) if partial + else [mxfp4_moe(x, w)]) + assert all(torch.equal(a, b) for a, b in zip(eager, replayed)) + + +@pytest.mark.parametrize("m", [96, 320, 321, 512, 513, 577, 640, 641]) +def test_entry_contract_side_shared(weights, m): + """96-640 rows (the routed launches alone next to the side-stream shared expert) and the BM128 launch from 641: + the routed slots against the FP32 reference on the entry's own routing and MXFP4 operands, the shared output + against ``shared_mxfp8_ref``.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import shared_mxfp8_ref + from .workload import make_shared + raw, w = weights + x = make_activation(m, seed=7 * m) + parts, shared = mxfp4_moe(x, w, return_partials=True) + wt, ids = route(x, w) + got = sum(parts[:, k].float() for k in range(4)) + ref = _w4a4_ref(x, raw, wt, ids) + assert ((got - ref).norm() / ref.norm()).item() < 4e-3 + sref = shared_mxfp8_ref(x[:64], *make_shared()[0]) + assert torch.nn.functional.cosine_similarity(shared[:64].float(), sref, dim=1).min() >= 0.999 + + +def test_dispatch(): + """W4A4 at every M: no a16w4 / a8w4 range is served.""" + assert [variant_for(m) for m in [1, 8, 191, 192, 257, 384, 512, 513, 640, 641, 1024, 2048, 4096, 4097, 32768]] == [ + "a4w4_small"] * 9 + ["a4w4_bm128"] * 2 + ["a4w4_bm128_router_quant"] * 2 + ["a4w4_router_quant"] * 2 + for m in [0, -1, 87382]: + with pytest.raises(ValueError): + variant_for(m) + + + +@pytest.mark.parametrize("m", [513, 1024, 2049, 4096, 8192]) +def test_a4w4_entry_partials_equal_two_launch(weights, m): + """The BM128 and one-launch BM256 hosts' slots equal the BM256 host's two launches bit for bit.""" + _, w = weights + x = make_activation(m, seed=m) + wt, ids = make_routing(m, seed=m, skewed=True) + xq = mxfp4_quant(x) + assert torch.equal(two_launch_slots(xq, w, wt, ids), mxfp4_moe_experts(xq, wt, ids, w)) + + +@pytest.mark.parametrize("m", [513, 4096, 8193, 16384]) +def test_hosts_against_reference(weights, m): + """The BM128 and BM256 hosts' slots (the oracle of the one-launch and short-tile tests) against the FP32 reference + on the same MXFP4 operands: relative L2 of the slot sum < 4e-3 (README contract).""" + raw, w = weights + x = make_activation(m, seed=m) + wt, ids = make_routing(m, seed=m, skewed=True) + parts = mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + got = sum(parts[:, k].float() for k in range(4)) + ref = _w4a4_ref(x, raw, wt, ids) + assert ((got - ref).norm() / ref.norm()).item() < 4e-3 + + +@pytest.mark.parametrize("m", [1, 3, 5, 8, 12, 15, 17, 95, 96, 191, 192, 257, 320, 321, 384, 385, 448, 480, 512, 513, 577, 640, 641, 1024, 1536, 1537, 1792, 2048, 2560, 3072, 3583, 3584, 4096, 4097]) +def test_output_and_scratch_guards(weights, monkeypatch, m): + """Every Python-allocated scratch / output has 128-byte guards on both sides; NaN-poisoned outputs also catch the + buffer-resource OOB stores the hardware drops instead of writing a guard.""" + import math + _, w = weights + x = make_activation(m, seed=m) + original = torch.empty + guarded = [] + + def allocate(*size, **kwargs): + dims = tuple(size[0]) if isinstance(size[0], (tuple, list)) else size + dtype = kwargs.get("dtype", torch.float32) + guard = 128 // torch.tensor([], dtype=dtype).element_size() + storage = original((math.prod(dims) + 2 * guard,), **kwargs) + storage.fill_(17) + result = storage[guard:-guard].view(dims) + if dtype in (torch.bfloat16, torch.float32): + result.fill_(float("nan")) + guarded.append((storage, guard)) + return result + + monkeypatch.setattr(torch, "empty", allocate) + for partial in (False, True): + out = mxfp4_moe(x, w, return_partials=True) if partial else (mxfp4_moe(x, w),) + assert all(torch.isfinite(t).all() for t in out) + torch.cuda.synchronize() + assert guarded + for storage, guard in guarded: + assert (storage[:guard] == 17).all() + assert (storage[-guard:] == 17).all() + + +@pytest.mark.parametrize("m", [1, 8, 192, 257, 384, 512]) +def test_all_slots_on_one_expert(weights, m): + raw, w = weights + x = make_activation(m, seed=m) + ids = torch.zeros((m, 4), device="cuda", dtype=torch.int32) + wt = torch.full((m, 4), 0.5, device="cuda") + parts, _ = small_slots(x, w, wt, ids) + ref = _w4a4_ref(x, raw, wt, ids) + cosine = torch.nn.functional.cosine_similarity(sum(parts[:, k].float() for k in range(4)), ref, dim=1) + assert cosine.min() >= 0.999 + assert torch.equal(parts, small_slots(x, w, wt, ids)[0]) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gather_ranks2.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gather_ranks2.py new file mode 100644 index 0000000..7248009 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gather_ranks2.py @@ -0,0 +1,117 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two ranks (TP2 sequence-parallel prefill): each prepares its shard, ``gather_mxfp4_moe`` gathers the compact payload +through a two-rank communicator (here RCCL's all-gather behind the ``all_gather(outs, ins)`` contract; to 16384 rows per +rank), and the entry on the gathered input equals the entry on the rank-ordered concatenation of both shards' payloads, +eager and replayed.""" +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +ROWS = (1, 129, 513, 2049, 4096, 8192, 16384) + + +class _RcclGather: + """``AllGatherComm``'s contract: every output concatenates the ranks' inputs along the first axis.""" + + def __init__(self): + self.group = dist.new_group(backend="nccl") + + def all_gather(self, outs, ins): + for out, x in zip(outs, ins): + dist.all_gather_into_tensor(out, x, group=self.group) + + +def _weights(dev): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, SharedWeights + from .workload import H, I, make_experts + torch.cuda.set_device(dev) + raw, _ = make_experts() + g = torch.Generator(device=dev).manual_seed(607) + + def fp8(n, k): + q = (torch.randn((n, k), device=dev, generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + return q, torch.randint(118, 124, (n, k // 32), device=dev, dtype=torch.uint8, generator=g) + + shared = SharedWeights.from_row_major(*fp8(2 * I, H), *fp8(H, I)) + gate = torch.randn((128, H), device=dev, generator=g) / H**.5 + bias = torch.randn((128,), device=dev, generator=g) * .01 + return MoeWeights.from_row_major(gate, bias, *raw, shared=shared) + + +def _worker(rank, errors): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import (MoeInputCompact, gather_mxfp4_moe, mxfp4_moe, + prepare_mxfp4_moe) + from .workload import make_activation + dev = torch.device("cuda", rank) + w = _weights(dev) + comm = _RcclGather() + for rows in ROWS: + shards = [make_activation(rows, seed=100 * rows + r) for r in range(2)] + local = prepare_mxfp4_moe(shards[rank], w) + payloads = [prepare_mxfp4_moe(x, w).compact() for x in shards] + expected = MoeInputCompact(*(torch.cat(ts) for ts in zip(*(p.tensors() for p in payloads)))) + gathered = gather_mxfp4_moe(local, comm, world=2) + torch.cuda.synchronize() + if not all(torch.equal(a.view(torch.uint8), b.view(torch.uint8)) + for a, b in zip(gathered.tensors(), expected.tensors())): + errors.append(f"rank {rank} rows {rows}: gathered payload differs") + want = mxfp4_moe(expected, w, return_partials=True) + got = mxfp4_moe(gathered, w, return_partials=True) + if not all(torch.equal(a, b) for a, b in zip(got, want)): + errors.append(f"rank {rank} rows {rows}: entry on the gathered payload differs") + reduced = mxfp4_moe(gathered, w) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + replayed = mxfp4_moe(gathered, w) + replayed.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + if not torch.equal(replayed, reduced) or not torch.equal(reduced, mxfp4_moe(expected, w)): + errors.append(f"rank {rank} rows {rows}: replay differs") + del graph + try: + gather_mxfp4_moe(prepare_mxfp4_moe(make_activation(16385, seed=1), w), comm, world=2) + errors.append("16385 rows per rank accepted") + except ValueError: + pass + + +def test_prepared_gather_two_ranks(): + run_on_2_gpus(_worker, timeout=1800) + + +def _routed_worker(rank, errors): + """``routed_only``: the four routed fields (3296 B/row) through the pull ``AllGatherComm``; the routed partials of + the gathered input equal those of the rank-ordered concatenation.""" + from dataclasses import replace + + from amd_kernels.collectives.pull_collectives_ranks2 import AllGatherComm + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import (MoeInput, SharedWeights, gather_mxfp4_moe, mxfp4_moe, + prepare_mxfp4_moe) + from .workload import make_activation, make_full_shared + dev = torch.device("cuda", rank) + w = _weights(dev) + w = replace(w, shared_weights=SharedWeights.from_row_major(*make_full_shared()[0], tp_rank=rank)) + comm = AllGatherComm(rank, dist.group.WORLD, dev, 16384 * 3296 + 4 * 256) + comm.warm_up() + for rows in (1, 129, 1024, 2049, 8192, 16384): + shards = [make_activation(rows, seed=100 * rows + r) for r in range(2)] + local = prepare_mxfp4_moe(shards[rank], w) + payloads = [prepare_mxfp4_moe(x, w).routed() for x in shards] + expected = MoeInput(*(torch.cat(ts) for ts in zip(*(p.tensors()[:4] for p in payloads)))) + gathered = gather_mxfp4_moe(local, comm, world=2, routed_only=True) + torch.cuda.synchronize() + if not gathered.routed_only or not all(torch.equal(a.view(torch.uint8), b.view(torch.uint8)) + for a, b in zip(gathered.tensors()[:4], expected.tensors()[:4])): + errors.append(f"rank {rank} rows {rows}: gathered routed payload differs") + got, none = mxfp4_moe(gathered, w, return_partials=True) + want, _ = mxfp4_moe(expected, w, return_partials=True) + torch.cuda.synchronize() + if none is not None or not torch.equal(got, want): + errors.append(f"rank {rank} rows {rows}: routed partials of the gathered payload differ") + + +def test_routed_gather_two_ranks(): + run_on_2_gpus(_routed_worker, timeout=1800) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm12_a4w4.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm12_a4w4.py new file mode 100644 index 0000000..3acbfdb --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm12_a4w4.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""W4A4 GEMM1 + GEMM2 in one launch (``gemm12_a4w4``, 128- and 256-row tiles): the weighted slots equal the two-launch +chain bit for bit on every routing, the m-block counters are zero after every call, a graph replays changed inputs.""" + +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4 as a2 +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4_bm128 as aw +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + +from .workload import H, make_activation, make_experts, make_routing, require_gfx950, two_launch_slots + +LO, HI = aw.GEMM12_TOKENS +LO2, HI2 = a2.GEMM12_TOKENS +MS = sorted({LO, LO + 1, LO + 63, aw.GEMM12_ADAPTIVE_MAX, aw.GEMM12_ADAPTIVE_MAX + 1, 1024, 1407, 1408, + 1536, 1537, 2047, 2048, 2049, 2311, 3583, 3584, 3585, 3841, 3968, HI - 1, HI}) +MS2 = sorted({LO2, LO2 + 1, 5000, 7424, 7425, 7680, 8191, HI2}) + + +@pytest.fixture(scope="module") +def weights(): + require_gfx950() + raw, _ = make_experts() + gen = torch.Generator(device="cuda").manual_seed(77) + gate = torch.randn((128, H), generator=gen, device="cuda") / H**0.5 + bias = torch.randn((128,), generator=gen, device="cuda") * 0.01 + return MoeWeights.from_row_major(gate, bias, *raw) + + +def _routing(m, routing, seed): + wt, ids = make_routing(m, seed=seed, skewed=routing == "hot") + if routing == "four": + ids = torch.arange(4, device="cuda", dtype=torch.int32).expand(m, 4).contiguous() + if routing == "one-hot": + ids = ids % 127 + 1 + ids[:, 0] = 0 + return wt, ids + + +def _two_launch(x, wt, ids, w): + return two_launch_slots(mxfp4_quant(x), w, wt, ids) + + +def _counters_zero(): + torch.cuda.synchronize() + return all(int(t.count_nonzero()) == 0 for t in aw._ready.values()) + + +def test_range_is_served(weights, monkeypatch): + """The entry takes the one-launch path inside the range and only there.""" + calls = [] + for mod in (aw, a2): + real = mod._gemm12 + monkeypatch.setattr(mod, "_gemm12", lambda *a, _r=real, **k: calls.append(a) or _r(*a, **k)) + for m in (LO, HI, HI + 1, LO2, HI2, HI2 + 1): + x = make_activation(m, seed=m) + wt, ids = _routing(m, "uniform", m) + mxfp4_moe_experts(mxfp4_quant(x), wt, ids, weights) + assert len(calls) == 4 + (HI + 1 == LO2) + + +@pytest.mark.parametrize("m", MS + MS2) +@pytest.mark.parametrize("routing", ["uniform", "hot", "four", "one-hot"]) +def test_gemm12_a4w4_equals_two_launches(weights, m, routing): + w = weights + for seed in (m, m + 1000): # a changed input reuses the reset counters + x = make_activation(m, seed=seed) + wt, ids = _routing(m, routing, seed) + expect = _two_launch(x, wt, ids, w) + assert torch.isfinite(expect.float()).all() + for _ in range(3): + got = mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + assert torch.equal(got, expect) + assert _counters_zero() + + +@pytest.mark.parametrize("m", [LO, aw.GEMM12_ADAPTIVE_MAX + 1, 1024, 1536, 2048, 2049, 3584, HI, LO2, HI2]) +def test_gemm12_a4w4_graph_replay_changed_inputs(weights, m): + """Capture once, replay on inputs written into the captured buffers, the output NaN-poisoned before each replay.""" + w = weights + x = make_activation(m, seed=1) + wt, ids = _routing(m, "uniform", 1) + mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + for seed, routing in [(2, "uniform"), (3, "hot"), (4, "one-hot"), (5, "four"), (6, "uniform")]: + x.copy_(make_activation(m, seed=seed)) + nwt, nids = _routing(m, routing, seed) + wt.copy_(nwt) + ids.copy_(nids) + out.fill_(float("nan")) + graph.replay() + torch.cuda.synchronize() + expect = _two_launch(x, wt, ids, w) + assert torch.equal(out, expect) + assert _counters_zero() + + +@pytest.mark.parametrize("m", [513, 1024, 1536, 2048, 3584, 4095, 4096, 4097, 8192, 16384, 32768]) +def test_gemm12_next_to_mfma(weights, m): + """The state handoff and the epilogues give the two-launch bits next to another MFMA stream.""" + from tests.mfma_spinner import mfma_spinner + x = make_activation(m, seed=m) + wt, ids = _routing(m, "uniform", m) + expect = _two_launch(x, wt, ids, weights) + mxfp4_moe_experts(mxfp4_quant(x), wt, ids, weights) + spin = mfma_spinner(64) + foreign = torch.cuda.Stream(priority=-1) + foreign.wait_stream(torch.cuda.current_stream()) + spin(2_000_000, foreign) + for _ in range(3): + got = mxfp4_moe_experts(mxfp4_quant(x), wt, ids, weights) + assert torch.equal(got, expect) + assert _counters_zero() + foreign.synchronize() diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_adaptive.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_adaptive.py new file mode 100644 index 0000000..efb83ef --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_adaptive.py @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The adaptive GEMM2's column splits follow the actual padded expert blocks, with exact slot ownership.""" + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_activation, make_routing, replay_check, two_launch_slots + + +def routing_with_blocks(m, blocks): + """Expert counts with exactly ``blocks`` padded 128-row tiles; duplicate ids in a token stress slot ownership.""" + active = min(blocks, 128) + counts = [1] * active + counts[0] += (blocks - active) * 128 + capacity = [128] * active + capacity[0] += (blocks - active) * 128 + left = m * 4 - sum(counts) + assert left >= 0 + for e in range(active): + take = min(left, capacity[e] - counts[e]) + counts[e] += take + left -= take + assert left == 0 + assert sum((c + 127) // 128 for c in counts) == blocks + ids = torch.repeat_interleave(torch.arange(active, dtype=torch.int32), torch.tensor(counts)) + return ids.reshape(m, 4).cuda() + + +@pytest.mark.parametrize("m,blocks", [ + (513, 17), (513, 21), (513, 22), (513, 42), (513, 43), (513, 64), + (513, 65), (513, 85), (513, 86), (513, 127), (513, 128), (513, 129), + (3584, 170), (3584, 171), (3584, 192), (3584, 193), +]) +def test_column_split_thresholds(weights, m, blocks): + _, w = weights + x = make_activation(m, seed=m) + wt, _ = make_routing(m, seed=m) + ids = routing_with_blocks(m, blocks) + expect = two_launch_slots(mxfp4_quant(x), w, wt, ids) + run = lambda: mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + got = run() + assert torch.isfinite(got).all() + assert torch.equal(got, expect) + assert torch.equal(got, run()) + eager, replayed = replay_check(lambda: [run()]) + assert torch.equal(eager[0], expect) + assert torch.equal(replayed[0], expect) + + +@pytest.mark.parametrize("m", [513, 640, 641, 1535, 1536, 1537, 3583, 3584, 3585, 4095, 4096]) +def test_adaptive_dispatch_edges(weights, m): + _, w = weights + x = make_activation(m, seed=m) + wt, ids = make_routing(m, seed=m, skewed=True) + expect = two_launch_slots(mxfp4_quant(x), w, wt, ids) + got = mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + assert torch.equal(got, expect) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_bm256_epilogue_groups.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_bm256_epilogue_groups.py new file mode 100644 index 0000000..877c1b6 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_bm256_epilogue_groups.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""BM256 group skips and wave probes retain every valid row across poisoned replays.""" +import math + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4 as aw +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4_bm128 as a128 +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_activation, make_routing, two_launch_slots + + +def _group_ids(m, changed=False): + # Every 16-row boundary of one 256-row tile (15/16/17 ... 255/256/257), then + # second-tile boundaries; the last expert takes the rest (full tiles). + counts = [1] + [b + d for b in range(16, 257, 16) for d in (-1, 0, 1)] + counts += [271, 272, 273, 383, 384, 385, 511, 512, 513] + if changed: + counts.reverse() + counts.append(m * 4 - sum(counts)) + return torch.repeat_interleave(torch.arange(len(counts)), torch.tensor(counts)).int().cuda().view(m, 4) + + +@pytest.mark.parametrize("m", [4097, 6144, 7424, 7425, 8192, 8193, 16384, 32768]) +def test_bm256_changed_guarded_group_boundaries(weights, monkeypatch, m): + """Against the two launches (full-height GEMM1, unconditional GEMM2 epilogue): counts cross every 16-row boundary + and change inside one graph; above 7424 rows this also covers the sixteen-lane padding ballot and the wave-private + LDS write / read order without an explicit write drain, at six and four column chunks.""" + _, w = weights + wt, _ = make_routing(m, seed=m) + ids = _group_ids(m) + xq, xs = mxfp4_quant(make_activation(m, seed=m)) + + def run(): + return aw.a4w4_experts((xq, xs), w.w13, w.w2, w.a4w4_scales, wt, ids) + + def expected(): + return two_launch_slots((xq, xs), w, wt, ids) + + expect = expected() + assert torch.equal(run(), expect) + torch.cuda.synchronize() + original = torch.empty + allocations = [] + + def guarded(*size, **kw): + shape = tuple(size[0]) if isinstance(size[0], (tuple, list)) else size + guard = 128 // torch.tensor([], dtype=kw.get("dtype", torch.float32)).element_size() + storage = original((math.prod(shape) + 2 * guard,), **kw) + value = storage[guard:-guard].view(shape) + allocations.append((storage, value, guard)) + return value + + with monkeypatch.context() as mp: + mp.setattr(torch, "empty", guarded) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + for storage, _, _ in allocations: + storage.view(torch.uint8).fill_(0xA5) + for changed in (False, True, False): + ids.copy_(_group_ids(m, changed)) + q, s = mxfp4_quant(make_activation(m, seed=m + int(changed))) + xq.copy_(q) + xs.copy_(s) + expect = expected() + for _, value, _ in allocations: + value.view(torch.uint8).fill_(0xFF) + for _ in range(2): + graph.replay() + torch.cuda.synchronize() + assert torch.isfinite(out).all() + assert torch.equal(out, expect) + assert all(int(t.count_nonzero()) == 0 for t in a128._ready.values()) + for storage, _, guard in allocations: + assert (storage[:guard].view(torch.uint8) == 0xA5).all() + assert (storage[-guard:].view(torch.uint8) == 0xA5).all() + + +def test_bm256_one_launch_serves_the_group_body_to_7424(): + """The one launch serves the group body at 4097-7424; above it the + half-row body stages output byte offsets.""" + assert aw.GEMM12_EPILOGUE_GROUPS_MAX_TOKENS == 7424 + assert aw.gemm12_for(4097) and aw.gemm12_for(8192) and not aw.gemm12_for(8193) + ns = aw.GEMM2_N_SPLIT + assert aw._gemm12(128, 4, ns, True).kernel_name == ( + f"kernel_gemm12_mxfp4_h6144_i1536_bm256_half_rows_epilogue_groups_ns{ns}") + assert aw._gemm12(128, 4, ns).kernel_name == f"kernel_gemm12_mxfp4_h6144_i1536_bm256_half_rows_ns{ns}" + k2 = aw._gemm2(128, 4, ns) + assert k2.kernel_name == "kernel_gemm2_mxfp4_h6144_i1536_bm256_output_offsets" + + +def test_bm256_host_flag_by_tokens(weights, monkeypatch): + """7424 tokens reach the group body, 7425 the output-offset body.""" + _, w = weights + seen = [] + real = aw._gemm12 + + def spy(*a): + seen.append(a[3] if len(a) > 3 else False) + return real(*a) + + monkeypatch.setattr(aw, "_gemm12", spy) + for m in (7424, 7425): + wt, ids = make_routing(m, seed=m) + xq, xs = mxfp4_quant(make_activation(m, seed=m)) + aw.a4w4_experts((xq, xs), w.w13, w.w2, w.a4w4_scales, wt, ids) + assert seen == [True, False] diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_epilogue_groups.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_epilogue_groups.py new file mode 100644 index 0000000..e1b77b1 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_epilogue_groups.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Wave-private epilogue skips retain every valid row group and CTA barrier.""" +import math + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4_bm128 as aw +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_activation, make_routing, two_launch_slots + + +@pytest.mark.parametrize("m", [513, 640, 704, 705, 1024, 1536, 1537, 2048, 2049, 3072, 3584, 4096]) +def test_changed_guarded_group_boundaries(weights, monkeypatch, m): + """Against the two launches (full-height GEMM1, unconditional GEMM2 epilogue): counts cross every 16-row boundary, + include repeated full tiles and change inside one graph.""" + _, w = weights + wt, _ = make_routing(m, seed=m) + + def routing(changed): + counts = [1] + [b + d for b in range(16, 129, 16) for d in (-1, 0, 1)] + if changed: + counts.reverse() + counts.append(m * 4 - sum(counts)) + return torch.repeat_interleave(torch.arange(len(counts)), torch.tensor(counts)).int().cuda().view(m, 4) + + ids = routing(False) + xq, xs = mxfp4_quant(make_activation(m, seed=m)) + + def run(): + return aw.a4w4_experts((xq, xs), w.w13, w.w2, w.a4w4_scales, wt, ids) + + def expected(): + return two_launch_slots((xq, xs), w, wt, ids) + + expect = expected() + assert torch.equal(run(), expect) + torch.cuda.synchronize() + original = torch.empty + allocations = [] + + def guarded(*size, **kw): + shape = tuple(size[0]) if isinstance(size[0], (tuple, list)) else size + guard = 128 // torch.tensor([], dtype=kw.get("dtype", torch.float32)).element_size() + storage = original((math.prod(shape) + 2 * guard,), **kw) + value = storage[guard:-guard].view(shape) + allocations.append((storage, value, guard)) + return value + + with monkeypatch.context() as mp: + mp.setattr(torch, "empty", guarded) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + for storage, _, _ in allocations: + storage.view(torch.uint8).fill_(0xA5) + for changed in (False, True, False): + ids.copy_(routing(changed)) + q, s = mxfp4_quant(make_activation(m, seed=m + int(changed))) + xq.copy_(q) + xs.copy_(s) + expect = expected() + for _, value, _ in allocations: + value.view(torch.uint8).fill_(0xFF) + for _ in range(3): + graph.replay() + torch.cuda.synchronize() + assert torch.isfinite(out).all() + assert torch.equal(out, expect) + assert all(int(t.count_nonzero()) == 0 for t in aw._ready.values()) + for storage, _, guard in allocations: + assert (storage[:guard].view(torch.uint8) == 0xA5).all() + assert (storage[-guard:].view(torch.uint8) == 0xA5).all() diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_row_probes.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_row_probes.py new file mode 100644 index 0000000..fe37291 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm2_row_probes.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Wave row probes and skipped padding epilogues preserve every slot output.""" + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_activation, make_routing, replay_check, two_launch_slots + + +@pytest.mark.parametrize("m", [1536, 1537, 1600, 1792, 2047, 2048, 2049, 2304, 2560, 2816, 3072, 3328, 3583, 3584, 4095]) +def test_row_group_boundaries(weights, m): + _, w = weights + x = make_activation(m, seed=m) + wt, _ = make_routing(m, seed=m) + counts = [1] + [b + d for b in range(16, 129, 16) for d in (-1, 0, 1)] + counts += [m * 4 - sum(counts)] + ids = torch.repeat_interleave(torch.arange(len(counts), dtype=torch.int32), + torch.tensor(counts)).reshape(m, 4).cuda() + expect = two_launch_slots(mxfp4_quant(x), w, wt, ids) + + def run(): + return mxfp4_moe_experts(mxfp4_quant(x), wt, ids, w) + + assert torch.isfinite(expect).all() + for _ in range(3): + assert torch.equal(run(), expect) + eager, replayed = replay_check(lambda: [run()]) + assert torch.equal(eager[0], expect) + assert torch.equal(replayed[0], expect) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm_half_rows.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm_half_rows.py new file mode 100644 index 0000000..285855a --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_gemm_half_rows.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The BM256 and BM128 short bodies own every valid byte across the half-tile boundary.""" +import math + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4 as aw +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4_bm128 as a128 +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from .test_entry import weights +from .workload import make_activation, make_routing, two_launch_slots + + +def _boundary_ids(m, changed=False): + counts = [1, 31, 32, 63, 64, 65, 127, 128, 129, 255, 256, 257, 383, 384, 385] + if changed: + counts = counts[::-1] + counts.append(m * 4 - sum(counts)) + return torch.repeat_interleave(torch.arange(len(counts)), torch.tensor(counts)).int().cuda().view(m, 4) + + +@pytest.mark.parametrize("m", [4097, 4098, 5000, 7424, 7425, 8191, 8192, 8193, 16384, 24576, 32767, 32768, 65536]) +def test_short_and_full_tiles_changed_poisoned_replay(weights, monkeypatch, m): + """Against the two launches (full-height GEMM1): changed routing crosses 127/128/129 and 255/256/257 rows inside + one graph, a long last expert forces full tiles, repeated claims and XCD tails; the six-chunk body stages output + byte offsets in LDS (above the signed-i32 range at 65536 tokens), the four-chunk body packed ids.""" + _, w = weights + wt, _ = make_routing(m, seed=m) + ids = _boundary_ids(m) + xq, xs = mxfp4_quant(make_activation(m, seed=m)) + # the one-launch range pinned: the test does not follow the host's measured bounds + monkeypatch.setattr(aw, "GEMM12_TOKENS", (4097, 8192)) + + def run(): + return aw.a4w4_experts((xq, xs), w.w13, w.w2, w.a4w4_scales, wt, ids) + + def expected(): + return two_launch_slots((xq, xs), w, wt, ids) + + expect = expected() + assert torch.equal(run(), expect) + torch.cuda.synchronize() + original = torch.empty + allocations = [] + + def guarded_empty(*size, **kw): + shape = tuple(size[0]) if isinstance(size[0], (tuple, list)) else size + dtype = kw.get("dtype", torch.float32) + guard = 128 // torch.tensor([], dtype=dtype).element_size() + storage = original((math.prod(shape) + 2 * guard,), **kw) + result = storage[guard:-guard].view(shape) + allocations.append((storage, result, guard)) + return result + + with monkeypatch.context() as mp: + mp.setattr(torch, "empty", guarded_empty) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + for storage, _, _ in allocations: + storage.view(torch.uint8).fill_(0xA5) + for changed in (False, True, False): + ids.copy_(_boundary_ids(m, changed)) + q, s = mxfp4_quant(make_activation(m, seed=m + int(changed))) + xq.copy_(q) + xs.copy_(s) + expect = expected() + for _, value, _ in allocations: + value.view(torch.uint8).fill_(0xFF) + graph.replay() + torch.cuda.synchronize() + assert torch.isfinite(out).all() + assert torch.equal(out, expect) + assert all(int(t.count_nonzero()) == 0 for t in a128._ready.values()) + for storage, _, guard in allocations: + assert (storage[:guard].view(torch.uint8) == 0xA5).all() + assert (storage[-guard:].view(torch.uint8) == 0xA5).all() + + +def _boundary_ids128(m, changed=False): + counts = [1, 31, 32, 63, 64, 65, 127, 128, 129, 191, 192, 193, 255, 256, 257] + if changed: + counts = counts[::-1] + counts.append(m * 4 - sum(counts)) + return torch.repeat_interleave(torch.arange(len(counts)), torch.tensor(counts)).int().cuda().view(m, 4) + + +@pytest.mark.parametrize("m", [513, 1024, 1407, 1408, 1536, 2048, 2049, 3072, 4095, 4096]) +def test_bm128_short_and_full_tiles_changed_poisoned_replay(weights, monkeypatch, m): + """The 128-row path's 64-row GEMM1 body against the two launches (full-height GEMM1): changed routing crosses + 63/64/65 and 127/128/129 rows inside one graph, every scratch guarded and poisoned.""" + _, w = weights + wt, _ = make_routing(m, seed=m) + ids = _boundary_ids128(m) + xq, xs = mxfp4_quant(make_activation(m, seed=m)) + def run(): + return a128.a4w4_experts((xq, xs), w.w13, w.w2, w.a4w4_scales, wt, ids) + + def expected(): + return two_launch_slots((xq, xs), w, wt, ids) + + expect = expected() + assert torch.equal(run(), expect) + torch.cuda.synchronize() + original = torch.empty + allocations = [] + + def guarded_empty(*size, **kw): + shape = tuple(size[0]) if isinstance(size[0], (tuple, list)) else size + dtype = kw.get("dtype", torch.float32) + guard = 128 // torch.tensor([], dtype=dtype).element_size() + storage = original((math.prod(shape) + 2 * guard,), **kw) + result = storage[guard:-guard].view(shape) + allocations.append((storage, result, guard)) + return result + + with monkeypatch.context() as mp: + mp.setattr(torch, "empty", guarded_empty) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + for storage, _, _ in allocations: + storage.view(torch.uint8).fill_(0xA5) + for changed in (False, True, False): + ids.copy_(_boundary_ids128(m, changed)) + q, s = mxfp4_quant(make_activation(m, seed=m + int(changed))) + xq.copy_(q) + xs.copy_(s) + expect = expected() + for _, value, _ in allocations: + value.view(torch.uint8).fill_(0xFF) + graph.replay() + torch.cuda.synchronize() + assert torch.isfinite(out).all() + assert torch.equal(out, expect) + assert all(int(t.count_nonzero()) == 0 for t in a128._ready.values()) + for storage, _, guard in allocations: + assert (storage[:guard].view(torch.uint8) == 0xA5).all() + assert (storage[-guard:].view(torch.uint8) == 0xA5).all() diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_id_rows_experts.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_id_rows_experts.py new file mode 100644 index 0000000..5af10dd --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_id_rows_experts.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""GEMM1 builds its 16-row blocks from topk_ids (gemm1_a4w4_id_rows): slot bytes equal the sorted path, the published +block map is canonical, the rows run in pair order; under skew, changed routing in replay, poison and MFMA load.""" +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4_small as ex +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from .workload import make_experts, make_activation, make_routing, require_gfx950 +from .test_shared_expert import weights as shared_weights + + +@pytest.fixture(scope='module') +def experts(): + require_gfx950() + return make_experts(seed=9411)[1] + + +def canonical(ids): + """Block -> expert list and the rows (token | slot << 24, padding token M) in pair order.""" + m = ids.shape[0] + values = ids.cpu().view(-1).tolist() + blocks, rows = [], [] + for e in range(128): + pairs = [(q // 4) | ((q % 4) << 24) for q, v in enumerate(values) if v == e] + for b in range((len(pairs) + 15) // 16): + blocks.append(e) + chunk = pairs[b * 16:(b + 1) * 16] + rows.extend(chunk + [m] * (16 - len(chunk))) + return blocks, rows + + +def same(a, b): + a = a if isinstance(a, tuple) else (a,) + b = b if isinstance(b, tuple) else (b,) + return all(torch.equal(x.view(torch.uint8), y.view(torch.uint8)) for x, y in zip(a, b)) + + +def spy_tables(monkeypatch): + """Capture the map and row table the id-rows host allocates (the 2nd/3rd int32 empties).""" + seen = [] + real = torch.empty + + def empty(*shape, **kw): + t = real(*shape, **kw) + if kw.get('dtype') == torch.int32 and len(shape) == 1: + seen.append(t) + return t + monkeypatch.setattr(ex.torch, 'empty', empty) + return seen + + +@pytest.mark.parametrize('m', [16, 17, 24, 32, 33, 48, 57, 58, 64, 65, 66, 72, 80, 81, 88, 95, 96]) +@pytest.mark.parametrize('kind', ['uniform', 'hot', 'four', 'one']) +def test_skew_changed_graph(experts, shared_weights, m, kind, monkeypatch): + assert ex.id_rows_for(m) + w13, w2, scales = experts + x = make_activation(m, seed=9412 + m); xq = mxfp4_quant(x) + wt, ids = make_routing(m, seed=9413 + m, skewed=kind == 'hot') + if kind == 'four': ids.copy_(torch.arange(4, device='cuda', dtype=torch.int32).expand(m, 4)) + elif kind == 'one': ids.fill_(127) + shared = (*shared_quant(x), shared_weights[1]) + args = (xq, w13, w2, scales, wt, ids) + expected = ex.a4w4_experts(*args, inline=False, shared=shared) + + def call(): + with monkeypatch.context() as p: + def fail(*a, **kw): raise AssertionError('id-rows consumers called the standalone sort') + p.setattr(ex, 'moe_sort_a16w4', fail) + seen = spy_tables(p) + out = ex.a4w4_experts(*args, shared=shared, id_rows=True) + return out, seen[0], seen[1] + out0, tile, rows = call() + torch.cuda.synchronize() + assert same(out0, expected) + blocks, rows_ref = canonical(ids) + assert int(tile[-1].item()) == len(blocks) + assert tile[:len(blocks)].cpu().tolist() == blocks + assert rows[:len(rows_ref)].cpu().tolist() == rows_ref + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): out, tile, rows = call() + for change in (False, True): + if change: + ids.copy_(torch.randint(0, 128, (m, 4), device='cuda', dtype=torch.int32)) + ids[:, 1] = ids[:, 0] # duplicate ids within a token + xnew = mxfp4_quant(make_activation(m, seed=9512 + m)) + for dst, src in zip(xq, xnew): dst.copy_(src) + expected = ex.a4w4_experts(*args, inline=False, shared=shared) + blocks, rows_ref = canonical(ids) + for _ in range(3): + tile.fill_(-1); rows.fill_(-7) + for t in (out if isinstance(out, tuple) else (out,)): t.fill_(float('nan')) + graph.replay(); torch.cuda.synchronize() + assert same(out, expected) + assert int(tile[-1].item()) == len(blocks) + assert tile[:len(blocks)].cpu().tolist() == blocks + assert rows[:len(rows_ref)].cpu().tolist() == rows_ref + + +@pytest.mark.parametrize('m', [16, 33, 64, 65, 80, 96]) +def test_next_to_mfma(experts, shared_weights, m): + from tests.mfma_spinner import mfma_spinner + w13, w2, scales = experts + x = make_activation(m, seed=9612 + m); xq = mxfp4_quant(x) + wt, ids = make_routing(m, seed=9613 + m, skewed=True) + shared = (*shared_quant(x), shared_weights[1]) + + def call(): + return ex.a4w4_experts(xq, w13, w2, scales, wt, ids, shared=shared, id_rows=True) + expected = call(); torch.cuda.synchronize() + spinner = mfma_spinner(256); stream = torch.cuda.Stream(priority=-1) + spinner(5_000_000, stream) + actual = call(); torch.cuda.synchronize() + assert same(actual, expected) + + +@pytest.mark.parametrize('m', [65, 80, 96]) +@pytest.mark.parametrize('kind', ['uniform', 'one']) +def test_guarded_row_table_allocations(experts, shared_weights, m, kind, monkeypatch): + """Guard every scratch / output allocation of the 512-slot path, duplicate ids included.""" + import math + w13, w2, scales = experts + x = make_activation(m, seed=9712 + m) + xq = mxfp4_quant(x) + wt, ids = make_routing(m, seed=9713 + m) + if kind == 'one': + ids.fill_(127) + shared = (*shared_quant(x), shared_weights[1]) + args = (xq, w13, w2, scales, wt, ids) + expected = ex.a4w4_experts(*args, inline=False, shared=shared) + real_empty, guards = torch.empty, [] + + def guarded(*shape, **kw): + dims = tuple(shape[0]) if len(shape) == 1 and isinstance(shape[0], (tuple, list, torch.Size)) else shape + dtype = kw.get('dtype', torch.float32) + g = 128 // real_empty(0, dtype=dtype).element_size() + base = real_empty(math.prod(dims) + 2 * g, **kw).fill_(93) + guards.append((base, g)) + return base[g:-g].view(dims) + + with monkeypatch.context() as p: + p.setattr(ex.torch, 'empty', guarded) + actual = ex.a4w4_experts(*args, shared=shared, id_rows=True) + torch.cuda.synchronize() + assert same(actual, expected) + assert guards + for base, g in guards: + assert bool((base[:g] == 93).all()) + assert bool((base[-g:] == 93).all()) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_prepared_input.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_prepared_input.py new file mode 100644 index 0000000..4369eca --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_prepared_input.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SP payloads keep direct BF16 quantization and bypass full-M routing and quant.""" +from dataclasses import replace +import importlib + +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeInput, mxfp4_moe, prepare_mxfp4_moe +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import mxfp4_moe_experts +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert import shared_expert +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import router_line_layout +from .test_shared_expert import weights, complete_weights +from .workload import make_activation, make_routing, replay_check + + +@pytest.mark.parametrize('m', [1,8,256,512,513,1024,1025,2047,2048,2049,2689,4096,4097, + 5377,8192,8193,10881,16384,16385,21761,24577,32768]) +def test_prepared_quantization_and_replay(complete_weights,m): + w=complete_weights;x=make_activation(m,seed=m);x[0,:64]=0 + expected=(*mxfp4_quant(x),*shared_quant(x)) + eager,replay=replay_check(lambda:list(prepare_mxfp4_moe(x,w).tensors())) + assert all(torch.equal(a,b) for a,b in zip(eager,replay)) + q,s,wt,ids,q8,s8=replay + assert all(torch.equal(a,b.view(torch.uint8)) for a,b in zip((q,s,q8,s8),expected)) + assert s8.shape==(m,192) # no per-shard tiled padding in the wire payload + refwt,refids=importlib.import_module( + 'amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain').route(x,w) + assert torch.equal(ids,refids) and torch.equal(wt,refwt) + + +@pytest.mark.parametrize('m',[8,128,256,257,513,1025,2048,4096,8192,16384,32768]) +def test_prepared_public_entry(complete_weights,m,monkeypatch): + w=complete_weights + x=make_activation(m,seed=m) + prepared=prepare_mxfp4_moe(x,w) + expected=list(mxfp4_moe(x,w,return_partials=True)) + # An already-gathered call must never revisit BF16 routing or quantization. + def fail(*args,**kw):raise AssertionError('prepared call rerouted or requantized its input') + monkeypatch.setattr('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain._prepare',fail) + monkeypatch.setattr('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant.mxfp4_quant',fail) + monkeypatch.setattr('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert.shared_quant',fail) + def call(): + return list(mxfp4_moe(prepared,w,return_partials=True)) + eager,replay=replay_check(call) + assert all(torch.equal(a,b) and torch.equal(a,c) for a,b,c in zip(expected,eager,replay)) + acc=torch.zeros_like(expected[0][:,0],dtype=torch.float32) + for k in range(4):acc=acc+expected[0][:,k].float() + acc=acc+expected[1].float() + assert torch.equal(mxfp4_moe(prepared,w),acc.bfloat16()) + + +@pytest.mark.parametrize('rows',[3,129,513,2049,8192]) +def test_four_arbitrary_shards(complete_weights,rows): + """Four shards gathered in rank order on one GPU (odd boundaries, padded tail tokens): the payload layout and its + consumption, not IPC; the experts against the same routing, the routing against FP64.""" + w=complete_weights + gen=torch.Generator(device='cuda').manual_seed(rows) + gate=torch.randn((128,6144),device='cuda',generator=gen)/6144**.5 + w=replace(w,router_weight=gate,router_weight_lines=router_line_layout(gate)) + x=make_activation(rows*4,seed=rows);x[-2:]=0 + local=[prepare_mxfp4_moe(t,w) for t in x.chunk(4)] + payload=MoeInput(*(torch.cat(ts) for ts in zip(*(p.tensors() for p in local)))) + payload.validate(w) + q,s=mxfp4_quant(x);q8,s8=shared_quant(x) + assert all(torch.equal(a,b.view(torch.uint8)) for a,b in zip( + (payload.routed_codes,payload.routed_scales,payload.shared_codes,payload.shared_scales),(q,s,q8,s8))) + scores=(x[:64].double()@gate.double().t()).sigmoid() + ids=(scores+w.correction_bias.double()).topk(4).indices.int() + assert torch.equal(ids,payload.topk_ids[:64]) + wt=scores.gather(1,ids.long());wt=(wt/wt.sum(1,keepdim=True)*2).float() + torch.testing.assert_close(wt,payload.topk_weights[:64],atol=1e-6,rtol=1e-6) + if rows*4<=512: # the shared tiles folded into the small-tile launches + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_small import a4w4_experts + expected,expected_shared=a4w4_experts((q,s),w.w13,w.w2,w.a4w4_scales,payload.topk_weights,payload.topk_ids, + shared=(q8,s8,w.shared_weights)) + else: + expected=mxfp4_moe_experts((q,s),payload.topk_weights,payload.topk_ids,w) + expected_shared=shared_expert(x,w.shared_weights,x_quant=(q8,s8)) + def call():return list(mxfp4_moe(payload,w,return_partials=True)) + eager,replay=replay_check(call) + assert torch.equal(expected,replay[0]) + assert not expected_shared[-2:].any() and not replay[1][-2:].any() + cos=torch.nn.functional.cosine_similarity(expected_shared[:-2].float(),replay[1][:-2].float(),dim=1) + assert cos.min()>=.9999 + assert all(torch.equal(a,b) for a,b in zip(eager,replay)) + + +@pytest.mark.parametrize('routing',['uniform','hot','four']) +def test_prepared_changed_inputs_and_routing(complete_weights,routing): + w=complete_weights;m=2052 + payload=prepare_mxfp4_moe(make_activation(m),w) + mxfp4_moe(payload,w);torch.cuda.synchronize() + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph):out=mxfp4_moe(payload,w) + previous=None + for seed in [67,68,69]: + updated=prepare_mxfp4_moe(make_activation(m,seed=seed),w) + wt,ids=make_routing(m,seed=seed,skewed=routing=='hot') + if routing=='four':ids[:]=torch.arange(4,device='cuda',dtype=torch.int32) + updated=replace(updated,topk_weights=wt,topk_ids=ids) + for dst,src in zip(payload.tensors(),updated.tensors()):dst.copy_(src) + out.fill_(float('nan'));graph.replay();torch.cuda.synchronize() + expected=mxfp4_moe(updated,w) + assert torch.equal(out,expected) and torch.isfinite(out).all() + if previous is not None:assert not torch.equal(out,previous) + previous=out.clone() + + +@pytest.mark.parametrize('field,value',[('routed_codes',None),('routed_scales',None), + ('shared_codes',None),('shared_scales',None),('topk_ids',None)]) +def test_invalid_prepared_payload(complete_weights,field,value): + p=prepare_mxfp4_moe(make_activation(8),complete_weights) + bad=replace(p,**{field:value}) + with pytest.raises(ValueError):mxfp4_moe(bad,complete_weights) + + +@pytest.mark.parametrize('m',[1025,2049,4097,8193]) +def test_row_major_dual_quant_guards(complete_weights,m,monkeypatch): + import math + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import shared_mxfp8_quant_ref,mxfp4_quant_ref + w=complete_weights;x=make_activation(m,seed=m) + x[:16]*=torch.exp2(torch.linspace(-96,96,16,device=x.device))[:,None] + x[0]=0 + reference=(*mxfp4_quant_ref(x),*shared_mxfp8_quant_ref(x)) + original=torch.empty;guards=[] + def empty(*size,**kw): + shape=tuple(size[0]) if isinstance(size[0],(tuple,list,torch.Size)) else tuple(size) + dt=kw.get('dtype',torch.float32);g=128//torch.tensor([],dtype=dt).element_size() + storage=original((math.prod(shape)+2*g,),**kw);storage.fill_(19) + out=storage[g:-g].view(shape);out.fill_(float('nan') if dt.is_floating_point else 255) + guards.append((storage,g));return out + monkeypatch.setattr(torch,'empty',empty) + p=prepare_mxfp4_moe(x,w);torch.cuda.synchronize() + assert all(torch.equal(a,b) for a,b in zip( + (p.routed_codes,p.routed_scales,p.shared_codes,p.shared_scales),reference)) + assert all((t[:g]==19).all() and (t[-g:]==19).all() for t,g in guards) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_quant.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_quant.py new file mode 100644 index 0000000..4b22c80 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_quant.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The chain input's two MXFP4 quantizers against ``reference.mxfp4_quant_ref`` bit for bit: ``quant.mxfp4_quant`` of +bf16 (one binade per 32-block from 2^-20 to 2^20, all-zero blocks, many values on e2m1 midpoints) and +``mxfp4_from_mxfp8`` of a gathered MXFP8 input (every e4m3 code but NaN, scales 2^-17 .. 2^17); three runs bitwise +equal, graph replay into 0xA5-filled outputs.""" + +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.mxfp4_from_mxfp8 import mxfp4_from_mxfp8 +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.prefill_h6144_i1536 import H +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import mxfp4_quant_ref + +from .workload import make_mxfp8, replay_check, require_gfx950 + +SIZES = [1, 300, 4097, 32768] + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def _bf16_blocks(m, gen): + """bf16 [m, H] with a binade per 32-block from 2^-20 to 2^20 and some all-zero + blocks; bf16's 8-bit mantissa puts many values exactly on e2m1 midpoints.""" + x = torch.randn((m, H // 32, 32), generator=gen) + x *= torch.exp2(torch.randint(-20, 21, (m, H // 32, 1), generator=gen).float()) + x[torch.rand((m, H // 32), generator=gen) < 0.02] = 0.0 + return x.view(m, H).bfloat16().cuda() + + +def _check(fn, ref): + q, s = fn() + rq, rs = ref + assert torch.equal(s, rs) + assert torch.equal(q, rq) + for _ in range(2): + q2, s2 = fn() + assert torch.equal(q2, q) and torch.equal(s2, s), "not bitwise reproducible" + eager, replayed = replay_check(fn) + for a, b in zip(eager, replayed): + assert torch.equal(a, b), "graph replay differs" + + +@pytest.mark.parametrize("m", SIZES) +def test_mxfp4_quant_matches_reference(m): + x = _bf16_blocks(m, torch.Generator().manual_seed(m)) + _check(lambda: mxfp4_quant(x), mxfp4_quant_ref(x)) + + +@pytest.mark.parametrize("m", SIZES) +def test_mxfp4_from_mxfp8_matches_reference(m): + codes, scales = make_mxfp8(m, seed=m) + x = codes.view(torch.float8_e4m3fn).float() * torch.exp2(scales.float() - 127.0).repeat_interleave(32, dim=1) + _check(lambda: mxfp4_from_mxfp8(codes, scales), mxfp4_quant_ref(x)) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_line_layout.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_line_layout.py new file mode 100644 index 0000000..b172434 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_line_layout.py @@ -0,0 +1,107 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router-side shared operand matches separate quantization, including padded scales.""" +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_quant_line_layout import router_mxfp4_quant_line_layout +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_mxfp8_quant_line_layout import router_mxfp4_mxfp8_quant_line_layout +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_dense import _shuffle_scales +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.launch import _run_compiled +from .test_entry import weights +from .workload import make_activation + +@pytest.mark.parametrize('m',[769,1024,1025,1536,2047,2048,2049,2688,2689,4096,4097,5376,5377,8192,8193,10880,10881,16384,16385,21760,21761,24576,24577,32768]) +def test_dual_quant_line_layout(weights,m): + _,w=weights;x=make_activation(m,seed=m);x[0,:64]=0 + q4,s4,parts=router_mxfp4_quant_line_layout(x,w.router_weight_lines) + q8,s8=shared_quant(x) + ss=torch.empty(((m+31)//32*32,192),device=x.device,dtype=torch.uint8) + _run_compiled(_shuffle_scales(6144),s8.view(-1),ss.view(-1),m,torch.cuda.current_stream()) + expected=[q4,s4,q8,ss,parts] + def call(): + a,b,p=router_mxfp4_mxfp8_quant_line_layout(x,w.router_weight_lines) + return [*a,*b,p] + actual=call() + assert all(torch.equal(a,b) for a,b in zip(actual,expected)),m + g=torch.cuda.CUDAGraph() + with torch.cuda.graph(g):out=call() + for _ in range(3): + for t in out[:-1]:t.fill_(255) + out[-1].fill_(float('nan')) + g.replay();torch.cuda.synchronize() + assert all(torch.equal(a,b) for a,b in zip(out,expected)),m + +@pytest.mark.parametrize('m',[257,300,513,577,640,1000,1024,2047]) +def test_topk_dual_quant_s8_tiles(weights,m,monkeypatch): + """The fused top-k + dual quant's tiled FP8 scales (padding rows zero) equal its row-major + scales through the dense scale shuffle; everything else bit-equal, no byte outside.""" + import math + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_gemm import router_gemm_partials + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk_mxfp4_mxfp8_quant import splitk_biased_topk_mxfp4_mxfp8_quant + _,w=weights;x=make_activation(m,seed=m);x[0,:64]=0 + x[1:9]*=torch.exp2(torch.linspace(-60,60,8,device=x.device))[:,None] + parts=router_gemm_partials(x,w.router_weight) + def run(tiles): + ids=torch.empty((m,4),dtype=torch.int32,device=x.device);wt=torch.empty((m,4),dtype=torch.float32,device=x.device) + return ids,wt,*splitk_biased_topk_mxfp4_mxfp8_quant(parts,w.correction_bias,wt,ids,w.routed_scaling_factor,x,s8_tiles=tiles) + ids,wt,(q4,s4),(q8,s8)=run(False) + ss=torch.empty(((m+31)//32*32,192),device=x.device,dtype=torch.uint8) + _run_compiled(_shuffle_scales(6144),s8.view(-1),ss.view(-1),m,torch.cuda.current_stream()) + original=torch.empty;guards=[] + def empty(*size,**kw): + shape=tuple(size[0]) if isinstance(size[0],(tuple,list,torch.Size)) else tuple(size) + dt=kw.get('dtype',torch.float32);g=128//torch.tensor([],dtype=dt).element_size() + storage=original((math.prod(shape)+2*g,),**kw);storage.fill_(19) + out=storage[g:-g].view(shape);out.fill_(255 if dt==torch.uint8 else float('nan') if dt.is_floating_point else -7) + guards.append((storage,g));return out + with monkeypatch.context() as mp: + mp.setattr(torch,'empty',empty) + got=run(True) + torch.cuda.synchronize() + i2,w2,(a4,b4),(a8,b8)=got + assert b8.shape==ss.shape + for a,b in zip([i2,w2,a4,b4,a8,b8],[ids,wt,q4,s4,q8,ss]): + assert torch.equal(a,b),m + assert all((t[:g]==19).all() and (t[-g:]==19).all() for t,g in guards) + + +@pytest.mark.parametrize('m',[1025,2049,4097,8193,16385,32769]) +def test_dual_quant_guards_and_wide_exponents(weights,m,monkeypatch): + import math + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import shared_mxfp8_quant_ref + _,w=weights;x=make_activation(m,seed=m) + x[:16]*=torch.exp2(torch.linspace(-96,96,16,device=x.device))[:,None] + x[0]=0 + refq,refs=shared_mxfp8_quant_ref(x) + original=torch.empty;guards=[] + def empty(*size,**kw): + shape=tuple(size[0]) if isinstance(size[0],(tuple,list,torch.Size)) else tuple(size) + dt=kw.get('dtype',torch.float32);g=128//torch.tensor([],dtype=dt).element_size() + storage=original((math.prod(shape)+2*g,),**kw);storage.fill_(19) + out=storage[g:-g].view(shape);out.fill_(255 if dt==torch.uint8 else float('nan')) + guards.append((storage,g));return out + monkeypatch.setattr(torch,'empty',empty) + _,(q,s),_=router_mxfp4_mxfp8_quant_line_layout(x,w.router_weight_lines) + torch.cuda.synchronize() + assert torch.equal(q,refq) + # Independent torch inverse of the dense scale permutation (incl. padding zeros). + row=s.view(-1,24,4,16,2,2).permute(0,5,3,1,4,2).reshape(-1,192) + assert torch.equal(row[:m],refs) and not row[m:].any() + assert all((t[:g]==19).all() and (t[-g:]==19).all() for t,g in guards) + +@pytest.mark.parametrize('m',[1025,2048,4097,8192,32768]) +@pytest.mark.parametrize('shared_scales_tiled',[False,True]) +def test_dual_quant_next_to_mfma(weights,m,shared_scales_tiled): + from tests.mfma_spinner import mfma_spinner + _,w=weights;x=make_activation(m,seed=m) + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_mxfp8_quant_row_scales import router_mxfp4_mxfp8_quant_row_scales + quant = router_mxfp4_mxfp8_quant_line_layout if shared_scales_tiled else router_mxfp4_mxfp8_quant_row_scales + def call(): + a,b,p=quant(x,w.router_weight_lines) + return [*a,*b,p] + ref=[t.clone() for t in call()];torch.cuda.synchronize() + spinner=mfma_spinner(256);stream=torch.cuda.Stream(priority=-1) + spinner(5_000_000,stream) + out=call();torch.cuda.synchronize() + assert all(torch.equal(a,b) for a,b in zip(out,ref)) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_tasks.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_tasks.py new file mode 100644 index 0000000..491e0fb --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_dual_quant_tasks.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Independent top-k/quant tasks retain routing ties, quant bits and replay writes.""" +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk import splitk_biased_topk +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import mxfp4_quant_ref +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk_mxfp4_mxfp8_quant_sorted_tasks import splitk_biased_topk_mxfp4_mxfp8_quant as sorted_tasks +from .workload import require_gfx950, make_activation + + +@pytest.mark.parametrize('m',[1,3,8,12,13,16,17,32,64,65,96,128,192,193,256,257,384,385,512,513,769,1024]) +def test_bits_guards_changed_graph(m,monkeypatch): + require_gfx950() + g=torch.Generator(device='cuda').manual_seed(m) + split=48 if m<=64 else 16 + # Non-dense token/split strides and top-k row strides exercise pointer bounds. + storage=torch.randn(m,split,132,device='cuda',generator=g) + part=storage[:,:,:128] + bias=torch.randn(128,device='cuda',generator=g)*.1 + x=make_activation(m,seed=m) + x[0,:32]=0 + wt=torch.full((m,8),float('nan'),device='cuda')[:,1:5] + ids=torch.full((m,8),-1,device='cuda',dtype=torch.int32)[:,1:5] + original=torch.empty;guards=[] + def alloc(*args,**kwargs): + shape=tuple(args[0]) if isinstance(args[0],(tuple,list)) else tuple(args) + total=1 + for n in shape:total*=n + t=torch.full((total+128,),0xA5,device=kwargs['device'],dtype=kwargs['dtype']) + guards.append(t) + return t[64:-64].view(shape) + def reference(): + splitk_biased_topk(part,bias,wt,ids,2.0) + q,s=mxfp4_quant_ref(x) + q8,s8=shared_quant(x) + return [wt.clone(),ids.clone(),q,s,q8,s8] + def run(fn): + a,b=fn(part,bias,wt,ids,2.,x) + return [wt,ids,*a,*b] + expected=reference() + monkeypatch.setattr(torch,'empty',alloc) + actual=run(sorted_tasks) + assert all(torch.equal(a.view(torch.uint8),b.view(torch.uint8)) for a,b in zip(actual,expected)) + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph):out=run(sorted_tasks) + monkeypatch.setattr(torch,'empty',original) + for change in range(3): + if change: + part.copy_(torch.randn(part.shape,device='cuda',generator=g)) + x.copy_(make_activation(m,seed=m+change)) + if change==2: + part.zero_();bias.zero_() # all experts tied, established strict tree + expected=reference() + for t in out:t.fill_(0 if t.dtype==torch.uint8 else -1) + graph.replay();torch.cuda.synchronize() + assert all(torch.equal(a.view(torch.uint8),b.view(torch.uint8)) for a,b in zip(out,expected)) + q,s=mxfp4_quant_ref(x) + assert torch.equal(out[2],q) and torch.equal(out[3],s) + q8,s8=shared_quant(x) + assert torch.equal(out[4],q8) and torch.equal(out[5],s8) + assert all((t[:64]==0xA5).all() and (t[-64:]==0xA5).all() for t in guards) + assert (wt._base[:,0].isnan()).all() and (wt._base[:,5:].isnan()).all() + assert (ids._base[:,0]==-1).all() and (ids._base[:,5:]==-1).all() diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_dual_quant_tasks.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_dual_quant_tasks.py new file mode 100644 index 0000000..28e761e --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_dual_quant_tasks.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Router/quant placement retains partials, routing, quant bytes and replay writes.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm as rg +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_gemm_topk_mxfp4_mxfp8_quant import router_gemm_topk_mxfp4_mxfp8_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk_mxfp4_mxfp8_quant import splitk_biased_topk_mxfp4_mxfp8_quant +from .workload import make_activation, require_gfx950 + + +@pytest.mark.parametrize("m", [58, 59, 61, 63, 64, 65, 66, 80, 97, 127, 128, 193, 194, 223, 224, 255, 256]) +@pytest.mark.parametrize("case", ["random", "ties", "skew", "no_choices"]) +def test_router_quant_tasks_bytes_and_graph(m, case, monkeypatch): + require_gfx950() + torch.manual_seed(m) + x = make_activation(m, seed=m) + gate = torch.randn((128, 6144), device="cuda") / 6144**0.5 + bias = torch.randn(128, device="cuda") * 0.01 + if case == "ties": + gate.zero_() + bias.zero_() + if case == "skew": + bias.zero_() + bias[[0, 31, 63, 127]] = 100.0 + + if case == "no_choices": + bias.fill_(float("-inf")) + + def reference(): + p = rg.router_gemm_partials(x, gate) + ids = torch.empty((m, 4), device="cuda", dtype=torch.int32) + wt = torch.empty((m, 4), device="cuda") + q4, q8 = splitk_biased_topk_mxfp4_mxfp8_quant(p, bias, wt, ids, 2.0, x) + return [*q4, *q8, wt, ids, p] + + def run(): + q4, q8, wt, ids, p = router_gemm_topk_mxfp4_mxfp8_quant(x, gate, bias, 2.0) + return [*q4, *q8, wt, ids, p] + + expected = reference() + original = torch.empty + guards = [] + + def alloc(*shape, **kw): + shape = tuple(shape[0]) if len(shape) == 1 and isinstance(shape[0], (tuple, list)) else shape + total = 1 + for n in shape: + total *= n + t = torch.full((total + 128,), 7, **kw) + guards.append(t) + return t[64:-64].view(shape) + + monkeypatch.setattr(torch, "empty", alloc) + actual = run() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = run() + monkeypatch.setattr(torch, "empty", original) + assert all(torch.equal(a.view(torch.uint8), b.view(torch.uint8)) for a, b in zip(actual, expected)) + for seed in (m, m + 1, m): + x.copy_(make_activation(m, seed=seed)) + expected = reference() + for t in out: + t.view(torch.uint8).fill_(0xA5) + graph.replay() + assert all(torch.equal(a.view(torch.uint8), b.view(torch.uint8)) for a, b in zip(out, expected)) + assert all((t[:64] == 7).all() and (t[-64:] == 7).all() for t in guards) + + +def test_quant_tiers(): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_topk_mxfp4_mxfp8_quant as q + placement = [q.router_quant_placement(m) for m in range(1, 300)] + assert placement == [0] * 57 + [1] * 7 + [2] * 64 + [0] * 64 + [1] * 64 + [0] * 43 + assert all(rg.nblocks_config(m) is not None for m in range(1, 300) if q.router_dual_quant_for(m)) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks.py new file mode 100644 index 0000000..8e14ca6 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The 16-row MFMA router tiers through ``router_gemm_direct`` (1-16 rows; 1-32 from the line-layout gate copy) and +``router_gemm_nblocks`` (1-384 rows): partials equal ``router_gemm.py``'s kernel bit for bit (strided and contiguous +x), graph replay equals eager, repeated runs equal, guard bytes kept.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm as rg +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_nblocks as rn +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_direct as rd +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import router_line_layout +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import router_gemm_partials_ref +from .workload import make_activation, require_gfx950 + +M = [1, 2, 7, 8, 9, 15, 16, 17, 31, 32, 33, 40, 48, 52, 57, 58, 63, 64, 65, 97, 128, 129, 160, 192, 193, + 255, 256, 257, 320, 384] + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def test_tiers(): + for m in range(1, 2048): + cfg = rg._kernel_config(m) + nb = rg.nblocks_config(m) + if m <= 384: + assert nb is not None and cfg[0] == 16 and cfg[4] == 0 + assert nb[:2] == cfg[1:3] and nb[3] == 1 + assert nb[2] == (2 if m <= rg.NBLOCKS_TWO_WAVES_MAX else cfg[3]) + else: + assert nb is None + + +@pytest.mark.parametrize("m", M) +@pytest.mark.parametrize("strided", [False, True]) +@pytest.mark.parametrize("kernel", ["nblocks", "nblocks_x_direct", "direct", "direct_lines"]) +def test_partials_equal_and_graph(m, strided, kernel): + if kernel == "direct" and m > 16: + pytest.skip("direct kernel: one m-tile") + if kernel == "direct_lines" and m > rg.DIRECT_LINES_MAX: + pytest.skip("line-layout direct kernel: two m-tiles") + if kernel == "nblocks_x_direct" and rg.X_DIRECT_FROM <= m <= rg.X_DIRECT_MAX: + pytest.skip("served there: the nblocks case") + torch.manual_seed(m) + if strided: + source = torch.empty((m, 6146), dtype=torch.bfloat16, device="cuda") + x = source[:, :6144] + else: + x = torch.empty((m, 6144), dtype=torch.bfloat16, device="cuda") + x.copy_(make_activation(m, seed=m)) + w = torch.randn((128, 6144), device="cuda") / 6144**0.5 + lines = router_line_layout(w) + cfg = rg._kernel_config(m) + split = cfg[1] + base = rg._get_kernel(*cfg) + original = torch.empty((split, m, 128), device="cuda") + + def launch_base(): + rg._run_compiled(base, x.data_ptr(), w.data_ptr(), original.data_ptr(), m, x.stride(0), + torch.cuda.current_stream()) + + storage = torch.full((split * m * 128 + 64,), 17.0, device="cuda") + p = storage[32:-32].view(split, m, 128) + if kernel == "direct": + k = rd._get_kernel(48, 64, 1) + + def launch(): + rd._run_compiled(k, x.data_ptr(), w.data_ptr(), p.data_ptr(), m, x.stride(0), + torch.cuda.current_stream()) + elif kernel == "nblocks_x_direct": # every tier, not only the served rows + k = rn._get_kernel(*rg.nblocks_config(m), 0, True) + + def launch(): + rn._run_compiled(k, x.data_ptr(), w.data_ptr(), p.data_ptr(), m, x.stride(0), 0, 0, 0, 0, + torch.cuda.current_stream()) + elif kernel == "direct_lines": + k = rd._get_kernel(48, 64, (m + 15) // 16, 2, True) + + def launch(): + rd._run_compiled(k, x.data_ptr(), lines.data_ptr(), p.data_ptr(), m, x.stride(0), + torch.cuda.current_stream()) + else: + # the served form (x through LDS, or from global memory at X_DIRECT_FROM..X_DIRECT_MAX) + k = rn._get_kernel(*rg.nblocks_config(m), 0, rg.X_DIRECT_FROM <= m <= rg.X_DIRECT_MAX) + + def launch(): + rn._run_compiled(k, x.data_ptr(), w.data_ptr(), p.data_ptr(), m, x.stride(0), 0, 0, 0, 0, + torch.cuda.current_stream()) + + launch_base() + p.fill_(float("nan")) + launch() + assert torch.equal(p.view(torch.int32), original.view(torch.int32)) + # the public call takes this kernel + pub = rg.router_gemm_partials(x, w) + assert pub.shape == (m, split, 128) + assert torch.equal(pub.permute(1, 0, 2).view(torch.int32), original.view(torch.int32)) + # with the line-layout copy (the entry's MoeWeights): the same partials at every tier + pub = rg.router_gemm_partials(x, w, lines) + assert torch.equal(pub.permute(1, 0, 2).view(torch.int32), original.view(torch.int32)) + exact, bound = router_gemm_partials_ref(x, w, split_k=split) + assert ((p.double() - exact).abs() <= 2**-18 * bound).all() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + launch() + for seed in (901, 902, 901): + x.copy_(make_activation(m, seed=seed)) + launch_base() + p.fill_(float("nan")) + g.replay() + assert torch.equal(p.view(torch.int32), original.view(torch.int32)) + p.fill_(float("nan")) + g.replay() + assert torch.equal(p.view(torch.int32), original.view(torch.int32)) + torch.cuda.synchronize() + assert (storage[:32] == 17).all() and (storage[-32:] == 17).all() + + +def test_direct_rejects_untested_splits(): + # only S48 is served: S16 / S24 gave wrong partials + for split in (8, 16, 24): + with pytest.raises(AssertionError): + rd.compile_router_gemm_direct(split, 64, 1) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks_lines.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks_lines.py new file mode 100644 index 0000000..5ea1a37 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_gemm_nblocks_lines.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The two-wave line-layout router writes the nblocks kernel's split-48 partials bit for bit: partial tiles, strided +inputs, guarded outputs, changed-input replay and the public dispatch.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm as rg +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_nblocks as old +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_nblocks_lines as new +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import router_line_layout +from .workload import make_activation, require_gfx950 + + +@pytest.mark.parametrize("m", [17, 18, 23, 24, 30, 31, 32]) +@pytest.mark.parametrize("strided", [False, True]) +def test_line_partial_bytes_guards_and_replay(m, strided): + require_gfx950() + g = torch.Generator(device="cuda").manual_seed(1107 + m) + x = torch.empty((m, 6146 if strided else 6144), dtype=torch.bfloat16, device="cuda")[:, :6144] + x.copy_(make_activation(m, seed=m)) + weight = torch.randn((128, 6144), generator=g, device="cuda") / 6144**0.5 + lines = router_line_layout(weight) + ref = torch.empty((48, m, 128), device="cuda") + guard = torch.full((ref.numel() + 64,), 17.0, device="cuda") + out = guard[32:-32].view_as(ref) + baseline = old._get_kernel(*rg.nblocks_config(m)) + candidate = new._get_kernel() + + def reference(): + rg._run_compiled(baseline, x.data_ptr(), weight.data_ptr(), ref.data_ptr(), m, x.stride(0), + 0, 0, 0, 0, torch.cuda.current_stream()) + + def run(): + rg._run_compiled(candidate, x.data_ptr(), lines.data_ptr(), out.data_ptr(), m, x.stride(0), + torch.cuda.current_stream()) + + reference() + run() + assert torch.equal(out.view(torch.uint8), ref.view(torch.uint8)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for seed in (901, 902, 901): + x.copy_(make_activation(m, seed=seed)) + reference() + for _ in range(2): + out.fill_(float("nan")) + graph.replay() + assert torch.equal(out.view(torch.uint8), ref.view(torch.uint8)) + public = rg.router_gemm_partials(x, weight, lines) + assert torch.equal(public.permute(1, 0, 2).contiguous().view(torch.uint8), ref.view(torch.uint8)) + assert (guard[:32] == 17).all() and (guard[-32:] == 17).all() diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_quant_line_layout.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_quant_line_layout.py new file mode 100644 index 0000000..4e558d5 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_quant_line_layout.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The fused BF16x3 router + MXFP4 quant on the line-layout gate: codes and scales against ``quant.mxfp4_quant`` bit +for bit, the router partials within the router contract, repeated and replayed bit for bit; the K split table.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_quant_line_layout import ( + router_mxfp4_quant_line_layout, +) +from .test_entry import weights # noqa: F401 +from .workload import make_activation + + +# every K-split tier (router_k_split) and its first row past a bound, incl. row tails +@pytest.mark.parametrize("m", [1025, 1536, 2047, 2048, 2049, 2689, 4096, 4097, 5377, + 8192, 8193, 10881, 16384, 16385, 21761, 24576, 24577, 32768]) +def test_router_quant_line_layout(weights, m): + _, w = weights + x = make_activation(m, seed=m) + x[0, :32] = 0 + expected = router_mxfp4_quant_line_layout(x, w.router_weight_lines) + q, s, p = expected + rq, rs = mxfp4_quant(x) + assert torch.equal(q, rq) and torch.equal(s, rs) + assert torch.isfinite(p).all() + logits = sum(p[:, k] for k in range(p.shape[1])) + reference = x.double() @ w.router_weight.double().t() + error = (logits.double() - reference).abs() + bound = x.double().abs() @ w.router_weight.double().abs().t() + assert (error <= 2**-18 * bound).all(), error.max().item() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = router_mxfp4_quant_line_layout(x, w.router_weight_lines) + for _ in range(2): + for t in captured[:2]: + t.fill_(255) + captured[2].fill_(float("nan")) + torch.cuda.synchronize() + graph.replay() + torch.cuda.synchronize() + assert all(torch.equal(a, b) for a, b in zip(captured, expected)) + + +# the K split per row count (the table's bounds and the first row past each) +@pytest.mark.parametrize("m,ks", [(2048, 16), (2049, 12), (2688, 12), (2689, 8), (4096, 8), (4097, 6), + (5376, 6), (5377, 4), (8192, 4), (8193, 3), (10880, 3), (10881, 2), + (16384, 2), (16385, 3), (21760, 3), (21761, 2), (32768, 2)]) +def test_router_k_split_table(m, ks): + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_quant import router_k_split + + assert router_k_split(m) == ks diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_rounds.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_rounds.py new file mode 100644 index 0000000..9aa86d8 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_rounds.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Token-wave geometry: the base config's partials bit for bit, the partial contract, guards, changed graph inputs.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm as rg +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import router_gemm_partials_ref +from .workload import make_activation, require_gfx950 + + +@pytest.mark.parametrize('m,waves', [(512,4), (513,5), (640,5), (641,6), (768,6), + (769,8), (896,8), (897,8), (1024,8), (1025,5), (1280,5), (1281,6), + (1536,6), (1537,7), (1792,7), (1793,8), (2047,8)]) +def test_round_geometry(m, waves): + active, helpers = rg._waves(32, m) + assert active == waves + if m >= 513: + assert active + helpers == 8 + groups = (m + 32 * active - 1) // (32 * active) + assert (groups + 3) // 4 == (m + 1023) // 1024 + if active > 1 and not 769 <= m <= 896: + smaller = (m + 32 * (active - 1) - 1) // (32 * (active - 1)) + assert (smaller + 3) // 4 > (m + 1023) // 1024 + + +@pytest.mark.parametrize('m', [193, 224, 256, 257, 320, 384, 385, 512, 513, 640, 641, 768, 769, 896, 897, 1024, + 1025, 1280, 1281, 1536, 1537, 1792, 1793, 2047]) +def test_partial_contract_and_graph(m): + require_gfx950() + torch.manual_seed(456) + # An even, noncontiguous row stride exercises the public router contract. + source = torch.empty((m, 6146), dtype=torch.bfloat16, device='cuda') + x = source[:, :6144] + x.copy_(make_activation(m, seed=m)) + w = torch.randn((128, 6144), device='cuda') / 6144**.5 + size = 16 * m * 128 + storage = torch.full((size + 64,), 17., device='cuda') + p = storage[32:-32].view(16, m, 128) + f = rg._get_kernel(*rg._kernel_config(m)) + base = rg._get_kernel(32, 16, 64, 8, 0) + original = torch.empty_like(p) + def launch(exe, out): + rg._run_compiled(exe, x.data_ptr(), w.data_ptr(), out.data_ptr(), m, + x.stride(0), torch.cuda.current_stream()) + launch(base, original) + p.fill_(float('nan')) + launch(f, p) + assert torch.equal(p, original) + exact, bound = router_gemm_partials_ref(x, w) + assert ((p.double()-exact).abs() <= 2**-18 * bound).all() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + launch(f, p) + for seed in [901, 902, 901]: + x.copy_(make_activation(m, seed=seed)) + launch(base, original) + p.fill_(float('nan')) + g.replay() + assert torch.equal(p, original) + # Repeat without changing inputs: deterministic, including output poisoning. + p.fill_(float('nan')) + g.replay() + assert torch.equal(p, original) + torch.cuda.synchronize() + assert (storage[:32] == 17).all() and (storage[-32:] == 17).all() + + +@pytest.fixture(scope='module') +def spinner(): + require_gfx950() + from tests.mfma_spinner import mfma_spinner + return mfma_spinner(256) + + +@pytest.mark.parametrize('m', [193, 256, 384, 513, 641, 769, 1025, 1537]) +def test_partials_next_to_mfma(spinner, m): + x = make_activation(m, seed=m) + w = torch.randn((128, 6144), device='cuda') / 6144**.5 + expected = rg.router_gemm_partials(x, w).clone() + torch.cuda.synchronize() + foreign = torch.cuda.Stream(priority=-1) + foreign.wait_stream(torch.cuda.current_stream()) + spinner(2_000_000, foreign) + results = [rg.router_gemm_partials(x, w) for _ in range(3)] + torch.cuda.synchronize() + assert all(torch.equal(p, expected) for p in results) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_token_partials.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_token_partials.py new file mode 100644 index 0000000..518c5ef --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_token_partials.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Token-major S48 stores preserve logical partials, bounds, strides and replay bytes.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm as rg +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_direct as rd +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_nblocks as rn +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_gemm_nblocks_lines as rl +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import router_line_layout +from .workload import require_gfx950 + + +@pytest.mark.parametrize("m", [1, 2, 3, 7, 8, 9, 12, 15, 16, 17, 24, 31, 32, 33, 47, 48, 57, 58, 63, 64, 65, 128, 256]) +@pytest.mark.parametrize("lines", [False, True]) +@pytest.mark.parametrize("strided", [False, True]) +def test_token_partials_match_split_major(m, lines, strided): + require_gfx950() + gen = torch.Generator(device="cuda").manual_seed(771 + m) + backing = torch.randn((m, 6152 if strided else 6144), device="cuda", dtype=torch.bfloat16, generator=gen) + x = backing[:, :6144] + gate = torch.randn((128, 6144), device="cuda", generator=gen) + wl = router_line_layout(gate) if lines else None + split, bk, waves, nb = rg.nblocks_config(m) + guard = 64 + buffers = [torch.full((split * m * 128 + 2 * guard,), -777., device="cuda") for _ in range(2)] + parts = [v[guard:-guard].view(split, m, 128) for v in buffers] + + def run(index): + # token-major stores: the standalone router's through 64, the quant-in-router entry's (S16) also at 65-128 + token_major = index == 1 and m <= 128 + args = (x.data_ptr(), (wl if lines and m <= 32 else gate).data_ptr(), parts[index].data_ptr(), m, x.stride(0)) + if m <= 16: + kernel = rd._get_kernel(48, 64, 1, 2 if lines else 0, lines, token_major) + rg._run_compiled(kernel, *args, torch.cuda.current_stream()) + elif lines and m <= 32: + rg._run_compiled(rl._get_kernel(token_major), *args, torch.cuda.current_stream()) + else: + kernel = rn._get_kernel(split, bk, waves, nb, 0, rg.X_DIRECT_FROM <= m <= rg.X_DIRECT_MAX, token_major) + rg._run_compiled(kernel, *args, 0, 0, 0, 0, torch.cuda.current_stream()) + return parts[index].view(m, split, 128) if token_major else parts[index].permute(1, 0, 2) + + def equal(a, b): + return torch.equal(a.contiguous().view(torch.uint8), b.contiguous().view(torch.uint8)) + + expected = run(0).clone() + actual = run(1) + public = rg.router_gemm_partials(x, gate, wl) + assert equal(expected, actual) and equal(expected, public) + assert public.stride() == ((split * 128, 128, 1) if m <= 64 else (128, m * 128, 1)) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run(1) + for change in (False, True, False): + if change: + x.copy_(torch.randn(x.shape, device="cuda", dtype=torch.bfloat16, generator=gen)) + expected = run(0).clone() + parts[1].fill_(float("nan")) + graph.replay() + assert equal(expected, actual) + for buf in buffers: + assert torch.all(buf[:guard] == -777) and torch.all(buf[-guard:] == -777) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_sorted_tasks.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_sorted_tasks.py new file mode 100644 index 0000000..24809e0 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_sorted_tasks.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Stable local queues retain strict-tree ties, unbiased weight bits and graph writes.""" +import struct + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_topk as rt +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_topk_mxfp4_mxfp8_quant_sorted_tasks as new +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_topk_sorted as plain_new +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import mxfp4_quant_ref +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from .workload import make_activation, require_gfx950 + + +@pytest.mark.parametrize("m", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 32, 33, 64, 65, 128, 255, 256]) +@pytest.mark.parametrize("case", ["random", "ties", "biased_ties", "saturation", "no_choices", "few_choices", "nan_logits"]) +@pytest.mark.parametrize("token_major", [False, True]) +@pytest.mark.parametrize("plain", [False, True]) +def test_sorted_queues_match_router_topk(m, case, token_major, plain): + """Against ``router_topk``'s ids and weights; the tasks kernel's codes and scales against the reference MXFP4 + quant and ``shared_quant``.""" + require_gfx950() + s = 48 if m <= 64 else 16 + gen = torch.Generator(device="cuda").manual_seed(531 + m) + p = (torch.randn((s, m, 128), device="cuda", generator=gen) * 0.1).permute(1, 0, 2) + bias = torch.randn(128, device="cuda", generator=gen) * 0.01 + if case == "ties": + p.copy_(p[:, :, :8].repeat(1, 1, 16)) + bias.zero_() + if case == "biased_ties": + # At 2**24 every sigmoid in [0,1] rounds to the same choice, while + # the unbiased weights differ. This checks the queue's carried sigmoid. + p.mul_(10) + bias.fill_(2**24) + if case == "saturation": + p[:, :, ::2] = 2.0 + p[:, :, 1::2] = -2.0 + bias.zero_() + if case == "no_choices": + bias.fill_(float("-inf")) + if case == "few_choices": + bias.fill_(float("nan")) + bias[127] = 0.0 + if case == "nan_logits": + p[:, :, ::3] = float("nan") + p[:, :, 1::3] = float("inf") + if token_major: + if s != 48 and not plain: + pytest.skip("constant offsets serve S48 only") + p = p.contiguous() + x = make_activation(m, seed=m) + outputs = [torch.empty((m, 4), device="cuda"), + torch.empty((m, 4), dtype=torch.int32, device="cuda"), + *[torch.empty((m, h), dtype=torch.uint8, device="cuda") for h in (3072, 192, 6144, 192)]] + actual_module = plain_new if plain else new + guards = None + if plain: + guards = [torch.full((m, 7), -777., device="cuda"), + torch.full((m, 7), -777, device="cuda", dtype=torch.int32)] + outputs = [t[:, :4] for t in guards] + scale = struct.unpack(" 1: + p[:, 1, 3:40] = 0.5 + if n > 2: + p[:, 2] = p[:, 2, :8].repeat(1, E // 8) + if n > 3: + p[:, n - 1, ::5] = 1.0 + return p.permute(1, 0, 2) # [n, s, E] view of [s, n, E], as the routers write + + +def _run(tpw, parts, bias, wt, ids): + n, s, _ = parts.shape + end = (n - 1) * parts.stride(0) + (s - 1) * parts.stride(1) + E + rt._run_compiled( + rt._get_kernel(E, TOPK, s, tpw), + parts.data_ptr(), + bias.data_ptr(), + wt.data_ptr(), + ids.data_ptr(), + end * 4, + n, + parts.stride(0), + parts.stride(1), + ids.stride(0), + 0x40000000, # 2.0 + torch.cuda.current_stream(), + ) + + +@pytest.mark.parametrize( + "n,s", [(1, 16), (2, 16), (3, 48), (2047, 16), (2048, 16), (2049, 16), (4095, 8), + (4097, 4), (8193, 4), (16385, 4), (24577, 2), (32768, 2), + (2049, 12), (4097, 6), (8193, 3), (16385, 3), (24577, 1), (32768, 1)] +) +def test_two_tokens_per_wave_bitwise(n, s): + parts = _partials(n, s, n) + bias = torch.randn((E,), device="cuda", generator=torch.Generator(device="cuda").manual_seed(1)) * 0.1 + bias[::7] = 0.0 + out = [] + for tpw in (1, 2): + wt = torch.full((n, TOPK), float("nan"), device="cuda") + ids = torch.full((n, TOPK), -1, dtype=torch.int32, device="cuda") + _run(tpw, parts, bias, wt, ids) + out.append((wt, ids)) + torch.cuda.synchronize() + (w1, i1), (w2, i2) = out + assert torch.equal(i1, i2) + assert torch.equal(w1.view(torch.int32), w2.view(torch.int32)) + assert bool((i1 >= 0).all()) and bool(torch.isfinite(w1).all()) + + +def test_public_entry_switches_at_tpw2_from(): + """``splitk_biased_topk`` serves tpw 2 from TPW2_FROM rows, inside a graph too: + replay with changed partials equals eager tpw 1, with no extra writes past M.""" + n, s = rt.TPW2_FROM + 1, 16 + bias = torch.zeros((E,), device="cuda") + parts = _partials(n, s, 5) + wt = torch.full((n + 1, TOPK), float("nan"), device="cuda") + ids = torch.full((n + 1, TOPK), -1, dtype=torch.int32, device="cuda") + rt.splitk_biased_topk(parts, bias, wt[:n], ids[:n], 2.0) + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + rt.splitk_biased_topk(parts, bias, wt[:n], ids[:n], 2.0) + parts.copy_(_partials(n, s, 6)) + g.replay() + w1 = torch.empty((n, TOPK), device="cuda") + i1 = torch.empty((n, TOPK), dtype=torch.int32, device="cuda") + _run(1, parts, bias, w1, i1) + torch.cuda.synchronize() + assert torch.equal(ids[:n], i1) and torch.equal(wt[:n].view(torch.int32), w1.view(torch.int32)) + assert bool(torch.isnan(wt[n]).all()) and bool((ids[n] == -1).all()) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_vec2.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_vec2.py new file mode 100644 index 0000000..9b59c26 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_router_topk_vec2.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-expert loads restore the four-expert argmax layout, including its ties.""" + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import router_topk as rt +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk_vec2 import compile_router_topk_vec2 +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import ( + WEIGHT_RTOL, biased_topk_reference, decided, splitk_logits_reference, +) +from .workload import require_gfx950 + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def _run(fn, p, bias, wt, ids, scale_bits=0x40000000): + m, s, e = p.shape + end = (m - 1) * p.stride(0) + (s - 1) * p.stride(1) + e + rt._run_compiled(fn, p.data_ptr(), bias.data_ptr(), wt.data_ptr(), ids.data_ptr(), + end * 4, m, p.stride(0), p.stride(1), ids.stride(0), scale_bits, + torch.cuda.current_stream()) + + +# (split, tokens): S48 serves 1..64 tokens, S16 65..512 and 1025..2048, S8 2049..4096 +SPLIT_M = ([(48, m) for m in (1, 2, 7, 8, 15, 16, 17, 26, 31, 32, 33, 48, 63, 64)] + + [(16, m) for m in (1, 65, 97, 128, 129, 255, 256, 257, 511, 512, 1025, 2047, 2048)] + + [(12, m) for m in (1, 2049, 2432, 2688)] + + [(8, m) for m in (1, 2049, 2689, 3071, 4095, 4096)]) + + +@pytest.mark.parametrize('split,m', SPLIT_M) +@pytest.mark.parametrize('case', ['random', 'ties', 'saturation']) +@pytest.mark.parametrize('strided', [False, True]) +def test_vec2_bitwise_and_contract(split, m, case, strided): + gen = torch.Generator(device='cuda').manual_seed(m) + storage = torch.randn((split, m * (2 if strided else 1), 128), device='cuda', generator=gen) * 0.1 + p = storage[:, ::2 if strided else 1].permute(1, 0, 2) + bias = torch.randn(128, device='cuda', generator=gen) * 0.01 + if case == 'ties': + p.copy_(p[:, :, :8].repeat(1, 1, 16)) + p[0] = 0.0 + bias.zero_() + if case == 'saturation': + p[:, :, ::2] = 2.0 + p[:, :, 1::2] = -2.0 + bias.zero_() + refs = (torch.empty((m, 4), device='cuda'), torch.empty((m, 4), device='cuda', dtype=torch.int32)) + _run(rt.compile_router_topk(128, 4, split), p, bias, *refs) + # Both sides of the output view and the padding of every row are guards. + weights = torch.full((m + 2, 8), float('nan'), device='cuda') + ids = torch.full((m + 2, 8), -1, device='cuda', dtype=torch.int32) + out = weights[1:-1, 2:6], ids[1:-1, 2:6] + fn = compile_router_topk_vec2(128, 4, split) + for _ in range(3): + out[0].fill_(float('nan')); out[1].fill_(-1) + _run(fn, p, bias, *out) + assert torch.equal(out[0].view(torch.int32), refs[0].view(torch.int32)) + assert torch.equal(out[1], refs[1]) + assert torch.isfinite(out[0]).all() + mask = torch.ones_like(ids, dtype=torch.bool) + mask[1:-1, 2:6] = False + assert torch.isnan(weights[mask]).all() and (ids[mask] == -1).all() + rw, ri, vals = biased_topk_reference(splitk_logits_reference(p), bias, 4, 2.0) + known = decided(vals) + assert torch.equal(out[1][known].long(), ri[known]) + if case == 'random': + # a row's weights are decided when its whole top-k set and order are + rows = known.all(-1) + torch.testing.assert_close(out[0][rows].double(), rw[rows], atol=0, rtol=WEIGHT_RTOL) + + +@pytest.mark.parametrize('split,m', [(48, 1), (48, 8), (48, 17), (48, 32), (48, 64), + (16, 65), (16, 256), (16, 512), (16, 2048), (12, 2688), (8, 4096)]) +def test_vec2_graph_changed_inputs(split, m): + p = torch.randn((split, m, 128), device='cuda').permute(1, 0, 2) * 0.1 + bias = torch.zeros(128, device='cuda') + out = torch.empty((m, 4), device='cuda'), torch.empty((m, 4), device='cuda', dtype=torch.int32) + ref = torch.empty_like(out[0]), torch.empty_like(out[1]) + fn = compile_router_topk_vec2(128, 4, split) + _run(fn, p, bias, *out) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _run(fn, p, bias, *out) + for _ in range(3): + p.normal_(0, 0.1) + bias.normal_(0, 0.01) + _run(rt.compile_router_topk(128, 4, split), p, bias, *ref) + out[0].fill_(float('nan')); out[1].fill_(-1) + graph.replay() + assert torch.equal(out[0].view(torch.int32), ref[0].view(torch.int32)) + assert torch.equal(out[1], ref[1]) + + +def test_public_factory_uses_vec2_only_for_the_served_splits(): + assert '_vec2_' in rt._get_kernel(128, 4, 48).kernel_name + assert '_vec2_' in rt._get_kernel(128, 4, 16).kernel_name + assert '_vec2_' in rt._get_kernel(128, 4, 8).kernel_name + for e, k, s, tpw in [(128, 4, 4, 1), (128, 4, 8, 2), (128, 4, 16, 2), (128, 4, 48, 2), (256, 4, 48, 1), (128, 2, 48, 1)]: + assert '_vec2_' not in rt._get_kernel(e, k, s, tpw).kernel_name diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_scale_sort_tile_map.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_scale_sort_tile_map.py new file mode 100644 index 0000000..20855a0 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_scale_sort_tile_map.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The merged activation-scale sort + gemm1 tile map (``common.scale_sort_tile_map``) after the BM256 host's routing sort +(``common.sort_count_all`` / ``common.sort_tile_map``, 256-row blocks) against the torch gather + AITER shuffle and the +torch tile map.""" + +import pytest +import torch +from aiter.utility.fp4_utils import e8m0_shuffle +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4 as prefill +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.scale_sort_tile_map import scale_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort import SortBuffers +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_count_all import moe_sort_count_all +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_tile_map import moe_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.tile_map import CLAIM_COUNTERS, tile_map_grid +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import E, H, TOPK, I + +from .workload import make_routing, require_gfx950, tile_map_ref + +SERVED_TOKENS = [513, 1024, 2048, 4096, 8192, 16384, 32768] + +BM = prefill.BM +NB_N = I // 128 + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def run_sort(n, ids, w): + bufs = SortBuffers.allocate(n, E, TOPK, BM, "cuda") + grid = tile_map_grid(bufs.max_sorted // BM, I) + scratch = torch.empty((grid + 1,), dtype=torch.int32, device="cuda") + if n <= prefill.SORT_COUNT_ALL_MAX_TOKENS: + moe_sort_count_all(bufs, ids, w, scratch, grid, n, E=E, I=I) + else: + moe_sort_tile_map(bufs, ids, w, scratch, grid, n, E=E) + return bufs + + +@pytest.mark.parametrize("n", SERVED_TOKENS) +def test_scale_sort_tile_map_matches_torch(n): + w, ids = make_routing(n, seed=n + 7, skewed=True) + g = torch.Generator(device="cuda").manual_seed(n) + xs = torch.randint(100, 140, (n, H // 32), dtype=torch.uint8, device="cuda", generator=g) + bufs = run_sort(n, ids, w) + rows = bufs.max_sorted + nv = int(bufs.num_valid_ids[0]) + grid = tile_map_grid(rows // BM, I) + gbj = 0 + + def run(): + a_s = torch.full(((rows + 31) // 32 * 32, H // 32), 0xA5, dtype=torch.uint8, device="cuda") + tm = torch.full((grid + 1,), 0x5A5A5A5A, dtype=torch.int32, device="cuda") + scale_sort_tile_map( + xs, bufs.sorted_ids, bufs.sorted_expert_ids, bufs.num_valid_ids, a_s, tm, + grid, n, BM, a_group_bj=gbj, + ) # fmt: skip + return a_s, tm + + a_s, tm = run() + # scales: each valid row its token's bytes, padding rows 0, AITER-shuffled; + # tiles past num_valid are not written (the GEMM never reads them) + sid = bufs.sorted_ids[:rows].long() + tok = sid & 0xFFFFFF + valid = (torch.arange(rows, device="cuda") < nv) & (tok < n) + gathered = torch.zeros(((rows + 31) // 32 * 32, H // 32), dtype=torch.uint8, device="cuda") + gathered[:rows][valid] = xs[tok[valid]] + want = e8m0_shuffle(gathered).view(torch.uint8).reshape(-1) + written = (nv + 31) // 32 * 32 * (H // 32) + assert torch.equal(a_s.reshape(-1)[:written], want[:written]) + # tile map: the torch table, entry for entry + assert torch.equal(tm, tile_map_ref(bufs.sorted_expert_ids, bufs.num_valid_ids, grid, BM)) + # and its contents: every valid (m-block, n-tile) once, -1 after, the count last + valid_blocks = nv // BM + entries = tm[: valid_blocks * NB_N].long() + n_bits = (NB_N - 1).bit_length() + m_blk, n_tile = entries >> n_bits, entries & ((1 << n_bits) - 1) + key = torch.sort(m_blk * NB_N + n_tile).values + assert torch.equal(key, torch.arange(valid_blocks * NB_N, device="cuda")) + assert bool((tm[valid_blocks * NB_N : grid] == -1).all()) + assert grid - valid_blocks * NB_N >= CLAIM_COUNTERS + assert int(tm[grid]) == valid_blocks * NB_N + for _ in range(2): + a2, tm2 = run() + assert torch.equal(a2.reshape(-1)[:written], a_s.reshape(-1)[:written]) + assert torch.equal(tm2, tm) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_epilogue_early.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_epilogue_early.py new file mode 100644 index 0000000..6c50a49 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_epilogue_early.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Early previous-tile stores preserve FP32 SwiGLU and every valid MXFP8 byte.""" +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import shared_dense as sd +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_gemm1_a8w8_epilogue_early import ( + compile_shared_gemm1_a8w8_epilogue_early, +) +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.mxfp8 import shared_dense_4wave as impl +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.launch import _run_compiled +from .test_shared_expert import weights +from .workload import make_activation + + +def tile_end_gemm1(): + return impl.compile_mxfp8_gemm_4w_gateup( + K=6144, alpha=1.702, beta=1.0, limit=7.0, + epilogue_type=sd.StoreSharedFp32, + kernel_prefix='shared_fp32_swiglu_mxfp8_h6144_i1536_', tiles=(4, 4)) + + +def row_scales(s, m): + return s.view(-1, 6, 4, 16, 2, 2).permute(0, 5, 3, 1, 4, 2).reshape(-1, 48)[:m] + + +@pytest.mark.parametrize('m', [1, 257, 1025, 2048, 4097, 8192, 8193, 5376, 5377, 10752, 16384, 32768, 32769, 87381]) +@pytest.mark.parametrize('schedule', ['early', 'single']) +def test_early_epilogue_changed_graph_inputs(weights, m, schedule): + _, w = weights + if schedule == 'single' and m > 5376: + pytest.skip('single-tile grid requires ceil(M/256)*12 <= 256') + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_gemm1_a8w8_single_tile import compile_shared_gemm1_a8w8_single_tile + ref = tile_end_gemm1() + early = (compile_shared_gemm1_a8w8_epilogue_early() if schedule == 'early' + else compile_shared_gemm1_a8w8_single_tile()) + x = make_activation(m, seed=m) + q, s = shared_quant(x) + padded = (m + 31) // 32 * 32 + ss = torch.empty((padded, 192), device='cuda', dtype=torch.uint8) + _run_compiled(sd._shuffle_scales(6144), s.view(-1), ss.view(-1), m, torch.cuda.current_stream()) + guards = [] + def guarded(shape): + size = 1 + for dim in shape: + size *= dim + storage = torch.full((size + 256,), 19, device='cuda', dtype=torch.uint8) + guards.append(storage) + return storage[128:-128].view(shape) + expected = (guarded((m, 1536)), guarded((padded, 48))) + actual = (guarded((m, 1536)), guarded((padded, 48))) + def launch(fn, out): + parents = (m + 255) // 256 * 12 + _run_compiled(fn, q.view(torch.int8).view(-1), w.w13.view(torch.int8).view(-1), + out[0].view(torch.int8).view(-1), out[1].view(-1), ss.view(-1), + w.s13.view(-1), m, 3072, 0, parents, torch.cuda.current_stream()) + launch(ref, expected) + launch(early, actual) + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + launch(early, actual) + for seed in [901, 902, 901]: + q1, s1 = shared_quant(make_activation(m, seed=seed)) + q.copy_(q1) + _run_compiled(sd._shuffle_scales(6144), s1.view(-1), ss.view(-1), m, torch.cuda.current_stream()) + launch(ref, expected) + for _ in range(2): + actual[0].fill_(255) + actual[1].fill_(255) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(actual[0], expected[0]) + assert torch.equal(row_scales(actual[1], m), row_scales(expected[1], m)) + assert all((t[:128] == 19).all() and (t[-128:] == 19).all() for t in guards) + + +def test_full_tile_epilogue_variants(): + assert '_early_' in sd._gemm1((4, 4), schedule='early').kernel_name + assert '_early_' not in sd._gemm1((4, 4)).kernel_name + assert 'single_tile' in sd._gemm1((4, 4), schedule='single').kernel_name + assert '_early_' not in sd._gemm1((2, 2), schedule='early').kernel_name + assert '_early_' not in sd._gemm1((2, 4), schedule='early').kernel_name + + +@pytest.mark.parametrize('m', [8193, 32769]) +def test_early_epilogue_next_to_mfma(weights, m, monkeypatch): + from tests.mfma_spinner import mfma_spinner + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert import shared_expert + _, w = weights + x = make_activation(m, seed=m) + original = sd._gemm1 + with monkeypatch.context() as mp: + mp.setattr(sd, '_gemm1', lambda tiles=(4, 4), **kw: original(tiles)) + expected = shared_expert(x, w) + shared_expert(x, w) + torch.cuda.synchronize() + spinner = mfma_spinner(256) + stream = torch.cuda.Stream(priority=-1) + stream.wait_stream(torch.cuda.current_stream()) + spinner(5_000_000, stream) + outputs = [shared_expert(x, w) for _ in range(3)] + torch.cuda.synchronize() + assert all(torch.equal(y, expected) for y in outputs) + + +@pytest.mark.parametrize('m,schedule', [(1024, 'deferred'), (1025, 'single'), + (5376, 'single'), (5377, 'early'), (10753, 'early'), (32768, 'early')]) +def test_shared_gemm1_schedule(m, schedule): + assert sd._gemm1_schedule_for(m) == schedule + name = sd._gemm1((4, 4), schedule=schedule).kernel_name + assert ('single_tile' in name) == (schedule == 'single') + assert ('_early_' in name) == (schedule == 'early') diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_expert.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_expert.py new file mode 100644 index 0000000..a57b252 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_expert.py @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert import SharedWeights,shared_expert +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import shared_mxfp8_ref,shared_mxfp8_quant_ref +from .workload import make_activation,require_gfx950,replay_check + + +@pytest.fixture(scope='module') +def weights(): + require_gfx950();g=torch.Generator(device='cuda').manual_seed(607) + def weight(n,k): + q=(torch.randn((n,k),device='cuda',generator=g)*20).to(torch.float8_e4m3fn).view(torch.uint8) + s=torch.randint(118,124,(n,k//32),device='cuda',dtype=torch.uint8,generator=g) + return q,s + raw=(*weight(3072,6144),*weight(6144,1536)) + return raw,SharedWeights.from_row_major(*raw) + + +@pytest.fixture +def separate_shared(monkeypatch): + """Keep the standalone scheduling tests independent of the composed launch.""" + import importlib + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + es=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_bm128') + monkeypatch.setattr(sh,'shared_in_persistent_for',lambda n:False) + def fail(*args,**kwargs):raise AssertionError('standalone oracle called the composed kernel') + monkeypatch.setattr(es,'_gemm12_shared',fail) + + +@pytest.mark.parametrize('m',[96,97,128,192,256,512,513,1024,2048,4096,8192,16384,32768]) +def test_shared_contract(weights,m): + raw,w=weights;x=make_activation(m,seed=m) + eager,replay=replay_check(lambda:[shared_expert(x,w)]) + assert torch.equal(eager[0],replay[0]) and torch.isfinite(replay[0]).all() + ref=shared_mxfp8_ref(x[:min(m,64)],*raw) + cos=torch.nn.functional.cosine_similarity(replay[0][:len(ref)].float(),ref,dim=1) + assert cos.min()>=.999,(m,cos.min().item()) + + +def test_shared_quant_rule(weights): + x=make_activation(8192);x[:3,:64]=0 + refq,refs=shared_mxfp8_quant_ref(x) + for m in [1,8,512,8192]: + q,s=shared_quant(x[:m]) + assert torch.equal(q,refq[:m]) and torch.equal(s,refs[:m]) + + +@pytest.fixture(scope='module') +def complete_weights(weights): + from .workload import make_experts + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights + raw,w=weights + routed,_=make_experts() + gate=torch.zeros((128,6144),device='cuda');bias=torch.arange(128,device='cuda').float()/1280 + return MoeWeights.from_row_major(gate,bias,*routed,shared=w) + + +@pytest.mark.parametrize('m',[1,8,16,128,257,513,2048,4097,32768]) +@pytest.mark.parametrize('parallel',[False,True]) +def test_complete_entry(complete_weights,m,parallel,monkeypatch,separate_shared): + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + monkeypatch.setattr(sh,'shared_parallel_for',lambda n:parallel) + w=complete_weights;x=make_activation(m,seed=m) + eager,replay=replay_check(lambda:list(mxfp4_moe(x,w,return_partials=True))) + assert all(torch.equal(a,b) for a,b in zip(eager,replay)) + p,s=replay + assert p.shape==(m,4,6144) and s.shape==(m,6144) + reduced=mxfp4_moe(x,w) + expected=torch.zeros_like(s,dtype=torch.float32) + for k in range(4):expected=expected+p[:,k].float() + assert torch.equal(reduced,(expected+s.float()).bfloat16()) + + +@pytest.mark.parametrize('m',[1,2,5,8,12,13,16,17,33,64,95,96,97,128,129,256,257,336,337,448,512]) +def test_shared_folded_into_routed_launches(weights,complete_weights,m): + """Below 96 rows the shared tiles run first in the routed 16-row GEMM1 / GEMM2 launches (from 96 the standalone + shared expert next to the routed launches): replay equal to eager, the shared output within the FP32 reference + and, from 96 rows, cos >= 0.9999 to the standalone shared expert.""" + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + raw,_=weights;w=complete_weights;x=make_activation(m,seed=300+m) + assert sh.shared_folded_for(m)==(m<96) + eager,replay=replay_check(lambda:list(mxfp4_moe(x,w,return_partials=True))) + assert all(torch.equal(a,b) for a,b in zip(eager,replay)) + p,s=replay + assert torch.isfinite(p).all() and torch.isfinite(s).all() + again=mxfp4_moe(x,w,return_partials=True) + assert torch.equal(again[0],p) and torch.equal(again[1],s) + ref=shared_mxfp8_ref(x[:min(m,64)],*raw) + assert torch.nn.functional.cosine_similarity(s[:len(ref)].float(),ref,dim=1).min()>=.999 + if m>=96: # the standalone shared expert (shared_dense) on the same operands + s0=shared_expert(x,w.shared_weights,x_quant=shared_quant(x)) + cos=torch.nn.functional.cosine_similarity(s.float(),s0.float(),dim=1) + assert cos.min()>=.9999,(m,cos.min().item()) + + +@pytest.mark.parametrize('m',[1,17,33,64,95]) +@pytest.mark.parametrize('tile',[(16,4,2),(16,2,2),(32,2,2),(32,4,2),(16,1,2),(32,1,2),(64,4,1),(64,2,1),(32,2,1)]) +def test_shared_fold_tiles_agree(weights,complete_weights,m,tile,monkeypatch): + """Every folded shared tile (rows, GEMM1 K-waves, prefetch) gives the routed slots of the separate + launches bit for bit and the shared output within FP32 summation order of the served tile.""" + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + es=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_small') + if m<=15 and tile!=(16,4,2): + pytest.skip('inline routing keeps 16-row 4-K-wave shared tiles') + w=complete_weights;x=make_activation(m,seed=500+m) + p0,s0=mxfp4_moe(x,w,return_partials=True) + monkeypatch.setattr(es,'shared_fold_tile_for',lambda n:tile) + eager,replay=replay_check(lambda:list(mxfp4_moe(x,w,return_partials=True))) + assert all(torch.equal(a,b) for a,b in zip(eager,replay)) + p,s=replay + assert torch.equal(p,p0) and torch.isfinite(s).all() + cos=torch.nn.functional.cosine_similarity(s.float(),s0.float(),dim=1) + assert cos.min()>=.9999,(m,tile,cos.min().item()) + + +def test_shared_weight_bits(weights): + raw,w=weights + restored=(w.w13.permute(0,3,1,2,4).reshape(3072,6144), + w.s13.permute(0,5,3,1,4,2).reshape(3072,192), + w.w2.permute(0,3,1,2,4).reshape(6144,1536), + w.s2.permute(0,5,3,1,4,2).reshape(6144,48)) + assert all(torch.equal(a,b.view(torch.uint8)) for a,b in zip(restored,raw)) + assert sum(getattr(w,k).numel() for k in ['w13','s13','w2','s2'])==sum(x.numel() for x in raw) + + +@pytest.mark.parametrize('m',[1,8,64,65,128,512,1024]) +def test_dual_input_quant(complete_weights,m): + """The top-k with both input quants in one launch: the MXFP4 and MXFP8 quantizers' bits, the plain top-k's routing.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_gemm import router_gemm_partials + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk import splitk_biased_topk + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_topk_mxfp4_mxfp8_quant import splitk_biased_topk_mxfp4_mxfp8_quant + w=complete_weights;x=make_activation(m,seed=m) + parts=router_gemm_partials(x,w.router_weight) + wt=torch.empty((m,4),dtype=torch.float32,device=x.device);ids=torch.empty((m,4),dtype=torch.int32,device=x.device) + splitk_biased_topk(parts,w.correction_bias,wt,ids,2.0) + w2=torch.empty_like(wt);i2=torch.empty_like(ids) + a,b=splitk_biased_topk_mxfp4_mxfp8_quant(parts,w.correction_bias,w2,i2,2.0,x) + assert all(torch.equal(x,y.view(torch.uint8)) for x,y in zip(a,mxfp4_quant(x))) + assert all(torch.equal(x,y) for x,y in zip(b,shared_quant(x))) + assert torch.equal(w2,wt) and torch.equal(i2,ids) + eager,replay=replay_check(lambda:[mxfp4_moe(x,w)]) + assert torch.equal(eager[0],replay[0]) + + +@pytest.mark.parametrize('m',[3,128,513,2048,8192,32768]) +def test_shared_dense(weights,m,monkeypatch): + import importlib + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + raw,w=weights;x=make_activation(m,seed=m) + monkeypatch.setattr(sh,'shared_dense_for',lambda n:True) + eager,replay=replay_check(lambda:[shared_expert(x,w)]) + assert torch.equal(eager[0],replay[0]) and torch.isfinite(replay[0]).all() + ref=shared_mxfp8_ref(x[:min(m,64)],*raw) + cos=torch.nn.functional.cosine_similarity(replay[0][:len(ref)].float(),ref,dim=1) + assert cos.min()>=.999,(m,cos.min().item()) + + +@pytest.mark.parametrize('m',[3,128,513,8192]) +@pytest.mark.parametrize('g1_tiles',[(2,2),(2,4)]) +def test_shared_dense_guards(weights,m,g1_tiles,monkeypatch): + import importlib,math + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + monkeypatch.setattr(sh,'shared_dense_for',lambda n:True) + monkeypatch.setattr(sh,'shared_dense_config_for',lambda n:(g1_tiles,(2,2))) + _,w=weights;x=make_activation(m) + shared_expert(x,w);torch.cuda.synchronize() + original=torch.empty;guards=[] + def allocate(*size,**kw): + dims=tuple(size[0]) if isinstance(size[0],(tuple,list)) else size + dtype=kw.get('dtype',torch.float32);g=128//torch.tensor([],dtype=dtype).element_size() + storage=original((math.prod(dims)+2*g,),**kw);storage.fill_(19) + out=storage[g:-g].view(dims) + if dtype==torch.bfloat16:out.fill_(float('nan')) + guards.append((storage,g));return out + monkeypatch.setattr(torch,'empty',allocate) + eager,replay=replay_check(lambda:[shared_expert(x,w)]) + assert torch.equal(eager[0],replay[0]) and torch.isfinite(replay[0]).all() + assert all((t[:g]==19).all() and (t[-g:]==19).all() for t,g in guards) + + +def test_shared_across_tiles(weights,monkeypatch): + import importlib + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + _,w=weights;x=make_activation(8192) + monkeypatch.setattr(sh,'shared_dense_for',lambda n:True) + small=shared_expert(x[:8],w) + large=shared_expert(x,w)[:8] + cos=torch.nn.functional.cosine_similarity(small.float(),large.float(),dim=1) + assert cos.min()>=.9999 + + +@pytest.mark.parametrize('m',[3,128,129,256,513,1024,8192]) +@pytest.mark.parametrize('g1_tiles',[(2,2),(2,4)]) +def test_shared_dense_compact(weights,m,g1_tiles,monkeypatch): + import importlib + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + _,w=weights;x=make_activation(m,seed=m) + monkeypatch.setattr(sh,'shared_dense_for',lambda n:True) + monkeypatch.setattr(sh,'shared_dense_config_for',lambda n:((4,4),(4,4))) + large=shared_expert(x,w) + monkeypatch.setattr(sh,'shared_dense_config_for',lambda n:(g1_tiles,(2,2))) + eager,replay=replay_check(lambda:[shared_expert(x,w)]) + assert torch.equal(eager[0],replay[0]) + cos=torch.nn.functional.cosine_similarity(large.float(),replay[0].float(),dim=1) + assert cos.min()>=.9999 + + +@pytest.mark.parametrize('m',[513,1024,2048]) +def test_dual_dense_overlap(complete_weights,m,monkeypatch,separate_shared): + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + w=complete_weights;x=make_activation(m,seed=m) + monkeypatch.setattr(sh,'shared_dense_for',lambda n:True) + monkeypatch.setattr(sh,'shared_parallel_for',lambda n:False) + monkeypatch.setattr(sh,'shared_dual_quant_for',lambda n:False) + reference=mxfp4_moe(x,w) + monkeypatch.setattr(sh,'shared_parallel_for',lambda n:True) + monkeypatch.setattr(sh,'shared_dual_quant_for',lambda n:n<=1024) + eager,replay=replay_check(lambda:[mxfp4_moe(x,w)]) + assert torch.equal(eager[0],replay[0]) and torch.equal(reference,eager[0]) + + +@pytest.mark.parametrize('m',[128,513,8192]) +def test_shared_mfma_corunner(weights,m): + from tests.mfma_spinner import mfma_spinner + _,w=weights;x=make_activation(m) + ref=shared_expert(x,w).clone();torch.cuda.synchronize() + spinner=mfma_spinner(256);stream=torch.cuda.Stream(priority=-1) + spinner(5_000_000,stream) + out=shared_expert(x,w);torch.cuda.synchronize() + assert torch.equal(out,ref) + +@pytest.mark.parametrize('m',[1025,1536,2048,2049,4096,4097,8192,8193,16384,16385,24577,32768]) +@pytest.mark.parametrize('parallel',[False,True]) +def test_router_dual_shared_entry(complete_weights,m,parallel,monkeypatch,separate_shared): + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + monkeypatch.setattr(sh,'shared_parallel_for',lambda n:parallel) + monkeypatch.setattr(sh,'shared_dual_quant_for',lambda n:n<=1024) + w=complete_weights;x=make_activation(m,seed=m) + old=mxfp4_moe(x,w) + monkeypatch.setattr(sh,'shared_dual_quant_for',lambda n:True) + eager,replay=replay_check(lambda:[mxfp4_moe(x,w)]) + assert torch.equal(old,eager[0]) and torch.equal(eager[0],replay[0]) + +@pytest.mark.parametrize('m',[4096,8192,16384,32768]) +def test_router_dual_graph_changed_inputs(complete_weights,m,monkeypatch): + from dataclasses import replace + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import router_line_layout + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + gate=torch.randn((128,6144),device='cuda')/6144**.5 + w=replace(complete_weights,router_weight=gate,router_weight_lines=router_line_layout(gate)) + x=make_activation(m,seed=m) + mxfp4_moe(x,w);torch.cuda.synchronize() + graph=torch.cuda.CUDAGraph() + with torch.cuda.graph(graph):out=mxfp4_moe(x,w) + previous=None + for seed in [619,620,621]: + x.copy_(make_activation(m,seed=seed)) + if seed==621:x.zero_() + out.fill_(float('nan')) + graph.replay();torch.cuda.synchronize() + with monkeypatch.context() as mp: + mp.setattr(sh,'shared_dual_quant_for',lambda n:n<=1024) + mp.setattr(sh,'shared_in_persistent_for',lambda n:False) + expected=mxfp4_moe(x,w) + torch.cuda.synchronize() + assert torch.equal(out,expected) + if previous is not None:assert not torch.equal(out,previous) + previous=out.clone() + +@pytest.mark.parametrize('m',[4032,4096]) +def test_partial_mode_uses_separate_shared_prep(complete_weights,m,monkeypatch,separate_shared): + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + mod=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.router_mxfp4_mxfp8_quant_line_layout') + original=mod.router_mxfp4_mxfp8_quant_line_layout;calls=[] + def counted(*args,**kw): + calls.append(1) + return original(*args,**kw) + monkeypatch.setattr(mod,'router_mxfp4_mxfp8_quant_line_layout',counted) + x=make_activation(m,seed=m);w=complete_weights + p,s=mxfp4_moe(x,w,return_partials=True) + assert not calls + out=mxfp4_moe(x,w) + assert len(calls)==1 + ref=torch.zeros_like(s,dtype=torch.float32) + for k in range(4):ref=ref+p[:,k].float() + assert torch.equal(out,(ref+s.float()).bfloat16()) + + +def test_shared_schedule_tiers(): + """Folded below 96 rows, second stream 96-30720 (when not folded), serial outside.""" + import importlib + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + es=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_small') + assert [sh.shared_in_persistent_for(m) for m in (448,449,512,513,640,641,1024,1025,1536,2048,3583,3584,3840,4096,4097)]==[False,False,False,False,False,True,True,True,True,True,True,False,False,False,False] + assert [sh.shared_folded_for(m) for m in (1, 95, 96, 256, 448)] == [True, True, False, False, False] + assert [es.shared_fold_tile_for(m) for m in (1, 95)] == [(16, 4, 2), (16, 4, 2)] + assert [es.small_config_for(m) for m in (1, 320, 321, 512)] == [(4, 16), (4, 16), (4, 32), (4, 32)] + assert [sh.shared_parallel_for(m) for m in (1, 95, 96, 16384, 16385, 28672, 28673, 30720, 30721, 32768)] == [ + False, False, True, True, True, True, True, True, False, False] + + +@pytest.mark.parametrize('m',[16385,28673,30720,30721]) +def test_parallel_tier_bit_equal_serial(complete_weights,m,monkeypatch): + """The served schedule (second stream through 30720) gives the serial schedule's bits, under replay too.""" + import importlib + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + sh=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_expert') + w=complete_weights;x=make_activation(m,seed=m) + eager,replay=replay_check(lambda:list(mxfp4_moe(x,w,return_partials=True))) + monkeypatch.setattr(sh,'shared_parallel_for',lambda n:False) + serial=list(mxfp4_moe(x,w,return_partials=True)) + assert all(torch.equal(a,b) and torch.equal(a,c) for a,b,c in zip(eager,replay,serial)) + + +@pytest.mark.parametrize('m',[1025,2047,2048,8192,32768]) +def test_shared_gemm2_nt_bit_equal(weights,m,monkeypatch): + """Non-temporal shared GEMM2 stores from 2048 rows give the default policy's bits, under replay too.""" + import importlib + sd=importlib.import_module('amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_dense') + assert sd._gemm2_nt_for(m)==(m>=2048) + assert sd._gemm2((4,4),nt=True) is not sd._gemm2((4,4)) + _,w=weights;x=make_activation(m,seed=m) + eager,replay=replay_check(lambda:[shared_expert(x,w)]) + monkeypatch.setattr(sd,'_gemm2_nt_for',lambda n:n= .999 per token), eager == replay, on a side stream the same bits.""" + raw, ws = full_weights + w = ws[0][0] + x = make_activation(n, seed=n) + local = prepare_mxfp4_moe(x, w) + sw = w.shared_weights + eager, replay = replay_check(lambda: [shared_expert_full(local.shared_codes, local.shared_scales, sw)]) + assert torch.equal(eager[0], replay[0]) and bool(torch.isfinite(replay[0]).all()) + ref = shared_mxfp8_ref(x[:256], *raw) + cos = torch.nn.functional.cosine_similarity(replay[0][:256].float(), ref, dim=1) + assert cos.min() >= .999, (n, cos.min().item()) + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + out = shared_expert_full(local.shared_codes, local.shared_scales, sw, stream=side) + torch.cuda.current_stream().wait_stream(side) + assert torch.equal(out, eager[0]) + + +@pytest.mark.parametrize("n", [1024, 1280, 2048, 4096]) +def test_full_schedules_bitwise(full_weights, n): + """Every gate/up schedule and tile of the full expert stores the same bits (the sweep's candidates).""" + _, ws = full_weights + w = ws[0][0] + local = prepare_mxfp4_moe(make_activation(n, seed=3), w) + q, s, sw = local.shared_codes, local.shared_scales, w.shared_weights + want = shared_dense_full(q, s, sw, config=(((4, 4), "deferred"), ((4, 4), False))) + for t1, sched in (((2, 4), "deferred"), ((4, 4), "early"), ((4, 4), "single"), ((2, 4), "single")): + if sched == "single" and ((n + 255) // 256) * 24 * (4 // t1[0]) > 256: + continue + for t2 in ((4, 4), (2, 4)): + got = shared_dense_full(q, s, sw, config=((t1, sched), (t2, True))) + assert torch.equal(got, want), (n, t1, sched, t2) + + +@pytest.mark.parametrize("m", [1, 17, 96, 513, 640, 641, 2048, 3584, 4096, 4097, 8192, 32768]) +def test_routed_only_partials(full_weights, m): + """The routed-only prepared input: the BF16 entry's routed slots bit for bit, no shared output.""" + _, ws = full_weights + w = ws[0][0] + x = make_activation(m, seed=m) + routed = prepare_mxfp4_moe(x, w).routed() + assert routed.routed_only and routed.shared_codes is None + eager, replay = replay_check(lambda: [mxfp4_moe(routed, w, return_partials=True)[0]]) + slots, shared = mxfp4_moe(routed, w, return_partials=True) + assert shared is None and torch.equal(eager[0], replay[0]) and torch.equal(slots, eager[0]) + assert torch.equal(slots, mxfp4_moe(x, w, return_partials=True)[0]), m + with pytest.raises(ValueError): + mxfp4_moe(routed, w) + + +def test_unsupported_reason_full_and_shard(): + E = 128 + routed = (torch.empty((E, 2 * I, H // 2), dtype=torch.uint8), torch.empty((E, 2 * I, H // 32), dtype=torch.uint8), + torch.empty((E, H, I // 2), dtype=torch.uint8), torch.empty((E, H, I // 32), dtype=torch.uint8)) + f8 = torch.float8_e4m3fn + for i in (I, 2 * I): + shared = (torch.empty((2 * i, H), dtype=f8), torch.empty((2 * i, H // 32), dtype=torch.uint8), + torch.empty((H, i), dtype=f8), torch.empty((H, i // 32), dtype=torch.uint8)) + assert unsupported_reason(*routed, *shared) is None, i + bad = (torch.empty((4 * I, H), dtype=f8), torch.empty((4 * I, H // 32), dtype=torch.uint8), + torch.empty((H, I), dtype=f8), torch.empty((H, I // 32), dtype=torch.uint8)) + assert unsupported_reason(*routed, *bad) is not None + + +def test_full_needs_rank(): + raw = (torch.empty((4 * I, H), dtype=torch.uint8), torch.empty((4 * I, H // 32), dtype=torch.uint8), + torch.empty((H, 2 * I), dtype=torch.uint8), torch.empty((H, 2 * I // 32), dtype=torch.uint8)) + with pytest.raises(AssertionError): + SharedWeights.from_row_major(*raw) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_persistent.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_persistent.py new file mode 100644 index 0000000..70b1a29 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_shared_persistent.py @@ -0,0 +1,98 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Mixed-format persistent launch: independent oracle, changed graphs and metadata reuse.""" +import importlib +import pytest +import torch +from .test_shared_expert import weights,complete_weights +from .workload import make_activation,small_slots +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.reference import shared_mxfp8_ref + +PKG='amd_kernels.moe.mxfp4_h6144_i1536_e128_top4' + +def old_entry(x,w,monkeypatch): + """The routed and shared GEMMs in separate launches: at 449-640 rows the small-tile host's routed slots (the folded + launches: the same routed tiles) and the standalone shared expert, elsewhere the entry with the combined launch + switched off.""" + if 448=.999 + reduced=mxfp4_moe(x,w) + acc=torch.zeros_like(captured[1],dtype=torch.float32) + for k in range(4):acc=acc+captured[0][:,k].float() + assert torch.equal(reduced,(acc+captured[1].float()).bfloat16()) + + +@pytest.mark.parametrize('m',[449,512,640,641,768,769,1024,1025,1536,2048,3072,4096]) +def test_shared_persistent_mfma_corunner(complete_weights,m,monkeypatch): + from tests.mfma_spinner import mfma_spinner + es=importlib.import_module(PKG+'.experts_a4w4_bm128') + x=make_activation(m,seed=m);w=complete_weights + expected=old_entry(x,w,monkeypatch) + torch.cuda.synchronize() + spinner=mfma_spinner(256);stream=torch.cuda.Stream(priority=-1) + spinner(5_000_000,stream) + out=mxfp4_moe(x,w,return_partials=True) + torch.cuda.synchronize() + assert all(torch.equal(a,b) for a,b in zip(out,expected)) + assert all(torch.count_nonzero(t).item()==0 for t in es._ready.values()) + + +@pytest.mark.parametrize('m',[512,640,641,768,769,1024,1025]) +def test_shared_persistent_router_path(complete_weights,m,monkeypatch): + """The combined launch's tiled shared scales come from the split-K top-k below + ROUTER_QUANT_FROM rows and from the line-layout router from there.""" + ch=importlib.import_module(PKG+'.chain') + ll=importlib.import_module(PKG+'.router_mxfp4_mxfp8_quant_line_layout') + rt=importlib.import_module(PKG+'.router_topk_mxfp4_mxfp8_quant') + calls=[] + def counted(name,fn): + def f(*args,**kw): + calls.append((name,kw.get('s8_tiles')));return fn(*args,**kw) + return f + monkeypatch.setattr(ll,'router_mxfp4_mxfp8_quant_line_layout',counted('line',ll.router_mxfp4_mxfp8_quant_line_layout)) + monkeypatch.setattr(rt,'splitk_biased_topk_mxfp4_mxfp8_quant',counted('splitk',rt.splitk_biased_topk_mxfp4_mxfp8_quant)) + mxfp4_moe(make_activation(m,seed=m),complete_weights,return_partials=True) + sh=importlib.import_module(PKG+'.shared_expert') + tiled=sh.shared_in_persistent_for(m) + assert calls==([('splitk',tiled)] if m expert list and per expert the set of (token | slot << 24, weight bits) rows equal a CPU reference (experts +ascending, padded with token = M, weight 0); sets, since LDS atomic order places the rows inside an expert.""" +import collections + +import pytest +import torch + +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.sort_a16w4 import moe_sort_a16w4 +from .workload import TOPK, make_routing, require_gfx950 + +E, BM = 128, 16 + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def _routing(m, kind): + wt, ids = make_routing(m, seed=31 + m, skewed=kind == "skewed") + if kind == "one": # every pair on expert 77 (the slots keep their own rows) + ids = torch.full_like(ids, 77) + elif kind == "four": # every token on experts 3, 64, 65, 127 + ids = torch.tensor([3, 64, 65, 127], dtype=torch.int32, device="cuda").expand(m, TOPK).contiguous() + return wt, ids + + +def _sets(sorted_ids, sorted_w, eids, nv, bm=BM): + n = int(nv[0]) + rows = sorted_ids[:n].cpu().tolist() + w = sorted_w[:n].view(torch.int32).cpu().tolist() + be = eids[:n // bm].cpu().tolist() + per = collections.defaultdict(list) + for b, e in enumerate(be): + per[e] += list(zip(rows[b * bm:(b + 1) * bm], w[b * bm:(b + 1) * bm])) + return n, be, {e: sorted(v) for e, v in per.items()} + + +def _reference(wt, ids, bm=BM): + m = ids.shape[0] + idl, wl = ids.cpu().tolist(), wt.view(torch.int32).cpu().tolist() + per = collections.defaultdict(list) + for t in range(m): + for s in range(TOPK): + per[idl[t][s]].append((t | s << 24, wl[t][s])) + n, be, out = 0, [], {} + for e in sorted(per): + rows = per[e] + pad = -len(rows) % bm + rows = sorted(rows + [(m, 0)] * pad) + out[e] = rows + be += [e] * (len(rows) // bm) + n += len(rows) + return n, be, out + + +# every SORT_TOKEN_TIERS edge (64 / 128 / 192 / 256 / 384 / 448 / 512 / 640) and the next count; 32-row blocks only +# above 512 tokens (at most 256 blocks) +@pytest.mark.parametrize("m", [16, 17, 31, 32, 33, 64, 65, 97, 128, 129, 192, 193, 255, 256, 257, 384, 385, 448, + 449, 512, 513, 577, 640]) +@pytest.mark.parametrize("kind", ["uniform", "skewed", "one", "four"]) +@pytest.mark.parametrize("rank", [True, False]) +@pytest.mark.parametrize("bm", [16, 32]) +def test_sort_sets_and_graph(m, kind, rank, bm): + if m > 512 and bm == 16: + pytest.skip("16-row blocks to 512 tokens") + wt, ids = _routing(m, kind) + ref = _reference(wt, ids, bm) + out = moe_sort_a16w4(ids, wt, E, bm, rank=rank) + torch.cuda.synchronize() + assert _sets(*out, bm=bm) == ref + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + got = moe_sort_a16w4(ids, wt, E, bm, rank=rank) + for seed in (5, 6): + wt2, ids2 = make_routing(m, seed=seed * 1000 + m, skewed=kind == "skewed") + wt.copy_(wt2) + ids.copy_(ids2) + for t in got: + t.fill_(-7) + g.replay() + torch.cuda.synchronize() + assert _sets(*got, bm=bm) == _reference(wt, ids, bm) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_count_all.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_count_all.py new file mode 100644 index 0000000..e4cc43f --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_count_all.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The one-launch routing sort + tile map (``common.sort_count_all``: every CTA counts all pairs) against the routing +(padded totals, block experts, per-expert (row id, weight) sets) and the torch tile map, on uniform, skewed and +4-expert routing; then the scale sort without tile-map CTAs after it, against the torch gather + AITER shuffle.""" + +import pytest +import torch +from aiter.utility.fp4_utils import e8m0_shuffle +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.scale_sort_tile_map import scale_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort import SortBuffers +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_count_all import moe_sort_count_all +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.tile_map import tile_map_grid + +from .workload import E, H, I, TOPK, make_routing, require_gfx950, tile_map_ref + +TOKENS = [1, 7, 513, 777, 1024, 2047, 2048, 3001, 4096] + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def routing(n, kind): + if kind == "four_experts": + w, _ = make_routing(n, seed=n) + ids = torch.arange(TOPK, dtype=torch.int32, device="cuda").expand(n, TOPK).contiguous() + return w, ids + return make_routing(n, seed=n + 3, skewed=kind == "skewed") + + +def run(n, ids, w, bm): + bufs = SortBuffers.allocate(n, E, TOPK, bm, "cuda") + bufs.sorted_ids.fill_(-7) + bufs.sorted_weights.fill_(float("nan")) + bufs.sorted_expert_ids.fill_(-7) + grid = tile_map_grid(bufs.max_sorted // bm, I) + tm = torch.full((grid + 1,), 0x5A5A5A5A, dtype=torch.int32, device="cuda") + moe_sort_count_all(bufs, ids, w, tm, grid, n, E=E, I=I) + return bufs, tm, grid + + +@pytest.mark.parametrize("bm", [128, 256]) +@pytest.mark.parametrize("kind", ["uniform", "skewed", "four_experts"]) +@pytest.mark.parametrize("n", TOKENS) +def test_sort_count_all_matches_routing_and_tile_map(n, kind, bm): + w, ids = routing(n, kind) + got, tm, grid = run(n, ids, w, bm) + counts = torch.bincount(ids.flatten().long(), minlength=E) + padded = (counts + bm - 1) // bm * bm + starts = torch.cumsum(padded, 0) - padded + nv = got.num_valid_ids.tolist() + assert nv[0] == int(padded.sum()) and nv[1] == int(starts[-1]) + blocks = nv[0] // bm + expect_eid = torch.repeat_interleave(torch.arange(E, device="cuda"), padded // bm) + assert torch.equal(got.sorted_expert_ids[:blocks].long(), expect_eid) + sid = got.sorted_ids[: nv[0]].long() + sw = got.sorted_weights[: nv[0]] + pad = sid == n + assert torch.equal(sw[pad], torch.zeros_like(sw[pad])) + tok, slot = sid[~pad] & 0xFFFFFF, sid[~pad] >> 24 + row_expert = torch.repeat_interleave(torch.arange(E, device="cuda"), padded)[~pad] + pair = tok * TOPK + slot + assert torch.equal(torch.sort(pair).values, torch.arange(n * TOPK, device="cuda")) + assert torch.equal(ids.flatten().long()[pair], row_expert) + assert torch.equal(w.flatten()[pair], sw[~pad]) + within = torch.arange(nv[0], device="cuda") - torch.repeat_interleave(starts, padded) + assert torch.equal(pad, within >= torch.repeat_interleave(counts, padded)) + ref_tm = tile_map_ref(got.sorted_expert_ids, got.num_valid_ids, grid, bm) + assert torch.equal(tm, ref_tm) + # the scale sort without tile-map CTAs: each valid row its token's e8m0 bytes, padding rows 0, AITER-shuffled; the + # tile map untouched + g = torch.Generator(device="cuda").manual_seed(n) + xs = torch.randint(100, 140, (n, H // 32), dtype=torch.uint8, device="cuda", generator=g) + a_s = torch.full(((got.max_sorted + 31) // 32 * 32, H // 32), 0xA5, dtype=torch.uint8, device="cuda") + scale_sort_tile_map( + xs, got.sorted_ids, got.sorted_expert_ids, got.num_valid_ids, a_s, tm, grid, n, bm, with_tile_map=False + ) # fmt: skip + gathered = torch.zeros((nv[0], H // 32), dtype=torch.uint8, device="cuda") + gathered[~pad] = xs[tok] + want = e8m0_shuffle(gathered).view(torch.uint8).reshape(-1)[: nv[0] * (H // 32)] + assert torch.equal(a_s.reshape(-1)[: nv[0] * (H // 32)], want) + assert torch.equal(tm, ref_tm) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_scales_all.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_scales_all.py new file mode 100644 index 0000000..2984268 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_scales_all.py @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The routing sort + tile map + sorted scale tiles in one launch (``common.sort_scales_all``) against the routing, the +``sort_count_all`` + scale sort pair (padded totals, block experts, per-expert (row id, weight) sets, tile map) and the +torch gather + AITER shuffle, on uniform, skewed, 4-expert and duplicate-id routing, odd token counts, every +tokens-per-thread tier and both block sizes; rows come out in a fixed order (two runs, identical bits).""" + +import pytest +import torch +from aiter.utility.fp4_utils import e8m0_shuffle +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.scale_sort_tile_map import scale_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort import SortBuffers +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_count_all import moe_sort_count_all +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_scales_all import MAX_TOKENS, moe_sort_scales_all +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.tile_map import tile_map_grid + +from .workload import E, H, I, TOPK, make_routing, require_gfx950 + +TOKENS = [1, 7, 513, 777, 1024, 1025, 2047, 2048, 3001, 4096, 8192] + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +def routing(n, kind): + if kind == "four_experts": + w, _ = make_routing(n, seed=n) + return w, torch.arange(TOPK, dtype=torch.int32, device="cuda").expand(n, TOPK).contiguous() + if kind == "duplicates": # two slots of a token on one expert + w, ids = make_routing(n, seed=n + 5) + ids[:, 1] = ids[:, 0] + ids[: n // 2, 3] = 9 + return w, ids.contiguous() + return make_routing(n, seed=n + 3, skewed=kind == "skewed") + + +def scales(n): + g = torch.Generator(device="cuda").manual_seed(n) + return torch.randint(100, 140, (n, H // 32), dtype=torch.uint8, device="cuda", generator=g) + + +def run(n, ids, w, xs, bm, fused): + bufs = SortBuffers.allocate(n, E, TOPK, bm, "cuda") + bufs.sorted_ids.fill_(-7) + bufs.sorted_weights.fill_(float("nan")) + bufs.sorted_expert_ids.fill_(-7) + grid = tile_map_grid(bufs.max_sorted // bm, I) + tm = torch.full((grid + 1,), 0x5A5A5A5A, dtype=torch.int32, device="cuda") + a_s = torch.full(((bufs.max_sorted + 31) // 32 * 32, H // 32), 0xA5, dtype=torch.uint8, device="cuda") + if fused: + moe_sort_scales_all(bufs, ids, w, xs, a_s, tm, grid, n, E=E, I=I) + else: + moe_sort_count_all(bufs, ids, w, tm, grid, n, E=E, I=I) + scale_sort_tile_map( + xs, bufs.sorted_ids, bufs.sorted_expert_ids, bufs.num_valid_ids, a_s, tm, grid, n, bm, + with_tile_map=False, + ) # fmt: skip + return bufs, tm, a_s + + +def per_expert(bufs, bm): + nv = int(bufs.num_valid_ids[0]) + eids = bufs.sorted_expert_ids[: nv // bm].tolist() + ids, ws = bufs.sorted_ids[:nv].tolist(), bufs.sorted_weights[:nv].tolist() + per = {} + for i, e in enumerate(eids): + per.setdefault(e, []).extend(zip(ids[i * bm : (i + 1) * bm], ws[i * bm : (i + 1) * bm])) + return {e: sorted(v) for e, v in per.items()} + + +@pytest.mark.parametrize("bm", [128, 256]) +@pytest.mark.parametrize("kind", ["uniform", "skewed", "four_experts", "duplicates"]) +@pytest.mark.parametrize("n", TOKENS) +def test_sort_scales_all_matches_routing_and_two_launches(n, kind, bm): + w, ids = routing(n, kind) + xs = scales(n) + got, tm, a_s = run(n, ids, w, xs, bm, True) + counts = torch.bincount(ids.flatten().long(), minlength=E) + padded = (counts + bm - 1) // bm * bm + starts = torch.cumsum(padded, 0) - padded + nv = got.num_valid_ids.tolist() + assert nv[0] == int(padded.sum()) and nv[1] == int(starts[-1]) + blocks = nv[0] // bm + expect_eid = torch.repeat_interleave(torch.arange(E, device="cuda"), padded // bm) + assert torch.equal(got.sorted_expert_ids[:blocks].long(), expect_eid) + sid = got.sorted_ids[: nv[0]].long() + sw = got.sorted_weights[: nv[0]] + pad = sid == n + assert torch.equal(sw[pad], torch.zeros_like(sw[pad])) + tok, slot = sid[~pad] & 0xFFFFFF, sid[~pad] >> 24 + row_expert = torch.repeat_interleave(torch.arange(E, device="cuda"), padded)[~pad] + pair = tok * TOPK + slot + assert torch.equal(torch.sort(pair).values, torch.arange(n * TOPK, device="cuda")) + assert torch.equal(ids.flatten().long()[pair], row_expert) + assert torch.equal(w.flatten()[pair], sw[~pad]) + within = torch.arange(nv[0], device="cuda") - torch.repeat_interleave(starts, padded) + assert torch.equal(pad, within >= torch.repeat_interleave(counts, padded)) + # scale tiles: each valid row its token's e8m0 bytes, padding rows 0, AITER-shuffled + gathered = torch.zeros((nv[0], H // 32), dtype=torch.uint8, device="cuda") + gathered[~pad] = xs[tok] + want = e8m0_shuffle(gathered).view(torch.uint8).reshape(-1)[: nv[0] * (H // 32)] + assert torch.equal(a_s.reshape(-1)[: nv[0] * (H // 32)], want) + # the two-launch pair: the same table, rows per expert and tile map + ref, ref_tm, _ = run(n, ids, w, xs, bm, False) + assert ref.num_valid_ids.tolist() == nv + assert per_expert(got, bm) == per_expert(ref, bm) + assert torch.equal(tm, ref_tm) + # a fixed row order: a second run writes the same bits + again, tm2, a_s2 = run(n, ids, w, xs, bm, True) + assert torch.equal(again.sorted_ids[: nv[0]], got.sorted_ids[: nv[0]]) + assert torch.equal(again.sorted_weights[: nv[0]], got.sorted_weights[: nv[0]]) + assert torch.equal(a_s2.reshape(-1)[: nv[0] * (H // 32)], a_s.reshape(-1)[: nv[0] * (H // 32)]) + assert torch.equal(tm2, tm) + + +def test_sort_scales_all_rejects_more_tokens(): + w, ids = make_routing(MAX_TOKENS + 1, seed=1) + with pytest.raises(ValueError): + run(MAX_TOKENS + 1, ids, w, scales(MAX_TOKENS + 1), 256, True) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_tile_map.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_tile_map.py new file mode 100644 index 0000000..d2a22c7 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sort_tile_map.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""CTA-major histogram sort + exact GEMM1 tile map (``common.sort_tile_map``): the workspace against per-CTA torch +histograms, row / weight sets against the routing, the map against the torch tile map, gathered scales +against torch + AITER, every writable buffer guarded, eager and replayed.""" +import pytest +import torch +from aiter.utility.fp4_utils import e8m0_shuffle +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.scale_sort_tile_map import scale_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort import SORT_CTAS, SortBuffers +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.sort_tile_map import moe_sort_tile_map +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.common.tile_map import tile_map_grid +from .test_sort_scales_all import routing, scales +from .workload import E, TOPK, H, I, require_gfx950, tile_map_ref + + +@pytest.fixture(autouse=True) +def _gpu(): + require_gfx950() + + +@pytest.mark.parametrize('n', [8192, 8193, 12289, 16384, 16385, 32767, 32768, 65536, 87381]) +@pytest.mark.parametrize('kind', ['uniform', 'skewed', 'four_experts', 'duplicates']) +def test_cta_histogram_sort_and_map(n, kind): + bm = 256 + weights, ids = routing(n, kind) + xs = scales(n) + b = SortBuffers.allocate(n, E, TOPK, bm, 'cuda') + grid = tile_map_grid(b.max_sorted // bm, I) + guards = [] + + def guarded(size, dtype): + whole = torch.full((size + 128,), 37, dtype=dtype, device='cuda') + guards.append(whole) + return whole[64:-64] + + for name in ['sorted_ids', 'sorted_expert_ids', 'num_valid_ids', 'sorted_weights', 'block_offsets']: + old = getattr(b, name) + setattr(b, name, guarded(old.numel(), old.dtype)) + tm = guarded(grid + 1, torch.int32) + out = guarded(b.max_sorted * (H // 32), torch.uint8) + + def run(): + moe_sort_tile_map(b, ids, weights, tm, grid, n, E=E) + scale_sort_tile_map(xs, b.sorted_ids, b.sorted_expert_ids, b.num_valid_ids, + out, tm, grid, n, bm, with_tile_map=False) + + run() + torch.cuda.synchronize() + counts = torch.bincount(ids.long().flatten(), minlength=E) + padded = (counts + bm - 1) // bm * bm + starts = padded.cumsum(0) - padded + nv = int(padded.sum()) + assert b.num_valid_ids.tolist() == [nv, int(starts[-1])] + assert torch.equal(b.sorted_expert_ids[:nv // bm].long(), + torch.repeat_interleave(torch.arange(E, device='cuda'), padded // bm)) + pair_cta = torch.arange(n * TOPK, device='cuda') // ((n * TOPK + SORT_CTAS - 1) // SORT_CTAS) + want_hist = torch.bincount(pair_cta * E + ids.long().flatten(), minlength=SORT_CTAS * E) + assert torch.equal(b.block_offsets.long(), want_hist) + sid = b.sorted_ids[:nv].long() + tok, slot = sid & 0xFFFFFF, sid >> 24 + valid = tok < n + pair = tok[valid] * TOPK + slot[valid] + assert torch.equal(pair.sort().values, torch.arange(n * TOPK, device='cuda')) + row_e = torch.repeat_interleave(torch.arange(E, device='cuda'), padded) + assert torch.equal(row_e[valid], ids.flatten().long()[pair]) + assert torch.equal(b.sorted_weights[:nv][valid], weights.flatten()[pair]) + assert torch.equal(b.sorted_weights[:nv][~valid], torch.zeros_like(b.sorted_weights[:nv][~valid])) + within = torch.arange(nv, device='cuda') - torch.repeat_interleave(starts, padded) + assert torch.equal(~valid, within >= torch.repeat_interleave(counts, padded)) + independent_tm = tile_map_ref(b.sorted_expert_ids, b.num_valid_ids, grid, bm) + assert torch.equal(tm, independent_tm) + gathered = torch.zeros((nv, H // 32), dtype=torch.uint8, device='cuda') + gathered[valid] = xs[tok[valid]] + want_scale = e8m0_shuffle(gathered).view(torch.uint8).flatten()[:nv * (H // 32)] + assert torch.equal(out[:nv * (H // 32)], want_scale) + for whole in guards: + assert torch.all(whole[:64] == 37) and torch.all(whole[-64:] == 37) + # Rebuild all persistent sort state, then replay over poisoned output buffers. + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + run() + for name in ['sorted_ids', 'sorted_expert_ids', 'num_valid_ids', 'block_offsets']: + getattr(b, name).fill_(-1) + b.sorted_weights.fill_(float('nan')) + tm.fill_(-1) + out.fill_(165) + torch.cuda.synchronize() + graph.replay() + torch.cuda.synchronize() + assert b.num_valid_ids.tolist() == [nv, int(starts[-1])] + assert torch.equal(tm, independent_tm) + # Sorting may change row order; compare scales to this replay's row order. + tok = b.sorted_ids[:nv].long() & 0xFFFFFF + valid = tok < n + gathered.zero_() + gathered[valid] = xs[tok[valid]] + want_scale = e8m0_shuffle(gathered).view(torch.uint8).flatten()[:nv * (H // 32)] + assert torch.equal(out[:nv * (H // 32)], want_scale) + for whole in guards: + assert torch.all(whole[:64] == 37) and torch.all(whole[-64:] == 37) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sp_prefill_warm_up.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sp_prefill_warm_up.py new file mode 100644 index 0000000..9f084f3 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_sp_prefill_warm_up.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""vLLM's start-up order with the library's SP prefill, in a fresh process: the BF16 entry called at every warm-up +size, its partials mode, then sp_prefill_akr's first call (the compact partials input of a 32-row shard). The compact +mode's warm-up must launch nothing: a launch there runs the GEMM2 the BF16 calls loaded at 97-128 rows in its 17-row +step, on a sort only compiled there, and the server faults on every rank.""" +import json +import os +import subprocess +import sys + +import pytest + +# the compact shapes the SP path reaches: the 17-row step that faulted, the warm-up shard, the smallest SP prefill +COMPACT_ROWS = (17, 32, 96, 4096) + + +def scenario(): + import torch + from flydsl.compiler.jit_function import CompiledFunction + from amd_kernels import warm_up + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, mxfp4_moe, prepare_mxfp4_moe + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.chain import WARM_UP_TOKENS + from .test_shared_expert import weights + from .workload import H, make_activation, make_experts + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + raw, _ = make_experts() + g = torch.Generator(device='cuda').manual_seed(5) + w = MoeWeights.from_row_major(torch.randn(128, H, device='cuda', generator=g) / H**.5, + torch.randn(128, device='cuda', generator=g) * 1e-2, *raw) + _, w.shared_weights = weights.__wrapped__() + in_warm_up = [] + launch = CompiledFunction.__call__ + + def spy(self, *args): # a loaded handle's launch; the warm-up may launch none + if warm_up._in_warm_up.get(): + in_warm_up.append(1) + return launch(self, *args) + + CompiledFunction.__call__ = spy + + def zeros(m): + return torch.zeros((m, H), device='cuda', dtype=torch.bfloat16) + + with torch.inference_mode(): + for m in WARM_UP_TOKENS: # akr_moe.warm_up + mxfp4_moe(zeros(m), w) + mxfp4_moe(zeros(WARM_UP_TOKENS[0]), w, return_partials=True) + # sp_prefill_akr._warm_up: the compact mode's first call + mxfp4_moe(prepare_mxfp4_moe(zeros(32), w).compact(), w, return_partials=True) + torch.cuda.synchronize() + late = warm_up.late_compiles() + same = [] + for m in COMPACT_ROWS: # the gathered input's contract: the plain entry's partials, bit for bit + x = make_activation(m, seed=m) + expected = mxfp4_moe(x, w, return_partials=True) + got = mxfp4_moe(prepare_mxfp4_moe(x, w).compact(), w, return_partials=True) + same.append(all(torch.equal(a, b) and bool(torch.isfinite(a).all()) for a, b in zip(expected, got))) + torch.cuda.synchronize() + return {'launches_in_warm_up': len(in_warm_up), 'late_before': late, 'late': warm_up.late_compiles() - late, + 'same': same} + + +def test_sp_prefill_compact_warm_up_launches_nothing(): + import torch + if not torch.cuda.is_available(): + pytest.skip('needs a GPU') + proc = subprocess.run([sys.executable, '-m', __name__], capture_output=True, text=True, timeout=900, + env={**os.environ, 'FLYDSL_RUNTIME_ENABLE_CACHE': '1'}) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + print(result) + assert result['launches_in_warm_up'] == 0 + assert result['late_before'] == 0 and result['late'] == 0 + assert all(result['same']), dict(zip(COMPACT_ROWS, result['same'])) + + +if __name__ == '__main__': + print(json.dumps(scenario())) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_unsupported_reason.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_unsupported_reason.py new file mode 100644 index 0000000..16ece15 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_unsupported_reason.py @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``unsupported_reason``: None for the checkpoint layout ``MoeWeights.from_row_major`` takes, a reason for a wrong +shape, dtype or SwiGLU constant (meta tensors: no GPU).""" + +import torch +from amd_kernels.moe import mxfp4_h6144_i1536_e128_top4 as P + +H, I, E = P.H, P.I, P.E # noqa: E741 + + +def _weights(**override): + m = dict(device="meta") + w = dict( + w13=torch.empty((E, 2 * I, H // 2), dtype=torch.uint8, **m), + w13_scale=torch.empty((E, 2 * I, H // 32), dtype=torch.uint8, **m), + w2=torch.empty((E, H, I // 2), dtype=torch.uint8, **m), + w2_scale=torch.empty((E, H, I // 32), dtype=torch.uint8, **m), + shared_w13=torch.empty((2 * I, H), dtype=torch.float8_e4m3fn, **m), + shared_s13=torch.empty((2 * I, H // 32), dtype=torch.uint8, **m), + shared_w2=torch.empty((H, I), dtype=torch.float8_e4m3fn, **m), + shared_s2=torch.empty((H, I // 32), dtype=torch.uint8, **m), + ) + w.update(override) + return w + + +def test_checkpoint_layout_is_supported(): + assert P.unsupported_reason(**_weights()) is None + + +def test_reasons(): + assert "w2:" in P.unsupported_reason(**_weights(w2=torch.empty((E, H, I), dtype=torch.uint8, device="meta"))) + bf16 = torch.empty((2 * I, H), dtype=torch.bfloat16, device="meta") + assert "shared w13:" in P.unsupported_reason(**_weights(shared_w13=bf16)) + assert "SwiGLU" in P.unsupported_reason(**_weights(), swiglu=(1.0, 7.0, 1.0)) + assert "w13:" in P.unsupported_reason(**_weights(w13=None)) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_full_shared.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_full_shared.py new file mode 100644 index 0000000..7b1c6ba --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_full_shared.py @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``warm_up_mxfp4_moe(w, partials=True, sp=True)`` on the full shared expert copy in a fresh process and an empty +FlyDSL cache: the BF16 entry's view variants (both modes), the preparation, the routed-only partials and +``shared_expert_full``; afterwards the served row counts compile nothing and no warm-up launched a kernel.""" +import json +import os +import subprocess +import sys + +import pytest + +BF16_ROWS = (1, 8, 16, 17, 33, 95, 96, 300, 640, 641, 1024, 2049, 3583, 3584, 4097, 8191, 16385) +SP_ROWS = (2048, 2049, 4096, 4097, 8192, 8885, 16384, 16385, 31490, 32768) +FULL_ROWS = (1, 1024, 1280, 1281, 2048, 2560, 2561, 4096, 4097, 7873, 16384) + + +def scenario(): + import torch + from flydsl.compiler.jit_function import CompiledFunction, MlirCompiler + + from amd_kernels import warm_up + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import (MoeWeights, SharedWeights, mxfp4_moe, prepare_mxfp4_moe, + shared_expert_full, warm_up_mxfp4_moe) + from .workload import H, make_activation, make_experts, make_full_shared + + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + torch.set_num_threads(8) + raw, _ = make_experts() + full, _ = make_full_shared() + w = MoeWeights.from_row_major(torch.randn(128, H, device="cuda") / H**0.5, torch.zeros(128, device="cuda"), *raw, + shared=SharedWeights.from_row_major(*full, tp_rank=1)) + in_warm_up = [] + launch = CompiledFunction.__call__ + + def spy_launch(self, *args): + if warm_up._in_warm_up.get(): + in_warm_up.append(1) + return launch(self, *args) + + CompiledFunction.__call__ = spy_launch + warm_up_mxfp4_moe(w, partials=True, sp=True) + x = torch.zeros((4096, H), device="cuda", dtype=torch.bfloat16) + prepare_mxfp4_moe(x, w) # the preparation's own entry (the hand-offs warm it in their shard compute) + CompiledFunction.__call__ = launch + count = [] + original = MlirCompiler.compile + + def spy(self, *args, **kw): + if warm_up._in_warm_up.get() or warm_up._current.get() is not None: + count.append(1) + return original(self, *args, **kw) + + MlirCompiler.compile = spy + late = [] + for m in BF16_ROWS: + x = make_activation(m, seed=m) + mxfp4_moe(x, w) + mxfp4_moe(x, w, return_partials=True) + torch.cuda.synchronize() + if count: + late.append(("bf16", m)) + count.clear() + for m in SP_ROWS: + p = prepare_mxfp4_moe(make_activation(m, seed=m), w) + mxfp4_moe(p.routed(), w, return_partials=True) + torch.cuda.synchronize() + if count: + late.append(("routed", m)) + count.clear() + for m in FULL_ROWS: + p = prepare_mxfp4_moe(make_activation(m, seed=m), w) + shared_expert_full(p.shared_codes, p.shared_scales, w.shared_weights) + torch.cuda.synchronize() + if count: + late.append(("full", m)) + count.clear() + modes = sorted(str(m) for name, m in warm_up.warmed()) + return {"late": late, "launches_in_warm_up": len(in_warm_up), "modes": modes, + "late_reported": warm_up.late_compiles()} + + +def test_sp_full_shared_warm_up(tmp_path): + import torch + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=2400, + env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", + "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + print(result) + assert not result["late"] and result["late_reported"] == 0 + assert result["launches_in_warm_up"] == 0 + assert any("routed" in m and "full" in m for m in result["modes"]), result["modes"] + assert any("'bf16', 'full'" in m for m in result["modes"]), result["modes"] + + +if __name__ == "__main__": + print(json.dumps(scenario())) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_modes.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_modes.py new file mode 100644 index 0000000..033067c --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/test_warm_up_modes.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``warm_up_mxfp4_moe`` in a fresh process: the reduced mode, with ``partials`` the partials mode too, once per mode +(a repeat calls nothing), the node's default worker count; afterwards the served row counts compile nothing.""" +import json +import os +import subprocess +import sys + +import pytest + +ENTRY = "moe/mxfp4_h6144_i1536_e128_top4" +ROWS = (1, 8, 300, 2049, 8191) + + +def scenario(): + import torch + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels import warm_up + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, chain, warm_up_mxfp4_moe + + from .test_shared_expert import weights + from .workload import H, make_activation, make_experts + + warm_up.set_auto(True) # tests/__init__.py turns it off for the single-size tests + torch.set_num_threads(8) + raw, _ = make_experts() + w = MoeWeights.from_row_major(torch.randn(128, H, device="cuda") / H**0.5, torch.zeros(128, device="cuda"), *raw) + _, w.shared_weights = weights.__wrapped__() + + def modes(): + return sorted(bool(m[3]) for name, m in warm_up.warmed() if name == ENTRY) + + warm_up_mxfp4_moe(w) + reduced = modes() + warm_up_mxfp4_moe(w, partials=True) + both = modes() + calls = [] + real = chain.mxfp4_moe + chain.mxfp4_moe = lambda *a, **kw: calls.append(1) or real(*a, **kw) + warm_up_mxfp4_moe(w, partials=True) + chain.mxfp4_moe = real + count = [] + original = MlirCompiler.compile + + def spy(self, *args, **kw): + if warm_up._in_warm_up.get() or warm_up._current.get() is not None: + count.append(1) + return original(self, *args, **kw) + + MlirCompiler.compile = spy + late = [] + for m in ROWS: + x = make_activation(m, seed=m) + real(x, w) + real(x, w, return_partials=True) + torch.cuda.synchronize() + if count: + late.append(m) + count.clear() + return {"reduced": reduced, "both": both, "repeat_calls": len(calls), "late": late, + "workers": warm_up._workers, "node_workers": chain._node_workers()} + + +def test_warm_up_modes_once_each(tmp_path): + import torch + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + proc = subprocess.run([sys.executable, "-m", __name__], capture_output=True, text=True, timeout=1800, + env={**os.environ, "FLYDSL_RUNTIME_ENABLE_CACHE": "1", + "FLYDSL_RUNTIME_CACHE_DIR": str(tmp_path / "flydsl")}) + assert proc.returncode == 0, proc.stdout[-4000:] + proc.stderr[-4000:] + result = json.loads(proc.stdout.strip().splitlines()[-1]) + print(result) + assert result["reduced"] == [False] and result["both"] == [False, True] + assert result["repeat_calls"] == 0 + assert result["workers"] == result["node_workers"] + assert not result["late"] + + +if __name__ == "__main__": + print(json.dumps(scenario())) diff --git a/tests/moe/mxfp4_h6144_i1536_e128_top4/workload.py b/tests/moe/mxfp4_h6144_i1536_e128_top4/workload.py new file mode 100644 index 0000000..2137c63 --- /dev/null +++ b/tests/moe/mxfp4_h6144_i1536_e128_top4/workload.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Shared test and benchmark inputs: MXFP4 routed experts in the chain's layouts, routing, activations, replay.""" + +import pytest +import torch +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.layout import w2_layout, w13_layout +from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.prefill_h6144_i1536 import H, I, Mxfp4PrefillScales + +E, TOPK = 128, 4 # the served shape: 128 routed experts, top-4 (the shared slot runs elsewhere) +ROUTED_SCALING = 2.0 # MiniMax-M3.1; the top-k kernel folds it into the weights + +# vLLM's W4A4 call sizes (W4A4_FROM_TOKENS 1536 up to max_num_batched_tokens 65536), some off the 32 / 256-row grid +SERVED_TOKENS = (1536, 2049, 4097, 16384, 32768, 65536) + + +def require_gfx950(): + if not torch.cuda.is_available(): + pytest.skip("needs a GPU") + arch = torch.cuda.get_device_properties(0).gcnArchName.split(":")[0] + if arch != "gfx950": + pytest.skip(f"gfx950 only, found {arch}") + + +def make_experts(num_experts: int = E, seed: int = 0, cpu_gen: bool = False): + """Random MXFP4 experts, scales over 6 binades (the checkpoint's span <= 7): (raw, (w13, w2, scales)), ``raw`` = + (w13 codes, w13 e8m0, w2 codes, w2 e8m0) row-major. ``cpu_gen``: vLLM's test's CPU generator (its exact tensors).""" + if cpu_gen: + g = torch.Generator().manual_seed(seed) + + def rnd(lo, hi, shape): + return torch.randint(lo, hi, shape, dtype=torch.uint8, generator=g).cuda() + + w13, s13 = rnd(0, 256, (num_experts, 2 * I, H // 2)), rnd(118, 124, (num_experts, 2 * I, H // 32)) + w2, s2 = rnd(0, 256, (num_experts, H, I // 2)), rnd(118, 124, (num_experts, H, I // 32)) + else: + g = torch.Generator(device="cuda").manual_seed(seed) + + def rnd(lo, hi, shape): + return torch.randint(lo, hi, shape, dtype=torch.uint8, device="cuda", generator=g) + + w13, s13 = rnd(0, 256, (num_experts, 2 * I, H // 2)), rnd(118, 124, (num_experts, 2 * I, H // 32)) + w2, s2 = rnd(0, 256, (num_experts, H, I // 2)), rnd(118, 124, (num_experts, H, I // 32)) + ops = (w13_layout(w13), w2_layout(w2), Mxfp4PrefillScales.from_row_major(s13, s2)) + return (w13, s13, w2, s2), ops + + +def make_shared(seed: int = 607): + """Random MXFP8 shared-expert weights: (raw, SharedWeights), ``raw`` = (w13 e4m3 [3072, 6144] u8, e8m0 [3072, 192], + w2 e4m3 [6144, 1536] u8, e8m0 [6144, 48]) row-major.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import SharedWeights + g = torch.Generator(device="cuda").manual_seed(seed) + + def weight(n, k): + q = (torch.randn((n, k), device="cuda", generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + s = torch.randint(118, 124, (n, k // 32), device="cuda", dtype=torch.uint8, generator=g) + return q, s + + raw = (*weight(2 * I, H), *weight(H, I)) + return raw, SharedWeights.from_row_major(*raw) + + +def make_full_shared(seed: int = 607): + """The random full MXFP8 shared expert (I 3072, row-major ``raw`` = w13 [6144, 6144] u8 (gate rows, then up rows), + e8m0 [6144, 192], w2 [6144, 3072] u8, e8m0 [6144, 96]) and its TP2 shards: (raw, [shard raw of rank 0, of rank + 1]), each shard what a TP2 checkpoint loader gives the rank.""" + g = torch.Generator(device="cuda").manual_seed(seed) + + def weight(n, k): + q = (torch.randn((n, k), device="cuda", generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + s = torch.randint(118, 124, (n, k // 32), device="cuda", dtype=torch.uint8, generator=g) + return q, s + + raw = (*weight(4 * I, H), *weight(H, 2 * I)) + w13, s13, w2, s2 = raw + shards = [] + for r in range(2): + rows = torch.cat([torch.arange(r * I, (r + 1) * I), torch.arange(2 * I + r * I, 2 * I + (r + 1) * I)]).cuda() + shards.append((w13[rows].contiguous(), s13[rows].contiguous(), w2[:, r * I:(r + 1) * I].contiguous(), + s2[:, r * I // 32:(r + 1) * I // 32].contiguous())) + return raw, shards + + +def small_slots(x, w, topk_weights, topk_ids, **kwargs): + """The small-tile host with the shared tiles folded in (the BF16 entry below 96 rows; ``id_rows`` at its rows): + (slots [M, 4, H], shared output [M, H]) of ``MoeWeights`` ``w`` on the BF16 ``x`` and the given routing.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.experts_a4w4_small import a4w4_experts, id_rows_for + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.quant import mxfp4_quant + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4.shared_quant import shared_quant + kwargs.setdefault("id_rows", id_rows_for(x.shape[0])) + return a4w4_experts(mxfp4_quant(x), w.w13, w.w2, w.a4w4_scales, topk_weights, topk_ids, + shared=(*shared_quant(x), w.shared_weights), **kwargs) + + +def make_routing(tokens: int, num_experts: int = E, seed: int = 11, skewed: bool = False): + """(fp32 weights [M, 4], int32 ids [M, 4]) as the top-k kernel hands the chain: distinct experts per token, weights + renormalized times the routed scaling. ``skewed``: expert probabilities ~ rank^-1.2 (some experts get no row).""" + g = torch.Generator(device="cuda").manual_seed(seed) + if skewed: + p = torch.arange(1, num_experts + 1, device="cuda", dtype=torch.float32) ** -1.2 + ids = torch.multinomial(p.expand(tokens, num_experts), TOPK, generator=g) + else: + ids = torch.rand((tokens, num_experts), device="cuda", generator=g).topk(TOPK).indices + w = torch.rand((tokens, TOPK), device="cuda", generator=g) + 0.5 + w = w / w.sum(1, keepdim=True) * ROUTED_SCALING + return w.contiguous(), ids.int().contiguous() + + +def make_activation(tokens: int, seed: int = 13, scale: float = 0.5): + g = torch.Generator(device="cuda").manual_seed(seed) + return (torch.randn((tokens, H), device="cuda", generator=g) * scale).bfloat16() + + +def make_mxfp8(tokens: int, seed: int = 17): + """A gathered MXFP8 input: (e4m3 codes [M, H] u8 without NaN, e8m0 [M, H/32]).""" + g = torch.Generator(device="cuda").manual_seed(seed) + codes = torch.randint(0, 256, (tokens, H), dtype=torch.uint8, device="cuda", generator=g) + codes[(codes & 0x7F) == 0x7F] = 0 # e4m3 NaN + scales = torch.randint(110, 145, (tokens, H // 32), dtype=torch.uint8, device="cuda", generator=g) + return codes, scales + + +def two_launch_slots(x_quant, w, topk_weights, topk_ids): + """The slots [M, 4, H] of the BM256 host's two launches at any M (``experts_a4w4`` above ``GEMM12_TOKENS``: the + persistent full-height GEMM1, then a GEMM2 that stores every valid row): the bitwise oracle of the one-launch and + short-tile kernels.""" + from unittest import mock + + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import experts_a4w4 + with mock.patch.object(experts_a4w4, "GEMM12_TOKENS", (0, 0)): + return experts_a4w4.a4w4_experts(x_quant, w.w13, w.w2, w.a4w4_scales, topk_weights, topk_ids) + + +def tile_map_ref(sorted_expert_ids, num_valid_ids, grid, bm): + """GEMM1's tile map (``common/tile_map.py``) of a routing sort's block experts, in torch: each expert's run of + m-blocks n-major, -1 to ``grid``, the valid entry count last.""" + nb_n = I // 128 + n_bits = (nb_n - 1).bit_length() + vb = int(num_valid_ids[0]) // bm + eids = sorted_expert_ids[:vb].tolist() + tm = [-1] * (grid + 1) + lo = 0 + while lo < vb: + hi = lo + while hi < vb and eids[hi] == eids[lo]: + hi += 1 + for m in range(lo, hi): + for n in range(nb_n): + tm[nb_n * lo + n * (hi - lo) + (m - lo)] = m << n_bits | n + lo = hi + tm[grid] = vb * nb_n + return torch.tensor(tm, dtype=torch.int32, device="cuda") + + +def replay_check(fn, outputs_of=lambda out: out): + """Run ``fn`` eagerly, capture it in a HIP graph, poison the graph's outputs (NaN for floats, 0xA5 bytes otherwise), + replay, and return the (eager, replayed) output lists.""" + eager = [t.clone() for t in outputs_of(fn())] + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = outputs_of(fn()) + for t in captured: + if t.dtype.is_floating_point and t.element_size() > 1: + t.fill_(float("nan")) + else: + t.view(torch.uint8).fill_(0xA5) + graph.replay() + torch.cuda.synchronize() + return eager, captured diff --git a/tests/multi_gpu.py b/tests/multi_gpu.py index 950f56a..54ad3a1 100644 --- a/tests/multi_gpu.py +++ b/tests/multi_gpu.py @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project -"""Run a test body on 4 GPUs: one process per GPU, a gloo process group (a TCPStore on a free local port: AITER's -custom all-reduce exchanges its IPC handles through a pure TCP store). The 4 rank processes start at the first call -and serve every later call of the same pytest process (a call of its own would pay for 4 new ones: Python, torch, a -HIP context, the group and the body's warm-up again). A failed call kills them; the next call starts new ones. A body -sees what earlier calls of its file left in the process (as it already saw their FlyDSL disk cache). The body -appends mismatches to ``errors`` instead of raising, so every rank reaches the same collectives; an exception is -recorded and ends that rank.""" +"""Run a test body on 2 or 4 GPUs: one process per GPU, a gloo process group (a TCPStore on a free local port: +AITER's custom all-reduce exchanges its IPC handles through a pure TCP store). The rank processes start at the first +call and serve every later call of the same world size in the same pytest process (a call of its own would pay for +new ones: Python, torch, a HIP context, the group and the body's warm-up again). A failed call kills them; the next +call starts new ones. A body sees what earlier calls of its file left in the process (as it already saw their FlyDSL +disk cache). The body appends mismatches to ``errors`` instead of raising, so every rank reaches the same +collectives; an exception is recorded and ends that rank.""" import atexit import os @@ -23,7 +23,7 @@ WORLD = 4 -CPU_THREADS = 8 # per rank: 4 ranks x all cores of torch's default spin on each other +CPU_THREADS = 8 # per rank: ranks on all cores (torch's default) spin on each other EXIT_GRACE_S = 10 # after one rank failed: how long the others get to report before all are killed @@ -37,11 +37,11 @@ def _free_port() -> int: return s.getsockname()[1] -def _serve(rank, port, jobs, results): +def _serve(rank, world, port, jobs, results): torch.set_num_threads(CPU_THREADS) torch.cuda.set_device(rank) dist.init_process_group( - "gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=WORLD + "gloo", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=world ) parent = os.getppid() while True: @@ -70,17 +70,18 @@ def _serve(rank, port, jobs, results): class _Ranks: - """The 4 rank processes, one job queue each, one result queue.""" + """The rank processes, one job queue each, one result queue.""" - def __init__(self): + def __init__(self, world): + self.world = world ctx = mp.get_context("spawn") self.env = _env() self.results = ctx.Queue() - self.jobs = [ctx.Queue() for _ in range(WORLD)] + self.jobs = [ctx.Queue() for _ in range(world)] port = _free_port() self.procs = [ - ctx.Process(target=_serve, args=(r, port, self.jobs[r], self.results)) - for r in range(WORLD) + ctx.Process(target=_serve, args=(r, world, port, self.jobs[r], self.results)) + for r in range(world) ] for p in self.procs: p.start() @@ -91,7 +92,7 @@ def run(self, fn, args, timeout): q.put((fn, args)) done = {} deadline = time.monotonic() + timeout - while len(done) < WORLD and time.monotonic() < deadline: + while len(done) < self.world and time.monotonic() < deadline: try: rank, errors, ok = self.results.get(timeout=1) done[rank] = (errors, ok) @@ -125,23 +126,33 @@ def _close_ranks(): _ranks = None -def run_on_4_gpus(fn, *args, timeout=900): - """``fn(rank, errors, *args)`` on 4 GPUs; fails on any recorded error, an +def run_on_gpus(fn, *args, world, timeout=900): + """``fn(rank, errors, *args)`` on ``world`` GPUs; fails on any recorded error, an exception, a rank that exits or a timeout.""" global _ranks - if torch.cuda.device_count() < WORLD: - pytest.skip("needs 4 GPUs") - if _ranks is not None and _ranks.env != _env(): - _close_ranks() # the ranks run with the environment they started with + if torch.cuda.device_count() < world: + pytest.skip(f"needs {world} GPUs") + if _ranks is not None and (_ranks.env != _env() or _ranks.world != world): + _close_ranks() # the ranks run with the environment and world they started with if _ranks is None: - _ranks = _Ranks() + _ranks = _Ranks(world) done = _ranks.run(fn, args, timeout) msgs = [m for r in sorted(done) for m in done[r][0]] - ok = len(done) == WORLD and all(o for _, o in done.values()) and not msgs + ok = len(done) == world and all(o for _, o in done.values()) and not msgs if not ok: codes = [p.exitcode for p in _ranks.procs] _ranks.close(kill=True) _ranks = None - missing = [r for r in range(WORLD) if r not in done] + missing = [r for r in range(world) if r not in done] assert not missing, f"ranks {missing} did not finish in {timeout} s (exit codes {codes}); {msgs[:4]}" assert not msgs, "\n".join(msgs[:8]) + + +def run_on_4_gpus(fn, *args, timeout=900): + """``fn(rank, errors, *args)`` on 4 GPUs.""" + run_on_gpus(fn, *args, world=4, timeout=timeout) + + +def run_on_2_gpus(fn, *args, timeout=900): + """``fn(rank, errors, *args)`` on 2 GPUs.""" + run_on_gpus(fn, *args, world=2, timeout=timeout) diff --git a/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_allreduce_ranks2.py b/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_allreduce_ranks2.py new file mode 100644 index 0000000..6796f17 --- /dev/null +++ b/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_allreduce_ranks2.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two-rank contract, bf16 quant rule, odd rows, strided residuals, both residual forms, graph overwrite, rank skew.""" +import torch +import torch.distributed as dist +from amd_kernels.norm.allreduce_gemma_bf16_mxfp8_h6144_ranks2 import Bf16ArNormComm, host +from amd_kernels.norm.allreduce_gemma_bf16_mxfp8_h6144_ranks2.reference import ( + allreduce_norm_reference, + reduce_reference, +) +from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an +from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import ( + CODE_MISMATCH, + MXFP8_COS, + SCALE_MISMATCH, + compare_mxfp8, + unshuffle_scales, + untile_codes, +) +from tests.multi_gpu import run_on_2_gpus + +# every rows-per-batch tier and form edge, and the DSpark-7 draft rows (7 per request) +ROWS = (1, 7, 17, 23, 24, 63, 64, 65, 70, 112, 128, 129, 224, 256, 257, 448, 512, 513, 896, 1025, 1792, 2048) + + +def bits(t): + return t.contiguous().view(torch.uint8) + + +def check(got, ref, m, tile, own=None): + """``ref``: the one-GPU bf16 norm of the reduced rows (out, residual out). bf16 out bit for bit; MXFP8 out (of the + bf16-rounded norm) within the family reference's tolerances; the residual bit for bit.""" + if tile is None: + assert torch.equal(bits(got[0]), bits(ref[0])) + else: + q = untile_codes(got[0], m) if tile else got[0][:m] + st = compare_mxfp8(bits(q), unshuffle_scales(got[1], m), ref[0].double(), bf16_input=True) + assert st['scale_max_diff'] <= 1 and st['scale_mismatch'] <= SCALE_MISMATCH, ('s', m, tile, st) + assert st['code_mismatch'] <= CODE_MISMATCH and st['code_far'] == 0 and st['cos'] >= MXFP8_COS, ('q', m, tile, + st) + lo, hi = own if own else (0, m) + assert torch.equal(bits(got[-1][lo:hi]), bits(ref[-1][lo:hi])), ('res', m, tile) + + +def same(got, want, m, tile, own=None): + """Two calls' outputs bit for bit (codes and scales on the written rows; residual on ``own``).""" + if tile is None: + assert torch.equal(bits(got[0]), bits(want[0])) + else: + q = untile_codes(got[0], m) if tile else got[0][:m] + eq = untile_codes(want[0], m) if tile else want[0][:m] + assert torch.equal(bits(q), bits(eq)), ('q', m, tile) + assert torch.equal(unshuffle_scales(got[1], m), unshuffle_scales(want[1], m)), ('s', m, tile) + lo, hi = own if own else (0, m) + assert torch.equal(bits(got[-1][lo:hi]), bits(want[-1][lo:hi])), ('res', m, tile) + + +def worker(rank, errors, rows=ROWS): + dev = torch.device('cuda', rank) + c = Bf16ArNormComm(rank, dist.group.WORLD, dev) + w = (torch.randn(6144, generator=torch.Generator().manual_seed(72)) * 0.2).to(dev, dtype=torch.bfloat16) + c.warm_up(w) + from tests.mfma_spinner import mfma_spinner + spinner = mfma_spinner(256) + side = torch.cuda.Stream(priority=-1) + for m in rows: + torch.manual_seed(m) + exponents = torch.randint(-5, 6, (m, 192, 1), device=dev).expand(m, 192, 32).reshape(m, 6144) + parts = [(torch.randn(m, 6144, device=dev) * torch.exp2(exponents.float())).bfloat16() for _ in range(2)] + for x in parts: + x[0].zero_() + parts[0][:, 64:72] = 2.0**-133 # denormal operands: flushed by the rank sum + parts[1][:, 72:80] = -0.0 + parts[0][:, 128:160] = 2.0**-125 # normal operands that cancel into a denormal + parts[1][:, 128:160] = -(2.0**-125) + 2.0**-133 + rbase = torch.randn((m, 6152), device=dev, dtype=torch.bfloat16) + r = rbase[:, :6144] + for tile in (None, 0, 16, 32, 64, 128): + ar = reduce_reference(parts) + y64, rr = allreduce_norm_reference(parts, r, w) + fn = lambda: (c.ar_add_rmsnorm(parts[rank], r, w) if tile is None # noqa: E731 + else c.ar_add_rmsnorm_mxfp8(parts[rank], r, w, 1e-6, tile)) + ref = an.gemma_add_rmsnorm(ar, r, w, 1e-6) + eager = fn() + torch.cuda.synchronize() + check(eager, ref, m, tile) + eager = [t.clone() for t in eager] + if tile is None: + cos = torch.nn.functional.cosine_similarity(eager[0].double(), y64, dim=1) + assert cos.min() > 0.99999, (m, float(cos.min())) + g = torch.cuda.CUDAGraph() + with c.capture(), torch.cuda.graph(g): + out = fn() + for rep in range(3): + for t in out: + t.view(torch.uint8).fill_(255) + if rep == 2 and tile == 16 and m in (7, 224, 896, 1792): + spinner(100_000, side) + g.replay() + torch.cuda.synchronize() + same(out, eager, m, tile) + # changed graph inputs must be consumed; every rank regenerates the same sign flip for its oracle + for x in parts: + x.neg_() + ar2 = reduce_reference(parts) + ref2 = an.gemma_add_rmsnorm(ar2, r, w, 1e-6) + g.replay() + torch.cuda.synchronize() + check(out, ref2, m, tile) + full = [t.clone() for t in out] + # the owned-residual form: the same outputs, residual on the owned rows only + own = c.owned_rows(m) + fo = lambda: (c.ar_add_rmsnorm(parts[rank], r, w, owned_residual=True) if tile is None # noqa: E731 + else c.ar_add_rmsnorm_mxfp8(parts[rank], r, w, 1e-6, tile, owned_residual=True)) + same(fo(), full, m, tile, own) + go = torch.cuda.CUDAGraph() + with c.capture(), torch.cuda.graph(go): + oo = fo() + for t in oo: + t.view(torch.uint8).fill_(255) + go.replay() + torch.cuda.synchronize() + same(oo, full, m, tile, own) + if rank == 0: + print('contract PASS rows', m, flush=True) + c.close() + + +def test_allreduce_contract(): + run_on_2_gpus(worker, timeout=900) + + +def worker_owned_chain(rank, errors): + """The draft's chain: owned-residual calls feed each other's residual (only the owned rows are valid between).""" + dev = torch.device('cuda', rank) + c = Bf16ArNormComm(rank, dist.group.WORLD, dev) + w = (torch.randn(6144, generator=torch.Generator().manual_seed(3)) * 0.2).to(dev, dtype=torch.bfloat16) + c.warm_up(w) + for m in (7, 21, 28, 63, 64, 224, 448, 1792): + g = torch.Generator(device=dev).manual_seed(m) + resid_ref = torch.randn(m, 6144, generator=g, device=dev).bfloat16() + resid = resid_ref.clone() + lo, hi = c.owned_rows(m) if m >= host.OWNER_MIN_ROWS else (0, m) + for step in range(10): + parts = [torch.randn(m, 6144, generator=g, device=dev).bfloat16() for _ in range(2)] + tile = None if step == 9 else (16, 0)[step % 2] + ref = an.gemma_add_rmsnorm(reduce_reference(parts), resid_ref, w, 1e-6) + if tile is None: + got = c.ar_add_rmsnorm(parts[rank], resid, w, owned_residual=True) + else: + got = c.ar_add_rmsnorm_mxfp8(parts[rank], resid, w, 1e-6, tile, owned_residual=True) + torch.cuda.synchronize() + check(got, ref, m, tile, (lo, hi)) + resid_ref, resid = ref[-1], got[-1] + c.close() + + +def test_owned_residual_chain(): + run_on_2_gpus(worker_owned_chain, timeout=600) + + +def worker_async(rank, errors): + import time + dev = torch.device('cuda', rank) + c = Bf16ArNormComm(rank, dist.group.WORLD, dev) + w = torch.ones(6144, dtype=torch.bfloat16, device=dev) + c.warm_up(w) + history = [] + for call in range(36): + m = (7, 70, 224, 448, 896, 1792, 1, 65, 513)[call % 9] + parts = [torch.randn((m, 6144), generator=torch.Generator().manual_seed(1000 * call + r), + dtype=torch.bfloat16).pin_memory() for r in range(2)] + residual = torch.zeros((m, 6144), dtype=torch.bfloat16, device=dev) + if rank == call % 2: + time.sleep(0.01) + x = parts[rank].to(dev, non_blocking=True) + tile = 16 if call % 2 else None + owner = call % 3 == 0 + out = (c.ar_add_rmsnorm(x, residual, w, owned_residual=owner) if tile is None + else c.ar_add_rmsnorm_mxfp8(x, residual, w, 1e-6, tile, owned_residual=owner)) + history.append((parts, residual, out, m, tile, owner)) + torch.cuda.synchronize() + for parts, residual, out, m, tile, owner in history: + reduced = reduce_reference([p.to(dev) for p in parts]) + ref = an.gemma_add_rmsnorm(reduced, residual, w, 1e-6) + check(out, ref, m, tile, c.owned_rows(m) if owner else None) + c.close() + + +def test_eager_rank_skew_and_slot_reuse(): + run_on_2_gpus(worker_async, timeout=300) diff --git a/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_compile_ranks2.py b/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_compile_ranks2.py new file mode 100644 index 0000000..a9cf78e --- /dev/null +++ b/tests/norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2/test_compile_ranks2.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Warm-up covers all row tiers, code layouts, output types and both residual forms, eager and in graphs.""" +import torch +import torch.distributed as dist +from amd_kernels.norm.allreduce_gemma_bf16_mxfp8_h6144_ranks2 import Bf16ArNormComm +from tests.multi_gpu import run_on_2_gpus + + +def worker(rank, errors): + from flydsl.compiler.jit_function import MlirCompiler + dev = torch.device('cuda', rank) + c = Bf16ArNormComm(rank, dist.group.WORLD, dev) + w = torch.ones(6144, device=dev, dtype=torch.bfloat16) + c.warm_up(w) + orig = MlirCompiler.compile + + def unexpected(*a, **kw): + raise AssertionError('compile after warm-up') + MlirCompiler.compile = unexpected + try: + for m in (1, 7, 8, 63, 64, 65, 70, 112, 128, 129, 224, 256, 257, 448, 512, 513, 896, 1024, 1025, 1792, 2048): + x = torch.zeros((m, 6144), device=dev, dtype=torch.bfloat16) + g = torch.cuda.CUDAGraph() + for ctx in (None, g): + def calls(): + for owner in (False, True): + c.ar_add_rmsnorm(x, x, w, owned_residual=owner) + for tile in (0, 16, 32, 64, 128): + c.ar_add_rmsnorm_mxfp8(x, x, w, 1e-6, tile, owned_residual=owner) + if ctx is None: + calls() + else: + with c.capture(), torch.cuda.graph(g): + calls() + g.replay() + torch.cuda.synchronize() + finally: + MlirCompiler.compile = orig + c.close() + + +def test_warmup(): + run_on_2_gpus(worker, timeout=300) diff --git a/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_compile.py b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_compile.py new file mode 100644 index 0000000..c932f64 --- /dev/null +++ b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_compile.py @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Rule 4 at two ranks: after ``DecoderArNorm``'s warm-ups (both communicators on one buffer pair), no row count of +1-4095 reaches a kernel variant the warm-up did not compile, and probes at every tier edge compile nothing, eager and +in graphs, both code layouts and every output.""" + +import torch +import torch.distributed as dist + +from tests.multi_gpu import run_on_2_gpus + +H = 6144 +EPS = 1e-6 + + +def _worker(rank, errors): + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + from amd_kernels.norm.gemma_add_rmsnorm_h6144.allreduce_add_rmsnorm_ranks2 import host + + dev = torch.device("cuda", rank) + w = torch.ones(H, dtype=torch.bfloat16, device=dev) + real_kernel, warm = host._kernel, set() + + def spy(*key): + warm.add(key) + return real_kernel(*key) + + host._kernel = spy + try: + comms = arn.DecoderArNorm(rank, dist.group.WORLD, dev, w, EPS) + finally: + host._kernel = real_kernel + for rows in range(1, host.MAX_ROWS + 1): + for out, slots in (("bf16", False), ("tiled", False), ("rowmajor", False), ("tiled", True), + ("rowmajor", True)): + rpb, _, _, coded = host.config(rows, out != "bf16", slots) + if (out, slots, rpb, coded) not in warm: + errors.append(f"rank {rank} rows {rows} {out} slots {slots}: variant {rpb, coded} not warmed up") + return + probes = sorted({1, 2, 8, 48, 49, 64, 256, 257, 1024, 1025, 2048, 2049, 4095, 4096, 7700} + | set(range(1, host.MAX_ROWS + 1, 97))) + + def unexpected(*a, **kw): + raise AssertionError("compile after warm-up") + + real = MlirCompiler.compile + MlirCompiler.compile = unexpected + try: + for rows in probes: + x = torch.zeros((rows, H), dtype=torch.bfloat16, device=dev) + slots = torch.zeros((rows, 4, H), dtype=torch.bfloat16, device=dev) + r = torch.zeros_like(x) + + def calls(): + comms.staged.ar_add_rmsnorm(x, r, w, EPS) + for tile in (0, 16, 32, 64): + comms.staged.ar_add_rmsnorm_mxfp8(x, r, w, EPS, tile) + comms.slots.slots4_ar_add_rmsnorm_mxfp8(slots, x, r, w, EPS, tile) + + calls() + g = torch.cuda.CUDAGraph() + with comms.capture(), torch.cuda.graph(g): + calls() + g.replay() + torch.cuda.synchronize() + except AssertionError as e: + errors.append(f"rank {rank}: {e}") + finally: + MlirCompiler.compile = real + + +def test_ranks2_no_compile_after_warm_up(): + run_on_2_gpus(_worker, timeout=1800) diff --git a/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_decoder_tail.py b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_decoder_tail.py new file mode 100644 index 0000000..8294fe2 --- /dev/null +++ b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_decoder_tail.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``allreduce_add_rmsnorm_ranks2.DecoderArNorm``: ``add_rmsnorm`` writes the bytes of the communicator call it routes +to, and ``post_attention`` / ``pre_attention`` write the bytes of the per-kernel chain they run (the norm, the MoE +entry ``moe.mxfp4_h6144_i1536_e128_top4`` or the dense MLP of ``decoder.dense_mxfp8_h6144_i6144_ranks2``, the next +norm), on 2 gfx950 GPUs; ``moe=`` warms the MoE modes the tail runs. Without the dense package a dense layer's tail +returns gate_up's codes (checked) and the dense chain case skips.""" + +import importlib.util + +import pytest +import torch +import torch.distributed as dist + +from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm +from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn +from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import H, unshuffle_scales, untile_codes + +from tests.multi_gpu import run_on_2_gpus + +EPS = 1e-6 +I_DENSE = 6144 +QKV_N = 4992 +DENSE_PKG = "amd_kernels.decoder.dense_mxfp8_h6144_i6144_ranks2" +ROUTE_ROWS = (1, 8, 48, 49, 256, 1000, 1752, 4095) +MOE_ROWS = (1, 8, 64, 300, 2049, 4095) +FINAL_ROWS = (8, 300) + + +def _codes(q, rows, gemm_weight): + tile = add_rmsnorm.code_tile_rows_for(rows, gemm_weight) + return untile_codes(q, rows) if tile else q.view(torch.uint8)[:rows] + + +def _same_codes(a, b, rows, gemm_weight): + (q, s), (wq, ws) = a, b + return torch.equal(_codes(q, rows, gemm_weight), _codes(wq, rows, gemm_weight)) and torch.equal( + unshuffle_scales(s, rows), unshuffle_scales(ws, rows)) + + +def _route_worker(rank, errors): + dev = torch.device("cuda", rank) + g = torch.Generator(device=dev).manual_seed(7) + weight = (torch.randn(H, generator=g, device=dev) * 0.1 + 1).bfloat16() + comms = arn.DecoderArNorm(rank, dist.group.WORLD, dev, weight, EPS) + gemm_weight = torch.empty((QKV_N, H), dtype=torch.float8_e4m3fn, device=dev) + for rows in ROUTE_ROWS: + g = torch.Generator(device=dev).manual_seed(rows * 10 + rank) + x = torch.randn(rows, H, generator=g, device=dev).bfloat16() + slots = torch.randn(rows, 4, H, generator=g, device=dev).bfloat16() + shared = torch.randn(rows, H, generator=g, device=dev).bfloat16() + res = torch.randn(rows, H, generator=g, device=dev).bfloat16() + out, r = comms.add_rmsnorm(res.clone(), weight, EPS, partial=x) + want, wr = comms.staged.ar_add_rmsnorm(x, res.clone(), weight, EPS) + if not (torch.equal(out, want) and torch.equal(r, wr)): + errors.append(f"rank {rank} rows {rows}: bf16 out differs") + for name, kw, direct in ( + ("staged", dict(partial=x), + lambda: comms.staged.ar_add_rmsnorm_mxfp8(x, res.clone(), weight, EPS, gemm_weight=gemm_weight)), + ("slots4", dict(slots=slots, shared=shared), + lambda: comms.slots.slots4_ar_add_rmsnorm_mxfp8( + slots, shared, res.clone(), weight, EPS, gemm_weight=gemm_weight)), + ): + (q, s), r = comms.add_rmsnorm(res.clone(), weight, EPS, gemm_weight=gemm_weight, **kw) + wq, ws, wr = direct() + if not (_same_codes((q, s), (wq, ws), rows, gemm_weight) and torch.equal(r, wr)): + errors.append(f"rank {rank} rows {rows}: {name} MXFP8 out differs") + + +def test_ranks2_decoder_ar_norm_routes(): + run_on_2_gpus(_route_worker) + + +def _moe(rank, g, full=False): + """The rank's MoE block; ``full``: the full shared expert copy (I 3072), the rank's shard computed by view.""" + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import MoeWeights, SharedWeights + from tests.moe.mxfp4_h6144_i1536_e128_top4.workload import make_experts + + raw, _ = make_experts(seed=rank) + gate = torch.randn((128, H), generator=g, device="cuda") / H**0.5 + bias = torch.randn((128,), generator=g, device="cuda") * 0.01 + i = 3072 if full else 1536 + sq = (torch.randn((2 * i, H), device="cuda", generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + ss = torch.randint(117, 122, (2 * i, H // 32), device="cuda", dtype=torch.uint8, generator=g) + dq = (torch.randn((H, i), device="cuda", generator=g) * 20).to(torch.float8_e4m3fn).view(torch.uint8) + ds = torch.randint(117, 122, (H, i // 32), device="cuda", dtype=torch.uint8, generator=g) + shared = SharedWeights.from_row_major(sq, ss, dq, ds, tp_rank=rank if full else None) + return MoeWeights.from_row_major(gate, bias, *raw, shared=shared) + + +def _tail_worker(rank, errors, dense, full=False): + from amd_kernels.moe import mxfp4_h6144_i1536_e128_top4 as moe_pkg + from amd_kernels.moe.mxfp4_h6144_i1536_e128_top4 import mxfp4_moe + + dev = torch.device("cuda", rank) + g = torch.Generator(device=dev).manual_seed(11 + rank) + norm_w = (torch.randn(H, generator=g, device=dev) * 0.1 + 1).bfloat16() + moe = _moe(rank, g, full) + warmed = [] + real = moe_pkg.warm_up_mxfp4_moe + moe_pkg.warm_up_mxfp4_moe = lambda w, **kw: warmed.append((w, kw)) + try: + comms = arn.DecoderArNorm(rank, dist.group.WORLD, dev, norm_w, EPS, moe=moe) + finally: + moe_pkg.warm_up_mxfp4_moe = real + if len(warmed) != 1 or warmed[0][0] is not moe or warmed[0][1] != {"partials": True}: + errors.append(f"rank {rank}: moe= warmed {[kw for _, kw in warmed]}") + qkv = torch.empty((QKV_N, H), dtype=torch.float8_e4m3fn, device=dev) + + def inputs(rows): + gi = torch.Generator(device=dev).manual_seed(rows * 10 + rank) + return tuple(torch.randn(rows, H, generator=gi, device=dev).bfloat16() for _ in range(3)) + + def check(name, rows, ok): + if not ok: + errors.append(f"rank {rank} rows {rows}: {name} differs") + + if dense: + from amd_kernels.decoder import dense_mxfp8_h6144_i6144_ranks2 as D + from amd_kernels.gemm.mxfp8_common import gemm as G + + def e4m3(n, k, scale): + q = (torch.randn((n, k), device="cuda", generator=g) * scale).to(torch.float8_e4m3fn) + return q, torch.randint(117, 122, (n, k // 32), device="cuda", dtype=torch.uint8, generator=g) + + mlp = arn.DenseMlp(*G.preshuffle_mxfp8_weight(*e4m3(2 * I_DENSE, H, 0.6)), + *G.preshuffle_mxfp8_weight(*e4m3(H, I_DENSE, 0.6)), 1.702, 1.0, 7.0) + for rows in (1, 8, 513, D.MAX_ROWS, D.MAX_ROWS + 1): + o, res, res2 = inputs(rows) + x, r = comms.post_attention(o.clone(), res.clone(), norm_w, EPS, mlp) + (q, s), wr = comms.add_rmsnorm(res.clone(), norm_w, EPS, partial=o.clone(), gemm_weight=mlp.gate_up) + check("dense residual", rows, torch.equal(r, wr)) + if rows > D.MAX_ROWS: + check("dense codes", rows, _same_codes(x, (q, s), rows, mlp.gate_up)) + continue + want = D.mxfp8_swiglu_oai_mlp(q, s, rows, mlp.gate_up, mlp.gate_up_scale, mlp.down, mlp.down_scale, + mlp.alpha, mlp.beta, mlp.limit, form="fp32") + check("dense partial", rows, isinstance(x, arn.FfnPartial) and torch.equal(x.partial, want)) + got, r2 = comms.pre_attention(x, res2.clone(), norm_w, EPS, gemm_weight=qkv) + exp, wr2 = comms.add_rmsnorm(res2.clone(), norm_w, EPS, partial=want, gemm_weight=qkv) + check("dense next norm", rows, _same_codes(got, exp, rows, qkv) and torch.equal(r2, wr2)) + return + if arn._dense_package() is None: + # without the dense MLP package a dense layer's tail returns gate_up's codes for the caller's MLP + gate_up = torch.zeros((2 * I_DENSE, H), dtype=torch.float8_e4m3fn, device=dev) + down = torch.zeros((H, I_DENSE), dtype=torch.float8_e4m3fn, device=dev) + mlp = arn.DenseMlp(gate_up, None, down, None, 1.702, 1.0, 7.0) + for rows in (8, 513): + o, res, _ = inputs(rows) + x, r = comms.post_attention(o.clone(), res.clone(), norm_w, EPS, mlp) + want, wr = comms.add_rmsnorm(res.clone(), norm_w, EPS, partial=o.clone(), gemm_weight=gate_up) + check("dense codes without the MLP package", rows, + isinstance(x, tuple) and _same_codes(x, want, rows, gate_up) and torch.equal(r, wr)) + for final in (False, True): + for rows in FINAL_ROWS if final else MOE_ROWS: + o, res, res2 = inputs(rows) + x, r = comms.post_attention(o.clone(), res.clone(), norm_w, EPS, moe, final=final) + xb, wr = comms.add_rmsnorm(res.clone(), norm_w, EPS, partial=o.clone()) + check("moe residual", rows, torch.equal(r, wr)) + if final: + want = mxfp4_moe(xb, moe) + check("moe final partial", rows, x.slots is None and torch.equal(x.partial, want)) + got, r2 = comms.pre_attention(x, res2.clone(), norm_w, EPS) + exp, wr2 = comms.add_rmsnorm(res2.clone(), norm_w, EPS, partial=want) + check("final norm", rows, torch.equal(got, exp) and torch.equal(r2, wr2)) + continue + slots, shared = mxfp4_moe(xb, moe, return_partials=True) + check("moe slots", rows, torch.equal(x.slots, slots) and torch.equal(x.shared, shared)) + got, r2 = comms.pre_attention(x, res2.clone(), norm_w, EPS, gemm_weight=qkv) + exp, wr2 = comms.add_rmsnorm(res2.clone(), norm_w, EPS, slots=slots, shared=shared, gemm_weight=qkv) + check("moe next norm", rows, _same_codes(got, exp, rows, qkv) and torch.equal(r2, wr2)) + + +def test_ranks2_decoder_tail_moe_matches_the_chain(): + run_on_2_gpus(_tail_worker, False, timeout=2400) # the MoE entry compiles each row count on its first call here + + +def test_ranks2_decoder_tail_moe_full_shared_copy(): + """The decode tail on the full shared expert copy (each rank's TP shard read by view).""" + run_on_2_gpus(_tail_worker, False, True, timeout=2400) + + +@pytest.mark.skipif(importlib.util.find_spec(DENSE_PKG) is None, reason=f"needs {DENSE_PKG}") +def test_ranks2_decoder_tail_dense_matches_the_chain(): + run_on_2_gpus(_tail_worker, True, timeout=2400) diff --git a/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_slots4.py b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_slots4.py new file mode 100644 index 0000000..3c8fd1f --- /dev/null +++ b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_slots4.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The MoE combine fused into the two-rank all-reduce + add + Gemma RMSNorm + MXFP8 quant +(``allreduce_add_rmsnorm_ranks2.slots4_comm``) on 2 gfx950 GPUs: + +- against the contract: ``reference.slots4_allreduce_reference`` of the 2 ranks' slots and shared outputs, then the + one-GPU ``gemma_add_rmsnorm_mxfp8`` on it, bit for bit (codes, scales, residual out; written bytes only), and the + MXFP8 tolerances of reference.py on the first and last call; denormals and signed zeros in the slots; +- every rows-per-batch tier and its edges, tiled and row-major codes, 1-4095 rows; +- eager calls, each with its own inputs, the residual flowing from call to call; then the same calls captured in a + graph and replayed into guarded outputs: bit for bit the eager results, with changed-input replays, next to MFMA. +""" + +import torch +import torch.distributed as dist +from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import ( + CODE_MISMATCH, + MXFP8_COS, + SCALE_MISMATCH, + H, + add_rmsnorm_reference, + compare_mxfp8, + slots4_allreduce_reference, + unshuffle_scales, + untile_codes, +) + +from tests.multi_gpu import run_on_2_gpus + +WORLD = 2 +EPS = 1e-6 +CALLS = 4 +TILE = 16 + + +def _inputs(rows, call, dev, sensitive=False): + """Both ranks' (slots [rows, 4, H], shared [rows, H]) of call ``call``, bf16.""" + out = [] + for p in range(WORLD): + g = torch.Generator(device=dev).manual_seed(rows * 1000 + call * 10 + p) + s = torch.randn(rows, 4, H, generator=g, device=dev) * (0.5 + p) + sh = torch.randn(rows, H, generator=g, device=dev) + s[:, :, :32] *= 40 + s[:, :, 64:72] = 2.0**-133 # a bf16 denormal: kept by the local sum, flushed by the rank sum + s[:, 1, 72:80] = -0.0 + sh[:, 64:80] = -0.0 + if sensitive: + # local denormals cross into the normal range before the rank flush + s[:, :, 96:128] = 2.0**-128 + sh[:, 96:128] = 0.0 + # normal rank operands cancel into a denormal, which the rank sum flushes + s[:, :, 128:160] = 0.0 + sh[:, 128:160] = 0.0 + s[:, 0, 128:160] = 2.0**-125 if p == 0 else -(2.0**-125) + 2.0**-133 + s[:, :, 160:192] = 0.0 + sh[:, 160:192] = 0.0 + out.append((s.bfloat16(), sh.bfloat16())) + return out + + +def _residual(rows, dev): + g = torch.Generator(device=dev).manual_seed(rows + 7919) + return torch.randn(rows, H, generator=g, device=dev).bfloat16() + + +def _bits(t): + return t.contiguous().view(torch.uint8) + + +def _written(o, rows, tile=TILE): + q, s, res = o + return [untile_codes(q, rows) if tile else q[:rows].view(torch.uint8), unshuffle_scales(s, rows), res] + + +def _worker(rank, errors, row_counts, tile=TILE, spin=False, sensitive=False): + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + + dev = torch.device("cuda", rank) + if spin: + from tests.mfma_spinner import mfma_spinner + + spinner = mfma_spinner(32) + side = torch.cuda.Stream(priority=-1) + w = (torch.randn(H, generator=torch.Generator().manual_seed(7)) * 0.5).bfloat16().to(dev) + comm = arn.slots4_comm(rank, dist.group.WORLD, dev) + comm.warm_up(w, EPS, tile) + compiled = [] + real = MlirCompiler.compile + + def counting(fn, *args, **kwargs): + compiled.append(type(fn).__name__) + return real(fn, *args, **kwargs) + + for rows in row_counts: + tag = f"rank {rank} rows {rows} tile {tile}" + every = [_inputs(rows, c, dev, sensitive) for c in range(CALLS)] + mine = [every[c][rank] for c in range(CALLS)] + r0 = _residual(rows, dev) + if sensitive: + r0[:, 96:160] = 0.0 + r0[:, 160:192] = 2.0**-133 + MlirCompiler.compile = counting + try: + eager, r = [], r0 + for s, sh in mine: + o = comm.slots4_ar_add_rmsnorm_mxfp8(s, sh, r, w, EPS, tile) + eager.append(o) + r = o[2] + torch.cuda.synchronize() + finally: + MlirCompiler.compile = real + if compiled: + errors.append(f"{tag}: compiled {compiled} after warm_up") + compiled.clear() + r = r0 + for c in range(CALLS): + x = slots4_allreduce_reference([every[c][p][0] for p in range(WORLD)], + [every[c][p][1] for p in range(WORLD)]) + ref = an.gemma_add_rmsnorm_mxfp8(x, r, w, EPS, TILE) + for name, a, b in zip(("codes", "scales", "res"), _written(eager[c], rows, tile), _written(ref, rows)): + if not torch.equal(_bits(a), _bits(b)): + k = int((_bits(a) != _bits(b)).sum()) + errors.append(f"{tag} call {c} {name}: {k} bytes differ from the reference chain") + y, res_ref = add_rmsnorm_reference(x, r, w, EPS) + if not torch.equal(_bits(eager[c][2]), _bits(res_ref)): + errors.append(f"{tag} call {c}: residual out differs from the reference") + if c in (0, CALLS - 1): + q, s, _ = eager[c] + st = compare_mxfp8(_written(eager[c], rows, tile)[0], unshuffle_scales(s, rows), y) + ok = (st["scale_max_diff"] <= 1 and st["scale_mismatch"] <= SCALE_MISMATCH + and st["code_mismatch"] <= CODE_MISMATCH and st["code_far"] == 0 and st["cos"] >= MXFP8_COS) + if not ok: + errors.append(f"{tag} call {c}: contract {st}") + r = eager[c][2] + guards = [] + + def guarded(t): + buf = torch.full((t.numel() * t.element_size() + 32,), 0x5A, dtype=torch.uint8, device=dev) + guards.append(buf) + return buf[16:-16].view(t.dtype).view(t.shape) + + outs = [[guarded(t) for t in o] for o in eager] + graph = torch.cuda.CUDAGraph() + MlirCompiler.compile = counting + try: + with torch.cuda.graph(graph): + r = r0 + for (s, sh), o in zip(mine, outs): + comm.slots4_ar_add_rmsnorm_mxfp8(s, sh, r, w, EPS, tile, q_out=o[0], scale_out=o[1], + res_out=o[2]) + r = o[2] + finally: + MlirCompiler.compile = real + if compiled: + errors.append(f"{tag}: compiled {compiled} in graph after warm_up") + compiled.clear() + for rep in range(3): + if rep == 1: + for inputs in every: + for slots0, shared0 in inputs: + slots0.neg_() + shared0.add_(0.5) + eager, r = [], r0 + for slots0, shared0 in mine: + o = comm.slots4_ar_add_rmsnorm_mxfp8(slots0, shared0, r, w, EPS, tile) + eager.append(o) + r = o[2] + for o in outs: + for t in o: + t.view(torch.uint8).fill_(0xFF) + dist.barrier() + torch.cuda.synchronize() + if spin: + side.wait_stream(torch.cuda.current_stream()) + spinner(100000, side) + graph.replay() + torch.cuda.synchronize() + for buf in guards: + if not bool((buf[:16] == 0x5A).all() & (buf[-16:] == 0x5A).all()): + errors.append(f"{tag} replay {rep}: output guard overwritten") + for c in range(CALLS): + for i, (a, b) in enumerate(zip(_written(outs[c], rows, tile), _written(eager[c], rows, tile))): + if not torch.equal(_bits(a), _bits(b)): + errors.append(f"{tag} replay {rep} call {c} output {i}: != eager") + if rank == 0: + print(f"{tag}: eager + 3 replays checked", flush=True) + del graph, outs, eager, every, mine + + +def test_ranks2_slots4_tiled_rows(): + run_on_2_gpus(_worker, (1, 2, 5, 8, 16, 24, 47, 48, 49, 64, 65, 128, 255, 256, 257, 512, 1000, 1024, 1025, + 2048, 2049, 4095, 4096, 7681)) + + +def test_ranks2_slots4_rowmajor_rows(): + run_on_2_gpus(_worker, (1, 8, 13, 48, 49, 256, 257, 1025, 2049, 4095), 0) + + +def test_ranks2_slots4_denormal_next_to_mfma(): + run_on_2_gpus(_worker, (1, 8, 48, 49, 300, 1025, 2049, 4095), TILE, True, True) + run_on_2_gpus(_worker, (1, 49, 1029), 0, True, True) diff --git a/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_staged.py b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_staged.py new file mode 100644 index 0000000..f5bc8f8 --- /dev/null +++ b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_staged.py @@ -0,0 +1,354 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""The attention tail's two-rank all-reduce + add + Gemma RMSNorm (``allreduce_add_rmsnorm_ranks2.staged_comm``), bf16 +out and MXFP8 out (tiled and row-major codes), on 2 gfx950 GPUs: + +- against the contract: ``reference.allreduce_rows_reference`` of the 2 partials, then the one-GPU + ``gemma_add_rmsnorm`` (``gemma_add_rmsnorm_mxfp8``, tile 16) on it, bit for bit (out or written codes and scales, and + residual out; row-major codes = the tiled norm's codes untiled), and the fp64 norm / MXFP8 contract of reference.py + on the first and last call; denormals and signed zeros in the partials, cancellations into denormals; +- every rows-per-batch tier and its edges, the CUDA graph batch sizes of a DSpark-7 decode (8 rows per request); +- eager calls, each with its own inputs, the residual flowing from call to call; then the same calls captured in a + graph and replayed into guarded outputs: bit for bit the eager results, with changed-input replays and eager calls + interleaved; next to MFMA waves on a high-priority stream; strided partials and residuals; +- a producer that overwrites its input right after each call, and rank skew with mixed row counts and outputs. +""" + +import torch +import torch.distributed as dist +from amd_kernels.norm.gemma_add_rmsnorm_h6144.reference import ( + CODE_MISMATCH, + MXFP8_COS, + OUT_COS, + OUT_MISMATCH, + SCALE_MISMATCH, + H, + add_rmsnorm_reference, + allreduce_rows_reference, + compare_mxfp8, + compare_out, + unshuffle_scales, + untile_codes, +) + +from tests.multi_gpu import run_on_2_gpus + +WORLD = 2 +EPS = 1e-6 +CALLS = 4 +DECODE_ROWS = (8, 16, 24, 64, 128, 256, 512, 1024) + + +def _inputs(rows, call, dev, sensitive=False): + """Both ranks' bf16 [rows, H] partials of call ``call`` (each rank makes both: the reference needs them).""" + out = [] + for p in range(WORLD): + g = torch.Generator(device=dev).manual_seed(rows * 1000 + call * 10 + p) + x = torch.randn(rows, H, generator=g, device=dev) * (0.5 + p) + x[:, :32] *= 40 + x[:, 64:72] = 2.0**-133 # bf16 denormals: flushed by the rank sum + x[:, 72:80] = -0.0 + if sensitive: + # normal operands cancel into a denormal, which the rank sum flushes + x[:, 128:160] = 2.0**-125 if p == 0 else -(2.0**-125) + 2.0**-133 + # zero partials: a denormal residual must survive + x[:, 160:192] = 0.0 + # finite operands whose first row's sum of squares overflows fp32: the norm returns signed zeros + x[0, 192:224] = 1.0e20 + out.append(x.bfloat16()) + return out + + +def _residual(rows, dev, sensitive=False): + g = torch.Generator(device=dev).manual_seed(rows + 7919) + r = torch.randn(rows, H, generator=g, device=dev).bfloat16() + if sensitive: + r[:, 128:160] = 0.0 + r[:, 160:192] = 2.0**-133 + return r + + +def _bits(t): + return t.contiguous().view(torch.uint8) + + +def _written(o, rows, tile): + """The bytes an output holds for its rows: (codes row-major, scales unshuffled, residual) for MXFP8, (out, residual) + for bf16.""" + if tile is None: + return list(o) + q, s, res = o + return [untile_codes(q, rows) if tile else q[:rows].view(torch.uint8), unshuffle_scales(s, rows), res] + + +def _worker(rank, errors, row_counts, spin=False, sensitive=False, tile=None, strided=False): + """``tile`` None: bf16 out; else MXFP8 out with that code tile (0: row-major).""" + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + + dev = torch.device("cuda", rank) + if spin: + from tests.mfma_spinner import mfma_spinner + + spinner = mfma_spinner(32) + side = torch.cuda.Stream(priority=-1) + w = (torch.randn(H, generator=torch.Generator().manual_seed(7)) * 0.5).bfloat16().to(dev) + comm = arn.staged_comm(rank, dist.group.WORLD, dev) + comm.warm_up(w, EPS) + compiled = [] + real = MlirCompiler.compile + + def counting(fn, *args, **kwargs): + compiled.append(type(fn).__name__) + return real(fn, *args, **kwargs) + + mx = tile is not None + + def call(x, r, o=None): + if mx: + if o is None: + return comm.ar_add_rmsnorm_mxfp8(x, r, w, EPS, tile) + return comm.ar_add_rmsnorm_mxfp8(x, r, w, EPS, tile, q_out=o[0], scale_out=o[1], res_out=o[2]) + if o is None: + return comm.ar_add_rmsnorm(x, r, w, EPS) + return comm.ar_add_rmsnorm(x, r, w, EPS, out=o[0], res_out=o[1]) + + for rows in row_counts: + tag = f"rank {rank} rows {rows}" + (f" tile {tile}" if mx else "") + every = [_inputs(rows, c, dev, sensitive) for c in range(CALLS)] + if strided: + for inputs in every: + for p, x in enumerate(inputs): + backing = torch.full((rows, H + 16), float("nan"), dtype=x.dtype, device=dev) + backing[:, :H].copy_(x) + inputs[p] = backing[:, :H] + mine = [every[c][rank] for c in range(CALLS)] + r0 = _residual(rows, dev, sensitive) + if strided: + backing = torch.full((rows, H + 8), float("nan"), dtype=r0.dtype, device=dev) + backing[:, :H].copy_(r0) + r0 = backing[:, :H] + MlirCompiler.compile = counting + try: + eager, r = [], r0 + for x in mine: + o = call(x, r) + eager.append(o) + r = o[-1] + torch.cuda.synchronize() + finally: + MlirCompiler.compile = real + if compiled: + errors.append(f"{tag}: compiled {compiled} after warm_up") + compiled.clear() + r = r0 + for c in range(CALLS): + x = allreduce_rows_reference(every[c]) + if mx: + ref = _written(an.gemma_add_rmsnorm_mxfp8(x, r, w, EPS, 16), rows, 16) + names = ("codes", "scales", "res") + else: + ref = an.gemma_add_rmsnorm(x, r, w, EPS) + names = ("out", "res") + for name, a, b in zip(names, _written(eager[c], rows, tile), ref): + if not torch.equal(_bits(a), _bits(b)): + k = int((_bits(a) != _bits(b)).sum()) + errors.append(f"{tag} call {c} {name}: {k} bytes differ from the reference chain") + y, res_ref = add_rmsnorm_reference(x, r, w, EPS) + if not torch.equal(_bits(eager[c][-1]), _bits(res_ref)): + errors.append(f"{tag} call {c}: residual out differs from the reference") + if c in (0, CALLS - 1): + if mx: + wq, ws, _ = _written(eager[c], rows, tile) + st = compare_mxfp8(wq, ws, y) + ok = st["scale_max_diff"] <= 1 and st["scale_mismatch"] <= SCALE_MISMATCH + ok = ok and st["code_mismatch"] <= CODE_MISMATCH and st["code_far"] == 0 + ok = ok and st["cos"] >= MXFP8_COS + else: + st = compare_out(eager[c][0], y) + ok = st["nonfinite_ok"] and st["bad_ulp"] == 0 and st["mismatch"] <= OUT_MISMATCH + ok = ok and st["cos"] >= OUT_COS + if not ok: + errors.append(f"{tag} call {c}: contract {st}") + r = eager[c][-1] + guards = [] + + def guarded(t): + buf = torch.full((t.numel() * t.element_size() + 32,), 0x5A, dtype=torch.uint8, device=dev) + guards.append(buf) + return buf[16:-16].view(t.dtype).view(t.shape) + + outs = [[guarded(t) for t in o] for o in eager] + graph = torch.cuda.CUDAGraph() + MlirCompiler.compile = counting + try: + with comm.capture(), torch.cuda.graph(graph): + r = r0 + for x, o in zip(mine, outs): + call(x, r, o) + r = o[-1] + finally: + MlirCompiler.compile = real + if compiled: + errors.append(f"{tag}: compiled {compiled} in graph after warm_up") + compiled.clear() + for rep in range(3): + if rep == 1: + # change every rank's inputs, then interleave eager calls + for inputs in every: + for x in inputs: + x.neg_() + eager, r = [], r0 + for x in mine: + o = call(x, r) + eager.append(o) + r = o[-1] + for o in outs: + for t in o: + t.view(torch.uint8).fill_(0xFF) + dist.barrier() + torch.cuda.synchronize() + if spin: + side.wait_stream(torch.cuda.current_stream()) + spinner(100000, side) + graph.replay() + torch.cuda.synchronize() + for buf in guards: + if not bool((buf[:16] == 0x5A).all() & (buf[-16:] == 0x5A).all()): + errors.append(f"{tag} replay {rep}: output guard overwritten") + for c in range(CALLS): + for i, (a, b) in enumerate(zip(_written(outs[c], rows, tile), _written(eager[c], rows, tile))): + if not torch.equal(_bits(a), _bits(b)): + errors.append(f"{tag} replay {rep} call {c} output {i}: != eager") + if rank == 0: + print(f"{tag}: eager + 3 replays checked", flush=True) + del graph, outs, eager, every, mine + + +TIER_ROWS = (1, 2, 3, 7, 8, 9, 47, 48, 49, 63, 65, 127, 255, 256, 257, 511, 1000, 1024, 1025, 1536) +LARGE_ROWS = (2048, 2049, 3073, 4095, 4096, 8191) # above 4095: launches of CHUNK_ROWS + + +def test_ranks2_bf16_rows(): + run_on_2_gpus(_worker, TIER_ROWS + DECODE_ROWS + LARGE_ROWS) + + +def test_ranks2_bf16_denormal_next_to_mfma(): + run_on_2_gpus(_worker, (1, 8, 48, 49, 257, 1025, 2049, 4095), True, True) + + +def test_ranks2_mxfp8_tiled_rows(): + run_on_2_gpus(_worker, TIER_ROWS + DECODE_ROWS + LARGE_ROWS, False, False, 16) + run_on_2_gpus(_worker, (1, 8, 48, 49, 257, 1000), True, True, 32) + + +def test_ranks2_mxfp8_rowmajor_rows(): + run_on_2_gpus(_worker, TIER_ROWS + LARGE_ROWS, False, False, 0) + run_on_2_gpus(_worker, (1, 8, 49, 300, 1029, 2049, 4095), True, True, 0) + + +def test_ranks2_strided_inputs_and_residuals(): + rows = (1, 3, 8, 13, 48, 49, 320, 513, 1027, 2050) + run_on_2_gpus(_worker, rows, True, True, None, True) + run_on_2_gpus(_worker, rows, True, True, 16, True) + + +def _reuse_worker(rank, errors, tile=None, rows_list=(1, 2, 7, 8, 16, 48, 49, 200, 257, 1025, 2049, 4095)): + """A producer recycles the same input allocation right after every tail.""" + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + from tests.mfma_spinner import mfma_spinner + + dev = torch.device("cuda", rank) + w = (torch.randn(H, generator=torch.Generator().manual_seed(7)) * 0.5).bfloat16().to(dev) + comm = arn.staged_comm(rank, dist.group.WORLD, dev) + comm.warm_up(w, EPS) + spin = mfma_spinner(128) + side = torch.cuda.Stream(priority=-1) + calls = 12 + for rows in rows_list: + every = [_inputs(rows, c, dev) for c in range(calls)] + x = torch.empty((rows, H), dtype=torch.bfloat16, device=dev) + residual = _residual(rows, dev) + outputs = [] + graph = torch.cuda.CUDAGraph() + with comm.capture(), torch.cuda.graph(graph): + for inputs in every: + x.copy_(inputs[rank]) + if tile is None: + outputs.append(comm.ar_add_rmsnorm(x, residual, w, EPS)) + else: + outputs.append(comm.ar_add_rmsnorm_mxfp8(x, residual, w, EPS, tile)) + x.fill_(float("nan")) + for rep in range(2): + if rep: + for inputs in every: + for source in inputs: + source.neg_() + torch.cuda.synchronize() + dist.barrier() + if rank == 1: + spin(200000, side) + graph.replay() + torch.cuda.synchronize() + for c, inputs in enumerate(every): + red = allreduce_rows_reference(inputs) + if tile is None: + want = an.gemma_add_rmsnorm(red, residual, w, EPS) + else: + want = an.gemma_add_rmsnorm_mxfp8(red, residual, w, EPS, 16) + got = _written(outputs[c], rows, tile) + for k, (a, b) in enumerate(zip(got, _written(want, rows, 16 if tile is not None else None))): + if not torch.equal(_bits(a), _bits(b)): + errors.append(f"rank {rank} rows {rows} rep {rep} call {c} output {k}: input reuse") + del graph, outputs, every + + +def test_ranks2_producer_reuses_input_immediately(): + run_on_2_gpus(_reuse_worker) + run_on_2_gpus(_reuse_worker, 16) + run_on_2_gpus(_reuse_worker, 0, (1, 49, 1029, 4095)) + + +def _skew_worker(rank, errors): + """Eager calls with mixed row counts and outputs (every rows-per-batch tier, the flag words of one batch tier read + by another) while one rank sleeps between calls: the call counts and staged copies stay paired.""" + import time + + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import add_rmsnorm as an + from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as arn + + dev = torch.device("cuda", rank) + comm = arn.staged_comm(rank, dist.group.WORLD, dev) + w = torch.ones(H, dtype=torch.bfloat16, device=dev) + comm.warm_up(w, EPS) + history = [] + pattern = (7, 300, 64, 1, 49, 2048, 8, 257, 4095, 130, 3, 512, 3500, 48, 1025) + for call in range(48): + m = pattern[call % len(pattern)] + parts = [torch.randn((m, H), generator=torch.Generator().manual_seed(1000 * call + r)).bfloat16().pin_memory() + for r in range(WORLD)] + residual = torch.randn((m, H), generator=torch.Generator().manual_seed(77 + call)).bfloat16().to(dev) + if rank == call % 2 and call % 3 == 0: + time.sleep(0.01) + x = parts[rank].to(dev, non_blocking=True) + tile = (None, 16, 0)[call % 3] + out = comm.ar_add_rmsnorm(x, residual, w, EPS) if tile is None else comm.ar_add_rmsnorm_mxfp8( + x, residual, w, EPS, tile) + history.append((parts, residual, out, m, tile)) + torch.cuda.synchronize() + for parts, residual, out, m, tile in history: + red = allreduce_rows_reference([p.to(dev) for p in parts]) + if tile is None: + want = an.gemma_add_rmsnorm(red, residual, w, EPS) + else: + want = an.gemma_add_rmsnorm_mxfp8(red, residual, w, EPS, 16) + for k, (a, b) in enumerate(zip(_written(out, m, tile), _written(want, m, 16 if tile is not None else None))): + if not torch.equal(_bits(a), _bits(b)): + errors.append(f"rank {rank} rows {m} tile {tile} output {k}: skewed eager call differs") + + +def test_ranks2_eager_rank_skew_mixed_tiers(): + run_on_2_gpus(_skew_worker, timeout=600) diff --git a/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_unsupported_reason.py b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_unsupported_reason.py new file mode 100644 index 0000000..cccb866 --- /dev/null +++ b/tests/norm/gemma_add_rmsnorm_h6144/test_ranks2_unsupported_reason.py @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``allreduce_add_rmsnorm_ranks2.unsupported_reason``: the conditions a model integration asks the library about.""" + +from amd_kernels.norm.gemma_add_rmsnorm_h6144 import allreduce_add_rmsnorm_ranks2 as AR + + +def test_reasons(): + assert "2 ranks" in AR.unsupported_reason(4) + assert "2 ranks" in AR.unsupported_reason(1) + assert "hidden size 6144" in AR.unsupported_reason(2, 4096) + assert AR.unsupported_reason(2) in (None, "needs gfx950 with FlyDSL") diff --git a/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py b/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/test_batch_sharded.py b/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/test_batch_sharded.py new file mode 100644 index 0000000..c869914 --- /dev/null +++ b/tests/sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/test_batch_sharded.py @@ -0,0 +1,301 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Two spawned ranks: the block (``BatchShardedSampling``) against the per-kernel entries it sequences, bit for bit; +its target logits against the torch reference; the local path (a copy of the peer's lm_head shard) against the +exchange, bit for bit; graph replay; the first-call warm-up compiles every variant.""" +import socket +from datetime import timedelta + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +WORLD, V, H, RANK, N = 2, 200064, 6144, 256, 100032 +MAX_REQS, SLOTS = 256, 300 + + +def _port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _init(rank, port): + torch.set_num_threads(8) + torch.cuda.set_device(rank) + dev = torch.device("cuda", rank) + dist.init_process_group("nccl", init_method=f"tcp://127.0.0.1:{port}", rank=rank, world_size=WORLD, + timeout=timedelta(minutes=10), device_id=dev) + return dev, dist.new_group(backend="gloo") + + +def _shard(rank, dev): + from amd_kernels.gemm.bf16_n100032_k6144 import shuffle_weight + g = torch.Generator(device=dev).manual_seed(500 + rank) + w = (torch.randn(N, H, device=dev, generator=g) * 0.02).to(torch.bfloat16) + return w, shuffle_weight(w) + + +def _draft_inputs(steps, dev): + """The same on every rank (one seed): the speculator's buffers for MAX_REQS requests.""" + g = torch.Generator(device=dev).manual_seed(1234 + steps) + q = steps + 1 + t = MAX_REQS * q + hidden = torch.randn(t, H, device=dev, generator=g).to(torch.bfloat16) + req = torch.arange(MAX_REQS, device=dev).repeat_interleave(steps) + step = torch.arange(steps, device=dev).repeat(MAX_REQS) + sample_indices = (req * q + step + 1).to(torch.int64) + slot_of = torch.randperm(SLOTS, device=dev, generator=g)[:MAX_REQS] + sample_idx_mapping = slot_of[req].to(torch.int32) + sample_pos = torch.randint(1, 1 << 20, (MAX_REQS,), device=dev, generator=g)[req] + step + input_ids = torch.randint(0, V, (t,), device=dev, generator=g).to(torch.int32) + anchor = torch.arange(MAX_REQS, device=dev) * q + temperature = torch.tensor([0.0, 1.0, 0.7, 1.3], device=dev)[torch.randint(0, 4, (SLOTS,), device=dev, + generator=g)] + seeds = torch.randint(0, 1 << 62, (SLOTS,), device=dev, generator=g) + w1 = (torch.randn(V, RANK, device=dev, generator=g) * 0.05).to(torch.bfloat16) + w2 = (torch.randn(V, RANK, device=dev, generator=g) * 0.05).to(torch.bfloat16) + return dict(hidden=hidden, sample_indices=sample_indices, sample_idx_mapping=sample_idx_mapping, + sample_pos=sample_pos, input_ids=input_ids, anchor=anchor, temperature=temperature, seeds=seeds, + w1=w1, w2=w2) + + +def _per_kernel_markov(comm_a2a, comm_gather, tables, steps, num_reqs, x, lm, d, cache, draft_tokens): + """The chain as separate per-kernel entry calls (the sequence the block replaces).""" + from amd_kernels.gemm.bf16_n100032_k6144.host import bf16_gemm_n100032_k6144 + from amd_kernels.gemm.bf16_n200064_k256.host import bf16_gemm_packed_rows, pack_bf16_rows, padded_rows + from amd_kernels.sampling.gumbel_combine_markov_embed_v200064_r256.host import gumbel_combine_markov_embed_packed + from amd_kernels.sampling.gumbel_partials_bias_bf16_v200064.host import gumbel_partials_bias + t = tables[num_reqs] + cap, dev = t.cap, x.device + rows = d["sample_indices"][t.send_rows] + base = torch.empty((cap * steps, V), dtype=torch.bfloat16, device=dev) + bf16_gemm_n100032_k6144(x, lm, shuffled=True, row_indices=rows, publish=comm_a2a.slot()) + comm_a2a.pull(rows.numel(), t.counts, base) + base = base.view(cap, steps, V) + slots = torch.where(t.valid, d["sample_idx_mapping"][t.slot_rows], -1).to(torch.int32) + pos = d["sample_pos"][t.slot_rows] + prev = d["input_ids"][t.anchor_rows].to(torch.int64) + rows_ar = torch.arange(cap, dtype=torch.int64, device=dev) + packed = torch.empty((padded_rows(cap), RANK), dtype=torch.bfloat16, device=dev) + pack_bf16_rows(d["w1"], prev, packed) + bias = torch.empty((cap, V), dtype=torch.bfloat16, device=dev) + lmax = torch.empty((cap, 196), dtype=torch.float32, device=dev) + larg = torch.empty((cap, 196), dtype=torch.int64, device=dev) + tokens = torch.empty((cap, 7), dtype=torch.int64, device=dev) + prev_out = torch.empty(cap, dtype=torch.int64, device=dev) + for i in range(steps): + bf16_gemm_packed_rows(packed, d["w2"], cap, bias) + gumbel_partials_bias(base[:, i], slots[i], d["temperature"], d["seeds"], pos[i], cache, i, bias, rows_ar, + lmax, larg, pos_offset=-1) + for lo in range(0, cap, 64): + hi = min(cap, lo + 64) + gumbel_combine_markov_embed_packed(lmax[lo:hi], larg[lo:hi], rows_ar[lo:hi], d["w1"], tokens[lo:hi], i, + prev_out, packed[lo:lo + padded_rows(hi - lo)]) + full = torch.empty((num_reqs, 7), dtype=torch.int64, device=dev) + comm_gather(tokens, t.gathered_src, cap, full) + draft_tokens[:num_reqs].copy_(full[:, :steps]) + + +def _worker(rank, port): + dev, group = _init(rank, port) + from amd_kernels import warm_up + warm_up.set_auto(False) # single sizes here; the warm-up test runs it (a spawned child may skip tests/__init__) + from amd_kernels.collectives.all_gather_i64_c7_ranks2 import I64C7AllGather + from amd_kernels.collectives.all_to_all_bf16_n100032_ranks2 import RowAllToAll + from amd_kernels.gemm.bf16_n100032_k6144.reference import bf16_gemm_n100032_k6144_reference + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2 import BatchShardedSampling + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2.reference import owner_order_reference + w, lm = _shard(rank, dev) + # target logits: the block vs the reference of every owner's rows (cosine) and vs the per-kernel pair (bits) + s = BatchShardedSampling(rank, group, dev, max_reqs=MAX_REQS, target_rows=MAX_REQS * 8, lm_weight=lm) + old_a2a = RowAllToAll(rank, group, dev, max_rows=MAX_REQS * 8) + g = torch.Generator(device=dev).manual_seed(7) + hidden = torch.randn(MAX_REQS * 8, H, device=dev, generator=g).to(torch.bfloat16) + for m in (1, 7, 64, 128, 129, 300, 512, 1023, 2048): + rows = torch.randperm(hidden.shape[0], device=dev, generator=g)[:m].to(torch.int64) + counts = [m // WORLD + int(r < m % WORLD) for r in range(WORLD)] + out = s.target_logits(hidden, lm, rows, counts, counts[rank]) + from amd_kernels.gemm.bf16_n100032_k6144.host import bf16_gemm_n100032_k6144 + bf16_gemm_n100032_k6144(hidden, lm, shuffled=True, row_indices=rows, publish=old_a2a.slot()) + ref_out = torch.empty_like(out) + old_a2a.pull(m, torch.tensor(counts, dtype=torch.int32, device=dev), ref_out) + torch.cuda.synchronize() + assert torch.equal(out.view(torch.int16), ref_out.view(torch.int16)), (rank, m) + # owner r gets owner-sorted rows [sum(counts[:r]), sum(counts[:r + 1])); shard s at columns s * 100032 + lo = sum(counts[:rank]) + mine = hidden[rows[lo:lo + counts[rank]]] + part = bf16_gemm_n100032_k6144_reference(mine, w) # this rank's shard of its own rows + if counts[rank]: + cos = torch.nn.functional.cosine_similarity(out[:, rank * N:(rank + 1) * N].float().flatten(), + part.float().flatten(), dim=0) + assert cos > 0.9999, (rank, m, float(cos)) + none = s.target_logits(hidden, lm, torch.empty(0, dtype=torch.int64, device=dev), [0] * WORLD, 0) + assert none.shape == (0, V) and none.dtype == torch.bfloat16 + # sampler outputs: a known pattern in batch order + num_reqs = 13 + n_loc = len(owner_order_reference(num_reqs)[rank]) + tok = torch.arange(n_loc * 8, device=dev).view(n_loc, 8) + 1000 * rank + ns = torch.full((n_loc,), 3 + rank, dtype=torch.int64, device=dev) + nr = torch.full((n_loc,), rank, dtype=torch.int64, device=dev) + nn = torch.full((n_loc,), 2 * rank, dtype=torch.int64, device=dev) + cu = torch.arange(n_loc + 1, dtype=torch.int32, device=dev) # one NaN-count row per request + cap = -(-num_reqs // WORLD) + req = torch.arange(num_reqs, device=dev) + src = ((req % WORLD) * cap + req // WORLD).to(torch.int64) + out_t, out_s, out_r, out_n = s.gather_sampler_output(tok, ns, nr, nn, cu, src, cap, num_reqs, 4, True) + torch.cuda.synchronize() + for i in range(num_reqs): + owner, local = i % WORLD, i // WORLD + assert out_t[i].tolist() == [1000 * owner + local * 8 + j for j in range(4)], (rank, i) + assert (int(out_s[i]), int(out_r[i]), int(out_n[i])) == (3 + owner, owner, 2 * owner), (rank, i) + # the Markov draft: the block vs the per-kernel chain, every request count of a sample, DSpark 3 and 7 + for steps in (3, 7): + d = _draft_inputs(steps, dev) + s.setup_markov(steps, MAX_REQS, d["anchor"], d["w1"], d["w2"]) + old_a2a_d = RowAllToAll(rank, group, dev, max_rows=min(2048, WORLD * (MAX_REQS // WORLD) * steps)) + old_gather = I64C7AllGather(rank, group, dev) + for num_reqs in (1, 2, 3, 5, 13, 18, 19, 31, 63, 64, 65, 127, 128, 129, 200, 255, 256): + cache_a = torch.zeros(SLOTS, steps, V, dtype=torch.bfloat16, device=dev) + cache_b = torch.zeros_like(cache_a) + ids_a = torch.full((MAX_REQS, steps), -1, dtype=torch.int64, device=dev) + ids_b = ids_a.clone() + s.markov_draft(num_reqs, d["hidden"], lm, d["sample_indices"], d["sample_idx_mapping"], d["sample_pos"], + d["input_ids"], d["temperature"], d["seeds"], cache_a, ids_a) + _per_kernel_markov(old_a2a_d, old_gather, s.markov.tables, steps, num_reqs, d["hidden"], lm, d, cache_b, + ids_b) + torch.cuda.synchronize() + assert torch.equal(ids_a, ids_b), (rank, steps, num_reqs) + diff = (cache_a.view(torch.int16) != cache_b.view(torch.int16)) + if diff.any(): + where = diff.any(dim=2).nonzero().tolist() + na = (cache_a.view(torch.int16) != 0).any(dim=2).nonzero().tolist() + nb = (cache_b.view(torch.int16) != 0).any(dim=2).nonzero().tolist() + mine = owner_order_reference(num_reqs)[rank] + exp = sorted(set(d["sample_idx_mapping"].view(MAX_REQS, steps)[mine, 0].tolist())) + raise AssertionError((rank, steps, num_reqs, "diff", where[:20], "a", na[:20], "b", nb[:20], "mine", + exp, int(diff.sum()))) + assert (ids_a[:num_reqs] >= 0).all() and (ids_a[:num_reqs] < V).all() + # graph replay with changed inputs (the tables and the request count are fixed per graph) + num_reqs = 13 if steps == 3 else 150 + cache_g = torch.zeros(SLOTS, steps, V, dtype=torch.bfloat16, device=dev) + ids_g = torch.full((MAX_REQS, steps), -1, dtype=torch.int64, device=dev) + args = (d["hidden"], lm, d["sample_indices"], d["sample_idx_mapping"], d["sample_pos"], d["input_ids"], + d["temperature"], d["seeds"]) + s.markov_draft(num_reqs, *args, cache_g, ids_g) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + s.markov_draft(num_reqs, *args, cache_g, ids_g) + for k in range(3): + d["hidden"].copy_(torch.randn(d["hidden"].shape, device=dev, generator=torch.Generator(device=dev) + .manual_seed(k)).to(torch.bfloat16)) + d["seeds"].add_(1) + graph.replay() + cache_e = torch.zeros_like(cache_g) + ids_e = torch.full_like(ids_g, -1) + s.markov_draft(num_reqs, *args, cache_e, ids_e) + torch.cuda.synchronize() + assert torch.equal(ids_g, ids_e), (rank, steps, "graph", k) + mine = owner_order_reference(num_reqs)[rank] + slots = d["sample_idx_mapping"].view(MAX_REQS, steps)[mine, 0].long() + assert torch.equal(cache_g[slots].view(torch.int16), cache_e[slots].view(torch.int16)), (rank, k) + del graph + old_a2a_d.close() + old_gather.close() + old_a2a.close() + s.close() + dist.barrier() + dist.destroy_process_group() + + +def _warm_worker(rank, port): + """Each mode's first call warms it up; later calls at other sizes compile nothing.""" + dev, group = _init(rank, port) + from flydsl.compiler.jit_function import MlirCompiler + from amd_kernels import warm_up + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2 import BatchShardedSampling + warm_up.set_auto(True) + warm_up.set_workers(4) + _, lm = _shard(rank, dev) + steps, max_reqs = 7, 48 + d = _draft_inputs(steps, dev) + s = BatchShardedSampling(rank, group, dev, max_reqs=max_reqs, target_rows=max_reqs * 8, lm_weight=lm) + s.setup_markov(steps, max_reqs, d["anchor"], d["w1"], d["w2"]) + late, original = [], MlirCompiler.compile + + def spy(*a, **k): + if warm_up._current.get() is not None and not warm_up._in_warm_up.get(): + late.append(1) + return original(*a, **k) + + MlirCompiler.compile = spy + cache = torch.zeros(SLOTS, steps, V, dtype=torch.bfloat16, device=dev) + ids = torch.zeros(MAX_REQS, steps, dtype=torch.int64, device=dev) + args = (d["hidden"], lm, d["sample_indices"], d["sample_idx_mapping"], d["sample_pos"], d["input_ids"], + d["temperature"], d["seeds"], cache, ids) + for num_reqs in (1, 2, 3, 5, 6, 9, 10, 11, 17, 21, 33, 40, 47, 48): + s.markov_draft(num_reqs, *args) + m = num_reqs * 8 + rows = torch.arange(m, dtype=torch.int64, device=dev) + counts = [m // WORLD + int(r < m % WORLD) for r in range(WORLD)] + s.target_logits(d["hidden"], lm, rows, counts, counts[rank]) + n_loc = len(range(rank, num_reqs, WORLD)) + t = torch.zeros(n_loc, 8, dtype=torch.int64, device=dev) + c = torch.ones(n_loc, dtype=torch.int32, device=dev) + cu = torch.arange(n_loc + 1, dtype=torch.int32, device=dev) + cap = -(-num_reqs // WORLD) + req = torch.arange(num_reqs, device=dev) + src = ((req % WORLD) * cap + req // WORLD).to(torch.int64) + for nans in (False, True): + s.gather_sampler_output(t, c, c, c, cu, src, cap, num_reqs, 8, nans) + torch.cuda.synchronize() + MlirCompiler.compile = original + assert late == [] and warm_up.late_compiles() == 0, (rank, len(late), warm_up.late_compiles()) + assert len([k for k in warm_up.warmed() if k[0].startswith("sampling/batch_sharded")]) == 2, warm_up.warmed() + s.close() + dist.barrier() + dist.destroy_process_group() + + +def _need_gpus(): + if torch.cuda.device_count() < WORLD: + pytest.skip("requires two gfx950 GPUs") + + +def test_block_matches_per_kernel_entries(): + _need_gpus() + mp.spawn(_worker, args=(_port(),), nprocs=WORLD, join=True) + + +def test_first_call_warm_up_compiles_every_variant(tmp_path, monkeypatch): + _need_gpus() + monkeypatch.setenv("FLYDSL_RUNTIME_CACHE_DIR", str(tmp_path / "flydsl")) + mp.spawn(_warm_worker, args=(_port(),), nprocs=WORLD, join=True) + + +def test_sampling_blocker(): + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2 import sampling_blocker + + for max_reqs, per_req in ((2, 1), (256, 8), (256, 1), (32, 8)): + assert sampling_blocker(WORLD, max_reqs, per_req) is None, (max_reqs, per_req) + for world, max_reqs, per_req in ((4, 64, 4), (8, 64, 4), (2, 1, 1), (2, 257, 1), (2, 64, 0), (2, 64, 9), + (2, 512, 8)): + assert sampling_blocker(world, max_reqs, per_req) is not None, (world, max_reqs, per_req) + + +def test_markov_blocker(): + from amd_kernels.sampling.batch_sharded_markov_v200064_h6144_r256_ranks2 import markov_blocker + + w = torch.empty(V, RANK, dtype=torch.bfloat16) + cache = torch.empty(16, 3, V, dtype=torch.bfloat16) + assert markov_blocker(3, 64, w, w) is None + assert markov_blocker(3, 256, w, w) is None + assert markov_blocker(3, 64, w, w, cache=cache) is None + for steps, reqs, w1 in ((0, 64, w), (8, 64, w), (3, 257, w), (3, 64, w.float()), (3, 64, w[:-1])): + assert markov_blocker(steps, reqs, w1, w) is not None, (steps, reqs) + for bad in (cache.float(), cache[:, :2], torch.empty(16, 3, V - 1, dtype=torch.bfloat16)): + assert markov_blocker(3, 64, w, w, cache=bad) is not None + for kw in ("fp64_noise", "confidences", "logit_transform", "reduced_vocab"): + assert markov_blocker(3, 64, w, w, **{kw: True}) is not None, kw + assert markov_blocker(3, 64, w, w, lm_weight_shuffled=False) is not None diff --git a/tests/sampling/top_p_mask_f32_v200064/test_top_p_mask_rows.py b/tests/sampling/top_p_mask_f32_v200064/test_top_p_mask_rows.py new file mode 100644 index 0000000..d9d0af7 --- /dev/null +++ b/tests/sampling/top_p_mask_f32_v200064/test_top_p_mask_rows.py @@ -0,0 +1,162 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""``top_p_mask`` above 64 rows (vLLM's one-program-per-row kernel) against ``reference.py``, bit for bit: LM-like +rows at temperatures 0.6-1.5, a heavy tail (Gumbel body), integer ties (boundary duplicates), -inf columns, flat +rows (the full-buffer search), p 1 rows, a constant row, an all -inf row; the three outlier paths (threshold on e, +Triton's division per element, the exact outlier mass); more rows than CUs; a caller's buffer; row stride; graph +replay with new inputs; the warm-up compiles the kernel and nothing compiles later. Bit-equality with vLLM's Triton +kernel needs vLLM, so it is checked outside these tests.""" + +import flydsl.compiler as flyc +import pytest +import torch +from amd_kernels.sampling import top_p_mask_f32_v200064 as tp +from amd_kernels.sampling.top_p_mask_f32_v200064 import V, top_p_mask, top_p_mask_reference +from amd_kernels.sampling.top_p_mask_f32_v200064 import host as tp_host +from amd_kernels.sampling.top_p_mask_f32_v200064 import kernel_rows + + +def require_gfx950(): + if not tp.available(): + pytest.skip("requires gfx950 with FlyDSL") + + +def lm_rows(n, gen, temp=1.0): + x = torch.randn(n, V, generator=gen) * 2.0 + x.scatter_(1, torch.randint(0, V, (n, 64), generator=gen), torch.rand(n, 64, generator=gen) * 12 + 6) + return (x.to(torch.bfloat16).float() / temp).cuda() + + +def case(name, n, gen): + p = torch.full((n,), 0.95, device="cuda") + if name == "lm": + return lm_rows(n, gen), p + if name == "cold": + return lm_rows(n, gen, 0.6), p + if name == "hot": + return lm_rows(n, gen, 1.5), torch.rand(n, generator=gen).cuda() * 0.9 + 0.05 + if name == "tail": + x = -torch.log(-torch.log(torch.rand(n, V, generator=gen).clamp(1e-12, 1 - 1e-7))) * 1.3 + x.scatter_(1, torch.randint(0, V, (n, 8), generator=gen), torch.rand(n, 8, generator=gen) * 10 + 8) + return x.to(torch.bfloat16).float().cuda(), p + if name == "ties": + return torch.randint(-4, 5, (n, V), generator=gen).float().cuda(), torch.full((n,), 0.9, device="cuda") + if name == "flat": + return (torch.randn(n, V, generator=gen) * 0.5).cuda(), p + if name == "masked": + x = lm_rows(n, gen) + x[:, ::3] = float("-inf") + x[: n // 4, 1::3] = float("-inf") + return x, p + if name == "mixed": # p 1 rows, a one-finite row, a constant row, an all -inf row, 100 equal maxima + x = lm_rows(n, gen) + p[::4] = 1.0 + x[1] = float("-inf") + x[1, 1234] = 3.0 + x[2] = 0.0 + x[3] = float("-inf") + x[5, :100] = 50.0 + return x, p + raise ValueError(name) + + +def same(a, b): + return torch.equal(a.view(torch.int32), b.view(torch.int32)) + + +def bad_rows(a, b): + return (a.view(torch.int32) != b.view(torch.int32)).any(1).nonzero().flatten().tolist() + + +@pytest.mark.parametrize("n", [65, 97, 300]) +@pytest.mark.parametrize("name", ["lm", "cold", "hot", "tail", "ties", "flat", "masked", "mixed"]) +def test_matches_reference(n, name): + require_gfx950() + if n == 300 and name not in ("lm", "mixed"): + pytest.skip("more rows than CUs: two cases") + gen = torch.Generator().manual_seed(n * 7 + len(name)) + x, p = case(name, n, gen) + y = top_p_mask(x.clone(), p) + r = top_p_mask_reference(x, p) + torch.cuda.synchronize() + assert same(y, r), (name, n, bad_rows(y, r)) + keep = p >= 1 + assert same(y[keep], x[keep]) + + +@pytest.mark.parametrize("path", ["exact", "mass"]) +def test_outlier_paths(path): + """The per-element division and the exact outlier mass (the fallbacks of the threshold path) on every row.""" + require_gfx950() + gen = torch.Generator().manual_seed(21) + x = torch.cat([case("lm", 40, gen)[0], case("hot", 20, gen)[0], case("ties", 10, gen)[0], + case("mixed", 10, gen)[0]]) + p = torch.rand(80, generator=gen).cuda() * 0.9 + 0.05 + p[::7] = 1.0 + exe = kernel_rows.compile_top_p_mask_rows(path=path) + buf = torch.empty(80, V, device="cuda") + y = x.clone() + flyc.compile(exe, y.data_ptr(), y.stride(0), p.data_ptr(), tp_host._sigma_table(y.device).data_ptr(), + buf.data_ptr(), 80, 80, torch.cuda.current_stream()) + r = top_p_mask_reference(x, p) + torch.cuda.synchronize() + assert same(y, r), bad_rows(y, r) + + +def test_buffer_and_row_stride(): + require_gfx950() + gen = torch.Generator().manual_seed(22) + x, p = case("lm", 70, gen) + big = torch.full((70, V + 64), 7.0, device="cuda") + big[:, :V] = x + buf = torch.empty(tp.rows_grid(70, 0), V, device="cuda") + top_p_mask(big[:, :V], p, buffer=buf) + assert same(big[:, :V], top_p_mask(x.clone(), p)) and bool((big[:, V:] == 7.0).all()) + + +def test_graph_replay_new_inputs(): + require_gfx950() + gen = torch.Generator().manual_seed(23) + n = 90 + x, p = case("lm", n, gen) + buf = x.clone() + work = torch.empty(tp.rows_grid(n, 0), V, device="cuda") + tp.warmup_top_p_mask(torch.device("cuda")) + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + top_p_mask(buf, p, buffer=work) + torch.cuda.synchronize() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + top_p_mask(buf, p, buffer=work) + for name in ("hot", "ties", "mixed", "flat"): + x2, p2 = case(name, n, gen) + buf.copy_(x2) + p.copy_(p2) + g.replay() + torch.cuda.synchronize() + assert same(buf, top_p_mask_reference(x2, p2)), name + + +def test_warm_up_compiles_rows_kernel(): + """``warmup_top_p_mask`` compiles the > 64-row kernel; calls at any row count compile nothing after it.""" + require_gfx950() + from flydsl.compiler.jit_function import MlirCompiler + + tp.warmup_top_p_mask(torch.device("cuda")) + assert torch.device("cuda", 0) not in tp_host._ROWS_WORKSPACE + original = MlirCompiler.compile + + def unexpected(*a, **k): + raise AssertionError("compile after warm-up") + + MlirCompiler.compile = unexpected + try: + gen = torch.Generator().manual_seed(24) + for n in (65, 128, 257): + x, p = case("lm", n, gen) + top_p_mask(x, p) + torch.cuda.synchronize() + finally: + MlirCompiler.compile = original diff --git a/tools/ci/targets.tsv b/tools/ci/targets.tsv index 93278c9..147f878 100644 --- a/tools/ci/targets.tsv +++ b/tools/ci/targets.tsv @@ -25,6 +25,7 @@ collectives/pull_collectives_ranks4/test_norm_gather_h6144.py 4 collectives/pull_collectives_ranks4/test_unpack_q2048_shuffled.py 4 collectives/seq_parallel_handoffs_h6144_q2048_ranks4/test_handoffs.py 4 collectives/sdma_push_ranks4/test_transport.py 4 +collectives/sdma_push_ranks4/test_agents.py 4 collectives/seq_parallel_handoffs_h6144_q2048_ranks4/test_warm_up.py 4 collectives/seq_parallel_handoffs_h6144_q2048_ranks4/test_shard.py 4 norm/gemma_add_rmsnorm_h6144/test_stream_hoist_weight.py 4 @@ -38,6 +39,27 @@ norm/gemma_add_rmsnorm_h6144/test_decoder_tail_ranks4.py 4 norm/gemma_add_rmsnorm_h6144/test_slots4_ar_add_rmsnorm.py 4 norm/gemma_add_rmsnorm_h6144/test_slots4_lossless.py 4 +indexer/indexer_nvfp4_d128_p128_top16/test_split2_ranks2.py 2 +moe/mxfp4_h6144_i1536_e128_top4/test_gather_ranks2.py 2 +collectives/pull_collectives_ranks2 2 +collectives/sdma_push_ranks2 2 +collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_handoffs.py 2 +collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_shard.py 2 +collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_dbo.py 2 +collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_coded_tail.py 2 +collectives/seq_parallel_handoffs_h6144_q4096_ranks2/test_warm_up.py 2 +collectives/all_to_all_bf16_n100032_ranks2/test_exchange.py 2 +collectives/all_gather_sampler_i64_c8_ranks2/test_sampler_gather.py 2 +collectives/all_gather_i64_c7_ranks2/test_i64_c7_gather.py 2 +norm/allreduce_gemma_bf16_mxfp8_h6144_ranks2 2 +norm/gemma_add_rmsnorm_h6144/test_ranks2_staged.py 2 +norm/gemma_add_rmsnorm_h6144/test_ranks2_slots4.py 2 +norm/gemma_add_rmsnorm_h6144/test_ranks2_compile.py 2 +norm/gemma_add_rmsnorm_h6144/test_ranks2_decoder_tail.py 2 +gemm/bf16_concat_linear_n3072_k30720/test_gather.py 2 +gemm/bf16_concat_linear_n3072_k30720/test_compile.py 2 +gemm/bf16_n100032_k6144/test_publish.py 2 python3 -c "import tests, torch; from amd_kernels.gemm.bf16_n100032_k6144 import warmup_bf16_gemm_n100032_k6144 as w; [w(torch.device('cuda'), True, m) for m in ('contiguous', 'publish')]" + # compile tests: compile-only mode, sharing one card (the same FlyDSL compiles, kernel and cache key, as a normal run) attention/norm_rope_fp8_insert_d128_rot64/test_compile.py compile attention/norm_rope_nvfp4_insert_d128_rot64_p128/test_compile.py compile @@ -47,11 +69,17 @@ attention/swa_decode_fp8kv_gqa16_d128/test_compile.py compile gemm/mxfp8_common/test_compile.py compile indexer/indexer_nvfp4_d128_p128_top16/test_compile.py compile moe/mxfp4_h6144_i768_e128_top4/test_compile_entry.py compile +moe/mxfp4_h6144_i1536_e128_top4/test_compile_entry.py compile norm/gemma_add_rmsnorm_h6144/test_compile.py compile # precompiled targets: their cold time is mostly compile (cold -> warm: test_moe 528 -> 21 s, test_gemm 347 -> 15, # test_dispatch 194 -> 8 (AITER's hipcc build of its CK module), test_publish 178 -> 17) gemm/bf16_n50016_k6144/test_gemm.py 1 same +gemm/bf16_n100032_k6144/test_gemm.py 1 same +gemm/mxfp8_common/test_shape_rows.py 1 same attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks4.py 4 sampling/batch_sharded_markov_v200064_h6144_r256_ranks4/test_batch_sharded.py 4 decoder/dense_mxfp8_h6144_i3072_ranks4/test_decoder_ranks4.py 4 +attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks2.py 2 +sampling/batch_sharded_markov_v200064_h6144_r256_ranks2/test_batch_sharded.py 2 +decoder/dense_mxfp8_h6144_i6144_ranks2/test_decoder_ranks2.py 2