From dcc5b20499aaf4e14def0dea8f747815298d46c4 Mon Sep 17 00:00:00 2001 From: BinDing Date: Sat, 14 Nov 2026 08:06:17 +0000 Subject: [PATCH 1/3] add mega_moe_tp kernel --- amd_kernels/README.md | 2 + amd_kernels/moe/mega_moe_tp/README.md | 46 + amd_kernels/moe/mega_moe_tp/__init__.py | 28 + amd_kernels/moe/mega_moe_tp/bigm_compute.py | 640 ++++++ amd_kernels/moe/mega_moe_tp/common.py | 519 +++++ amd_kernels/moe/mega_moe_tp/communication.py | 1392 ++++++++++++ amd_kernels/moe/mega_moe_tp/fly_utils.py | 196 ++ amd_kernels/moe/mega_moe_tp/gemm.py | 1219 ++++++++++ amd_kernels/moe/mega_moe_tp/ipc.py | 69 + amd_kernels/moe/mega_moe_tp/layer.py | 514 +++++ amd_kernels/moe/mega_moe_tp/layout.py | 29 + amd_kernels/moe/mega_moe_tp/mega_moe_tp.py | 944 ++++++++ .../mega_moe_tp/mega_moe_tp_bigm_tuned.csv | 6 + .../moe/mega_moe_tp/mega_moe_tp_config.py | 423 ++++ .../moe/mega_moe_tp/mega_moe_tp_kernel.py | 2002 +++++++++++++++++ .../moe/mega_moe_tp/mega_moe_tp_tuned.csv | 11 + amd_kernels/moe/mega_moe_tp/reference.py | 153 ++ amd_kernels/moe/mega_moe_tp/schedule.py | 887 ++++++++ amd_kernels/moe/mega_moe_tp/sdma.py | 182 ++ amd_kernels/moe/mega_moe_tp/sp_rs_norm.py | 852 +++++++ 20 files changed, 10114 insertions(+) create mode 100644 amd_kernels/moe/mega_moe_tp/README.md create mode 100644 amd_kernels/moe/mega_moe_tp/__init__.py create mode 100644 amd_kernels/moe/mega_moe_tp/bigm_compute.py create mode 100644 amd_kernels/moe/mega_moe_tp/common.py create mode 100644 amd_kernels/moe/mega_moe_tp/communication.py create mode 100644 amd_kernels/moe/mega_moe_tp/fly_utils.py create mode 100644 amd_kernels/moe/mega_moe_tp/gemm.py create mode 100644 amd_kernels/moe/mega_moe_tp/ipc.py create mode 100644 amd_kernels/moe/mega_moe_tp/layer.py create mode 100644 amd_kernels/moe/mega_moe_tp/layout.py create mode 100644 amd_kernels/moe/mega_moe_tp/mega_moe_tp.py create mode 100644 amd_kernels/moe/mega_moe_tp/mega_moe_tp_bigm_tuned.csv create mode 100644 amd_kernels/moe/mega_moe_tp/mega_moe_tp_config.py create mode 100644 amd_kernels/moe/mega_moe_tp/mega_moe_tp_kernel.py create mode 100644 amd_kernels/moe/mega_moe_tp/mega_moe_tp_tuned.csv create mode 100644 amd_kernels/moe/mega_moe_tp/reference.py create mode 100644 amd_kernels/moe/mega_moe_tp/schedule.py create mode 100644 amd_kernels/moe/mega_moe_tp/sdma.py create mode 100644 amd_kernels/moe/mega_moe_tp/sp_rs_norm.py diff --git a/amd_kernels/README.md b/amd_kernels/README.md index bbc9b9a..e9de1d6 100644 --- a/amd_kernels/README.md +++ b/amd_kernels/README.md @@ -21,6 +21,8 @@ One line per kernel package. Each package's README has what it does, its entry p - [`moe/mxfp4_h6144_i768_e128_top4`](moe/mxfp4_h6144_i768_e128_top4/README.md): routed MXFP4 experts (A4W4) with the MXFP8 shared expert. +- [`moe/mega_moe_tp`](moe/mega_moe_tp/README.md): the tensor-parallel MXFP4 MoE layer as one persistent kernel + (all-gather / all-reduce, both GEMMs, reduce-scatter), and SpRsNorm. ## Attention diff --git a/amd_kernels/moe/mega_moe_tp/README.md b/amd_kernels/moe/mega_moe_tp/README.md new file mode 100644 index 0000000..77ca5e7 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/README.md @@ -0,0 +1,46 @@ +# MegaMoE TP: the tensor-parallel MXFP4 MoE layer as one kernel (gfx950) + +## What it does + +One launch runs a TP rank's whole MoE layer: experts replicated over the group, `inter_dim` sharded; the collectives +around GEMM1 + activation (SiLU, SwiGLU-OAI, SiTUv2) + GEMM2 of this rank's inter slice. `comm_mode` `ag_rs`: each rank +passes its own tokens `[m, H]` bf16 and gets its rows of the summed output (sequence parallel); `ar`: replicated +`[M, H]` in, the all-reduced `[M, H]` out on every rank. Weights MXFP4 with e8m0 scales per 32 in `layout.py`'s +layouts; activations MXFP4, exchanged rows FP8 (`comm_dtype="bf16"`: bf16). The kernel is persistent (one CTA per CU, +CTAs and ranks spin on each other) and needs the whole GPU; a wait times out after 2 s and sets a sticky error bit. +Optional fused tail (ag_rs): residual add + (Gemma)RMSNorm + FP8 rows. Big batches (beyond the small-batch engine's +capacity, e.g. 2048 local tokens on 256 CUs) run a chunked big-M kernel with SDMA transfers; it needs MORI built with +SDMA (importable, or `AKR_MORI_PATH`). Launch configs per batch: `mega_moe_tp_tuned.csv` / +`mega_moe_tp_bigm_tuned.csv` (`AKR_CONFIG_MEGA_MOE_TP`, `AKR_CONFIG_MEGA_MOE_TP_BIGM`). Contract (`reference.py`): +`mega_moe_tp_ref` summed over the ranks within rel. L2 0.1; graph replay = eager bit for bit; ar output identical on +every rank. SpRsNorm: SP int8 reduce-scatter + residual add + Gemma RMSNorm (+ sigmoid top-k router) in one launch. + +## Entry points + +- `MegaMoeTP`: one layer (`MegaMoeTPConfig`); `forward` is a collective (same call count, order and local token + count on every rank); `prepare(local_tokens)` before serving and graph capture compiles, runs and checks every launch + config (rule 4); `set_weights` reuses one instance across layers of one shape; `error_flag` / `check_errors` / + `reset` / `close` for the watchdog and teardown (module docstring of `layer.py`). +- `shuffle_weight`, `shuffle_scale`: the checkpoint's row-major MXFP4 codes / e8m0 scales into the kernel layouts. +- `SpRsNorm`: the sequence-parallel tail of an MoE layer whose partials are not reduced by the kernel. +- `ActivationType`, `mega_moe_tp_supported`, `MegaMoeTPLDSError`, `ERR_BIGM`, `COMM_MODES`. + +Dispatch: `mega_moe_tp.MegaMoeTPEngine.config` picks the schedule and row tile per batch (tuned CSV, else defaults); +`layer.MegaMoeTP.forward` sends big batches to `bigm_compute.MegaMoeTPBigM`. + +## Performance + +4 x MI355X, m3 (H 6144, I 768 per rank, 129 experts, top 5, SwiGLU), fp8 comm, random routing; graph replay of 10 +calls over 4 inputs, median of 20 replays, slowest rank, us per call. No baseline measured here. + +| M (global tokens) | 8 | 16 | 32 | 64 | 128 | 256 | 512 | 1024 | 2048 | 4096 | 8192 | +|---|---|---|---|---|---|---|---|---|---|---|---| +| ag_rs | 66.3 | 98.4 | 136.0 | 163.4 | 209.1 | 243.6 | 258.3 | 264.8 | 412.3 | 611.7 | 1148.3 | +| ar | 63.5 | 96.2 | 132.0 | 164.0 | 193.4 | 211.1 | 239.8 | 320.4 | 490.1 | 825.7 | 1587.8 | + +## Tests + +```bash +python3 -m pytest tests/moe/mega_moe_tp -q # 4 GPUs; big-M cases need MORI +python3 benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py # 4 GPUs +``` diff --git a/amd_kernels/moe/mega_moe_tp/__init__.py b/amd_kernels/moe/mega_moe_tp/__init__.py new file mode 100644 index 0000000..e5ddeb3 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/__init__.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Tensor-parallel MXFP4 MoE layer as one persistent kernel (MegaMoE TP), and SpRsNorm (README).""" + +from .layer import ( + COMM_MODES, + ERR_BIGM, + ActivationType, + MegaMoeTP, + MegaMoeTPConfig, + MegaMoeTPLDSError, + mega_moe_tp_supported, +) +from .layout import shuffle_scale, shuffle_weight +from .sp_rs_norm import SpRsNorm + +__all__ = [ + "COMM_MODES", + "ERR_BIGM", + "ActivationType", + "MegaMoeTP", + "MegaMoeTPConfig", + "MegaMoeTPLDSError", + "SpRsNorm", + "mega_moe_tp_supported", + "shuffle_scale", + "shuffle_weight", +] diff --git a/amd_kernels/moe/mega_moe_tp/bigm_compute.py b/amd_kernels/moe/mega_moe_tp/bigm_compute.py new file mode 100644 index 0000000..60be328 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/bigm_compute.py @@ -0,0 +1,640 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/bigm_compute.py (ROCm/aiter edf58c67a). +"""Host side of the big-M path of MegaMoeTP.forward (the kernel: compile_bigm_kernel in +mega_moe_tp_kernel.py): MegaMoeTPBigM (the SDMA queues and hipIpc arena, buffers, per-batch +plans, launches), the bf16 row gather of the fused tail. +""" + +import ctypes +import inspect +import time + +import torch +import torch.distributed as dist +from flydsl.utils import env as fenv + +from .common import device_key, nearest, tuned_config, tuned_rows +from .fly_utils import _preload_compiled, _run_compiled +from .gemm import bigm_lds +from .mega_moe_tp import LDS_LIMIT +from .mega_moe_tp_config import ( + BIGM_CTRL_INTS, + C_ERR, + CMAX, + NQ, + PAD, + UNIT_ROWS, + UQ, + i8_row_bytes, + unit_cap, +) +from .mega_moe_tp_kernel import compile_bigm_kernel +from .sdma import SdmaComm, compile_signal_read + +__all__ = ["MegaMoeTPBigM"] + + +def _bigm_cached(launch, args): + """Collective-free: compile launch(*args) without dispatching a kernel. FlyDSL's disk cache + (keyed on the traced IR and the linked SDMA bitcode) makes this a load in a warm process. + """ + _preload_compiled(launch, *args) + + +_HIP = None + + +def _hip(): + """the HIP runtime torch uses (ctypes): stream write / wait value""" + global _HIP + if _HIP is None: + torch.cuda.init() + with open("/proc/self/maps") as maps: + path = next(l.split()[-1] for l in maps if "libamdhip64" in l) + lib = ctypes.CDLL(path) + lib.hipStreamWriteValue32.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_uint32, + ctypes.c_uint, + ] + lib.hipStreamWaitValue32.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_uint32, + ctypes.c_uint, + ctypes.c_uint32, + ] + lib.hipStreamWriteValue32.restype = lib.hipStreamWaitValue32.restype = ( + ctypes.c_int + ) + _HIP = lib + return _HIP + + +def _hip_check(err): + if err != 0: + raise RuntimeError(f"MegaMoeTP big-M: HIP stream value op failed ({err})") + + +def _kcopy(dst, src): + """dst <- src by a shader kernel (dst: a peer's memory). Not copy_: a contiguous copy_ is a + hipMemcpyAsync, which ROCm runs on the SDMA engine of that xGMI link -- the engines the + big-M kernel's MORI queues use; ROCr's own queue there slows every later big-M put. + """ + torch.mul(src, 1, out=dst) + + +class _CudaBuf: + """__cuda_array_interface__ of raw device bytes (a peer's arena as mapped here).""" + + def __init__(self, ptr, nbytes): + self.__cuda_array_interface__ = { + "shape": (nbytes,), + "typestr": "|u1", + "data": (ptr, False), + "version": 3, + } + + +class MegaMoeTPBigM: + """Host side of the big-M dispatch: the SDMA queues and arena (same regions on every + rank), scratch buffers and the launch. + moe(x [m, H], topk_weights / topk_ids [m, K1], out, tail) (MegaMoeTP.forward, ag_rs) + -> y [m, H], with tail = (res_in, res_out, norm_w) (y, q [T, H] fp8, scales [T]). + Per batch the chunk plan and options come from the tuned csv, else defaults. + """ + + REGIONS = ("xq", "xs", "ids", "tw", "eq", "erecv", "q", "qs") + + def __init__( + self, + *, + hidden, + inter, + experts, + topk, + max_tokens, + eps, + gemma, + swiglu_limit, + w1, + w1_scale, + w2, + w2_scale, + group=None, + device=None, + opts=None, + act="swiglu", + ygather=False, + ): + self.H, self.I, self.E, self.K = hidden, inter, experts, topk + self.K1, self.NE = topk + 1, experts + 1 + self.tp = dist.get_world_size(group) + self.rank = dist.get_rank(group) + self.Tmax = max_tokens + self.act = act # the units' activation ("swiglu", "silu") + self.ygather = bool(ygather) # moe(gather=True): the "q" region holds bf16 rows + self._plans = {} + self.cfg = (float(eps), bool(gemma), float(swiglu_limit)) + self.opts = tuple(sorted((opts or {}).items())) # tuned BIGM_* options + self.w = ( + w1.view(torch.uint8), + w1_scale.view(torch.uint8), + w2.view(torch.uint8), + w2_scale.view(torch.uint8), + ) + H, T, K1, tp = hidden, max_tokens, self.K1, self.tp + RB = i8_row_bytes(H) + # the kernel's 32-bit offsets into these regions (the route rows: per chunk, _chunks) + for what, n in (("E rows", T * RB), ("y gather rows", T * H * 2)): + if n >= 1 << 31: + raise ValueError( + f"MegaMoeTP big-M: {what} of {T} tokens exceed 32-bit offsets" + ) + sizes = { + "xq": T * H // 2, + "xs": T * H // 32, + "ids": T * K1 * 4, + "tw": T * K1 * 4, + "eq": T * RB, + "erecv": T * RB, + "q": T * H * (2 if self.ygather else 1), + "qs": T * 4, + } + sd = self.sd = SdmaComm(group=group, num_queues=NQ, device=device) + for r_ in self.REGIONS: + sd.reserve(r_, sizes[r_]) + # gather_rows: every rank's bf16 tail rows (not a kernel region) + sd.reserve("brows", 2 * T * H * 2) # two buffers in turn + sd.reserve("bflag", tp * 64) # per-rank epoch flags (gather_rows) + sd.commit() + # the signal-base buffer each launch reads (the other: the next launch's, from GW) + self._cpar = 0 + self._cons = tuple(int(c.data_ptr()) for c in sd.consumed) + if self._cons[0] % (2 * tp * NQ * 4) or self._cons[1] != self._cons[0] ^ ( + tp * NQ * 4 + ): + raise RuntimeError(f"MegaMoeTP big-M: misaligned signal bases {self._cons}") + dev = self.device = sd.device + self.offs = torch.tensor( + [sd.offset(r_) for r_ in self.REGIONS], dtype=torch.int64, device=dev + ) + m = T // tp + bf = torch.bfloat16 + self.normed = torch.empty((m, H), dtype=bf, device=dev) + self.resout = torch.empty((m, H), dtype=bf, device=dev) + self.ybuf = torch.empty((m, H), dtype=bf, device=dev) + # route rows / lists of every chunk packed (chunk k from route tp * o_k * K1) + self.routes = torch.empty(T * K1 * H, dtype=bf, device=dev) + self.lrix = torch.empty(T * K1, dtype=torch.int32, device=dev) + self.lwt = torch.empty(T * K1, dtype=torch.float32, device=dev) + # unit slots of every chunk ([C][8][ucap of the batch]); _chunks grows it if a plan + # needs more + self.uslots = 2 * (-(-tp * m * K1 // UNIT_ROWS) // 8) + CMAX * ( + -(-self.NE // 8) + UQ + 2 + ) + self.utab = torch.zeros(8 * self.uslots * 4, dtype=torch.int32, device=dev) + self.ctrl = torch.zeros(BIGM_CTRL_INTS + 32, dtype=torch.int32, device=dev) + self.chunks_t = {} + # moe()'s padding buffers (m not a multiple of PAD), the largest padded size + self._pads = ( + torch.zeros((m, H), dtype=bf, device=dev), + torch.zeros((m, K1), dtype=torch.float32, device=dev), + torch.full((m, K1), -1, dtype=torch.int32, device=dev), + torch.zeros((m, H), dtype=bf, device=dev), + torch.empty((T, H), dtype=torch.uint8, device=dev), + torch.empty((T,), dtype=torch.float32, device=dev), + ) + self._pad_dirty = 0 # rows below may hold an earlier call's tokens / routes + self._ready = set() + self._brows_peer = None # gather_rows' state (set up on its first call) + self.cu = torch.cuda.get_device_properties(dev).multi_processor_count + self.nab = self._nab() + + def _nab(self): + """the deepest A ring (dividing the GEMM1 chunk count) whose LDS (+ the kernel's + row-phase scratch, < 4 KiB) fits a CU""" + kch = self.H // 256 + for n in (4, 3, 2): + if ( + kch % n == 0 + and bigm_lds(UNIT_ROWS, n, self.I)["BYTES"] + 4096 <= LDS_LIMIT + ): + return n + raise ValueError( + f"MegaMoeTP big-M: no A ring depth fits hidden {self.H} / inter {self.I}" + ) + + def gather_rows(self, rows, local=False): + """Collective: every rank's rows [m, H] bf16 -> [tp * m, H] (rank r's at r * m), a + view of the arena valid until the next call. local: this rank's rows only (placed + at rank * m; no peer writes). Each rank writes its rows into every + peer's arena (hipIpc mapping; one side stream per peer), then stream-ordered flags: + hipStreamWriteValue32 of this call's epoch into every peer's flag slot of this rank, + hipStreamWaitValue32 on every peer's slot here. No host sync (the host keeps running + ahead of the GPU). Two buffers in turn: a peer's writes of call k + 2 follow its wait + for this rank's flag of call k + 1, which this stream writes after whatever it ran on + call k's rows.""" + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("MegaMoeTP big-M: gather_rows is not graph-capturable") + m, H = rows.shape + T = m * self.tp + if T > self.Tmax or rows.dtype != torch.bfloat16 or H != self.H: + raise ValueError( + f"MegaMoeTP big-M: gather_rows needs bf16 [m, {self.H}] rows, m * tp <= " + f"{self.Tmax}; got {rows.dtype} {tuple(rows.shape)}" + ) + stream = torch.cuda.current_stream() + if self._brows_peer is None: + off, half = self.sd.offset("brows"), self.Tmax * H * 2 + bases = self.sd.arena.base_ptrs + self._brows_peer = [ + [ + torch.as_tensor( + _CudaBuf(b_ + off + i * half, half), device=self.device + ) + .view(torch.bfloat16) + .view(self.Tmax, H) + for b_ in bases + ] + for i in range(2) + ] + foff = self.sd.offset("bflag") + # flag slot of rank r (64 B apart) in rank p's arena, as mapped here + self._bflag = [[b_ + foff + r * 64 for r in range(self.tp)] for b_ in bases] + self.sd.view("bflag", (self.tp * 16,), torch.int32).zero_() + self._brows_streams = [ + torch.cuda.Stream(self.device) for _ in range(self.tp) + ] + self._bepoch = 0 + torch.cuda.synchronize(self.device) + dist.barrier(group=self.sd.group) + self._bepoch += 1 + ep = self._bepoch & 0x7FFFFFFF + dst = self._brows_peer[self._bepoch & 1] + src = rows.contiguous() + lo, hi = self.rank * m, (self.rank + 1) * m + if local: + _kcopy(dst[self.rank][lo:hi], src) + return dst[self.rank][:T] + ev = torch.cuda.Event() + ev.record(stream) + for p in range(self.tp): + if p == self.rank: + continue + s_ = self._brows_streams[p] + s_.wait_event(ev) + with torch.cuda.stream(s_): + _kcopy(dst[p][lo:hi], src) + src.record_stream(s_) + _kcopy(dst[self.rank][lo:hi], src) + hip, sh = _hip(), stream.cuda_stream + for p in range(self.tp): + if p != self.rank: + stream.wait_stream(self._brows_streams[p]) + for p in range(self.tp): # this rank's rows landed in p's arena + if p != self.rank: + _hip_check( + hip.hipStreamWriteValue32(sh, self._bflag[p][self.rank], ep, 0) + ) + for p in range(self.tp): # every peer's rows landed here + if p != self.rank: + _hip_check( + hip.hipStreamWaitValue32( + sh, self._bflag[self.rank][p], ep, 0, 0x7FFFFFFF + ) + ) + return dst[self.rank][:T] + + def poll_errors(self) -> int: + """Nonzero if a wait on a peer inside the kernel gave up since the last clear.""" + return int(self.ctrl[C_ERR].item()) + + def clear_errors(self) -> None: + self.ctrl[C_ERR:].zero_() + + def error_flag(self): + """The sticky watchdog word as a 1-element device tensor (no sync): 1 if a wait gave + up since the last clear.""" + return self.ctrl[C_ERR : C_ERR + 1] + + def reset(self) -> None: + """Collective: re-base the per-(peer, queue) signal bases on the received signals and + clear the watchdog word -- after a wait gave up the bases may be out of step with the + peers' puts (every later wait on that queue would time out).""" + torch.cuda.synchronize(self.device) + dist.barrier(group=self.sd.group) + time.sleep(0.2) # the puts of the peers' finished launches land + got = torch.zeros(self.tp * NQ, dtype=torch.int32, device=self.device) + _run_compiled( + compile_signal_read(self.tp, NQ), + self.sd.dev_comm.ptr, + got.data_ptr(), + torch.cuda.current_stream(), + ) + self.sd.consumed[self._cpar].copy_(got) + self.ctrl.zero_() + torch.cuda.synchronize(self.device) + dist.barrier(group=self.sd.group) + + def close(self) -> None: + """Collective: free the SDMA queues and the arena (unusable afterwards).""" + self.sd.close() + + def _plan(self, T): + """(chunk weights, BIGM_* options) of a batch of T tokens (T / tp a multiple + of PAD): the tuned csv's, else the defaults; at most one chunk per PAD local rows + """ + p = self._plans.get(T) + if p is None: + o = dict(self.opts) + key = { + **device_key(self.device), + "model_dim": self.H, + "inter_dim": self.I, + "experts": self.NE, + "topk": self.K1, + "tp": self.tp, + } + path = tuned_config("mega_moe_tp_bigm") + tu = nearest(tuned_rows(path, key, "tokens"), T, clamp=True) + if tu is not None: + w = [int(x) for x in tu["chunks"].split(",")] + opts = (kv.split("=", 1) for kv in tu.get("opts", "").split(";") if kv) + o = {**dict(opts), **o} + else: + w = [1, 1] if T <= 16384 else [1, 2, 2, 1] + if not 0 < len(w) <= CMAX or min(w) <= 0: + raise ValueError(f"MegaMoeTP big-M: bad chunk weights {w} (T={T})") + n = max(1, T // self.tp // PAD) + if len(w) > n: # few rows: fewer chunks (the largest weights) + w = sorted(w, reverse=True)[:n] + p = self._plans[T] = (tuple(w), tuple(sorted(o.items()))) + return p + + def _split(self, m): + w = self._plan(m * self.tp)[0] + tot = sum(w) + # multiples of 64 rows (moe() pads m to PAD): whole input row items (ZX) in every head + # sub-block (BIGM_NSBA <= 4) + g = PAD + sizes = [max(g, (m * x // tot) // g * g) for x in w] + sizes[w.index(max(w))] += m - sum(sizes) + if m % g or sum(sizes) != m or min(sizes) <= 0 or any(x % g for x in sizes): + raise RuntimeError(f"MegaMoeTP big-M: cannot split {m} rows by {w}") + return sizes + + def _chunks(self, T): + m = T // self.tp + sizes = tuple(self._split(m)) + c = self.chunks_t.get(T) + if c is None: + # the kernel's 32-bit offsets of a chunk's route rows (rows * H bf16) + if self.tp * max(sizes) * self.K1 * self.H * 2 >= 1 << 31: + raise ValueError( + f"MegaMoeTP big-M: chunks {sizes} of {T} tokens: a chunk's route rows " + "exceed 32-bit offsets (more chunks: the tuned csv)" + ) + need = len(sizes) * self._ucap(T) + if need > self.uslots: + self.uslots = need + self.utab = torch.zeros( + 8 * need * 4, dtype=torch.int32, device=self.device + ) + os_ = [sum(sizes[:i]) for i in range(len(sizes))] + c = self.chunks_t[T] = torch.tensor( + [[o, s] for o, s in zip(os_, sizes)], + dtype=torch.int32, + device=self.device, + ) + return c + + def set_weights(self, w1, w1_scale, w2, w2_scale): + """the next launches use another layer's weights of the same shape""" + self.w = ( + w1.view(torch.uint8), + w1_scale.view(torch.uint8), + w2.view(torch.uint8), + w2_scale.view(torch.uint8), + ) + + def _ucap(self, T): + return unit_cap( + self.tp, + max(self._split(T // self.tp)), + self.K1, + self.NE, + UNIT_ROWS, + ) + + def _run(self, launch, args): + if launch not in self._ready: + # compile (or load) on every rank first: the kernel's waits on peers have a 2 s + # watchdog, a launch must not start while a peer is still compiling + _bigm_cached(launch, args) + torch.cuda.synchronize() + dist.barrier(group=self.sd.group) + self._ready.add(launch) + _run_compiled(launch, *args) + + def moe( + self, x, topk_weights, topk_ids, out=None, tail=None, local=False, gather=False + ): + """The MoE of every rank's tokens (ag_rs): x [m, H] bf16 (this rank's rows, global + rows rank * m + i), topk_weights f32 / topk_ids int32 [m, K1] (the shared expert E in + any column; ids outside [0, experts] are no route) -> y [m, H] bf16 = this rank's rows of + sum_r MoE_r(every row); tail = (res_in, res_out, norm_w): also res_out = y + res_in and + the (Gemma)RMSNorm of res_out as FP8 rows (+ f32 scales) of every rank -> (y, q [T, H] + fp8 (uint8), scales [T]); local: only this rank's q rows (rank * m ..) are written. + gather (ygather, no tail): y of every rank's rows -> [tp * m, H] (rank r's at r * m; out, + else a view of the arena valid until the next launch), gathered inside the kernel; + copies out of the arena by shader kernels (_kcopy). + Any m: the kernel runs on m rounded up to a multiple of 64 (PAD; the extra rows are zero, + with no routes), its results cut back to m rows.""" + if gather and not (self.ygather and tail is None): + raise ValueError("MegaMoeTP big-M: gather needs ygather and no tail") + m = x.shape[0] + if ( + x.dtype != torch.bfloat16 + or tuple(x.shape) != (m, self.H) + or not x.is_contiguous() + or tuple(topk_ids.shape) != (m, self.K1) + or tuple(topk_weights.shape) != (m, self.K1) + or topk_ids.dtype != torch.int32 + or topk_weights.dtype != torch.float32 + or not (topk_ids.is_contiguous() and topk_weights.is_contiguous()) + ): + raise ValueError( + f"MegaMoeTP big-M: need contiguous bf16 x [m, {self.H}], int32 topk_ids / " + f"f32 topk_weights [m, {self.K1}]; got {x.dtype} {tuple(x.shape)}, " + f"{topk_ids.dtype} {tuple(topk_ids.shape)}, {topk_weights.dtype} " + f"{tuple(topk_weights.shape)}" + ) + mp = -(-m // PAD) * PAD + if mp * self.tp > self.Tmax: + raise ValueError( + f"MegaMoeTP big-M: {m} local tokens (padded {mp}) exceed " + f"{self.Tmax // self.tp}" + ) + if mp == m: + if gather: + ya = self._moe(x, topk_weights, topk_ids, None, None, local, gather) + if out is not None: + _kcopy(out, ya) + return ya if out is None else out + return self._moe(x, topk_weights, topk_ids, out, tail, local) + # one set of padded buffers (allocated at init, the largest padded size), sliced to mp: + # rows m.. of x / tw / ids / res must be zero / no route; rows below _pad_dirty may hold + # an earlier call's tokens + xp, twp, idp, rp, qc, qsc = self._pads + hi_ = min(self._pad_dirty, mp) + if hi_ > m: # rows m .. mp of this call: clear what an earlier call left there + xp[m:hi_].zero_() + twp[m:hi_].zero_() + idp[m:hi_].fill_(-1) + rp[m:hi_].zero_() + # dirty from now on: this call's rows (< m), plus an earlier call's beyond mp + self._pad_dirty = self._pad_dirty if self._pad_dirty > mp else m + xp, twp, idp, rp = xp[:mp], twp[:mp], idp[:mp], rp[:mp] + qc, qsc = qc[: self.tp * m], qsc[: self.tp * m] + xp[:m].copy_(x) + twp[:m].copy_(topk_weights) + idp[:m].copy_(topk_ids) + if tail is not None: + res_in, res_out, norm_w = tail + rp[:m].copy_(res_in) + y, q, qs = self._moe(xp, twp, idp, None, (rp, rp, norm_w), local) + res_out.copy_(rp[:m]) + # q / scales as [rank][m] rows (the kernel's are [rank][mp]) + src = [self.rank] if local else range(self.tp) + for r in src: + qc[r * m : (r + 1) * m].copy_(q[r * mp : r * mp + m]) + qsc[r * m : (r + 1) * m].copy_(qs[r * mp : r * mp + m]) + q, qs = qc, qsc + elif gather: + ya = self._moe(xp, twp, idp, None, None, local, gather) + if out is None: + out = torch.empty( + (self.tp * m, self.H), dtype=ya.dtype, device=ya.device + ) + for r in range(self.tp): + _kcopy(out[r * m : (r + 1) * m], ya[r * mp : r * mp + m]) + return out + else: + y = self._moe(xp, twp, idp, None, None, local) + y = y[:m] + if out is not None: + y = out.copy_(y) + return y if tail is None else (y, q, qs) + + @staticmethod + def tailq(tail, local, gather=False): + """the kernel variant (compile_bigm_kernel's tailq) of a moe() call""" + return 3 if gather else 0 if not tail else 2 if local else 1 + + def _io_launch(self, m, tailq): + """the kernel of m (a multiple of PAD) local rows""" + T = m * self.tp + eps, gemma, lim = self.cfg + w_, opts = self._plan(T) + order = 4 if self.tp * max(self._split(m)) >= 8192 else 3 + return compile_bigm_kernel( + self.H, + self.I, + self.E, + self.K, + self.tp, + len(w_), + eps, + gemma, + lim, + NAB=self.nab, + order=order, + opts=opts, + tailq=tailq, + act=self.act, + ) + + def io_rows_per_kernel(self, lo, hi): + """One local row count per distinct kernel (whatever the tail) among the batches of + lo .. hi local rows (padded to PAD as moe() does): warming these up compiles every big-M + kernel such batches can launch.""" + reps = {} + for m in range(-(-max(lo, 1) // PAD) * PAD, -(-hi // PAD) * PAD + 1, PAD): + reps.setdefault(self._io_launch(m, 0), m) + return sorted(reps.values()) + + def precompile(self, rows, tailq): + """Collective: the kernels of these local row counts (tailq: 0 - 3 as moe()) into + the disk cache, split over the ranks -- rank r compiles every tp-th -- so a cold cache + costs about one compile, not one per kernel; the moe() warm-ups then load them. No-op + without a disk cache.""" + if not fenv.runtime.enable_cache: + return + mine = [m for i, m in enumerate(rows) if i % self.tp == self.rank] + for m in mine: + launch = self._io_launch(m, tailq) + if launch in self._ready: + continue + # (compile only: the argument values do not matter, their types are the launch's) + n = len(inspect.signature(launch.func).parameters) + _bigm_cached(launch, (0,) * (n - 1) + (torch.cuda.current_stream(),)) + dist.barrier(group=self.sd.group) + + def _moe(self, x, topk_weights, topk_ids, out, tail, local, gather=False): + """moe() of m (a multiple of PAD) rows""" + m = x.shape[0] + T = m * self.tp + w1, w1s, w2, w2s = self.w + y = self.normed[:m] if out is None else out + if tail is not None: + res_in, res_out, norm_w = tail + else: + res_in, res_out, norm_w = x, self.resout, x + launch = self._io_launch(m, self.tailq(tail is not None, local, gather)) + chunks = self._chunks(T) # (first: it may grow the unit table) + self.ctrl[:BIGM_CTRL_INTS].zero_() + self._run( + launch, + ( + self.sd.base, + self.sd.bases.data_ptr(), + self.sd.dev_comm.ptr, + self._cons[self._cpar], + self.offs.data_ptr(), + x.data_ptr(), + topk_ids.data_ptr(), + topk_weights.data_ptr(), + norm_w.data_ptr(), + res_in.data_ptr(), + y.data_ptr(), + res_out.data_ptr(), + self.ybuf.data_ptr(), + self.routes.data_ptr(), + self.lrix.data_ptr(), + self.lwt.data_ptr(), + self.utab.data_ptr(), + w1.data_ptr(), + w1s.data_ptr(), + w2.data_ptr(), + w2s.data_ptr(), + self.ctrl.data_ptr(), + chunks.data_ptr(), + self.rank, + m, + self._ucap(T), + self.cu, + torch.cuda.current_stream(), + ), + ) + self._cpar ^= 1 # (the next launch reads what this one's GW wrote) + if gather: + return self.sd.view("q", (T, self.H), torch.bfloat16) + if tail is None: + return y + q = self.sd.view("q", (T, self.H), torch.uint8) + qs = self.sd.view("qs", (T,), torch.float32) + return y, q, qs diff --git a/amd_kernels/moe/mega_moe_tp/common.py b/amd_kernels/moe/mega_moe_tp/common.py new file mode 100644 index 0000000..6349d0f --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/common.py @@ -0,0 +1,519 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/common.py (ROCm/aiter edf58c67a). +"""Device helpers shared by the MegaMoE TP and SpRsNorm kernels.""" + +from __future__ import annotations + +import csv +import functools +import os + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl._mlir.dialects import llvm as _llvm +from flydsl.expr import math as fmath +from flydsl.expr import range_constexpr, rocdl +from flydsl.expr.typing import T + +from amd_kernels.common import buffer_ops + +from .fly_utils import _e8m0_from_amax, _fabs_f32 + +MAX_TP = 8 +DEADLINE = 200_000_000 +AUX_SC1 = 16 +AUX_SYS = 1 | 16 + +MONO, ACQ, REL, ACQ_REL = ( + fx.AtomicOrdering.Monotonic, + fx.AtomicOrdering.Acquire, + fx.AtomicOrdering.Release, + fx.AtomicOrdering.AcqRel, +) + + +def _attr(v): + return ir.IntegerAttr.get(ir.IntegerType.get_signless(32), int(v)) + + +def _u(v): + return v.ir_value() if hasattr(v, "ir_value") else v + + +def i32(v): + raw = v if isinstance(v, ir.Value) else getattr(v, "ir_value", lambda: None)() + if raw is not None and isinstance(raw.type, ir.IndexType): + return fx.Int32(fx.index_cast(T.i32, raw)) + return fx.Int32(v) + + +class _LazyTy: + def __init__(self, fn): + self.fn = fn + + +V4I = _LazyTy(lambda: T.vec(4, T.i32)) +V4F = _LazyTy(lambda: T.vec(4, T.f32)) +V2I = _LazyTy(lambda: T.vec(2, T.i32)) + + +def _ty(t): + return t.fn() if isinstance(t, _LazyTy) else t + + +# LDS is one byte arena carved by computed offsets (no typed views): pointers +# are built from the integer address. +def lds_ptr(base, off, ty, align=4): + ty = _ty(ty) + if isinstance(ty, ir.VectorType): + ty = ty.element_type + return fx.inttoptr( + fx.PointerType.get(ty, fx.AddressSpace.Shared, align), base + i32(off) + ) + + +def lds_ld(base, off, ty=None, align=4): + ty = _ty(ty) if ty is not None else T.i32 + return fx.ptr_load(lds_ptr(base, off, ty, align), result_type=ty) + + +def lds_ld_i32(base, off): + return i32(lds_ld(base, off)) + + +def lds_st(base, off, val, align=4): + v = _u(val) + fx.ptr_store(v, lds_ptr(base, off, v.type, align)) + + +def lds_p(base, off): + return fx.inttoptr( + fx.PointerType.get(T.i32, fx.AddressSpace.Shared, 4), + fx.Int32(base + i32(off)), + ) + + +def gptr(addr): + return fx.inttoptr( + fx.PointerType.get(T.i32, fx.AddressSpace.Global, 4), fx.Int64(addr) + ) + + +def lds_atomic_add(base, off, v, ordering=MONO): + return i32( + fx.atomic_add( + lds_p(base, off), i32(v), syncscope="workgroup", ordering=ordering + ) + ) + + +def lds_atomic_or(base, off, v): + return i32(fx.atomic_or(lds_p(base, off), i32(v), syncscope="workgroup")) + + +def lds_ld_acq(base, off): + return i32( + fx.generic_load( + lds_p(base, off), + dtype=fx.Int32, + memory_order=MONO, + syncscope="workgroup", + volatile=True, + ) + ) + + +def lds_st_rel(base, off, v): + fx.generic_store(lds_p(base, off), i32(v), memory_order=REL, syncscope="workgroup") + + +def lds_cas(base, off, cmp, new): + return i32( + fx.atomic_cas(lds_p(base, off), i32(cmp), i32(new), syncscope="workgroup")[0] + ) + + +def _ctpop(v): + return i32(fx.ctpop(i32(v))) + + +def g_ld_rel(addr, scope): + return i32( + fx.generic_load( + gptr(addr), + dtype=fx.Int32, + memory_order=MONO, + syncscope=scope, + volatile=True, + ) + ) + + +def g_ld_sys(addr): + return g_ld_rel(addr, "one-as") + + +def g_st_sys(addr, v): + fx.generic_store(gptr(addr), i32(v), memory_order=MONO, syncscope="one-as") + + +def g_add_agent(addr, v): + return i32(fx.atomic_add(gptr(addr), i32(v), syncscope="agent")) + + +def g_or_agent(addr, v): + fx.atomic_or(gptr(addr), i32(v), syncscope="agent") + + +def g_ld_i32(addr): + return i32(fx.generic_load(gptr(addr), dtype=fx.Int32)) + + +def uni(v): + return i32(rocdl.readfirstlane(T.i32, i32(v))) + + +def uni64(v): + v = fx.Int64(v) + lo = uni(fx.Int32(v & fx.Int64(0xFFFFFFFF))) + hi = uni(fx.Int32(v >> 32)) + return (fx.Int64(hi) << 32) | (fx.Int64(lo) & fx.Int64(0xFFFFFFFF)) + + +# Buffer resources from raw (peer / arena) addresses plus per-lane byte offsets +# and explicit cache-policy bits (sc0 / sc1 / nt): no layout-tensor form. +def rsrc(addr, nbytes=None): + if nbytes is None: + return buffer_ops.create_buffer_resource_from_addr(uni64(addr)) + return buffer_ops.create_buffer_resource_from_addr( + uni64(addr), num_records_bytes=fx.Int64(uni(nbytes)) + ) + + +def bld(rs, voff, soff, ty, aux=0): + return rocdl.raw_ptr_buffer_load(_ty(ty), rs, i32(voff), i32(soff), aux=aux) + + +def bst(val, rs, voff, soff, aux=0): + rocdl.raw_ptr_buffer_store( + _u(val), rs, _u(i32(voff)), _u(i32(soff)), aux=_attr(aux) + ) + + +# LDS-DMA (buffer_load ... lds, M0 = LDS base): the async form, so the compiler's +# waitcnt pass does not wait on it; waits are explicit (wait_vm). +def _dma(lds_addr, rs, voff, soff, nbytes, aux): + lds_ptr = fx.to_llvm_ptr( + fx.inttoptr(fx.PointerType.get(T.i8, fx.AddressSpace.Shared), uni(lds_addr)) + ) + fx.rocdl.raw_ptr_buffer_load_async_lds( + rs, + lds_ptr, + _u(i32(nbytes)), + _u(i32(voff)), + _u(uni(soff)), + _u(i32(0)), + aux=_attr(aux), + ) + + +def dma16(lds_addr, rs, voff, soff, nt=False): + _dma(lds_addr, rs, voff, soff, 16, 2 if nt else 0) + + +def dma4(lds_addr, rs, voff, soff, sys=False): + _dma(lds_addr, rs, voff, soff, 4, AUX_SYS if sys else 0) + + +def wait_vm(n): + rocdl.s_waitcnt(vmcnt=min(int(n), 63)) + + +def wait_lgkm0(): + rocdl.s_waitcnt(lgkmcnt=0) + + +def amax(vals): + am = _fabs_f32(vals[0]) + for v in vals[1:]: + am = am.maximumf(_fabs_f32(v)) + return am + + +def fp4_pack(vals, qs): + pk = i32(0) + for k in range(len(vals) // 2): + pk = rocdl.cvt_scalef32_pk_fp4_f32( + T.i32, pk, vals[2 * k], vals[2 * k + 1], qs, k + ) + return fx.Int32(pk) + + +def fp8x4_pack(f, qs): + zero = fx.full(2, 0, fx.Int16) + w = rocdl.cvt_scalef32_pk_fp8_f32( + T.vec(2, T.i16), _u(zero), _u(f[0]), _u(f[1]), _u(qs), False + ) + w = rocdl.cvt_scalef32_pk_fp8_f32( + T.vec(2, T.i16), w, _u(f[2]), _u(f[3]), _u(qs), True + ) + return fx.Vector(w).bitcast(fx.Int32)[0] + + +def fp8x4_unpack(d, sc): + out = [] + for sel in (False, True): + v = fx.Vector( + rocdl.cvt_scalef32_pk_f32_fp8(T.vec(2, T.f32), _u(i32(d)), _u(sc), sel) + ) + out += [fx.Float32(v[0]), fx.Float32(v[1])] + return out + + +def e8_scale(e8): + return ((fx.Int32(e8) & i32(0xFF)) << i32(23)).bitcast(fx.Float32) + + +def fp8x8_decode(ld): + dv = fx.Vector(ld[0]) + sc = e8_scale(ld[1]) + return fp8x4_unpack(fx.Int32(dv[0]), sc) + fp8x4_unpack(fx.Int32(dv[1]), sc) + + +def mxfp8x8(acc): + am = amax(acc) + am = am.maximumf(am.shuffle_xor(i32(1), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(2), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=448.0) + return fp8x4_pack(acc[0:4], qs), fp8x4_pack(acc[4:8], qs), fx.Int32(e8) & i32(0xFF) + + +def scales4(e8i): + e = e8i.bitcast(fx.Float32) + sc = e8i + for k in range_constexpr(1, 4): + sc = sc | (e.shuffle_xor(i32(4 * k), i32(64)).bitcast(fx.Int32) << i32(8 * k)) + return sc + + +def bf16_bits(f): + # Preserve the packed conversion's code generation on gfx950. + return fx.Int32( + fx.Vector.from_elements([fx.Float32(f).to(fx.BFloat16)], fx.BFloat16).bitcast( + fx.Int16 + )[0] + ) & i32(0xFFFF) + + +def pack_bf16x2(a, b): + return bf16_bits(a) | (bf16_bits(b) << i32(16)) + + +def bf16x8_to_f32(d): + dv = fx.Vector(d) + out = [] + for q in range_constexpr(4): + w = fx.Int32(dv[q]) + out.append((w << i32(16)).bitcast(fx.Float32)) + out.append((w & i32(-65536)).bitcast(fx.Float32)) + return out + + +def pack_bf16x8(acc): + return fx.Vector.from_elements( + [pack_bf16x2(acc[2 * q], acc[2 * q + 1]) for q in range(4)], fx.Int32 + ) + + +def sum_live(acc, vals, live): + return [x + live.select(y, fx.Float32(0.0)) for x, y in zip(acc, vals)] + + +def wave_red(v, lane, op): + for k in (1, 2, 4, 8, 16, 32): + v = op(v, i32(rocdl.ds_bpermute(T.i32, (lane ^ i32(k)) * i32(4), v))) + return uni(v) + + +def wave_rank(hit): + b = fx.Int64(rocdl.ballot(T.i64, hit)) + lo = i32(b & fx.Int64(0xFFFFFFFF)) + hi = i32(b >> 32) + below = rocdl.mbcnt_lo(T.i32, _u(lo), _u(i32(0))) + below = i32(rocdl.mbcnt_hi(T.i32, _u(hi), below)) + return below, _ctpop(lo) + _ctpop(hi) + + +def _swap(fn, x, y): + st = _llvm.StructType.get_literal([T.i32, T.i32]) + r = fn(st, _u(i32(x)), _u(i32(y)), False, False) + return ( + i32(_llvm.ExtractValueOp(T.i32, r, [0]).res), + i32(_llvm.ExtractValueOp(T.i32, r, [1]).res), + ) + + +def swap16(x, y): + return _swap(rocdl.permlane16_swap, x, y) + + +def swap32(x, y): + return _swap(rocdl.permlane32_swap, x, y) + + +def widen(v4): + z = fx.full(4, 0, fx.Int32) + return fx.Vector(v4).shuffle(z, list(range(8))) + + +# Scaled MFMA with per-lane E8M0 scales read from LDS: no MMA-atom form. +def mfma(a4, b4, cacc, sa, sb, b8=False): + bx = b4 if b8 else widen(b4) + return fx.Vector( + rocdl.mfma_scale_f32_16x16x128_f8f6f4( + _ty(V4F), + [widen(a4), bx, cacc, 4, 0 if b8 else 4, 0, i32(sa), 0, i32(sb)], + ) + ) + + +def cat8(lo, hi): + return _u(fx.Vector(lo).shuffle(fx.Vector(hi), list(range(8)))) + + +# s_memrealtime (100 MHz) has no wrapper. +def now(): + return fx.Int64( + _llvm.call_intrinsic(T.i64, "llvm.amdgcn.s.memrealtime", [], [], []) + ) + + +def before(cur, target): + """cur has not reached target. Wrap-safe for int32 launch epochs (serial-number + order: correct while every compared flag is within 2**31 launches of target).""" + return (cur - target) < i32(0) + + +def alive(t0): + return (now() - t0) < fx.Int64(DEADLINE) + + +@flyc.jit +def spin_lds_ge(L, off, target): + cur = lds_ld_acq(L, off) + while cur < target: + rocdl.s_sleep(0) + cur = lds_ld_acq(L, off) + + +@flyc.jit +def spin0(L, lane, off, target): + if lane == i32(0): + spin_lds_ge(L, off, target) + rocdl.sched_barrier(0) + + +@flyc.jit +def poll_sys_ge(addr, target, nap=2): + cur = g_ld_sys(addr) + t0 = now() + while before(cur, target) & alive(t0): + rocdl.s_sleep(nap) + cur = g_ld_sys(addr) + return cur + + +def i8x4_pack(f, inv): + """4 f32 * inv -> int8 (round to even, clamped to +-127) packed in an i32""" + w = i32(0) + for k in range_constexpr(4): + q = fx.Int32(fmath.roundeven(f[k] * inv)) + q = fx.max(fx.min(q, i32(127)), i32(-127)) + w = w | ((q & i32(0xFF)) << i32(8 * k)) + return w + + +def i8x4_unpack(d, sc): + out = [] + for k in range_constexpr(4): + b = (fx.Int32(d) << i32(24 - 8 * k)) >> i32(24) + out.append(b.to(fx.Float32) * sc) + return out + + +def f32_bits_add(x, y): + """f32 add on i32 bit patterns (block reductions over i32 LDS words)""" + return (x.bitcast(fx.Float32) + y.bitcast(fx.Float32)).bitcast(fx.Int32) + + +def f32_bits_max(x, y): + return x.bitcast(fx.Float32).maximumf(y.bitcast(fx.Float32)).bitcast(fx.Int32) + + +def i4x8_pack(f, inv): + w = i32(0) + for k in range_constexpr(8): + q = fx.Int32(fmath.roundeven(f[k] * inv)) + q = fx.max(fx.min(q, i32(7)), i32(-8)) + w = w | ((q & i32(0xF)) << i32(4 * k)) + return w + + +def i4x8_unpack(d, sc): + out = [] + for k in range_constexpr(8): + b = (fx.Int32(d) << i32(28 - 4 * k)) >> i32(28) + out.append(b.to(fx.Float32) * sc) + return out + + +# ---- tuned-config CSVs ---- +@functools.cache +def _read(path: str, mtime: float) -> tuple: + if not os.path.isfile(path): + return () + with open(path) as f: + return tuple(csv.DictReader(f)) + + +def tuned_rows(path: str, key: dict, tokens_col: str) -> dict: + """{tokens: row} of the rows of path matching every key column (compared as str).""" + path = os.path.abspath(path) + mtime = os.path.getmtime(path) if os.path.isfile(path) else 0.0 + want = {k: str(v) for k, v in key.items()} + return { + int(r[tokens_col]): r + for r in _read(path, mtime) + if all(r.get(k) == v for k, v in want.items()) + } + + +def nearest(rows: dict, tokens: int, clamp: bool = False): + """The row of the nearest tuned count <= tokens inside [min, max] of rows; outside it + None, or (clamp) the row of the nearer end.""" + if not rows: + return None + if not min(rows) <= tokens <= max(rows): + if not clamp: + return None + tokens = min(max(tokens, min(rows)), max(rows)) + return rows[max(t for t in rows if t <= tokens)] + + +def tuned_config(name: str) -> str: + """Path of the tuned-config CSV ``name`` (mega_moe_tp, mega_moe_tp_bigm): ``AKR_CONFIG_``, else the + package's ``_tuned.csv``.""" + default = os.path.join(os.path.dirname(os.path.abspath(__file__)), f"{name}_tuned.csv") + return os.environ.get(f"AKR_CONFIG_{name.upper()}", default) + + +def device_key(device=None) -> dict: + """gfx / cu_num of a device (the CSVs' leading key columns).""" + import torch + + props = torch.cuda.get_device_properties(device or torch.cuda.current_device()) + gfx = props.gcnArchName.split(":")[0] + return {"gfx": gfx, "cu_num": int(props.multi_processor_count)} diff --git a/amd_kernels/moe/mega_moe_tp/communication.py b/amd_kernels/moe/mega_moe_tp/communication.py new file mode 100644 index 0000000..5bf8eb3 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/communication.py @@ -0,0 +1,1392 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/communication.py (ROCm/aiter edf58c67a). +"""Per-instance device helpers (ctrl / peer addressing, LL packets, barriers, error +reports, route gather) and the communication: the input all-gather, and the +ReduceScatter of the GEMM2 partials (ar: plus the all-gather of the sums) with its +final sums, LL packets, comm wave roles and the per-launch init / finish.""" + +from __future__ import annotations + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import gpu, range_constexpr, rocdl +from flydsl.expr.typing import T + +from .fly_utils import _activation_mul_batch, _e8m0_from_amax, ceildiv +from .common import ( + ACQ, + ACQ_REL, + AUX_SC1, + AUX_SYS, + DEADLINE, + MAX_TP, + REL, + V2I, + V4I, + _ctpop, + alive, + amax, + before, + bf16x8_to_f32, + bld, + bst, + cat8, + e8_scale, + fp4_pack, + fp8x4_pack, + fp8x4_unpack, + fp8x8_decode, + g_add_agent, + g_ld_i32, + g_ld_rel, + g_ld_sys, + g_or_agent, + g_st_sys, + i32, + lds_atomic_add, + lds_atomic_or, + lds_cas, + lds_ld, + lds_ld_acq, + lds_ld_i32, + lds_st, + lds_st_rel, + mxfp8x8, + now, + pack_bf16x8, + poll_sys_ge, + rsrc, + scales4, + spin0, + sum_live, + uni, + wait_lgkm0, + wait_vm, + wave_rank, + wave_red, +) +from .mega_moe_tp_config import ( + C_AGFREE, + C_ARDY, + C_BARCNT, + C_BARGEN, + C_CDONE, + C_CLAIM, + C_CNT, + C_DONE, + C_DYNP, + C_EPOCH, + C_FBITS, + C_FRDY, + C_GEXP, + C_LRED, + C_MBOX, + C_NSIG, + C_PBITS, + C_PRDY, + C_PULL, + C_RLAND, + C_UL, + C_YAGM, + CTRL_AGG, + CTRL_AGX, + CTRL_CLM, + CTRL_EPB, + CTRL_ERR, + CTRL_LRDY, + CTRL_SC, + CTRL_XES, + ERR_CHUNK, + ERR_COMM, + ERR_FLAG, + ERR_META, + ERR_YAG, + FLAG_AGM, + FLAG_AGQ, + FLAG_RDY, + FLAG_YAG, + LRDY_STRIDE, + N_XCD, + NCTA_MAX, + NT, + NTT, + NW, + POLL_SLEEP, + SC_ALL, + SC_LINES, + SC_PUSH, + KernelCtx, +) + + +def build_communication(kc: KernelCtx) -> dict: + """The communication device functions of one instance (added to kc).""" + ACB, AG8, ARLL, CW, DLL, E, H, I = kc.get("ACB AG8 ARLL CW DLL E H I") + L_CTL, L_DCNT, L_DPRE, L_DYN = kc.get("L_CTL L_DCNT L_DPRE L_DYN") + L_INTER, L_INTERS, L_RIX, L_WT, MLL = kc.get("L_INTER L_INTERS L_RIX L_WT MLL") + MT, NBW, NCH, NCK, NMETA, NPC, NV = kc.get("MT NBW NCH NCK NMETA NPC NV") + RG, SCAN_G, SCAN_IT, SI_STRIDE = kc.get("RG SCAN_G SCAN_IT SI_STRIDE") + TMAX, TN, TOPK, TPC, VPL, XB, XLPR = kc.get("TMAX TN TOPK TPC VPL XB XLPR") + a8, act, agr, ar, comm_bf16 = kc.get("a8 act agr ar comm_bf16") + const_expr, lb, ll_rs, route_fp8 = kc.get("const_expr lb ll_rs route_fp8") + situ_beta, situ_linear_beta = kc.get("situ_beta situ_linear_beta") + swiglu_limit = kc.swiglu_limit + + def ctrl_at(a, idx): + return fx.Int64(a["ctrl"]) + fx.Int64(i32(idx) * i32(4)) + + def lrdy_at(a, c): + return ctrl_at(a, i32(CTRL_LRDY) + c * i32(LRDY_STRIDE)) + + def peer_sel(a, p): + v = fx.Int64(a["peer"][0]) + for j in range_constexpr(1, MAX_TP): + v = (p == i32(j)).select(fx.Int64(a["peer"][j]), v) + return v + + def peer_rs(a, p, key, nbytes=None): + return rsrc(fx.Int64(a["peer"][p]) + fx.Int64(a[key]), nbytes) + + def own_rs(a, key, nbytes=None): + return rsrc(a["mine"] + fx.Int64(a[key]), nbytes) + + def route_region_bytes(ttot): + return ttot * i32(TOPK * (H + H // 32 if route_fp8 and not DLL else 2 * H)) + + def ll_pkt(a, d0, d1, e8): + tag = ((a["epoch"] & i32(0xFFFFFF)) << i32(8)) | e8 + return fx.Vector.from_elements([d0, a["epoch"], d1, tag], fx.Int32) + + def ll_ok(a, q): + return (fx.Int32(q[1]) == a["epoch"]) & ( + fx.Int32(q[3]).shrui(i32(8)) == (a["epoch"] & i32(0xFFFFFF)) + ) + + def ll_vals(q): + sc = e8_scale(q[3]) + return fp8x4_unpack(fx.Int32(q[0]), sc) + fp8x4_unpack(fx.Int32(q[2]), sc) + + def ll_pending(a, lane, pkts): + bad = i32(0) + for q, live in pkts: + bad = fx.max( + bad, (live & (ll_ok(a, q) == fx.Boolean(False))).select(i32(1), i32(0)) + ) + return wave_red(bad, lane, fx.max) + + def route_load(rs, region, ttot, ridx, col, take=None): + if route_fp8: + doff = region + ridx * i32(H) + col + soff = region + ttot * i32(TOPK * H) + ridx * i32(H // 32) + col // i32(32) + if take is not None: + end = region + route_region_bytes(ttot) * i32(NPC) + doff, soff = take.select(doff, end), take.select(soff, end) + return (bld(rs, doff, 0, V2I, AUX_SC1), bld(rs, soff, 0, T.i8, AUX_SC1)) + off = region + (ridx * i32(H) + col) * i32(2) + if take is not None: + off = take.select(off, region + route_region_bytes(ttot) * i32(NPC)) + return bld(rs, off, 0, V4I, AUX_SC1) + + def route_decode(ld): + return fp8x8_decode(ld) if route_fp8 else bf16x8_to_f32(ld) + + def a_frag(abuf, row, k, q4): + if const_expr(a8): + lo = (i32(k * 8) + q4) ^ (row & i32(7)) + hi = (i32(k * 8 + 4) + q4) ^ (row & i32(7)) + return cat8( + lds_ld(abuf, row * i32(ACB) + lo * i32(16), V4I, 16), + lds_ld(abuf, row * i32(ACB) + hi * i32(16), V4I, 16), + ) + col = (i32(k * 4) + q4) ^ (row & i32(7)) + return lds_ld(abuf, row * i32(ACB) + col * i32(16), V4I, 16) + + def act_batch(gs, us): + return _activation_mul_batch( + gs, + us, + act=act, + situ_beta=situ_beta, + situ_linear_beta=situ_linear_beta, + swiglu_limit=swiglu_limit, + ) + + @flyc.jit + def cbar(L, tid): + wait_lgkm0() + if (tid % i32(64)) == i32(0): + g = lds_ld_acq(L, L_CTL + C_BARGEN * 4) + old = lds_atomic_add(L, L_CTL + C_BARCNT * 4, 1, ACQ_REL) + if old == i32(NW - 1): + lds_st(L, L_CTL + C_BARCNT * 4, i32(0)) + lds_st_rel(L, L_CTL + C_BARGEN * 4, g + i32(1)) + else: + cur = lds_ld_acq(L, L_CTL + C_BARGEN * 4) + while cur == g: + rocdl.s_sleep(0) + cur = lds_ld_acq(L, L_CTL + C_BARGEN * 4) + rocdl.sched_barrier(0) + + def report(a, code): + g_or_agent(ctrl_at(a, CTRL_ERR), code) + + @flyc.jit + def _report_if(a, bad, code): + if bad: + report(a, code) + + @flyc.jit + def poll_zero(fn, a=None, lane=None, err=0): + pend = fn() + t0 = now() + while (pend != i32(0)) & alive(t0): + rocdl.s_sleep(1) + pend = fn() + if const_expr(err != 0): + _report_if(a, (pend != i32(0)) & (lane == i32(0)), err) + + @flyc.jit + def spin_sys_ge(addr, target, a=None): + cur = poll_sys_ge(addr, target) + if const_expr(a is not None): + _report_if(a, before(cur, target), ERR_FLAG) + + def masked(e): + return fx.Int32(e).bitcast(fx.Uint32) >= fx.Uint32(E) + + @flyc.jit + def gather_routes_chunk(L, tid, a, expert, cc, ce): + key = expert + i32(1) + (cc << i32(12)) + (ce << i32(20)) + if lds_ld_i32(L, L_CTL + C_GEXP * 4) != key: + if ce == i32(1): + _gather_scan(L, tid, a, expert, key) + else: + inv = fx.Float32(1.0) / ce.to(fx.Float32) + _gather_scan(L, tid, a, expert, key, (cc, ce, inv)) + return fx.min(lds_ld_i32(L, L_CTL + C_CNT * 4), i32(TMAX)) + + def _in_chunk(idx, chunk): + if chunk is None: + return fx.Boolean(True) + cc, ce, inv = chunk + t = idx // i32(TOPK) + q = ((t.to(fx.Float32) + fx.Float32(0.5)) * inv).to(fx.Int32) + return (t - q * ce) == cc + + @flyc.jit + def _gather_scan(L, tid, a, expert, key, chunk=None): + ids_addr, tw_addr, ttot = a["ids"], a["tw"], a["ttot"] + cbar(L, tid) + if tid == i32(0): + lds_st(L, L_CTL + C_CNT * 4, i32(0)) + cbar(L, tid) + n = ttot * i32(TOPK) + rid = rsrc(ids_addr, n * i32(16 if MLL else 4)) + if const_expr(MLL): + for i_ in range(tid, n, i32(NT)): + i = i32(i_) + pk = fx.Vector(bld(rid, i * i32(16), 0, V4I, 0)) + hit = (fx.Int32(pk[0]) == expert) & _in_chunk(i, chunk) + _gather_one(L, i, hit, fx.Int32(pk[2])) + else: + n4 = n // i32(4) + rtw = rsrc(tw_addr, n * i32(4)) + for g0 in range_constexpr(0, SCAN_IT, SCAN_G): + its = list(range(g0, min(g0 + SCAN_G, SCAN_IT))) + vs, ws = [], [] + for it in its: + q = fx.min(tid + i32(it * NT), n4 - i32(1)) + vs.append(fx.Vector(bld(rid, q * i32(16), 0, V4I, 0))) + ws.append(fx.Vector(bld(rtw, q * i32(16), 0, V4I, 0))) + hits, run = [], i32(0) + for x, it in enumerate(its): + q = tid + i32(it * NT) + for j in range_constexpr(4): + idx = q * i32(4) + i32(j) + hit = ( + (q < n4) + & (fx.Int32(vs[x][j]) == expert) + & _in_chunk(idx, chunk) + ) + pos, cnt = wave_rank(hit) + hits.append((hit, run + pos, idx, ws[x][j])) + run = run + cnt + base = _wave_claim(L, tid, run) + for hit, pos, idx, wv in hits: + slot = base + pos + if hit & (slot < i32(TMAX)): + lds_st(L, L_RIX + slot * i32(4), idx) + lds_st(L, L_WT + slot * i32(4), fx.Int32(wv)) + for idx_ in range(n4 * i32(4) + tid, n, i32(NT)): + idx = i32(idx_) + e = g_ld_i32(fx.Int64(ids_addr) + fx.Int64(idx) * 4) + wv = g_ld_i32(fx.Int64(tw_addr) + fx.Int64(idx) * 4) + _gather_one(L, idx, (e == expert) & _in_chunk(idx, chunk), wv) + cbar(L, tid) + if tid == i32(0): + lds_st(L, L_CTL + C_GEXP * 4, key) + cbar(L, tid) + + @flyc.jit + def zero_masked(L, lane, a, w0, nw): + n = a["ttot"] * i32(TOPK) + nblk = i32(gpu.grid_dim.x) + per = ceildiv(n, nblk) + lo = i32(gpu.block_id("x")) * per + hi = fx.min(lo + per, n) + rid = rsrc(a["ids"], n * i32(16 if MLL else 4)) + if const_expr(MLL): + spin0(L, lane, L_CTL + C_RLAND * 4, i32(1)) + for b_ in range(lo + i32(w0 * 64), hi, i32(nw * 64)): + i = i32(b_) + lane + off = fx.min(i, n - i32(1)) * i32(16 if MLL else 4) + e = fx.Int32(bld(rid, off, 0, T.i32, 0)) + bal = fx.Int64(rocdl.ballot(T.i64, (i < hi) & masked(e))) + if bal != 0: + for j_ in range(i32(0), i32(64), i32(1)): + j = i32(j_) + if ((bal >> fx.Int64(j)) & 1) != 0: + _zero_route(a, lane, i32(b_) + j) + wait_vm(0) + + @flyc.jit + def _zero_route(a, lane, ridx): + rb = route_region_bytes(a["ttot"]) + z = fx.Vector.from_elements([i32(0)] * 4, fx.Int32) + for p in range_constexpr(NPC): + rs = rsrc(a["routes"], rb) if p == 0 else rsrc(a["proutes"], i32(p) * rb) + base = i32(0) if p == 0 else i32(p - 1) * rb + for q_ in range(lane, i32(H // 8), i32(64)): + q = i32(q_) + if const_expr(DLL): + v = ll_pkt(a, i32(0), i32(0), i32(0)) + bst(v, rs, base + (ridx * i32(H) + q * i32(8)) * i32(2), 0, AUX_SC1) + elif const_expr(route_fp8): + if q < i32(H // 16): + bst(z, rs, base + ridx * i32(H) + q * i32(16), 0, AUX_SC1) + if q < i32(H // 512): + soff = a["ttot"] * i32(TOPK * H) + ridx * i32(H // 32) + bst(z, rs, base + soff + q * i32(16), 0, AUX_SC1) + else: + bst(z, rs, base + (ridx * i32(H) + q * i32(8)) * i32(2), 0, AUX_SC1) + + @flyc.jit + def _wave_claim(L, tid, n): + got = i32(0) + if ((tid % i32(64)) == i32(0)) & (n > i32(0)): + got = lds_atomic_add(L, L_CTL + C_CNT * 4, n) + return uni(got) + + @flyc.jit + def _gather_one(L, idx, hit, wv): + if hit: + slot = lds_atomic_add(L, L_CTL + C_CNT * 4, 1) + if slot < i32(TMAX): + lds_st(L, L_RIX + slot * i32(4), idx) + lds_st(L, L_WT + slot * i32(4), wv) + + def act_quant_store(L, lane, w, accg, accu, nb, mte=None): + q4 = lane // i32(16) + for m in range_constexpr(MT if mte is None else mte): + row = i32(m * 16) + lane % i32(16) + gs, us = [], [] + for t in range_constexpr(2): + ga, ua = fx.Vector(accg[m * 2 + t]), fx.Vector(accu[m * 2 + t]) + for v in range_constexpr(4): + gs.append(fx.Float32(ga[v])) + us.append(fx.Float32(ua[v])) + xs = [ + fx.Float32(x).to(fx.BFloat16).to(fx.Float32) for x in act_batch(gs, us) + ] + am = amax(xs) + am = am.maximumf(am.shuffle_xor(i32(16), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(32), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=448.0 if a8 else 6.0) + for t in range_constexpr(2): + col = nb * i32(128) + w * i32(32) + i32(t * 16) + q4 * i32(4) + if const_expr(a8): + pk = fp8x4_pack(xs[t * 4 : t * 4 + 4], qs) + lds_st(L, L_INTER + row * i32(SI_STRIDE) + col, pk, align=4) + else: + pk = fp4_pack(xs[t * 4 : t * 4 + 4], qs) + lds_st( + L, + L_INTER + row * i32(SI_STRIDE) + col // i32(2), + fx.Int16(pk & i32(0xFFFF)), + align=2, + ) + _q0_store(L, q4, row, nb, w, e8) + + @flyc.jit + def _q0_store(L, q4, row, nb, w, e8): + if q4 == i32(0): + lds_st( + L, L_INTERS + row * i32(I // 32) + nb * i32(4) + w, fx.Int8(e8), align=1 + ) + + def claim_addr(a, bank_off): + bank = (a["epoch"] + i32(bank_off)) & i32(1) + return ctrl_at(a, i32(CTRL_CLM) + bank * i32(LRDY_STRIDE)) + + @flyc.jit + def col_claim(L, tid, a): + cbar(L, tid) + if tid == i32(0): + v = g_add_agent(claim_addr(a, 0), 1) + for w in range_constexpr(NW): + lds_st(L, L_CTL + (C_CLAIM * 4) + i32(w * 4), v) + cbar(L, tid) + + def push_units(ttot): + nblk = i32(gpu.grid_dim.x) + return fx.min(i32(2) * nblk, ttot) + + def push_first_unit(ns, cidx): + nblk = i32(gpu.grid_dim.x) + rot = (cidx * ns) % nblk + return (i32(gpu.block_id("x")) - rot + nblk) % nblk + + def token_dst(a, t): + owner = t // a["m"] + r_dst = rsrc(peer_sel(a, owner) + fx.Int64(a["off_part"])) + return r_dst, a["rank"] * a["mmax"] + (t - owner * a["m"]) + + def part_offs(a, prow, col): + soff = a["tp"] * a["mmax"] * i32(H) + prow * i32(H // 32) + col // i32(32) + return prow * i32(H) + col, soff + + @flyc.jit + def _push_store(a, r_dst, prow, c0, v, acc, ok): + col = c0 + v * i32(8) + if const_expr(comm_bf16): + if ok: + bst(pack_bf16x8(acc), r_dst, (prow * i32(H) + col) * i32(2), 0, AUX_SYS) + else: + _push_store_fp8(a, r_dst, prow, col, v, acc, ok) + + @flyc.jit + def _push_store_fp8(a, r_dst, prow, col, v, acc, ok): + d0, d1, e8i = mxfp8x8(acc) + if const_expr(ll_rs): + pkt = ll_pkt(a, d0, d1, e8i) + if ok: + bst(pkt, r_dst, (prow * i32(H) + col) * i32(2), 0, AUX_SYS) + else: + doff, soff = part_offs(a, prow, col) + sc = scales4(e8i) + if ok: + bst( + fx.Vector.from_elements([d0, d1], fx.Int32), r_dst, doff, 0, AUX_SYS + ) + if (v & i32(15)) == i32(0): + bst(sc, r_dst, soff, 0, AUX_SYS) + + def sc_addr(a, cidx, slot): + return ctrl_at( + a, i32(CTRL_SC) + (cidx * i32(SC_LINES) + slot) * i32(LRDY_STRIDE) + ) + + @flyc.jit + def _push_done_all(a, epoch, cidx, target, n): + cnt_addr = sc_addr(a, cidx, i32(SC_PUSH)) + if g_add_agent(cnt_addr, n) + n == target: + g_st_sys(cnt_addr, i32(0)) + fo = fx.Int64((i32(FLAG_RDY) + cidx * i32(MAX_TP) + a["rank"]) * i32(4)) + for p in range_constexpr(MAX_TP): + if i32(p) < a["tp"]: + g_st_sys( + fx.Int64(a["peer"][p]) + fx.Int64(a["off_flag"]) + fo, epoch + ) + + def fin_split(a): + m = fx.max(a["m"], i32(1)) + nblk = i32(gpu.grid_dim.x) + return (m < nblk).select(fx.min(i32(NV), nblk // m), i32(1)) + + def fin_key(a): + bid = i32(gpu.block_id("x")) + return (fin_split(a) > i32(1)).select(bid % fx.max(a["m"], i32(1)), bid) + + def fin_owned(a): + bid = i32(gpu.block_id("x")) + m = fx.max(a["m"], i32(1)) + S = fin_split(a) + sidx = bid // m + own = i32(0) + for c in range_constexpr(NV): + own = own | ((i32(c) % S) == sidx).select(i32(1 << c), i32(0)) + return (bid < m * S).select(own, i32(0)) + + def recv_pkts(a, r_recv, row, rstride, c0, vc): + out = [] + for p in range_constexpr(MAX_TP): + pc = fx.min(i32(p), a["tp"] - i32(1)) + off = ((pc * rstride + row) * i32(H) + c0 + vc * i32(8)) * i32(2) + out.append(fx.Vector(bld(r_recv, off, 0, V4I, AUX_SYS))) + return out + + @flyc.jit + def final_chunk(L, tid, a, cidx): + lane = tid % i32(64) + nblk = gpu.grid_dim.x + c0 = cidx * i32(CW) + r_recv = own_rs(a, "off_part") + step = (fin_split(a) > i32(1)).select(a["m"], i32(nblk)) + for row_ in range(fin_key(a), a["m"], step): + row = i32(row_) + _final_row(L, tid, a, r_recv, row, c0) + if const_expr(ar): # noqa: SIM102 + if lane == i32(0): + lds_atomic_or(L, L_CTL + C_YAGM * 4, i32(1) << cidx) + + @flyc.jit + def _final_row(L, tid, a, r_recv, row, c0): + lane = tid % i32(64) + for j in range_constexpr(VPL): + v = lane + i32(j * 64) + vc = fx.min(v, i32(CW // 8 - 1)) + if const_expr(ll_rs): + _final_ll(tid, a, r_recv, row, c0, v, vc) + continue + acc = [fx.Float32(0.0)] * 8 + if const_expr(comm_bf16): + pk = recv_pkts(a, r_recv, row, a["mmax"], c0, vc) + for p in range_constexpr(MAX_TP): + acc = sum_live(acc, bf16x8_to_f32(pk[p]), i32(p) < a["tp"]) + else: + lds_ = [] + for p in range_constexpr(MAX_TP): + pc = fx.min(i32(p), a["tp"] - i32(1)) + doff, soff = part_offs(a, pc * a["mmax"] + row, c0 + vc * i32(8)) + lds_.append( + ( + bld(r_recv, doff, 0, V2I, AUX_SYS), + bld(r_recv, soff, 0, T.i8, AUX_SYS), + ) + ) + for p in range_constexpr(MAX_TP): + acc = sum_live(acc, fp8x8_decode(lds_[p]), i32(p) < a["tp"]) + _y_store(a, row, c0, v, acc) + + @flyc.jit + def _final_ll(tid, a, r_recv, row, c0, v, vc): + lane = tid % i32(64) + pk = recv_pkts(a, r_recv, row, a["mmax"], c0, vc) + live = [(i32(p) < a["tp"]) & (v < i32(CW // 8)) for p in range(MAX_TP)] + pend = ll_pending(a, lane, zip(pk, live)) + t0 = now() + while (pend != i32(0)) & alive(t0): + rocdl.s_sleep(1) + pk = recv_pkts(a, r_recv, row, a["mmax"], c0, vc) + pend = ll_pending(a, lane, zip(pk, live)) + _report_if(a, (pend != i32(0)) & (lane == i32(0)), ERR_COMM) + acc = [fx.Float32(0.0)] * 8 + for p in range_constexpr(MAX_TP): + acc = sum_live(acc, ll_vals(pk[p]), i32(p) < a["tp"]) + _y_store(a, row, c0, v, acc) + + @flyc.jit + def _y_store(a, row, c0, v, acc): + if const_expr(AG8): + _y_store_ag8(a, row, c0, v, acc) + else: + _y_store_bf16(a, row, c0, v, pack_bf16x8(acc)) + + @flyc.jit + def _y_store_bf16(a, row, c0, v, o): + if v < i32(CW // 8): + if const_expr(ar): + off = ((a["rank"] * a["m"] + row) * i32(H) + c0 + v * i32(8)) * i32(2) + for p in range_constexpr(TPC): + bst(o, peer_rs(a, p, "off_yall"), off, 0, AUX_SYS) + else: + bst( + o, + rsrc(a["y"]), + (row * i32(H) + c0 + v * i32(8)) * i32(2), + 0, + AUX_SYS if TN else 0, + ) + + @flyc.jit + def _y_store_ag8(a, row, c0, v, acc): + col = c0 + v * i32(8) + grow = a["rank"] * a["m"] + row + d0, d1, e8i = mxfp8x8(acc) + sc = scales4(e8i) + f = fp8x4_unpack(d0, e8_scale(e8i)) + fp8x4_unpack(d1, e8_scale(e8i)) + if v < i32(CW // 8): + bst(pack_bf16x8(f), rsrc(a["y"]), (grow * i32(H) + col) * i32(2), 0, 0) + doff, soff = part_offs(a, grow, col) + dv = fx.Vector.from_elements([d0, d1], fx.Int32) + for p in range_constexpr(TPC): + if i32(p) != a["rank"]: + rd = peer_rs(a, p, "off_yall") + bst(dv, rd, doff, 0, AUX_SYS) + if (v & i32(15)) == i32(0): + bst(sc, rd, soff, 0, AUX_SYS) + + @flyc.jit + def ag8_convert(tid, a): + bid = i32(gpu.block_id("x")) + if bid < a["m"]: + step = (fin_split(a) > i32(1)).select(a["m"], i32(gpu.grid_dim.x)) + ry = own_rs(a, "off_yall") + yo = rsrc(a["y"]) + nq = i32(H // 8) + for row_ in range(bid, a["m"], step): + row = i32(row_) + for q_ in range(tid, i32(TPC * (H // 8)), i32(NTT)): + q = i32(q_) + p = q // nq + col = (q - p * nq) * i32(8) + if p != a["rank"]: + grow = p * a["m"] + row + doff, soff = part_offs(a, grow, col) + ld = ( + bld(ry, doff, 0, V2I, AUX_SYS), + bld(ry, soff, 0, T.i8, AUX_SYS), + ) + bst( + pack_bf16x8(fp8x8_decode(ld)), + yo, + (grow * i32(H) + col) * i32(2), + 0, + 0, + ) + + @flyc.jit + def yag_flush(L, tid, a): + if tid < i32(64): + mask = lds_ld_i32(L, L_CTL + C_YAGM * 4) + for it in range_constexpr(ceildiv(NV * TPC, 64)): + e = i32(it * 64) + tid + v = e // i32(TPC) + p = e - v * i32(TPC) + vc = fx.min(v, i32(NV - 1)) + idx = ( + i32(FLAG_YAG) + + (vc * i32(MAX_TP) + a["rank"]) * i32(NCTA_MAX) + + fin_key(a) + ) + ok = (v < i32(NV)) & (((mask >> vc) & i32(1)) == i32(1)) + _yag_post(a, p, idx, ok) + + @flyc.jit + def _yag_post(a, p, idx, ok): + if ok: + fo = fx.Int64(a["off_flag"]) + fx.Int64(idx * i32(4)) + g_st_sys(peer_sel(a, p) + fo, a["epoch"]) + + def _yag_pending(a, lane, epoch): + rf = own_rs(a, "off_flag") + bid = i32(gpu.block_id("x")) + pend = i32(0) + for it in range_constexpr(ceildiv(NV * MAX_TP, 64)): + e = i32(it * 64) + lane + cidx = e // i32(MAX_TP) + src = e - cidx * i32(MAX_TP) + ok = (cidx < i32(NV)) & (src < a["tp"]) + idx = ( + i32(FLAG_YAG) + + (fx.min(cidx, i32(NV - 1)) * i32(MAX_TP) + src) * i32(NCTA_MAX) + + bid + ) + f = fx.Int32(bld(rf, idx * i32(4), 0, T.i32, AUX_SYS)) + pend = fx.max(pend, (ok & before(f, epoch)).select(i32(1), i32(0))) + return wave_red(pend, lane, fx.max) + + @flyc.jit + def yag_wait(tid, a, epoch): + if (tid < i32(64)) & (i32(gpu.block_id("x")) < a["m"]): + lane = tid % i32(64) + poll_zero(lambda: _yag_pending(a, lane, epoch), a, lane, ERR_YAG) + + def comm_idle(a): + bid = i32(gpu.block_id("x")) + ns = push_units(a["ttot"]) + some = (ns * i32(NV) >= i32(gpu.grid_dim.x)) | (bid < ns * i32(NV)) + return (some.select(i32(0), i32(1)) == i32(1)) & (fin_owned(a) == i32(0)) + + VMASK = (1 << NV) - 1 + + def _lowest(v): + return _ctpop((v & (i32(0) - v)) - i32(1)) + + @flyc.jit + def _claim_bit(L, rdy_off, bits_off, cnt_off, w): + res = i32(-1) + n = lds_ld_acq(L, L_CTL + cnt_off * 4) + avail = ( + lds_ld_acq(L, L_CTL + rdy_off * 4) + & (lds_ld_acq(L, L_CTL + bits_off * 4) ^ i32(-1)) + & i32(VMASK) + ) + if (n < i32(NV)) & (avail != i32(0)): + c = _lowest(avail) + old = lds_atomic_or(L, L_CTL + bits_off * 4, i32(1) << c) + if ((old >> c) & i32(1)) == i32(0): + lds_atomic_add(L, L_CTL + cnt_off * 4, i32(1)) + res = c + lds_st(L, L_CTL + (C_MBOX * 4) + w * i32(4), res) + + @flyc.jit + def claim_push(L, lane, w): + if lane == i32(0): + _claim_bit(L, C_PRDY, C_PBITS, C_LRED, w) + rocdl.sched_barrier(0) + return uni(lds_ld_acq(L, L_CTL + (C_MBOX * 4) + w * i32(4))) + + @flyc.jit + def claim_final(L, lane, w): + if lane == i32(0): + _claim_bit(L, C_FRDY, C_FBITS, C_PULL, w) + rocdl.sched_barrier(0) + return uni(lds_ld_acq(L, L_CTL + (C_MBOX * 4) + w * i32(4))) + + @flyc.jit + def poll_ready(L, lane, a, epoch): + c = fx.min(lane, i32(NV - 1)) + live = lane < i32(NV) + v = g_ld_sys(lrdy_at(a, c)) + pm = i32( + fx.Int64( + rocdl.ballot(T.i64, live & (before(v, epoch) == fx.Boolean(False))) + ) + & fx.Int64(VMASK) + ) + fm = i32( + fx.Int64(rocdl.ballot(T.i64, live & _final_ready(a, c, epoch))) + & fx.Int64(VMASK) + ) + pm = uni(pm) + fm = uni(fm) + old_p = lds_ld_acq(L, L_CTL + C_PRDY * 4) + old_f = lds_ld_acq(L, L_CTL + C_FRDY * 4) + if lane == i32(0): + lds_atomic_or(L, L_CTL + C_PRDY * 4, pm) + lds_atomic_or(L, L_CTL + C_FRDY * 4, fm) + return (((pm & (old_p ^ i32(-1))) | (fm & (old_f ^ i32(-1)))) != i32(0)).select( + i32(1), i32(0) + ) + + def _comm_pending(L): + full = i32(VMASK) + push = (lds_ld_acq(L, L_CTL + C_LRED * 4) < i32(NV)) & ( + (lds_ld_acq(L, L_CTL + C_PRDY * 4) & full) != full + ) + fin = (lds_ld_acq(L, L_CTL + C_PULL * 4) < i32(NV)) & ( + ( + (lds_ld_acq(L, L_CTL + C_FRDY * 4) | lds_ld_acq(L, L_CTL + C_FBITS * 4)) + & full + ) + != full + ) + return (lds_ld_acq(L, L_CTL + C_NSIG * 4) < i32(NCK)) | push | fin + + @flyc.jit + def poll_loop(L, tid, a, epoch): + t0 = now() + while _comm_pending(L) & alive(t0): + p1 = comm_signal(L, tid, a, epoch) + p2 = poll_ready(L, tid % i32(64), a, epoch) + if (p1 | p2) == i32(0): + rocdl.s_sleep(POLL_SLEEP) + _report_if( + a, + (now() - t0 >= fx.Int64(DEADLINE)) & ((tid % i32(64)) == i32(0)), + ERR_COMM, + ) + + def _final_ready(a, c, epoch): + if const_expr(ll_rs): + r_recv = own_rs(a, "off_part") + r1 = fx.Boolean(True) + for p in range_constexpr(MAX_TP): + pc = fx.min(i32(p), a["tp"] - i32(1)) + off = ( + (pc * a["mmax"] + fin_key(a)) * i32(H) + c * i32(CW) + i32(CW - 8) + ) * i32(2) + t = fx.Int32(bld(r_recv, off + i32(4), 0, T.i32, AUX_SYS)) + r1 = r1 & ((i32(p) >= a["tp"]) | (t == epoch)) + return r1 + rf = own_rs(a, "off_flag") + fo = (i32(FLAG_RDY) + c * i32(MAX_TP)) * i32(4) + fl = fx.Vector(bld(rf, fo, 0, V4I, AUX_SYS)) + fh = fx.Vector(bld(rf, fo + i32(16), 0, V4I, AUX_SYS)) + r1 = fx.Boolean(True) + for p in range_constexpr(MAX_TP): + f = fx.Int32((fl if p < 4 else fh)[p % 4]) + r1 = r1 & ((i32(p) >= a["tp"]) | (before(f, epoch) == fx.Boolean(False))) + return r1 + + @flyc.jit + def push_chunk_dyn(L, tid, a, epoch, cidx): + lane = tid % i32(64) + ttot = a["ttot"] + nblk = gpu.grid_dim.x + ns = push_units(ttot) + u0 = push_first_unit(ns, cidx) + c0 = cidx * i32(CW) + routes_bytes = route_region_bytes(ttot) + r_routes = rsrc(a["routes"], routes_bytes) + r_pr = rsrc(a["proutes"], i32(NPC - 1) * routes_bytes) + P = uni(lds_ld_i32(L, L_CTL + C_DYNP * 4)) + n = fx.max(ceildiv(ns - u0, i32(nblk)), i32(0)) + for u_ in range(u0, ns, i32(nblk)): + u = i32(u_) + for t_ in range(u, ttot, ns): + t = i32(t_) + for j in range_constexpr(VPL): + v = lane + i32(j * 64) + vc = fx.min(v, i32(CW // 8 - 1)) + lds_ = [] + for k in range_constexpr(TOPK): + ridx = t * i32(TOPK) + i32(k) + lds_.append( + route_load(r_routes, i32(0), ttot, ridx, c0 + vc * i32(8)) + ) + for p in range_constexpr(1, NPC): + lds_.append( + route_load( + r_pr, + i32(p - 1) * routes_bytes, + ttot, + ridx, + c0 + vc * i32(8), + i32(p) < P, + ) + ) + acc = [fx.Float32(0.0)] * 8 + for ld in lds_: + acc = [x + y for x, y in zip(acc, route_decode(ld))] + r_dst, prow = token_dst(a, t) + _push_store(a, r_dst, prow, c0, v, acc, v < i32(CW // 8)) + if const_expr(not ll_rs): + wait_vm(0) + if (lane == i32(0)) & (n > i32(0)): + _push_done_all(a, epoch, cidx, ns, n) + + @flyc.jit + def comm_work(L, tid, a, epoch): + lane = tid % i32(64) + w = tid // i32(64) + cp = claim_final(L, lane, w) + if cp >= i32(0): + final_chunk(L, tid, a, cp) + cr = claim_push(L, lane, w) + if cr >= i32(0): + push_chunk_dyn(L, tid, a, epoch, cr) + return ((cr >= i32(0)) | (cp >= i32(0))).select(i32(1), i32(0)) + + @flyc.jit + def comm_signal(L, tid, a, epoch): + lane = tid % i32(64) + w = tid // i32(64) + mn = i32(NCK) + for wv in range_constexpr(NW): + mn = fx.min(mn, lds_ld_acq(L, L_CTL + C_DONE * 4 + wv * 4)) + start = uni(lds_ld_acq(L, L_CTL + C_NSIG * 4)) + end = uni(fx.min(mn + i32(1), i32(NCK))) + _signal_claim(L, lane, w, start, end) + rocdl.sched_barrier(0) + won = uni(lds_ld_acq(L, L_CTL + (C_MBOX * 4) + w * i32(4))) + if won == i32(1): # noqa: SIM102 + if start + lane < end: + _signal_one(a, epoch, start + lane) + rocdl.sched_barrier(0) + return (end > start).select(i32(1), i32(0)) + + @flyc.jit + def _signal_claim(L, lane, w, start, end): + if lane == i32(0): + won = i32(0) + if end > start: + got = lds_cas(L, L_CTL + C_NSIG * 4, start, end) + won = (got == start).select(i32(1), i32(0)) + lds_st(L, L_CTL + (C_MBOX * 4) + w * i32(4), won) + + def _signal_one(a, epoch, cidx): + nblk = i32(gpu.grid_dim.x) + x = i32(gpu.block_id("x")) % i32(N_XCD) + _signal_part(a, epoch, cidx, x, ceildiv(nblk - x, i32(N_XCD))) + + @flyc.jit + def _signal_part(a, epoch, cidx, slot, target): + xa = sc_addr(a, cidx, slot) + if g_add_agent(xa, 1) + i32(1) == target: + g_st_sys(xa, i32(0)) + _signal_count(a, epoch, cidx) + + @flyc.jit + def _signal_count(a, epoch, cidx): + cnt_addr = sc_addr(a, cidx, i32(SC_ALL)) + parts = fx.min(i32(gpu.grid_dim.x), i32(N_XCD)) + if g_add_agent(cnt_addr, 1) + i32(1) == parts: + g_st_sys(cnt_addr, i32(0)) + g_st_sys(lrdy_at(a, cidx), epoch) + + @flyc.jit + def signal_loop(L, tid, a, epoch): + t0 = now() + while (lds_ld_acq(L, L_CTL + C_NSIG * 4) < i32(NCK)) & alive(t0): + if comm_signal(L, tid, a, epoch) == i32(0): + rocdl.s_sleep(1) + + @flyc.jit + def comm_wave(L, tid, a, epoch): + t0 = now() + while ( + (lds_ld_acq(L, L_CTL + C_NSIG * 4) < i32(NCK)) + | (lds_ld_acq(L, L_CTL + C_LRED * 4) < i32(NV)) + | (lds_ld_acq(L, L_CTL + C_PULL * 4) < i32(NV)) + ) & alive(t0): + p1 = comm_signal(L, tid, a, epoch) + if const_expr(not ARLL): + p1 = p1 | poll_ready(L, tid % i32(64), a, epoch) + p2 = comm_work(L, tid, a, epoch) + if (p1 | p2) == i32(0): + rocdl.s_sleep(POLL_SLEEP) + _report_if( + a, + (now() - t0 >= fx.Int64(DEADLINE)) & ((tid % i32(64)) == i32(0)), + ERR_COMM, + ) + + @flyc.jit + def comm_help(L, tid, a, epoch): + t0 = now() + while ( + (lds_ld_acq(L, L_CTL + C_LRED * 4) < i32(NV)) + | (lds_ld_acq(L, L_CTL + C_PULL * 4) < i32(NV)) + ) & alive(t0): + if comm_work(L, tid, a, epoch) == i32(0): + rocdl.s_sleep(POLL_SLEEP) + + AG_SROW = XB + AG_SCB = agr * AG_SROW + assert AG_SCB + agr * (H // 32) <= L_INTERS - L_INTER + RG * (I // 32) + NCHA = H // 256 + AG_WIN = max(1, 64 // TPC) + + def ag_split(a): + m = fx.max(a["m"], i32(1)) + return fx.min(fx.max(i32(gpu.grid_dim.x) // m, i32(1)), i32(NCHA)) + + def ag_rows(a): + bid = i32(gpu.block_id("x")) + nblk = i32(gpu.grid_dim.x) + S = ag_split(a) + nr = fx.min(fx.max(ceildiv(a["m"] - bid, nblk), i32(0)), i32(agr)) + m = fx.max(a["m"], i32(1)) + s = bid // m + split = S > i32(1) + live = s < S + row0 = split.select(bid - s * m, bid) + nrows = split.select(live.select(i32(1), i32(0)), nr) + qb = split.select(s, i32(0)) + qs = split.select(S, i32(1)) + cnt = split.select(live.select(ceildiv(i32(NCHA) - s, S), i32(0)), i32(NCHA)) + return row0, nblk, nrows, qb, qs, cnt + + def _row32(rs, row, g, aux): + f = [] + for v in range_constexpr(4): + f += bf16x8_to_f32( + bld( + rs, + ((row * i32(H)) + g * i32(32) + i32(v * 8)) * i32(2), + 0, + V4I, + aux, + ) + ) + return f + + def ag_row_vals(a, rxl, i, g): + return _row32(rxl, (a["rank"] * a["m"] if ar else i32(0)) + i, g, 0) + + @flyc.jit + def quant_chunks(L, t0, stride, a, k_lo, k_hi): + row0, stride_r, nrows, qb, qstep, cnt = ag_rows(a) + k_hi = fx.min(k_hi, cnt) + nk = fx.max(k_hi - k_lo, i32(0)) + rxl = rsrc(a["x"], (a["ttot"] if ar else a["m"]) * i32(H * 2)) + for q_ in range(t0, nrows * nk * i32(8), i32(stride)): + q = i32(q_) + r = q // (nk * i32(8)) + e = q - r * nk * i32(8) + g = (qb + (k_lo + e // i32(8)) * qstep) * i32(8) + e % i32(8) + f = ag_row_vals(a, rxl, row0 + r * stride_r, g) + e8, qs = _e8m0_from_amax(amax(f), max_norm=448.0 if a8 else 6.0) + if const_expr(a8): + words = [fp8x4_pack(f[dw * 4 : dw * 4 + 4], qs) for dw in range(8)] + else: + words = [fp4_pack(f[dw * 8 : dw * 8 + 8], qs) for dw in range(4)] + lds_st( + L, + L_INTER + r * i32(AG_SROW) + g * i32(4 * len(words)), + fx.Vector.from_elements(words, fx.Int32), + 16, + ) + lds_st( + L, L_INTER + i32(AG_SCB) + r * i32(H // 32) + g, fx.Int8(e8), align=1 + ) + + @flyc.jit + def ag_send(L, lane, a): + row0, stride, nrows, qb, qs, cnt = ag_rows(a) + rank = a["rank"] + if const_expr(not ar and not MLL): + mp = (ag_split(a) > i32(1)).select( + i32(0), _meta_pending(a, lane, a["epoch"], own=True) + ) + t0 = now() + while (mp != i32(0)) & alive(t0): + rocdl.s_sleep(1) + mp = _meta_pending(a, lane, a["epoch"], own=True) + r = lane // i32(XLPR) + j = lane - r * i32(XLPR) + rok = r < nrows + i = row0 + fx.min(r, fx.max(nrows - i32(1), i32(0))) * stride + gslot = rank * a["m"] + i + x_bytes = a["ttot"] * i32(XB) + xs_bytes = a["ttot"] * i32(H // 32) + rx = [peer_rs(a, p, "off_x", x_bytes) for p in range(TPC)] + rs = [peer_rs(a, p, "off_xs", xs_bytes) for p in range(TPC)] + for k in range_constexpr(NCHA): + kl = i32(k) < cnt + q = fx.min(qb + i32(k) * qs, i32(NCHA - 1)) + if kl: + ag_chunk(L, lane, a, q, r, j, rok, gslot, rx, rs, x_bytes, xs_bytes) + if const_expr(k == NCHA - 1): + wait_lgkm0() + if lane == i32(0): + lds_atomic_add(L, L_CTL + C_AGFREE * 4, 1, REL) + if const_expr(k >= 1): # noqa: SIM102 + if i32(k - 1) < cnt: + if kl: + wait_vm(2 * TPC) + else: + wait_vm(0) + _ag_bump(a, lane, qb + i32(k - 1) * qs) + rocdl.sched_barrier(0) + wait_vm(0) + if i32(NCHA - 1) < cnt: + _ag_bump(a, lane, qb + i32(NCHA - 1) * qs) + + @flyc.jit + def ag_chunk(L, lane, a, q, r, j, rok, gslot, rx, rs, x_bytes, xs_bytes): + rr = fx.min(r, i32(agr - 1)) + cb = q * i32(XLPR * 16) + j * i32(16) + dv = lds_ld(L, L_INTER + rr * i32(AG_SROW) + cb, V4I, 16) + sv = lds_ld(L, L_INTER + i32(AG_SCB) + rr * i32(H // 32) + q * i32(8), V2I, 8) + doff = rok.select(gslot * i32(XB) + cb, x_bytes) + soff = (rok & (j == i32(0))).select(gslot * i32(H // 32) + q * i32(8), xs_bytes) + for p in range_constexpr(TPC): + bst(dv, rx[p], doff, 0, AUX_SYS) + bst(sv, rs[p], soff, 0, AUX_SYS) + + @flyc.jit + def ag_stage_free(L, lane, a): + spin0(L, lane, L_CTL + C_AGFREE * 4, i32(1)) + + def _ag_senders(a, q, x): + m = fx.max(a["m"], i32(1)) + S = ag_split(a) + lo = (S > i32(1)).select((q % S) * m, i32(0)) + hi = (S > i32(1)).select(lo + m, i32(gpu.grid_dim.x)) + return ceildiv(hi - x, i32(N_XCD)) - ceildiv(lo - x, i32(N_XCD)) + + @flyc.jit + def _ag_bump(a, lane, q): + if lane == i32(0): + x = i32(gpu.block_id("x")) % i32(N_XCD) + xa = ctrl_at(a, i32(CTRL_AGX) + (q * i32(N_XCD) + x) * i32(LRDY_STRIDE)) + if g_add_agent(xa, 1) + i32(1) == _ag_senders(a, q, x): + g_st_sys(xa, i32(0)) + _ag_bump_rank(a, q) + + @flyc.jit + def _ag_bump_rank(a, q): + ga = ctrl_at(a, i32(CTRL_AGG) + q * i32(LRDY_STRIDE)) + m = fx.max(a["m"], i32(1)) + nx = fx.min((ag_split(a) > i32(1)).select(m, i32(gpu.grid_dim.x)), i32(N_XCD)) + if g_add_agent(ga, 1) + i32(1) == nx: + g_st_sys(ga, i32(0)) + idx = i32(FLAG_AGQ) + q * i32(MAX_TP) + a["rank"] + for p in range_constexpr(TPC): + bst(a["epoch"], peer_rs(a, p, "off_flag"), idx * i32(4), 0, AUX_SYS) + + def _ag_nmeta(a): + return fx.max(ceildiv(a["m"] * i32(TOPK) // i32(4), i32(64)), i32(1)) + + @flyc.jit + def _ag_send_meta(lane, a): + bid = i32(gpu.block_id("x")) + if bid < _ag_nmeta(a): + n = a["m"] * i32(TOPK) + n4 = n // i32(4) + rid = rsrc(a["ids_in"], n * i32(4)) + rtw = rsrc(a["tw_in"], n * i32(4)) + dst0 = a["rank"] * n * i32(4) + v = bid * i32(64) + lane + idv = bld(rid, v * i32(16), 0, V4I, 0) + wvv = bld(rtw, v * i32(16), 0, V4I, 0) + e = n4 * i32(4) + lane + ide = bld(rid, e * i32(4), 0, T.i32, 0) + wte = bld(rtw, e * i32(4), 0, T.i32, 0) + big = i32(1 << 30) + vo = (v < n4).select(dst0 + v * i32(16), big) + eo = ((bid == i32(0)) & (e < n)).select(dst0 + e * i32(4), big) + for p in range_constexpr(TPC): + ri = peer_rs(a, p, "off_ids", big) + rw = peer_rs(a, p, "off_w", big) + bst(idv, ri, vo, 0, AUX_SYS) + bst(wvv, rw, vo, 0, AUX_SYS) + bst(ide, ri, eo, 0, AUX_SYS) + bst(wte, rw, eo, 0, AUX_SYS) + wait_vm(0) + if lane == i32(0): + fo = (i32(FLAG_AGM) + a["rank"] * i32(NMETA) + bid) * i32(4) + bst(a["epoch"], own_rs(a, "off_flag"), fo, 0, AUX_SYS) + for p in range_constexpr(TPC): + if i32(p) != a["rank"]: + bst(a["epoch"], peer_rs(a, p, "off_flag"), fo, 0, AUX_SYS) + + @flyc.jit + def _ag_send_meta_ll(lane, a): + n = a["m"] * i32(TOPK) + rid = rsrc(a["ids_in"], n * i32(4)) + rtw = rsrc(a["tw_in"], n * i32(4)) + big = i32(1 << 30) + for i_ in range( + i32(gpu.block_id("x")) * i32(64) + lane, n, i32(gpu.grid_dim.x) * i32(64) + ): + i = i32(i_) + e = fx.Int32(bld(rid, i * i32(4), 0, T.i32, 0)) + wv = fx.Int32(bld(rtw, i * i32(4), 0, T.i32, 0)) + pkt = fx.Vector.from_elements([e, a["epoch"], wv, a["epoch"]], fx.Int32) + off = (a["rank"] * n + i) * i32(16) + for p in range_constexpr(TPC): + bst(pkt, peer_rs(a, p, "off_ids", big), off, 0, AUX_SYS) + + @flyc.jit + def meta_ll_wait(tid, a): + n = a["ttot"] * i32(TOPK) + rid = rsrc(a["ids"]) + for i_ in range(tid, n, i32(NT)): + i = i32(i_) + t = fx.Int32(fx.Vector(bld(rid, i * i32(16), 0, V4I, AUX_SYS))[1]) + t0 = now() + while (t != a["epoch"]) & alive(t0): + rocdl.s_sleep(1) + t = fx.Int32(fx.Vector(bld(rid, i * i32(16), 0, V4I, AUX_SYS))[1]) + _report_if(a, t != a["epoch"], ERR_META) + + def _meta_pending(a, lane, epoch, own=False): + rf = own_rs(a, "off_flag") + nmeta = _ag_nmeta(a) + pend = i32(0) + for it in range_constexpr(MAX_TP * NMETA // 64): + e = i32(it * 64) + lane + p = e // i32(NMETA) + j = e - p * i32(NMETA) + ok = ((p == a["rank"]) if own else (p < a["tp"])) & (j < nmeta) + f = fx.Int32( + bld( + rf, (i32(FLAG_AGM) + p * i32(NMETA) + j) * i32(4), 0, T.i32, AUX_SYS + ) + ) + pend = fx.max(pend, (ok & before(f, epoch)).select(i32(1), i32(0))) + return wave_red(pend, lane, fx.max) + + @flyc.jit + def ag_wait_meta(a, tid, epoch): + if tid < i32(64): + lane = tid % i32(64) + poll_zero(lambda: _meta_pending(a, lane, epoch), a, lane, ERR_META) + + def _ag_first_pending(lane, a, epoch, c_lo, nwin): + rf = own_rs(a, "off_flag") + end = fx.min(c_lo + i32(nwin), i32(NCH)) + first = end + for it in range_constexpr(ceildiv(TPC * nwin, 64)): + e = i32(it * 64) + lane + cc = c_lo + e // i32(TPC) + src = e % i32(TPC) + idx = i32(FLAG_AGQ) + fx.min(cc, i32(NCH - 1)) * i32(MAX_TP) + src + f = fx.Int32(bld(rf, idx * i32(4), 0, T.i32, AUX_SYS)) + first = fx.min(first, ((cc < end) & before(f, epoch)).select(cc, end)) + return wave_red(first, lane, fx.min) + + @flyc.jit + def ag_wait_chunk(L, lane, a, epoch, cc): + rdy = _ag_first_pending(lane, a, epoch, cc, AG_WIN) + if cc == i32(0): + rdy = _ag_first_pending(lane, a, epoch, i32(0), NCH) + t0 = now() + while (rdy <= cc) & alive(t0): + rocdl.s_sleep(1) + rdy = _ag_first_pending(lane, a, epoch, cc, AG_WIN) + _report_if(a, (rdy <= cc) & (lane == i32(0)), ERR_CHUNK) + if lane == i32(0): + lds_st(L, L_CTL + C_ARDY * 4, rdy) + rocdl.sched_barrier(0) + + @flyc.jit + def finish(tid, a, epoch): + if tid == i32(0): + bid = i32(gpu.block_id("x")) + g_st_sys(ctrl_at(a, i32(CTRL_EPB) + bid), epoch) + x = bid % i32(N_XCD) + nx = ceildiv(i32(gpu.grid_dim.x) - x, i32(N_XCD)) + xc = ctrl_at(a, i32(CTRL_XES) + x * i32(LRDY_STRIDE)) + if g_add_agent(xc, 1) == nx - i32(1): + g_st_sys(xc, i32(0)) + fx.memory_fence(syncscope="one-as", ordering=ACQ) + + @flyc.jit + def _dyn_zero(L, tid): + if (tid >= i32(NT + 64)) & (tid < i32(NT + 64 + NBW)): + lds_st(L, L_DYN + (tid - i32(NT + 64)) * i32(4), i32(0)) + for e_ in range(tid, i32(E), i32(NTT)): + lds_st(L, L_DCNT + i32(e_) * i32(4), i32(0)) + if const_expr(lb): + for e_ in range(tid, i32(E), i32(NTT)): + lds_st(L, L_DPRE + i32(e_) * i32(4), i32(0)) + + @flyc.jit + def init_lds(L, tid, a): + bid = i32(gpu.block_id("x")) + if tid < i32(C_UL): + is_done = (tid >= i32(C_DONE)) & (tid < i32(C_DONE + NW)) + ep = g_ld_rel(ctrl_at(a, i32(CTRL_EPB) + bid), "agent") + i32(1) + v = is_done.select(i32(-1), (tid == i32(C_EPOCH)).select(ep, i32(0))) + stage = (tid == i32(C_LRED)) | (tid == i32(C_PULL)) + v = (stage & comm_idle(a)).select(i32(NV), v) + if const_expr(DLL): + v = ((tid == i32(C_LRED)) | (tid == i32(C_NSIG))).select(i32(NCK), v) + if const_expr(lb): + v = (tid == i32(C_NSIG)).select(i32(NCK), v) + own = fin_owned(a) + v = (tid == i32(C_PULL)).select(i32(NV) - _ctpop(own), v) + v = (tid == i32(C_FBITS)).select(i32((1 << NV) - 1) & (own ^ i32(-1)), v) + if const_expr(ARLL): + v = (tid == i32(C_PULL)).select(i32(NCK), v) + lds_st(L, L_CTL + tid * i32(4), v) + + def _dll_rows(a, t, c0, vc, P): + rb = route_region_bytes(a["ttot"]) + r_routes = rsrc(a["routes"], rb) + r_pr = rsrc(a["proutes"], i32(NPC - 1) * rb) + out = [] + for k in range_constexpr(TOPK): + off = ((t * i32(TOPK) + i32(k)) * i32(H) + c0 + vc * i32(8)) * i32(2) + out.append(fx.Vector(bld(r_routes, off, 0, V4I, AUX_SC1))) + for p in range_constexpr(1, NPC): + o2 = (i32(p) < P).select(i32(p - 1) * rb + off, i32(NPC) * rb) + out.append(fx.Vector(bld(r_pr, o2, 0, V4I, AUX_SC1))) + return out + + def _dll_items(ws): + nblk = i32(gpu.grid_dim.x) + return i32(gpu.block_id("x")) + i32(ws) * nblk, nblk * i32(4) + + @flyc.jit + def push_dll(L, tid, a, ws=0): + lane = tid % i32(64) + spin0(L, lane, L_CTL + C_CDONE * 4, i32(1)) + P = uni(lds_ld_acq(L, L_CTL + C_DYNP * 4)) + ttot = a["ttot"] + it0, istep = _dll_items(ws) + for it_ in range(it0, ttot * i32(NCK), istep): + it = i32(it_) + c = it // ttot + t = it - c * ttot + c0 = c * i32(CW) + r_dst, prow = token_dst(a, t) + if const_expr(ARLL): + prow = a["rank"] * a["mmax"] * a["tp"] + t + for j in range_constexpr(VPL): + v = lane + i32(j * 64) + vc = fx.min(v, i32(CW // 8 - 1)) + live = [(v < i32(CW // 8)) & (i32(p) < P) for p in range(NPC)] * TOPK + if const_expr(TOPK * NPC <= 48): + pk = _dll_rows(a, t, c0, vc, P) + pend = ll_pending(a, lane, zip(pk, live)) + t0 = now() + while (pend != i32(0)) & alive(t0): + rocdl.s_sleep(8) + pk = _dll_rows(a, t, c0, vc, P) + pend = ll_pending(a, lane, zip(pk, live)) + else: + pend = ll_pending(a, lane, zip(_dll_rows(a, t, c0, vc, P), live)) + t0 = now() + while (pend != i32(0)) & alive(t0): + rocdl.s_sleep(8) + pend = ll_pending( + a, lane, zip(_dll_rows(a, t, c0, vc, P), live) + ) + pk = _dll_rows(a, t, c0, vc, P) + _report_if(a, (pend != i32(0)) & (lane == i32(0)), ERR_COMM) + acc = [fx.Float32(0.0)] * 8 + for k in range_constexpr(TOPK): + for p in range_constexpr(NPC): + acc = sum_live(acc, ll_vals(pk[k * NPC + p]), i32(p) < P) + if const_expr(ARLL): + for pr in range_constexpr(TPC): + _push_store( + a, + peer_rs(a, pr, "off_part"), + prow, + c0, + v, + acc, + v < i32(CW // 8), + ) + else: + _push_store(a, r_dst, prow, c0, v, acc, v < i32(CW // 8)) + if const_expr(ARLL): + final_all_ll(tid, a, ws) + + @flyc.jit + def final_all_ll(tid, a, ws=0): + lane = tid % i32(64) + ttot = a["ttot"] + trows = a["mmax"] * a["tp"] + r_recv = own_rs(a, "off_part") + ry = rsrc(a["y"], ttot * i32(H * 2)) + it0, istep = _dll_items(ws) + for it_ in range(it0, ttot * i32(NCK), istep): + it = i32(it_) + c = it // ttot + t = it - c * ttot + c0 = c * i32(CW) + for j in range_constexpr(VPL): + v = lane + i32(j * 64) + vc = fx.min(v, i32(CW // 8 - 1)) + live = [(i32(p) < a["tp"]) & (v < i32(CW // 8)) for p in range(MAX_TP)] + pk = recv_pkts(a, r_recv, t, trows, c0, vc) + pend = ll_pending(a, lane, zip(pk, live)) + t0 = now() + while (pend != i32(0)) & alive(t0): + rocdl.s_sleep(1) + pk = recv_pkts(a, r_recv, t, trows, c0, vc) + pend = ll_pending(a, lane, zip(pk, live)) + _report_if(a, (pend != i32(0)) & (lane == i32(0)), ERR_COMM) + acc = [fx.Float32(0.0)] * 8 + for p in range_constexpr(MAX_TP): + acc = sum_live(acc, ll_vals(pk[p]), i32(p) < a["tp"]) + if v < i32(CW // 8): + bst( + pack_bf16x8(acc), + ry, + (t * i32(H) + c0 + v * i32(8)) * i32(2), + 0, + 0, + ) + + return locals() diff --git a/amd_kernels/moe/mega_moe_tp/fly_utils.py b/amd_kernels/moe/mega_moe_tp/fly_utils.py new file mode 100644 index 0000000..e8fdc7c --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/fly_utils.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2025-2026 FlyDSL Project Contributors +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Device helpers of the MegaMoE TP kernels and their launch helpers. + +From AITER ``aiter/ops/flydsl/kernels/{kernels_common,act,mxfp4_gemm_common,communication_ops_utils,tensor_shim}.py`` +(ROCm/aiter edf58c67a), only what this package uses. +""" + +import os +import threading + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl.expr import math as fmath +from flydsl.expr import rocdl +from flydsl.expr.typing import T +from flydsl.utils import env + +LOG2E = 1.4426950408889634 + + +def ceildiv(numer, denom): + """Ceiling division preserving Python-int or DSL-scalar operand types.""" + return (numer + denom - 1) // denom + + +def _to_raw(v): + """ArithValue / Numeric (Int32, Boolean, ...) -> raw ir.Value.""" + if isinstance(v, ir.Value): + return v + if hasattr(v, "ir_value"): + return _to_raw(v.ir_value()) + return ir.Value._CAPICreate(v._CAPIPtr) + + +# ---- activations ---- +def sigmoid_batch(xs, *, alpha=1.0): + """Emit all exponentials before their reciprocals to preserve batch scheduling.""" + e = [fx.exp2(x * fx.Float32(-alpha * LOG2E), fastmath="afn") for x in xs] + return [fx.Float32(rocdl.rcp(T.f32, _to_raw(fx.Float32(1.0) + ei))) for ei in e] + + +def sigmoid_f32(g, *, alpha=1.0): + return sigmoid_batch([g], alpha=alpha)[0] + + +def tanh_batch(xs): + """Sign-restored tanh with exp2 and reciprocal operations grouped by stage.""" + neg_two_log2e = fx.Float32(-2.0 * LOG2E) + es = [fx.exp2(fx.max(x, -x) * neg_two_log2e, fastmath="afn") for x in xs] + recips = [fx.Float32(rocdl.rcp(T.f32, _to_raw(fx.Float32(1.0) + e))) for e in es] + zero = fx.Float32(0.0) + out = [] + for i, x in enumerate(xs): + tanh_abs = (fx.Float32(1.0) - es[i]) * recips[i] + out.append((x > zero).select(tanh_abs, -tanh_abs)) + return out + + +def tanh_f32(x): + return tanh_batch([x])[0] + + +def _silu_mul_batch(gs, us): + sig = sigmoid_batch(gs) + return [gs[i] * sig[i] * us[i] for i in range(len(gs))] + + +def _clamp_gate_up_batch(gate_values, up_values, limit=7.0): + limit_f32 = fx.Float32(float(limit)) + neg_limit_f32 = fx.Float32(-float(limit)) + gates = [fx.min(gate, limit_f32) for gate in gate_values] + ups = [fx.max(fx.min(up, limit_f32), neg_limit_f32) for up in up_values] + return gates, ups + + +def _swiglu_mul_batch(gate_values, up_values, limit): + gates, ups = _clamp_gate_up_batch(gate_values, up_values, limit) + # alpha folded into the exp2 constant (this kernel's rounding) + sigmoids = sigmoid_batch(gates, alpha=1.702) + return [gate * sigmoid * (up + 1.0) for gate, sigmoid, up in zip(gates, sigmoids, ups)] + + +def _situ_mul_batch(gate_values, up_values, beta=1.0, linear_beta=1.0): + # unclamped, host-computed reciprocals, left-associated products + beta_f32 = fx.Float32(float(beta)) + beta_rcp = fx.Float32(1.0 / float(beta)) + linear_beta_f32 = fx.Float32(float(linear_beta)) + linear_beta_rcp = fx.Float32(1.0 / float(linear_beta)) + return [ + beta_f32 * tanh_f32(gate * beta_rcp) * sigmoid_f32(gate) * linear_beta_f32 * tanh_f32(up * linear_beta_rcp) + for gate, up in zip(gate_values, up_values) + ] + + +def _activation_mul_batch(gate_values, up_values, act="silu", situ_beta=1.0, situ_linear_beta=1.0, + swiglu_limit=7.0): + if act == "swiglu": + return _swiglu_mul_batch(gate_values, up_values, limit=swiglu_limit) + if act == "situv2": + return _situ_mul_batch(gate_values, up_values, beta=situ_beta, linear_beta=situ_linear_beta) + # SiLU clamps gate / up before the multiply; the host passes +inf for an unclamped SiLU + gate_values, up_values = _clamp_gate_up_batch(gate_values, up_values, swiglu_limit) + return _silu_mul_batch(gate_values, up_values) + + +# ---- MX scales ---- +def _fabs_f32(x): + return fmath.absf(x) + + +def _e8m0_roundup(amax_f32, max_norm=6.0): + wi = (amax_f32 * fx.Float32(1.0 / float(max_norm))).bitcast(fx.Int32) + bexp = (wi + fx.Int32(0x7FFFFF)).shrui(fx.Int32(23)) & fx.Int32(0xFF) + lt = fx.Uint32(bexp) < 254 + return lt.select(bexp, fx.Int32(254)) + + +def _e8m0_from_amax(amax_f32, max_norm=6.0): + e8m0 = _e8m0_roundup(amax_f32, max_norm=max_norm) + qscale = (e8m0 << fx.Int32(23)).bitcast(fx.Float32) + return e8m0, qscale + + +# ---- global memory ordering ---- +def _to_ptr_global(v, dtype=fx.Int32, alignment=4): + ptr_type = fx.PointerType.get(dtype.ir_type, fx.AddressSpace.Global, alignment) + return fx.inttoptr(ptr_type, fx.Int64(v)) + + +def load_i32_global_agent(addr_i64): + """Volatile monotonic i32 load with agent-one-as visibility.""" + return fx.generic_load(_to_ptr_global(addr_i64), volatile=True, memory_order=fx.AtomicOrdering.Monotonic, + syncscope=fx.rocdl.SyncScope.AgentOneAs) + + +def fence_release(syncscope): + fx.memory_fence(ordering=fx.AtomicOrdering.Release, syncscope=syncscope) + + +def atomic_add_agent(addr_i64, val): + """Agent-scope monotonic global fetch-and-add.""" + dtype = fx.Int32 if isinstance(val, int) else fx.Int64 if fx.as_ir_value(val).type == T.i64 else fx.Int32 + ptr = _to_ptr_global(addr_i64, dtype, dtype.width // 8) + return fx.atomic_add(ptr, dtype(val), syncscope=fx.rocdl.SyncScope.Agent) + + +# ---- launch ---- +_PRELOAD_COMPILE_LOCK = threading.RLock() + + +def _compile_and_run(exe, *args): + """``flyc.compile(exe, *args)``: compiles and runs, returns the ``CompiledFunction`` (None in compile-only).""" + try: + return flyc.compile(exe, *args) + except Exception: + # flyc.compile leaks ir.Context on failure; pop it so a retry takes the right path. + try: + while ir.Context.current is not None: + ir.Context.current.__exit__(None, None, None) + except Exception: # noqa: BLE001, S110 + pass + raise + + +def _run_compiled(exe, *args): + """First call: compile and run; later calls: dispatch the cached ``CompiledFunction``.""" + if env.compile.compile_only: + # a callable warmed before a compile-only preload must not dispatch: keep it for the run + _compile_and_run(exe, *args) + return + cf = getattr(exe, "_cf", None) + if cf is not None: + cf(*args) + return + exe._cf = _compile_and_run(exe, *args) + + +def _preload_compiled(exe, *args): + """Compile (or load from the disk cache) without dispatching a kernel: ``JitFunction.preload`` where FlyDSL has + it, else FlyDSL's ``COMPILE_ONLY`` mode around ``flyc.compile``.""" + preload = getattr(exe, "preload", None) + if callable(preload): + return preload(*args) + with _PRELOAD_COMPILE_LOCK: + old = os.environ.get("COMPILE_ONLY") + os.environ["COMPILE_ONLY"] = "1" + try: + return flyc.compile(exe, *args) + finally: + if old is None: + os.environ.pop("COMPILE_ONLY", None) + else: + os.environ["COMPILE_ONLY"] = old diff --git a/amd_kernels/moe/mega_moe_tp/gemm.py b/amd_kernels/moe/mega_moe_tp/gemm.py new file mode 100644 index 0000000..58ecaae --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/gemm.py @@ -0,0 +1,1219 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/gemm.py (ROCm/aiter edf58c67a). +"""GEMM1 (gate / up + activation + quant of the intermediate into LDS) with its A +loader wave, and GEMM2 (down projection) of a unit into the route regions.""" + +from __future__ import annotations + +import functools +import math + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import gpu, range_constexpr, rocdl +from flydsl.expr.typing import T + +from .fly_utils import _activation_mul_batch, _e8m0_from_amax, ceildiv +from .common import ( + AUX_SC1, + REL, + V4I, + amax, + bld, + bst, + cat8, + dma4, + dma16, + fp4_pack, + fp8x4_pack, + i32, + lds_atomic_add, + lds_ld, + lds_ld_acq, + lds_ld_i32, + lds_st, + lds_st_rel, + mfma, + pack_bf16x2, + rsrc, + spin0, + spin_lds_ge, + swap16, + swap32, + uni, + wait_lgkm0, + wait_vm, +) +from .mega_moe_tp_config import ( + C_AFREE, + C_ARDY, + C_ASEQ, + C_DONE, + C_G2V, + C_G2VN, + C_LPUB, + C_LQ, + C_NCH, + C_PFW, + C_PLAN, + C_QBASE, + C_UL, + C_UNIT, + C_UROWS, + C_USEQ, + KCS, + NW, + OPS, + SLOT, + KernelCtx, + _nsk2_for, +) + + +def build_gemm(kc: KernelCtx) -> dict: + """The gemm device functions of one instance (added to kc).""" + ACB, ADEPTH, AUX_RT, CGW, CH1, CH2 = kc.get("ACB ADEPTH AUX_RT CGW CH1 CH2") + DLL, G2, GPC, H, I, KS1, L_A, L_AS = kc.get("DLL G2 GPC H I KS1 L_A L_AS") + L_B1, L_B1S, L_CTL, L_DCH, L_INTER = kc.get("L_B1 L_B1S L_CTL L_DCH L_INTER") + L_INTERS, L_RING, L_RIX, L_WT, MT = kc.get("L_INTERS L_RING L_RIX L_WT MT") + MTSKIP, NA_L, NA_ROWOPS, NAB, NCG = kc.get("MTSKIP NA_L NA_ROWOPS NAB NCG") + NCH, NCK, NSC_BLK, RG, SI_STRIDE = kc.get("NCH NCK NSC_BLK RG SI_STRIDE") + TOPK, XB, a8, a_frag = kc.get("TOPK XB a8 a_frag") + act_quant_store, ag_stage_free = kc.get("act_quant_store ag_stage_free") + ag_wait_chunk, cbar, const_expr = kc.get("ag_wait_chunk cbar const_expr") + ll_pkt, npp, nsk, route_fp8 = kc.get("ll_pkt npp nsk route_fp8") + route_region_bytes = kc.route_region_bytes + + @flyc.jit + def wait_a_chunk(L, lane, buf, q): + if lane == i32(0): + spin_lds_ge(L, L_CTL + (C_ASEQ * 4) + buf * i32(4), q + i32(1)) + rocdl.sched_barrier(0) + + @flyc.jit + def release_a_chunk(L, lane, buf): + if lane == i32(0): + lds_atomic_add(L, L_CTL + C_AFREE * 4 + buf * i32(4), 1, REL) + + def g1_group(nnb): + if const_expr(npp > 1): + return ((nnb % i32(npp)) == i32(0)).select(i32(npp), i32(1)) + return i32(1) + + @flyc.jit + def _gemm1_disp(L, tid, a, expert, i0, nnb, mte=None): + if const_expr(npp > 1): + if (nnb % i32(npp)) == i32(0): + _gemm1(L, tid, a, expert, i0, nnb, npp, mte) + else: + _gemm1(L, tid, a, expert, i0, nnb, 1, mte) + else: + _gemm1(L, tid, a, expert, i0, nnb, 1, mte) + + @flyc.jit + def gemm1(L, tid, a, expert, i0, nnb, rows=None): + nnb = uni(nnb) + if const_expr(MTSKIP and rows is not None): + if uni(rows) <= i32(3 * 16): + _gemm1_disp(L, tid, a, expert, i0, nnb, 3) + else: + _gemm1_disp(L, tid, a, expert, i0, nnb) + else: + _gemm1_disp(L, tid, a, expert, i0, nnb) + wait_vm(0) + cbar(L, tid) + + def _g1_operands(tid, a, expert, i0): + lane = tid % i32(64) + w = uni(tid // i32(64)) + e = uni(expert) + rw = rsrc(fx.Int64(a["w1"]) + fx.Int64(e) * fx.Int64(2 * I * (H // 2))) + rws = rsrc(a["w1s"]) + icol0 = uni(i0) + w * i32(32) + vg0 = icol0 * i32(H // 2) + lane * i32(16) + vg1 = vg0 + i32(16 * (H // 2)) + vu0 = vg0 + i32(I * (H // 2)) + vu1 = vu0 + i32(16 * (H // 2)) + vsl = lane * i32(4) + sg0 = uni((e * i32(2 * I) + icol0) // i32(32)) * i32(CH1 * 256) + su0 = uni((e * i32(2 * I) + i32(I) + icol0) // i32(32)) * i32(CH1 * 256) + return lane, w, rw, rws, (vg0, vg1, vu0, vu1), vsl, sg0, su0 + + def _g1_where(g, total, P): + gc = fx.min(g, total - i32(1)) + grp = gc // i32(KS1 * P) + r = gc - grp * i32(KS1 * P) + kk = r // i32(P) + return grp * i32(P) + (r - kk * i32(P)), kk + + def _g1_so(pas, kk): + return pas * i32(128 * (H // 2)) + kk * i32(1024) + + def _g1_sso(pas, kk): + return pas * i32((128 // 32) * CH1 * 256) + (kk // i32(2)) * i32(256) + + def _g1_chunk(L, lane, qb, ks): + k = ks % KCS + q = qb + i32(ks // KCS) + buf = q % i32(NAB) + rocdl.sched_barrier(0) + if k == 0: + wait_a_chunk(L, lane, buf, q) + abuf = L + i32(L_A) + buf * i32(RG * ACB) + asbuf = ( + L + + i32(L_AS) + + buf * i32(KCS * NSC_BLK * 64 * 4) + + i32(k * NSC_BLK * 64 * 4) + ) + return k, buf, abuf, asbuf + + def _g1_mfma(acc, o, b, sgw, suw, kh, abuf, asbuf, lane, k, MTE): + sc = [ + (x >> i32(8 * (kh * 2 + h))) & i32(0xFF) for x in (sgw, suw) for h in (0, 1) + ] + # all A fragments / scales of the k-step first, then the MFMAs: the LDS reads + # overlap instead of one exposed lgkmcnt(0) wait per 16-row block + afs, sas = [], [] + for m in range_constexpr(MTE): + row = i32(m * 16) + lane % i32(16) + afs.append(a_frag(abuf, row, k, lane // i32(16))) + sas.append( + fx.Int32(lds_ld(asbuf, row * i32(4) + lane // i32(16), T.i8, 1)) + & i32(0xFF) + ) + for m in range_constexpr(MTE): + for t in range_constexpr(4): + j = o + (t // 2) * 2 * MTE + m * 2 + t % 2 + acc[j] = mfma(b[t], afs[m], acc[j], sc[t], sas[m], a8) + + def _g1_out(L, lane, w, a, acc, grp, P, MTE): + ag_stage_free(L, lane, a) + for pp in range_constexpr(P): + o = pp * 4 * MTE + act_quant_store( + L, + lane, + w, + acc[o : o + 2 * MTE], + acc[o + 2 * MTE : o + 4 * MTE], + grp * i32(P) + i32(pp), + MTE, + ) + + @flyc.jit + def _gemm1(L, tid, a, expert, i0, nnb, P, MTE=None): + MTE = MT if MTE is None else MTE + lane, w, rw, rws, vs, vsl, sg0, su0 = _g1_operands(tid, a, expert, i0) + ring = uni(L + i32(L_RING) + w * i32(nsk * SLOT)) + total = nnb * i32(KS1) + qbase = lds_ld_i32(L, L_CTL + C_QBASE * 4 + w * i32(4)) + GS = KS1 * P + IL = nsk * (P * KCS) // math.gcd(nsk, P * KCS) + assert GS % IL == 0 + + def issue_b(g, slot_idx): + pas, kk = _g1_where(g, total, P) + slot = ring + i32(slot_idx * SLOT) + so = _g1_so(pas, kk) + for t in range_constexpr(4): + dma16(slot + i32(t * 1024), rw, vs[t], so, nt=True) + sso = _g1_sso(pas, kk) + dma4(slot + i32(4096), rws, vsl, sg0 + sso) + dma4(slot + i32(4096 + 256), rws, vsl, su0 + sso) + + for kk in range_constexpr(nsk): + issue_b(i32(kk), kk) + zero = fx.full(4, 0.0, fx.Float32) + for grp_ in range(i32(0), nnb // i32(P), i32(1)): + grp = i32(grp_) + init = [zero] * (4 * MTE * P) + for g0_, st in range( + grp * i32(GS), (grp + i32(1)) * i32(GS), i32(IL), init=init + ): + g0 = i32(g0_) + acc = list(st) + qb = qbase + grp * i32(NCH) + (g0 - grp * i32(GS)) // i32(P * KCS) + for ks in range_constexpr(IL // P): + k, buf, abuf, asbuf = _g1_chunk(L, lane, qb, ks) + for pp in range_constexpr(P): + s_ = ks * P + pp + g = g0 + i32(s_) + wait_vm((nsk - 1) * OPS) + slot = ring + i32((s_ % nsk) * SLOT) + b = [ + lds_ld(slot, i32(t * 1024) + lane * i32(16), V4I, 16) + for t in range(4) + ] + sgw = lds_ld_i32(slot, i32(4096) + lane * i32(4)) + suw = lds_ld_i32(slot, i32(4096 + 256) + lane * i32(4)) + _g1_mfma( + acc, + pp * 4 * MTE, + b, + sgw, + suw, + ks & 1, + abuf, + asbuf, + lane, + k, + MTE, + ) + wait_lgkm0() + rocdl.sched_barrier(0) + issue_b(g + i32(nsk), s_ % nsk) + rocdl.sched_barrier(0) + if k == KCS - 1: + release_a_chunk(L, lane, buf) + res = yield acc + _g1_out(L, lane, w, a, res, grp, P, MTE) + qend = qbase + (nnb // i32(P)) * i32(NCH) + if lane == i32(0): + lds_st(L, L_CTL + C_QBASE * 4 + w * i32(4), qend) + + def unit_fields(L, u): + f = [ + lds_ld_i32(L, L_CTL + (i32(C_UL) + u * i32(4) + i32(k)) * i32(4)) + for k in range(4) + ] + return f[0], f[1], f[2], f[3] + + @flyc.jit + def a_loader(L, tid, a): + lane = tid % i32(64) + rx = rsrc(a["ax"]) + rxs = rsrc(a["axs"]) + spin0(L, lane, L_CTL + C_PLAN * 4, i32(1)) + ub = uni(lds_ld_acq(L, L_CTL + C_UNIT * 4)) + ue = uni(lds_ld_acq(L, L_CTL + (C_UNIT + 1) * 4)) + for u_ in range(ub, ue, i32(1)): + u = i32(u_) + if lane == i32(0): + spin_lds_ge(L, L_CTL + C_USEQ * 4, u - ub + i32(1)) + _a_unit(L, lane, a, rx, rxs, u) + + @flyc.jit + def _a_unit(L, lane, a, rx, rxs, u): + R = uni(lds_ld_acq(L, L_CTL + C_UROWS * 4)) + nnb = unit_fields(L, u)[2] // i32(128) + for r0_ in range(i32(0), R, i32(RG)): + r0 = i32(r0_) + rows = fx.min(R - r0, i32(RG)) + arow = [] + for j in range_constexpr(NA_ROWOPS): + if const_expr(a8): + row = i32(j * 4) + lane // i32(16) + col = (lane % i32(16)) ^ (row & i32(7)) + else: + row = i32(j * 8) + lane // i32(8) + col = (lane % i32(8)) ^ (row & i32(7)) + rr = (row < rows).select(row, i32(0)) + arow.append( + (lds_ld_i32(L, L_RIX + (r0 + rr) * i32(4)) // i32(TOPK)) * i32(XB) + + col * i32(16) + ) + ascl = [] + for j in range_constexpr(NSC_BLK): + row = i32(j * 64) + lane + rr = (row < rows).select(row, i32(0)) + ascl.append( + (lds_ld_i32(L, L_RIX + (r0 + rr) * i32(4)) // i32(TOPK)) + * i32(H // 32) + ) + nq = (nnb // g1_group(nnb)) * i32(NCH) + if lane == i32(0): + lds_st(L, L_CTL + C_ARDY * 4, i32(0)) + for cidx_ in range(i32(0), nq, i32(1)): + cidx = i32(cidx_) + q = lds_ld_i32(L, L_CTL + C_LQ * 4) + b = q % i32(NAB) + if (lane == i32(0)) & (q >= i32(NAB)): + spin_lds_ge( + L, + L_CTL + C_AFREE * 4 + b * i32(4), + i32(NW) * (q // i32(NAB)), + ) + rocdl.sched_barrier(0) + cc = cidx % i32(NCH) + if cidx < i32(NCH): # noqa: SIM102 + if cidx >= uni(lds_ld_i32(L, L_CTL + C_ARDY * 4)): + ag_wait_chunk(L, lane, a, a["epoch"], cc) + abase = L + i32(L_A) + b * i32(RG * ACB) + for j in range_constexpr(NA_ROWOPS): + dma16(abase + i32(j * 1024), rx, arow[j], cc * i32(ACB)) + for k in range_constexpr(KCS): + for j in range_constexpr(NSC_BLK): + dst = ( + L + + i32(L_AS) + + b * i32(KCS * NSC_BLK * 64 * 4) + + i32((k * NSC_BLK + j) * 64 * 4) + ) + dma4(dst, rxs, ascl[j], cc * i32(8) + i32(k * 4), sys=True) + _loader_advance(L, lane, q) + wait_vm(0) + _loader_flush(L, lane) + + @flyc.jit + def _loader_advance(L, lane, q): + pub = lds_ld_i32(L, L_CTL + C_LPUB * 4) + if q + i32(1) - pub >= i32(ADEPTH): + wait_vm(NA_L * (ADEPTH - 1)) + if lane == i32(0): + lds_st_rel( + L, L_CTL + C_ASEQ * 4 + (pub % i32(NAB)) * i32(4), pub + i32(1) + ) + lds_st(L, L_CTL + C_LPUB * 4, pub + i32(1)) + if lane == i32(0): + lds_st(L, L_CTL + C_LQ * 4, q + i32(1)) + rocdl.sched_barrier(0) + + @flyc.jit + def _loader_flush(L, lane): + if lane == i32(0): + pub = lds_ld_i32(L, L_CTL + C_LPUB * 4) + q = lds_ld_i32(L, L_CTL + C_LQ * 4) + for p_ in range(pub, q, i32(1)): + p = i32(p_) + lds_st_rel(L, L_CTL + C_ASEQ * 4 + (p % i32(NAB)) * i32(4), p + i32(1)) + lds_st(L, L_CTL + C_LPUB * 4, q) + rocdl.sched_barrier(0) + + @flyc.jit + def report_chunk(L, lane, w, cidx): + if lane == i32(0): + lds_st_rel(L, L_CTL + C_DONE * 4 + w * i32(4), cidx) + + @flyc.jit + def maybe_report(L, lane, w, gi, signal, rlag_wait, lag=1): + if ( + signal + & (((gi + i32(1)) % i32(GPC)) == i32(0)) + & (gi + i32(1) > i32(lag * GPC)) + ): + wait_vm(rlag_wait) + report_chunk(L, lane, w, (gi // i32(GPC)) - i32(lag)) + + def store_route_fp8(rs, accs, wt, rix, n0, q4, ok, oob, a): + d, e8s = [], [] + for half in range_constexpr(2): + f = [ + fx.Float32(fx.Vector(accs[2 * half + h])[i]) * wt + for h in range(2) + for i in range(4) + ] + am = amax(f) + am = am.maximumf(am.shuffle_xor(i32(16), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(32), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=448.0) + d += [fp8x4_pack(f[0:4], qs), fp8x4_pack(f[4:8], qs)] + e8s.append(fx.Int32(e8) & i32(0xFF)) + d0, d1 = swap16(d[0], d[1]) + d2, d3 = swap16(d[2], d[3]) + d0, d2 = swap32(d0, d2) + d1, d3 = swap32(d1, d3) + lo, hi = [d0, d1], [d2, d3] + col = n0 + q4 * i32(16) + if const_expr(DLL): + e8 = ((q4 >> i32(1)) == i32(0)).select(e8s[0], e8s[1]) + off = (rix * i32(H) + col) * i32(2) + for h in range_constexpr(2): + d = lo if h == 0 else hi + pkt = ll_pkt(a, d[0], d[1], e8) + bst(pkt, rs, ok.select(off + i32(16 * h), oob), 0, AUX_SC1) + return + ov = fx.Vector.from_elements(lo + hi, fx.Int32) + bst(ov, rs, ok.select(rix * i32(H) + col, oob), 0, AUX_RT) + sc = fx.Int16(e8s[0] | (e8s[1] << i32(8))) + soff = a["ttot"] * i32(TOPK * H) + rix * i32(H // 32) + n0 // i32(32) + bst(sc, rs, (ok & (q4 == i32(0))).select(soff, oob), 0, AUX_RT) + + def _w2_off(n0, t, k): + return (n0 + i32(t * 16)) * i32(I // 2) + k * i32(1024) + + def g2_slot_dma(a, lane, w, slot, e, gi, k): + n0 = (w + i32(NW) * gi) * i32(64) + for t in range_constexpr(4): + dma16( + slot + i32(t * 1024), + rsrc(a["w2"]), + lane * i32(16), + e * i32(H * (I // 2)) + + (n0 + i32(t * 16)) * i32(I // 2) + + k * i32(1024), + nt=True, + ) + for p in range_constexpr(2): + rb = (e * i32(H) + n0 + i32(p * 32)) // i32(32) + dma4( + slot + i32(4096 + p * 256), + rsrc(a["w2s"]), + lane * i32(4), + (rb * i32(CH2) + k // i32(2)) * i32(256), + ) + + @flyc.jit + def gemm2( + L, tid, a, expert, ks0, r0, rows, signal, NKS, pidx, gi_lo=None, *args, **kw + ): + g = functools.partial( + _gemm2, + L, + tid, + a, + expert, + ks0, + r0, + rows, + signal, + NKS, + pidx, + gi_lo, + *args, + **kw, + ) + if const_expr(MTSKIP): + if uni(rows) <= i32(3 * 16): + g(mte=3) + else: + g() + else: + g() + + @flyc.jit + def _gemm2( + L, + tid, + a, + expert, + ks0, + r0, + rows, + signal, + NKS, + pidx, + gi_lo=None, + gi_hi=None, + pre=None, + buf=None, + progress=None, + mte=None, + nxt=None, + ): + MTE = MT if mte is None else mte + if const_expr(buf is None): + ib0, is0 = i32(L_INTER), i32(L_INTERS) + rix0, wt0 = i32(L_RIX), i32(L_WT) + else: + b0 = buf == i32(0) + ib0 = b0.select(i32(L_INTER), i32(L_B1)) + is0 = b0.select(i32(L_INTERS), i32(L_B1S)) + rix0 = i32(L_RIX) + buf * i32(128 * 4) + wt0 = i32(L_WT) + buf * i32(128 * 4) + NSK2 = _nsk2_for(NKS, G2) + GPI = (NSK2 * NKS // math.gcd(NSK2, NKS)) // NKS + assert G2 % GPI == 0 + WAIT_B2 = (NSK2 - 1) * OPS + LAG = max(1, ceildiv(NSK2, GPC * NKS)) + RLAG = GPC * NKS * OPS + lane = tid % i32(64) + w = uni(tid // i32(64)) + e = uni(expert) + ks0 = uni(ks0) + rw = rsrc(fx.Int64(a["w2"]) + fx.Int64(e) * fx.Int64(H * (I // 2))) + rws = rsrc(a["w2s"]) + routes_bytes = route_region_bytes(a["ttot"]) + pidx = uni(pidx) + dst = (pidx == i32(0)).select( + fx.Int64(a["routes"]), + fx.Int64(a["proutes"]) + fx.Int64(pidx - i32(1)) * fx.Int64(routes_bytes), + ) + r_routes = rsrc(dst, routes_bytes) + oob = routes_bytes + ring = uni(L + i32(L_RING) + w * i32(nsk * SLOT)) + q4 = lane // i32(16) + g_lo = i32(0) if gi_lo is None else uni(gi_lo) + g_hi = i32(G2) if gi_hi is None else uni(gi_hi) + + XPF = nxt is not None + + def issue_x(q, slot_idx): + tq = g_hi * i32(NKS) + if q == tq: + vn = lds_ld_acq(L, L_CTL + C_G2VN * 4) + ok = (vn == nxt + i32(2)) & ( + lds_ld_i32(L, L_CTL + C_G2V * 4) + < lds_ld_i32(L, L_CTL + C_NCH * 4) * i32(NCG) + ) + if lane == i32(0): + lds_st(L, L_CTL + (C_PFW + w) * i32(4), ok.select(i32(1), i32(0))) + pf = lds_ld_i32(L, L_CTL + (C_PFW + w) * i32(4)) == i32(1) + nun_ = fx.max(lds_ld_i32(L, L_CTL + C_NCH * 4), i32(1)) + v = lds_ld_i32(L, L_CTL + C_G2V * 4) + cc_ = v // nun_ + ent_ = lds_ld_i32(L, L_DCH + fx.min(v - cc_ * nun_, nun_ - i32(1)) * i32(4)) + ee = uni(pf.select(ent_ & i32(0xFFFF), e)) + qc = uni(pf.select(q - tq + cg_lo(cc_) * i32(GPC * NKS), tq - i32(1))) + gi = qc // i32(NKS) + g2_slot_dma( + a, + lane, + w, + ring + i32(slot_idx * SLOT), + ee, + gi, + ks0 + qc - gi * i32(NKS), + ) + + def issue(q, slot_idx): + if const_expr(XPF): + if q >= g_hi * i32(NKS): + issue_x(q, slot_idx) + else: + issue_n(q, slot_idx) + else: + issue_n(q, slot_idx) + + def issue_n(q, slot_idx): + qc = fx.min(q, g_hi * i32(NKS) - i32(1)) + gi = qc // i32(NKS) + k = ks0 + qc - gi * i32(NKS) + n0 = (w + i32(NW) * gi) * i32(64) + slot = ring + i32(slot_idx * SLOT) + for t in range_constexpr(4): + dma16( + slot + i32(t * 1024), rw, lane * i32(16), _w2_off(n0, t, k), nt=True + ) + for p in range_constexpr(2): + rb = (e * i32(H) + n0 + i32(p * 32)) // i32(32) + dma4( + slot + i32(4096 + p * 256), + rws, + lane * i32(4), + (rb * i32(CH2) + k // i32(2)) * i32(256), + ) + + if const_expr(XPF): + if lds_ld_i32(L, L_CTL + (C_PFW + w) * i32(4)) == i32(0): + for qq in range_constexpr(NSK2): + issue(g_lo * i32(NKS) + i32(qq), qq) + if lane == i32(0): + lds_st(L, L_CTL + (C_PFW + w) * i32(4), i32(0)) + else: + for qq in range_constexpr(NSK2): + issue(g_lo * i32(NKS) + i32(qq), qq) + if const_expr(pre is not None): + pre() + zero = fx.full(4, 0.0, fx.Float32) + + row_meta = [] + for m in range_constexpr(MTE): + R = i32(m * 16) + lane % i32(16) + ok = R < rows + Rc = ok.select(R, i32(0)) + row_meta.append( + ( + ok, + fx.Float32(lds_ld(L, wt0 + (r0 + Rc) * i32(4), T.f32)), + lds_ld_i32(L, rix0 + (r0 + Rc) * i32(4)), + ) + ) + + def groups(gi0): + for j in range_constexpr(GPI): + gi = gi0 + i32(j) + acc = [zero] * (4 * MTE) + for k in range_constexpr(NKS): + sidx = (j * NKS + k) % NSK2 + if const_expr(k < NSK2 and NKS >= NSK2): + if const_expr(j > 0): # noqa: SIM114 + wait_vm(WAIT_B2 + 2 * MTE) + elif gi > g_lo: + wait_vm(WAIT_B2 + 2 * MTE) + else: + wait_vm(WAIT_B2) + else: + wait_vm(WAIT_B2) + slot = ring + i32(sidx * SLOT) + b = [ + lds_ld(slot, i32(t * 1024) + lane * i32(16), V4I, 16) + for t in range(4) + ] + s0 = lds_ld_i32(slot, i32(4096) + lane * i32(4)) + s1 = lds_ld_i32(slot, i32(4096 + 256) + lane * i32(4)) + sh0 = ((ks0 + i32(k)) & i32(1)) * i32(16) + sb = [ + (s0 >> sh0) & i32(0xFF), + (s0 >> (sh0 + i32(8))) & i32(0xFF), + (s1 >> sh0) & i32(0xFF), + (s1 >> (sh0 + i32(8))) & i32(0xFF), + ] + af_l, sa_l = [], [] + for m in range_constexpr(MTE): + row = i32(m * 16) + lane % i32(16) + ib = ib0 + row * i32(SI_STRIDE) + if const_expr(a8): + kb = ib + i32(k * 128) + q4 * i32(16) + af_l.append( + cat8( + lds_ld(L, kb, V4I, 16), + lds_ld(L, kb + i32(64), V4I, 16), + ) + ) + else: + af_l.append( + lds_ld(L, ib + i32(k * 64) + q4 * i32(16), V4I, 16) + ) + sa_l.append( + fx.Int32( + lds_ld( + L, + is0 + row * i32(I // 32) + i32(k * 4) + q4, + T.i8, + 1, + ) + ) + & i32(0xFF) + ) + rocdl.sched_barrier(0) + rocdl.s_setprio(1) + for m in range_constexpr(MTE): + for t in range_constexpr(4): + acc[m * 4 + t] = mfma( + b[t], af_l[m], acc[m * 4 + t], sb[t], sa_l[m], a8 + ) + rocdl.s_setprio(0) + wait_lgkm0() + rocdl.sched_barrier(0) + issue(gi * i32(NKS) + i32(k + NSK2), sidx) + rocdl.sched_barrier(0) + n0 = (w + i32(NW) * gi) * i32(64) + ccol = (q4 & i32(1)) * i32(16) + (q4 >> i32(1)) * i32(8) + for m in range_constexpr(MTE): + ok, wt, rix = row_meta[m] + if const_expr(route_fp8): + store_route_fp8( + r_routes, + acc[m * 4 : m * 4 + 4], + wt, + rix, + n0, + q4, + ok, + oob, + a, + ) + else: + pk = [] + for t in range_constexpr(4): + av = fx.Vector(acc[m * 4 + t]) + pk.append( + [ + pack_bf16x2( + fx.Float32(av[2 * h]) * wt, + fx.Float32(av[2 * h + 1]) * wt, + ) + for h in range(2) + ] + ) + for half in range_constexpr(2): + ta, tb = 2 * half, 2 * half + 1 + lo, hi = [], [] + for h in range_constexpr(2): + x, y = swap16(pk[ta][h], pk[tb][h]) + lo.append(x) + hi.append(y) + ov = fx.Vector.from_elements(lo + hi, fx.Int32) + col = n0 + i32(half * 32) + ccol + voff = (rix * i32(H) + col) * i32(2) + bst(ov, r_routes, ok.select(voff, oob), 0, AUX_RT) + if const_expr(not DLL): + maybe_report(L, lane, w, gi, signal, LAG * RLAG, LAG) + + for gi0_ in range(g_lo, g_hi, i32(GPI)): + if const_expr(progress is not None): # noqa: SIM102 + if i32(gi0_) + i32(GPI) >= g_hi: + progress() + groups(i32(gi0_)) + wait_vm(0) + if const_expr(not DLL): # noqa: SIM102 + if signal: + report_chunk(L, lane, w, i32(NCK - 1)) + + def cg_lo(cc): + v = i32(0) + for k in range_constexpr(1, NCG + 1): + v = (cc >= i32(k)).select(i32(sum(CGW[:k])), v) + return v + + return locals() + + +# ---- big-M fused units (compile_bigm_kernel) ---- +const_expr = fx.const_expr + +KCK = 2 # GEMM1 k-steps (of 128) per chunk + + +def bigm_lds(RG, NAB, I): + """LDS layout (bytes) of one CTA.""" + c = {} + o = 0 + + def take(n, align=16): + nonlocal o + o = (o + align - 1) // align * align + r = o + o += n + return r + + # A codes: [buf][row][128 B], 16 B columns swizzled + c["LA"] = take(NAB * RG * 64 * KCK) + c["LAS"] = take(NAB * NW * 64 * 4) # A scales: [buf][wave][lane] words (dma4 order) + c["LRIX"] = take(RG * 4) + c["LWT"] = take(RG * 4) + c["SIS"] = I // 2 + 16 + c["LINT"] = take(RG * c["SIS"]) # intermediate MXFP4 rows + c["ISS"] = I // 32 + c["LINTS"] = take(RG * c["ISS"]) # their E8M0 scales + c["LCTL"] = take(64) + c["BYTES"] = (o + 15) // 16 * 16 + return c + + +def build_bigm_unit(H, I, TOPK, RG, NAB, BST, BS2, act, swiglu_limit, R8): + """The fused unit device function: unit_body(L, tid, e, r0, rows, rr, rwt, rx, rxs, w1, + w2, rws1, rws2, r_routes, oob): rows rr / rwt[r0 : r0 + rows] (route index, weight) of expert + e -> route rows wt * GEMM2(act(GEMM1(A[route // TOPK]))) stored at route * H (bf16). + R8: route rows as MXFP8 (e4m3 + one e8m0 scale per 32 columns): codes [rows][H] at r_routes, + scales [rows][H / 32] after them (at oob / 2: oob = the route count * H * 2). + Returns (LDS layout, unit_body).""" + assert RG % 16 == 0 and RG <= 128 and H % 256 == 0 and I % 128 == 0 + MTE = RG // 16 # 16-row tiles (every wave computes every row of the unit) + KH = H // 2 # A / w1 row bytes + KCH = H // (128 * KCK) # GEMM1 chunks (KCK k-steps of 128) + RBA = 64 * KCK # A row bytes per chunk + LPR = RBA // 16 # DMA lanes per A row (16 B each) + RPJ = 64 // LPR # A rows per dma16 + PASSES = I // (NW * 32) # GEMM1 passes + NG2 = H // (NW * 64) # GEMM2 column groups per wave + KS2 = I // 128 # GEMM2 k-steps + assert I % (NW * 32) == 0 + CH1, CH2 = ceildiv(H // 32, 8), ceildiv(I // 32, 8) + assert KCH % NAB == 0 and KCH % BST == 0 and KS2 % BS2 == 0 + lay = bigm_lds(RG, NAB, I) + ISS = lay["ISS"] + LA, LAS, LRIX, LWT = lay["LA"], lay["LAS"], lay["LRIX"], lay["LWT"] + SIS, LINT, LINTS = lay["SIS"], lay["LINT"], lay["LINTS"] + AR = RG // NW # A rows per wave (DMA) + assert AR in (16, 32) and 64 % (AR * KCK) == 0 + NAJ = AR // RPJ # dma16 per wave per chunk (RPJ rows x RBA B each) + NA = NAJ + 1 # A ops per wave per chunk: dma16s + 1 x dma4 (scales) + # B ops per wave per chunk: 2 k-steps x 4 tiles + gate / up scale words + NB = 4 * KCK + 2 + # sched barrier mask after the GEMM2 B issue groups (only VMEM is held) + SCM = 0x38F + NB2 = 6 # GEMM2 B ops per k-step: 4 tiles + 2 scale words per 64 columns + NST2 = 2 * MTE # route stores per wave per group + # steady-state GEMM1 issue order per chunk c: B(c + BST), then A(c + NAB - 1). By the + # count alone A(c) has landed once (NA + NB) * (NAB - 2) newer ops remain, but its LDS + # writes (direct-to-LDS loads) can still be in flight then: one chunk's ops more are + # awaited (else a unit now and then reads stale A rows of a chunk, small units mostly) + WAIT1 = max(min(NA + (NA + NB) * (BST - 1), (NA + NB) * (NAB - 3)), 0) + assert WAIT1 <= 63 + + def _wait2(kk): + # GEMM2: B2 of step s was issued BS2 steps earlier; every step issues NB2 B2 ops and + # the last k-step of a group then NST2 route stores + n = 0 + for j in range(1, BS2 + 1): + pos = (kk - j) % KS2 + if j < BS2: + n += NB2 + if pos == KS2 - 1: + n += NST2 + return min(n, 63) + + WAIT2 = [_wait2(kk) for kk in range(KS2)] + + def act_batch(gs, us): + return _activation_mul_batch(gs, us, act=act, swiglu_limit=swiglu_limit) + + @flyc.jit + def part_fill(L, tid, r0, rows, rr, rwt): + gpu.barrier() + if tid < i32(RG): + rc = (tid < rows).select(tid, i32(0)) + lds_st( + L, + i32(LRIX) + tid * i32(4), + fx.Int32(bld(rr, (r0 + rc) * i32(4), 0, T.i32)), + ) + lds_st( + L, + i32(LWT) + tid * i32(4), + fx.Int32(bld(rwt, (r0 + rc) * i32(4), 0, T.i32)), + ) + gpu.barrier() + + @flyc.jit + def part_g1(L, tid, e, rx, rxs, w1, rws1): + lane = tid % i32(64) + w = rocdl.readfirstlane(T.i32, tid // i32(64)) # the wave: its column group + q4 = lane // i32(16) + zero = fx.Vector.filled(4, 0.0, fx.Float32) + # ---- A operand addressing (this wave's rows) ---- + arow = [] + for j in range_constexpr(NAJ): + row = w * i32(AR) + i32(j * RPJ) + lane // i32(LPR) + col = (lane % i32(LPR)) ^ (row & i32(LPR - 1)) + tok = lds_ld_i32(L, i32(LRIX) + row * i32(4)) // i32(TOPK) + arow.append(tok * i32(KH) + col * i32(16)) + # scales: lane -> (row w * AR + lane % AR, k-step (lane // AR) % KCK) + srow = w * i32(AR) + lane % i32(AR) + stok = lds_ld_i32(L, i32(LRIX) + srow * i32(4)) // i32(TOPK) + soff = stok * i32(H // 32) + ((lane // i32(AR)) % i32(KCK)) * i32(4) + + def issue_a(c, buf): + base = L + i32(LA + buf * RG * RBA) + for j in range_constexpr(NAJ): + dma16( + base + (w * i32(AR) + i32(j * RPJ)) * i32(RBA), + rx, + arow[j], + c * i32(RBA), + ) + dma4( + L + i32(LAS + buf * NW * 256) + w * i32(256), + rxs, + soff, + c * i32(4 * KCK), + ) + + # one per-lane base + compile-time offsets (row tile m, k-step, ring buffer) for the A + # fragment / scale reads (one wave's rows are every row: the swizzle depends on the lane + # only), instead of per-tile address registers (spilled under the register pressure) + l16 = lane % i32(16) + swz = l16 & i32(LPR - 1) + abase = [ + L + i32(LA) + l16 * i32(RBA) + ((i32(k * 4) + q4) ^ swz) * i32(16) + for k in range(KCK) + ] + sbase = L + i32(LAS) + l16 * i32(4) + q4 + + def a_frag_m(buf, m, k): + return lds_ld(abase[k], buf * RG * RBA + m * 16 * RBA, V4I, 16) + + def a_scale_m(buf, m, k): + wd0 = (m * 16) // AR * 64 + k * AR + (m * 16) % AR + return fx.Int32(lds_ld(sbase, buf * NW * 256 + wd0 * 4, T.i8, 1)) & i32( + 0xFF + ) + + # ---- GEMM1 B (w1 of expert e): pass p, chunk c ---- + rw1 = rsrc(fx.Int64(w1) + fx.Int64(e) * fx.Int64(2 * I * KH)) + + def issue_b(p, c): + icol0 = p * i32(NW * 32) + w * i32(32) + vg0 = icol0 * i32(KH) + lane * i32(16) + vs = (vg0, vg0 + i32(16 * KH), vg0 + i32(I * KH), vg0 + i32((I + 16) * KH)) + b = [] + for k in range_constexpr(KCK): + for t in range_constexpr(4): + b.append( + fx.Vector( + bld(rw1, vs[t], (c * i32(KCK) + i32(k)) * i32(1024), V4I) + ) + ) + # one scale word = 2 k-steps + cw = (c * i32(KCK)) // i32(2) + sg = ((e * i32(2 * I) + icol0) // i32(32)) * i32(CH1 * 256) + cw * i32(256) + su = ((e * i32(2 * I) + i32(I) + icol0) // i32(32)) * i32(CH1 * 256) + su = su + cw * i32(256) + vso = lane * i32(4) + b.append(fx.Int32(bld(rws1, vso, sg, T.i32))) + b.append(fx.Int32(bld(rws1, vso, su, T.i32))) + return b + + def g1_chunk(acc, bq, buf, c): + # c: the chunk (compile time) + sgw, suw = bq[4 * KCK], bq[4 * KCK + 1] + for k in range_constexpr(KCK): + kp = (c * KCK + k) % 2 # the global k-step's half of the scale word + sc = [ + (x >> i32(8 * (kp * 2 + h))) & i32(0xFF) + for x in (sgw, suw) + for h in (0, 1) + ] + afs = [a_frag_m(buf, m, k) for m in range(MTE)] + sas = [a_scale_m(buf, m, k) for m in range(MTE)] + for m in range_constexpr(MTE): + for t in range_constexpr(4): + j = (t // 2) * 2 * MTE + m * 2 + t % 2 + acc[j] = mfma(bq[k * 4 + t], afs[m], acc[j], sc[t], sas[m]) + return acc + + def act_store(acc, p): + for m in range_constexpr(MTE): + row = i32(m * 16) + lane % i32(16) + gs, us_ = [], [] + for t in range_constexpr(2): + ga = fx.Vector(acc[m * 2 + t]) + ua = fx.Vector(acc[2 * MTE + m * 2 + t]) + for v in range_constexpr(4): + gs.append(fx.Float32(ga[v])) + us_.append(fx.Float32(ua[v])) + xs_ = [ + fx.Float32(x).to(fx.BFloat16).to(fx.Float32) + for x in act_batch(gs, us_) + ] + am = amax(xs_) + am = am.maximumf(am.shuffle_xor(i32(16), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(32), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=6.0) + for t in range_constexpr(2): + col = p * i32(NW * 32) + w * i32(32) + i32(t * 16) + q4 * i32(4) + pk = fp4_pack(xs_[t * 4 : t * 4 + 4], qs) + lds_st( + L, + i32(LINT) + row * i32(SIS) + col // i32(2), + fx.Int16(pk & i32(0xFFFF)), + align=2, + ) + if q4 == i32(0): + lds_st( + L, + i32(LINTS) + row * i32(ISS) + p * i32(NW) + w, + fx.Int8(e8), + align=1, + ) + + # ---- GEMM1: PASSES x KCH chunks, A ring NAB, B ring BST (carried over passes) ---- + # prologue = the steady-state issue order of the BST virtual steps before c = 0 + # (steps before the first real op of a kind issue dummy loads of the same count) + bpre = [] + for i in range_constexpr(-max(BST, NAB - 1), 0): + if const_expr(i + BST >= 0): + bpre.append(issue_b(i32(0), i32(i + BST))) + else: + issue_b(i32(0), i32(0)) + if const_expr(i + NAB - 1 >= 0): + issue_a(i32(i + NAB - 1), (i + NAB - 1) % NAB) + else: + for _ in range_constexpr(NA): + bld(rx, lane * i32(16), 0, V4I) + init = [zero] * (4 * MTE) + [v for b in bpre for v in b] + phi = i32(PASSES) + for p_, stt in range(i32(0), phi, i32(1), init=init): + p = i32(p_) + acc = [zero] * (4 * MTE) + ring = [ + list(stt[4 * MTE + s * NB : 4 * MTE + (s + 1) * NB]) for s in range(BST) + ] + for c in range_constexpr(KCH): + rocdl.sched_barrier(0) + wait_vm(WAIT1) + gpu.barrier() + # nothing crosses the chunk barrier: the A ring reads after every wave's DMA landed + # (else the scheduler may hoist them above s_barrier: a race) + rocdl.sched_barrier(0) + acc = g1_chunk(acc, ring[c % BST], c % NAB, c) + # refill: B of step c + BST (next pass past the end; the last pass + # reloads its own chunks: harmless, keeps the counts static) + cn = c + BST + pn = p + i32(cn // KCH) + pn = (pn < phi).select(pn, phi - i32(1)) + ring[cn % BST] = issue_b(pn, i32(cn % KCH)) + issue_a(i32((c + NAB - 1) % KCH), (c + NAB - 1) % NAB) + act_store(acc, p) + # (the yield's value binding is FlyDSL's loop-carried form) + res = yield [zero] * (4 * MTE) + [ # noqa: F841 + v for s in range(BST) for v in ring[s] + ] + wait_vm(0) + wait_lgkm0() # the intermediate's LDS writes land before other waves read them + gpu.barrier() + + @flyc.jit + def part_g2(L, tid, e, rows, w2, rws2, r_routes, oob): + lane = tid % i32(64) + w = rocdl.readfirstlane(T.i32, tid // i32(64)) # the wave: its column group + q4 = lane // i32(16) + zero = fx.Vector.filled(4, 0.0, fx.Float32) + # ---- GEMM2 from the LDS intermediate: NG2 groups x KS2 k-steps, B ring BS2 ---- + rw2 = rsrc(fx.Int64(w2) + fx.Int64(e) * fx.Int64(H * (I // 2))) + + def issue_b2(gi, kk): # one 64-column group + n0 = (w + i32(NW) * gi) * i32(64) + b = [ + fx.Vector( + bld( + rw2, + lane * i32(16), + (n0 + i32(t * 16)) * i32(I // 2) + kk * i32(1024), + V4I, + ) + ) + for t in range(4) + ] + for pp in range_constexpr(2): + rb = (e * i32(H) + n0 + i32(pp * 32)) // i32(32) + vo = lane * i32(4) + b.append( + fx.Int32( + bld(rws2, vo, (rb * i32(CH2) + kk // i32(2)) * i32(256), T.i32) + ) + ) + # the loads stay where the ring issues them (no sinking) + rocdl.sched_barrier(SCM) + return b + + row_meta = [] + for m in range_constexpr(MTE): + R = i32(m * 16) + lane % i32(16) + ok = R < rows + Rc = ok.select(R, i32(0)) + row_meta.append( + ( + ok, + lds_ld_i32(L, i32(LWT) + Rc * i32(4)).bitcast(fx.Float32), + lds_ld_i32(L, i32(LRIX) + Rc * i32(4)), + ) + ) + + def epi2(acc, gi): + # route rows of group gi (rows past the unit's: stores dropped, counts kept) + n0 = (w + i32(NW) * gi) * i32(64) + ccol = (q4 & i32(1)) * i32(16) + (q4 >> i32(1)) * i32(8) + if const_expr(R8): + for m in range_constexpr(MTE): + # MXFP8 route rows (the small-M engine's store_route_fp8 layout) + ok, wt, rix = row_meta[m] + d, e8s = [], [] + for half in range_constexpr(2): + f = [ + fx.Float32(fx.Vector(acc[m * 4 + 2 * half + h])[i]) * wt + for h in range(2) + for i in range(4) + ] + am = amax(f) + am = am.maximumf(am.shuffle_xor(i32(16), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(32), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=448.0) + d += [fp8x4_pack(f[0:4], qs), fp8x4_pack(f[4:8], qs)] + e8s.append(fx.Int32(e8) & i32(0xFF)) + d0, d1 = swap16(d[0], d[1]) + d2, d3 = swap16(d[2], d[3]) + d0, d2 = swap32(d0, d2) + d1, d3 = swap32(d1, d3) + ov = fx.Vector.from_elements([d0, d1, d2, d3], fx.Int32) + sc = fx.Int16(e8s[0] | (e8s[1] << i32(8))) + bst( + ov, + r_routes, + ok.select(rix * i32(H) + n0 + q4 * i32(16), oob), + 0, + 0, + ) + soff = oob // i32(2) + rix * i32(H // 32) + n0 // i32(32) + bst(sc, r_routes, (ok & (q4 == i32(0))).select(soff, oob), 0, 0) + else: + for m in range_constexpr(MTE): + ok, wt, rix = row_meta[m] + pk = [] + for t in range_constexpr(4): + av = fx.Vector(acc[m * 4 + t]) + pk.append( + [ + pack_bf16x2( + fx.Float32(av[2 * h]) * wt, + fx.Float32(av[2 * h + 1]) * wt, + ) + for h in range(2) + ] + ) + for half in range_constexpr(2): + ta, tb = 2 * half, 2 * half + 1 + lo, hi = [], [] + for h in range_constexpr(2): + x, y = swap16(pk[ta][h], pk[tb][h]) + lo.append(x) + hi.append(y) + ov = fx.Vector.from_elements(lo + hi, fx.Int32) + col = n0 + i32(half * 32) + ccol + bst( + ov, + r_routes, + ok.select((rix * i32(H) + col) * i32(2), oob), + 0, + 0, + ) + + # prologue = the steady issue order of the BS2 steps before (0, 0): their B2 loads, + # then (the previous group's last step) NST2 route stores, here dropped (OOB) + b2pre = [issue_b2(i32(s_ // KS2), i32(s_ % KS2)) for s_ in range(BS2)] + for _ in range_constexpr(NST2): + bst(fx.Vector.filled(4, 0, fx.Int32), r_routes, oob, 0) + ghi = i32(NG2) + init2 = [v for b in b2pre for v in b] + for gi_, st2 in range(i32(0), ghi, i32(1), init=init2): + gi = i32(gi_) + ring2 = [list(st2[s * NB2 : (s + 1) * NB2]) for s in range(BS2)] + acc = [zero] * (4 * MTE) + for kk in range_constexpr(KS2): + wait_vm(WAIT2[kk]) + bq = ring2[kk % BS2] + sh0 = (i32(kk) & i32(1)) * i32(16) + afs, sas = [], [] + for m in range_constexpr(MTE): + row = i32(m * 16) + lane % i32(16) + afs.append( + lds_ld( + L, + i32(LINT) + row * i32(SIS) + i32(kk * 64) + q4 * i32(16), + V4I, + 16, + ) + ) + sas.append( + fx.Int32( + lds_ld( + L, + i32(LINTS) + row * i32(ISS) + i32(kk * 4) + q4, + T.i8, + 1, + ) + ) + & i32(0xFF) + ) + s0, s1 = bq[4], bq[5] + sb = [ + (s0 >> sh0) & i32(0xFF), + (s0 >> (sh0 + i32(8))) & i32(0xFF), + (s1 >> sh0) & i32(0xFF), + (s1 >> (sh0 + i32(8))) & i32(0xFF), + ] + for m in range_constexpr(MTE): + for t in range_constexpr(4): + j = m * 4 + t + acc[j] = mfma(bq[t], afs[m], acc[j], sb[t], sas[m]) + sn = kk + BS2 + gn = gi + i32(sn // KS2) + gn = (gn < ghi).select(gn, ghi - i32(1)) + ring2[kk % BS2] = issue_b2(gn, i32(sn % KS2)) + epi2(acc, gi) + res2 = yield [v for s in range(BS2) for v in ring2[s]] # noqa: F841 + wait_vm(0) + + @flyc.jit + def unit_body( + L, tid, e, r0, rows, rr, rwt, rx, rxs, w1, w2, rws1, rws2, r_routes, oob + ): + part_fill(L, tid, r0, rows, rr, rwt) + part_g1(L, tid, e, rx, rxs, w1, rws1) + part_g2(L, tid, e, rows, w2, rws2, r_routes, oob) + + return lay, unit_body diff --git a/amd_kernels/moe/mega_moe_tp/ipc.py b/amd_kernels/moe/mega_moe_tp/ipc.py new file mode 100644 index 0000000..930c89e --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/ipc.py @@ -0,0 +1,69 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2025 FlyDSL Project Contributors +# Modifications Copyright (C) 2026 Advanced Micro Devices, Inc. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""HIP IPC handles of the symmetric arena (``mega_moe_tp.SymmetricArena``). + +From AITER ``aiter/ops/flydsl/ipc_heap.py`` (``UncachedIpcHeap``, ROCm/aiter edf58c67a), the calls this package uses. +""" + +import ctypes +import functools + +from amd_kernels.common.hip_runtime import load_hip_runtime + +_HANDLE_BYTES = 64 +_LAZY_ENABLE_PEER_ACCESS = 0x1 + + +class _Handle(ctypes.Structure): + _fields_ = [("reserved", ctypes.c_byte * _HANDLE_BYTES)] + + +@functools.cache +def load_hip() -> ctypes.CDLL: + """torch's HIP runtime with the IPC calls declared (a handle of its own: amd_kernels.common.hip_runtime).""" + hip = load_hip_runtime() + hip.hipIpcGetMemHandle.restype = ctypes.c_int + hip.hipIpcGetMemHandle.argtypes = [ctypes.POINTER(_Handle), ctypes.c_void_p] + hip.hipIpcOpenMemHandle.restype = ctypes.c_int + hip.hipIpcOpenMemHandle.argtypes = [ctypes.POINTER(ctypes.c_void_p), _Handle, ctypes.c_uint] + hip.hipIpcCloseMemHandle.restype = ctypes.c_int + hip.hipIpcCloseMemHandle.argtypes = [ctypes.c_void_p] + hip.hipGetErrorString.restype = ctypes.c_char_p + hip.hipGetErrorString.argtypes = [ctypes.c_int] + return hip + + +def hip_check(err: int, *, what: str) -> None: + if int(err) == 0: + return + try: + s = load_hip().hipGetErrorString(int(err)) + msg = s.decode("utf-8", errors="replace") if s else f"hipError({err})" + except Exception: # noqa: BLE001 + msg = f"hipError({err})" + raise RuntimeError(f"{what} failed: {msg}") + + +def get_mem_handle_bytes(base_ptr: int) -> bytes: + h = _Handle() + err = load_hip().hipIpcGetMemHandle(ctypes.byref(h), ctypes.c_void_p(int(base_ptr))) + hip_check(err, what="hipIpcGetMemHandle") + return bytes(ctypes.string_at(ctypes.byref(h), _HANDLE_BYTES)) + + +def open_mem_handle(handle_bytes: bytes) -> int: + if len(handle_bytes) != _HANDLE_BYTES: + raise ValueError(f"Expected {_HANDLE_BYTES}B handle") + h = _Handle() + ctypes.memmove(ctypes.byref(h), bytes(handle_bytes), _HANDLE_BYTES) + out = ctypes.c_void_p() + err = load_hip().hipIpcOpenMemHandle(ctypes.byref(out), h, ctypes.c_uint(_LAZY_ENABLE_PEER_ACCESS)) + hip_check(err, what="hipIpcOpenMemHandle") + return int(out.value) + + +def close_mem_handle(base_ptr: int) -> None: + err = load_hip().hipIpcCloseMemHandle(ctypes.c_void_p(int(base_ptr))) + hip_check(err, what="hipIpcCloseMemHandle") diff --git a/amd_kernels/moe/mega_moe_tp/layer.py b/amd_kernels/moe/mega_moe_tp/layer.py new file mode 100644 index 0000000..d2e32ed --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/layer.py @@ -0,0 +1,514 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/mega_moe_tp.py (ROCm/aiter edf58c67a). +"""Tensor-parallel MoE layer as one kernel (MegaMoE TP), gfx950. + +Experts are replicated over the TP group and ``inter_dim`` is sharded; one launch +runs the collectives around GEMM1 + activation + GEMM2 of this rank's inter slice. +``comm_mode``: ``"ag_rs"`` (sequence-parallel in/out) or ``"ar"`` (replicated in, +all-reduced out). Weights are MXFP4 (``layout.shuffle_weight`` + ``layout.shuffle_scale``). + +Contract: + +* ``forward`` is a collective: every rank calls it the same number of times, in the + same order, with the same local token count (and, ar, the same input and routing). + Layers of one shape may share an instance (``set_weights``): their forwards form one + collective sequence. +* ``prepare(local_tokens)`` before CUDA graph capture: it compiles every launch config, + runs each once and checks every rank's watchdog (reset + retry on a timeout). +* The kernel is persistent (one CTA per CU, CTAs and ranks spin on each other): it + needs the whole GPU. Do not run it concurrently with other kernels (other streams, + other processes, CU-masked / partitioned GPUs). +* A wait that times out (2 s) sets a sticky bit and the launch's output is invalid. + ``error_flag()`` is that bit as a device tensor (copy it asynchronously, e.g. every N + steps); ``check_errors()`` synchronizes and raises; ``reset()`` (collective) restarts + the cross-rank state, which a timed-out launch leaves inconsistent. +* The returned tensors (``out=None``, ar outputs, the fused tail's rows) are views of + internal buffers, valid until the next forward of this instance; pass ``out=`` to keep + the output. +* Launch epochs are int32, compared wrap-safe; a flag slot unused for 2**31 launches can + read stale: ``reset()`` at least that often for very long-lived instances. +* The symmetric memory uses hipIpc handles; in containers on the host network ROCm 7.1 + needs ``HSA_ENABLE_IPC_MODE_LEGACY=1``. + +Large batches (Swiglu or Silu, fp4, from ``bigm_min_tokens`` global tokens; default: the +batches the small-batch engine cannot hold, e.g. beyond 2048 local tokens on 256 CUs; 0 +disables) run the big-M kernel (chunked, SDMA, per-batch tuned configs from +mega_moe_tp_bigm_tuned.csv) behind the same ``forward``; it needs MORI built with +SDMA (importable, or its install directory in ``AKR_MORI_PATH``). ``max_local_tokens`` must +then cover the largest batch (e.g. 8192 for 32768 tokens on tp4); the small-batch engine +holds the batches below the threshold. ``prepare()`` sets the big-M path up (a collective: +SDMA queues, buffers, kernel compiles); big-M batches are not graph-capturable. +``launch_config()`` is None for a big-M batch (``bigm_plan()``: its chunks and options); +``error_flag()`` carries ``ERR_BIGM`` for a big-M timeout; ``close()`` frees the IPC memory +and the SDMA queues. In the ar mode each rank runs its 1 / tp of the replicated rows through +it (ag_rs) and the kernel gathers every rank's output rows (no fused tail there). + +Schedules (planned in the kernel from the routing): up to 256 global tokens the dynamic +one, from ``lb_min`` (default 256) global tokens up the large-batch (LB) one; its row tile +is 16 * ``lb_mt`` rows (``"T1:N1,...,N"``: up to Ti tokens Ni), 16 * ``lb_mt_small`` up to +``lb_small_max`` tokens; ``lb_npp`` GEMM1 column blocks per A pass; ``lb_q`` output column +chunks per GEMM2 unit. Fields set explicitly take precedence; unset ones use the per-batch +tuned configs (mega_moe_tp_tuned.csv, ``AKR_CONFIG_MEGA_MOE_TP``; inside its tuned +range), else the defaults (lb_min 256, lb_mt 5, lb_npp 2, lb_q 3). +""" + +from __future__ import annotations + +import enum +from dataclasses import dataclass + +import torch + +from .bigm_compute import MegaMoeTPBigM, _kcopy +from .mega_moe_tp import ( + COMM_MODES, + LaunchCfg, + MegaMoeTPEngine, + MegaMoeTPLDSError, +) +from .mega_moe_tp_config import PAD + +# MegaMoeTP.error_flag(): a big-M wait timed out (the small engine's bits are below it) +ERR_BIGM = 32 + +__all__ = [ + "COMM_MODES", + "ERR_BIGM", + "ActivationType", + "MegaMoeTP", + "MegaMoeTPConfig", + "MegaMoeTPLDSError", + "mega_moe_tp_supported", +] + + +class ActivationType(enum.Enum): + """The gate / up activations of GEMM1 (same names as AITER's ``ActivationType``, which is also accepted).""" + + Silu = "silu" + Swiglu = "swiglu" + Situv2 = "situv2" + + +_ACTS = {a.name.lower() for a in ActivationType} + + +def _act_name(activation) -> str: + """silu / swiglu / situv2 of an ActivationType (this one's or AITER's) or of its name.""" + return getattr(activation, "name", str(activation)).lower() + + +def mega_moe_tp_supported(gfx: str | None = None) -> bool: + """Whether this device has the kernel (gfx950).""" + if gfx is None: + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + gfx = props.gcnArchName.split(":")[0] + return gfx == "gfx950" + + +@dataclass(frozen=True) +class MegaMoeTPConfig: + rank: int + world_size: int + model_dim: int + inter_dim: int + experts: int + topk: int + # The largest batch of one forward, in local tokens (ag_rs: this rank's rows; ar: the + # replicated rows / world_size): set it to the caller's largest step, e.g. a scheduler's + # max_num_batched_tokens / world_size; larger batches must be split by the caller. It + # does not change the speed of smaller batches; it sizes the buffers: beyond what the + # small-batch engine holds (2048 local tokens on 256 CUs) the big-M path is set up, e.g. + # M3 tp4 ~0.5 GiB per rank at 1024, ~0.8 at 2048, ~2.8 at 4096, ~4.8 at 8192. + max_local_tokens: int + activation: ActivationType = ActivationType.Silu + beta: float | None = None + linear_beta: float | None = None + swiglu_limit: float | None = None + comm_mode: str = "ag_rs" + comm_dtype: str = "fp8" + ar_gather: str = "auto" + schedule: str = "dynamic" + act_dtype: str = "fp4" + lb_min: int | None = None + lb_mt: int | str | None = None + lb_mt_small: int | None = None + lb_small_max: int | None = None + lb_npp: int | None = None + lb_q: int | None = None + # the fused tail's norm: GemmaRMSNorm (scale 1 + w) or RMSNorm (w), and its eps + tail_eps: float = 1e-6 + tail_gemma: bool = True + # global tokens from which forward runs the big-M kernel (None: the batches the small + # engine cannot hold, i.e. (its capacity + 1) * world_size; 0: never) + bigm_min_tokens: int | None = None + # max_local_tokens is an upper bound: without a big-M path, lowered to what the small + # engine holds (MegaMoeTP.max_local_tokens) instead of raising + fit_max_local: bool = False + + +class MegaMoeTP: + """One fused TP MoE layer (see the module docstring).""" + + def __init__( + self, + cfg: MegaMoeTPConfig, + *, + w1: torch.Tensor, + w1_scale: torch.Tensor, + w2: torch.Tensor, + w2_scale: torch.Tensor, + group=None, + device: torch.device | None = None, + ): + act = _act_name(cfg.activation) + if act not in _ACTS: + raise ValueError(f"MegaMoeTP: unsupported activation {cfg.activation}") + if not mega_moe_tp_supported(): + raise ValueError("MegaMoeTP: needs gfx950") + if cfg.comm_mode not in COMM_MODES: + raise ValueError(f"MegaMoeTP: unknown comm_mode {cfg.comm_mode!r}") + situ = act == "situv2" + self.cfg = cfg + bmin = cfg.bigm_min_tokens + if bmin is not None and bmin < 0: + raise ValueError(f"MegaMoeTP: bigm_min_tokens {bmin} < 0") + # the shapes the big-M kernel takes: H % 256, < 512 routed experts (+ the last one, + # counted apart), topk - 1 <= 64, inter % 128 (else the small engine) + big_ok = ( + act in ("swiglu", "silu") + and cfg.act_dtype == "fp4" + and cfg.model_dim % 256 == 0 + and 0 < cfg.experts - 1 < 512 + and cfg.topk - 1 <= 64 + and cfg.inter_dim % 128 == 0 + ) + # local tokens from which forward runs the big-M kernel (0: never; set below for + # the default threshold, from the small engine's capacity) + self.bigm_m = -(-bmin // cfg.world_size) if bmin and big_ok else 0 + if self.bigm_m and self.bigm_m < PAD: + raise ValueError( + f"MegaMoeTP: bigm_min_tokens {bmin} < {PAD * cfg.world_size} (the big-M " + f"kernel runs at least {PAD} local rows)" + ) + self._bigm = None + self._bigm_args = { + "w1": w1, + "w1_scale": w1_scale, + "w2": w2, + "w2_scale": w2_scale, + "group": group, + "device": device, + } + small_max = cfg.max_local_tokens + if self.bigm_m and cfg.max_local_tokens >= self.bigm_m: + small_max = self.bigm_m - 1 + # default threshold: the small engine holds what it can, big-M takes the rest + fit = (big_ok and bmin is None) or cfg.fit_max_local + self.engine = MegaMoeTPEngine( + rank=cfg.rank, + world_size=cfg.world_size, + model_dim=cfg.model_dim, + inter_dim=cfg.inter_dim, + experts=cfg.experts, + topk=cfg.topk, + max_local_tokens=small_max, + w1=w1, + w1_scale=w1_scale, + w2=w2, + w2_scale=w2_scale, + activation=act, + situ_beta=cfg.beta if situ and cfg.beta is not None else 1.0, + situ_linear_beta=( + cfg.linear_beta if situ and cfg.linear_beta is not None else 1.0 + ), + swiglu_limit=cfg.swiglu_limit, + comm_mode=cfg.comm_mode, + comm_dtype=cfg.comm_dtype, + ar_gather=cfg.ar_gather, + schedule=cfg.schedule, + act_dtype=cfg.act_dtype, + lb={ + "min": cfg.lb_min, + "mt": cfg.lb_mt, + "mt_small": cfg.lb_mt_small, + "small_max": cfg.lb_small_max, + "npp": cfg.lb_npp, + "q": cfg.lb_q, + }, + tail_eps=cfg.tail_eps, + tail_gemma=cfg.tail_gemma, + group=group, + device=device, + fit_max_local=fit, + ) + if big_ok and bmin is None and self.engine.mmax < cfg.max_local_tokens: + self.bigm_m = self.engine.mmax + 1 + + def forward( + self, + x_local: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + out: torch.Tensor | None = None, + tail=None, + bf16: bool = False, + ): + """ag_rs: x [m, H] (own tokens); ar: x [M, H] replicated. tail (ag_rs, see + tail_ok): (res_in, res_out, norm_w) -> also res_out = y + res_in and the FP8 rows + (+ fp32 scales; bf16: + the bf16 rows) of the norm of res_out: returns (y, q, + scale[, rows]), each [tp * m, ...] with rank r's rows at r * m. engine.tn_local + (default True): only this rank's rows are written (the others are undefined); + False: every rank's.""" + if self._big(self._local(x_local.shape[0])): + # the small engine's input checks (up to max_local_tokens), then the big-M + # kernel's input types (int32 ids, f32 weights, contiguous; no-ops if so) + self.engine._local_tokens( + x_local, + topk_weights, + topk_ids, + limit=self.cfg.max_local_tokens, + uneven=True, + ) + self.engine._check_out(out, x_local.shape[0]) + if tail is not None: + self.engine._check_tail(tail, x_local.shape[0]) + x_local = x_local.contiguous() + topk_weights = topk_weights.to(torch.float32).contiguous() + topk_ids = topk_ids.to(torch.int32).contiguous() + if self.cfg.comm_mode == "ar": + return self._bigm_ar(x_local, topk_weights, topk_ids, out, tail) + local = bool(self.engine.tn_local) + res = self.bigm().moe( + x_local, topk_weights, topk_ids, out=out, tail=tail, local=local + ) + if tail is not None and bf16: + res = (*res, self._bigm_rows(tail[1], tail[2], local)) + return res + return self.engine(x_local, topk_weights, topk_ids, out, tail=tail, bf16=bf16) + + def _local(self, rows: int) -> int: + """local tokens of a forward of these input rows (ar: replicated, rows / tp per rank)""" + tp = self.cfg.world_size + return -(-rows // tp) if self.cfg.comm_mode == "ar" else rows + + def _bigm_ar(self, x, topk_weights, topk_ids, out, tail): + """ar on the big-M kernel: this rank's rows of the replicated batch (ag_rs), every + rank's y gathered inside the kernel -> y [M, H] (out, else a new tensor)""" + if tail is not None: + raise ValueError("MegaMoeTP: no fused tail in the ar mode's big-M path") + M, H = x.shape + tp, r = self.cfg.world_size, self.cfg.rank + m = -(-M // tp) + lo, hi = min(r * m, M), min((r + 1) * m, M) + xs, tw, ids = x[lo:hi], topk_weights[lo:hi], topk_ids[lo:hi] + if hi - lo < m: # the last ranks' rows of an uneven batch: zero / no route + n = hi - lo + xs = torch.cat([xs, x.new_zeros((m - n, H))]) + tw = torch.cat([tw, tw.new_zeros((m - n, tw.shape[1]))]) + ids = torch.cat([ids, ids.new_full((m - n, ids.shape[1]), -1)]) + if out is None: + out = torch.empty_like(x) + xs, tw, ids = xs.contiguous(), tw.contiguous(), ids.contiguous() + if M == tp * m: # (moe copies every rank's rows into out) + self.bigm().moe(xs, tw, ids, out=out, gather=True) + else: + # (a shader copy: copy_ of the arena may run on an SDMA engine) + _kcopy(out, self.bigm().moe(xs, tw, ids, gather=True)[:M]) + return out + + def _bigm_rows(self, res_out, norm_w, local): + """the big-M tail's bf16 rows [tp * m, H]: (Gemma)RMSNorm of res_out at this rank's + position, the other ranks' rows too unless local (the kernel writes the FP8 rows + only), exchanged through the big-M arena (device copies over xGMI; the caller's group + may be a CPU group).""" + r = res_out.to(torch.float32, copy=True) # (scaled in place below) + w = norm_w.float() + 1.0 if self.engine.tn_gemma else norm_w.float() + inv = torch.rsqrt(r.pow(2).mean(-1, keepdim=True) + float(self.engine.tn_eps)) + rows = r.mul_(inv).mul_(w).to(torch.bfloat16) + return self.bigm().gather_rows(rows, local=local) + + def _big(self, local_tokens: int) -> bool: + return bool(self.bigm_m) and local_tokens >= self.bigm_m + + def bigm(self): + """The big-M kernel's host object (collective on first use: its SDMA queues, arena). + prepare() creates it (and compiles its kernels) when big batches are reachable. + """ + if self._bigm is None: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + "MegaMoeTP: the big-M path first used inside CUDA graph capture; " + "call prepare(local_tokens) first (big-M batches are not " + "graph-capturable)" + ) + + cfg, a = self.cfg, self._bigm_args + self._bigm = MegaMoeTPBigM( + hidden=cfg.model_dim, + inter=cfg.inter_dim, + experts=cfg.experts - 1, # bigm counts the last expert apart (shared) + topk=cfg.topk - 1, + # (the big-M kernel runs on local rows padded to a multiple of PAD) + max_tokens=-(-cfg.max_local_tokens // PAD) * PAD * cfg.world_size, + eps=self.engine.tn_eps, + gemma=self.engine.tn_gemma, + swiglu_limit=self.engine.swiglu_limit, + w1=a["w1"], + w1_scale=a["w1_scale"], + w2=a["w2"], + w2_scale=a["w2_scale"], + group=a["group"], + device=a["device"], + act=_act_name(cfg.activation), + ygather=cfg.comm_mode == "ar", + # comm_dtype "bf16" (the accurate mode): bf16 route rows (MXFP8 otherwise) + opts={"BIGM_R8": "0"} if cfg.comm_dtype == "bf16" else None, + ) + return self._bigm + + __call__ = forward + + def set_weights( + self, + *, + w1: torch.Tensor, + w1_scale: torch.Tensor, + w2: torch.Tensor, + w2_scale: torch.Tensor, + ) -> None: + """Run the next forwards on another layer's weights of the same shape.""" + self.engine.set_weights(w1, w1_scale, w2, w2_scale) + self._bigm_args.update(w1=w1, w1_scale=w1_scale, w2=w2, w2_scale=w2_scale) + if self._bigm is not None: + self._bigm.set_weights(w1, w1_scale, w2, w2_scale) + + def prepare( + self, + local_tokens, + tail: bool = False, + tail_bf16: bool = False, + warmup: bool = True, + ) -> None: + """Collective: compile, arm and (warmup) run once and check the launch configs + of these local token counts (tail / tail_bf16: their fused-tail variants). When the + big-M path is reachable (max_local_tokens >= bigm_m) it is set up (SDMA queues, arena) + and (warmup) every big-M kernel a batch up to max_local_tokens can launch is compiled + (or loaded from the disk cache) and run once: with tail / tail_bf16 its fused-tail + variant (what a fused caller launches), else the plain one. Then no big-M batch + compiles while serving (a compile stalls every rank) or inside CUDA graph capture. + """ + ts = [local_tokens] if isinstance(local_tokens, int) else list(local_tokens) + ar = self.cfg.comm_mode == "ar" + btail = (tail or tail_bf16) and not ar # (the ar big-M path has no tail) + small = [t for t in ts if not self._big(t)] + if small: + self.engine.prepare(small, tail, tail_bf16, warmup) + hi = max([self.cfg.max_local_tokens, *ts]) + if not self._big(hi): + return + b = self.bigm() + if not warmup: + return + rows = b.io_rows_per_kernel(self.bigm_m, hi) + b.precompile(rows, b.tailq(btail, self.engine.tn_local, ar)) + cfg, dev = self.cfg, b.device + for m in rows: + m = m * cfg.world_size if ar else m # (ar: the replicated batch) + x = torch.zeros((m, cfg.model_dim), dtype=torch.bfloat16, device=dev) + # balanced routing: token i's slot j to expert (i (topk - 1) + j) % (experts - 1), + # the shared expert (experts - 1) last + r = torch.arange(m, device=dev)[:, None] * (cfg.topk - 1) + torch.arange( + cfg.topk - 1, device=dev + ) + ids = torch.cat( + [ + r % (cfg.experts - 1), + torch.full((m, 1), cfg.experts - 1, device=dev), + ], + 1, + ) + ids = ids.to(torch.int32).contiguous() + tw = torch.full( + (m, cfg.topk), 1.0 / cfg.topk, dtype=torch.float32, device=dev + ) + if btail: + res = torch.zeros_like(x) + self.forward(x, tw, ids, tail=(res, res, x[0]), bf16=tail_bf16) + else: + self.forward(x, tw, ids) + torch.cuda.synchronize() + + @property + def max_local_tokens(self) -> int: + return ( + self.cfg.max_local_tokens + if self._big(self.cfg.max_local_tokens) + else self.engine.mmax + ) + + def tail_ok(self, local_tokens: int) -> bool: + """forward(tail=...) runs for this local token count.""" + if self._big(local_tokens): + return self.cfg.comm_mode == "ag_rs" + return self.engine.tail_ok(local_tokens) + + def launch_config(self, local_tokens: int) -> LaunchCfg | None: + """The small engine's launch config of this local token count; None: the batch + runs the big-M kernel (see bigm_plan).""" + if self._big(local_tokens): + return None + return self.engine.config(local_tokens) + + def bigm_plan(self, local_tokens: int): + """The big-M kernel's plan of this local token count: ((chunk weights), + ((BIGM_* option, value), ...)); None: the small engine runs it.""" + if not self._big(local_tokens): + return None + m = -(-local_tokens // PAD) * PAD + return self.bigm()._plan(m * self.cfg.world_size) + + def error_flag(self) -> torch.Tensor: + """Sticky watchdog bits (nonzero: a launch timed out) as a device tensor (no + sync): the small engine's, | ERR_BIGM for a big-M one.""" + e = self.engine.error_flag() + if self._bigm is None: + return e + return e | (self._bigm.error_flag() * ERR_BIGM) + + def poll_errors(self) -> int: + """Nonzero if a wait inside the kernel gave up (a peer never arrived).""" + e = self.engine.poll_errors() + if not e and self._bigm is not None: + e = self._bigm.poll_errors() + return e + + def check_errors(self) -> None: + """Raise RuntimeError (and clear) if a wait gave up since the last check.""" + if self._bigm is not None and self._bigm.poll_errors(): + self._bigm.clear_errors() + raise RuntimeError( + "MegaMoeTP: a big-M wait on a peer gave up (results invalid)" + ) + self.engine.check_errors() + + def clear_errors(self) -> None: + self.engine.clear_errors() + if self._bigm is not None: + self._bigm.clear_errors() + + def reset(self) -> None: + """Collective: restart the cross-rank state (after a timeout / desync).""" + self.engine.reset() + if self._bigm is not None: + self._bigm.reset() + + def close(self) -> None: + """Collective: release the IPC memory and the MORI SDMA queues (the instance is + unusable afterwards).""" + if self._bigm is not None: + self._bigm.close() + self._bigm = None + self.engine.arena.close() diff --git a/amd_kernels/moe/mega_moe_tp/layout.py b/amd_kernels/moe/mega_moe_tp/layout.py new file mode 100644 index 0000000..7ea6920 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/layout.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/shuffle.py (shuffle_weight, layout (16, 16) on packed FP4; shuffle_scale; ROCm/aiter edf58c67a). +"""The MXFP4 weight layouts MegaMoeTP reads: ``shuffle_weight`` (codes) and ``shuffle_scale`` (e8m0 scales).""" + +import torch + + +def shuffle_weight(w: torch.Tensor) -> torch.Tensor: + """Row-major MXFP4 codes [E, N, K / 2] (uint8 or float4_e2m1fn_x2) -> 16 x 16-byte tiles: [E, N / 16, K / 32, + 16 rows, 16 bytes], same shape and dtype as the input.""" + dtype = w.dtype + u = w.view(torch.uint8) + E, N, KB = u.shape + assert N % 16 == 0 and KB % 16 == 0, (N, KB) + u = u.view(E, N // 16, 16, KB // 16, 16).permute(0, 1, 3, 2, 4).contiguous() + return u.view(E, N, KB).view(dtype) + + +def shuffle_scale(s: torch.Tensor) -> torch.Tensor: + """e8m0 scales [rows, K / 32] (all experts' rows stacked) -> the MFMA scale layout, rows padded to 256 and columns + to 8 with 0x7F (1.0).""" + m, n = s.shape + padded = torch.full(((m + 255) // 256 * 256, (n + 7) // 8 * 8), 0x7F, dtype=torch.uint8, device=s.device) + padded[:m, :n] = s.view(torch.uint8) + sm, sn = padded.shape + out = padded.view(sm // 32, 2, 16, sn // 8, 2, 4).permute(0, 3, 5, 2, 4, 1).contiguous() + return out.view(sm, sn).view(s.dtype) diff --git a/amd_kernels/moe/mega_moe_tp/mega_moe_tp.py b/amd_kernels/moe/mega_moe_tp/mega_moe_tp.py new file mode 100644 index 0000000..c37bf05 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/mega_moe_tp.py @@ -0,0 +1,944 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/mega_moe_tp.py (ROCm/aiter edf58c67a). +"""Host side of the single-kernel TP MegaMoE (launch configs, buffers, launches).""" + +from __future__ import annotations + +import contextlib +import ctypes +import dataclasses +import os +from dataclasses import dataclass + +import torch +import torch.distributed as dist + +from . import ipc as _Ipc +from .common import device_key, nearest, tuned_config, tuned_rows +from .fly_utils import _preload_compiled, _run_compiled, ceildiv +from .mega_moe_tp_config import ( + CTRL_ERR, + CTRL_INTS, + DYN_MAX, + ERR_CHUNK, + ERR_COMM, + ERR_FLAG, + ERR_META, + ERR_YAG, + FLAG_INTS, + MAX_TP, + NCHLB_MAX, + NCTA_MAX, + NMETA_CAP, + TN_MAX, + gemm2_chunk_groups, + gemm2_group_step, + mega_moe_tp_consts, + mega_moe_tp_shape_supported, +) +from .mega_moe_tp_kernel import compile_mega_moe_tp + +__all__ = [ + "LaunchCfg", + "MegaMoeTPEngine", + "mega_moe_tp_shape_supported", +] + +LDS_LIMIT = 160 * 1024 +LB_PF_MAX = 1024 +AG8_MIN = 512 +COMM_MODES = ("ag_rs", "ar") +_ERR_NAMES = ( + (ERR_FLAG, "flag"), + (ERR_META, "routing"), + (ERR_CHUNK, "input chunk"), + (ERR_COMM, "reduce"), + (ERR_YAG, "output gather"), +) + + +@dataclass(frozen=True) +class LaunchCfg: + """One kernel variant: row tile 16 * mt; ll / llr: LL reduce-scatter packets / LL + route rows too; npp: GEMM1 column blocks per A pass; ag8: MXFP8 all-reduce second + hop; lb: large-batch schedule (pf: GEMM2 prefetch, lp: forced GEMM1 inter pieces); + tn: fused tail (1: FP8 rows, 2: + bf16 rows).""" + + mt: int + ll: bool = False + llr: bool = False + npp: int = 1 + ag8: bool = False + lb: bool = False + tn: int = 0 + pf: bool = False + lp: int = 0 + q: int = 0 # LB GEMM2 output split (0: the engine's lb_q) + + +# --- intra-node symmetric memory (hipIpc, via ipc_heap) -------------------------- + +_ALIGN = 256 +# a peer allocation maps once per process (arenas can share one): handle -> [ptr, refs] +_OPEN: dict[bytes, list] = {} + + +def _allocation_base(ptr: int) -> int: + hip = _Ipc.load_hip() + hip.hipMemGetAddressRange.argtypes = [ + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ctypes.c_void_p, + ] + base, size = ctypes.c_void_p(), ctypes.c_size_t() + err = hip.hipMemGetAddressRange( + ctypes.byref(base), ctypes.byref(size), ctypes.c_void_p(ptr) + ) + _Ipc.hip_check(err, what="hipMemGetAddressRange") + return int(base.value or 0) + + +def _ipc_open(handle: bytes) -> int: + ent = _OPEN.get(handle) + if ent is None: + ent = _OPEN[handle] = [_Ipc.open_mem_handle(handle), 0] + ent[1] += 1 + return ent[0] + + +def _ipc_close(handle: bytes) -> None: + ent = _OPEN[handle] + ent[1] -= 1 + if ent[1] == 0: + del _OPEN[handle] + _Ipc.close_mem_handle(ent[0]) + + +@contextlib.contextmanager +def _no_expandable_segments(): + # hipIpcGetMemHandle cannot export expandable-segment (VMM) memory + conf = os.environ.get("PYTORCH_HIP_ALLOC_CONF") or os.environ.get( + "PYTORCH_CUDA_ALLOC_CONF", "" + ) + on = "expandable_segments:true" in conf.replace(" ", "").lower() + if on: + torch.cuda.memory._set_allocator_settings("expandable_segments:False") + try: + yield + finally: + if on: + torch.cuda.memory._set_allocator_settings("expandable_segments:True") + + +@dataclass +class _Slice: + offset: int + nbytes: int + shape: tuple + dtype: torch.dtype + local: torch.Tensor | None = None + + +class SymmetricArena: + """A same-layout device arena (torch caching-allocator memory) on every rank of + ``group``, mapped into all peers with hipIpc. The exported handle covers the + allocator segment holding the arena (peers address it by its offset in it). + Destruction is not collective: synchronize every rank before dropping an instance. + In containers on the host network ROCm 7.1 needs HSA_ENABLE_IPC_MODE_LEGACY=1.""" + + def __init__(self, *, group=None, device: torch.device | None = None): + self.group = group + self.device = device or torch.device("cuda", torch.cuda.current_device()) + self.rank = dist.get_rank(group=group) + self.world_size = dist.get_world_size(group=group) + self._slices: dict[str, _Slice] = {} + self._cursor = 0 + self._storage: torch.Tensor | None = None + self._base_ptrs: tuple[int, ...] = () + self._opened: list[bytes] = [] + + def reserve(self, name: str, shape, dtype: torch.dtype) -> _Slice: + """Carve out a named region. Must run in the same order on every rank.""" + if self._storage is not None or name in self._slices: + raise RuntimeError(f"cannot reserve {name!r}") + shape = tuple(int(s) for s in shape) + offset = ceildiv(self._cursor, _ALIGN) * _ALIGN + nbytes = torch.Size(shape).numel() * torch.empty((), dtype=dtype).element_size() + self._cursor = offset + nbytes + self._slices[name] = _Slice(offset, nbytes, shape, dtype) + return self._slices[name] + + def commit(self) -> SymmetricArena: + """Collective: allocate and map every rank's arena (raises on every rank if any + rank fails).""" + total = ceildiv(self._cursor, _ALIGN) * _ALIGN + with _no_expandable_segments(): + self._storage = torch.zeros(total, dtype=torch.uint8, device=self.device) + base_ptr = int(self._storage.data_ptr()) + for s in self._slices.values(): + s.local = self._storage[s.offset : s.offset + s.nbytes].view(s.dtype) + s.local = s.local.view(s.shape) + try: + with torch.cuda.device(self.device): + handle = _Ipc.get_mem_handle_bytes(base_ptr) + payload = (handle, base_ptr - _allocation_base(base_ptr), total) + err = "" + except RuntimeError as exc: + payload, err = None, str(exc) + torch.cuda.synchronize(self.device) + gathered: list = [None] * self.world_size + dist.all_gather_object(gathered, (payload, err), group=self.group) + bad = {r: e for r, (_, e) in enumerate(gathered) if e} + if not bad and any(p[2] != total for p, _ in gathered): + bad = {r: f"{p[2]} B, not {total} B" for r, (p, _) in enumerate(gathered)} + ptrs, err = [], "" + if not bad: + try: + with torch.cuda.device(self.device): + for r, ((handle, off, _), _) in enumerate(gathered): + if r == self.rank: + ptrs.append(base_ptr) + else: + ptrs.append(_ipc_open(handle) + off) + self._opened.append(handle) + except RuntimeError as exc: + err = str(exc) + opened: list = [None] * self.world_size + dist.all_gather_object(opened, err, group=self.group) + bad = {r: e for r, e in enumerate(opened) if e} + if bad: + self.close() + raise RuntimeError(f"SymmetricArena: mapping failed on ranks {bad}") + self._base_ptrs = tuple(ptrs) + dist.barrier(group=self.group) + return self + + def close(self) -> None: + """Unmap the peers' arenas (no kernel may use them any more).""" + if self._opened: + torch.cuda.synchronize(self.device) + for handle in self._opened: + _ipc_close(handle) + self._opened = [] + self._base_ptrs = () + + def __del__(self): + try: + self.close() + except Exception: # noqa: BLE001, S110 (interpreter teardown) + pass + + def barrier(self) -> None: + """Collective; also waits for this device, so a barrier implemented as a GPU + kernel (NCCL) never runs next to the next (persistent) launch.""" + dist.barrier(group=self.group) + torch.cuda.synchronize(self.device) + + @property + def storage(self) -> torch.Tensor: + return self._storage + + @property + def base_ptrs(self) -> tuple[int, ...]: + return self._base_ptrs + + +class MegaMoeTPLDSError(ValueError): + """max_local_tokens does not fit in LDS (retry with a smaller one).""" + + +# the tuned launch-config fields of a batch (mega_moe_tp_tuned.csv) +_TUNED_FIELDS = ("mt", "npp", "q", "pf", "lp", "ll", "llr") + + +def _mt_table(spec) -> tuple[int, list[tuple[int, int]]]: + *rows, last = [x.strip() for x in str(spec).split(",") if x.strip()] + table = sorted(tuple(int(v) for v in r.split(":")) for r in rows) + return int(last), table + + +class MegaMoeTPEngine: + def __init__( + self, + *, + rank: int, + world_size: int, + model_dim: int, + inter_dim: int, + experts: int, + topk: int, + max_local_tokens: int, + w1: torch.Tensor, + w1_scale: torch.Tensor, + w2: torch.Tensor, + w2_scale: torch.Tensor, + activation: str = "silu", + situ_beta: float = 1.0, + situ_linear_beta: float = 1.0, + swiglu_limit: float | None = None, + comm_mode: str = "ag_rs", + comm_dtype: str = "fp8", + ar_gather: str = "auto", + schedule: str = "dynamic", + act_dtype: str = "fp4", + lb: dict | None = None, + tail_eps: float = 1e-6, + tail_gemma: bool = True, + group=None, + device: torch.device | None = None, + fit_max_local: bool = False, + ): + """fit_max_local: max_local_tokens is an upper bound, lowered to the largest count + the kernel takes (its tables, 32-bit offsets, LDS; see capacity) instead of + raising.""" + if not mega_moe_tp_shape_supported(model_dim, inter_dim, world_size): + raise ValueError( + f"fused TP MegaMoE does not tile h{model_dim} i{inter_dim} tp{world_size}" + ) + if comm_mode not in COMM_MODES: + raise ValueError(f"unknown comm_mode {comm_mode!r}") + if comm_dtype not in ("fp8", "bf16"): + raise ValueError(f"unknown comm_dtype {comm_dtype!r}") + self.comm_bf16 = comm_dtype == "bf16" + if ar_gather not in ("bf16", "fp8", "auto"): + raise ValueError(f"unknown ar_gather {ar_gather!r}") + self.ag_fp8 = ar_gather == "fp8" + self.ag_auto = AG8_MIN if ar_gather == "auto" else 0 + if not 1 <= topk <= experts or max_local_tokens < 1: + raise ValueError( + f"need 1 <= topk ({topk}) <= experts ({experts}), max_local_tokens >= 1" + ) + self.mode = comm_mode + self.check = os.environ.get("AKR_MEGA_MOE_TP_CHECK", "0") == "1" + if schedule != "dynamic": + raise ValueError(f'unknown schedule {schedule!r} (only "dynamic")') + if act_dtype not in ("fp4", "fp8"): + raise ValueError(f"unknown act_dtype {act_dtype!r}") + self.a8 = act_dtype == "fp8" + lb = {k: v for k, v in (lb or {}).items() if v is not None} + # the LB fields set explicitly: they take precedence over the tuned configs + self._lb_set = frozenset(lb) + # above DYN_MAX tokens always the LB schedule (fp8 activations: never) + lb_min = int(lb.get("min", 256)) + lb_min = lb_min if 0 < lb_min <= DYN_MAX else DYN_MAX + 1 + self.lb_min = 0 if self.a8 else lb_min + self.lb_mt, self.lb_mt_table = _mt_table(lb.get("mt", 5)) + self.lb_mt_small = int(lb.get("mt_small", 3)) + self.lb_small_max = int(lb.get("small_max", 512)) + self.lb_npp = int(lb.get("npp", 2)) + lb_q = int(lb.get("q", 3)) + nck = mega_moe_tp_consts(model_dim, inter_dim, 1, 1)["NCK"] + gpc = gemm2_chunk_groups(model_dim, inter_dim) + step = gemm2_group_step(model_dim, inter_dim) + valid = [d for d in range(1, nck + 1) if nck % d == 0 and d * gpc % step == 0] + self.lb_q = max((d for d in valid if d <= lb_q), default=min(valid)) + self._lb_q_valid = valid + self.ar = comm_mode == "ar" + self.rank, self.tp = int(rank), int(world_size) + self.H, self.I, self.E, self.K = model_dim, inter_dim, experts, topk + self.mmax = int(max_local_tokens) + device = torch.device(device if device is not None else "cuda") + if device.index is None: + device = torch.device(device.type, torch.cuda.current_device()) + self.device = device + self.activation = activation + self.situ = (float(situ_beta), float(situ_linear_beta)) + if swiglu_limit is None: + swiglu_limit = 7.0 if activation == "swiglu" else float("inf") + self.swiglu_limit = float(swiglu_limit) + self._bind_weights(w1, w1_scale, w2, w2_scale) + + props = torch.cuda.get_device_properties(self.device) + self.n_cta = int(props.multi_processor_count) + if self.n_cta > NCTA_MAX: + raise ValueError(f"at most {NCTA_MAX} CTAs (flag layout)") + self.gfx = getattr(props, "gcnArchName", "").split(":")[0] + if self.gfx != "gfx950": + raise ValueError(f"fused TP MegaMoE needs gfx950, not {self.gfx}") + if fit_max_local: + self.mmax = self.capacity(self.mmax) + self.agr = max(1, ceildiv(self.mmax, self.n_cta)) + self._check_limits() + if not self._fits(self.mmax): + ok = self.mmax + while ok > 1 and not self._fits(ok): + ok -= max(1, ok // 64) + raise MegaMoeTPLDSError( + f"max_local_tokens={self.mmax} does not fit in LDS for h{model_dim} " + f"i{inter_dim} e{experts} k{topk} tp{world_size}: at most {ok}" + ) + + tot = self.mmax * self.tp + H, K = model_dim, topk + arena = SymmetricArena(group=group, device=self.device) + self._x = arena.reserve("x", (tot, H if self.a8 else H // 2), torch.uint8) + self._xs = arena.reserve("xs", (tot, H // 32), torch.uint8) + self._ids = arena.reserve("ids", (2, tot, 4 * K), torch.int32) + self._w = arena.reserve("w", (2, tot, K), torch.float32) + self._recv = arena.reserve( + "recv", (self.tp, tot if self.ar else self.mmax, H), torch.bfloat16 + ) + self._flag = arena.reserve("flag", (FLAG_INTS,), torch.int32) + self._yall = arena.reserve( + "yall", (tot, H) if self.ar else (1,), torch.bfloat16 + ) + tn_ok = comm_mode == "ag_rs" + self._qall = arena.reserve( + "qall", (tot * H * 3,) if tn_ok else (1,), torch.uint8 + ) + self._sall = arena.reserve("sall", (tot,) if tn_ok else (1,), torch.float32) + # the fused tail's norm (compiled in; part of the launcher key) + self.tn_eps = float(tail_eps) + self.tn_gemma = bool(tail_gemma) + # tail q/scales (bf16: + rows) of own rows only, at their [T, H] position; the other + # ranks' rows are not written (no peer stores / waits). Default: an SP caller (ATOM) + # consumes its own rows only; False: every rank's rows (the caller needs them all) + self.tn_local = True + arena.commit() + self.arena = arena + self.ctrl = torch.zeros(CTRL_INTS, dtype=torch.int32, device=self.device) + self.routes = torch.zeros( + (tot * topk + 1, H), dtype=torch.bfloat16, device=self.device + ) + self.y = torch.empty( + (tot if self.ar else self.mmax, H), dtype=torch.bfloat16, device=self.device + ) + + self.dyn_max = min(tot, DYN_MAX) + # partial rows of the GEMM2 pieces past the first (dynamic schedule) + extra = (inter_dim // 128 - 1) * self.dyn_max * topk + if extra * H * 2 >= 1 << 31: + raise ValueError("split-expert partial rows exceed 32-bit buffer offsets") + self.proutes = torch.zeros( + (extra + 1, H), dtype=torch.bfloat16, device=self.device + ) + self.xg = torch.empty( + tot * topk * (inter_dim // 2 + inter_dim // 32 + 8), + dtype=torch.uint8, + device=self.device, + ) + self._cfgs: dict = {} + self._launchers: dict = {} + self._armed: set = set() + self._args = (None, None) + + def capacity(self, upto: int) -> int: + """The largest local token count <= upto the kernel takes (0: none).""" + + def ok(m): + return not self._limit_errors(m) and self._fits(m) + + if ok(upto): + return upto + lo, hi = 0, upto # ok(lo) or lo == 0; not ok(hi) + while hi - lo > 1: + mid = (lo + hi) // 2 + lo, hi = (mid, hi) if ok(mid) else (lo, mid) + return lo + + def _check_limits(self) -> None: + """The kernel's fixed-size tables and 32-bit offsets, checked up front.""" + bad = self._limit_errors(self.mmax) + if bad: + raise ValueError( + f"MegaMoeTP h{self.H} i{self.I} e{self.E} k{self.K} tp{self.tp} " + f"max_local_tokens={self.mmax}: need " + ", ".join(bad) + ) + + def _limit_errors(self, mmax: int) -> list: + tot, H, K = mmax * self.tp, self.H, self.K + agr = max(1, ceildiv(mmax, self.n_cta)) + bad = [] + if agr > (4 if self.a8 else 8): + bad.append(f"max_local_tokens <= {(4 if self.a8 else 8) * self.n_cta}") + if not self.ar and ceildiv(mmax * K, 256) > min(NMETA_CAP, self.n_cta): + bad.append(f"max_local_tokens * topk <= {256 * min(NMETA_CAP, self.n_cta)}") + if self.lb_min and self.lb_min <= tot and self.I // 128 >= 2: + mts = [self.lb_mt, *(n for _, n in self.lb_mt_table)] + if self.lb_small_max >= self.lb_min: + mts.append(self.lb_mt_small) + mt = min(self._fit_mt(t, lb=True, mmax=mmax) for t in mts) + if self.E + ceildiv(tot * K, 16 * mt) + 1 > NCHLB_MAX: + bad.append(f"LB row chunks <= {NCHLB_MAX} (larger LB row tiles)") + for what, n in ( + ("route rows", tot * K * 2 * H), + ("gathered rows", self.tp * tot * H * 2), + ("tail rows", tot * H * 3), + ("intermediate rows", tot * K * (self.I // 2 + self.I // 32 + 8)), + ): + if n >= 1 << 31: + bad.append(f"{what} within 32-bit offsets") + return bad + + def _bind_weights(self, w1, w1_scale, w2, w2_scale) -> None: + w = tuple(t.view(torch.uint8) for t in (w1, w1_scale, w2, w2_scale)) + H, I, E = self.H, self.I, self.E + for name, t, n in ( + ("w1", w[0], E * 2 * I * H // 2), + ("w1_scale", w[1], E * 2 * I * H // 32), + ("w2", w[2], E * H * I // 2), + ("w2_scale", w[3], E * H * I // 32), + ): + if t.device != self.device or not t.is_contiguous() or t.numel() < n: + raise ValueError( + f"{name}: need a contiguous tensor of >= {n} bytes on {self.device}" + ) + self.w1, self.w1s, self.w2, self.w2s = w + + def set_weights(self, w1, w1_scale, w2, w2_scale) -> None: + self._bind_weights(w1, w1_scale, w2, w2_scale) + self._args = (None, None) + + def _chunks(self, mt: int, mmax: int = 0, lb: bool = False) -> tuple[int, int]: + rch = 16 * mt + tot = (mmax or self.mmax) * self.tp + if not lb: + tot = min(tot, DYN_MAX) + return rch, self.E + ceildiv(tot * self.K, rch) + 1 + + def _consts(self, mt: int, nab: int = 4, mmax: int = 0, lb: bool = False) -> dict: + mmax = mmax or self.mmax + return mega_moe_tp_consts( + self.H, + self.I, + mt, + mmax * self.tp, + max(1, ceildiv(mmax, self.n_cta)), + self.E, + nab, + self._chunks(mt, mmax, lb)[1], + self.a8, + lb, + ) + + def _nab(self, mt: int, mmax: int = 0, lb: bool = False) -> int: + if lb: + return 3 + return 4 if self._consts(mt, 4, mmax)["LDS_BYTES"] <= LDS_LIMIT else 3 + + def _lds(self, mt: int, mmax: int = 0, lb: bool = False) -> int: + nab = self._nab(mt, mmax, lb) + return self._consts(mt, nab, mmax, lb)["LDS_BYTES"] + + def _fits(self, mmax: int) -> bool: + # the smallest row tile of each schedule these batches can take fits in LDS + lb = bool(self.lb_min) and mmax * self.tp >= self.lb_min + return self._lds(1, mmax) <= LDS_LIMIT and ( + not lb or self._lds(1, mmax, lb=True) <= LDS_LIMIT + ) + + def _fit_mt(self, mt: int, lb: bool = False, mmax: int = 0) -> int: + mt = max(1, min(6, int(mt))) + while mt > 1 and self._lds(mt, mmax, lb=lb) > LDS_LIMIT: + mt -= 1 + return mt + + def _fit_npp(self, npp: int) -> int: + ok = npp >= 1 and 4 % (2 * npp) == 0 and (self.H // 128 * npp) % 4 == 0 + return npp if ok else 1 + + def _tuned(self, lb: bool, tot: int) -> dict: + """The tuned fields of the nearest tuned batch (same schedule, <= tot, inside the + tuned range: common.nearest), else {} (the defaults).""" + key = { + **device_key(self.device), + "model_dim": self.H, + "inter_dim": self.I, + "experts": self.E, + "topk": self.K, + "tp": self.tp, + "comm_mode": self.mode, + "act": "fp8" if self.a8 else "fp4", + "lb": int(lb), + } + row = nearest( + tuned_rows(tuned_config("mega_moe_tp"), key, "tot"), tot + ) + if row is None: + return {} + return {k: int(row[k]) for k in _TUNED_FIELDS if row.get(k, "") != ""} + + def _lb_mt_ok(self, mt: int) -> int: + # LB row chunks of the largest batch fit the kernel's table: grow the tile if not + tot = self.mmax * self.tp + while mt < 6 and self.E + ceildiv(tot * self.K, 16 * mt) + 1 > NCHLB_MAX: + mt += 1 + return mt + + def config(self, m: int) -> LaunchCfg: + cfg = self._cfgs.get(m) + if cfg is None: + tot = m * self.tp + lb = bool(self.lb_min) and tot >= self.lb_min + fp8 = not self.comm_bf16 + tu = self._tuned(lb, tot) + if lb: + small = tot <= self.lb_small_max + # an explicit config field, else the tuned one, else the default + if ("mt_small" if small else "mt") in self._lb_set: + tu.pop("mt", None) + if "npp" in self._lb_set and not small: + tu.pop("npp", None) + if "q" in self._lb_set: + tu.pop("q", None) + mt = next((n for t, n in self.lb_mt_table if tot <= t), self.lb_mt) + lp = tu.get("lp", 0 if tot <= LB_PF_MAX else 3) + q = tu.get("q", 0) + cfg = LaunchCfg( + self._fit_mt( + self._lb_mt_ok(tu.get("mt", self.lb_mt_small if small else mt)), + lb=True, + ), + npp=self._fit_npp(tu.get("npp", 2 if small else self.lb_npp)), + ag8=self.ar and bool(self.ag_auto) and tot >= self.ag_auto, + lb=True, + pf=bool(tu.get("pf", tot <= LB_PF_MAX)), + lp=lp if self.I // 128 % max(lp, 1) == 0 else 0, + q=q if q in self._lb_q_valid else 0, + ) + elif tot <= self.dyn_max: + cfg = LaunchCfg( + self._fit_mt( + tu.get("mt", 1 if tot <= 32 else 2 if tot <= 64 else 3) + ), + # LL packets carry FP8 rows only (the tuned rows are fp8-comm tunings) + ll=fp8 and bool(tu.get("ll", tot <= 256)), + llr=fp8 and bool(tu.get("llr", tot <= 128)), + ag8=self.ar and bool(self.ag_auto) and tot >= self.ag_auto, + ) + else: + raise ValueError( + f'act_dtype="fp8": {tot} tokens are beyond the dynamic schedule' + ) + self._cfgs[m] = cfg + return cfg + + def _arll(self, cfg: LaunchCfg) -> bool: + return self.ar and cfg.ll and cfg.llr + + def _ag8(self, cfg: LaunchCfg) -> bool: + return self.ar and (self.ag_fp8 or cfg.ag8) and not self._arll(cfg) + + def _key(self, cfg: LaunchCfg): + return (cfg, self.tn_eps, self.tn_gemma, self.tn_local) if cfg.tn else cfg + + def _launcher(self, cfg: LaunchCfg): + fn = self._launchers.get(self._key(cfg)) + if fn is None: + rch, nch = self._chunks(cfg.mt, lb=cfg.lb) + fn = compile_mega_moe_tp( + H=self.H, + I=self.I, + TOPK=self.K, + MT=cfg.mt, + TMAX=self.mmax * self.tp, + E=self.E, + act=self.activation, + situ_beta=self.situ[0], + situ_linear_beta=self.situ[1], + swiglu_limit=self.swiglu_limit, + route_fp8=cfg.ll and cfg.llr, + agr=self.agr, + tp=self.tp, + ar=self.ar, + nab=self._nab(cfg.mt, lb=cfg.lb), + npp=cfg.npp, + ll_rs=cfg.ll, + ll_route=cfg.ll and cfg.llr, + comm_bf16=self.comm_bf16, + ag8=self._ag8(cfg), + rch=rch, + nch=nch, + a8=self.a8, + lb=cfg.lb, + lbq=(cfg.q or self.lb_q) if cfg.lb else 1, + tn=cfg.tn, + tn_eps=self.tn_eps, + tn_gemma=self.tn_gemma, + tn_local=self.tn_local, + lbpf=cfg.pf, + lbp=cfg.lp, + ) + self._launchers[self._key(cfg)] = fn + return fn + + def _local_tokens( + self, x, topk_weights, topk_ids, limit: int = 0, uneven: bool = False + ) -> int: + """forward's input checks -> its local token count (limit: the largest, default + max_local_tokens; uneven: ar batches need not be a multiple of tp, their local + count rounded up)""" + rows = int(x.shape[0]) if x.dim() == 2 else -1 + for name, t in ( + ("x", x), + ("topk_weights", topk_weights), + ("topk_ids", topk_ids), + ): + if t.device != self.device: + raise ValueError(f"{name} is on {t.device}, the layer on {self.device}") + if x.dim() != 2 or x.shape[1] != self.H or x.dtype != torch.bfloat16: + raise ValueError( + f"x: need bf16 [tokens, {self.H}], got {x.dtype} {tuple(x.shape)}" + ) + want = (rows, self.K) + if tuple(topk_ids.shape) != want or tuple(topk_weights.shape) != want: + raise ValueError( + f"topk_ids / topk_weights: need {list(want)}, got " + f"{list(topk_ids.shape)} / {list(topk_weights.shape)}" + ) + if topk_ids.dtype not in (torch.int32, torch.int64): + raise ValueError(f"topk_ids: need int32 (or int64), got {topk_ids.dtype}") + if not topk_weights.is_floating_point(): + raise ValueError(f"topk_weights: need float32, got {topk_weights.dtype}") + m = rows + if self.ar: + if m % self.tp and not uneven: + raise ValueError( + f"{self.mode}: {m} tokens are not a multiple of tp={self.tp}" + ) + m = ceildiv(m, self.tp) + limit = limit or self.mmax + if m > limit: + raise ValueError(f"{m} local tokens exceed max_local_tokens {limit}") + if ( + self.check + and dist.is_initialized() + and not torch.cuda.is_current_stream_capturing() + ): + self._check_replicas(m, x, topk_weights, topk_ids) + return m + + def _check_replicas(self, m, x, topk_weights, topk_ids) -> None: + if self.ar: + fp = [ + float(t.double().sum()) + for t in (x, topk_weights, topk_ids, (x.float() * x.float()).sum(1)) + ] + else: + fp = [] + got = [None] * self.tp + dist.all_gather_object(got, (m, fp), group=self.arena.group) + if len({g[0] for g in got}) > 1: + raise ValueError( + f"{self.mode}: local token counts differ across ranks: " + f"{[g[0] for g in got]}" + ) + if self.ar and any(g[1] != fp for g in got): + raise ValueError( + f"{self.mode}: x / topk_ids / topk_weights must be the same on every " + "rank (replicated input)" + ) + + def prepare( + self, local_tokens, tail: bool = False, tail_bf16: bool = False, warmup=True + ) -> None: + """Collective: compile and arm the launch configs of these local token counts + (and their fused-tail variants), then (warmup) run each once eagerly and check + every rank's watchdog: on a timeout reset() and retry once, else raise.""" + runs = [] + for m in sorted({int(m) for m in local_tokens if 0 < m <= self.mmax}): + cfg = self.config(m) + self._arm(cfg, m) + runs.append((m, 0)) + if self.tail_ok(m): + for on, tn in ((tail, 1), (tail_bf16, 2)): + if on: + self._arm(dataclasses.replace(cfg, tn=tn), m) + runs.append((m, tn)) + if not warmup: + return + for attempt in range(2): + for m, tn in runs: + self._warmup(m, tn) + torch.cuda.synchronize(self.device) + errs = [None] * self.tp + dist.all_gather_object(errs, self.poll_errors(), group=self.arena.group) + if not any(errs): + return + self.reset() + raise RuntimeError(f"MegaMoeTP.prepare: warmup launches timed out ({errs})") + + def _warmup(self, m: int, tn: int = 0) -> None: + rows = m * self.tp if self.ar else m + dev = self.device + x = torch.zeros((rows, self.H), dtype=torch.bfloat16, device=dev) + ids = torch.arange(rows * self.K, dtype=torch.int32, device=dev) % self.E + w = torch.full((rows, self.K), 1.0 / self.K, device=dev) + tail = None + if tn: + res = torch.zeros((m, self.H), dtype=torch.bfloat16, device=dev) + nw = torch.zeros(self.H, dtype=torch.bfloat16, device=dev) + tail = (res, res, nw) + self.forward(x, w, ids.view(rows, self.K), tail=tail, bf16=tn == 2) + + def tail_ok(self, m: int) -> bool: + return ( + self.mode == "ag_rs" + and 0 < m <= min(TN_MAX, self.mmax) + and self.config(m).lb + ) + + def _arm(self, cfg: LaunchCfg, m: int = 0) -> None: + if self._key(cfg) in self._armed: + return + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError( + f"MegaMoeTP: launch config {cfg} first used inside CUDA graph " + "capture; call prepare(local_tokens) (or run it eagerly) first" + ) + fn = self._launcher(cfg) + m = m or next(m for m, c in self._cfgs.items() if c == cfg) + args = self._launch_args(m, cfg, self.y, None, None, None) + _preload_compiled(fn, *args, torch.cuda.current_stream()) + torch.cuda.synchronize(self.device) + self.arena.barrier() + self._armed.add(self._key(cfg)) + + def _launch_args(self, m, cfg, y, x, ids, tw, tail=None): + peers = [int(b) for b in self.arena.base_ptrs] + [0] * (MAX_TP - self.tp) + + def ptr(t): + return 0 if t is None else t.data_ptr() + + return ( + self.w1.data_ptr(), + self.w1s.data_ptr(), + self.w2.data_ptr(), + self.w2s.data_ptr(), + ptr(x), + ptr(ids), + ptr(tw), + y.data_ptr(), + self.routes.data_ptr(), + self.proutes.data_ptr(), + self.ctrl.data_ptr(), + *peers, + self._x.offset, + self._xs.offset, + self._ids.offset, + self._w.offset, + self._recv.offset, + self._flag.offset, + self._yall.offset, + self.rank, + self.tp, + m, + self.mmax, + self.xg.data_ptr(), + *(ptr(t) for t in (tail or (None, None, None))), + self._qall.offset, + self._sall.offset, + self.n_cta, + ) + + def _check_out(self, out, rows: int) -> None: + if out is not None and ( + tuple(out.shape) != (rows, self.H) + or out.dtype != torch.bfloat16 + or out.device != self.device + or not out.is_contiguous() + ): + raise ValueError(f"out: need a contiguous bf16 [{rows}, {self.H}]") + + def _check_tail(self, tail, m: int) -> None: + res_in, res_out, nw = tail + for t, shp in ((res_in, (m, self.H)), (res_out, (m, self.H)), (nw, (self.H,))): + if ( + tuple(t.shape) != shp + or t.dtype != torch.bfloat16 + or t.device != self.device + or not t.is_contiguous() + ): + raise ValueError(f"tail: need contiguous bf16 {shp} on {self.device}") + + def forward( + self, + x_local: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + out: torch.Tensor | None = None, + tail=None, + bf16: bool = False, + ): + m = self._local_tokens(x_local, topk_weights, topk_ids) + rows = m * self.tp if self.ar else m + self._check_out(out, rows) + if m == 0: + return x_local.new_empty((0, self.H)) if out is None else out + x = x_local.contiguous() + ids = topk_ids.to(torch.int32).contiguous() + tw = topk_weights.to(torch.float32).contiguous() + cfg = self.config(m) + if tail is not None: + if not self.tail_ok(m): + raise ValueError( + "tail: ag_rs, the LB schedule, at most TN_MAX local tokens" + ) + self._check_tail(tail, m) + cfg = dataclasses.replace(cfg, tn=2 if bf16 else 1) + local_y = not self.ar or self._arll(cfg) or self._ag8(cfg) + y = out if out is not None and local_y else self.y + tptr = tuple(t.data_ptr() for t in tail) if tail is not None else () + key = (m, x.data_ptr(), ids.data_ptr(), tw.data_ptr(), y.data_ptr(), cfg, tptr) + if self._args[0] != key: + args = self._launch_args(m, cfg, y, x, ids, tw, tail) + self._args = (key, (args, x, ids, tw)) + self._arm(cfg, m) + _run_compiled( + self._launcher(cfg), *self._args[1][0], torch.cuda.current_stream() + ) + if self.check and not torch.cuda.is_current_stream_capturing(): + self.check_errors() + if not local_y: + y = self._yall.local[:rows] + return y if out is None else out.copy_(y) + y = y[:rows] if out is None else out + if tail is not None: + T_ = m * self.tp + qall = self._qall.local + q = qall[: T_ * self.H].view(T_, self.H) + if bf16: + b0 = self.mmax * self.tp * self.H + b = ( + qall[b0 : b0 + 2 * T_ * self.H] + .view(torch.bfloat16) + .view(T_, self.H) + ) + return y, q, self._sall.local[:T_], b + return y, q, self._sall.local[:T_] + return y + + __call__ = forward + + def clear_errors(self) -> None: + self.ctrl[CTRL_ERR] = 0 + + def poll_errors(self) -> int: + return int(self.ctrl[CTRL_ERR].item()) + + def error_flag(self) -> torch.Tensor: + """The sticky watchdog bits as a 1-element device tensor (no sync).""" + return self.ctrl[CTRL_ERR : CTRL_ERR + 1] + + def check_errors(self) -> None: + err = self.poll_errors() + if err: + self.clear_errors() + what = [n for b, n in _ERR_NAMES if err & b] + raise RuntimeError( + f"MegaMoeTP rank {self.rank}: a wait inside the kernel timed out " + f"({'|'.join(what)}); the output is invalid. Ranks out of step " + "(different call counts or local token counts) need reset()." + ) + + def reset(self) -> None: + torch.cuda.synchronize(self.device) + self.arena.barrier() + self.arena.storage.zero_() + self.ctrl.zero_() + self.routes.zero_() + self.proutes.zero_() + torch.cuda.synchronize(self.device) + self.arena.barrier() diff --git a/amd_kernels/moe/mega_moe_tp/mega_moe_tp_bigm_tuned.csv b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_bigm_tuned.csv new file mode 100644 index 0000000..384ddc6 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_bigm_tuned.csv @@ -0,0 +1,6 @@ +gfx,cu_num,model_dim,inter_dim,experts,topk,tp,tokens,chunks,opts,us,q_rel,route_match +gfx950,256,6144,768,129,5,4,8192,"1,1",BIGM_NSBA=2;BIGM_ORD=3;BIGM_R8=1;BIGM_SXW=0;BIGM_BS2=6;BIGM_E4=1,1043.0,0.0856, +gfx950,256,6144,768,129,5,4,16384,"1,1",BIGM_NSBA=2;BIGM_ORD=4;BIGM_UA=24;BIGM_R8=1;BIGM_SXW=0;BIGM_BS2=6;BIGM_E4=1,1650.6,0.0856, +gfx950,256,6144,768,129,5,4,32768,"2,5,2",BIGM_NSBA=2;BIGM_ORD=4;BIGM_UA=20;BIGM_R8=1;BIGM_SXW=0;BIGM_BS2=6;BIGM_E4=1,2862.2,0.0856, +gfx950,256,4096,512,257,6,4,32768,"2,2,2,1",BIGM_NSBA=2;BIGM_ORD=4;BIGM_UA=20;BIGM_R8=1;BIGM_SXW=0;BIGM_BS2=6,3029.0,0.0270, +gfx950,256,6144,512,257,9,4,32768,"2,2,2,2,1",BIGM_NSBA=2;BIGM_ORD=4;BIGM_UA=20;BIGM_R8=1;BIGM_SXW=0;BIGM_BS2=6,4728.0,0.0271, diff --git a/amd_kernels/moe/mega_moe_tp/mega_moe_tp_config.py b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_config.py new file mode 100644 index 0000000..fc38817 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_config.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/mega_moe_tp_config.py (ROCm/aiter edf58c67a). +"""Compile-time layout (flag / ctrl / LDS slots), shape limits and the per-instance +KernelCtx shared by the parts of the fused TP MegaMoE kernel.""" + +from __future__ import annotations + +import functools +import math +import struct +import types + +import flydsl.expr as fx + +from .fly_utils import ceildiv +from .common import ( + AUX_SC1, + AUX_SYS, + DEADLINE, + MAX_TP, +) + +# tokens per launch (all ranks) up to which the host may pick the dynamic schedule +DYN_MAX, NCTA_MAX, NCK_MAX, NCHA_MAX = 256, 256, 64, 64 +# routing-meta flags per rank (one per sending CTA: m * topk <= 64 * NMETA_CAP) +NMETA_CAP = 256 +# Flag arena (ints, written by peers, one writer each, values = launch epoch, compared +# wrap-safe: see common.before): +# FLAG_RDY + c*MAX_TP + r rank r pushed RS chunk c +# FLAG_AGM + r*NMETA + b rank r's routing, CTA b's part +# FLAG_YAG + (c*MAX_TP + r)*NCTA + b AR: rank r sent output chunk c of CTA b's rows +# FLAG_AGQ + q*MAX_TP + r every CTA of rank r landed K-chunk q +# (per rank, not per row: per-row polls by +# every CTA flood the lines peers write) +# FLAG_TN + r*TN_MAX + i tail (tn): rank r sent its output row i +FLAG_RDY = 0 +FLAG_AGM = FLAG_RDY + MAX_TP * NCK_MAX +FLAG_YAG = FLAG_AGM + MAX_TP * NMETA_CAP +FLAG_AGQ = FLAG_YAG + NCK_MAX * MAX_TP * NCTA_MAX +TN_MAX = 2048 +FLAG_TN = FLAG_AGQ + NCHA_MAX * MAX_TP +FLAG_INTS = FLAG_TN + MAX_TP * TN_MAX + +# Local control ints. Polled/atomic counters sit on their own 128 B lines (stride +# LRDY_STRIDE): same-line device atomics from every CTA serialize. Counters are reset +# by their last arriver (a timed-out launch leaves residue: reset()). +CTRL_ERR = 20 +CTRL_XF = 32 +ERR_FLAG, ERR_META, ERR_CHUNK, ERR_COMM, ERR_YAG = 1, 2, 4, 8, 16 +N_XCD = 8 +LRDY_STRIDE = 32 +CTRL_LRDY = 64 + 2 * NCK_MAX +CTRL_SC = CTRL_LRDY + NCK_MAX * LRDY_STRIDE +# per RS chunk: one line per XCD (CTAs done), all XCDs done, push units done +SC_ALL, SC_PUSH = N_XCD, N_XCD + 1 +SC_LINES = SC_PUSH + 1 +CTRL_EPB = CTRL_SC + NCK_MAX * SC_LINES * LRDY_STRIDE +CTRL_XES = CTRL_EPB + NCTA_MAX +CTRL_AGX = CTRL_XES + N_XCD * LRDY_STRIDE +CTRL_AGG = CTRL_AGX + NCHA_MAX * N_XCD * LRDY_STRIDE +CTRL_CLM = CTRL_AGG + NCHA_MAX * LRDY_STRIDE +# LB schedule: launches counted by CTRL_LBSEQ (bumped by CTA 0 once every CTA read it); +# its parity picks the bank of the counters below, and every LB launch zeroes the other +# bank for the next one. +# CTRL_LBR + (bank*N_XCD + x)*LRDY_STRIDE CTAs of XCD x whose route list slice landed +# CTRL_COLC + (bank*NCK_MAX + c)*LRDY_STRIDE GEMM2 units done of column group c +# CTRL_G1C + (bank*NCHLB_MAX + j)*G1C_STRIDE GEMM1 pieces done of row chunk j +# CTRL_TNC + i tail (tn): CTAs done with row i +NCHLB_MAX = 2048 +G1C_STRIDE = 16 +CTRL_LBSEQ = CTRL_CLM + 2 * LRDY_STRIDE +CTRL_LBR = CTRL_LBSEQ + LRDY_STRIDE +CTRL_COLC = CTRL_LBR + 2 * N_XCD * LRDY_STRIDE +CTRL_G1C = CTRL_COLC + 2 * NCK_MAX * LRDY_STRIDE +CTRL_TNC = CTRL_G1C + 2 * NCHLB_MAX * G1C_STRIDE +CTRL_INTS = CTRL_TNC + TN_MAX +# unit records (expert, i0, icnt, kind); LB: kind = UNIT_G1X | piece << 8 | chunk << XQ_SHIFT +# (a GEMM1 piece exporting its intermediate) +UNIT_G1X = 3 +XQ_SHIFT = 20 + +NW = 4 +NT = NW * 64 +NTT = NT + 4 * 64 +NSK = 4 +SLOT = 4 * 1024 + 2 * 256 +OPS = 6 +KCS = 2 +ALOAD_DEPTH = 3 +POLL_SLEEP = 16 + + +def mega_moe_tp_shape_supported(model_dim: int, inter_dim: int, tp: int) -> bool: + return ( + model_dim % 512 == 0 + and (model_dim // 64) % NW == 0 + and model_dim // 256 <= NCHA_MAX + and model_dim // 256 // gemm2_chunk_groups(model_dim, inter_dim) <= 31 + and inter_dim % 128 == 0 + and inter_dim >= 256 + and 1 <= tp <= MAX_TP + ) + + +def _nsk2_for(nks: int, g2: int) -> int: + if g2 % (NSK * nks // math.gcd(NSK, nks) // nks) == 0: + return NSK + return 3 if nks % 3 == 0 else 2 + + +def gemm2_group_step(H: int, I: int) -> int: + ks2, g2 = I // 128, H // 64 // NW + nsk2 = _nsk2_for(ks2, g2) + return nsk2 * ks2 // math.gcd(nsk2, ks2) // ks2 + + +def gemm2_chunk_groups(H: int, I: int) -> int: + return 2 if (H // 64 // NW) % 2 == 0 else 1 + + +@functools.cache +def mega_moe_tp_consts( + H: int, + I: int, + MT: int, + TMAX: int, + agr: int = 1, + dyn_e: int = 0, + nab: int = 4, + nch: int = 0, + a8: bool = False, + lb: bool = False, +) -> dict: + RG = MT * 16 + KS1 = H // 128 + KS2 = I // 128 + G2 = H // 64 // NW + NSC_BLK = ceildiv(RG, 64) + acb = KCS * (128 if a8 else 64) + NA_ROWOPS = RG * acb // 1024 + GPC = gemm2_chunk_groups(H, I) + c = { + "RG": RG, + "KS1": KS1, + "NCH": KS1 // KCS, + "KS2": KS2, + "G2": G2, + "CH1": ceildiv(H // 32, 8), + "CH2": ceildiv(I // 32, 8), + "ACB": acb, + "XB": H if a8 else H // 2, + # row pad 16: every intermediate row 16 B aligned for the 128-bit LDS ops + "SI_STRIDE": (I if a8 else I // 2) + 16, + "NA_ROWOPS": NA_ROWOPS, + "NSC_BLK": NSC_BLK, + "NA_L": NA_ROWOPS + NSC_BLK * KCS, + "GPC": GPC, + "NCK": G2 // GPC, + "CW": GPC * NW * 64, + "NAB": nab, + "NBW": ceildiv(dyn_e, 32), + } + off = 0 + + def take(n): + nonlocal off + off = ceildiv(off, 16) * 16 + start, off = off, off + n + return start + + c["L_RING"] = take(NW * NSK * SLOT) + c["L_A"] = take(nab * RG * acb) + c["L_AS"] = take(nab * KCS * NSC_BLK * 64 * 4) + c["L_B1"] = c["L_A"] + c["L_B1S"] = c["L_A"] + RG * c["SI_STRIDE"] + c["B1_FITS"] = ( + c["L_B1S"] + RG * (I // 32) <= c["L_AS"] + nab * KCS * NSC_BLK * 64 * 4 + ) + c["L_INTER"] = take( + max(RG * c["SI_STRIDE"], agr * (c["XB"] + H // 32) - RG * (I // 32)) + ) + c["L_INTERS"] = take(RG * (I // 32)) + nrix = 256 if lb else min(TMAX, DYN_MAX) + c["L_RIX"] = take(nrix * 4) + c["L_WT"] = take(nrix * 4) + c["L_CTL"] = take(128 * 4) + c["L_DYN"] = take((2 * c["NBW"] + dyn_e) * 4) + c["L_DCNT"] = take(dyn_e * 4) if nch else 0 + c["L_DCH"] = take((nch + 2 * NW) * 4) if nch else 0 + c["L_EOFF"] = take(dyn_e * 4) if lb else 0 + c["L_DPRE"] = take(dyn_e * 4) if lb else 0 + c["LDS_BYTES"] = ceildiv(off, 128) * 128 + assert H // 256 <= NCHA_MAX + assert KS1 % NSK == 0 and NSK % KCS == 0 + assert (NSK - 1) * OPS <= 63 and c["NA_L"] * (max(ALOAD_DEPTH, nab) - 1) <= 63 + assert not lb or RG <= 256 + assert c["NCK"] <= NCK_MAX and c["NCK"] <= 31 + return c + + +# LDS control slots (ints at L_CTL); NW-wide ones: C_DONE, C_ASEQ, C_AFREE, C_QBASE (one +# per compute wave), C_MBOX (one per wave), C_UNIT (6: unit range + first record), C_CLAIM. +# C_GEXP: routes key (expert + 1, plus the row chunk) held by L_RIX / L_WT / C_CNT. +C_CNT, C_BARCNT, C_BARGEN, C_LRED, C_PULL, C_EPOCH, C_USEQ, C_UROWS = range(8) +C_NSIG, C_LQ, C_LPUB, C_GEXP, C_DONE = range(8, 13) +C_ASEQ, C_AFREE, C_QBASE, C_MBOX = 16, 20, 24, 28 +C_FBITS, C_YAGM, C_NCH, C_AGFREE, C_PLAN, C_NACT = 36, 39, 40, 41, 42, 43 +C_ARDY, C_DYNP, C_CDONE, C_UNIT = 44, 45, 46, 48 +C_CLAIM, C_PRDY, C_FRDY, C_PBITS, C_RLAND = 56, 60, 61, 62, 63 +# LB: C_LBB bank of this launch's LB counters, C_LBN route lists seen, C_G1FIN no more +# GEMM1 units, C_TNL tail row taken, C_G2*: GEMM2 prefetch ring (C_G2U: 2 x 4 ints), +# C_PFW: per compute wave, the ring holds the next unit's first slots, C_LBJ2: first row +# chunk run as single-block pieces (LB mix). +C_LBB, C_LBN, C_G1FIN, C_TNL = 64, 65, 67, 69 +C_G2RDY, C_G2FREE, C_G2END, C_G2V, C_G2U = 76, 77, 78, 79, 80 +C_G2LAST, C_G2VN, C_PFW, C_LBJ2 = 88, 89, 90, 94 +# per-CTA unit list (dynamic / LB): UL_MAX records of 4 ints +C_UL = 96 +UL_MAX = (128 - C_UL) // 4 + + +class KernelCtx(types.SimpleNamespace): + """Compile-time names of one kernel instance: the compile_mega_moe_tp arguments, + the values derived from them and the device functions of the parts built so far.""" + + def get(self, names: str) -> tuple: + return tuple(getattr(self, n) for n in names.split()) + + def add(self, env: dict) -> None: + vars(self).update((n, v) for n, v in env.items() if n != "kc") + + +def kernel_config( + *, + H: int, + I: int, + TOPK: int, + MT: int, + TMAX: int, + E: int, + act: str = "silu", + situ_beta: float = 1.0, + situ_linear_beta: float = 1.0, + swiglu_limit: float | None = None, + route_fp8: bool = False, + agr: int = 1, + tp: int = MAX_TP, + ar: bool = False, + nab: int = 4, + npp: int = 1, + ll_rs: bool = False, + ll_route: bool = False, + comm_bf16: bool = False, + ag8: bool = False, + rch: int = 16, + nch: int = 0, + a8: bool = False, + lb: bool = False, + lbq: int = 1, + tn: int = 0, + tn_eps: float = 1e-6, + tn_gemma: bool = True, + tn_local: bool = False, + lbpf: bool = False, + lbp: int = 0, +): + """The derived compile-time values of one compile_mega_moe_tp instance.""" + if swiglu_limit is None: + swiglu_limit = 7.0 if act == "swiglu" else float("inf") + TN = tn in (1, 2) + assert not TN or not ar + nsk = NSK + c = mega_moe_tp_consts(H, I, MT, TMAX, agr, E, nab, nch, a8, lb) + RG, KS1, NCH, KS2, G2 = c["RG"], c["KS1"], c["NCH"], c["KS2"], c["G2"] + CH1, CH2, SI_STRIDE = c["CH1"], c["CH2"], c["SI_STRIDE"] + ACB, XB = c["ACB"], c["XB"] + NA_ROWOPS, NSC_BLK, NA_L = c["NA_ROWOPS"], c["NSC_BLK"], c["NA_L"] + GPC, NCK, CW, NAB, NBW = c["GPC"], c["NCK"], c["CW"], c["NAB"], c["NBW"] + L_RING, L_A, L_AS, L_DYN = c["L_RING"], c["L_A"], c["L_AS"], c["L_DYN"] + L_INTER, L_INTERS, L_RIX, L_WT = c["L_INTER"], c["L_INTERS"], c["L_RIX"], c["L_WT"] + L_CTL, LDS_BYTES = c["L_CTL"], c["LDS_BYTES"] + L_B1, L_B1S = c["L_B1"], c["L_B1S"] + L_DCNT, L_DCH = c["L_DCNT"], c["L_DCH"] + L_EOFF, L_DPRE = c["L_EOFF"], c["L_DPRE"] + L_WSUM = L_DCH + nch * 4 + assert npp >= 1 and nsk % (npp * KCS) == 0 and (KS1 * npp) % nsk == 0 + assert 3 <= NAB <= 4 + VPL = ceildiv(CW // 8, 64) + SCAN_IT = ceildiv(min(TMAX, DYN_MAX) * TOPK // 4, NT) + SCAN_G = 16 + NPC = 1 if lb else KS2 + assert not (comm_bf16 and ll_rs) + DYN_PS = [p for p in range(1, KS2 + 1) if KS2 % p == 0] + DLL = bool(ll_rs and ll_route) + assert not DLL or route_fp8 + MLL = DLL and not ar + ARLL = DLL and ar + AG8 = bool(ag8 and ar and not ARLL) + NV = NCK + TPC = tp + # routing-meta flag slots per rank: one per sending CTA (64 lanes x 4 routes each) + NMETA = max(32, 1 << (ceildiv(TMAX // tp * TOPK, 256) - 1).bit_length()) + assert NMETA <= NMETA_CAP, "m * topk > 64 * NMETA_CAP" + assert not lb or (not ll_rs and not a8 and rch == RG) + assert not lb or nch <= NCHLB_MAX + LBQ = lbq if lb else 1 + assert NCK % LBQ == 0 + assert not lb or LBQ * GPC % gemm2_group_step(H, I) == 0 + NCG = NCK // LBQ + CGW = [LBQ] * NCG + _cg = lb and NCK == 12 and LBQ == 3 + if _cg: + CGW = [3, 3, 3, 2, 1] + NCG = len(CGW) + LB_B = min(ceildiv(TMAX * TOPK // 4, NT), 10) + LB_PS = [p for p in range(1, KS2 + 1) if KS2 % p == 0] + if lbp and lbp in LB_PS: + LB_PS = [lbp] + # LB mix: one round of 2-piece GEMM1 units; the chunks past it run as KS2 single-block + # pieces, H2 = KS2 // 2 (one 2-piece unit's work) on each CTA without a unit, one more + # on the first H2 * X CTAs + H2 = KS2 // 2 + LBMIX = ( + lb + and MT <= 3 + and len(LB_PS) > 1 + and 2 in LB_PS + and KS2 % 2 == 0 + and H2 < UL_MAX + ) + LBMIX_DIV = 4 + XLPR = (256 if a8 else 128) // 16 + assert agr <= 64 // XLPR + AUX_RT = AUX_SYS if lb else AUX_SC1 + ADEPTH = min(ALOAD_DEPTH, NAB - 1) if lb and NAB > 3 else ALOAD_DEPTH + LBPF = bool(lb and c["B1_FITS"] and lbpf) + MTSKIP = lb and MT >= 4 + # tail rows stay local (no peer writes / waits): the caller gathers q itself + TNL = TN and tn_local + name = ( + f"mega_moe_tp_fused_h{H}_i{I}_e{E}_k{TOPK}_mt{MT}_t{TMAX}_{act}" + + (f"_lim{swiglu_limit:g}" if math.isfinite(swiglu_limit) else "") + + (f"_p{NPC}" if NPC > 1 else "") + + ("_r8" if route_fp8 else "") + + f"_ag{agr}_tp{tp}" + + ("_ar" if ar else "") + + (f"_nab{NAB}" if NAB != 4 else "") + + (f"_npp{npp}" if npp > 1 else "") + + ("_ll" if ll_rs else "") + + ("_llr" if DLL else "") + + ("_cb16" if comm_bf16 else "") + + ("_ag8" if AG8 else "") + + ("_a8" if a8 else "") + + ("_lb" if lb else "") + + (f"_q{LBQ}" if LBQ > 1 else "") + + (("_cg" + "x".join(str(w) for w in CGW)) if _cg else "") + + (f"_lp{LB_PS[0]}" if lb and len(LB_PS) == 1 else "") + + ( + f"_tn{tn}e{struct.unpack(' 256: C_ECNT / C_OFF / C_FILL here instead, [chunk][NE] 512 apart +BIGM_CTRL_INTS = C_XW + 3 * 8 * 512 +C_ERR = BIGM_CTRL_INTS # sticky watchdog word after the per-launch ctrl block (not zeroed per launch) + +CMAX = 7 # chunks per launch (ctrl layout) +PAD = 64 # the kernel's local rows (and chunk rows) are multiples of this (moe() pads) +# rows per unit, one CTA (NW waves) per CU: the validated geometry (64-row +# units: 16% slower at 16384 tokens and failing with two CTAs per CU / three chunks) +UNIT_ROWS = 128 + + +def i8_row_bytes(H): + return H + H // 8 + + +def unit_cap(tp, s, K1, NE, RG): + """Upper bound of the units one XCD gets for a chunk of s rows per rank.""" + ubound = -(-tp * s * K1 // RG) + NE + return -(-ubound // 8) + UQ + 1 diff --git a/amd_kernels/moe/mega_moe_tp/mega_moe_tp_kernel.py b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_kernel.py new file mode 100644 index 0000000..e66141e --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_kernel.py @@ -0,0 +1,2002 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/mega_moe_tp_kernel.py (ROCm/aiter edf58c67a). +"""Fused TP MegaMoE kernel (a4w4, gfx950): the fused tail norm, the wave roles, the +kernel entry and its launcher. The device code is built per instance by the build_* +parts (communication, gemm, schedule) over one shared KernelCtx.""" + +from __future__ import annotations + +import functools +import inspect +import struct + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import gpu, range_constexpr, rocdl +from flydsl.expr import math as fmath +from flydsl.expr.typing import T + +from .fly_utils import ( + _e8m0_from_amax, + atomic_add_agent, + ceildiv, + fence_release, + load_i32_global_agent, +) +from .common import ( + AUX_SYS, + MAX_TP, + V4I, + alive, + amax, + before, + bf16x8_to_f32, + bld, + bst, + f32_bits_add, + f32_bits_max, + fp4_pack, + fp8x4_pack, + fp8x8_decode, + g_add_agent, + g_ld_sys, + g_st_sys, + i4x8_pack, + i4x8_unpack, + i8x4_pack, + i8x4_unpack, + i32, + lds_atomic_add, + lds_ld_acq, + lds_ld_i32, + lds_st, + now, + pack_bf16x8, + rsrc, + sum_live, + uni, + wait_vm, + wave_red, +) +from .communication import build_communication +from .gemm import build_bigm_unit, build_gemm +from .mega_moe_tp_config import ( + C_ECNT, + C_EPOCH, + C_ERR, + C_FILL, + C_OFF, + C_PH, + C_SB, + C_SBH, + C_TK, + C_TNL, + C_TURN, + C_UN, + C_XW, + CTRL_TNC, + ERR_YAG, + FLAG_TN, + NKIND, + NQ, + NT, + NTT, + NW, + P_RB, + P_RF, + P_RQ, + P_S1, + P_S2, + P_S3, + P_U, + P_UB, + QE, + QID, + QQ, + QS, + QUEUES, + QX, + QXS, + SRT, + TN_MAX, + UNIT_ROWS, + UQ, + ZQ, + i8_row_bytes, + kernel_config, +) +from .schedule import build_schedule + +__all__ = ["compile_bigm_kernel", "compile_mega_moe_tp"] + + +@functools.cache +def compile_mega_moe_tp( + *, + H: int, + I: int, + TOPK: int, + MT: int, + TMAX: int, + E: int, + act: str = "silu", + situ_beta: float = 1.0, + situ_linear_beta: float = 1.0, + swiglu_limit: float | None = None, + route_fp8: bool = False, + agr: int = 1, + tp: int = MAX_TP, + ar: bool = False, + nab: int = 4, + npp: int = 1, + ll_rs: bool = False, + ll_route: bool = False, + comm_bf16: bool = False, + ag8: bool = False, + rch: int = 16, + nch: int = 0, + a8: bool = False, + lb: bool = False, + lbq: int = 1, + tn: int = 0, + tn_eps: float = 1e-6, + tn_gemma: bool = True, + tn_local: bool = False, + lbpf: bool = False, + lbp: int = 0, +): + """Build the launcher of one (shape, variant) instance; LaunchCfg (host) picks them. + + Units are planned in the kernel over the active experts: an expert routed by R + tokens runs as ceil(R / rch) row chunks (chunk c takes its tokens t with + t % chunks == c; nch bounds the chunk table), each in P equal inter pieces; piece k + writes its GEMM2 partial into route region k. + ar: every rank holds the same input and routing of all tokens (each rank quantizes + and all-gathers its 1/tp of the rows) and gets every token's sum (all-reduce); + else (ag_rs) each rank holds its own rows and gets their sums (reduce-scatter). + ll_rs: LL ReduceScatter packets; ll_route: LL route rows too. + comm_bf16: bf16 (not MXFP8) ReduceScatter / all-reduce partials (no LL). + ag8: all-reduce whose second hop carries MXFP8 rows (else bf16). + a8: activations as MXFP8 (E4M3 + E8M0 per 32) instead of MXFP4. + lb: (rch = 16 * MT) large batches in two phases without partial sums: + GEMM1 units (row chunk x inter piece) export their quantized intermediate; GEMM2 + units (column group x row chunk) take the full inter dim and are claimed by + whichever CTA is free; a column chunk is pushed once all its units are done. + lbq: output column chunks per GEMM2 unit; lbpf: the A loader wave prefetches the + GEMM2 units; lbp: forced GEMM1 inter pieces (0: picked per batch). + tn: (ag_rs, lb) the next layer's input fused in: each output row r of this rank, + once final, becomes res_out[r] = y[r] + res_in[r] and the per-token FP8 quant of + GemmaRMSNorm(res_out[r]; w, tn_eps) (RMSNorm unless tn_gemma), gathered by every + rank into qall / sall; tn 2: the bf16 normed rows too (qall after the FP8 rows). + """ + kc = kernel_config(**locals()) + kc.add(build_communication(kc)) + kc.add(build_gemm(kc)) + kc.add(build_schedule(kc)) + AG8, ARLL, DLL, L_CTL, L_RING, LBPF = kc.get("AG8 ARLL DLL L_CTL L_RING LBPF") + LDS_BYTES, MLL, NCHA, TN, TPC = kc.get("LDS_BYTES MLL NCHA TN TPC") + a_loader, ag8_convert, ag_send = kc.get("a_loader ag8_convert ag_send") + _ag_send_meta, _ag_send_meta_ll = kc.get("_ag_send_meta _ag_send_meta_ll") + claim_addr, comm_help, comm_wave = kc.get("claim_addr comm_help comm_wave") + compute_units, const_expr, ctrl_at = kc.get("compute_units const_expr ctrl_at") + (TNL,) = kc.get("TNL") + _drop_stale, _dyn_zero, fin_key = kc.get("_drop_stale _dyn_zero fin_key") + fin_owned, fin_split, finish = kc.get("fin_owned fin_split finish") + init_lds, layout_tag, lb_finish = kc.get("init_lds layout_tag lb_finish") + lb_g2_prefetch, name, peer_rs = kc.get("lb_g2_prefetch name peer_rs") + peer_sel, poll_loop, poll_zero = kc.get("peer_sel poll_loop poll_zero") + push_dll, quant_chunks, signal_loop = kc.get("push_dll quant_chunks signal_loop") + yag_flush, yag_wait, zero_masked = kc.get("yag_flush yag_wait zero_masked") + + TN_V = H // 8 + TN_IT = ceildiv(TN_V, NTT) + + @flyc.jit + def blk_red(L, tid, v, op, slot): + r = wave_red(v, tid % i32(64), op) + if (tid % i32(64)) == i32(0): + lds_st( + L, i32(L_RING) + (i32(slot * (NTT // 64)) + tid // i32(64)) * i32(4), r + ) + gpu.barrier() + t = lds_ld_i32(L, i32(L_RING) + i32(slot * (NTT // 64) * 4)) + for k in range_constexpr(1, NTT // 64): + t = op(t, lds_ld_i32(L, i32(L_RING) + i32((slot * (NTT // 64) + k) * 4))) + return t + + @flyc.jit + def tn_rows(L, tid, a): + m = a["m"] + S = fin_split(a) + step = (S > i32(1)).select(m, i32(gpu.grid_dim.x)) + mine = fin_owned(a) != i32(0) + start = ((S > i32(1)) & (i32(gpu.block_id("x")) >= m * S)).select(m, fin_key(a)) + for r_ in range(start, m, step): + r = i32(r_) + if tid == i32(0): + last = i32(1) + if S > i32(1): + ca = ctrl_at(a, i32(CTRL_TNC) + r) + old = g_add_agent(ca, 1) + last = (old == S - i32(1)).select(i32(1), i32(0)) + if last == i32(1): + g_st_sys(ca, i32(0)) + lds_st(L, L_CTL + C_TNL * 4, mine.select(last, i32(0))) + gpu.barrier() + if lds_ld_i32(L, L_CTL + C_TNL * 4) == i32(1): + tn_row(L, tid, a, r, True) + gpu.barrier() + + def own_q(a): + return rsrc(a["mine"] + fx.Int64(a["off_qall"])) + + @flyc.jit + def tn_row(L, tid, a, r, sysy): + H8 = i32(TN_V) + ry = rsrc(a["y"]) + rres = rsrc(a["tn_res"]) + rout = rsrc(a["tn_out"]) + rw = rsrc(a["tn_w"]) + f, ss = [], fx.Float32(0.0) + for k in range_constexpr(TN_IT): + q = tid + i32(k * NTT) + qc = fx.min(q, H8 - i32(1)) + off = (r * i32(H) + qc * i32(8)) * i32(2) + yv = bf16x8_to_f32(bld(ry, off, 0, V4I, AUX_SYS if sysy else 0)) + rv = bf16x8_to_f32(bld(rres, off, 0, V4I, 0)) + fk = [x + y for x, y in zip(yv, rv)] + live = q < H8 + if live: + bst(pack_bf16x8(fk), rout, off, 0, 0) + fk = [live.select(x, fx.Float32(0.0)) for x in fk] + for x in fk: + ss = ss + x * x + f.append(fk) + tot = blk_red( + L, + tid, + ss.bitcast(fx.Int32), + lambda x, y: (x.bitcast(fx.Float32) + y.bitcast(fx.Float32)).bitcast( + fx.Int32 + ), + 0, + ) + rcp = fx.Float32( + fmath.rsqrt( + tot.bitcast(fx.Float32) / fx.Float32(float(H)) + + fx.Float32(float(tn_eps)) + ) + ) + xs, am = [], fx.Float32(0.0) + for k in range_constexpr(TN_IT): + q = fx.min(tid + i32(k * NTT), H8 - i32(1)) + wv = bf16x8_to_f32(bld(rw, q * i32(16), 0, V4I, 0)) + xk = [ + ( + fx.Float32(x * rcp * ((w + fx.Float32(1.0)) if tn_gemma else w)) + .to(fx.BFloat16) + .to(fx.Float32) + ) + for x, w in zip(f[k], wv) + ] + am = am.maximumf(amax(xk)) + xs.append(xk) + amx = blk_red( + L, + tid, + am.bitcast(fx.Int32), + lambda x, y: x.bitcast(fx.Float32) + .maximumf(y.bitcast(fx.Float32)) + .bitcast(fx.Int32), + 1, + ) + scale = amx.bitcast(fx.Float32).maximumf(fx.Float32(1e-10)) / fx.Float32(448.0) + inv = fx.Float32(1.0) / scale + one = fx.Float32(1.0) + grow = a["rank"] * a["m"] + r + for k in range_constexpr(TN_IT): + q = tid + i32(k * NTT) + qv = [x * inv for x in xs[k]] + d = fx.Vector.from_elements( + [fp8x4_pack(qv[0:4], one), fp8x4_pack(qv[4:8], one)], fx.Int32 + ) + if q < H8: + for p in range_constexpr(1 if TNL else TPC): + bst( + d, + own_q(a) if TNL else peer_rs(a, p, "off_qall"), + grow * i32(H) + q * i32(8), + 0, + 0 if TNL else AUX_SYS, + ) + b0 = a["mmax"] * i32(TPC * H) + for k in range_constexpr(TN_IT if tn == 2 else 0): + q = tid + i32(k * NTT) + d = pack_bf16x8(xs[k]) + if q < H8: + for p in range_constexpr(TPC): + bst( + d, + peer_rs(a, p, "off_qall"), + b0 + (grow * i32(H) + q * i32(8)) * i32(2), + 0, + AUX_SYS, + ) + if tid == i32(0): + for p in range_constexpr(1 if TNL else TPC): + bst( + scale.bitcast(fx.Int32), + ( + rsrc(a["mine"] + fx.Int64(a["off_sall"])) + if TNL + else peer_rs(a, p, "off_sall") + ), + grow * i32(4), + 0, + 0 if TNL else AUX_SYS, + ) + if const_expr(TNL): + return + wait_vm(0) + gpu.barrier() + if tid < i32(TPC): + fo = fx.Int64((i32(FLAG_TN) + a["rank"] * i32(TN_MAX) + r) * i32(4)) + g_st_sys(peer_sel(a, tid) + fx.Int64(a["off_flag"]) + fo, a["epoch"]) + + @flyc.jit + def tn_wait(tid, a): + if tid < i32(64): + nblk = i32(gpu.grid_dim.x) + t = i32(gpu.block_id("x")) + tid * nblk + live = t < a["ttot"] + tc = fx.min(t, a["ttot"] - i32(1)) + src = tc // a["m"] + row = tc - src * a["m"] + addr = ( + a["mine"] + + fx.Int64(a["off_flag"]) + + fx.Int64((i32(FLAG_TN) + src * i32(TN_MAX) + row) * i32(4)) + ) + poll_zero( + lambda: wave_red( + (live & before(g_ld_sys(addr), a["epoch"])).select(i32(1), i32(0)), + tid, + fx.max, + ), + a, + tid, + ERR_YAG, + ) + + @flyc.jit + def roles(L, tid, a, epoch): + if tid < i32(NT): + compute_units(L, tid, a) + comm_help(L, tid, a, epoch) + else: + if tid < i32(NT + 64): + ag_send(L, tid % i32(64), a) + if const_expr(DLL): + push_dll(L, tid, a) + comm_wave(L, tid, a, epoch) + else: + poll_loop(L, tid, a, epoch) + elif tid < i32(NT + 128): + a_loader(L, tid, a) + if const_expr(LBPF): + lb_g2_prefetch(L, tid % i32(64), a) + if const_expr(DLL): + push_dll(L, tid, a, 1) + signal_loop(L, tid, a, epoch) + comm_help(L, tid, a, epoch) + else: + if const_expr(DLL): + if tid < i32(NT + 192): + zero_masked(L, tid % i32(64), a, 0, 1) + push_dll(L, tid, a, 2) + else: + push_dll(L, tid, a, 3) + comm_help(L, tid, a, epoch) + + Shared = fx.struct( + type( + "Shared", (), {"__annotations__": {"buf": fx.Array[fx.Int8, LDS_BYTES, 16]}} + ) + ) + + def mega_moe_tp_kernel( + w1: fx.Int64, + w1s: fx.Int64, + w2: fx.Int64, + w2s: fx.Int64, + x: fx.Int64, + ids_in: fx.Int64, + tw_in: fx.Int64, + y: fx.Int64, + routes: fx.Int64, + proutes: fx.Int64, + ctrl: fx.Int64, + p0: fx.Int64, + p1: fx.Int64, + p2: fx.Int64, + p3: fx.Int64, + p4: fx.Int64, + p5: fx.Int64, + p6: fx.Int64, + p7: fx.Int64, + off_x: fx.Int64, + off_xs: fx.Int64, + off_ids: fx.Int64, + off_w: fx.Int64, + off_part: fx.Int64, + off_flag: fx.Int64, + off_yall: fx.Int64, + rank: fx.Int32, + tp: fx.Int32, + m: fx.Int32, + mmax: fx.Int32, + xg: fx.Int64, + tn_res: fx.Int64, + tn_out: fx.Int64, + tn_w: fx.Int64, + off_qall: fx.Int64, + off_sall: fx.Int64, + ): + a = dict(locals()) + tid = fx.Int32(gpu.thread_id("x")) + assert layout_tag and name + lds = fx.SharedAllocator().allocate(Shared).peek() + L = uni(fx.Int32(fx.ptrtoint(lds.buf.ptr))) + peers = [p0, p1, p2, p3, p4, p5, p6, p7] + mine = peers[0] + for j in range_constexpr(1, MAX_TP): + mine = (rank == i32(j)).select(peers[j], mine) + a.update(peer=peers, mine=fx.Int64(mine), ttot=tp * m) + _dyn_zero(L, tid) + init_lds(L, tid, a) + gpu.barrier() + epoch = lds_ld_i32(L, L_CTL + C_EPOCH * 4) + a["epoch"] = epoch + _drop_stale(tid, a) + if (tid == i32(0)) & (i32(gpu.block_id("x")) == i32(0)): + g_st_sys(claim_addr(a, 1), i32(0)) + if const_expr(ar): + a["ids"] = fx.Int64(ids_in) + a["tw"] = fx.Int64(tw_in) + else: + par = fx.Int64(epoch & i32(1)) * fx.Int64(a["tp"] * a["mmax"] * i32(TOPK)) + a["off_ids"] = off_ids + par * 16 + a["off_w"] = off_w + par * 4 + a["ids"] = fx.Int64(mine) + a["off_ids"] + a["tw"] = fx.Int64(mine) + a["off_w"] + if (tid >= i32(NT)) & (tid < i32(NT + 64)): + if const_expr(MLL): + _ag_send_meta_ll(tid % i32(64), a) + else: + _ag_send_meta(tid % i32(64), a) + quant_chunks(L, tid, NTT, a, i32(0), i32(NCHA)) + gpu.barrier() + a["ax"] = fx.Int64(mine) + off_x + a["axs"] = fx.Int64(mine) + off_xs + roles(L, tid, a, epoch) + if const_expr(ar and not ARLL): + wait_vm(0) + if const_expr(TN): + wait_vm(0) + gpu.barrier() + if const_expr(ar and not ARLL): + yag_flush(L, tid, a) + yag_wait(tid, a, epoch) + if const_expr(TN): + tn_rows(L, tid, a) + if const_expr(not TNL): + tn_wait(tid, a) + if const_expr(AG8): + gpu.barrier() + ag8_convert(tid, a) + if const_expr(lb): + lb_finish(tid, a, L) + finish(tid, a, epoch) + + kargs = list(inspect.signature(mega_moe_tp_kernel).parameters) + mega_moe_tp_kernel = flyc.kernel(name=name, known_block_size=[NTT, 1, 1])( + mega_moe_tp_kernel + ) + + def launch( + w1: fx.Int64, + w1s: fx.Int64, + w2: fx.Int64, + w2s: fx.Int64, + x: fx.Int64, + ids_in: fx.Int64, + tw_in: fx.Int64, + y: fx.Int64, + routes: fx.Int64, + proutes: fx.Int64, + ctrl: fx.Int64, + p0: fx.Int64, + p1: fx.Int64, + p2: fx.Int64, + p3: fx.Int64, + p4: fx.Int64, + p5: fx.Int64, + p6: fx.Int64, + p7: fx.Int64, + off_x: fx.Int64, + off_xs: fx.Int64, + off_ids: fx.Int64, + off_w: fx.Int64, + off_part: fx.Int64, + off_flag: fx.Int64, + off_yall: fx.Int64, + rank: fx.Int32, + tp: fx.Int32, + m: fx.Int32, + mmax: fx.Int32, + xg: fx.Int64, + tn_res: fx.Int64, + tn_out: fx.Int64, + tn_w: fx.Int64, + off_qall: fx.Int64, + off_sall: fx.Int64, + i32_grid: fx.Int32, + stream: fx.Stream, + ): + loc = locals() + mega_moe_tp_kernel(*[loc[k] for k in kargs]).launch( + grid=(fx.Int64(i32_grid), 1, 1), block=(NTT, 1, 1), stream=stream + ) + + # launch forwards its arguments to the kernel by name + if list(inspect.signature(launch).parameters) != kargs + ["i32_grid", "stream"]: + raise ValueError("compile_mega_moe_tp: kernel and launch signatures differ") + return flyc.jit(launch) + + +# ---- big-M kernel ---- +# Large-batch (big-M) path of MegaMoeTP.forward: one persistent launch runs the MoE of T +# tokens (m = T / tp rows per rank, ag_rs: every rank's routing of its own rows), in C chunks: +# +# RB this rank's rows -> MXFP4 rows, + their routing -> SDMA gather of rows + routing (C) +# S1 wait C; routes per expert S2 route-list offsets, per-XCD unit queues +# S3 route lists (route index, weight) per expert +# U fused units (build_bigm_unit): GEMM1 -> act + MXFP4 in LDS -> GEMM2 -> route rows +# RQ topk sum: own rows bf16, peer rows int8 -> SDMA MoE-output RS (E) +# RF wait E; y = the sum (+ tail: res_out = y + res, next-layer norm, per-token FP8 +# -> SDMA gather (G); or the y gather of the ar mode) +# GW wait every G; advance the persistent signal base +# +# All communication is MORI SDMA (puts with remote signals) issued and awaited inside the +# kernel; the GEMM1 -> GEMM2 intermediate stays in LDS. CTAs (one per CU, 4 waves) claim +# items with a per-XCD atomic ticket in a fixed phase order; an item only waits on items +# before it (and on peers' puts of earlier items), so the schedule cannot deadlock. +# Data another XCD reads in the same launch is stored write-through (sc0 | sc1). +const_expr = fx.const_expr + + +@functools.cache +def compile_bigm_kernel( + H: int, + I: int, + E: int, + K: int, + tp: int, + C: int, + eps: float, + gemma: bool, + swiglu_limit: float, + NAB: int = 4, + order: int = 3, + opts: tuple = (), + tailq: int = 1, + act: str = "swiglu", +): + """act: the units' activation ("swiglu", "silu"); E < 512 routed experts (+ the shared one). + tailq: 0 no tail, 1 the tail with the q gather, 2 the local tail (this rank's q rows), + 3 no tail, y of every rank's rows (bf16, the "q" region: the y gather of the ar mode). + The MoE interface (MegaMoeTP.forward, ag_rs): the caller's normed own rows x [m, H] and their + routing (ids / tw [m, K1], the shared expert's column included) are gathered; the tail sums the + MoE output (y), and with tailq res_out = y + res_in, the next-layer norm + FP8 rows of every + rank (q gather). + """ + assert H % 256 == 0 and 0 < E < 512 and K <= 64 and act in ("swiglu", "silu") + RG = UNIT_ROWS # (one CTA of NW waves per CU) + K1, NE = K + 1, E + 1 + HV = H // 8 # 16 B (8 bf16) columns of a row + # row iterations per thread (the last one partial unless NT | HV) + PIT = ceildiv(HV, NT) + # per-expert ctrl / LDS entries (one thread per expert, NEB rounds of NT) + NEB = ceildiv(NE, NT) + NES = NEB * NT + if NE <= 256: + XST, CC_CNT, CC_OFF, CC_FILL = 256, C_ECNT, C_OFF, C_FILL + else: + XST, CC_CNT, CC_OFF, CC_FILL = 512, C_XW, C_XW + 8 * 512, C_XW + 16 * 512 + + od = dict(opts) # tuned BIGM_* options (mega_moe_tp_bigm_tuned.csv) + + def _opt(name, default): + return od.get(name, default) + + # cache policy of the data one CTA writes and another (on another XCD: separate L2s) reads in + # this launch -- route rows, own MoE rows, received E rows: stored cached and this XCD's L2 + # written back (buffer_wbl2 sc1) before a unit counts as done (release), loaded sc1 + CAUX = 16 + # GEMM2 B ring depth (6 = a whole group: the route stores of a group never sit between a + # GEMM2 load and its wait) + BS2 = int(_opt("BIGM_BS2", "3")) + # the B rings' depths divide the unit's GEMM1 chunks (H / 256) / GEMM2 k-steps (I / 128): the + # largest divisor up to the requested depth (H 4096: BST 3 -> 2; I 512: BS2 3 -> 2, 6 -> 4) + BST = max(d for d in range(1, 4) if (H // 256) % d == 0) + BS2 = max(d for d in range(1, BS2 + 1) if (I // 128) % d == 0) + # item order (see ORDER below); the host picks 3 (one unit round per chunk) or 4 (several) + ORD = int(_opt("BIGM_ORD", str(order))) + # tail sub-blocks: each chunk's topk sums / drains post their E / q puts per row sub-block + NSB = 4 + # head sub-blocks: each chunk's gather of the input rows + routing (C) per row sub-block + NSBA = int(_opt("BIGM_NSBA", "1")) + assert NSBA in (1, 2, 4) + UA = int(_opt("BIGM_UA", "32")) # ORD 4: first-round unit slots per XCD + # R8: route rows as MXFP8 (half the bytes of bf16 GEMM2 stores / topk-sum loads) + R8 = int(_opt("BIGM_R8", "0")) + # MoE-output RS rows as int4 (+ bf16 scale per 32) + E4 = int(_opt("BIGM_E4", "0")) + # SXW 0: the routing plan waits only for the gathered routing; units wait for the gathered rows + SXW = int(_opt("BIGM_SXW", "1")) + # topk sum: token pipeline depth (0: ZQ, a batch: every load first). bf16 route rows of many + # routes (GLM-5: 9 x 6144): a batch's topk-sum loads alone take the register file (512 VGPRs + # + scratch spills): pipelined instead + RQD = int(bool(R8) or K1 * PIT > 16) + # puts per chunk and queue; no q gather without the tail or with the local tail (own rows' + # q only); the y gather (tailq 3): bf16 rows on QQ, no scales + PPC = {QE: NSB, QQ: NSB, QS: NSB, QX: NSBA, QXS: NSBA, QID: 2 * NSBA} + if tailq not in (1, 3): + PPC[QQ] = PPC[QS] = 0 + if tailq == 3: + PPC[QS] = 0 + ZX = 16 # io: rows per input item (quant + routing copy) + EPL = ceildiv(E, 64) # experts per lane of the plan's wave + # int8 rows of the attention / MoE-output RS: one fp32 scale per 32 values + RB = i8_row_bytes(H) + SCO = H + lay, unit_body = build_bigm_unit(H, I, K1, RG, NAB, BST, BS2, act, swiglu_limit, R8) + # row-phase / plan scratch after the unit's LDS (also reused ranges) + LBASE = lay["BYTES"] + LSC = LBASE # 64 B scratch for block reductions (NW x 4 words) + LT = LBASE + 64 # ticket broadcast + LH = LBASE + 128 # [NE] histogram / ranks + LB2 = LH + 4 * NES # [NE] bases + LP = LB2 + 4 * NES # [SRT * K1] route positions of a sort item ([NE] plan scratch) + LTID = LP + max(4 * SRT * K1, 64 + 4 * max(NE, 64 * EPL)) + 64 + LDS_BYTES = LTID + 4 * NT # [NT] each thread's id (own_tid) + assert LDS_BYTES <= 160 * 1024, LDS_BYTES + + def cpart(kk): # row iteration kk is the partial last one + return HV % NT != 0 and kk == PIT - 1 + + def col(tid, kk): + """this thread's column of row iteration kk; past the row (a partial last iteration) the + same lane of the last full 4-column group: those threads redo its loads, math and stores + (identical values, its per-32 scales included; the fused tail: see own_col)""" + p = tid + i32(kk * NT) + if not cpart(kk): + return p + return (p < i32(HV)).select(p, i32(HV - 4) + (p & i32(3))) + + def own_col(tid, kk): + """None, else (a partial iteration) whether this thread's column is its own, not redone. + The fused tail updates res in place (res_in == res_out): a redone column's res load + can follow its owner's store, so there the redone columns store nothing and stay out + of the row's sums.""" + if not cpart(kk): + return None + return tid + i32(kk * NT) < i32(HV) + + def at_col(base, tid, kk): + """base + this thread's first element (bf16) of iteration kk""" + if not cpart(kk): + return base + tid * i32(8) + i32(kk * NT * 8) + return base + col(tid, kk) * i32(8) + + # item order: chunk k+1's prep (input rows RB, plan) ahead of chunk k's units and drain + # (a CTA spinning on an item cannot take later ones: items that wait for a transfer come + # after the work that overlaps it) chunk k's units, then chunk k+1's prep (claimed as + # chunk k's units drain), chunk k's topk sum, then chunk k-1's drain after k's units + def prep(k): + return [(k, P_RB), (k, P_S1), (k, P_S2), (k, P_S3)] + + assert ORD in (3, 4), ORD + if ORD == 4: + # U(k) = the first unit slot of every CTA, UB(k) the rest: chunk k+1's input rows run + # between them and its routing plan right after, so chunk k+1's units fill the CTAs + # chunk k's last unit round leaves idle + ORDER = prep(0) + for k in range(C): + ORDER += [(k, P_U)] + if k >= 1: + ORDER += [(k - 1, P_RQ)] + if k + 1 < C: + ORDER += [(k + 1, P_RB)] + ORDER += [(k, P_UB)] + if k >= 1: + ORDER += [(k - 1, P_RF)] + if k + 1 < C: + ORDER += [(k + 1, P_S1), (k + 1, P_S2), (k + 1, P_S3)] + ORDER += [(C - 1, P_RQ), (C - 1, P_RF)] + elif ORD == 3: + # chunk k's topk sum right after chunk k+1's prep (it fills the gap while chunk k+1's + # routing lands, on the whole machine) so E(k), RF(k), G(k) overlap chunk k+1's units + ORDER = prep(0) + for k in range(C): + ORDER += [(k, P_U)] + if k >= 1: # on the CTAs chunk k's units leave idle (E(k-1) is on its way) + ORDER += [(k - 1, P_RF)] + if k + 1 < C: + ORDER += prep(k + 1) + ORDER += [(k, P_RQ)] + ORDER += [(C - 1, P_RF)] + ORDER += [(k, P_UB) for k in range(C)] # empty: U(k) takes every unit slot + USPLIT = ORD == 4 + + def fb(v): + return f"{struct.unpack(' -8, -amax -> 8 clamped to 7); MoE output rel_l2 vs + # torch 0.0856 (symmetric [-7, 7]: 0.0882) + sc = fx.Float32( + fx.Float32(am.maximumf(fx.Float32(1e-30)) / fx.Float32(-8.0)) + .to(fx.BFloat16) + .to(fx.Float32) + ) + bst( + i4x8_pack(f, fx.Float32(1.0) / sc), + rd, + drow * i32(RBE) + p * i32(4), + 0, + AUX_SYS, + ) + if (lane & i32(3)) == i32(0): + bst( + fx.Int16(sc.bitcast(fx.Int32) >> i32(16)), + rd, + drow * i32(RBE) + i32(H // 2) + (p // i32(4)) * i32(2), + 0, + AUX_SYS, + ) + else: + i8_row_store(f, rd, drow, p, lane) + + @flyc.jit + def i8_row_store(f, rd, drow, p, lane): + am = amax(f) + am = am.maximumf(am.shuffle_xor(i32(1), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(2), i32(64))) + g = p // i32(4) + sc = am.maximumf(fx.Float32(1e-30)) / fx.Float32(127.0) + inv = fx.Float32(1.0) / sc + dv = fx.Vector.from_elements( + [i8x4_pack(f[0:4], inv), i8x4_pack(f[4:8], inv)], fx.Int32 + ) + bst(dv, rd, drow * i32(RB) + p * i32(8), 0, AUX_SYS) + if (lane & i32(3)) == i32(0): + bst( + sc.bitcast(fx.Int32), + rd, + drow * i32(RB) + i32(SCO) + g * i32(4), + 0, + AUX_SYS, + ) + + def kernel( + wb: fx.Int64, # this rank's data arena (SdmaComm: hipIpc, not a MORI window) + pbases: fx.Int64, # [tp] int64 every rank's arena as mapped here (the put destinations) + dc: fx.Int64, # MORI device communicator + consumed: fx.Int64, # [tp][NQ] int32 puts awaited before this launch (persistent; + # the next launch's at consumed ^ (tp * NQ * 4): two buffers in turn) + offs: fx.Int64, # [8] int64 arena region offsets + xin: fx.Int64, # [m, H] bf16 own rows + ids: fx.Int64, # [m, K1] int32 + tw: fx.Int64, # [m, K1] f32 + normw: fx.Int64, # [H] the next layer's norm weight (tail) + resin: fx.Int64, # [m, H] bf16 (tail) + yout: fx.Int64, # [m, H] bf16 out (the MoE output) + resout: fx.Int64, # [m, H] bf16 out + ybuf: fx.Int64, # [m, H] bf16 own MoE rows + routes: fx.Int64, # [T * K1, H] bf16 route rows (chunk k's from route tp * o_k * K1) + lrix: fx.Int64, # [C][T * K1] route lists + lwt: fx.Int64, + utab: fx.Int64, # [C][8][ucap][4] units + w1: fx.Int64, + w1s: fx.Int64, + w2: fx.Int64, + w2s: fx.Int64, + ctrl: fx.Int64, + chunks: fx.Int64, # [C][2] int32: (rows offset o, rows s) per rank + rank: fx.Int32, + m: fx.Int32, + ucap: fx.Int32, + ): + sa = fx.SharedAllocator() + st = sa.allocate(Shared).peek() + L = fx.Int32(fx.ptrtoint(st.raw.ptr)) + tid = fx.Int32(gpu.thread_id("x")) + lds_st(L, i32(LTID) + tid * i32(4), tid) + + def own_tid(): + # the thread id through a volatile LDS load, at the start of every item: values + # derived from it stay inside the item (not hoisted / CSE'd across items and live + # through the whole loop: register spills) + return lds_ld_acq(L, i32(LTID) + tid * i32(4)) + + lane = tid % i32(64) + bid = i32(gpu.block_id("x")) + xc = bid % i32(8) + roffs = rsrc(offs) + + def off_of(i): + return fx.Int64(bld(roffs, i32(i * 8), 0, T.i64)) + + a = { + "wb": wb, + "off_xq": off_of(0), + "off_xs": off_of(1), + "off_ids": off_of(2), + "off_tw": off_of(3), + "off_eq": off_of(4), + "off_erecv": off_of(5), + "off_q": off_of(6), + "off_qs": off_of(7), + } + + def ca(i): + return ctrl + fx.Int64(i * i32(4)) + + def wp(name): # local window address of a region + return wb + a["off_" + name] + + rch = rsrc(chunks) + + def chunk(k): # (rows offset o, rows s) of chunk k (traced) + return fx.Int32(bld(rch, k * i32(8), 0, T.i32)), fx.Int32( + bld(rch, k * i32(8) + i32(4), 0, T.i32) + ) + + def lbase(k): # chunk k's first route (route lists, route rows: chunks packed) + return i32(tp) * chunk(k)[0] * i32(K1) + + def rbase(k): # chunk k's route rows (bytes) + return fx.Int64(lbase(k)) * fx.Int64(H * 2) + + # this launch's signal base per (peer, queue) (GW writes the next launch's into the + # other buffer) + rcons = rsrc(consumed) + + def sig_base(p, q): + return fx.Int32( + bld(rcons, (p * i32(NQ) + i32(q)) * i32(4), 0, T.i32, AUX_SYS) + ) + + # ---- SDMA (MORI device functions) ---- + from flydsl.compiler.extern_link import link_extern + from flydsl.expr.extern import ffi + + from .sdma import sdma_bitcode + + bc = sdma_bitcode() + put_ptr = link_extern( + ffi( + "cco_sdma_put_ptr_rsig__thread", + ["uint64", "int32", "uint64", "uint64", "uint64", "int32"], + "int32", + ), + bitcode_path=bc, + ) + sig_slot = link_extern( + ffi("cco_sdma_signal_slot", ["uint64", "int32", "int32"], "uint64"), + bitcode_path=bc, + ) + + @flyc.jit + def take_turn(q, nth): + # posts on one queue in a fixed order, one poster at a time (MORI loses remote + # signals of concurrent puts on the same queue) + cur = fx.Int32(load_i32_global_agent(ca(i32(C_TURN + 32 * q)))) + while cur < nth: + rocdl.s_sleep(1) + cur = fx.Int32(load_i32_global_agent(ca(i32(C_TURN + 32 * q)))) + + def give_turn(q, nth): + g_st_sys(ca(i32(C_TURN + 32 * q)), nth + i32(1)) + + rpb = rsrc(pbases) + + def put(peer, dst_region, dst_off, src_region, src_off, nbytes, q): + # region + offset of the peer's arena <- the same of ours (remote-signalled) + dst = ( + fx.Int64(bld(rpb, peer * i32(8), 0, T.i64)) + + a["off_" + dst_region] + + fx.Int64(dst_off) + ) + src = wb + a["off_" + src_region] + fx.Int64(src_off) + put_ptr(dc, peer, dst, src, fx.Int64(nbytes), i32(q)) + + def wait_puts(q, n_per, k): + # every peer's first (k + 1) * n_per puts on queue q of this launch landed + wait_puts_n(q, (k + i32(1)) * i32(n_per)) + + @flyc.jit + def wait_puts_n(q, n): + if lane < i32(tp): + p = lane + if p != rank: + slot = fx.Int64(sig_slot(dc, p, i32(q))) + tgt = sig_base(p, q) + n + cur = g_ld_sys(slot) + t0 = now() + while before(cur, tgt) & alive(t0): + rocdl.s_sleep(2) + cur = g_ld_sys(slot) + if before( + cur, tgt + ): # gave up on a peer: flag it (the results are not valid) + g_st_sys(ca(i32(C_ERR)), i32(1)) + + def ph(k, kind): + return ca(i32(C_PH) + (k * i32(NKIND) + i32(kind)) * i32(32)) + + @flyc.jit + def done_last(k, kind, n): + """this CTA finished an item of (k, kind): True (uniform) for the last of n""" + wait_vm(0) + gpu.barrier() + if tid == i32(0): + v = fx.Int32(atomic_add_agent(ph(k, kind), i32(1))) + lds_st(L, i32(LT) + i32(4), (v == n - i32(1)).select(i32(1), i32(0))) + gpu.barrier() + return lds_ld_i32(L, i32(LT) + i32(4)) == i32(1) + + @flyc.jit + def wait_ph(k, kind, n): + wait_at(ph(k, kind), n) + + @flyc.jit + def wait_at(addr, n): + if tid == i32(0): + cur = fx.Int32(load_i32_global_agent(addr)) + while cur < n: + rocdl.s_sleep(1) + cur = fx.Int32(load_i32_global_agent(addr)) + gpu.barrier() + + def sbc(k, w, b): # sub-block done counter (w 0: RQ items, 1: RF items) + return ca(i32(C_SB) + ((k * i32(2) + i32(w)) * i32(16) + b) * i32(8)) + + @flyc.jit + def done_at(addr, n): + """like done_last, on a counter address (callers ran done_last first: stores drained)""" + if tid == i32(0): + v = fx.Int32(atomic_add_agent(addr, i32(1))) + lds_st(L, i32(LT) + i32(4), (v == n - i32(1)).select(i32(1), i32(0))) + gpu.barrier() + return lds_ld_i32(L, i32(LT) + i32(4)) == i32(1) + + def il(j, n): + # XCD-interleaved item order (XCD x's contiguous slice -> every 8th item) so the + # sub-blocks complete in order; identity when n is not a multiple of 8 + q = n // i32(8) + qq = fx.max(q, i32(1)) + x = j // qq + jn = (j - x * qq) * i32(8) + x + return ((n % i32(8) == i32(0)) & (q > i32(0))).select(jn, j) + + def claim(): + if tid == i32(0): + lds_st( + L, + i32(LT), + fx.Int32(atomic_add_agent(ca(i32(C_TK) + xc * i32(32)), i32(1))), + ) + gpu.barrier() + v = lds_ld_i32(L, i32(LT)) + gpu.barrier() + return v + + def part8(n): + q = n // i32(8) + r = n - q * i32(8) + return xc * q + (xc < r).select(xc, r), q + (xc < r).select(i32(1), i32(0)) + + # ---- item counts per (chunk, kind) (rows s per rank of the chunk) ---- + + def cnt(kind, s_): + if const_expr(kind == P_RB): # the input rows (item_rx) + return s_ // i32(ZX) + if const_expr(kind == P_RF): + return s_ // i32(ZQ) + if const_expr(kind in (P_S1, P_S3)): + return (i32(tp) * s_ + i32(SRT - 1)) // i32(SRT) + if const_expr(kind == P_S2): + return i32(1) + if const_expr(kind == P_U): + return ucap * i32(8) + return (i32(tp) * s_) // i32(ZQ) + + def chunk_v(k): # per-claim (not hoisted: system-scope load) + return fx.Int32(bld(rch, k * i32(8) + i32(4), 0, T.i32, AUX_SYS)) + + def slices_of(sk): + out = [] + tot_ = i32(0) + for oi in range_constexpr(len(ORDER)): + k, kind = ORDER[oi] + n = cnt(kind, sk[k]) + if const_expr(kind == P_S2): + lo_, c_ = i32(0), (xc == i32(0)).select(i32(1), i32(0)) + elif const_expr(kind == P_U): + lo_, c_ = i32(0), (fx.min(ucap, i32(UA)) if USPLIT else ucap) + elif const_expr(kind == P_UB): + lo_, c_ = i32(UA), ( + fx.max(ucap - i32(UA), i32(0)) if USPLIT else i32(0) + ) + else: + lo_, c_ = part8(n) + out.append((k, kind, lo_, tot_, tot_ + c_)) + tot_ = tot_ + c_ + return out, tot_ + + _, gw_lo = slices_of([chunk(i32(k))[1] for k in range(C)]) + nall = gw_lo + (xc == i32(0)).select(i32(1), i32(0)) + + r_xin, r_resin, r_ids = rsrc(xin), rsrc(resin), rsrc(ids) + r_tw, r_normw, r_y = rsrc(tw), rsrc(normw), rsrc(ybuf) + r_yout, r_out = rsrc(yout), rsrc(resout) + + def norm_w(rw, p): + return bf16x8_to_f32(bld(rw, p * i32(16), 0, T.vec(4, T.i32))) + + def gate_w(x, wk): + return x * ((wk + fx.Float32(1.0)) if gemma else wk) + + # ======================= items ======================= + + def sbh(k, w, b): # head sub-block done counter (w 1: input row items) + return ca(i32(C_SBH) + ((k * i32(2) + i32(w)) * i32(16) + b) * i32(8)) + + @flyc.jit + def post_c(k, bh): + # thread 0: this head sub-block's quantized rows + routing -> every peer + o, s = chunk(k) + B = i32(tp) * o + sa = s // i32(NSBA) + r0b = B + rank * s + bh * sa + t1 = k * i32(NSBA) + bh + CPUTS = ( + (QX, "xq", H // 2), + (QXS, "xs", H // 32), + (QID, "ids", K1 * 4), + (QID, "tw", K1 * 4), + ) + for ci in range_constexpr(4): + q_, reg, rowb = CPUTS[ci] + tq = t1 * i32(2) if const_expr(q_ == QID) else t1 + if const_expr(reg != "tw"): + take_turn(q_, tq) + for p_ in range_constexpr(tp): + if i32(p_) != rank: + put( + i32(p_), + reg, + r0b * i32(rowb), + reg, + r0b * i32(rowb), + sa * i32(rowb), + q_, + ) + if const_expr(reg in ("xq", "xs")): + give_turn(q_, tq) + if const_expr(reg == "tw"): + give_turn(q_, tq + i32(1)) + + @flyc.jit + def item_rx(k, j): + # io: ZX of this rank's (normed) rows -> MXFP4 rows + routing in the window; the last + # item of a head sub-block posts it to every peer (gather C) + tid = own_tid() + lane = tid % i32(64) + o, s = chunk(k) + B = i32(tp) * o + jx = il(j, cnt(P_RB, s)) + rq, rqs = rsrc(wp("xq")), rsrc(wp("xs")) + i0 = jx * i32(ZX) + ld = [ + [ + bld( + r_xin, + ((o + i0 + i32(ii)) * i32(H) + col(tid, kk) * i32(8)) * i32(2), + 0, + T.vec(4, T.i32), + ) + for kk in range(PIT) + ] + for ii in range(ZX) + ] + if tid < i32(ZX * K1): + rr_ = tid // i32(K1) + src = (o + i0 + rr_) * i32(K1) + (tid - rr_ * i32(K1)) + dst = (B + rank * s + i0) * i32(K1) + tid + bst( + fx.Int32(bld(r_ids, src * i32(4), 0, T.i32)), + rsrc(wp("ids")), + dst * i32(4), + 0, + AUX_SYS, + ) + bst( + fx.Int32(bld(r_tw, src * i32(4), 0, T.i32)), + rsrc(wp("tw")), + dst * i32(4), + 0, + AUX_SYS, + ) + for ii in range_constexpr(ZX): + row0 = B + rank * s + i0 + i32(ii) + for kk in range_constexpr(PIT): + p = col(tid, kk) + y = bf16x8_to_f32(ld[ii][kk]) + am = amax(y) + am = am.maximumf(am.shuffle_xor(i32(1), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(2), i32(64))) + e8, qs = _e8m0_from_amax(am, max_norm=6.0) + bst( + fp4_pack(y, qs), rq, row0 * i32(H // 2) + p * i32(4), 0, AUX_SYS + ) + if (lane & i32(3)) == i32(0): + bst( + fx.Int8(e8), + rqs, + row0 * i32(H // 32) + p // i32(4), + 0, + AUX_SYS, + ) + done_last(k, P_RB, cnt(P_RB, s)) + sa = s // i32(NSBA) + bh = i0 // sa + if done_at( # noqa: SIM102 - preserve FlyDSL branches + sbh(k, 1, bh), sa // i32(ZX) + ): + if tid == i32(0): + post_c(k, bh) + + @flyc.jit + def item_s1(k, j): + tid = own_tid() + o, s = chunk(k) + B = i32(tp) * o + if const_expr(SXW): + wait_puts(QX, NSBA, k) + wait_puts(QXS, NSBA, k) + wait_puts(QID, 2 * NSBA, k) + wait_ph(k, P_RB, cnt(P_RB, s)) # own rows' routing copied into the window + for b_ in range_constexpr(NEB): + if tid + i32(b_ * NT) < i32(NE): + lds_st(L, i32(LH) + (tid + i32(b_ * NT)) * i32(4), i32(0)) + gpu.barrier() + rids = rsrc(wp("ids")) + r0 = (B + j * i32(SRT)) * i32(K1) + nr = fx.min((B + i32(tp) * s) * i32(K1) - r0, i32(SRT * K1)) + for kk in range_constexpr((SRT * K1 + NT - 1) // NT): + r = tid + i32(kk * NT) + if r < nr: + e = fx.Int32(bld(rids, (r0 + r) * i32(4), 0, T.i32, AUX_SYS)) + if (e >= i32(0)) & (e < i32(NE)): # masked / invalid ids: no route + lds_atomic_add(L, i32(LH) + e * i32(4), i32(1)) + gpu.barrier() + for b_ in range_constexpr(NEB): + e = tid + i32(b_ * NT) + if e < i32(NE): + c = lds_ld_i32(L, i32(LH) + e * i32(4)) + if c > i32(0): + atomic_add_agent(ca(i32(CC_CNT) + k * i32(XST) + e), c) + done_last(k, P_S1, cnt(P_S1, s)) + + @flyc.jit + def item_s2(k): + tid = own_tid() + _o, s = chunk(k) + wait_ph(k, P_S1, cnt(P_S1, s)) + # route-list offsets (prefix of the counts, each thread its own expert) + for b_ in range_constexpr(NEB): + e = tid + i32(b_ * NT) + if e < i32(NE): + lds_st( + L, + i32(LH) + e * i32(4), + fx.Int32( + load_i32_global_agent(ca(i32(CC_CNT) + k * i32(XST) + e)) + ), + ) + gpu.barrier() + accs = [i32(0)] * NEB + for e_ in range_constexpr(NE - 1): + v = lds_ld_i32(L, i32(LH + 4 * e_)) + for b_ in range_constexpr(NEB): + accs[b_] = accs[b_] + (i32(e_) < tid + i32(b_ * NT)).select( + v, i32(0) + ) + for b_ in range_constexpr(NEB): + e = tid + i32(b_ * NT) + if e < i32(NE): + g_st_sys(ca(i32(CC_OFF) + k * i32(XST) + e), accs[b_]) + lds_st(L, i32(LB2) + e * i32(4), accs[b_]) + gpu.barrier() + s2_greedy(k) + done_last(k, P_S2, i32(1)) + + @flyc.jit + def s2_greedy(k): + tid = own_tid() + # units -> per-XCD queues: unit g of one round-robin sequence (the shared expert's + # units, then those of the routed experts with more than UQ units) to queue g % 8, + # slot g / 8; every other routed expert's units whole to the least-loaded queue (its + # weights shared in that XCD's L2) + rut = rsrc(utab) + ns = (lds_ld_i32(L, i32(LH + 4 * E)) + i32(RG - 1)) // i32(RG) + soff = lds_ld_i32(L, i32(LB2 + 4 * E)) + cs_ = lds_ld_i32(L, i32(LH + 4 * E)) + for u_ in range(tid, ns, i32(NT)): + u = i32(u_) + ent = ((k * i32(8) + u % i32(8)) * ucap + u // i32(8)) * i32(16) + bst(i32(E), rut, ent, 0, AUX_SYS) + bst(soff + u * i32(RG), rut, ent + i32(4), 0, AUX_SYS) + bst(fx.min(cs_ - u * i32(RG), i32(RG)), rut, ent + i32(8), 0, AUX_SYS) + # the routed experts' queue / first slot to LDS (wave 0), then every expert's units + LEQ = LP + 64 + # wave 0: the queue loads and unit counts wave-uniform; key = load * 8 + queue is + # unique per queue, so the least-loaded (lowest on ties) queue is the one whose key is + # the minimum; each expert's (queue, first slot) is kept by lane e % 64 + # (a spread expert's entry: bit 31 | its first g) + if tid < i32(64): + nuv = [ + ( + lds_ld_i32(L, i32(LH) + (tid + i32(64 * j)) * i32(4)) + + i32(RG - 1) + ) + // i32(RG) + for j in range(EPL) + ] + g_ = uni(ns) + qb = [i32(0)] * EPL + for e_ in range_constexpr(E): + nu = i32(rocdl.readlane(T.i32, nuv[e_ // 64], i32(e_ % 64))) + sp = nu > i32(UQ) + qb[e_ // 64] = ((tid == i32(e_ % 64)) & sp).select( + g_ | i32(-(1 << 31)), qb[e_ // 64] + ) + g_ = g_ + sp.select(nu, i32(0)) + ky = [ + ((g_ - i32(x) + i32(7)) // i32(8)) * i32(8) + i32(x) + for x in range(8) + ] + for e_ in range_constexpr(E): + nu = i32(rocdl.readlane(T.i32, nuv[e_ // 64], i32(e_ % 64))) + sp = nu > i32(UQ) + nu8 = sp.select(i32(0), nu * i32(8)) + m_ = fx.min( + fx.min(fx.min(ky[0], ky[1]), fx.min(ky[2], ky[3])), + fx.min(fx.min(ky[4], ky[5]), fx.min(ky[6], ky[7])), + ) + qb[e_ // 64] = (tid == i32(e_ % 64)).select( + sp.select( + qb[e_ // 64], (m_ & i32(7)) | ((m_ >> i32(3)) << i32(8)) + ), + qb[e_ // 64], + ) + for x in range_constexpr(8): + ky[x] = ky[x] + (ky[x] == m_).select(nu8, i32(0)) + for j in range_constexpr(EPL): + lds_st(L, i32(LEQ) + (tid + i32(64 * j)) * i32(4), qb[j]) + if tid == i32(0): + for x in range_constexpr(8): + lds_st(L, i32(LP + 4 * x), ky[x] >> i32(3)) + gpu.barrier() + for b_ in range_constexpr( + ceildiv(E, NT) + ): # (one round per NT routed experts) + e = tid if b_ == 0 else tid + i32(b_ * NT) + if e < i32(E): + c = lds_ld_i32(L, i32(LH) + e * i32(4)) + off = lds_ld_i32(L, i32(LB2) + e * i32(4)) + nu = (c + i32(RG - 1)) // i32(RG) + qb = lds_ld_i32(L, i32(LEQ) + e * i32(4)) + spr = qb < i32(0) + g0 = qb & i32(0x7FFFFFFF) + x_ = qb & i32(0xFF) + b0 = (qb >> i32(8)) & i32(0xFFFFFF) + for u_ in range(i32(0), nu, i32(1)): + u = i32(u_) + xs = spr.select((g0 + u) % i32(8), x_) + slot = fx.min( + spr.select((g0 + u) // i32(8), b0 + u), ucap - i32(1) + ) + ent = ((k * i32(8) + xs) * ucap + slot) * i32(16) + bst(e, rut, ent, 0, AUX_SYS) + bst(off + u * i32(RG), rut, ent + i32(4), 0, AUX_SYS) + bst( + fx.min(c - u * i32(RG), i32(RG)), + rut, + ent + i32(8), + 0, + AUX_SYS, + ) + gpu.barrier() + if tid < i32(8): + g_st_sys( + ca(i32(C_UN) + tid * i32(32) + k * i32(256)), + fx.min(lds_ld_i32(L, i32(LP) + tid * i32(4)), ucap), + ) + + @flyc.jit + def item_s3(k, j): + tid = own_tid() + o, s = chunk(k) + B = i32(tp) * o + wait_ph(k, P_S2, i32(1)) # the route-list offsets + for b_ in range_constexpr(NEB): + if tid + i32(b_ * NT) < i32(NE): + lds_st(L, i32(LH) + (tid + i32(b_ * NT)) * i32(4), i32(0)) + gpu.barrier() + rids, rtw = rsrc(wp("ids")), rsrc(wp("tw")) + r0 = (B + j * i32(SRT)) * i32(K1) + nr = fx.min((B + i32(tp) * s) * i32(K1) - r0, i32(SRT * K1)) + NRI = (SRT * K1 + NT - 1) // NT + es, rk, oks = [], [], [] + for kk in range_constexpr(NRI): + r = tid + i32(kk * NT) + e = fx.Int32( + bld(rids, (fx.min(r, nr - i32(1)) + r0) * i32(4), 0, T.i32, AUX_SYS) + ) + ok = ( + (r < nr) & (e >= i32(0)) & (e < i32(NE)) + ) # masked / invalid ids: no route + e = ok.select(e, i32(0)) + es.append(e) + oks.append(ok) + rk.append( + lds_atomic_add(L, i32(LH) + e * i32(4), ok.select(i32(1), i32(0))) + ) + gpu.barrier() + for b_ in range_constexpr(NEB): + e = tid + i32(b_ * NT) + if e < i32(NE): + c = lds_ld_i32(L, i32(LH) + e * i32(4)) + base = fx.Int32( + atomic_add_agent(ca(i32(CC_FILL) + k * i32(XST) + e), c) + ) + lds_st( + L, + i32(LB2) + e * i32(4), + fx.Int32( + load_i32_global_agent(ca(i32(CC_OFF) + k * i32(XST) + e)) + ) + + base, + ) + gpu.barrier() + rl, rlw = rsrc(lrix), rsrc(lwt) + for kk in range_constexpr(NRI): + if oks[kk]: + r = tid + i32(kk * NT) + pos = lds_ld_i32(L, i32(LB2) + es[kk] * i32(4)) + rk[kk] + lo_ = lbase(k) + pos + bst( + (r0 + r) - B * i32(K1), rl, lo_ * i32(4), 0, AUX_SYS + ) # chunk-local route index + bst( + fx.Int32(bld(rtw, (r0 + r) * i32(4), 0, T.i32, AUX_SYS)), + rlw, + lo_ * i32(4), + 0, + AUX_SYS, + ) + done_last(k, P_S3, cnt(P_S3, s)) + + @flyc.jit + def item_u_own(k, j0): + tid = own_tid() + o, s = chunk(k) + B = i32(tp) * o + wait_ph(k, P_S3, cnt(P_S3, s)) + if const_expr( + not SXW + ): # the plan only waited for the routing: the rows now + wait_puts(QX, NSBA, k) + wait_puts(QXS, NSBA, k) + gpu.barrier() + nu = fx.Int32( + load_i32_global_agent(ca(i32(C_UN) + k * i32(256) + xc * i32(32))) + ) + j = j0 + if j < nu: + rut = rsrc(utab) + ent = ((k * i32(8) + xc) * ucap + j) * i32(16) + e = uni(fx.Int32(bld(rut, ent, 0, T.i32, AUX_SYS))) + r0 = uni(fx.Int32(bld(rut, ent + i32(4), 0, T.i32, AUX_SYS))) + rows = uni(fx.Int32(bld(rut, ent + i32(8), 0, T.i32, AUX_SYS))) + rl = rsrc(lrix + fx.Int64(lbase(k) * i32(4))) + rlw = rsrc(lwt + fx.Int64(lbase(k) * i32(4))) + rx = rsrc(wp("xq") + fx.Int64(B * i32(H // 2))) + rxs = rsrc(wp("xs") + fx.Int64(B * i32(H // 32))) + nrt = i32(tp) * s * i32(K1) + r_rt = rsrc(routes + rbase(k), nrt * i32(H * 2)) + unit_body( + L, + tid, + e, + r0, + rows, + rl, + rlw, + rx, + rxs, + w1, + w2, + rsrc(w1s), + rsrc(w2s), + r_rt, + nrt * i32(H * 2), + ) + # release: the unit's route rows out of this XCD's L2 (agent-scope release + # fence: buffer_wbl2 sc1 + wait) + wait_vm(0) + gpu.barrier() + if tid == i32(0): + fence_release(fx.rocdl.SyncScope.Agent) + done_last(k, P_U, i32(-1)) # counts units done (no single last) + + @flyc.jit + def item_rq(k, j): + tid = own_tid() + lane = tid % i32(64) + o, s = chunk(k) + B = i32(tp) * o + wait_ph(k, P_S2, i32(1)) # the unit plan (C_UN) + un = i32(0) + for x in range_constexpr(8): + un = un + fx.Int32( + load_i32_global_agent(ca(i32(C_UN + 32 * x) + k * i32(256))) + ) + wait_ph(k, P_U, un) + rx = rsrc(routes + rbase(k)) + rd = rsrc(wp("eq")) + rids_ = rsrc(wp("ids")) + + def rvalid( + wrow, t + ): # (token, slot) routes to an expert: else its route row is not written + e_ = fx.Int32( + bld( + rids_, + ((B + wrow) * i32(K1) + i32(t)) * i32(4), + 0, + T.i32, + AUX_SYS, + ) + ) + return (e_ >= i32(0)) & (e_ < i32(NE)) + + sb = s // i32(NSB) + nb = (i32(tp) * sb) // i32(ZQ) + jx = il(j, cnt(P_RQ, s)) + b = jx // nb + rem = (jx - b * nb) * i32(ZQ) + pd = rem // sb + w0 = pd * s + b * sb + (rem - pd * sb) # first (dest rank, row) of the item + own = pd == rank + + nrt_ = i32(tp) * s * i32(K1) + + def rq_load( + ii, + ): # token ii of the item: its K1 route rows (this thread's columns) + if const_expr(R8): + return [ + [ + ( + bld( + rx, + ((w0 + i32(ii)) * i32(K1) + i32(t)) * i32(H) + + col(tid, kk) * i32(8), + 0, + T.vec(2, T.i32), + CAUX, + ), + bld( + rx, + nrt_ * i32(H) + + ((w0 + i32(ii)) * i32(K1) + i32(t)) * i32(H // 32) + + col(tid, kk) // i32(4), + 0, + T.i8, + CAUX, + ), + ) + for t in range(K1) + ] + for kk in range(PIT) + ] + return [ + [ + bld( + rx, + ( + at_col( + ((w0 + i32(ii)) * i32(K1) + i32(t)) * i32(H), + tid, + kk, + ) + ) + * i32(2), + 0, + T.vec(4, T.i32), + CAUX, + ) + for t in range(K1) + ] + for kk in range(PIT) + ] + + def rq_store(ii, ld): + wrow = w0 + i32(ii) + vt = [rvalid(wrow, t) for t in range(K1)] + for kk in range_constexpr(PIT): + p = col(tid, kk) + acc = [fx.Float32(0.0)] * 8 + for t in range_constexpr(K1): + v = ( + fp8x8_decode(ld[kk][t]) + if const_expr(R8) + else bf16x8_to_f32(ld[kk][t]) + ) + acc = [vt[t].select(a_ + b_, a_) for a_, b_ in zip(acc, v)] + yb = pack_bf16x8(acc) + if own: + bst( + yb, + r_y, + ((o + wrow - rank * s) * i32(H) + p * i32(8)) * i32(2), + 0, + AUX_SYS, + ) + else: + e_store(bf16x8_to_f32(yb), rd, B + wrow, p, lane) + + # token pipeline: the next D tokens' loads in flight while one is summed (D = ZQ: + # every load of the item first) + D = RQD or ZQ + rqp = [rq_load(ii) for ii in range(min(D, ZQ))] + for ii in range_constexpr(ZQ): + if const_expr(ii + D < ZQ): + rqp.append(rq_load(ii + D)) + rq_store(ii, rqp[ii]) + done_last(k, P_RQ, cnt(P_RQ, s)) + if done_at(sbc(k, 0, b), nb): # noqa: SIM102 - preserve FlyDSL branches + if tid == i32(0): + tn = k * i32(NSB) + b + take_turn(QE, tn) + for p_ in range_constexpr(tp): + if i32(p_) != rank: + put( + i32(p_), + "erecv", + (B + rank * s + b * sb) * i32(RBE), + "eq", + (B + i32(p_) * s + b * sb) * i32(RBE), + sb * i32(RBE), + QE, + ) + give_turn(QE, tn) + + def rf_head(k, j): + # an RF item (ZQ rows of chunk k, from i0, in sub-block b), once every peer's E + # rows of its sub-block and this rank's topk sums of it are in: + # (tid, o, s, B, sb, i0, b, the received E rows) + tid = own_tid() + o, s = chunk(k) + B = i32(tp) * o + sb = s // i32(NSB) + jx = il(j, cnt(P_RF, s)) + i0 = jx * i32(ZQ) + b = i0 // sb + wait_puts_n(QE, k * i32(NSB) + b + i32(1)) + wait_at(sbc(k, 0, b), (i32(tp) * sb) // i32(ZQ)) + return tid, o, s, B, sb, i0, b, rsrc(wp("erecv")) + + def y_sum(ly, lsrc): + # y of a row's 8 columns: this rank's partial (ly) + every peer's (lsrc(src): its + # E value, loaded here or earlier) + f = bf16x8_to_f32(ly) + for src_ in range_constexpr(tp): + f = sum_live(f, e_val(lsrc(src_)), i32(src_) != rank) + return f + + @flyc.jit + def item_rf(k, j): + tid, o, s, B, sb, i0, b, rv = rf_head(k, j) + rq, rqs = rsrc(wp("q")), rsrc(wp("qs")) + one = fx.Float32(1.0) + # every load of the item first (one memory round trip), then the math + ld = [] + for ii in range_constexpr(ZQ): + i = i0 + i32(ii) + lo = ((o + i) * i32(H)) * i32(2) + row = [] + for kk in range_constexpr(PIT): + off = lo + col(tid, kk) * i32(16) + row.append( + ( + bld(r_y, off, 0, T.vec(4, T.i32), CAUX), + [ + e_ld(rv, B + i32(src_) * s + i, col(tid, kk)) + for src_ in range(tp) + ], + bld(r_resin, off, 0, T.vec(4, T.i32), 0), + ) + ) + ld.append(row) + fss, sss = [], [] + for ii in range_constexpr(ZQ): + i = i0 + i32(ii) + lo = ((o + i) * i32(H)) * i32(2) + fs = [] + ss = fx.Float32(0.0) + for kk in range_constexpr(PIT): + off = lo + col(tid, kk) * i32(16) + ly, lr, lres = ld[ii][kk] + f = y_sum(ly, lambda src_, lr=lr: lr[src_]) + # the MoE output y (this rank's rows) + f = [fx.Float32(x.to(fx.BFloat16).to(fx.Float32)) for x in f] + live = own_col(tid, kk) + if const_expr(live is None): + bst(pack_bf16x8(f), r_yout, off, 0) + else: + yv = pack_bf16x8(f) + f = [x + yy for x, yy in zip(f, bf16x8_to_f32(lres))] + f = [fx.Float32(x.to(fx.BFloat16).to(fx.Float32)) for x in f] + if const_expr(live is None): + bst(pack_bf16x8(f), r_out, off, 0) + for x in f: + ss = ss + x * x + else: + if live: + bst(yv, r_yout, off, 0) + bst(pack_bf16x8(f), r_out, off, 0) + for x in f: + ss = ss + live.select(x * x, fx.Float32(0.0)) + fs.append(f) + fss.append(fs) + sss.append(ss.bitcast(fx.Int32)) + tots = blk_red_vec(L, tid, sss, f32_bits_add) + xss, ams = [], [] + for ii in range_constexpr(ZQ): + rcp = fx.Float32( + fmath.rsqrt( + tots[ii].bitcast(fx.Float32) / fx.Float32(float(H)) + + fx.Float32(float(eps)) + ) + ) + xs_, am = [], fx.Float32(0.0) + for kk in range_constexpr(PIT): + wv = norm_w(r_normw, col(tid, kk)) + xk = [ + fx.Float32( + fx.Float32(gate_w(x * rcp, wk)) + .to(fx.BFloat16) + .to(fx.Float32) + ) + for x, wk in zip(fss[ii][kk], wv) + ] + live = own_col(tid, kk) + if const_expr(live is None): + am = am.maximumf(amax(xk)) + else: + am = am.maximumf(live.select(amax(xk), fx.Float32(0.0))) + xs_.append(xk) + xss.append(xs_) + ams.append(am.bitcast(fx.Int32)) + amxs = blk_red_vec(L, tid, ams, f32_bits_max) + for ii in range_constexpr(ZQ): + i = i0 + i32(ii) + scl = amxs[ii].bitcast(fx.Float32).maximumf( + fx.Float32(1e-10) + ) / fx.Float32(448.0) + inv = one / scl + g = rank * m + o + i + for kk in range_constexpr(PIT): + p = col(tid, kk) + qv = [x * inv for x in xss[ii][kk]] + d = fx.Vector.from_elements( + [fp8x4_pack(qv[0:4], one), fp8x4_pack(qv[4:8], one)], + fx.Int32, + ) + live = own_col(tid, kk) + if const_expr(live is None): + bst(d, rq, g * i32(H) + p * i32(8), 0, AUX_SYS) + else: + if live: + bst(d, rq, g * i32(H) + p * i32(8), 0, AUX_SYS) + if tid == i32(0): + bst(scl.bitcast(fx.Int32), rqs, g * i32(4), 0, AUX_SYS) + done_last(k, P_RF, cnt(P_RF, s)) + # (nested ifs: preserve FlyDSL branches) + if done_at(sbc(k, 1, b), sb // i32(ZQ)) & ( # noqa: SIM102 + i32(int(tailq == 1)) == i32(1) + ): + if tid == i32(0): + g0 = rank * m + o + b * sb + tn = k * i32(NSB) + b + take_turn(QQ, tn) + take_turn(QS, tn) + for p_ in range_constexpr(tp): + if i32(p_) != rank: + put( + i32(p_), + "q", + g0 * i32(H), + "q", + g0 * i32(H), + sb * i32(H), + QQ, + ) + put( + i32(p_), + "qs", + g0 * i32(4), + "qs", + g0 * i32(4), + sb * i32(4), + QS, + ) + give_turn(QQ, tn) + give_turn(QS, tn) + + @flyc.jit + def item_rf_y(k, j): + # io without tail: y = own partial + the peers' MoE-output rows (this rank's rows); + # tailq 3: y rows to the "q" region (global row rank * m + ..), each sub-block then + # put to every peer + tid, o, s, B, sb, i0, b, rv = rf_head(k, j) + ryg = rsrc(wp("q")) + for ii in range_constexpr(ZQ): + i = i0 + i32(ii) + lo = ((o + i) * i32(H)) * i32(2) + for kk in range_constexpr(PIT): + off = lo + col(tid, kk) * i32(16) + f = y_sum( + bld(r_y, off, 0, T.vec(4, T.i32), CAUX), + lambda src_, i=i, kk=kk: e_ld( + rv, B + i32(src_) * s + i, col(tid, kk) + ), + ) + if const_expr(tailq == 3): + bst( + pack_bf16x8(f), + ryg, + (rank * m + o + i) * i32(H * 2) + col(tid, kk) * i32(16), + 0, + AUX_SYS, + ) + else: + bst(pack_bf16x8(f), r_yout, off, 0) + done_last(k, P_RF, cnt(P_RF, s)) + if const_expr(tailq == 3): # noqa: SIM102 - preserve FlyDSL branches + if done_at(sbc(k, 1, b), sb // i32(ZQ)): # noqa: SIM102 + if tid == i32(0): + g0 = rank * m + o + b * sb + tn = k * i32(NSB) + b + take_turn(QQ, tn) + for p_ in range_constexpr(tp): + if i32(p_) != rank: + put( + i32(p_), + "q", + g0 * i32(H * 2), + "q", + g0 * i32(H * 2), + sb * i32(H * 2), + QQ, + ) + give_turn(QQ, tn) + + @flyc.jit + def item_gw(): + for k in range_constexpr(C): + wait_ph(i32(k), P_RF, cnt(P_RF, chunk(i32(k))[1])) + if const_expr(tailq in (1, 3)): + wait_puts(QQ, NSB, i32(C - 1)) + if const_expr(tailq == 1): + wait_puts(QS, NSB, i32(C - 1)) + gpu.barrier() + # every put of this launch landed: the next launch's per-(peer, queue) base, into + # the other buffer (a late wait of this launch still reads this launch's base). + # One thread per peer (tid, not lane: every wave would store it) + if tid < i32(tp): # noqa: SIM102 - preserve FlyDSL branches + if tid != rank: + for qi in range_constexpr(len(QUEUES)): + q_ = QUEUES[qi] + addr = (consumed ^ fx.Int64(tp * NQ * 4)) + fx.Int64( + (tid * i32(NQ) + i32(q_)) * i32(4) + ) + g_st_sys(addr, sig_base(tid, q_) + i32(C * PPC[q_])) + + # ======================= the work loop ======================= + def decode(slices, t): + kind_r = i32(-1) + k_r = i32(0) + jj = i32(0) + for si in range_constexpr(len(slices)): + k, kind, lo_, b0, b1 = slices[si] + hit = (t >= b0) & (t < b1) + kind_r = hit.select(i32(kind), kind_r) + k_r = hit.select(i32(k), k_r) + jj = hit.select(lo_ + (t - b0), jj) + return kind_r, k_r, jj + + rf_item = item_rf_y if tailq in (0, 3) else item_rf + + t = claim() + alive_ = t < nall + while alive_: + kind_r, k_r, jj = decode( + slices_of([chunk_v(i32(k)) for k in range(C)])[0], t + ) + kind_r = uni(kind_r) + k_r = uni(k_r) + jj = uni(jj) + if kind_r == i32(P_RB): + item_rx(k_r, jj) + elif kind_r == i32(P_S1): + item_s1(k_r, jj) + elif kind_r == i32(P_S2): + item_s2(k_r) + elif kind_r == i32(P_S3): + item_s3(k_r, jj) + elif (kind_r == i32(P_U)) | (kind_r == i32(P_UB)): + item_u_own(k_r, jj) + elif kind_r == i32(P_RQ): + item_rq(k_r, jj) + elif kind_r == i32(P_RF): + rf_item(k_r, jj) + if t == gw_lo: + item_gw() + gpu.barrier() + t = claim() + alive_ = t < nall + + kargs = list(inspect.signature(kernel).parameters) + kernel = flyc.kernel(name=name, known_block_size=[NT, 1, 1])(kernel) + + def launch( + wb: fx.Int64, + pbases: fx.Int64, + dc: fx.Int64, + consumed: fx.Int64, + offs: fx.Int64, + xin: fx.Int64, + ids: fx.Int64, + tw: fx.Int64, + normw: fx.Int64, + resin: fx.Int64, + yout: fx.Int64, + resout: fx.Int64, + ybuf: fx.Int64, + routes: fx.Int64, + lrix: fx.Int64, + lwt: fx.Int64, + utab: fx.Int64, + w1: fx.Int64, + w1s: fx.Int64, + w2: fx.Int64, + w2s: fx.Int64, + ctrl: fx.Int64, + chunks: fx.Int64, + rank: fx.Int32, + m: fx.Int32, + ucap: fx.Int32, + grid: fx.Int32, + stream: fx.Stream, + ): + loc = locals() + kernel(*[loc[k] for k in kargs]).launch( + grid=(fx.Int64(grid), 1, 1), block=(NT, 1, 1), stream=stream + ) + + # launch forwards its arguments to the kernel by name + if list(inspect.signature(launch).parameters) != kargs + ["grid", "stream"]: + raise ValueError("compile_bigm_kernel: kernel and launch signatures differ") + launch = flyc.jit(launch) + return launch diff --git a/amd_kernels/moe/mega_moe_tp/mega_moe_tp_tuned.csv b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_tuned.csv new file mode 100644 index 0000000..a67383e --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/mega_moe_tp_tuned.csv @@ -0,0 +1,11 @@ +gfx,cu_num,model_dim,inter_dim,experts,topk,tp,comm_mode,act,lb,tot,tokens,mt,npp,q,pf,lp,ll,llr,us +gfx950,256,6144,768,129,5,4,ag_rs,fp4,1,256,256,4,2,3,1,2,0,0,228.7 +gfx950,256,6144,768,129,5,4,ag_rs,fp4,1,512,512,5,2,2,1,2,0,0,259.7 +gfx950,256,6144,768,129,5,4,ag_rs,fp4,1,1024,1024,5,2,3,0,3,0,0,354.4 +gfx950,256,6144,768,129,5,4,ag_rs,fp4,1,2048,2048,4,2,2,1,0,0,0,529.2 +gfx950,256,6144,768,129,5,4,ag_rs,fp4,1,4096,4096,5,2,2,1,3,0,0,864.5 +gfx950,256,6144,768,129,5,4,ar,fp4,0,8,8,1,1,0,0,0,1,1,83.2 +gfx950,256,6144,768,129,5,4,ar,fp4,0,16,16,2,1,0,0,0,1,1,107.2 +gfx950,256,6144,768,129,5,4,ar,fp4,0,64,64,1,1,0,0,0,1,1,175.7 +gfx950,256,6144,768,129,5,4,ar,fp4,0,96,96,1,1,0,0,0,1,1,196.6 +gfx950,256,6144,768,129,5,4,ar,fp4,0,128,128,1,1,0,0,0,1,1,216.8 diff --git a/amd_kernels/moe/mega_moe_tp/reference.py b/amd_kernels/moe/mega_moe_tp/reference.py new file mode 100644 index 0000000..0530a36 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/reference.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Torch references of MegaMoE TP and SpRsNorm: what "correct" means for the tests and benchmarks. + +MegaMoE TP (``mega_moe_tp_ref``, one rank's partial; the layer's output is the sum of the ranks' partials): + +- MXFP4 (E2M1, two codes per byte, low nibble first) with one e8m0 scale per 32 values along K. The scale is + ``2^ceil(log2(amax / 6))`` (round-up), the codes round to nearest even and saturate at 6 (``mxfp4_quant``). +- The input x [M, H] is quantized per row, GEMM1 runs in fp32 on the dequantized operands; gate = columns [0, I), + up = [I, 2I) of w1 [E, 2I, H]. The activation (``ACTIVATIONS``) runs in fp32 and rounds to bf16, which is quantized + again to MXFP4 per (token, slot) and multiplied with w2 [E, H, I] in fp32. Slot k is scaled by + ``topk_weights[:, k]``, the slots are summed in fp32 and the partial rounds to bf16 once. +- Masked routes (ids < 0 or >= E) contribute nothing. +- The kernel quantizes in other places (FP8 / bf16 exchange, its own MXFP4 rounding of the intermediate), so the + tests compare the summed output by relative L2 over every rank: ``rel_l2 <= RTOL`` (0.1); graph replay and eager + are bitwise equal, as are the ranks' outputs in the ar mode. + +The fused tail (``tail_ref``): res_out = bf16(y + res_in), (Gemma)RMSNorm of res_out in fp32, FP8 E4M3 rows with one +fp32 scale per row ``amax(bf16(normed)) / 448``. + +SpRsNorm (``sp_rs_norm_ref``): the fp32 sum of every rank's partial + the residual, Gemma RMSNorm; its router +(``sigmoid_topk_ref``): bf16 logits, sigmoid scores, top-k of score + bias, weights = scores / their sum * scale, then +the shared expert E with weight shared_w. Tests: rel L2 < 0.01 for the norm, >= 90% identical expert sets (bf16 logits +near ties) and weights within 2% on those rows. +""" + +import torch +import torch.nn.functional as F + +RTOL = 0.10 + +_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0) +# |x| -> E2M1 code, round to nearest even: (upper bound, inclusive?) of codes 0..6, code 7 above +_E2M1_BOUNDS = ((0.25, True), (0.75, False), (1.25, True), (1.75, False), (2.5, True), (3.5, False), (5.0, True)) + + +def _e8m0_roundup(amax: torch.Tensor) -> torch.Tensor: + """uint8 biased exponent of ceil_pow2(amax / 6).""" + u = (amax.float() / 6.0).view(torch.int32) + e = (u >> 23) & 0xFF + bump = ((u & 0x7FFFFF) != 0) & (e < 0xFF) + return torch.where(bump, e + 1, e).to(torch.uint8) + + +def e8m0_to_f32(s: torch.Tensor) -> torch.Tensor: + s = s.view(torch.uint8) + bits = s.to(torch.int32) << 23 + bits = torch.where(s == 0, torch.full_like(bits, 0x00400000), bits) + return bits.view(torch.float32) + + +def _e2m1_codes(y: torch.Tensor) -> torch.Tensor: + a = y.abs() + code = torch.zeros_like(a, dtype=torch.uint8) + for bound, inclusive in _E2M1_BOUNDS: + code += (a > bound if inclusive else a >= bound).to(torch.uint8) + return code | (torch.signbit(y).to(torch.uint8) << 3) + + +def mxfp4_quant(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """x [..., K] -> (codes uint8 [..., K / 2], e8m0 scales uint8 [..., K / 32]).""" + shape = x.shape + blocks = x.float().reshape(-1, 32) + scale = _e8m0_roundup(blocks.abs().amax(-1)) + code = _e2m1_codes(blocks / e8m0_to_f32(scale)[:, None]).view(-1, 2) + packed = code[:, 0] | (code[:, 1] << 4) + return packed.view(*shape[:-1], shape[-1] // 2), scale.view(*shape[:-1], shape[-1] // 32) + + +def mxfp4_dequant(codes: torch.Tensor, scales: torch.Tensor) -> torch.Tensor: + """(codes [..., K / 2], scales [..., K / 32]) -> fp32 [..., K].""" + codes = codes.view(torch.uint8) + lut = torch.tensor(_E2M1 + tuple(-v for v in _E2M1), dtype=torch.float32, device=codes.device) + vals = torch.stack([lut[(codes & 0xF).long()], lut[(codes >> 4).long()]], -1).flatten(-2) + k = vals.shape[-1] + vals = vals.view(*vals.shape[:-1], k // 32, 32) * e8m0_to_f32(scales.reshape(*vals.shape[:-1], k // 32))[..., None] + return vals.view(*vals.shape[:-2], k) + + +def mxfp4_qdq(x: torch.Tensor) -> torch.Tensor: + return mxfp4_dequant(*mxfp4_quant(x)) + + +def _swiglu(g, u, limit=None): + limit = 7.0 if limit is None else limit + g, u = g.clamp(max=limit), u.clamp(min=-limit, max=limit) + return g * torch.sigmoid(1.702 * g) * (u + 1) + + +def _silu(g, u, limit=None): + if limit: + g, u = g.clamp(max=limit), u.clamp(min=-limit, max=limit) + return F.silu(g) * u + + +def _situv2(g, u, beta=4.0, linear_beta=25.0): + return beta * torch.tanh(g / beta) * torch.sigmoid(g) * linear_beta * torch.tanh(u / linear_beta) + + +ACTIVATIONS = ("silu", "swiglu", "situv2") + + +@torch.no_grad() +def mega_moe_tp_ref(x, topk_weights, topk_ids, w1, w1_scale, w2, w2_scale, activation="silu", *, beta=4.0, + linear_beta=25.0, swiglu_limit=None) -> torch.Tensor: + """One rank's partial (bf16 values, fp32 tensor [M, H]) of the MoE of x [M, H] bf16 over this rank's inter slice: + w1 / w2 the row-major codes [E, 2I, H / 2] / [E, H, I / 2] with scales [E, 2I, H / 32] / [E, H, I / 32].""" + M, H = x.shape + E, I2 = w1.shape[0], w1.shape[1] + inter = I2 // 2 + ids = topk_ids.long() + valid = (ids >= 0) & (ids < E) + a1 = mxfp4_qdq(x) + out = torch.zeros((M, topk_ids.shape[1], H), dtype=torch.float32, device=x.device) + for e in torch.unique(ids[valid]).tolist(): + mask = ids == e + rows = mask.nonzero()[:, 0] + h = a1[rows] @ mxfp4_dequant(w1[e], w1_scale[e].view(I2, -1)).t() + g, u = h[:, :inter], h[:, inter:] + if activation == "swiglu": + act = _swiglu(g, u, swiglu_limit) + elif activation == "situv2": + act = _situv2(g, u, beta, linear_beta) + else: + act = _silu(g, u, swiglu_limit) + a2 = mxfp4_qdq(act.to(torch.bfloat16)) + out[mask] = a2 @ mxfp4_dequant(w2[e], w2_scale[e].view(H, -1)).t() + out.mul_(torch.where(valid, topk_weights.float(), 0.0)[..., None]) + return out.sum(1).to(torch.bfloat16).float() + + +def tail_ref(y, res, norm_w, eps, gemma=True): + """(res_out bf16, the fp32 normed rows, their per-row FP8 scales) of the fused tail.""" + r = y.float() + res.float() + w = norm_w.float() + 1.0 if gemma else norm_w.float() + xn = r * torch.rsqrt(r.pow(2).mean(-1, keepdim=True) + eps) * w + scale = xn.to(torch.bfloat16).float().abs().amax(-1).clamp_min(1e-10) / 448.0 + return r.to(torch.bfloat16), xn, scale + + +def sp_rs_norm_ref(part_sum, res, norm_w, eps): + """(res_out, out) fp32 of SpRsNorm's rows: part_sum the fp32 sum of the ranks' partials of those rows.""" + h = part_sum + res.float() + return h, h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + eps) * (norm_w.float() + 1) + + +def sigmoid_topk_ref(normed_bf16, wg, bias, topk, scale): + """(weights fp32 [M, topk], ids int32 [M, topk]) of SpRsNorm's router.""" + logits = (normed_bf16.float() @ wg.float().t()).to(torch.bfloat16).float() + score = torch.sigmoid(logits) + ids = (score + bias.float()).topk(topk, dim=-1).indices + w = score.gather(1, ids) + return w / w.sum(-1, keepdim=True) * scale, ids.to(torch.int32) diff --git a/amd_kernels/moe/mega_moe_tp/schedule.py b/amd_kernels/moe/mega_moe_tp/schedule.py new file mode 100644 index 0000000..743e9fb --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/schedule.py @@ -0,0 +1,887 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/schedule.py (ROCm/aiter edf58c67a). +"""Unit scheduling: dynamic (active-expert row-chunk plan) and LB (two-phase GEMM1 / +GEMM2 units claimed by whichever CTA is free).""" + +from __future__ import annotations + +import functools + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl.expr import gpu, range_constexpr, rocdl +from flydsl.expr.typing import T + +from .fly_utils import ceildiv +from .common import ( + ACQ, + AUX_SYS, + V4I, + _ctpop, + bld, + bst, + g_add_agent, + g_ld_sys, + g_st_sys, + i32, + lds_atomic_add, + lds_atomic_or, + lds_ld, + lds_ld_acq, + lds_ld_i32, + lds_st, + lds_st_rel, + rsrc, + spin0, + spin_lds_ge, + uni, + wait_lgkm0, + wait_vm, + wave_rank, +) +from .mega_moe_tp_config import ( + C_CDONE, + C_CLAIM, + C_DYNP, + C_G1FIN, + C_G2END, + C_G2FREE, + C_G2LAST, + C_G2RDY, + C_G2U, + C_G2V, + C_G2VN, + C_GEXP, + C_LBB, + C_LBJ2, + C_LBN, + C_NACT, + C_NCH, + C_PLAN, + C_RLAND, + C_UL, + C_UNIT, + C_UROWS, + C_USEQ, + CTRL_COLC, + CTRL_G1C, + CTRL_LBR, + CTRL_LBSEQ, + CTRL_XF, + G1C_STRIDE, + LRDY_STRIDE, + N_XCD, + NCHLB_MAX, + NCK_MAX, + NT, + NW, + UL_MAX, + UNIT_G1X, + XQ_SHIFT, + KernelCtx, +) + + +def build_schedule(kc: KernelCtx) -> dict: + """The schedule device functions of one instance (added to kc).""" + DLL, DYN_PS, E, GPC, H2, I, KS2 = kc.get("DLL DYN_PS E GPC H2 I KS2") + L_B1, L_B1S, L_CTL, L_DCH, L_DCNT = kc.get("L_B1 L_B1S L_CTL L_DCH L_DCNT") + L_DPRE, L_DYN, L_EOFF, L_INTER = kc.get("L_DPRE L_DYN L_EOFF L_INTER") + L_INTERS, L_RIX, L_WSUM, L_WT, LB_B = kc.get("L_INTERS L_RIX L_WSUM L_WT LB_B") + LB_PS, LBMIX, LBMIX_DIV, LBPF, MLL = kc.get("LB_PS LBMIX LBMIX_DIV LBPF MLL") + NBW, NCG, NCK, RG, SI_STRIDE, TMAX = kc.get("NBW NCG NCK RG SI_STRIDE TMAX") + TOPK, ag_stage_free, ag_wait_meta = kc.get("TOPK ag_stage_free ag_wait_meta") + ar, cbar, cg_lo, claim_addr = kc.get("ar cbar cg_lo claim_addr") + col_claim, const_expr, ctrl_at = kc.get("col_claim const_expr ctrl_at") + gather_routes_chunk, gemm1, gemm2 = kc.get("gather_routes_chunk gemm1 gemm2") + lb, lrdy_at, masked, meta_ll_wait = kc.get("lb lrdy_at masked meta_ll_wait") + rch, report_chunk, spin_sys_ge = kc.get("rch report_chunk spin_sys_ge") + unit_fields, zero_masked = kc.get("unit_fields zero_masked") + + @flyc.jit + def gemm2_dispatch(L, tid, a, expert, ks0, icnt, r0, rows, signal): + for q in DYN_PS: + if icnt == i32(I // q): + n = KS2 // q + gemm2(L, tid, a, expert, ks0, r0, rows, signal, n, ks0 // i32(n)) + + @flyc.jit + def _drop_stale(tid, a): + if tid < i32(64): + fx.memory_fence(syncscope="agent", ordering=ACQ) # drop stale L1 lines + bid = i32(gpu.block_id("x")) + if (tid < i32(64)) & (bid < i32(N_XCD)) & (a["epoch"] == i32(1)): + fx.memory_fence(syncscope="one-as", ordering=ACQ) + if tid == i32(0): + g_st_sys(ctrl_at(a, i32(CTRL_XF) + bid), a["epoch"]) + + @flyc.jit + def _stale_dropped(tid, a): + if (tid == i32(0)) & (a["epoch"] == i32(1)): + x = i32(gpu.block_id("x")) % i32(N_XCD) + spin_sys_ge(ctrl_at(a, i32(CTRL_XF) + x), a["epoch"]) + + def unit_tile(L, tid, a, expert, i0, icnt, r0, rows, sig): + gemm1(L, tid, a, expert, i0, icnt // i32(128)) + gemm2_dispatch(L, tid, a, expert, i0 // i32(128), icnt, r0, rows, sig) + + @flyc.jit + def compute_units(L, tid, a): + lane = tid % i32(64) + w = tid // i32(64) + if const_expr(MLL): + meta_ll_wait(tid, a) + elif const_expr(not ar): + ag_wait_meta(a, tid, a["epoch"]) + _stale_dropped(tid, a) + if const_expr(ar and not DLL): + zero_masked(L, tid % i32(64), a, tid // i32(64), NW) + cbar(L, tid) + if const_expr(MLL): # noqa: SIM102 + if tid == i32(0): + lds_st_rel(L, L_CTL + C_RLAND * 4, i32(1)) + if const_expr(not ar and not DLL): + zero_masked(L, tid % i32(64), a, tid // i32(64), NW) + if const_expr(lb): + lb_plan(L, tid, a) + else: + dyn_plan(L, tid, a) + ub = lds_ld_i32(L, L_CTL + C_UNIT * 4) + ue = lds_ld_i32(L, L_CTL + (C_UNIT + 1) * 4) + if ub == ue: + report_chunk(L, lane, w, i32(NCK - 1)) + for u_ in range(ub, ue, i32(1)): + u = i32(u_) + expert, i0, icnt, kind = unit_fields(L, u) + if const_expr(lb): + R = lb_routes(L, tid, a, expert) + else: + cc = expert.shrui(i32(24)) + ce = expert.shrui(i32(16)) & i32(0xFF) + expert = expert & i32(0xFFFF) + R = gather_routes_chunk(L, tid, a, expert, cc, ce) + if tid == i32(0): + lds_st(L, L_CTL + C_UROWS * 4, R) + lds_st_rel(L, L_CTL + C_USEQ * 4, u - ub + i32(1)) + if (u == ue - i32(1)) & (R == i32(0)): + report_chunk(L, lane, w, i32(NCK - 1)) + for r0_ in range(i32(0), R, i32(RG)): + r0 = i32(r0_) + rows = fx.min(R - r0, i32(RG)) + if const_expr(lb): + lb_g1_tile(L, tid, a, expert, i0, icnt, rows) + else: + sig = (u == ue - i32(1)) & (r0 + i32(RG) >= R) + unit_tile(L, tid, a, expert, i0, icnt, r0, rows, sig) + cbar(L, tid) + if const_expr(lb): # noqa: SIM102 + if tid == i32(0): + g_add_agent(lb_g1c(a, kind.shrui(i32(XQ_SHIFT)), lb_bank(L)), 1) + if const_expr(LBPF): + lb_g2_consume(L, tid, a) + elif const_expr(lb): + lb_g2_phase(L, tid, a) + if const_expr(DLL): # noqa: SIM102 + if tid == i32(0): + lds_st_rel(L, L_CTL + C_CDONE * 4, i32(1)) + + @flyc.jit + def dyn_plan(L, tid, a): + n = a["ttot"] * i32(TOPK) + rid = rsrc(a["ids"], n * i32(16 if MLL else 4)) + for idx_ in range(tid, n, i32(NT)): + idx = i32(idx_) + e = fx.Int32(bld(rid, idx * i32(16 if MLL else 4), 0, T.i32, 0)) + if masked(e) == fx.Boolean(False): + lds_atomic_or( + L, L_DYN + (e >> i32(5)) * i32(4), i32(1) << (e & i32(31)) + ) + lds_atomic_add(L, L_DCNT + e * i32(4), 1) + cbar(L, tid) + _dyn_prefix(L, tid) + cbar(L, tid) + for it in range_constexpr(ceildiv(E, NT)): + _dyn_list(L, tid + i32(it * NT)) + cbar(L, tid) + _chunk_table(L, tid) + cbar(L, tid) + _dyn_units(L, tid) + cbar(L, tid) + + @flyc.jit + def _dyn_prefix(L, tid): + if tid == i32(0): + acc = i32(0) + for w in range_constexpr(NBW): + lds_st(L, L_DYN + i32((NBW + w) * 4), acc) + acc = acc + _ctpop(lds_ld_i32(L, L_DYN + i32(w * 4))) + lds_st(L, L_CTL + C_NACT * 4, acc) + + @flyc.jit + def _dyn_list(L, e): + if e < i32(E): + wv = lds_ld_i32(L, L_DYN + (e >> i32(5)) * i32(4)) + if ((wv >> (e & i32(31))) & i32(1)) != i32(0): + low = (i32(1) << (e & i32(31))) - i32(1) + rank = lds_ld_i32( + L, L_DYN + (i32(NBW) + (e >> i32(5))) * i32(4) + ) + _ctpop(wv & low) + lds_st(L, L_DYN + (i32(2 * NBW) + rank) * i32(4), e) + + def _pick_p(nun, C, ps): + P = i32(ps[0]) + best = ceildiv(nun * i32(ps[0]), C) * i32(KS2 // ps[0]) + for q in ps[1:]: + cost = ceildiv(nun * i32(q), C) * i32(KS2 // q) + better = (cost < best) & (nun * i32(q) <= C * i32(UL_MAX)) + P = better.select(i32(q), P) + best = better.select(cost, best) + return P + + def _xcd_rank(C): + bid = i32(gpu.block_id("x")) + return (C % i32(N_XCD) == i32(0)).select( + (bid % i32(N_XCD)) * (C // i32(N_XCD)) + bid // i32(N_XCD), bid + ) + + @flyc.jit + def _dyn_units(L, tid): + if tid == i32(0): + nun = lds_ld_i32(L, L_CTL + C_NCH * 4) + C = i32(gpu.grid_dim.x) + P = _pick_p(nun, C, DYN_PS) + U = nun * P + bid = _xcd_rank(C) + u_lo = bid * U // C + u_hi = (bid + i32(1)) * U // C + icnt = i32(I) // P + for u_ in range(u_lo, u_hi, i32(1)): + u = i32(u_) + j = u // P + ul = L_CTL + (i32(C_UL) + (u - u_lo) * i32(4)) * i32(4) + lds_st(L, ul, lds_ld_i32(L, L_DCH + j * i32(4))) + lds_st(L, ul + i32(4), (u - j * P) * icnt) + lds_st(L, ul + i32(8), icnt) + lds_st(L, ul + i32(12), i32(0)) + lds_st(L, L_CTL + C_DYNP * 4, P) + lds_st(L, L_CTL + C_UNIT * 4, i32(0)) + lds_st(L, L_CTL + (C_UNIT + 1) * 4, u_hi - u_lo) + lds_st_rel(L, L_CTL + C_PLAN * 4, i32(1)) + + def lb_g1c(a, j, bank): + return ctrl_at(a, i32(CTRL_G1C) + (bank * i32(NCHLB_MAX) + j) * i32(G1C_STRIDE)) + + def lb_colc(a, c, bank): + return ctrl_at(a, i32(CTRL_COLC) + (bank * i32(NCK_MAX) + c) * i32(LRDY_STRIDE)) + + def lb_lbr(a, bank, x): + return ctrl_at(a, i32(CTRL_LBR) + (bank * i32(N_XCD) + x) * i32(LRDY_STRIDE)) + + def lb_lbr_wait(a, bank, x): + nblk = i32(gpu.grid_dim.x) + spin_sys_ge(lb_lbr(a, bank, x), ceildiv(nblk - x, i32(N_XCD)), a) + + @flyc.jit + def lb_none(L, tid, a, nun): + if (nun == i32(0)) & (tid == i32(0)) & (i32(gpu.block_id("x")) == i32(0)): + lb_lists_wait(a, lb_bank(L)) + for c in range_constexpr(NCK): + g_st_sys(lrdy_at(a, i32(c)), a["epoch"]) + + @flyc.jit + def lb_col_done(L, a, cc, nun, bank): + if g_add_agent(lb_colc(a, cc, bank), 1) == nun - i32(1): + for q_ in range(cg_lo(cc), cg_lo(cc + i32(1)), i32(1)): + g_st_sys(lrdy_at(a, i32(q_)), a["epoch"]) + + @flyc.jit + def lb_lists_wait(a, bank): + for x in range_constexpr(N_XCD): + lb_lbr_wait(a, bank, i32(x)) + + def lb_lists(a): + base = fx.Int64(a["xg"]) + fx.Int64(TMAX * TOPK * (I // 2 + I // 32)) + return rsrc(base), rsrc(base + fx.Int64(TMAX * TOPK * 4)) + + def lb_bank(L): + return lds_ld_i32(L, L_CTL + C_LBB * 4) + + @flyc.jit + def lb_plan(L, tid, a): + n = a["ttot"] * i32(TOPK) + nblk = i32(gpu.grid_dim.x) + per = ceildiv(n, nblk) + lo = fx.min(i32(gpu.block_id("x")) * per, n) + hi = fx.min(lo + per, n) + if tid == i32(0): + lds_st(L, L_CTL + C_LBB * 4, g_ld_sys(ctrl_at(a, i32(CTRL_LBSEQ))) & i32(1)) + rid = rsrc(a["ids"], n * i32(4)) + n4 = n // i32(4) + for r0_ in range(i32(0), n4, i32(NT * LB_B)): + r0 = i32(r0_) + vs = [ + fx.Vector( + bld( + rid, + fx.min(r0 + tid + i32(it * NT), n4 - i32(1)) * i32(16), + 0, + V4I, + 0, + ) + ) + for it in range(LB_B) + ] + for it in range_constexpr(LB_B): + q = r0 + tid + i32(it * NT) + for j in range_constexpr(4): + e = fx.Int32(vs[it][j]) + live, ahead = q < n4, q * i32(4) + i32(j) < lo + if live & (masked(e) == fx.Boolean(False)): + lds_atomic_add(L, L_DCNT + e * i32(4), 1) + if ahead: + lds_atomic_add(L, L_DPRE + e * i32(4), 1) + for idx_ in range(n4 * i32(4) + tid, n, i32(NT)): + idx = i32(idx_) + e = fx.Int32(bld(rid, idx * i32(4), 0, T.i32, 0)) + ahead = idx < lo + if masked(e) == fx.Boolean(False): + lds_atomic_add(L, L_DCNT + e * i32(4), 1) + if ahead: + lds_atomic_add(L, L_DPRE + e * i32(4), 1) + cbar(L, tid) + _lb_bitmap(L, tid) + cbar(L, tid) + _dyn_prefix(L, tid) + cbar(L, tid) + for it in range_constexpr(ceildiv(E, NT)): + _dyn_list(L, tid + i32(it * NT)) + cbar(L, tid) + _chunk_table(L, tid, True) + lb_scatter(L, tid, a, lo, hi) + _lb_units(L, tid) + cbar(L, tid) + + @flyc.jit + def _lb_bitmap(L, tid): + lane = tid % i32(64) + w = tid // i32(64) + for rnd in range_constexpr(ceildiv(E, NT)): + e = tid + i32(rnd * NT) + c = (e < i32(E)).select( + lds_ld_i32(L, L_DCNT + fx.min(e, i32(E - 1)) * i32(4)), i32(0) + ) + b = fx.Int64(rocdl.ballot(T.i64, c > i32(0))) + for h in range_constexpr(2): + wi = i32(rnd * NT // 32 + h) + w * i32(2) + if (lane == i32(0)) & (wi < i32(NBW)): + lds_st( + L, + L_DYN + wi * i32(4), + i32((b >> fx.Int64(32 * h)) & fx.Int64(0xFFFFFFFF)), + ) + + @flyc.jit + def lb_zero_next(tid, a, bank): + nb = bank ^ i32(1) + bid = i32(gpu.block_id("x")) + nblk = i32(gpu.grid_dim.x) + if tid < i32(64): + for j_ in range(bid + tid * nblk, i32(NCHLB_MAX), nblk * i32(64)): + g_st_sys(lb_g1c(a, i32(j_), nb), i32(0)) + if bid == i32(0): + if tid < i32(NCK): + g_st_sys(lb_colc(a, tid, nb), i32(0)) + if (tid >= i32(NCK)) & (tid < i32(NCK + N_XCD)): + g_st_sys(lb_lbr(a, nb, tid - i32(NCK)), i32(0)) + + def _wave_prefix(v, bits): + pre, tot = i32(0), i32(0) + for b in range_constexpr(bits): + below, cnt = wave_rank(((v >> i32(b)) & i32(1)) == i32(1)) + pre = pre + (below << i32(b)) + tot = tot + (cnt << i32(b)) + return pre, tot + + @flyc.jit + def _chunk_table(L, tid, offs=False): + nact = lds_ld_i32(L, L_CTL + C_NACT * 4) + w = tid // i32(64) + lane = tid % i32(64) + base = [i32(0), i32(0)] + for rnd in range_constexpr(ceildiv(E, NT)): + j = tid + i32(rnd * NT) + live = j < nact + e = live.select( + lds_ld_i32(L, L_DYN + (i32(2 * NBW) + fx.min(j, i32(E - 1))) * i32(4)), + i32(0), + ) + r = lds_ld_i32(L, L_DCNT + e * i32(4)) + r = live.select(r, i32(0)) if offs else r + ce = fx.min(fx.max(ceildiv(r, i32(rch)), i32(1)), i32(255)) + ce = live.select(ce, i32(0)) + sums = [_wave_prefix(ce, 8)] + ( + [_wave_prefix(r, TMAX.bit_length())] if offs else [] + ) + if lane == i32(0): + for q in range_constexpr(len(sums)): + lds_st(L, L_WSUM + (w + i32(q * NW)) * i32(4), sums[q][1]) + cbar(L, tid) + off = [] + for q in range_constexpr(len(sums)): + o, total = base[q] + sums[q][0], i32(0) + for v in range_constexpr(NW): + sv = lds_ld_i32(L, L_WSUM + i32((q * NW + v) * 4)) + o = o + (i32(v) < w).select(sv, i32(0)) + total = total + sv + off.append(o) + base[q] = base[q] + total + if const_expr(offs): # noqa: SIM102 + if live: + lds_st(L, L_EOFF + e * i32(4), off[1]) + for c_ in range(i32(0), ce, i32(1)): + c = i32(c_) + lds_st( + L, + L_DCH + (off[0] + c) * i32(4), + e | (ce << i32(16)) | (c << i32(24)), + ) + cbar(L, tid) + if tid == i32(0): + lds_st(L, L_CTL + C_NCH * 4, base[0]) + if const_expr(offs): + cbar(L, tid) + + @flyc.jit + def lb_scatter(L, tid, a, lo, hi): + n = a["ttot"] * i32(TOPK) + rid = rsrc(a["ids"], n * i32(4)) + rtw = rsrc(a["tw"], n * i32(4)) + rl, rw = lb_lists(a) + for idx_ in range(lo + tid, hi, i32(NT)): + idx = i32(idx_) + e = fx.Int32(bld(rid, idx * i32(4), 0, T.i32, 0)) + if masked(e) == fx.Boolean(False): + pos = lds_atomic_add(L, L_DPRE + e * i32(4), 1) + slot = lds_ld_i32(L, L_EOFF + e * i32(4)) + pos + wv = fx.Int32(bld(rtw, idx * i32(4), 0, T.i32, 0)) + bst(idx, rl, slot * i32(4), 0, AUX_SYS) + bst(wv, rw, slot * i32(4), 0, AUX_SYS) + wait_vm(0) + cbar(L, tid) + if tid == i32(0): + g_add_agent(lb_lbr(a, lb_bank(L), i32(gpu.block_id("x")) % i32(N_XCD)), 1) + + @flyc.jit + def _lb_units(L, tid): + if tid == i32(0): + nun = lds_ld_i32(L, L_CTL + C_NCH * 4) + C = i32(gpu.grid_dim.x) + P = _pick_p(nun, C, LB_PS) + U = nun * P + bid = _xcd_rank(C) + icnt = i32(I) // P + mix = fx.Boolean(False) + J2 = nun + if const_expr(LBMIX): + X = i32(2) * nun - C + K = ceildiv(X, i32(2)) + M = i32(2) * (nun - K) + mix = (X > i32(0)) & (X <= C // i32(LBMIX_DIV)) & (i32(H2) * X <= M) + P = mix.select(i32(2), P) + U = mix.select(i32(0), U) + J2 = mix.select(nun - K, nun) + if mix: + _lb_units_mix(L, bid, C, nun, K, M, X) + lds_st(L, L_CTL + C_LBJ2 * 4, J2) + for u_ in range(bid, U, C): + u = i32(u_) + j = u // P + s = u - j * P + ul = L_CTL + (i32(C_UL) + ((u - bid) // C) * i32(4)) * i32(4) + lds_st(L, ul, lds_ld_i32(L, L_DCH + j * i32(4))) + lds_st(L, ul + i32(4), s * icnt) + lds_st(L, ul + i32(8), icnt) + lds_st( + L, + ul + i32(12), + i32(UNIT_G1X) | (s << i32(8)) | (j << i32(XQ_SHIFT)), + ) + mine = (bid < U).select(ceildiv(U - bid, C), i32(0)) + if const_expr(LBMIX): + mine = mix.select(lds_ld_i32(L, L_CTL + (C_UNIT + 1) * 4), mine) + lds_st(L, L_CTL + C_DYNP * 4, P) + lds_st(L, L_CTL + C_UNIT * 4, i32(0)) + lds_st(L, L_CTL + (C_UNIT + 1) * 4, mine) + lds_st_rel(L, L_CTL + C_PLAN * 4, i32(1)) + + def lb_pieces(L, j): + P = lds_ld_i32(L, L_CTL + C_DYNP * 4) + if const_expr(LBMIX): + P = (j >= lds_ld_i32(L, L_CTL + C_LBJ2 * 4)).select(i32(KS2), P) + return P + + def _lb_ul(L, n, ent, i0, icnt, kind): + ul = L_CTL + (i32(C_UL) + n * i32(4)) * i32(4) + lds_st(L, ul, ent) + lds_st(L, ul + i32(4), i0) + lds_st(L, ul + i32(8), icnt) + lds_st(L, ul + i32(12), kind) + + @flyc.jit + def _lb_units_mix(L, r, C, nun, K, M, X): + J2 = nun - K + if r < M: + j = r // i32(2) + s_ = r - j * i32(2) + _lb_ul( + L, + i32(0), + lds_ld_i32(L, L_DCH + j * i32(4)), + s_ * i32(I // 2), + i32(I // 2), + i32(UNIT_G1X) | (s_ << i32(8)) | (j << i32(XQ_SHIFT)), + ) + lds_st(L, L_CTL + (C_UNIT + 1) * 4, i32(1)) + else: + for k in range_constexpr(H2): + f = (r - M) * i32(H2) + i32(k) + j = J2 + f // i32(KS2) + s_ = f - (j - J2) * i32(KS2) + _lb_ul( + L, + i32(k), + lds_ld_i32(L, L_DCH + j * i32(4)), + s_ * i32(128), + i32(128), + i32(UNIT_G1X) | (s_ << i32(8)) | (j << i32(XQ_SHIFT)), + ) + lds_st(L, L_CTL + (C_UNIT + 1) * 4, i32(H2)) + if r < i32(H2) * X: + f = (C - M) * i32(H2) + r + j = J2 + f // i32(KS2) + s_ = f - (j - J2) * i32(KS2) + nn = lds_ld_i32(L, L_CTL + (C_UNIT + 1) * 4) + _lb_ul( + L, + nn, + lds_ld_i32(L, L_DCH + j * i32(4)), + s_ * i32(128), + i32(128), + i32(UNIT_G1X) | (s_ << i32(8)) | (j << i32(XQ_SHIFT)), + ) + lds_st(L, L_CTL + (C_UNIT + 1) * 4, nn + i32(1)) + + @flyc.jit + def lb_routes(L, tid, a, ent): + e = ent & i32(0xFFFF) + r0 = ent.shrui(i32(24)) * i32(rch) + R = fx.min(lds_ld_i32(L, L_DCNT + e * i32(4)) - r0, i32(rch)) + base = lds_ld_i32(L, L_EOFF + e * i32(4)) + r0 + if (tid < i32(N_XCD)) & (lds_ld_i32(L, L_CTL + C_LBN * 4) == i32(0)): + lb_lbr_wait(a, lb_bank(L), tid) + cbar(L, tid) + if (tid == i32(0)) & (lds_ld_i32(L, L_CTL + C_LBN * 4) == i32(0)): + lds_st(L, L_CTL + C_LBN * 4, i32(1)) + cbar(L, tid) + rl, rw = lb_lists(a) + if tid < R: + lds_st( + L, + L_RIX + tid * i32(4), + fx.Int32(bld(rl, (base + tid) * i32(4), 0, T.i32, AUX_SYS)), + ) + lds_st( + L, + L_WT + tid * i32(4), + fx.Int32(bld(rw, (base + tid) * i32(4), 0, T.i32, AUX_SYS)), + ) + if tid == i32(0): + lds_st(L, L_CTL + C_GEXP * 4, i32(0)) + cbar(L, tid) + return R + + def lb_g1_tile(L, tid, a, ent, i0, icnt, rows): + gemm1(L, tid, a, ent & i32(0xFFFF), i0, icnt // i32(128), rows) + xq_export(L, tid, a, i0, icnt, i32(0), rows) + + def lb_g2_cap(U2, nblk): + return ceildiv(U2, nblk) * i32(4) + i32(4) + + @flyc.jit + def lb_g2_consume(L, tid, a): + nun = lds_ld_i32(L, L_CTL + C_NCH * 4) + nblk = i32(gpu.grid_dim.x) + lb_none(L, tid, a, nun) + if tid == i32(0): + lds_st_rel(L, L_CTL + C_G1FIN * 4, i32(1)) + cap = lb_g2_cap(nun * i32(NCG), nblk) + for it_ in range(i32(0), cap, i32(1)): + _lb_g2_take(L, tid, a, nun, i32(it_)) + + @flyc.jit + def spin_g2(L, target): + cur = lds_ld_acq(L, L_CTL + C_G2RDY * 4) + end = lds_ld_acq(L, L_CTL + C_G2END * 4) + while (cur < target) & (end == i32(0)): + rocdl.s_sleep(0) + cur = lds_ld_acq(L, L_CTL + C_G2RDY * 4) + end = lds_ld_acq(L, L_CTL + C_G2END * 4) + + @flyc.jit + def _lb_g2_take(L, tid, a, nun, n): + if tid == i32(0): + spin_g2(L, n + i32(1)) + cbar(L, tid) + if lds_ld_acq(L, L_CTL + C_G2RDY * 4) > n: + b = n % i32(2) + u = L_CTL + (i32(C_G2U) + b * i32(4)) * i32(4) + ent = lds_ld_i32(L, u) + cc = lds_ld_i32(L, u + i32(4)) + R = lds_ld_i32(L, u + i32(8)) + gemm2( + L, + tid, + a, + ent & i32(0xFFFF), + i32(0), + i32(0), + R, + fx.Boolean(False), + KS2, + i32(0), + cg_lo(cc) * i32(GPC), + cg_lo(cc + i32(1)) * i32(GPC), + buf=b, + progress=functools.partial(_g2_last, L, tid, n), + nxt=n, + ) + cbar(L, tid) + if tid == i32(0): + lds_st_rel(L, L_CTL + C_G2FREE * 4, n + i32(1)) + lb_col_done(L, a, cc, nun, lb_bank(L)) + + @flyc.jit + def _g2_last(L, tid, n): + if tid == i32(0): + lds_st_rel(L, L_CTL + C_G2LAST * 4, n + i32(1)) + + @flyc.jit + def lb_g2_prefetch(L, lane, a): + spin0(L, lane, L_CTL + C_G1FIN * 4, i32(1)) + nun = lds_ld_i32(L, L_CTL + C_NCH * 4) + U2 = nun * i32(NCG) + bank = lb_bank(L) + if (lane < i32(N_XCD)) & (lds_ld_i32(L, L_CTL + C_LBN * 4) == i32(0)): + lb_lbr_wait(a, bank, lane) + rocdl.sched_barrier(0) + cap = lb_g2_cap(U2, i32(gpu.grid_dim.x)) + for it_ in range(i32(0), cap, i32(1)): + _lb_g2_fetch(L, lane, a, nun, U2, bank, i32(it_)) + if lane == i32(0): + lds_st_rel(L, L_CTL + C_G2END * 4, i32(1)) + + @flyc.jit + def _lb_g2_fetch(L, lane, a, nun, U2, bank, n): + if lds_ld_acq(L, L_CTL + C_G2END * 4) == i32(0): + if (lane == i32(0)) & (n > i32(0)): + spin_lds_ge(L, L_CTL + C_G2LAST * 4, n) + rocdl.sched_barrier(0) + if lane == i32(0): + lds_st(L, L_CTL + C_G2V * 4, g_add_agent(claim_addr(a, 0), 1)) + lds_st_rel(L, L_CTL + C_G2VN * 4, n + i32(1)) + rocdl.sched_barrier(0) + v = uni(lds_ld_acq(L, L_CTL + C_G2V * 4)) + if v >= U2: + if lane == i32(0): + lds_st_rel(L, L_CTL + C_G2END * 4, i32(1)) + else: + _lb_g2_load(L, lane, a, nun, bank, n, v) + + @flyc.jit + def _lb_g2_load(L, lane, a, nun, bank, n, v): + b = n % i32(2) + if lane == i32(0): + spin_lds_ge(L, L_CTL + C_G2FREE * 4, n - i32(1)) + rocdl.sched_barrier(0) + cc = v // nun + j = v - cc * nun + ent = lds_ld_i32(L, L_DCH + j * i32(4)) + e = ent & i32(0xFFFF) + r0 = ent.shrui(i32(24)) * i32(rch) + R = fx.min(lds_ld_i32(L, L_DCNT + e * i32(4)) - r0, i32(rch)) + base = lds_ld_i32(L, L_EOFF + e * i32(4)) + r0 + if lane == i32(0): + spin_sys_ge(lb_g1c(a, j, bank), lb_pieces(L, j), a) + rocdl.sched_barrier(0) + rix0 = i32(L_RIX) + b * i32(128 * 4) + wt0 = i32(L_WT) + b * i32(128 * 4) + ib0 = (b == i32(0)).select(i32(L_INTER), i32(L_B1)) + is0 = (b == i32(0)).select(i32(L_INTERS), i32(L_B1S)) + rl, rw = lb_lists(a) + rv = [] + for k in range_constexpr(ceildiv(RG, 64)): + r = lane + i32(k * 64) + rc = fx.min(r, R - i32(1)) + rv.append( + ( + r, + bld(rl, (base + rc) * i32(4), 0, T.i32, AUX_SYS), + bld(rw, (base + rc) * i32(4), 0, T.i32, AUX_SYS), + ) + ) + for r, iv, wv in rv: + if r < i32(RG): + lds_st(L, rix0 + r * i32(4), fx.Int32(iv)) + lds_st(L, wt0 + r * i32(4), fx.Int32(wv)) + wait_lgkm0() + rx, rxs = xg_rs(a) + u16 = I // 2 // 16 + nq = fx.max(R * i32(u16), i32(1)) + NQ = ceildiv(RG * u16, 64) + for b0 in range_constexpr(0, NQ, 8): + dq = [] + for k in range_constexpr(b0, min(b0 + 8, NQ)): + q = fx.min(lane + i32(k * 64), nq - i32(1)) + row = q // i32(u16) + c = q - row * i32(u16) + rix = lds_ld_i32(L, rix0 + row * i32(4)) + dq.append( + (row, c, bld(rx, rix * i32(I // 2) + c * i32(16), 0, V4I, AUX_SYS)) + ) + for row, c, dv in dq: + lds_st(L, ib0 + row * i32(SI_STRIDE) + c * i32(16), dv, 16) + ns = fx.max(R * i32(I // 128), i32(1)) + ds = [] + for k in range_constexpr(ceildiv(RG * (I // 128), 64)): + q = fx.min(lane + i32(k * 64), ns - i32(1)) + row = q // i32(I // 128) + c = q - row * i32(I // 128) + rix = lds_ld_i32(L, rix0 + row * i32(4)) + ds.append( + (row, c, bld(rxs, rix * i32(I // 32) + c * i32(4), 0, T.i32, AUX_SYS)) + ) + for row, c, sv in ds: + lds_st(L, is0 + row * i32(I // 32) + c * i32(4), fx.Int32(sv)) + wait_lgkm0() + if lane == i32(0): + u = L_CTL + (i32(C_G2U) + b * i32(4)) * i32(4) + lds_st(L, u, ent) + lds_st(L, u + i32(4), cc) + lds_st(L, u + i32(8), R) + lds_st(L, u + i32(12), j) + lds_st_rel(L, L_CTL + C_G2RDY * 4, n + i32(1)) + rocdl.sched_barrier(0) + + @flyc.jit + def lb_g2_phase(L, tid, a): + nun = lds_ld_i32(L, L_CTL + C_NCH * 4) + U2 = nun * i32(NCG) + nblk = i32(gpu.grid_dim.x) + lb_none(L, tid, a, nun) + cap = ceildiv(U2, nblk) * i32(2) + i32(2) + col_claim(L, tid, a) + for it_ in range(i32(0), cap, i32(1)): + _lb_g2_unit(L, tid, a, nun, U2, i32(it_) < cap - i32(1)) + + @flyc.jit + def _lb_g2_unit(L, tid, a, nun, U2, more): + v = lds_ld_i32(L, L_CTL + C_CLAIM * 4) + if v < U2: + cc = v // nun + _lb_g2_run(L, tid, a, nun, cc, v - cc * nun) + if more: + col_claim(L, tid, a) + + @flyc.jit + def _lb_g2_run(L, tid, a, nun, cc, j): + ent = lds_ld_i32(L, L_DCH + j * i32(4)) + bank = lb_bank(L) + if tid == i32(0): + spin_sys_ge(lb_g1c(a, j, bank), lb_pieces(L, j), a) + R = lb_routes(L, tid, a, ent) + gemm2( + L, + tid, + a, + ent & i32(0xFFFF), + i32(0), + i32(0), + R, + fx.Boolean(False), + KS2, + i32(0), + cg_lo(cc) * i32(GPC), + cg_lo(cc + i32(1)) * i32(GPC), + pre=functools.partial(lb_import, L, tid, a, R), + ) + cbar(L, tid) + if tid == i32(0): + lb_col_done(L, a, cc, nun, bank) + + @flyc.jit + def lb_import(L, tid, a, rows): + ag_stage_free(L, tid % i32(64), a) + cbar(L, tid) + u16 = I // 2 // 16 + rx, rxs = xg_rs(a) + nq = fx.max(rows * i32(u16), i32(1)) + ns = fx.max(rows * i32(I // 128), i32(1)) + dq, ds = [], [] + for k in range_constexpr(ceildiv(RG * u16, NT)): + q = fx.min(tid + i32(k * NT), nq - i32(1)) + row = q // i32(u16) + c = q - row * i32(u16) + rix = lds_ld_i32(L, L_RIX + row * i32(4)) + dq.append( + (row, c, bld(rx, rix * i32(I // 2) + c * i32(16), 0, V4I, AUX_SYS)) + ) + for k in range_constexpr(ceildiv(RG * (I // 128), NT)): + q = fx.min(tid + i32(k * NT), ns - i32(1)) + row = q // i32(I // 128) + c = q - row * i32(I // 128) + rix = lds_ld_i32(L, L_RIX + row * i32(4)) + ds.append( + (row, c, bld(rxs, rix * i32(I // 32) + c * i32(4), 0, T.i32, AUX_SYS)) + ) + for row, c, v in dq: + lds_st(L, i32(L_INTER) + row * i32(SI_STRIDE) + c * i32(16), v, 16) + for row, c, sv in ds: + lds_st(L, i32(L_INTERS) + row * i32(I // 32) + c * i32(4), fx.Int32(sv)) + cbar(L, tid) + + @flyc.jit + def lb_finish(tid, a, L): + lb_zero_next(tid, a, lb_bank(L)) + if (tid == i32(0)) & (i32(gpu.block_id("x")) == i32(0)): + lb_lists_wait(a, lb_bank(L)) + g_add_agent(ctrl_at(a, i32(CTRL_LBSEQ)), 1) + + def xg_rs(a): + rx = rsrc(a["xg"]) + rxs = rsrc(fx.Int64(a["xg"]) + fx.Int64(a["ttot"] * i32(TOPK * (I // 2)))) + return rx, rxs + + @flyc.jit + def xq_export(L, tid, a, i0, icnt, r0, rows): + u16 = icnt // i32(32) + rx, rxs = xg_rs(a) + for q_ in range(tid, rows * u16, i32(NT)): + q = i32(q_) + row = q // u16 + c = q - row * u16 + rix = lds_ld_i32(L, L_RIX + (r0 + row) * i32(4)) + v = lds_ld(L, i32(L_INTER) + row * i32(SI_STRIDE) + c * i32(16), V4I, 16) + bst(v, rx, rix * i32(I // 2) + i0 // i32(2) + c * i32(16), 0, AUX_SYS) + nsd = icnt // i32(128) + for q_ in range(tid, rows * nsd, i32(NT)): + q = i32(q_) + row = q // nsd + c = q - row * nsd + rix = lds_ld_i32(L, L_RIX + (r0 + row) * i32(4)) + sv = lds_ld_i32(L, i32(L_INTERS) + row * i32(I // 32) + c * i32(4)) + bst(sv, rxs, rix * i32(I // 32) + i0 // i32(32) + c * i32(4), 0, AUX_SYS) + wait_vm(0) + + return locals() diff --git a/amd_kernels/moe/mega_moe_tp/sdma.py b/amd_kernels/moe/mega_moe_tp/sdma.py new file mode 100644 index 0000000..2e68651 --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/sdma.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MORI SDMA queues of the big-M kernel (``bigm_compute.MegaMoeTPBigM``). + +From AITER ``aiter/ops/communication.py`` (the MORI SDMA part, ROCm/aiter edf58c67a). + +MORI CCO provides the SDMA queues and their signal slots (its devComm); the kernel posts remote-signalled puts (each +lands on the peer, then bumps the peer's signal slot for (sender, queue)) on raw addresses: the data regions live in +a same-layout hipIpc arena on every rank (``SymmetricArena``, torch caching-allocator memory), not in a MORI window. A +MORI window is a VMM allocation (hipMemCreate), which ROCm can refuse after a pageable host copy pinned a low heap +address (KFD's SVM-range overlap check, reported as out of memory); hipMalloc'd memory is not affected. + +Needs MORI built with ``BUILD_CCO_SDMA=ON`` whose CCO device wrapper (libmori_cco_device.bc) provides +``cco_sdma_put_ptr_rsig__thread`` / ``cco_sdma_signal_slot``. MORI is imported on first use. +""" + +import contextlib +import functools +import os +import sys + +import torch +import torch.distributed as dist + + +@contextlib.contextmanager +def _mori_env(): + """For MORI's import / communicator setup only (the environment is restored): ``AKR_MORI_PATH`` (MORI's install + directory, if not importable) on sys.path, and MORI_ENABLE_SDMA=1 unless set (MORI builds the SDMA queues at + communicator creation).""" + path = os.environ.get("AKR_MORI_PATH") + added = bool(path) and path not in sys.path + if added: + sys.path.insert(0, path) + unset = "MORI_ENABLE_SDMA" not in os.environ + if unset: + os.environ["MORI_ENABLE_SDMA"] = "1" + try: + yield + finally: + if added and path in sys.path: + sys.path.remove(path) + if unset: + os.environ.pop("MORI_ENABLE_SDMA", None) + + +def _mori(): + with _mori_env(): + from mori.cco import GDA_CONNECTION_NONE, CCODevCommRequirements, Communicator + + return Communicator, CCODevCommRequirements, GDA_CONNECTION_NONE + + +def sdma_bitcode() -> str: + """MORI's CCO device bitcode (code object version 6), which the big-M kernel links.""" + with _mori_env(): + from mori.cco.device.bitcode import find_cco_bitcode + + return find_cco_bitcode(6) + + +class SdmaComm: + """MORI SDMA queues (``num_queues`` per peer) of ``group`` plus a same-layout data arena on every rank. + ``reserve`` carves named regions (same order on every rank), ``commit`` (collective) creates the MORI + communicator / devComm and allocates and maps the arena. ``base`` is this rank's arena address, ``bases`` [world] + int64 every rank's arena address as mapped here (the kernel's put destinations).""" + + def __init__(self, *, group=None, num_queues: int = 8, device=None): + from .mega_moe_tp import SymmetricArena + + Communicator, Reqs, gda_none = _mori() + self.group = group + self.rank = dist.get_rank(group) + self.world = dist.get_world_size(group) + self.device = device or torch.device("cuda", torch.cuda.current_device()) + self.nq = num_queues + self._comm_cls, self._reqs_cls, self._gda_none = Communicator, Reqs, gda_none + self.arena = SymmetricArena(group=group, device=self.device) + self._slices: dict = {} + self._committed = False + + def reserve(self, name: str, nbytes: int) -> None: + if self._committed or name in self._slices: + raise RuntimeError(f"SdmaComm: cannot reserve {name!r}") + self._slices[name] = self.arena.reserve(name, (max(int(nbytes), 1),), torch.uint8) + + def offset(self, name: str) -> int: + return self._slices[name].offset + + def commit(self) -> "SdmaComm": + uid = [self._comm_cls.get_unique_id() if self.rank == 0 else None] + src0 = dist.get_global_rank(self.group, 0) if self.group else 0 + dist.broadcast_object_list(uid, src=src0, group=self.group) + err = "" + try: + with _mori_env(): + # per_rank_vmm: only a virtual-address reservation (rounded up to 4 GiB by MORI); the data lives in + # the hipIpc arena + self.comm = self._comm_cls.init(self.world, self.rank, uid[0], per_rank_vmm=1 << 30) + reqs = self._reqs_cls() + reqs.gda_connection_type = self._gda_none + reqs.gda_signal_count = 0 + reqs.gda_counter_count = 0 + reqs.sdma_queue_count = self.nq + self.dev_comm = self.comm.create_dev_comm(reqs) + except Exception as exc: # noqa: BLE001 (reported on every rank below) + err = f"{type(exc).__name__}: {exc}" + errs: list = [None] * self.world + dist.all_gather_object(errs, err, group=self.group) + bad = {r: e for r, e in enumerate(errs) if e} + if bad: + raise RuntimeError(f"SdmaComm: MORI SDMA setup failed on ranks {bad}") + try: + self.arena.commit() # (raises on every rank if any rank fails) + except BaseException: + self.dev_comm.close() + self.comm.destroy() + raise + self.storage = self.arena.storage + self.base = int(self.storage.data_ptr()) + self.bases = torch.tensor(self.arena.base_ptrs, dtype=torch.int64, device=self.device) + # [2][world * nq] int32 per-(peer, queue) signal bases: each launch reads one buffer and writes the next + # launch's into the other + self.consumed = torch.zeros((2, self.world * self.nq), dtype=torch.int32, device=self.device) + self._committed = True + torch.cuda.synchronize(self.device) + dist.barrier(group=self.group) + return self + + def close(self) -> None: + """Collective: free the MORI device communicator / communicator (its SDMA queues) and the arena.""" + if not self._committed: + return + self._committed = False + try: + torch.cuda.synchronize(self.device) + dist.barrier(group=self.group) + try: + self.dev_comm.close() + finally: + self.comm.destroy() + finally: + self.arena.close() + + def view(self, name: str, shape, dtype: torch.dtype) -> torch.Tensor: + """A torch view of region ``name``.""" + sl = self._slices[name] + n = torch.Size(shape).numel() * dtype.itemsize + assert n <= sl.nbytes, (name, n, sl.nbytes) + return self.storage[sl.offset : sl.offset + n].view(dtype).view(shape) + + +@functools.cache +def compile_signal_read(tp, nq): + """launch(dc, out, stream): out [tp][nq] int32 <- the signal count received per (sender, queue) (MORI's signal + slots, low 32 bits).""" + import flydsl.compiler as flyc + import flydsl.expr as fx + from flydsl.compiler.extern_link import link_extern + from flydsl.expr import gpu + from flydsl.expr.extern import ffi + + from .common import g_ld_sys, g_st_sys, i32 + + sig_slot = link_extern( + ffi("cco_sdma_signal_slot", ["uint64", "int32", "int32"], "uint64"), + bitcode_path=sdma_bitcode(), + ) + + @flyc.kernel(name=f"sdma_signal_read_tp{tp}_q{nq}", known_block_size=[64, 1, 1]) + def kernel(dc: fx.Int64, out: fx.Int64): + lane = fx.Int32(gpu.thread_id("x")) + if lane < i32(tp * nq): + slot = fx.Int64(sig_slot(dc, lane // i32(nq), lane % i32(nq))) + g_st_sys(out + fx.Int64(lane * i32(4)), g_ld_sys(slot)) + + @flyc.jit + def launch(dc: fx.Int64, out: fx.Int64, stream: fx.Stream): + kernel(dc, out).launch(grid=(1, 1, 1), block=(64, 1, 1), stream=stream) + + return launch diff --git a/amd_kernels/moe/mega_moe_tp/sp_rs_norm.py b/amd_kernels/moe/mega_moe_tp/sp_rs_norm.py new file mode 100644 index 0000000..726434a --- /dev/null +++ b/amd_kernels/moe/mega_moe_tp/sp_rs_norm.py @@ -0,0 +1,852 @@ +# SPDX-License-Identifier: MIT +# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +# From AITER aiter/ops/flydsl/kernels/mega_moe_tp/sp_rs_norm.py (ROCm/aiter edf58c67a). +"""Sequence-parallel int8 reduce-scatter + residual add + RMSNorm (+ MoE router), one kernel.""" + +from __future__ import annotations + +import functools +import struct + +import flydsl.compiler as flyc +import flydsl.expr as fx +import torch +import torch.distributed as dist +from flydsl.expr import gpu, range_constexpr, rocdl +from flydsl.expr import math as fmath +from flydsl.expr.typing import T + +from .fly_utils import LOG2E, _preload_compiled, _run_compiled, ceildiv +from .common import ( + AUX_SYS, + MAX_TP, + amax, + before, + bf16x8_to_f32, + bld, + bst, + f32_bits_add, + g_or_agent, + g_st_sys, + gptr, + i8x4_pack, + i8x4_unpack, + i32, + lds_ld_i32, + lds_st, + pack_bf16x8, + poll_sys_ge, + rsrc, + sum_live, + uni, + wave_red, +) +from .mega_moe_tp import SymmetricArena + +__all__ = ["SpRsNorm"] + +NTH = 256 +KS_MAX = 16 +const_expr = fx.const_expr + + +ZT = 16 +NRT_MAX = 128 + + +@functools.cache +def compile_sp_rs_norm( + H: int, + tp: int, + eps: float, + E: int = 0, + topk: int = 0, + scale: float = 1.0, + shared_w: float = 0.0, + gemma: bool = True, + logit_bf16: bool = True, + nb: int = 256, + nrt: int = NRT_MAX, +): + NB = nb # CTAs: one per CU + ROWF = nrt * ZT # ctrl: per-row flags (rows per rank <= nrt * ZT) + ERR = NB + ROWF + nrt * KS_MAX # ctrl: sticky watchdog bit + RT = E > 0 + EPL = E // 64 + assert not RT or (E % 64 == 0 and EPL in (2, 4) and topk <= 64) + assert H % 256 == 0 + NP = H // 8 + KS_DIVS = [d for d in range(KS_MAX, 1, -1) if (H // 128) % d == 0] + NPF = min(3 if E <= 128 else 1, H // KS_MAX // 4 // 32) + PIT = ceildiv(NP, NTH) + assert NP % NTH == 0 + + def fbits(v): + return f"{struct.unpack('= i32(d))).select(i32(d), ks) + return rt, ks + + @flyc.jit + def router_tasks(L, a, tid): + m = a["m"] + bid = i32(gpu.block_id("x")) + lane = tid % i32(64) + w = tid // i32(64) + RT, KS = rt_tiles(m) + kw = i32(H) // KS + ro = rsrc(a["out"]) + rg = rsrc(a["wg"]) + rzp = rsrc(a["zp"]) + for t_ in range(i32(NB - 1) - bid, RT * KS, i32(NB)): + t = i32(t_) + rt = t // KS + ks = t - rt * KS + row = a["rank"] * m + rt * i32(ZT) + lane % i32(16) + k0 = ks * kw + w * (kw // i32(4)) + pf = [] + for j in range_constexpr(NPF): + kc = k0 + i32(j * 32) + (lane // i32(16)) * i32(8) + pf.append( + [ + bld( + rg, + ((i32(n * 16) + lane % i32(16)) * i32(H) + kc) * i32(2), + 0, + T.vec(4, T.i32), + ) + for n in range(E // 16) + ] + ) + if tid < i32(ZT): + r = rt * i32(ZT) + tid + addr = fx.Int64(a["ctrl"]) + fx.Int64((i32(NB) + r) * i32(4)) + wait_epoch(a, addr) + gpu.barrier() + + def step(acc, kc, bvs, row): + av = fx.Vector( + bld(ro, (row * i32(H) + kc) * i32(2), 0, T.vec(4, T.i32), AUX_SYS) + ).bitcast(fx.BFloat16) + out_ = [] + for n in range_constexpr(E // 16): + bv = fx.Vector(bvs[n]).bitcast(fx.BFloat16) + out_.append( + fx.Vector( + rocdl.mfma_f32_16x16x32_bf16( + T.vec(4, T.f32), [av, bv, acc[n], 0, 0, 0] + ) + ) + ) + return out_ + + acc0 = [fx.full(4, 0.0, fx.Float32) for _ in range(E // 16)] + for j in range_constexpr(NPF): + acc0 = step( + acc0, k0 + i32(j * 32) + (lane // i32(16)) * i32(8), pf[j], row + ) + for kk_, st in range(i32(NPF * 32), kw // i32(4), i32(32), init=acc0): + kc = k0 + i32(kk_) + (lane // i32(16)) * i32(8) + bvs = [ + bld( + rg, + ((i32(n * 16) + lane % i32(16)) * i32(H) + kc) * i32(2), + 0, + T.vec(4, T.i32), + ) + for n in range(E // 16) + ] + res = yield step(list(st), kc, bvs, row) + for n in range_constexpr(E // 16): + for i in range_constexpr(4): + lds_stf( + L, + i32(128) + + ( + ((w * i32(E // 16) + i32(n)) * i32(64) + lane) * i32(4) + + i32(i) + ) + * i32(4), + fx.Vector(res[n])[i], + ) + gpu.barrier() + if w == i32(0): + for n in range_constexpr(E // 16): + for i in range_constexpr(4): + v = fx.Float32(0.0) + for ww in range_constexpr(NTH // 64): + v = v + lds_f( + L, + i32(128) + + ( + (i32((ww * (E // 16) + n) * 64) + lane) * i32(4) + + i32(i) + ) + * i32(4), + ) + rl = (lane // i32(16)) * i32(4) + i32(i) + ex = i32(n * 16) + lane % i32(16) + zo = ( + ((rt * i32(KS_MAX) + ks) * i32(ZT) + rl) * i32(E) + ex + ) * i32(4) + bst(v.bitcast(fx.Int32), rzp, zo, 0, AUX_SYS) + rocdl.s_waitcnt(vmcnt=0) + if lane == i32(0): + g_st_sys(zflag(a, rt, ks), a["epoch"]) + if lane < KS: + wait_epoch(a, zflag(a, rt, lane)) + gpu.barrier() + for rl_ in range(ks + w * KS, i32(ZT), KS * i32(NTH // 64)): + router_row(a, lane, rt, i32(rl_), KS) + gpu.barrier() + + @flyc.jit + def router_row(a, lane, rt, rl, KS): + rzp = rsrc(a["zp"]) + rb = rsrc(a["bias"]) + r = rt * i32(ZT) + rl + vals, orig, idxs = [], [], [] + for i in range_constexpr(EPL): + e = lane + i32(i * 64) + vs = [] + for k in range_constexpr(KS_MAX): + kc = fx.min(i32(k), KS - i32(1)) + zo = (((rt * i32(KS_MAX) + kc) * i32(ZT) + rl) * i32(E) + e) * i32(4) + vs.append( + ( + i32(k) < KS, + fx.Int32(bld(rzp, zo, 0, T.i32, AUX_SYS)).bitcast(fx.Float32), + ) + ) + x = fx.Float32(0.0) + for live, v in vs: + x = x + live.select(v, fx.Float32(0.0)) + if const_expr(logit_bf16): + x = fx.Float32(x).to(fx.BFloat16).to(fx.Float32) + sc = 1.0 / (1.0 + fx.Float32(fmath.exp2(x * fx.Float32(-LOG2E)))) + orig.append(sc) + b = fx.Int32(bld(rb, e * i32(4), 0, T.i32)).bitcast(fx.Float32) + vals.append(sc + b) + idxs.append(e) + if const_expr(EPL == 2): + sw = vals[1] > vals[0] + v0, v1 = sw.select(vals[1], vals[0]), sw.select(vals[0], vals[1]) + o0, o1 = sw.select(orig[1], orig[0]), sw.select(orig[0], orig[1]) + i0, i1 = sw.select(idxs[1], idxs[0]), sw.select(idxs[0], idxs[1]) + cur = i32(0) + tot = fx.Float32(0.0) + my_id = i32(0) + my_w = fx.Float32(0.0) + ninf = fx.Float32(float("-inf")) + for k in range_constexpr(topk): + mv = (cur == i32(0)).select(v0, (cur == i32(1)).select(v1, ninf)) + mi = (cur == i32(0)).select(i0, i1) + mo = (cur == i32(0)).select(o0, o1) + mx = wave_red( + mv.bitcast(fx.Int32), + lane, + lambda p_, q_: p_.bitcast(fx.Float32) + .maximumf(q_.bitcast(fx.Float32)) + .bitcast(fx.Int32), + ).bitcast(fx.Float32) + bal = fx.Int64(rocdl.ballot(T.i64, mv == mx)) + win = i32(fx.ctpop((bal & (0 - bal)) - 1)) + win = (bal == 0).select(i32(0), win) + wid = i32(rocdl.readlane(T.i32, mi, win)) + wgt = fx.Int32( + rocdl.readlane(T.i32, mo.bitcast(fx.Int32), win) + ).bitcast(fx.Float32) + cur = cur + ((lane == win) & (cur < i32(2))).select(i32(1), i32(0)) + tot = tot + wgt + my_id = (lane == i32(k)).select(wid, my_id) + my_w = (lane == i32(k)).select(wgt, my_w) + else: + tot = fx.Float32(0.0) + my_id = i32(0) + my_w = fx.Float32(0.0) + ninf = fx.Float32(float("-inf")) + for k in range_constexpr(topk): + bv, bo, bi, bj = vals[0], orig[0], idxs[0], i32(0) + for j in range_constexpr(1, EPL): + t = vals[j] > bv + bv = t.select(vals[j], bv) + bo = t.select(orig[j], bo) + bi = t.select(idxs[j], bi) + bj = t.select(i32(j), bj) + mx = wave_red( + bv.bitcast(fx.Int32), + lane, + lambda p_, q_: p_.bitcast(fx.Float32) + .maximumf(q_.bitcast(fx.Float32)) + .bitcast(fx.Int32), + ).bitcast(fx.Float32) + bal = fx.Int64(rocdl.ballot(T.i64, bv == mx)) + win = i32(fx.ctpop((bal & (0 - bal)) - 1)) + win = (bal == 0).select(i32(0), win) + wid = i32(rocdl.readlane(T.i32, bi, win)) + wgt = fx.Int32( + rocdl.readlane(T.i32, bo.bitcast(fx.Int32), win) + ).bitcast(fx.Float32) + for j in range_constexpr(EPL): + vals[j] = ((lane == win) & (bj == i32(j))).select(ninf, vals[j]) + tot = tot + wgt + my_id = (lane == i32(k)).select(wid, my_id) + my_w = (lane == i32(k)).select(wgt, my_w) + f = fx.Float32(float(scale)) / tot.maximumf(fx.Float32(1e-20)) + ri = rsrc(a["ids"]) + rw = rsrc(a["tw"]) + if lane < i32(topk): + bst(my_id, ri, (r * i32(topk + 1) + lane) * i32(4), 0) + bst( + (my_w * f).bitcast(fx.Int32), rw, (r * i32(topk + 1) + lane) * i32(4), 0 + ) + if lane == i32(topk): + bst(i32(E), ri, (r * i32(topk + 1) + lane) * i32(4), 0) + bst( + fx.Float32(float(shared_w)).bitcast(fx.Int32), + rw, + (r * i32(topk + 1) + lane) * i32(4), + 0, + ) + + @flyc.jit + def blk_sum(L, tid, v): + r = wave_red(v.bitcast(fx.Int32), tid % i32(64), f32_bits_add) + if ((tid % i32(64)) == i32(0)) & (tid < i32(NTH)): + lds_st(L, i32(16) + (tid // i32(64)) * i32(4), r) + gpu.barrier() + t = fx.Float32(0.0) + for k in range_constexpr(NTH // 64): + t = t + lds_ld_i32(L, 16 + k * 4).bitcast(fx.Float32) + gpu.barrier() + return t + + @flyc.jit + def send_rows(a, tid): + m = a["m"] + bid = i32(gpu.block_id("x")) + lane = tid % i32(64) + rp = rsrc(a["part"]) + for it_ in range(bid, m * i32(tp - 1), i32(NB)): + it = i32(it_) + r = it // i32(tp - 1) + k = it - r * i32(tp - 1) + d = (a["rank"] + i32(1) + k) % i32(tp) + i = d * m + r + peer = a["peer"][0] + for j in range_constexpr(1, tp): + peer = (d == i32(j)).select(a["peer"][j], peer) + rx = rsrc(fx.Int64(peer) + fx.Int64(a["off_x"])) + rsx = rsrc(fx.Int64(peer) + fx.Int64(a["off_s"])) + slot = a["rank"] * m + r + for k in range_constexpr(PIT): + p = tid + i32(k * NTH) + f = bf16x8_to_f32( + bld(rp, (i * i32(H) + p * i32(8)) * i32(2), 0, T.vec(4, T.i32)) + ) + am = amax(f) + am = am.maximumf(am.shuffle_xor(i32(1), i32(64))) + am = am.maximumf(am.shuffle_xor(i32(2), i32(64))) + sc = am.maximumf(fx.Float32(1e-30)) / fx.Float32(127.0) + inv = fx.Float32(1.0) / sc + dv = fx.Vector.from_elements( + [i8x4_pack(f[0:4], inv), i8x4_pack(f[4:8], inv)], fx.Int32 + ) + bst(dv, rx, slot * i32(H) + p * i32(8), 0, AUX_SYS) + if (lane & i32(3)) == i32(0): + bst( + sc.bitcast(fx.Int32), + rsx, + (slot * i32(H // 32) + p // i32(4)) * i32(4), + 0, + AUX_SYS, + ) + + @flyc.jit + def post_flags(L, a, tid): + rocdl.s_waitcnt(vmcnt=0) + gpu.barrier() + if tid < i32(tp): + bid = i32(gpu.block_id("x")) + peer = a["peer"][0] + for j in range_constexpr(1, tp): + peer = (tid == i32(j)).select(a["peer"][j], peer) + fo = fx.Int64(a["off_f"]) + fx.Int64((a["rank"] * i32(NB) + bid) * i32(4)) + g_st_sys(fx.Int64(peer) + fo, a["epoch"]) + + @flyc.jit + def wait_row(a, tid, r): + if tid < i32(tp): + k = (a["rank"] - tid - i32(1) + i32(tp)) % i32(tp) + src_cta = (r * i32(tp - 1) + k) % i32(NB) + addr = ( + a["mine"] + + fx.Int64(a["off_f"]) + + fx.Int64((tid * i32(NB) + src_cta) * i32(4)) + ) + if tid != a["rank"]: + wait_epoch(a, addr) + gpu.barrier() + + @flyc.jit + def reduce_rows(L, a, tid): + m = a["m"] + bid = i32(gpu.block_id("x")) + rp = rsrc(a["part"]) + rr = rsrc(a["res"]) + ro = rsrc(a["res_out"]) + rout = rsrc(a["out"]) + rw = rsrc(a["w"]) + rx = rsrc(a["mine"] + fx.Int64(a["off_x"])) + rsx = rsrc(a["mine"] + fx.Int64(a["off_s"])) + for r_ in range(bid, m, i32(NB)): + r = i32(r_) + wait_row(a, tid, r) + g = a["rank"] * m + r + fs = [] + ss = fx.Float32(0.0) + for k in range_constexpr(PIT): + p = tid + i32(k * NTH) + off = (g * i32(H) + p * i32(8)) * i32(2) + f = bf16x8_to_f32(bld(rp, off, 0, T.vec(4, T.i32))) + rv = bf16x8_to_f32(bld(rr, off, 0, T.vec(4, T.i32))) + f = [x + y for x, y in zip(f, rv)] + for s in range_constexpr(tp): + src = i32(s) + live = src != a["rank"] + slot = src * m + r + d = fx.Vector( + bld(rx, slot * i32(H) + p * i32(8), 0, T.vec(2, T.i32), AUX_SYS) + ) + sc = fx.Int32( + bld( + rsx, + (slot * i32(H // 32) + p // i32(4)) * i32(4), + 0, + T.i32, + AUX_SYS, + ) + ).bitcast(fx.Float32) + v = i8x4_unpack(d[0], sc) + i8x4_unpack(d[1], sc) + f = sum_live(f, v, live) + bst(pack_bf16x8(f), ro, off, 0) + for x in f: + ss = ss + x * x + fs.append(f) + tot = blk_sum(L, tid, ss) + rcp = fx.Float32( + fmath.rsqrt(tot / fx.Float32(float(H)) + fx.Float32(float(eps))) + ) + for k in range_constexpr(PIT): + p = tid + i32(k * NTH) + off = (g * i32(H) + p * i32(8)) * i32(2) + wv = bf16x8_to_f32(bld(rw, p * i32(16), 0, T.vec(4, T.i32))) + bst( + pack_bf16x8( + [ + x * rcp * ((w + fx.Float32(1.0)) if gemma else w) + for x, w in zip(fs[k], wv) + ] + ), + rout, + off, + 0, + AUX_SYS if RT else 0, + ) + if const_expr(RT): + rocdl.s_waitcnt(vmcnt=0) + gpu.barrier() + if tid == i32(0): + g_st_sys( + fx.Int64(a["ctrl"]) + fx.Int64((i32(NB) + r) * i32(4)), + a["epoch"], + ) + + @flyc.kernel(name=name, known_block_size=[NTH, 1, 1]) + def sp_rs_norm_kernel( + part: fx.Int64, + res: fx.Int64, + res_out: fx.Int64, + out: fx.Int64, + w: fx.Int64, + ctrl: fx.Int64, + p0: fx.Int64, + p1: fx.Int64, + p2: fx.Int64, + p3: fx.Int64, + p4: fx.Int64, + p5: fx.Int64, + p6: fx.Int64, + p7: fx.Int64, + off_x: fx.Int64, + off_s: fx.Int64, + off_f: fx.Int64, + xbank: fx.Int64, + rank: fx.Int32, + m: fx.Int32, + wg: fx.Int64, + bias: fx.Int64, + ids: fx.Int64, + tw: fx.Int64, + zp: fx.Int64, + ): + tid = fx.Int32(gpu.thread_id("x")) + lds = fx.SharedAllocator().allocate(Shared).peek() + L = uni(fx.Int32(fx.ptrtoint(lds.buf.ptr))) + peers = [p0, p1, p2, p3, p4, p5, p6, p7] + mine = peers[0] + for j in range_constexpr(1, MAX_TP): + mine = (rank == i32(j)).select(peers[j], mine) + bid = i32(gpu.block_id("x")) + ea = ctrl + fx.Int64(bid * i32(4)) + epoch = i32(fx.generic_load(gptr(ea), dtype=fx.Int32)) + i32(1) + a = { + "part": part, + "res": res, + "res_out": res_out, + "out": out, + "w": w, + "peer": peers, + "mine": fx.Int64(mine), + # receive buffers double-buffered by launch parity: a peer's next launch + # never overwrites the rows this one still reads + "off_x": off_x + fx.Int64(epoch & i32(1)) * xbank, + "off_s": off_s + fx.Int64(epoch & i32(1)) * (xbank // 8), + "off_f": off_f, + "rank": rank, + "m": m, + "epoch": epoch, + "ctrl": ctrl, + "wg": wg, + "bias": bias, + "ids": ids, + "tw": tw, + "zp": zp, + } + if tid < i32(4): + lds_st(L, tid * i32(4), i32(0)) + gpu.barrier() + send_rows(a, tid) + post_flags(L, a, tid) + reduce_rows(L, a, tid) + if const_expr(RT): + router_tasks(L, a, tid) + if tid == i32(0): + fx.generic_store(gptr(ea), epoch) + + @flyc.jit + def launch( + part: fx.Int64, + res: fx.Int64, + res_out: fx.Int64, + out: fx.Int64, + w: fx.Int64, + ctrl: fx.Int64, + p0: fx.Int64, + p1: fx.Int64, + p2: fx.Int64, + p3: fx.Int64, + p4: fx.Int64, + p5: fx.Int64, + p6: fx.Int64, + p7: fx.Int64, + off_x: fx.Int64, + off_s: fx.Int64, + off_f: fx.Int64, + xbank: fx.Int64, + rank: fx.Int32, + m: fx.Int32, + wg: fx.Int64, + bias: fx.Int64, + ids: fx.Int64, + tw: fx.Int64, + zp: fx.Int64, + stream: fx.Stream, + ): + sp_rs_norm_kernel( + part, + res, + res_out, + out, + w, + ctrl, + p0, + p1, + p2, + p3, + p4, + p5, + p6, + p7, + off_x, + off_s, + off_f, + xbank, + rank, + m, + wg, + bias, + ids, + tw, + zp, + ).launch(grid=(NB, 1, 1), block=(NTH, 1, 1), stream=stream) + + return launch + + +class SpRsNorm: + """forward(part, res, w[, router]) -> (out, res_out) for this rank's rows. + + part / res: bf16 [T, H] (T = tp * m; this rank's partial of every token and the + residual), w: bf16 [H]; out / res_out: bf16 [T, H], rows rank*m:(rank+1)*m written. + router = (E, topk, scale, shared_w) at construction; forward(router=(wg, bias, ids, + tw)): wg bf16 [E, H], bias fp32 [E], ids int32 / tw fp32 [>= m, topk + 1] (the last + column is the shared expert E). A collective like MegaMoeTP (see its contract): a wait + that times out sets error_flag(); reset() (collective) restarts the state. + """ + + def __init__( + self, + hidden: int, + max_tokens: int, + eps: float, + group=None, + device=None, + router=None, + gemma: bool = True, + logit_bf16: bool = True, + ): + device = torch.device(device if device is not None else "cuda") + if device.index is None: + device = torch.device(device.type, torch.cuda.current_device()) + self.device = device + self.H = int(hidden) + self.eps = float(eps) + self.group = group + self.tp = dist.get_world_size(group) + self.rank = dist.get_rank(group) + mmax = ceildiv(int(max_tokens), self.tp) + self.mmax = mmax + # router row tiles per rank (ctrl / scratch sizes): the default up to 2048 rows per rank + self.nrt = max(NRT_MAX, ceildiv(mmax, ZT)) + self.router = router + E = int(router[0]) if router else 0 + topk = int(router[1]) if router else 0 + self.nb = int(torch.cuda.get_device_properties(device).multi_processor_count) + bad = [] + if self.tp > MAX_TP: + bad.append(f"tp <= {MAX_TP}") + if self.H % (8 * NTH): + bad.append(f"hidden % {8 * NTH} == 0") + if router and (E not in (128, 256) or not 0 < topk < 64): + bad.append("router experts in (128, 256) and 0 < topk < 64") + if bad: + raise ValueError(f"SpRsNorm h{self.H} tp{self.tp}: need " + ", ".join(bad)) + self._xbank = self.tp * mmax * self.H + arena = SymmetricArena(group=group, device=device) + self._x = arena.reserve("x", (2 * self._xbank,), torch.uint8) + self._s = arena.reserve("s", (2 * self._xbank // 8,), torch.uint8) + self._f = arena.reserve("f", (MAX_TP * self.nb,), torch.int32) + arena.commit() + self.arena = arena + self._err = self.nb + self.nrt * ZT + self.nrt * KS_MAX + self.ctrl = torch.zeros(self._err + 1, dtype=torch.int32, device=device) + self._zp = torch.empty( + (self.nrt * KS_MAX * ZT * max(E, 1),) if router else (1,), + dtype=torch.float32, + device=device, + ) + self._fn = compile_sp_rs_norm( + self.H, self.tp, self.eps, gemma=bool(gemma), nb=self.nb, nrt=self.nrt + ) + self._fn_rt = ( + compile_sp_rs_norm( + self.H, + self.tp, + self.eps, + E, + topk, + float(router[2]), + float(router[3]), + gemma=bool(gemma), + logit_bf16=bool(logit_bf16), + nb=self.nb, + nrt=self.nrt, + ) + if router + else None + ) + self._armed = set() + + def routes(self, tokens: int) -> bool: + """forward(router=...) runs for this batch.""" + m = tokens // self.tp + return ( + self._fn_rt is not None + and tokens % self.tp == 0 + and m % ZT == 0 + and 0 < m <= self.mmax + ) + + def _check_args(self, part, res, w, out, res_out, router): + T, H = part.shape[0], self.H + if T % self.tp or not 0 < T // self.tp <= self.mmax: + raise ValueError( + f"SpRsNorm: {T} tokens: need tp | T and T / tp <= {self.mmax}" + ) + for name, t, shp in ( + ("part", part, (T, H)), + ("res", res, (T, H)), + ("out", out, (T, H)), + ("res_out", res_out, (T, H)), + ("w", w, (H,)), + ): + if ( + tuple(t.shape) != shp + or t.dtype != torch.bfloat16 + or not t.is_contiguous() + ): + raise ValueError(f"SpRsNorm: {name} needs contiguous bf16 {list(shp)}") + if router is not None: + if not self.routes(T): + raise ValueError(f"SpRsNorm: {T} tokens cannot route (tp * 16 | T)") + E, k1 = int(self.router[0]), int(self.router[1]) + 1 + wg, bias, ids, tw = router + for name, t, dt, ok in ( + ("wg", wg, torch.bfloat16, tuple(wg.shape) == (E, H)), + ("bias", bias, torch.float32, tuple(bias.shape) == (E,)), + ("ids", ids, torch.int32, ids.dim() == 2 and ids.shape[1] == k1), + ("tw", tw, torch.float32, tw.dim() == 2 and tw.shape[1] == k1), + ): + if not ok or t.dtype != dt or not t.is_contiguous(): + raise ValueError(f"SpRsNorm: router {name}: wrong shape / dtype") + if ids.shape[0] < T // self.tp or tw.shape[0] < T // self.tp: + raise ValueError("SpRsNorm: router ids / tw need >= tokens / tp rows") + + def forward(self, part, res, w, out=None, res_out=None, router=None): + m = part.shape[0] // self.tp + out = torch.empty_like(part) if out is None else out + res_out = torch.empty_like(res) if res_out is None else res_out + self._check_args(part, res, w, out, res_out, router) + peers = [int(b) for b in self.arena.base_ptrs] + [0] * (MAX_TP - self.tp) + if router is not None: + wg, bias, ids, tw = router + rt = (wg.data_ptr(), bias.data_ptr(), ids.data_ptr(), tw.data_ptr()) + fn = self._fn_rt + else: + rt = (0, 0, 0, 0) + fn = self._fn + args = ( + part.data_ptr(), + res.data_ptr(), + res_out.data_ptr(), + out.data_ptr(), + w.data_ptr(), + self.ctrl.data_ptr(), + *peers, + self._x.offset, + self._s.offset, + self._f.offset, + self._xbank, + self.rank, + m, + *rt, + self._zp.data_ptr(), + ) + if fn not in self._armed: + if torch.cuda.is_current_stream_capturing(): + raise RuntimeError("SpRsNorm: first use inside graph capture") + _preload_compiled(fn, *args, torch.cuda.current_stream()) + torch.cuda.synchronize(self.device) + self.arena.barrier() + self._armed.add(fn) + _run_compiled(fn, *args, torch.cuda.current_stream()) + return out, res_out + + def warm(self) -> None: + """Collective: compile and run once (eagerly; graph capture cannot), then + check every rank's watchdog (reset + retry once on a timeout).""" + tp = self.tp + rows = tp * ZT if self._fn_rt is not None else tp + x = torch.zeros((rows, self.H), dtype=torch.bfloat16, device=self.device) + w = torch.zeros((self.H,), dtype=torch.bfloat16, device=self.device) + for _ in range(2): + self.forward(x, x.clone(), w) + if self._fn_rt is not None: + E, k = int(self.router[0]), int(self.router[1]) + dev = self.device + wg = torch.zeros((E, self.H), dtype=torch.bfloat16, device=dev) + bias = torch.zeros((E,), dtype=torch.float32, device=dev) + ids = torch.empty((ZT, k + 1), dtype=torch.int32, device=dev) + tw = torch.empty((ZT, k + 1), dtype=torch.float32, device=dev) + self.forward(x, x.clone(), w, router=(wg, bias, ids, tw)) + torch.cuda.synchronize(self.device) + errs = [None] * tp + dist.all_gather_object(errs, self.poll_errors(), group=self.group) + if not any(errs): + return + self.reset() + raise RuntimeError(f"SpRsNorm.warm: launches timed out ({errs})") + + def error_flag(self) -> torch.Tensor: + """Sticky watchdog bit (nonzero: a launch timed out) as a device tensor.""" + return self.ctrl[self._err : self._err + 1] + + def poll_errors(self) -> int: + return int(self.ctrl[self._err].item()) + + def check_errors(self) -> None: + if self.poll_errors(): + self.ctrl[self._err] = 0 + raise RuntimeError( + f"SpRsNorm rank {self.rank}: a wait timed out; the output is invalid " + "(ranks out of step need reset())" + ) + + def reset(self) -> None: + """Collective: drop the cross-rank state and restart the launch epochs.""" + torch.cuda.synchronize(self.device) + self.arena.barrier() + self.arena.storage.zero_() + self.ctrl.zero_() + torch.cuda.synchronize(self.device) + self.arena.barrier() + + __call__ = forward From 25f1b6ee855117337deae650c648386ffa5da3ef Mon Sep 17 00:00:00 2001 From: BinDing Date: Sat, 14 Nov 2026 08:12:07 +0000 Subject: [PATCH 2/3] add mega_moe_tp tests and benchmark Tests (4 GPUs) against reference.py, the warm-up check and SpRsNorm; the graph-replay benchmark; the tuned CSVs as package data; the CI target. Co-Authored-By: Claude Opus 5.5 (1M context) --- amd_kernels/moe/mega_moe_tp/README.md | 1 + .../moe/mega_moe_tp/benchmark_mega_moe_tp.py | 125 +++++++ pyproject.toml | 2 +- tests/moe/mega_moe_tp/__init__.py | 2 + tests/moe/mega_moe_tp/test_mega_moe_tp.py | 347 ++++++++++++++++++ .../test_mega_moe_tp_sp_rs_norm.py | 82 +++++ .../mega_moe_tp/test_mega_moe_tp_warm_up.py | 82 +++++ tests/moe/mega_moe_tp/workload.py | 156 ++++++++ tools/ci/targets.tsv | 1 + 9 files changed, 797 insertions(+), 1 deletion(-) create mode 100644 benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py create mode 100644 tests/moe/mega_moe_tp/__init__.py create mode 100644 tests/moe/mega_moe_tp/test_mega_moe_tp.py create mode 100644 tests/moe/mega_moe_tp/test_mega_moe_tp_sp_rs_norm.py create mode 100644 tests/moe/mega_moe_tp/test_mega_moe_tp_warm_up.py create mode 100644 tests/moe/mega_moe_tp/workload.py diff --git a/amd_kernels/moe/mega_moe_tp/README.md b/amd_kernels/moe/mega_moe_tp/README.md index 77ca5e7..20d3b7b 100644 --- a/amd_kernels/moe/mega_moe_tp/README.md +++ b/amd_kernels/moe/mega_moe_tp/README.md @@ -41,6 +41,7 @@ calls over 4 inputs, median of 20 replays, slowest rank, us per call. No baselin ## Tests ```bash +cd amd-kernels && source env.sh # or: PYTHONPATH=$PWD/.cache/flydsl-0.5.0:$PWD python3 -m pytest tests/moe/mega_moe_tp -q # 4 GPUs; big-M cases need MORI python3 benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py # 4 GPUs ``` diff --git a/benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py b/benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py new file mode 100644 index 0000000..da5844c --- /dev/null +++ b/benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MegaMoE TP on 4 ranks: CUDA graph replay of ``--calls`` back-to-back layer calls, each on its own input (the +weights of one rank, 0.9 GB for m3, exceed the LLC), us per call of the slowest rank (median of ``--iters`` replays); +plus GEMM TFLOPS and algorithmic TB/s over the cluster, and rel L2 vs ``reference.py`` of the first input. + + python3 benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py [--shape m3] [--modes ag_rs ar] [--tokens 8 64 ...] + [--comm fp8 bf16] [--json out.json] +""" + +import argparse +import json +import statistics + +import torch +import torch.distributed as dist + +from tests.multi_gpu import WORLD, run_on_4_gpus + + +def _graph_us(layer, cases, out, calls, iters): + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for c in cases[:3]: + layer(c.x, c.w, c.ids, out=out) + torch.cuda.current_stream().wait_stream(s) + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for i in range(calls): + c = cases[i % len(cases)] + layer(c.x, c.w, c.ids, out=out) + for _ in range(3): + g.replay() + torch.cuda.synchronize() + dist.barrier() + times = [] + for _ in range(iters): + a, b = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + a.record() + g.replay() + b.record() + b.synchronize() + times.append(a.elapsed_time(b) * 1e3 / calls) + dist.barrier() + t = torch.tensor([statistics.median(times)]) + dist.all_reduce(t, op=dist.ReduceOp.MAX) + return float(t.item()) + + +def _work(shape, wt, ids_all, tokens): + """cluster-wide GEMM FLOPs and algorithmic bytes (input, the active experts' MXFP4 weights, routing, output)""" + H, I, K = shape.model_dim, wt.inter, shape.topk + flops = 6 * tokens * K * H * I * WORLD + ids = ids_all.flatten() + active = int(torch.unique(ids[(ids >= 0) & (ids < shape.experts)]).numel()) + weights = active * (2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32)) + return flops, weights * WORLD + tokens * H * 2 * 2 + tokens * K * 8 + + +def _worker(rank, errors, args): + from tests.moe.mega_moe_tp.workload import SHAPES, build_weights, make_case, new_layer, rel_l2 + + dev = torch.device("cuda", rank) + shape = SHAPES[args.shape] + wt = build_weights(shape, rank, WORLD, dev) + max_local = max(t // WORLD for t in args.tokens) + rows = [] + for mode in args.modes: + for comm in args.comm: + layer = new_layer(shape, wt, rank, WORLD, mode, max_local, comm, "bf16") + layer.prepare(sorted({-(-t // WORLD) for t in args.tokens})) + for tokens in args.tokens: + cases = [make_case(shape, wt, rank, WORLD, mode, tokens, args.route, 1000 + i) + for i in range(args.inputs)] + out = torch.empty_like(cases[0].ref, dtype=torch.bfloat16) + layer(cases[0].x, cases[0].w, cases[0].ids, out=out) + torch.cuda.synchronize() + e = rel_l2(out, cases[0].ref) + us = _graph_us(layer, cases, out, args.calls, args.iters) + if layer.poll_errors(): + errors.append(f"rank {rank} {mode} {comm} M={tokens}: watchdog error") + flops, nbytes = _work(shape, wt, cases[0].ids_all, tokens) + cfg = layer.launch_config(tokens // WORLD) + plan = (f"mt{cfg.mt}{' lb' if cfg.lb else ''}" if cfg is not None + else "big-M " + ",".join(map(str, layer.bigm_plan(tokens // WORLD)[0]))) + rows.append({"shape": shape.name, "mode": mode, "comm": comm, "M": tokens, "config": plan, + "us": round(us, 1), "TFLOPS": round(flops / us / 1e6, 1), + "TB/s": round(nbytes / us / 1e6, 2), "rel_l2": round(e, 4)}) + del cases + layer.close() + if rank == 0: + cols = list(rows[0]) + print("| " + " | ".join(cols) + " |\n|" + "---|" * len(cols), flush=True) + for r in rows: + print("| " + " | ".join(str(r[c]) for c in cols) + " |", flush=True) + if args.json: + with open(args.json, "w") as f: + json.dump({"gpu": torch.cuda.get_device_properties(dev).name, "args": vars(args), "rows": rows}, f, + indent=1) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--shape", default="m3", choices=["m3", "glm5"]) + p.add_argument("--modes", nargs="*", default=["ag_rs", "ar"], choices=["ag_rs", "ar"]) + p.add_argument("--tokens", type=int, nargs="*", default=[8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192], + help="global tokens per call (multiples of 4)") + p.add_argument("--comm", nargs="*", default=["fp8"], choices=["fp8", "bf16"]) + p.add_argument("--route", default="random", choices=["balanced", "random", "hot", "subset"]) + p.add_argument("--inputs", type=int, default=4, help="distinct inputs the graph's calls cycle over") + p.add_argument("--calls", type=int, default=10, help="layer calls per graph") + p.add_argument("--iters", type=int, default=20, help="timed graph replays") + p.add_argument("--json") + args = p.parse_args() + bad = [t for t in args.tokens if t <= 0 or t % WORLD] + if bad: + p.error(f"tokens {bad}: positive multiples of {WORLD}") + run_on_4_gpus(_worker, args, timeout=7200) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index 80b39b4..43c26cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [] include = ["amd_kernels*"] [tool.setuptools.package-data] -"*" = ["README.md", "LICENSE"] +"*" = ["README.md", "LICENSE", "*.csv"] [tool.setuptools_scm] # Tags are v.+ (.github/workflows/tag-main.yml); diff --git a/tests/moe/mega_moe_tp/__init__.py b/tests/moe/mega_moe_tp/__init__.py new file mode 100644 index 0000000..3158f21 --- /dev/null +++ b/tests/moe/mega_moe_tp/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project diff --git a/tests/moe/mega_moe_tp/test_mega_moe_tp.py b/tests/moe/mega_moe_tp/test_mega_moe_tp.py new file mode 100644 index 0000000..7c2c8a3 --- /dev/null +++ b/tests/moe/mega_moe_tp/test_mega_moe_tp.py @@ -0,0 +1,347 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MegaMoE TP on 4 ranks vs ``reference.mega_moe_tp_ref`` (rel L2 <= RTOL over every rank): both comm modes and +formats at decode / prefill token counts and routings, then the layer's contract: varying m, ``out=``, graph replay == +eager on new inputs, both schedules, masked routes, m = 0, layers built and freed, the fused tail, host validation, +and the big-M path where MORI is importable.""" + +import gc + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import WORLD, run_on_4_gpus + +from amd_kernels.moe.mega_moe_tp import mega_moe_tp_supported +from amd_kernels.moe.mega_moe_tp.reference import RTOL, tail_ref + +from .workload import SEED, SHAPES, Case, build_weights, make_case, new_layer, rank_max, rel_l2, same_on_all_ranks + +MAX_LOCAL = 256 # max_local_tokens of the layers: 1024 tokens on 4 ranks +_WEIGHTS = {} + + +@pytest.fixture(autouse=True) +def _gfx950(): + if not torch.cuda.is_available() or not mega_moe_tp_supported(): + pytest.skip("needs gfx950") + + +def _weights(name, rank): + if name not in _WEIGHTS: + _WEIGHTS.clear() + gc.collect() + _WEIGHTS[name] = build_weights(SHAPES[name], rank, WORLD, torch.device("cuda", rank)) + return SHAPES[name], _WEIGHTS[name] + + +def _prepare(layer, tokens, **kw): + """the layer's contract: every launch config armed and run once (a watchdog timeout reset and retried) before + use; a first launch that loads its kernel can keep a rank past its peers' wait""" + ar = layer.cfg.comm_mode == "ar" + layer.prepare(sorted({-(-t // WORLD) if ar else t // WORLD for t in tokens}), **kw) + + +def _call(layer, c: Case, errors, what, out=None): + layer.clear_errors() + y = layer(c.x, c.w, c.ids, out=out) + torch.cuda.synchronize() + if rank_max(layer.poll_errors()): + errors.append(f"{what}: watchdog error") + if rank_max(float(torch.isnan(y).any())): + errors.append(f"{what}: NaN in the output") + return y + + +def _check(y, c: Case, errors, what): + e = rel_l2(y, c.ref) + if not e <= RTOL: + errors.append(f"{what}: rel_l2 {e:.4f} > {RTOL}") + + +def _expect_raise(fn, errors, what): + raised = 0.0 + try: + fn() + torch.cuda.synchronize() + except (ValueError, TypeError): + raised = 1.0 + if -rank_max(-raised) < 1.0: + errors.append(f"{what} was accepted (expected a host-side error)") + + +def _accuracy(rank, errors, name, mode, comm_dtype, ar_gather, tokens, routes): + shape, wt = _weights(name, rank) + layer = new_layer(shape, wt, rank, WORLD, mode, MAX_LOCAL, comm_dtype, ar_gather) + _prepare(layer, tokens) + for t in tokens: + for i, kind in enumerate(routes): + c = make_case(shape, wt, rank, WORLD, mode, t, kind, SEED + 1000 * i + t) + out = torch.empty_like(c.ref, dtype=torch.bfloat16) + what = f"{name} {mode} {comm_dtype}/{ar_gather} M={t} {kind}" + y = _call(layer, c, errors, what, out=out) + _check(y, c, errors, what) + if mode == "ar" and not same_on_all_ranks(y): + errors.append(f"{what}: the all-reduced output differs across ranks") + layer.close() + + +@pytest.mark.parametrize(("mode", "comm_dtype", "ar_gather"), [ + ("ag_rs", "fp8", "bf16"), ("ag_rs", "bf16", "bf16"), ("ar", "fp8", "bf16"), ("ar", "bf16", "bf16"), + ("ar", "fp8", "fp8"), +]) # fmt: skip +def test_mega_moe_tp_accuracy(mode, comm_dtype, ar_gather): + run_on_4_gpus(_accuracy, "m3", mode, comm_dtype, ar_gather, (8, 64, 256, 1024), ("balanced", "random", "hot")) + + +@pytest.mark.parametrize("mode", ["ag_rs", "ar"]) +def test_mega_moe_tp_accuracy_glm5(mode): + run_on_4_gpus(_accuracy, "glm5", mode, "fp8", "bf16", (8, 256), ("random",)) + + +def _features(rank, errors, mode): + shape, wt = _weights("m3", rank) + world = WORLD + layer = new_layer(shape, wt, rank, world, mode, MAX_LOCAL) + _prepare(layer, [m * world for m in (1, 4, 8, 16, 32, 64)]) + for m in (8, 1, 32, 1, 64, 8): # varying m on one layer + c = make_case(shape, wt, rank, world, mode, m * world, "random", SEED + m) + _check(_call(layer, c, errors, f"m={m}").clone(), c, errors, f"m={m}") + + c1 = make_case(shape, wt, rank, world, mode, 16 * world, "random", 1) + c2 = make_case(shape, wt, rank, world, mode, 16 * world, "random", 2) + out = torch.empty_like(c1.ref, dtype=torch.bfloat16) + if _call(layer, c1, errors, "out=", out=out).data_ptr() != out.data_ptr(): + errors.append("out= was not returned") + keep = out.clone() + _call(layer, c2, errors, "after out=") + if rank_max(float(not torch.equal(out, keep))): + errors.append("out= of call N changed by call N+1") + _check(keep, c1, errors, "out=") + + c = make_case(shape, wt, rank, world, mode, 16 * world, "random", 11, mask=0.2) + _check(_call(layer, c, errors, "masked").clone(), c, errors, "masked ids") + c = make_case(shape, wt, rank, world, mode, 16 * world, "random", 12) + _check(_call(layer, c, errors, "after masked").clone(), c, errors, "the call after masked ids") + + c = make_case(shape, wt, rank, world, mode, 4 * world, "random", 13) + y = _call(layer, Case(c.x[:0], c.w[:0], c.ids[:0], c.ref[:0], c.ids_all), errors, "m=0") + if tuple(y.shape) != (0, shape.model_dim): + errors.append(f"m=0 returned shape {tuple(y.shape)}") + _check(_call(layer, c, errors, "after m=0").clone(), c, errors, "the call after m=0") + + # graph replay on new inputs == eager (stale peer data would show) + cases = [make_case(shape, wt, rank, world, mode, 16 * world, "random", s) for s in (3, 4, 5)] + eager = [_call(layer, k, errors, "eager").clone() for k in cases] + x, w, ids = cases[0].x.clone(), cases[0].w.clone(), cases[0].ids.clone() + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + for _ in range(3): + layer(x, w, ids) + torch.cuda.current_stream().wait_stream(s) + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + y = layer(x, w, ids) + same = True + for i in range(12): + k = i % len(cases) + for dst, src in ((x, cases[k].x), (w, cases[k].w), (ids, cases[k].ids)): + dst.copy_(src) + g.replay() + same = same and torch.equal(y, eager[k]) + torch.cuda.synchronize() + if rank_max(layer.poll_errors()): + errors.append("watchdog error during graph replay") + if rank_max(float(not same)): + errors.append("graph replay != eager") + del g + layer.close() + + # layers built and freed between calls + layers = [new_layer(shape, wt, rank, world, mode, MAX_LOCAL) for _ in range(3)] + for rnd in range(2): + for lay in layers: + _prepare(lay, [8 * world]) + for i, lay in enumerate(layers): + c = make_case(shape, wt, rank, world, mode, 8 * world, "random", 100 * rnd + i) + _check(_call(lay, c, errors, f"layer {i}").clone(), c, errors, f"round {rnd} layer {i}") + for lay in layers[:2]: + lay.close() + del layers[:2] + gc.collect() + layers += [new_layer(shape, wt, rank, world, mode, MAX_LOCAL) for _ in range(2)] + for lay in layers: + lay.close() + + +@pytest.mark.parametrize("mode", ["ag_rs", "ar"]) +def test_mega_moe_tp_features(mode): + run_on_4_gpus(_features, mode) + + +def _schedules(rank, errors, mode, lb): + """the dynamic schedule up to 256 tokens; lb: the large-batch one from 1 token up at its smallest row tile (the + mixed GEMM2 split); odd local token counts and a narrow routing""" + shape, wt = _weights("m3", rank) + kw = {"lb_min": 1, "lb_mt": 3, "lb_mt_small": 3, "lb_npp": 2, "lb_q": 3} if lb else {"lb_min": 0} + layer = new_layer(shape, wt, rank, WORLD, mode, MAX_LOCAL, **kw) + _prepare(layer, (WORLD, 3 * WORLD, 33 * WORLD, MAX_LOCAL * WORLD)) + for t in (WORLD, 3 * WORLD, 33 * WORLD, MAX_LOCAL * WORLD): + for kind in ("random", "subset"): + c = make_case(shape, wt, rank, WORLD, mode, t, kind, SEED + 3 * t) + _check(_call(layer, c, errors, f"M={t} {kind}").clone(), c, errors, f"lb={lb} M={t} {kind}") + layer.close() + + +@pytest.mark.parametrize("lb", [False, True]) +@pytest.mark.parametrize("mode", ["ag_rs", "ar"]) +def test_mega_moe_tp_schedules(mode, lb): + run_on_4_gpus(_schedules, mode, lb) + + +def _tail(rank, errors): + """the fused tail in place, as a caller runs it: tail=(res, res, w)""" + shape, wt = _weights("m3", rank) + layer = new_layer(shape, wt, rank, WORLD, "ag_rs", MAX_LOCAL, lb_min=1) + _prepare(layer, [MAX_LOCAL * WORLD]) + eng = layer.engine + t = MAX_LOCAL * WORLD + if not layer.tail_ok(t // WORLD): + errors.append(f"no fused tail at {t} tokens") + layer.close() + return + c = make_case(shape, wt, rank, WORLD, "ag_rs", t, "random", 21) + dev = c.x.device + res = torch.randn(c.x.shape, generator=torch.Generator(device=dev).manual_seed(77 + rank), device=dev) + res = res.to(torch.bfloat16) + nw = (0.1 * torch.randn((shape.model_dim,), generator=torch.Generator(device=dev).manual_seed(78), device=dev)) + nw = nw.to(torch.bfloat16) + y0 = _call(layer, c, errors, "tail: plain").clone() + r_ref, xn, s_ref = tail_ref(y0, res, nw, eng.tn_eps) + # every rank's normed rows and scales (rank r's at r * m) + xn_all, s_all = [torch.zeros((WORLD,) + v.shape) for v in (xn, s_ref)] + xn_all[rank], s_all[rank] = xn.cpu(), s_ref.cpu() + dist.all_reduce(xn_all) + dist.all_reduce(s_all) + xn, s_ref = xn_all.flatten(0, 1).to(dev), s_all.flatten(0, 1).to(dev) + fp8 = torch.float8_e4m3fn + q_ref = (xn.to(torch.bfloat16).float() / s_ref[:, None]).clamp(-448, 448).to(fp8) + m = c.x.shape[0] + own = slice(rank * m, (rank + 1) * m) + # tn_local (the default): only this rank's rows of q / scales / bf16 rows are written; False: every rank's + for local in (True, False): + eng.tn_local = local + _prepare(layer, [t], tail=True, tail_bf16=True) + rows = own if local else slice(None) + xn_r, s_r = xn[rows], s_ref[rows] + e_ref = float((q_ref[rows].float() * s_r[:, None] - xn_r).norm() / xn_r.norm()) + for bf16 in (False, True): + r = res.clone() + outs = layer(c.x, c.w, c.ids, tail=(r, r, nw), bf16=bf16) + torch.cuda.synchronize() + y, q, s = outs[:3] + shapes_ok = q.shape[0] == s.shape[0] == m * WORLD and (not bf16 or outs[3].shape[0] == m * WORLD) + q, s = q[rows], s[rows] + e_q = float((q.view(fp8).float() * s[:, None] - xn_r).norm() / xn_r.norm()) + e_rows = float((outs[3][rows].float() - xn_r).norm() / xn_r.norm()) if bf16 else 0.0 + bad = [what for what, ok in ( + ("watchdog", not layer.poll_errors()), + ("shapes [T, ...]", shapes_ok), + ("y", torch.equal(y, y0)), + ("res_out", torch.equal(r, r_ref)), + (f"q rel {e_q:.4f} (fp8 {e_ref:.4f})", e_q < 1.05 * e_ref), + ("scale", float(((s - s_r).abs() / s_r).max()) < 1e-2), + (f"bf16 rows rel {e_rows:.4f}", e_rows < 0.01), + ) if not ok] # fmt: skip + if bad: + errors.append(f"rank {rank} tail tn_local={local} bf16={bf16}: {bad}") + eng.tn_local = True + layer.close() + + +def test_mega_moe_tp_fused_tail(): + run_on_4_gpus(_tail) + + +def _validation(rank, errors): + import os + + shape, wt = _weights("m3", rank) + _expect_raise(lambda: new_layer(shape, wt, rank, WORLD, "ag_rs", 4096, bigm_min_tokens=0), errors, + "max_local_tokens=4096 without the big-M path") + layer = new_layer(shape, wt, rank, WORLD, "ag_rs", MAX_LOCAL) + _prepare(layer, [8 * WORLD]) + c = make_case(shape, wt, rank, WORLD, "ag_rs", 8 * WORLD, "random", 5) + x, w, ids = c.x, c.w, c.ids + _expect_raise(lambda: layer(x.float(), w, ids), errors, "fp32 hidden states") + _expect_raise(lambda: layer(x[:, : shape.model_dim // 2].contiguous(), w, ids), errors, "hidden size") + _expect_raise(lambda: layer(x, w[:, :-1].contiguous(), ids[:, :-1].contiguous()), errors, "topk") + _expect_raise(lambda: layer(x, w, ids.to(torch.int16)), errors, "int16 topk_ids") + _expect_raise(lambda: layer(x, w, ids.cpu()), errors, "topk_ids on the cpu") + _expect_raise(lambda: layer(x, w, ids, out=torch.empty_like(x[:1])), errors, "out of the wrong shape") + layer.close() + # AKR_MEGA_MOE_TP_CHECK=1: the cross-rank input checks + os.environ["AKR_MEGA_MOE_TP_CHECK"] = "1" + try: + layer = new_layer(shape, wt, rank, WORLD, "ag_rs", MAX_LOCAL) + m = 16 if rank == 0 else 8 + c = make_case(shape, wt, rank, WORLD, "ag_rs", 16 * WORLD, "random", 14) + _expect_raise(lambda: layer(c.x[:m], c.w[:m], c.ids[:m]), errors, "ag_rs with uneven m") + layer.close() + layer = new_layer(shape, wt, rank, WORLD, "ar", MAX_LOCAL) + _prepare(layer, [8 * WORLD]) + c = make_case(shape, wt, rank, WORLD, "ar", 8 * WORLD, "random", 15) + ids = c.ids.clone() + if rank == 1: + ids[0, 0] = (ids[0, 0] + 1) % shape.experts + _expect_raise(lambda: layer(c.x, c.w, ids), errors, "ar with routing that differs across ranks") + _check(_call(layer, c, errors, "checked").clone(), c, errors, "checked call") + layer.close() + finally: + os.environ.pop("AKR_MEGA_MOE_TP_CHECK", None) + + +def test_mega_moe_tp_validation(): + run_on_4_gpus(_validation) + + +def _mori_importable() -> bool: + from amd_kernels.moe.mega_moe_tp import sdma + + try: + sdma._mori() + return True + except ImportError: + return False + + +def _big_m(rank, errors, mode): + pad = 64 + shape, wt = _weights("m3", rank) + layer = new_layer(shape, wt, rank, WORLD, mode, MAX_LOCAL, bigm_min_tokens=pad * WORLD) + if not layer.bigm_m: + errors.append("no big-M path") + return + tokens = [(pad - 1) * WORLD, pad * WORLD, 4 * pad * WORLD] + ([pad * WORLD + 1] if mode == "ar" else []) + _prepare(layer, tokens) + for t in tokens: + c = make_case(shape, wt, rank, WORLD, mode, t, "random", SEED + t) + out = torch.empty_like(c.ref, dtype=torch.bfloat16) + if _call(layer, c, errors, f"big-M M={t}", out=out).data_ptr() != out.data_ptr(): + errors.append(f"big-M M={t}: out= was not returned") + _check(out, c, errors, f"big-M M={t}") + c = make_case(shape, wt, rank, WORLD, mode, pad * WORLD, "random", SEED) + _expect_raise(lambda: layer(c.x, c.w, c.ids, out=torch.empty_like(c.ref, dtype=torch.float16)), errors, + "big-M out of the wrong dtype") + layer.close() + layer.close() + + +@pytest.mark.parametrize("mode", ["ag_rs", "ar"]) +def test_mega_moe_tp_big_m(mode): + if not _mori_importable(): + pytest.skip("needs MORI built with SDMA (importable or in AKR_MORI_PATH)") + run_on_4_gpus(_big_m, mode) diff --git a/tests/moe/mega_moe_tp/test_mega_moe_tp_sp_rs_norm.py b/tests/moe/mega_moe_tp/test_mega_moe_tp_sp_rs_norm.py new file mode 100644 index 0000000..eb8772a --- /dev/null +++ b/tests/moe/mega_moe_tp/test_mega_moe_tp_sp_rs_norm.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""SpRsNorm (MegaMoE TP's sequence-parallel reduce-scatter + residual add + Gemma RMSNorm, + its router) on 4 ranks +vs ``reference.sp_rs_norm_ref`` / ``reference.sigmoid_topk_ref``, twice back to back on new inputs.""" + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import WORLD, run_on_4_gpus + +from amd_kernels.moe.mega_moe_tp import SpRsNorm, mega_moe_tp_supported +from amd_kernels.moe.mega_moe_tp.reference import sigmoid_topk_ref, sp_rs_norm_ref + +H = 6144 +EPS = 1e-6 +ROUTER = (128, 4, 2.0, 0.5) # experts, top-k, routed scale, shared-expert weight +_OPS = {} + + +def _inputs(tokens, router, seed, rank, dev): + """this rank's partials of every token, the residual, the norm weight (+ the router's gate, bias, outputs)""" + E, K = ROUTER[:2] + gen = torch.Generator(device=dev).manual_seed(7) + wg = torch.randn((E, H), generator=gen, device=dev) * H**-0.5 * 3 + bias = (0.05 * torch.randn((E,), generator=gen, device=dev)).float() + w = (0.1 * torch.randn((H,), generator=gen, device=dev)).to(torch.bfloat16) + g = torch.Generator(device=dev).manual_seed(seed + rank + tokens) + part = torch.randn((tokens, H), generator=g, device=dev).to(torch.bfloat16) + res = torch.randn((tokens, H), generator=g, device=dev).to(torch.bfloat16) + rt = None + if router: + m = tokens // WORLD + ids = torch.empty((m, K + 1), dtype=torch.int32, device=dev) + tw = torch.empty((m, K + 1), dtype=torch.float32, device=dev) + rt = (wg.to(torch.bfloat16), bias, ids, tw) + return part, res, w, rt + + +def _worker(rank, errors, tokens, router): + dev = torch.device("cuda", rank) + if "op" not in _OPS: + _OPS["op"] = SpRsNorm(H, 1024, EPS, device=dev, router=ROUTER) + op = _OPS["op"] + if router and not op.routes(tokens): + errors.append(f"the router does not run at {tokens} tokens") + return + E, K, scale, shared_w = ROUTER + m = tokens // WORLD + rows = slice(rank * m, (rank + 1) * m) + for seed in (1000, 2000): + part, res, w, rt = _inputs(tokens, router, seed, rank, dev) + out, res_out = op(part, res, w, router=rt) + torch.cuda.synchronize() + tot = part.float().cpu() + dist.all_reduce(tot) + h, ref = sp_rs_norm_ref(tot[rows].to(dev), res[rows], w, EPS) + e = max(float((res_out[rows].float() - h).norm() / h.norm()), + float((out[rows].float() - ref).norm() / ref.norm())) + if op.poll_errors() or not e < 0.01: + errors.append(f"rank {rank} M={tokens} router={router}: rel_l2 {e:.4f}, watchdog {op.poll_errors()}") + if rt is None: + continue + _, _, ids, tw = rt + rtw, rid = sigmoid_topk_ref(ref.to(torch.bfloat16), rt[0], rt[1], K, scale) + same = (ids[:, :K].sort(1).values == rid.sort(1).values).all(1) + mine = torch.gather(tw[:, :K], 1, ids[:, :K].argsort(1)) + want = torch.gather(rtw, 1, rid.argsort(1)) + e_w = ((mine - want).abs() / want.abs().clamp_min(1e-6))[same] + e_w = float(e_w.max()) if e_w.numel() else 0.0 + # bf16 logits: a near tie may pick another expert (the share is over every rank's rows) + st = torch.tensor([float(same.sum()), m]) + dist.all_reduce(st) + frac = float(st[0] / st[1]) + if frac < 0.9 or e_w > 0.02 or not bool((ids[:, K] == E).all() and (tw[:, K] == shared_w).all()): + errors.append(f"rank {rank} M={tokens} router: same experts {frac:.3f}, weight err {e_w:.1e}") + + +@pytest.mark.parametrize(("tokens", "router"), [(64, True), (1024, True), (64, False), (1000, False)]) +def test_mega_moe_tp_sp_rs_norm(tokens, router): + if not torch.cuda.is_available() or not mega_moe_tp_supported(): + pytest.skip("needs gfx950") + run_on_4_gpus(_worker, tokens, router) diff --git a/tests/moe/mega_moe_tp/test_mega_moe_tp_warm_up.py b/tests/moe/mega_moe_tp/test_mega_moe_tp_warm_up.py new file mode 100644 index 0000000..6eb249a --- /dev/null +++ b/tests/moe/mega_moe_tp/test_mega_moe_tp_warm_up.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""Rule 4 on 4 ranks: after ``MegaMoeTP.prepare`` of every local token count (with the fused-tail variants), a walk +over those counts, eager and under CUDA graph capture, compiles, loads and arms nothing: no FlyDSL compile, no launcher +run for the first time, no launch config armed.""" + +import pytest +import torch +import torch.distributed as dist +from tests.multi_gpu import WORLD, run_on_4_gpus + +from amd_kernels.moe.mega_moe_tp import mega_moe_tp_supported + +from .workload import SHAPES, build_weights, make_case, new_layer + +MAX_LOCAL = 256 +WALK = (1, 2, 3, 7, 8, 16, 31, 32, 63, 64, 100, 128, 129, 200, 255, 256) + + +def _worker(rank, errors, mode): + from flydsl.compiler.jit_function import MlirCompiler + + from amd_kernels.moe.mega_moe_tp import fly_utils + from amd_kernels.moe.mega_moe_tp import mega_moe_tp as host + + shape = SHAPES["m3"] + wt = build_weights(shape, rank, WORLD, torch.device("cuda", rank)) + layer = new_layer(shape, wt, rank, WORLD, mode, MAX_LOCAL) + tail = mode == "ag_rs" + layer.prepare(range(1, MAX_LOCAL + 1), tail=tail, tail_bf16=tail) + events = [] + real = (MlirCompiler.compile, fly_utils._compile_and_run, host._preload_compiled) + + def spy(what, fn): + def wrapped(*a, **kw): + events.append(what) + return fn(*a, **kw) + + return wrapped + + MlirCompiler.compile = spy("compile", real[0]) + fly_utils._compile_and_run = spy("first run", real[1]) + host._preload_compiled = spy("arm", real[2]) + try: + for m in WALK: + c = make_case(shape, wt, rank, WORLD, mode, m * WORLD, "random", m) + layer(c.x, c.w, c.ids) + if tail and layer.tail_ok(m): + res = torch.zeros_like(c.x) + for bf16 in (False, True): + layer(c.x, c.w, c.ids, tail=(res, res, c.x[0]), bf16=bf16) + torch.cuda.synchronize() + dist.barrier() + s = torch.cuda.Stream() + s.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(s): + layer(c.x, c.w, c.ids) + torch.cuda.current_stream().wait_stream(s) + torch.cuda.synchronize() + dist.barrier() + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + layer(c.x, c.w, c.ids) + g.replay() + torch.cuda.synchronize() + dist.barrier() + if events: + errors.append(f"rank {rank} {mode} m={m}: {sorted(set(events))} after prepare") + events.clear() + if layer.poll_errors(): + errors.append(f"rank {rank} {mode} m={m}: watchdog error") + layer.clear_errors() + finally: + MlirCompiler.compile, fly_utils._compile_and_run, host._preload_compiled = real + layer.close() + + +@pytest.mark.parametrize("mode", ["ag_rs", "ar"]) +def test_mega_moe_tp_warm_up(mode): + if not torch.cuda.is_available() or not mega_moe_tp_supported(): + pytest.skip("needs gfx950") + run_on_4_gpus(_worker, mode, timeout=3600) diff --git a/tests/moe/mega_moe_tp/workload.py b/tests/moe/mega_moe_tp/workload.py new file mode 100644 index 0000000..64e8478 --- /dev/null +++ b/tests/moe/mega_moe_tp/workload.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the amd-kernels project +"""MegaMoE TP inputs for the tests and the benchmark: the served shapes, random MXFP4 weights of this rank's inter +slice (kernel layout + row-major for ``reference.py``), routed inputs every rank rebuilds from seeds (no tensor +collective: the test ranks run a gloo group), and the reference output summed over the ranks on the host.""" + +from dataclasses import dataclass + +import torch +import torch.distributed as dist + +from amd_kernels.moe.mega_moe_tp import MegaMoeTP, MegaMoeTPConfig, shuffle_scale, shuffle_weight +from amd_kernels.moe.mega_moe_tp.reference import mega_moe_tp_ref, mxfp4_quant + +SEED = 123 + + +@dataclass(frozen=True) +class Shape: + name: str + model_dim: int + inter_dim: int # all ranks' + experts: int + topk: int + act: str + + +# m3: H 6144, I 3072 (768 per rank on 4), 128 routed + 1 shared expert (top 4 + shared), SwiGLU-OAI +SHAPES = { + "m3": Shape("m3", 6144, 3072, 129, 5, "swiglu"), + "glm5": Shape("glm5", 6144, 2048, 257, 9, "silu"), +} + + +@dataclass +class Weights: + inter: int + w1: torch.Tensor + w1_scale: torch.Tensor + w2: torch.Tensor + w2_scale: torch.Tensor + w1_ref: torch.Tensor + w1_scale_ref: torch.Tensor + w2_ref: torch.Tensor + w2_scale_ref: torch.Tensor + + +def _quant_experts(experts, rows, cols, magnitude, seed, device, chunk_bytes=512 << 20): + chunk = max(1, min(experts, chunk_bytes // max(rows * cols * 2, 1))) + codes = torch.empty((experts, rows, cols // 2), dtype=torch.uint8, device=device) + scales = torch.empty((experts, rows, cols // 32), dtype=torch.uint8, device=device) + gen = torch.Generator(device=device) + for start in range(0, experts, chunk): + n = min(chunk, experts - start) + gen.manual_seed(seed + start) + w = torch.randn((n, rows, cols), dtype=torch.bfloat16, device=device, generator=gen) + codes[start : start + n], scales[start : start + n] = mxfp4_quant(w.mul_(magnitude)) + return codes, scales + + +def build_weights(shape: Shape, rank: int, world: int, device, seed=SEED) -> Weights: + inter = shape.inter_dim // world + H, E = shape.model_dim, shape.experts + base = seed + 1_000_000 * rank + w1, s1 = _quant_experts(E, 2 * inter, H, H**-0.25, base, device) + w2, s2 = _quant_experts(E, H, inter, inter**-0.25, base + 7, device) + return Weights(inter, shuffle_weight(w1), shuffle_scale(s1.view(E * 2 * inter, -1)), shuffle_weight(w2), + shuffle_scale(s2.view(E * H, -1)), w1, s1, w2, s2) + + +def route(rows, shape: Shape, kind: str, gen: torch.Generator, device): + """(topk weights fp32, topk ids int32) of ``rows`` tokens: balanced, random, hot (every token on the last topk + experts) or subset (~55% of the experts).""" + E, K = shape.experts, shape.topk + if kind == "balanced": + start = int(torch.randint(0, E, (1,), device=device, generator=gen)) + ids = (start + torch.arange(rows * K, device=device)) % E + score = torch.full((rows, E), -1e4, device=device) + score.scatter_(1, ids.view(rows, K), 0.0) + score += 1e-3 * torch.randn((rows, E), device=device, generator=gen) + else: + score = torch.randn((rows, E), device=device, generator=gen) + if kind == "hot": + score[:, E - K :] += 100.0 + elif kind == "subset": + score[:, E * 35 // 64 :] = -1e4 + w, ids = torch.softmax(score, dim=-1).topk(K, dim=-1) + w = w / w.sum(dim=-1, keepdim=True) + return w.float().contiguous(), ids.to(torch.int32).contiguous() + + +def _inputs(shape, rows, kind, seed, device, mask): + gen = torch.Generator(device=device).manual_seed(seed) + x = torch.randn((rows, shape.model_dim), dtype=torch.bfloat16, device=device, generator=gen) + w, ids = route(rows, shape, kind, gen, device) + if mask: + # a fraction of the routes masked: id -1 or an id >= E (no expert) + sel = torch.rand(ids.shape, device=device, generator=gen) + ids = torch.where(sel < mask / 2, -1, torch.where(sel < mask, shape.experts + 3, ids)).to(torch.int32) + return x, w, ids.contiguous() + + +@dataclass +class Case: + x: torch.Tensor # this rank's input (ag_rs: its m rows; ar: all M rows) + w: torch.Tensor + ids: torch.Tensor + ref: torch.Tensor # fp32, the rows this rank's output holds + ids_all: torch.Tensor # every token's ids + + +def make_case(shape: Shape, wt: Weights, rank: int, world: int, mode: str, tokens: int, kind: str, seed: int, + mask: float = 0.0) -> Case: + """``tokens`` global tokens; ag_rs: rank r's rows from seed + 7919 r, ar: one replicated batch from seed.""" + dev = wt.w1.device + if mode == "ag_rs": + m = tokens // world + parts = [_inputs(shape, m, kind, seed + 7919 * r, dev, mask) for r in range(world)] + x, w, ids = (torch.cat(t) for t in zip(*parts)) + own = parts[rank] + else: + x, w, ids = own = _inputs(shape, tokens, kind, seed, dev, mask) + part = mega_moe_tp_ref(x, w, ids, wt.w1_ref, wt.w1_scale_ref, wt.w2_ref, wt.w2_scale_ref, shape.act).cpu() + dist.all_reduce(part) + ref = part.to(dev) + if mode == "ag_rs": + ref = ref[rank * m : (rank + 1) * m] + return Case(*own, ref, ids) + + +def new_layer(shape: Shape, wt: Weights, rank: int, world: int, mode: str, max_local_tokens: int, + comm_dtype: str = "fp8", ar_gather: str = "bf16", **kw) -> MegaMoeTP: + cfg = MegaMoeTPConfig(rank=rank, world_size=world, model_dim=shape.model_dim, inter_dim=wt.inter, + experts=shape.experts, topk=shape.topk, max_local_tokens=max_local_tokens, + activation=shape.act, comm_mode=mode, comm_dtype=comm_dtype, ar_gather=ar_gather, **kw) + return MegaMoeTP(cfg, w1=wt.w1, w1_scale=wt.w1_scale, w2=wt.w2, w2_scale=wt.w2_scale, device=wt.w1.device) + + +def rel_l2(actual: torch.Tensor, expected: torch.Tensor) -> float: + """Relative L2 over every rank's rows (host all-reduce).""" + d = actual.float() - expected.float() + t = torch.stack([(d * d).sum(), (expected.float() ** 2).sum()]).cpu() + dist.all_reduce(t) + return float((t[0] / t[1]).sqrt()) if t[1] > 0 else float("nan") + + +def rank_max(v: float) -> float: + t = torch.tensor([float(v)]) + dist.all_reduce(t, op=dist.ReduceOp.MAX) + return float(t.item()) + + +def same_on_all_ranks(t: torch.Tensor) -> bool: + ref = t.cpu() + dist.broadcast(ref, src=0) + return rank_max(0.0 if torch.equal(ref, t.cpu()) else 1.0) == 0.0 diff --git a/tools/ci/targets.tsv b/tools/ci/targets.tsv index 93278c9..8481123 100644 --- a/tools/ci/targets.tsv +++ b/tools/ci/targets.tsv @@ -55,3 +55,4 @@ gemm/bf16_n50016_k6144/test_gemm.py 1 same attention/nvfp4_sparse_decode_gqa16_d128_p128_top16/test_index_decode_ranks4.py 4 sampling/batch_sharded_markov_v200064_h6144_r256_ranks4/test_batch_sharded.py 4 decoder/dense_mxfp8_h6144_i3072_ranks4/test_decoder_ranks4.py 4 +moe/mega_moe_tp 4 From 8e6c08535917b6d00679b0cdb70f4a92438ca687 Mon Sep 17 00:00:00 2001 From: BinDing Date: Sat, 14 Nov 2026 08:14:16 +0000 Subject: [PATCH 3/3] update --- amd_kernels/moe/mega_moe_tp/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/amd_kernels/moe/mega_moe_tp/README.md b/amd_kernels/moe/mega_moe_tp/README.md index 20d3b7b..77ca5e7 100644 --- a/amd_kernels/moe/mega_moe_tp/README.md +++ b/amd_kernels/moe/mega_moe_tp/README.md @@ -41,7 +41,6 @@ calls over 4 inputs, median of 20 replays, slowest rank, us per call. No baselin ## Tests ```bash -cd amd-kernels && source env.sh # or: PYTHONPATH=$PWD/.cache/flydsl-0.5.0:$PWD python3 -m pytest tests/moe/mega_moe_tp -q # 4 GPUs; big-M cases need MORI python3 benchmarks/moe/mega_moe_tp/benchmark_mega_moe_tp.py # 4 GPUs ```