diff --git a/CMakeLists.txt b/CMakeLists.txt index bf0cb2f4e..c08eac3fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(PYBIND11_ENABLE_EXTRAS ON) # Options for backends. +set(INFINI_OPS_DECLARATIONS "" CACHE STRING "Additional public operator APIs, even without active providers") + option(WITH_CPU "Enable CPU backend" OFF) option(WITH_NVIDIA "Enable CUDA backend" OFF) option(WITH_ILUVATAR "Enable Iluvatar GPU backend" OFF) @@ -430,10 +432,10 @@ if(WITH_NINETOOTHED) endif() if(WITH_TRITON) - if(NOT WITH_NVIDIA) + if(NOT WITH_NVIDIA AND NOT WITH_HYGON) message( FATAL_ERROR - "`WITH_TRITON` requires `WITH_NVIDIA=ON` because NVIDIA is the only implemented Triton JIT backend." + "`WITH_TRITON` requires `WITH_NVIDIA=ON` or `WITH_HYGON=ON`." ) endif() if(NOT GENERATE_PYTHON_BINDINGS) diff --git a/README.md b/README.md index fe72cdddd..73d1f0f39 100644 --- a/README.md +++ b/README.md @@ -85,3 +85,6 @@ Useful entry points: ## License See [LICENSE](LICENSE). + +Hygon Clover diffusion and native Triton JIT setup are described in +[the Clover operator guide](docs/clover-hygon.md). diff --git a/THIRD_PARTY.txt b/THIRD_PARTY.txt index e588b1443..3cc8a0337 100644 --- a/THIRD_PARTY.txt +++ b/THIRD_PARTY.txt @@ -113,3 +113,11 @@ from actual package metadata. That conservative package inventory can include materials for components outside these two DSOs; it is intentionally not filtered by fixed hashes or used to restrict build capability. A complete wheel or SDK redistribution must retain its own applicable notices, including secondary ones. + +Clover preview kernels +src/triton/ops/clover_block_conv1d/jit.py and +src/triton/ops/clover_local_memory/jit.py are adapted from SGLang PR #46927, +WilliamZR/sglang commit 058939af24d5cdc44bb4b8852d0ddfab555ed71e, +python/sglang/kernels/ops/{mamba/clover_conv_triton.py,attention/clover_gdn_triton.py}. +They retain Apache-2.0 licensing. Changes provide InfiniOps JIT entry points, +Hygon launch geometry, and bounds checks for the state pool. diff --git a/docs/clover-hygon.md b/docs/clover-hygon.md new file mode 100644 index 000000000..b17a51db1 --- /dev/null +++ b/docs/clover-hygon.md @@ -0,0 +1,77 @@ +# Hygon Clover operators + +These operators support the companion InfiniLM Clover implementation. Model +forward stays in C++; Hygon driver modules launch warmed Triton binaries. +Compilation uses the installed DTK Triton HIP target (for example `gfx936`, +wave size 64), not CUDA compute-capability arithmetic. + +## Build + +Install an InfiniRT revision containing the Hygon driver interface first +([InfiniRT PR #52](https://github.com/InfiniTensor/InfiniRT/pull/52)). Then: + +```sh +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INFINI_ROOT" -DINFINI_RT_ROOT="$INFINI_ROOT" \ + -DWITH_CPU=ON -DWITH_HYGON=ON -DHYGON_ARCH=gfx936 -DWITH_TRITON=ON \ + -DGENERATE_PYTHON_BINDINGS=ON -DAUTO_DETECT_DEVICES=OFF \ + -DAUTO_DETECT_BACKENDS=OFF +cmake --build build -j 12 +cmake --install build +``` + +The Python `infini.triton` package must be importable. CMake installs its package +contents at the prefix, while wheel installation supplies the `infini` namespace. +Include DTK's CUDA compatibility library directory in `LD_LIBRARY_PATH`. + +`INFINI_OPS_DECLARATIONS` accepts a semicolon-separated list of additional public +operator APIs. These can have no provider in the selected build: capability +queries return unsupported, and execution is rejected. This allows a downstream +application to compile its complete model registry without claiming that every +registered model runs on Hygon. InfiniLM supplies a matching declaration preset. + +## Operator contracts and source alignment + +The Clover-specific APIs target the released Clover model equations and state +layout, rather than claiming to be PyTorch standard operators. Preview kernel +provenance and licensing are recorded in `THIRD_PARTY.txt`. + +| Operator | Inputs and behavior | +| --- | --- | +| `CloverBlockConv1d` | Packed `[tokens,channels]` BF16/FP16 input, `[slots,channels,3]` clean history, slot indices, left `[channels,1,4]` and learned right `[channels,1,3]` weights, int8 validity, block width up to 32. Produces SiLU output without updating history. | +| `CloverLocalMemory` | Q/K/V `[1,tokens,heads,128]`, FP32 gates and `[slots,value_heads,128,128]` state, slot indices, validity and three per-head scalars. Implements bidirectional block attention plus the decayed clean-history bucket. Does not update state. | +| `CloverRecurrentGdn` | Packed Q/K/V, gates/beta, sequence offsets and initial/final slot vectors. Implements normalized causal delta recurrence with FP32 persistent state; writes sequence output and final state. Input/final slot indices must be valid and final destinations must not race. | +| `CloverCoreset` | Contiguous `[tokens,256]` router logits, block width and routing mass. Keeps the stable top-p support of aggregated per-token top-8 scores and masks other logits. Requires an explicit workspace. | +| `CloverLiveMask` | Current/previous int64 token IDs and the mask ID. Marks rows live if either token is masked, or refresh is forced; updates previous IDs. | +| `CloverMaskRoutes` | Replaces inactive rows' int32 expert IDs with the out-of-range sentinel `num_experts`. | +| `CloverMergeMoe` | Selects fresh routed output for live rows and cached output for other rows; updates the cache. Output and fresh input may alias. | +| `MoeFusedDense` (Hygon) | Existing packed dense-MoE API, with BF16/FP16 weights, top-k weights/IDs and aligned expert metadata. Triton computes gate/up, fused SiLU/down, and weighted reduction. Invalid routes contribute zero; alignment block size 16 is supported and expert biases are unsupported. Requires an explicit workspace. | + +The live-row protocol is an optional approximation for iterative diffusion, +adapted from the supplied dllm-h100 TV6 study. It is not a general guarantee that +fixed token IDs imply unchanged hidden states. InfiniLM refreshes every five +previews and always runs full causal replay before committing state. + +The standard `MoeAlignBlockSize`, causal convolution, gating, cast, fill, +multiplication, triangular masks and rotary providers also gain Hygon coverage. +They reuse this repository's existing native CUDA-like implementations. + +## Validation + +Actual BW/DTK runs compare preview operators to independent Torch equations and +verify bitwise state immutability, mixed-length causal state commits, coreset +selection, dense MoE, liveness and cache merging. Public-header and wrapper/JIT +unit tests cover the HIP architecture string and metadata without CUDA scratch +fields. The companion InfiniLM campaign runs full checkpoint generation. + +Workspace-bearing operators are invoked through the C++ API with an owning +workspace. The native test adapter in `tests/native/clover_workspace_smoke.cpp` +exercises that path. A missing workspace is rejected before launching a kernel. + +Build and run the workspace adapter against the installed SDK: + +```sh +python tests/native/build_clover_workspace.py +PYTHONPATH="build/clover-tests:$PYTHONPATH" python tests/native/check_clover_workspace.py +python -m pytest tests/test_clover_preview.py tests/test_clover_live_rows.py -q +``` diff --git a/docs/operator-api-alignment.md b/docs/operator-api-alignment.md index 1118420ab..c70da2387 100644 --- a/docs/operator-api-alignment.md +++ b/docs/operator-api-alignment.md @@ -238,3 +238,9 @@ unless callers need it. When applying these rules, record the selected upstream interface and any intentional C++ adaptations in the pull request. Follow `CONTRIBUTING.md` for compatibility, testing, and validation requirements. + +## Clover diffusion state operators + +The `Clover*` operators use the released Clover model equations, with packed +request/state-pool arguments for inference. Their exact contracts and source +provenance are documented in [the Hygon Clover guide](clover-hygon.md). diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py index bb03d2b57..269dd246c 100644 --- a/scripts/generate_wrappers.py +++ b/scripts/generate_wrappers.py @@ -2298,6 +2298,12 @@ def _dispatch_gen_batch_size(): help="Include Triton backend implementations.", ) + parser.add_argument( + "--declarations", + nargs="*", + default=[], + help="Also emit operator APIs with no active provider.", + ) args = parser.parse_args() for directory in (_BINDINGS_DIR, _GENERATED_SRC_DIR, _INCLUDE_DIR): @@ -2314,6 +2320,11 @@ def _dispatch_gen_batch_size(): with_native_w4=args.with_native_w4, ) + for name in _normalize_op_allowlist(args.declarations): + if not (_BASE_DIR / f"{name}.h").is_file(): + raise ValueError(f"Unknown operator declaration: {name}") + ops.setdefault(name, []) + if config_path is not None: config = ops_config.load_ops_config(config_path) ops = _select_ops_from_config(ops, config, config_path) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e685b4f12..49e7f8e3c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -158,6 +158,11 @@ if(WITH_NVIDIA) ) file(GLOB_RECURSE NVIDIA_SOURCES CONFIGURE_DEPENDS ${NVIDIA_PATTERNS}) + # The recursive top-level `native/cuda/*.cu` pattern also matches sibling + # vendor subtrees (hygon/iluvatar/metax/moore); drop them so their symbols + # are not defined twice when those backends are also enabled. + list(FILTER NVIDIA_SOURCES EXCLUDE REGEX + ".*/native/cuda/(hygon|iluvatar|metax|moore|thead|cambricon|ascend)/.*") # Optional pinned kernels have their own include closure and CUDA TUs. list(FILTER NVIDIA_SOURCES EXCLUDE REGEX "/ops/flash_attn_with_kvcache/") list(FILTER NVIDIA_SOURCES EXCLUDE REGEX "/ops/flash_attn_varlen_func/") @@ -249,6 +254,11 @@ if(WITH_TRITON OR GENERATE_PYTHON_BINDINGS) find_package(pybind11 CONFIG REQUIRED) endif() +if(WITH_TRITON AND WITH_HYGON) + target_link_directories(infiniops PRIVATE "${HYGON_CUDA_ROOT}/lib64") + target_link_libraries(infiniops PRIVATE cuda) +endif() + if(WITH_TRITON) target_link_libraries(infiniops PRIVATE Python::Python @@ -1303,6 +1313,9 @@ if(GENERATE_OPERATOR_CALL_INSTANTIATIONS OR GENERATE_PYTHON_BINDINGS) # failures. set(GENERATOR_ARGS --devices ${DEVICE_LIST}) + if(INFINI_OPS_DECLARATIONS) + list(APPEND GENERATOR_ARGS --declarations ${INFINI_OPS_DECLARATIONS}) + endif() if(_infini_ops_ops_config) list(APPEND GENERATOR_ARGS --ops-config "${_infini_ops_ops_config}") diff --git a/src/base/clover_block_conv1d.h b/src/base/clover_block_conv1d.h new file mode 100644 index 000000000..c6b90fd3c --- /dev/null +++ b/src/base/clover_block_conv1d.h @@ -0,0 +1,49 @@ +#pragma once + +#include "operator.h" + +namespace infini::ops { + +// Clover preview convolution reads a clean prefix without updating its state. +class CloverBlockConv1d : public Operator { + public: + CloverBlockConv1d(const Tensor input, const Tensor state, + const Tensor indices, const Tensor weight, + const Tensor right_weight, const Tensor valid, + size_t block_size, Tensor out) { + assert((input.ndim() == 2 && out.shape() == input.shape() && + state.ndim() == 3 && weight.ndim() == 3 && + weight.shape() == Tensor::Shape{input.size(1), 1, 4} && + right_weight.shape() == Tensor::Shape{input.size(1), 1, 3} && + state.size(1) == input.size(1) && state.size(2) == 3 && + indices.ndim() == 1 && block_size > 0 && block_size <= 32 && + input.size(0) == indices.numel() * block_size && + valid.shape() == Tensor::Shape{input.size(0)}) && + "CloverBlockConv1d: invalid block or state shape."); + assert((input.dtype() == DataType::kBFloat16 || + input.dtype() == DataType::kFloat16) && + "CloverBlockConv1d: expected BF16 or FP16 input."); + for (const auto& tensor : {state, weight, right_weight, out}) { + assert(tensor.dtype() == input.dtype() && + "CloverBlockConv1d: input and weight dtypes must match."); + } + assert((indices.dtype() == DataType::kInt32 || + indices.dtype() == DataType::kInt64) && + indices.IsContiguous() && valid.dtype() == DataType::kInt8 && + valid.IsContiguous() && + "CloverBlockConv1d: expected contiguous indices and int8 validity."); + for (const auto& tensor : + {state, indices, weight, right_weight, valid, out}) { + assert(tensor.device() == input.device() && + "CloverBlockConv1d: device mismatch."); + } + assert(!out.HasBroadcastDim() && "CloverBlockConv1d: overlapping output."); + } + + virtual void operator()(const Tensor input, const Tensor state, + const Tensor indices, const Tensor weight, + const Tensor right_weight, const Tensor valid, + size_t block_size, Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_coreset.h b/src/base/clover_coreset.h new file mode 100644 index 000000000..198890fc9 --- /dev/null +++ b/src/base/clover_coreset.h @@ -0,0 +1,23 @@ +#pragma once + +#include "operator.h" + +namespace infini::ops { + +class CloverCoreset : public Operator { + public: + CloverCoreset(const Tensor logits, size_t block_size, double top_p, + Tensor out) { + assert(logits.ndim() == 2 && logits.size(1) == 256 && block_size > 0 && + block_size <= 32 && logits.size(0) % block_size == 0 && + out.shape() == logits.shape() && out.dtype() == logits.dtype() && + logits.IsContiguous() && out.IsContiguous() && + logits.device() == out.device() && top_p > 0 && top_p <= 1 && + "CloverCoreset: invalid logits, block size or routing mass."); + } + + virtual void operator()(const Tensor logits, size_t block_size, double top_p, + Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_live_mask.h b/src/base/clover_live_mask.h new file mode 100644 index 000000000..f688402db --- /dev/null +++ b/src/base/clover_live_mask.h @@ -0,0 +1,22 @@ +#pragma once +#include "operator.h" + +namespace infini::ops { + +class CloverLiveMask : public Operator { + public: + CloverLiveMask(const Tensor ids, Tensor previous, int64_t mask_id, + bool refresh, Tensor live) { + assert(ids.IsContiguous() && previous.IsContiguous() && + live.IsContiguous() && ids.numel() == previous.numel() && + ids.numel() == live.numel() && ids.dtype() == DataType::kInt64 && + previous.dtype() == ids.dtype() && live.dtype() == DataType::kInt8 && + ids.device() == previous.device() && ids.device() == live.device() && + "CloverLiveMask: invalid token state."); + } + + virtual void operator()(const Tensor ids, Tensor previous, int64_t mask_id, + bool refresh, Tensor live) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_local_memory.h b/src/base/clover_local_memory.h new file mode 100644 index 000000000..f58757891 --- /dev/null +++ b/src/base/clover_local_memory.h @@ -0,0 +1,59 @@ +#pragma once + +#include "operator.h" + +namespace infini::ops { + +class CloverLocalMemory : public Operator { + public: + CloverLocalMemory(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor state, const Tensor indices, + const Tensor valid, const Tensor logit_scale, + const Tensor history_bias, const Tensor history_c, + size_t block_size, Tensor out) { + assert((q.ndim() == 4 && q.size(0) == 1 && k.shape() == q.shape() && + v.ndim() == 4 && v.size(0) == 1 && v.size(1) == q.size(1) && + q.size(2) > 0 && v.size(2) > 0 && v.size(2) % q.size(2) == 0 && + q.size(3) == 128 && v.size(3) == 128 && out.shape() == v.shape() && + state.ndim() == 4 && state.size(1) == v.size(2) && + state.size(2) == 128 && state.size(3) == 128 && + indices.ndim() == 1 && block_size > 0 && block_size <= 32 && + q.size(1) == indices.numel() * block_size && + g.shape() == Tensor::Shape{1, q.size(1), v.size(2)} && + valid.shape() == Tensor::Shape{q.size(1)}) && + "CloverLocalMemory: invalid sequence, head or state shape."); + assert( + (q.dtype() == DataType::kBFloat16 || q.dtype() == DataType::kFloat16) && + k.dtype() == q.dtype() && v.dtype() == q.dtype() && + out.dtype() == q.dtype() && state.dtype() == DataType::kFloat32 && + g.dtype() == DataType::kFloat32 && + "CloverLocalMemory: expected reduced data and FP32 state/gates."); + assert((indices.dtype() == DataType::kInt32 || + indices.dtype() == DataType::kInt64) && + indices.IsContiguous() && valid.dtype() == DataType::kInt8 && + valid.IsContiguous() && + "CloverLocalMemory: invalid index/validity dtype."); + for (const auto& scalar : {logit_scale, history_bias, history_c}) { + assert(scalar.ndim() == 1 && scalar.numel() == v.size(2) && + scalar.IsContiguous() && + (scalar.dtype() == q.dtype() || + scalar.dtype() == DataType::kFloat32) && + "CloverLocalMemory: invalid per-head scalars."); + } + for (const auto& tensor : {k, v, g, state, indices, valid, logit_scale, + history_bias, history_c, out}) { + assert(tensor.device() == q.device() && + "CloverLocalMemory: device mismatch."); + } + assert(!out.HasBroadcastDim() && "CloverLocalMemory: overlapping output."); + } + + virtual void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor state, + const Tensor indices, const Tensor valid, + const Tensor logit_scale, const Tensor history_bias, + const Tensor history_c, size_t block_size, + Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_mask_routes.h b/src/base/clover_mask_routes.h new file mode 100644 index 000000000..d04b5ad99 --- /dev/null +++ b/src/base/clover_mask_routes.h @@ -0,0 +1,22 @@ +#pragma once +#include "operator.h" + +namespace infini::ops { + +class CloverMaskRoutes : public Operator { + public: + CloverMaskRoutes(const Tensor ids, const Tensor live, int64_t num_experts, + Tensor out) { + assert(ids.ndim() == 2 && ids.IsContiguous() && live.IsContiguous() && + out.IsContiguous() && live.numel() == ids.size(0) && + out.shape() == ids.shape() && ids.dtype() == DataType::kInt32 && + out.dtype() == ids.dtype() && live.dtype() == DataType::kInt8 && + ids.device() == live.device() && ids.device() == out.device() && + "CloverMaskRoutes: invalid routes."); + } + + virtual void operator()(const Tensor ids, const Tensor live, + int64_t num_experts, Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_merge_moe.h b/src/base/clover_merge_moe.h new file mode 100644 index 000000000..6ccc62a81 --- /dev/null +++ b/src/base/clover_merge_moe.h @@ -0,0 +1,24 @@ +#pragma once +#include "operator.h" + +namespace infini::ops { + +class CloverMergeMoe : public Operator { + public: + CloverMergeMoe(const Tensor fresh, const Tensor live, Tensor cache, + Tensor out) { + assert(fresh.ndim() == 2 && live.numel() == fresh.size(0) && + cache.shape() == fresh.shape() && out.shape() == fresh.shape() && + cache.dtype() == fresh.dtype() && out.dtype() == fresh.dtype() && + live.dtype() == DataType::kInt8 && fresh.IsContiguous() && + cache.IsContiguous() && out.IsContiguous() && live.IsContiguous() && + fresh.device() == live.device() && + fresh.device() == cache.device() && fresh.device() == out.device() && + "CloverMergeMoe: invalid cached rows."); + } + + virtual void operator()(const Tensor fresh, const Tensor live, Tensor cache, + Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/base/clover_recurrent_gdn.h b/src/base/clover_recurrent_gdn.h new file mode 100644 index 000000000..10dd76066 --- /dev/null +++ b/src/base/clover_recurrent_gdn.h @@ -0,0 +1,51 @@ +#pragma once + +#include "operator.h" + +namespace infini::ops { + +class CloverRecurrentGdn : public Operator { + public: + CloverRecurrentGdn(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, const Tensor offsets, + const Tensor initial_indices, const Tensor final_indices, + Tensor state, Tensor out) { + assert((q.ndim() == 4 && q.size(0) == 1 && k.shape() == q.shape() && + v.ndim() == 4 && v.size(0) == 1 && q.size(1) == v.size(1) && + q.size(2) > 0 && v.size(2) % q.size(2) == 0 && q.size(3) == 128 && + v.size(3) == 128 && out.shape() == v.shape() && + g.shape() == Tensor::Shape{1, q.size(1), v.size(2)} && + beta.shape() == g.shape() && state.ndim() == 4 && + state.size(1) == v.size(2) && state.size(2) == 128 && + state.size(3) == 128 && offsets.ndim() == 1 && + offsets.numel() >= 2 && initial_indices.ndim() == 1 && + final_indices.shape() == initial_indices.shape() && + initial_indices.numel() + 1 == offsets.numel()) && + "CloverRecurrentGdn: invalid sequence or state shape."); + assert( + (q.dtype() == DataType::kBFloat16 || q.dtype() == DataType::kFloat16) && + k.dtype() == q.dtype() && v.dtype() == q.dtype() && + out.dtype() == q.dtype() && state.dtype() == DataType::kFloat32 && + g.dtype() == DataType::kFloat32 && beta.dtype() == DataType::kFloat32 && + "CloverRecurrentGdn: invalid dtype."); + for (const auto& index : {offsets, initial_indices, final_indices}) { + assert(index.IsContiguous() && + (index.dtype() == DataType::kInt32 || + index.dtype() == DataType::kInt64) && + "CloverRecurrentGdn: invalid sequence indices."); + } + for (const auto& tensor : + {k, v, g, beta, offsets, initial_indices, final_indices, state, out}) { + assert(tensor.device() == q.device() && + "CloverRecurrentGdn: device mismatch."); + } + } + + virtual void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, + const Tensor offsets, const Tensor initial_indices, + const Tensor final_indices, Tensor state, + Tensor out) const = 0; +}; + +} // namespace infini::ops diff --git a/src/native/cuda/hygon/caster.cuh b/src/native/cuda/hygon/caster.cuh index 4b381d03b..901fc1fd5 100644 --- a/src/native/cuda/hygon/caster.cuh +++ b/src/native/cuda/hygon/caster.cuh @@ -7,6 +7,9 @@ namespace infini::ops { +using infini::rt::cuda_bfloat16; +using infini::rt::cuda_bfloat162; + namespace detail { template <> diff --git a/src/native/cuda/hygon/ops/cast/kernel.h b/src/native/cuda/hygon/ops/cast/kernel.h new file mode 100644 index 000000000..250195f06 --- /dev/null +++ b/src/native/cuda/hygon/ops/cast/kernel.h @@ -0,0 +1,192 @@ +#ifndef INFINI_OPS_HYGON_CAST_KERNEL_H_ +#define INFINI_OPS_HYGON_CAST_KERNEL_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "base/cast.h" +#include "native/cuda/hygon/caster.cuh" + +namespace infini::ops { +namespace hygon_cast_detail { +struct Layout { + size_t rank{}; + size_t shape[16]{}; + ptrdiff_t input_strides[16]{}; + ptrdiff_t output_strides[16]{}; + bool contiguous{}; +}; +template +__global__ void Convert(const InT* input, OutT* out, std::size_t first, + std::size_t count, Layout layout) { + const auto local = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (local >= count) return; + size_t input_offset = first + local, output_offset = input_offset; + if (!layout.contiguous) { + size_t remaining = input_offset; + input_offset = output_offset = 0; + for (size_t axis = layout.rank; axis > 0; --axis) { + const auto coordinate = remaining % layout.shape[axis - 1]; + remaining /= layout.shape[axis - 1]; + input_offset += coordinate * layout.input_strides[axis - 1]; + output_offset += coordinate * layout.output_strides[axis - 1]; + } + } + out[output_offset] = + Caster::template Cast(input[input_offset]); +} +} // namespace hygon_cast_detail + +template <> +class Operator : public Cast { + public: + static bool supports(const Tensor input, Tensor out) { + if (input.device().type() != Device::Type::kHygon || + input.device() != out.device() || input.shape() != out.shape() || + !SupportedDtype(input.dtype()) || !SupportedDtype(out.dtype()) || + input.ndim() > 16 || !NonOverlapping(out)) + return false; + std::size_t input_bytes = 0, out_bytes = 0; + if (!SpanBytes(input, &input_bytes) || !SpanBytes(out, &out_bytes) || + !ValidAddress(input, input_bytes) || !ValidAddress(out, out_bytes)) + return false; + int current = -1; + if (cudaGetDevice(¤t) != cudaSuccess || + current != input.device().index()) + return false; + const auto ip = reinterpret_cast(input.data()); + const auto op = reinterpret_cast(out.data()); + const bool overlap = input_bytes && out_bytes && ip < op + out_bytes && + op < ip + input_bytes; + return !overlap || SameView(input, out); + } + + Operator(const Tensor input, Tensor out) + : Cast{Checked(input, out), out}, device_{input.device()} { + layout_.rank = input.ndim(); + layout_.contiguous = input.IsContiguous() && out.IsContiguous(); + for (size_t axis = 0; axis < input.ndim(); ++axis) { + layout_.shape[axis] = input.size(axis); + layout_.input_strides[axis] = input.stride(axis); + layout_.output_strides[axis] = out.stride(axis); + } + } + + void operator()(const Tensor input, Tensor out) const override { + if (input.dtype() != input_dtype_ || out.dtype() != out_dtype_ || + input.shape() != input_shape_ || out.shape() != out_shape_ || + input.strides() != input_strides_ || out.strides() != out_strides_ || + input.device() != device_ || !supports(input, out)) + throw std::invalid_argument( + "HYGON Cast: call metadata or alias changed after Make"); + if (output_size_ == 0 || SameView(input, out)) return; + const auto stream = static_cast(stream_); + DispatchFunc< + List, + List>( + {static_cast(input_dtype_), static_cast(out_dtype_)}, + [&](auto list_tag) { + using InT = TypeMapType(list_tag)>; + using OutT = TypeMapType(list_tag)>; + constexpr std::size_t cap = 65535u * 256u; + for (std::size_t first = 0; first < output_size_;) { + const auto count = std::min(cap, output_size_ - first); + const auto blocks = static_cast((count + 255) / 256); + hygon_cast_detail::Convert<<>>( + static_cast(input.data()), + static_cast(out.data()), first, count, layout_); + const auto status = cudaGetLastError(); + if (status != cudaSuccess) + throw std::runtime_error( + std::string("HYGON Cast: launch failed: ") + + cudaGetErrorString(status)); + first += count; + } + }, + "HYGON Cast"); + } + + private: + static bool SupportedDtype(DataType type) { + return type == DataType::kFloat32 || type == DataType::kFloat16 || + type == DataType::kBFloat16; + } + + static std::size_t Width(DataType type) { + return type == DataType::kFloat32 ? 4u : 2u; + } + + static bool SpanBytes(const Tensor t, std::size_t* bytes) { + if (t.shape().size() != t.strides().size()) return false; + bool empty = false; + for (std::size_t axis = 0; axis < t.ndim(); ++axis) { + if (t.stride(axis) < 0) return false; + empty = empty || t.size(axis) == 0; + } + if (empty) { + *bytes = 0; + return true; + } + const auto limit = + static_cast(std::numeric_limits::max()) / + Width(t.dtype()); + std::size_t span = 1; + for (std::size_t axis = 0; axis < t.ndim(); ++axis) { + const auto size = t.size(axis); + const auto stride = static_cast(t.stride(axis)); + if (size > 1 && stride > (limit - span) / (size - 1)) return false; + span += (size - 1) * stride; + } + *bytes = span * Width(t.dtype()); + return true; + } + + static bool NonOverlapping(const Tensor tensor) { + std::vector> axes; + for (size_t axis = 0; axis < tensor.ndim(); ++axis) { + if (tensor.stride(axis) < 0) return false; + if (tensor.size(axis) > 1) + axes.emplace_back(tensor.stride(axis), tensor.size(axis)); + } + std::sort(axes.begin(), axes.end()); + size_t extent = 1; + for (const auto& [stride, size] : axes) { + if (stride < extent || + (size - 1) > (std::numeric_limits::max() - extent) / stride) + return false; + extent += (size - 1) * stride; + } + return true; + } + + static bool ValidAddress(const Tensor t, std::size_t bytes) { + if (bytes == 0) return true; + const auto start = reinterpret_cast(t.data()); + return start && start % Width(t.dtype()) == 0 && + start <= std::numeric_limits::max() - bytes; + } + + static bool SameView(const Tensor input, const Tensor out) { + return input.dtype() == out.dtype() && input.data() == out.data() && + input.shape() == out.shape() && input.strides() == out.strides(); + } + + static Tensor Checked(const Tensor input, const Tensor out) { + if (!supports(input, out)) + throw std::invalid_argument( + "HYGON Cast: unsupported dtype/device/layout or alias"); + return input; + } + + Device device_; + hygon_cast_detail::Layout layout_; +}; +} // namespace infini::ops +#endif diff --git a/src/native/cuda/hygon/ops/causal_conv1d/kernel.h b/src/native/cuda/hygon/ops/causal_conv1d/kernel.h new file mode 100644 index 000000000..254d751e3 --- /dev/null +++ b/src/native/cuda/hygon/ops/causal_conv1d/kernel.h @@ -0,0 +1,202 @@ +#pragma once +#include +#include +#include + +#include + +#include "base/causal_conv1d.h" +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" + +namespace infini::ops { +namespace causal_conv1d_detail { +struct Layout { + ptrdiff_t strides[9][3]{}; + size_t tokens, channels, pool; + bool cu_i64, initial_i64, final_i64; +}; +__device__ inline int64_t index(const void* p, bool i64, int i, int fallback) { + return p ? (i64 ? static_cast(p)[i] + : static_cast(static_cast(p)[i])) + : fallback; +} +template +__device__ inline float history(const T* state, const T* qkv, int64_t pos, + int64_t begin, int batch, int channel, + ptrdiff_t base, const Layout& p) { + if (pos < 3) return static_cast(state[base + pos * p.strides[2][2]]); + const auto off = static_cast(batch) * p.strides[0][0] + + static_cast(begin + pos - 3) * p.strides[0][1] + + static_cast(channel) * p.strides[0][2]; + return static_cast(qkv[off]); +} +// Port of baseline InfiniCore-0.2.9 causal_conv1d_k4_kernel. One thread per +// request/channel reads the entire sequence before committing its final state. +template +__global__ void kernel(const T* qkv, const T* weight, T* state, const T* bias, + const void* cu, const void* initial, + const void* final_indices, T* out, T* final_state, + Layout p) { + const int request = blockIdx.x, channel = blockIdx.y; + if (static_cast(channel) >= p.channels || threadIdx.x != 0) return; + int64_t begin = 0, end = static_cast(p.tokens); + int batch = request; + if (cu) { + begin = index(cu, p.cu_i64, request, 0); + end = index(cu, p.cu_i64, request + 1, 0); + batch = 0; + // Preserve baseline device-value behavior: malformed range leaves the + // corresponding output/state untouched. This is not host validation. + if (begin < 0 || end < begin || end > static_cast(p.tokens)) + return; + } + const int64_t length = end - begin; + const int64_t read = index(initial, p.initial_i64, request, request); + const int64_t write = index(final_indices, p.final_i64, request, request); + if (read < 0 || write < 0 || read >= static_cast(p.pool) || + (final_indices && write >= static_cast(p.pool))) + return; + const auto base = static_cast(read) * p.strides[2][0] + + static_cast(channel) * p.strides[2][1]; + const auto wb = static_cast(channel) * p.strides[1][0]; + for (int64_t t = 0; t < length; ++t) { + float acc = static_cast(weight[wb]) * + history(state, qkv, t, begin, batch, channel, base, p) + + static_cast(weight[wb + p.strides[1][2]]) * + history(state, qkv, t + 1, begin, batch, channel, base, p) + + static_cast(weight[wb + 2 * p.strides[1][2]]) * + history(state, qkv, t + 2, begin, batch, channel, base, p) + + static_cast(weight[wb + 3 * p.strides[1][2]]) * + history(state, qkv, t + 3, begin, batch, channel, base, p); + if (bias) + acc += static_cast( + bias[static_cast(channel) * p.strides[3][0]]); + const auto off = static_cast(batch) * p.strides[7][0] + + static_cast(begin + t) * p.strides[7][1] + + static_cast(channel) * p.strides[7][2]; + out[off] = static_cast(acc); + } + T* target = final_indices ? state : final_state; + const int stride_index = final_indices ? 2 : 8; + const auto target_base = + static_cast(final_indices ? write : request) * + p.strides[stride_index][0] + + static_cast(channel) * p.strides[stride_index][1]; + // Same ordered commit as baseline. Exact initial/final alias is valid for + // short sequences because copied history source positions lie ahead of + // writes. + target[target_base] = static_cast( + history(state, qkv, length, begin, batch, channel, base, p)); + target[target_base + p.strides[stride_index][2]] = static_cast( + history(state, qkv, length + 1, begin, batch, channel, base, p)); + target[target_base + 2 * p.strides[stride_index][2]] = static_cast( + history(state, qkv, length + 2, begin, batch, channel, base, p)); +} +} // namespace causal_conv1d_detail + +template <> +class Operator : public CausalConv1d { + public: + Operator(const Tensor qkv, const Tensor weight, Tensor conv_state, + const std::optional bias, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_conv_state) + : CausalConv1d(qkv, weight, conv_state, bias, cu_seqlens, + initial_state_indices, final_state_indices, out, + final_conv_state) { + if (device_.type() != Device::Type::kHygon) + throw std::invalid_argument("causal conv requires HYGON"); + if (weight.shape()[2] != 4 || conv_state.shape()[2] != 3) + throw std::invalid_argument( + "HYGON causal conv kernel requires K4 and state_len3"); + if (tokens_ > static_cast(std::numeric_limits::max() - 3)) + throw std::overflow_error( + "HYGON causal conv history index exceeds int64"); + check_device(); + // Baseline uses int request/channel variables and CUDA's grid.y dimension. + if (requests_ >= static_cast(INT_MAX) || channels_ > 65535) + throw std::invalid_argument( + "causal conv request/channel launch extent unsupported"); + p_.tokens = tokens_; + p_.channels = channels_; + p_.pool = pool_; + for (size_t i = 0; i < metadata_.size(); ++i) + if (metadata_[i]) + for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) + p_.strides[i][j] = metadata_[i]->strides()[j]; + p_.cu_i64 = metadata_[4] && metadata_[4]->dtype() == DataType::kInt64; + p_.initial_i64 = metadata_[5] && metadata_[5]->dtype() == DataType::kInt64; + p_.final_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; + } + // Device capture/state-pool lifetime is not supported for this stateful path. + // Zero device allocations does not alone establish capture eligibility. + void operator()(const Tensor qkv, const Tensor weight, Tensor conv_state, + const std::optional bias, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_conv_state) const override { + const Views views{qkv, + weight, + conv_state, + bias, + cu_seqlens, + initial_state_indices, + final_state_indices, + out, + final_conv_state}; + for (size_t i = 0; i < views.size(); ++i) { + const auto& t = views[i]; + if (t.has_value() != metadata_[i].has_value() || + (t && (!std::equal_to{}(*t, *metadata_[i]) || + (t->numel() && !t->data())))) + throw std::invalid_argument( + "causal conv metadata changed or nonempty storage is null"); + } + check_device(); + if (!requests_ || !channels_) return; + switch (dtype_) { + case DataType::kFloat32: + launch(views); + break; + case DataType::kFloat16: + launch(views); + break; + case DataType::kBFloat16: + launch<__nv_bfloat16>(views); + break; + default: + throw std::invalid_argument("causal conv dtype unsupported"); + } + if (cudaPeekAtLastError() != cudaSuccess) + throw std::runtime_error("causal conv CUDA launch failed"); + } + + private: + causal_conv1d_detail::Layout p_{}; + void check_device() const { + int current = -1; + if (cudaGetDevice(¤t) != cudaSuccess) + throw std::runtime_error("causal conv device query failed"); + if (current != device_.index()) + throw std::invalid_argument("causal conv owner device mismatch"); + } + template + void launch(const Views& v) const { + auto pointer = [&](size_t i) -> void* { + return v[i] ? const_cast(v[i]->data()) : nullptr; + }; + const dim3 grid(static_cast(requests_), + static_cast(channels_), 1); + const auto stream = static_cast(stream_ ? stream_ : nullptr); + causal_conv1d_detail::kernel<<>>( + static_cast(pointer(0)), static_cast(pointer(1)), + static_cast(pointer(2)), static_cast(pointer(3)), + pointer(4), pointer(5), pointer(6), static_cast(pointer(7)), + static_cast(pointer(8)), p_); + } +}; +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/clover_block_conv1d/kernel.h b/src/native/cuda/hygon/ops/clover_block_conv1d/kernel.h new file mode 100644 index 000000000..d5c2ebe47 --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_block_conv1d/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_HYGON_CLOVER_BLOCK_CONV1D_KERNEL_H_ +#define INFINI_OPS_HYGON_CLOVER_BLOCK_CONV1D_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_block_conv1d/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverBlockConv1d> { + public: + using CudaCloverBlockConv1d< + Runtime>::CudaCloverBlockConv1d; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_coreset/kernel.h b/src/native/cuda/hygon/ops/clover_coreset/kernel.h new file mode 100644 index 000000000..eb5fae501 --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_coreset/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_HYGON_CLOVER_CORESET_KERNEL_H_ +#define INFINI_OPS_HYGON_CLOVER_CORESET_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_coreset/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverCoreset> { + public: + using CudaCloverCoreset>::CudaCloverCoreset; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_live_mask/kernel.h b/src/native/cuda/hygon/ops/clover_live_mask/kernel.h new file mode 100644 index 000000000..a71fc8f2a --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_live_mask/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_HYGON_CLOVER_live_mask_KERNEL_H_ +#define INFINI_OPS_HYGON_CLOVER_live_mask_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_live_mask/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverLiveMask> { + public: + using CudaCloverLiveMask>::CudaCloverLiveMask; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_local_memory/kernel.h b/src/native/cuda/hygon/ops/clover_local_memory/kernel.h new file mode 100644 index 000000000..f54bdc73c --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_local_memory/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_Hygon_CLOVER_LOCAL_MEMORY_KERNEL_H_ +#define INFINI_OPS_Hygon_CLOVER_LOCAL_MEMORY_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_local_memory/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverLocalMemory> { + public: + using CudaCloverLocalMemory< + Runtime>::CudaCloverLocalMemory; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_mask_routes/kernel.h b/src/native/cuda/hygon/ops/clover_mask_routes/kernel.h new file mode 100644 index 000000000..de76b2af3 --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_mask_routes/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_HYGON_CLOVER_mask_routes_KERNEL_H_ +#define INFINI_OPS_HYGON_CLOVER_mask_routes_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_mask_routes/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverMaskRoutes> { + public: + using CudaCloverMaskRoutes>::CudaCloverMaskRoutes; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_merge_moe/kernel.h b/src/native/cuda/hygon/ops/clover_merge_moe/kernel.h new file mode 100644 index 000000000..d9bd48665 --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_merge_moe/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_HYGON_CLOVER_merge_moe_KERNEL_H_ +#define INFINI_OPS_HYGON_CLOVER_merge_moe_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_merge_moe/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverMergeMoe> { + public: + using CudaCloverMergeMoe>::CudaCloverMergeMoe; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/clover_recurrent_gdn/kernel.h b/src/native/cuda/hygon/ops/clover_recurrent_gdn/kernel.h new file mode 100644 index 000000000..4d1932759 --- /dev/null +++ b/src/native/cuda/hygon/ops/clover_recurrent_gdn/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_Hygon_CLOVER_RECURRENT_GDN_KERNEL_H_ +#define INFINI_OPS_Hygon_CLOVER_RECURRENT_GDN_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/clover_recurrent_gdn/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverRecurrentGdn> { + public: + using CudaCloverRecurrentGdn< + Runtime>::CudaCloverRecurrentGdn; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/hygon/ops/fill/kernel.h b/src/native/cuda/hygon/ops/fill/kernel.h new file mode 100644 index 000000000..09a9ec9c6 --- /dev/null +++ b/src/native/cuda/hygon/ops/fill/kernel.h @@ -0,0 +1,16 @@ +#pragma once + +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/fill/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaFill> { + public: + using CudaFill>::CudaFill; +}; + +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/fused_gated_delta_net_gating/kernel.h b/src/native/cuda/hygon/ops/fused_gated_delta_net_gating/kernel.h new file mode 100644 index 000000000..8b35c724d --- /dev/null +++ b/src/native/cuda/hygon/ops/fused_gated_delta_net_gating/kernel.h @@ -0,0 +1,120 @@ +#pragma once +#include +#include +#include + +#include +#include +#include + +#include "base/fused_gated_delta_net_gating.h" +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" + +namespace infini::ops { +namespace gating_detail { +struct Layout { + ptrdiff_t strides[6][3]{}; + size_t total, seq, hidden; + float beta, threshold; +}; +template +__global__ void kernel(const T* A, const T* a, const T* b, const T* dt, + float* g, float* betao, Layout p) { + const size_t linear = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (linear >= p.total) return; + const size_t h = linear % p.hidden, s = (linear / p.hidden) % p.seq; + const size_t batch = linear / p.hidden / p.seq; + ptrdiff_t offsets[6]; + for (int i = 0; i < 6; ++i) + offsets[i] = static_cast(batch) * p.strides[i][0] + + static_cast(s) * p.strides[i][1] + + static_cast(h) * p.strides[i][2]; + const float x = static_cast(a[offsets[1]]) + + static_cast(dt[h * p.strides[3][0]]); + const float bx = p.beta * x; + const float softplus = bx <= p.threshold ? log1pf(expf(bx)) / p.beta : x; + g[offsets[4]] = -expf(static_cast(A[h * p.strides[0][0]])) * softplus; + const float bv = static_cast(b[offsets[2]]); + const float z = expf(bv >= 0 ? -bv : bv); + betao[offsets[5]] = bv >= 0 ? 1.0f / (1.0f + z) : z / (1.0f + z); +} +} // namespace gating_detail + +template <> +class Operator + : public FusedGatedDeltaNetGating { + public: + Operator(Tensor A, Tensor a, Tensor b, Tensor dt, float beta, float threshold, + Tensor g, Tensor betao) + : FusedGatedDeltaNetGating(A, a, b, dt, beta, threshold, g, betao) { + if (device_.type() != Device::Type::kHygon) + throw std::invalid_argument("gating requires HYGON device"); + check_device(); + if (total_ / 256 + (total_ % 256 != 0) > + std::numeric_limits::max()) + throw std::overflow_error("gating grid exceeds CUDA limit"); + p_.total = total_; + p_.seq = seq_; + p_.hidden = hidden_; + p_.beta = beta; + p_.threshold = threshold; + for (size_t i = 0; i < 6; ++i) + for (size_t j = 0; j < metadata_[i].strides().size(); ++j) + p_.strides[i][j] = metadata_[i].strides()[j]; + } + // All metadata is a kernel value argument. There is no allocation, host copy, + // synchronization or data-dependent host work in operator(); stream launch + // is capturable. Make owns no device resources and requires zero workspace. + bool capture_safe() const override { return true; } + void operator()(Tensor A, Tensor a, Tensor b, Tensor dt, float beta, + float threshold, Tensor g, Tensor betao) const override { + check_device(); + std::array tensors{A, a, b, dt, g, betao}; + for (size_t i = 0; i < 6; ++i) + if (!std::equal_to{}(tensors[i], metadata_[i])) + throw std::invalid_argument("gating metadata changed after Make"); + if (std::memcmp(&beta, &beta_, sizeof(float)) || + std::memcmp(&threshold, &threshold_, sizeof(float))) + throw std::invalid_argument("gating attributes changed after Make"); + if (!total_) return; + const dim3 grid( + static_cast(total_ / 256 + (total_ % 256 != 0))); + const auto stream = static_cast(stream_ ? stream_ : nullptr); + switch (metadata_[1].dtype()) { + case DataType::kFloat32: + launch(A, a, b, dt, g, betao, grid, stream); + break; + case DataType::kFloat16: + launch(A, a, b, dt, g, betao, grid, stream); + break; + case DataType::kBFloat16: + launch<__nv_bfloat16>(A, a, b, dt, g, betao, grid, stream); + break; + default: + throw std::invalid_argument("gating dtype unsupported"); + } + if (cudaPeekAtLastError() != cudaSuccess) + throw std::runtime_error("gating CUDA launch failed"); + } + + private: + gating_detail::Layout p_{}; + void check_device() const { + int current = -1; + if (cudaGetDevice(¤t) != cudaSuccess) + throw std::runtime_error("gating device query failed"); + if (current != device_.index()) + throw std::invalid_argument("gating owner device mismatch"); + } + template + void launch(Tensor A, Tensor a, Tensor b, Tensor dt, Tensor g, Tensor betao, + dim3 grid, cudaStream_t stream) const { + gating_detail::kernel<<>>( + static_cast(A.data()), static_cast(a.data()), + static_cast(b.data()), static_cast(dt.data()), + static_cast(g.data()), static_cast(betao.data()), p_); + } +}; +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cu b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cu new file mode 100644 index 000000000..76f5938a9 --- /dev/null +++ b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cu @@ -0,0 +1,81 @@ +#include + +#include +#include +#include + +#include "native/cuda/hygon/ops/moe_align_block_size/kernel.cuh" +#include "native/cuda/hygon/ops/moe_align_block_size/kernel.h" + +namespace infini::ops { +namespace { + +class DeviceGuard { + public: + explicit DeviceGuard(int device_index) { + auto status = cudaGetDevice(&previous_device_); + assert(status == cudaSuccess && + "`MoeAlignBlockSize` failed to query the current CUDA device"); + + if (previous_device_ != device_index) { + status = cudaSetDevice(device_index); + assert(status == cudaSuccess && + "`MoeAlignBlockSize` failed to select the input CUDA device"); + restore_ = true; + } + } + + ~DeviceGuard() { + if (restore_) { + const auto status = cudaSetDevice(previous_device_); + assert(status == cudaSuccess && + "`MoeAlignBlockSize` failed to restore the CUDA device"); + } + } + + private: + int previous_device_{0}; + + bool restore_{false}; +}; + +} // namespace + +Operator::Operator( + const Tensor topk_ids, const std::optional expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, Tensor num_tokens_post_pad) + : MoeAlignBlockSize{topk_ids, expert_map, num_experts, + block_size, sorted_token_ids, experts_ids, + num_tokens_post_pad}, + device_index_{topk_ids.device().index()} {} + +void Operator::Run( + const Tensor topk_ids, const std::optional maybe_expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, + Tensor num_tokens_post_pad) const { + DeviceGuard device_guard{device_index_}; + + constexpr int32_t kThreads = 256; + const auto shared_memory_size = + static_cast(num_experts_) * sizeof(int32_t); + moe_align_block_size_detail::MoeAlignBlockSizeKernel<<< + 1, kThreads, shared_memory_size, static_cast(stream_)>>>( + reinterpret_cast(topk_ids.data()), + maybe_expert_map + ? reinterpret_cast(maybe_expert_map->data()) + : nullptr, + reinterpret_cast(sorted_token_ids.data()), + reinterpret_cast(experts_ids.data()), + reinterpret_cast(num_tokens_post_pad.data()), numel_, + static_cast(num_experts_), static_cast(block_size_), + sorted_token_ids_size_, experts_ids_size_, + maybe_expert_map ? static_cast(maybe_expert_map->numel()) : 0); + + const auto status = cudaGetLastError(); + assert(status == cudaSuccess && + "`MoeAlignBlockSize` CUDA kernel launch failed"); +} + +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cuh b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cuh new file mode 100644 index 000000000..2055342e1 --- /dev/null +++ b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.cuh @@ -0,0 +1,86 @@ +#ifndef INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_CUH_ +#define INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_CUH_ + +#include +#include + +namespace infini::ops { +namespace moe_align_block_size_detail { + +__device__ __forceinline__ int32_t MapExpertId(int32_t expert_id, + const int32_t* expert_map, + int32_t num_experts, + int32_t map_size) { + if (expert_id < 0 || expert_id >= (expert_map ? map_size : num_experts)) { + return -1; + } + if (expert_map != nullptr) { + expert_id = expert_map[expert_id]; + } + + return expert_id >= 0 && expert_id < num_experts ? expert_id : -1; +} + +__global__ void MoeAlignBlockSizeKernel( + const int32_t* __restrict__ topk_ids, + const int32_t* __restrict__ expert_map, + int32_t* __restrict__ sorted_token_ids, int32_t* __restrict__ experts_ids, + int32_t* __restrict__ num_tokens_post_pad, size_t numel, + int32_t num_experts, int32_t block_size, size_t sorted_token_ids_size, + size_t experts_ids_size, int32_t map_size) { + extern __shared__ int32_t expert_offsets[]; + + for (size_t i = threadIdx.x; i < sorted_token_ids_size; i += blockDim.x) { + sorted_token_ids[i] = static_cast(numel); + } + for (size_t i = threadIdx.x; i < experts_ids_size; i += blockDim.x) { + experts_ids[i] = -1; + } + for (int32_t i = threadIdx.x; i < num_experts; i += blockDim.x) { + expert_offsets[i] = 0; + } + if (threadIdx.x == 0) { + *num_tokens_post_pad = 0; + } + __syncthreads(); + + for (size_t i = threadIdx.x; i < numel; i += blockDim.x) { + const auto expert_id = + MapExpertId(topk_ids[i], expert_map, num_experts, map_size); + if (expert_id >= 0) { + atomicAdd(&expert_offsets[expert_id], 1); + } + } + __syncthreads(); + + if (threadIdx.x == 0) { + int32_t token_offset = 0; + for (int32_t expert_id = 0; expert_id < num_experts; ++expert_id) { + const int32_t count = expert_offsets[expert_id]; + expert_offsets[expert_id] = token_offset; + const int32_t padded_count = + (count + block_size - 1) / block_size * block_size; + for (int32_t i = 0; i < padded_count; i += block_size) { + experts_ids[(token_offset + i) / block_size] = expert_id; + } + token_offset += padded_count; + } + *num_tokens_post_pad = token_offset; + } + __syncthreads(); + + if (threadIdx.x == 0) { + for (size_t i = 0; i < numel; ++i) { + const auto expert_id = + MapExpertId(topk_ids[i], expert_map, num_experts, map_size); + if (expert_id >= 0) { + sorted_token_ids[expert_offsets[expert_id]++] = static_cast(i); + } + } + } +} + +} // namespace moe_align_block_size_detail +} // namespace infini::ops + +#endif // INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_CUH_ diff --git a/src/native/cuda/hygon/ops/moe_align_block_size/kernel.h b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.h new file mode 100644 index 000000000..0984b5205 --- /dev/null +++ b/src/native/cuda/hygon/ops/moe_align_block_size/kernel.h @@ -0,0 +1,35 @@ +#ifndef INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_H_ +#define INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_H_ + +#include + +#include "base/moe_align_block_size.h" + +namespace infini::ops { + +template <> +class Operator + : public MoeAlignBlockSize { + public: + Operator(const Tensor topk_ids, const std::optional expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, + Tensor num_tokens_post_pad); + + using MoeAlignBlockSize::operator(); + + bool capture_safe() const override { return true; } + + protected: + void Run(const Tensor topk_ids, const std::optional maybe_expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, + Tensor num_tokens_post_pad) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_HYGON_MOE_ALIGN_BLOCK_SIZE_KERNEL_H_ diff --git a/src/native/cuda/hygon/ops/mrope/kernel.h b/src/native/cuda/hygon/ops/mrope/kernel.h new file mode 100644 index 000000000..7808c0627 --- /dev/null +++ b/src/native/cuda/hygon/ops/mrope/kernel.h @@ -0,0 +1,12 @@ +#pragma once +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/mrope/kernel.h" +namespace infini::ops { +template <> +class Operator + : public CudaMrope> { + public: + using CudaMrope>::CudaMrope; +}; +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/mul/kernel.h b/src/native/cuda/hygon/ops/mul/kernel.h new file mode 100644 index 000000000..b949617a7 --- /dev/null +++ b/src/native/cuda/hygon/ops/mul/kernel.h @@ -0,0 +1,16 @@ +#pragma once + +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/mul/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaMul> { + public: + using CudaMul>::CudaMul; +}; + +} // namespace infini::ops diff --git a/src/native/cuda/hygon/ops/separate_table_rope/kernel.h b/src/native/cuda/hygon/ops/separate_table_rope/kernel.h new file mode 100644 index 000000000..d4aa1316e --- /dev/null +++ b/src/native/cuda/hygon/ops/separate_table_rope/kernel.h @@ -0,0 +1,27 @@ +#ifndef INFINI_OPS_HYGON_SEPARATE_TABLE_ROPE_KERNEL_H_ +#define INFINI_OPS_HYGON_SEPARATE_TABLE_ROPE_KERNEL_H_ +#include "native/cuda/hygon/caster.cuh" +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/separate_table_rope/kernel.h" +namespace infini::ops { +template <> +class Operator + : public CudaSeparateTableRope> { + public: + using CudaSeparateTableRope< + Runtime>::CudaSeparateTableRope; + void operator()(const Tensor x, const Tensor p, const Tensor s, + const Tensor c, bool neox, Tensor y) const override { + CudaSeparateTableRope>::operator()(x, p, s, c, + neox, y); + if (x.numel() == 0) return; + // Same native HYGON launch-error API used by the installed Argmax source. + // Peek reports immediate launch failure; it does not synchronize or claim + // that device data access / position bounds are valid. + const auto status = cudaPeekAtLastError(); + if (status != cudaSuccess) + throw std::runtime_error(cudaGetErrorString(status)); + } +}; +} // namespace infini::ops +#endif diff --git a/src/native/cuda/hygon/ops/tril/kernel.h b/src/native/cuda/hygon/ops/tril/kernel.h new file mode 100644 index 000000000..532ccd490 --- /dev/null +++ b/src/native/cuda/hygon/ops/tril/kernel.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_HYGON_TRIL_KERNEL_H_ +#define INFINI_OPS_HYGON_TRIL_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/triangular/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public triangular_detail::CudaTriangular, + Tril, false> { + public: + using triangular_detail::CudaTriangular, Tril, + false>::CudaTriangular; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_HYGON_TRIL_KERNEL_H_ diff --git a/src/native/cuda/hygon/ops/triu/kernel.h b/src/native/cuda/hygon/ops/triu/kernel.h new file mode 100644 index 000000000..aa237ba7a --- /dev/null +++ b/src/native/cuda/hygon/ops/triu/kernel.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_HYGON_TRIU_KERNEL_H_ +#define INFINI_OPS_HYGON_TRIU_KERNEL_H_ + +#include "native/cuda/hygon/runtime_.h" +#include "native/cuda/ops/triangular/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public triangular_detail::CudaTriangular, + Triu, true> { + public: + using triangular_detail::CudaTriangular, Triu, + true>::CudaTriangular; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_HYGON_TRIU_KERNEL_H_ diff --git a/src/native/cuda/nvidia/ops/clover_block_conv1d/kernel.h b/src/native/cuda/nvidia/ops/clover_block_conv1d/kernel.h new file mode 100644 index 000000000..344114681 --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_block_conv1d/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_Nvidia_CLOVER_BLOCK_CONV1D_KERNEL_H_ +#define INFINI_OPS_Nvidia_CLOVER_BLOCK_CONV1D_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_block_conv1d/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverBlockConv1d> { + public: + using CudaCloverBlockConv1d< + Runtime>::CudaCloverBlockConv1d; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_coreset/kernel.h b/src/native/cuda/nvidia/ops/clover_coreset/kernel.h new file mode 100644 index 000000000..cccbfb07c --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_coreset/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_NVIDIA_CLOVER_CORESET_KERNEL_H_ +#define INFINI_OPS_NVIDIA_CLOVER_CORESET_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_coreset/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverCoreset> { + public: + using CudaCloverCoreset>::CudaCloverCoreset; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_live_mask/kernel.h b/src/native/cuda/nvidia/ops/clover_live_mask/kernel.h new file mode 100644 index 000000000..b4ae3dbae --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_live_mask/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_NVIDIA_CLOVER_live_mask_KERNEL_H_ +#define INFINI_OPS_NVIDIA_CLOVER_live_mask_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_live_mask/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverLiveMask> { + public: + using CudaCloverLiveMask>::CudaCloverLiveMask; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_local_memory/kernel.h b/src/native/cuda/nvidia/ops/clover_local_memory/kernel.h new file mode 100644 index 000000000..2dd1f59ba --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_local_memory/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_Nvidia_CLOVER_LOCAL_MEMORY_KERNEL_H_ +#define INFINI_OPS_Nvidia_CLOVER_LOCAL_MEMORY_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_local_memory/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverLocalMemory> { + public: + using CudaCloverLocalMemory< + Runtime>::CudaCloverLocalMemory; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_mask_routes/kernel.h b/src/native/cuda/nvidia/ops/clover_mask_routes/kernel.h new file mode 100644 index 000000000..4d5986001 --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_mask_routes/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_NVIDIA_CLOVER_mask_routes_KERNEL_H_ +#define INFINI_OPS_NVIDIA_CLOVER_mask_routes_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_mask_routes/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverMaskRoutes> { + public: + using CudaCloverMaskRoutes>::CudaCloverMaskRoutes; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_merge_moe/kernel.h b/src/native/cuda/nvidia/ops/clover_merge_moe/kernel.h new file mode 100644 index 000000000..36ff1b86b --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_merge_moe/kernel.h @@ -0,0 +1,18 @@ +#ifndef INFINI_OPS_NVIDIA_CLOVER_merge_moe_KERNEL_H_ +#define INFINI_OPS_NVIDIA_CLOVER_merge_moe_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_merge_moe/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverMergeMoe> { + public: + using CudaCloverMergeMoe>::CudaCloverMergeMoe; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/clover_recurrent_gdn/kernel.h b/src/native/cuda/nvidia/ops/clover_recurrent_gdn/kernel.h new file mode 100644 index 000000000..5684a29bd --- /dev/null +++ b/src/native/cuda/nvidia/ops/clover_recurrent_gdn/kernel.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_Nvidia_CLOVER_RECURRENT_GDN_KERNEL_H_ +#define INFINI_OPS_Nvidia_CLOVER_RECURRENT_GDN_KERNEL_H_ + +#include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/clover_recurrent_gdn/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaCloverRecurrentGdn> { + public: + using CudaCloverRecurrentGdn< + Runtime>::CudaCloverRecurrentGdn; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/ops/clover_block_conv1d/kernel.cuh b/src/native/cuda/ops/clover_block_conv1d/kernel.cuh new file mode 100644 index 000000000..2183637a9 --- /dev/null +++ b/src/native/cuda/ops/clover_block_conv1d/kernel.cuh @@ -0,0 +1,100 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_CUH_ + +#include + +#include "native/cuda/caster.cuh" + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverBlockConv1d, mirroring the Triton +// reference (SGLang lineage). Per (request, token, channel): +// left part : sum over 4 taps of input/state history . weight[c,0,tap] +// right part : sum over 3 taps of future-in-block input . right_weight[c,0,d-1] +// out = silu(left + right), zeroed for inactive rows. +// The state pool row for a request holds its last 3 pre-block values +// (clean prefix); only rows whose `valid` flag is set contribute. +// Grid: x covers channels in BLOCK_C tiles, y iterates (request, tile). +template +__global__ void CloverBlockConv1dKernel( + const Data* __restrict__ input, const Data* __restrict__ state, + const Index* __restrict__ indices, const Data* __restrict__ weight, + const Data* __restrict__ right_weight, const int8_t* __restrict__ valid, + Data* __restrict__ out, int64_t channels, int block_size, int num_slots, + int64_t stride_t, int64_t stride_c, int64_t state_slot_stride, + int64_t state_c_stride, int64_t state_w_stride, int64_t weight_c_stride, + int64_t weight_w_stride, int64_t right_c_stride, int64_t right_w_stride, + int64_t out_t_stride, int64_t out_c_stride, int token_tile_t, + int num_token_tiles) { + const int64_t ch = static_cast(blockIdx.x) * blockDim.x + + static_cast(threadIdx.x); + if (ch >= channels) return; + + const int token_program = static_cast(blockIdx.y); + const int request = token_program / num_token_tiles; + const int tile = token_program % num_token_tiles; + + const auto cache_index = indices[request]; + const bool request_active = cache_index >= 0 && cache_index < num_slots; + const auto safe_slot = cache_index >= 0 ? cache_index : 0; + + const int token_base = tile * token_tile_t; + for (int t_in_tile = 0; t_in_tile < token_tile_t; ++t_in_tile) { + const int t = token_base + t_in_tile; + if (t >= block_size) break; + const int64_t token = static_cast(request) * block_size + t; + const bool row_active = + request_active && (valid[token] != 0); + if (!row_active) { + out[token * out_t_stride + ch * out_c_stride] = Data(0); + continue; + } + + float acc = 0.0f; + // Left/history taps (WIDTH = 4): t-3 .. t come from the block input; + // earlier positions come from the clean state row (3 slots). + for (int tap = 0; tap < 4; ++tap) { + const int rel = t + tap - 3; + float value; + if (rel >= 0) { + const int64_t src = static_cast(request) * block_size + rel; + const bool src_active = valid[src] != 0; + value = src_active + ? Caster::template Cast( + input[src * stride_t + ch * stride_c]) + : 0.0f; + } else { + const int64_t w = rel + 3; // history position 0..2 + value = Caster::template Cast( + state[static_cast(safe_slot) * state_slot_stride + + ch * state_c_stride + w * state_w_stride]); + } + acc += value * Caster::template Cast( + weight[ch * weight_c_stride + tap * weight_w_stride]); + } + // Right/symmetric taps: future tokens inside the same block. + for (int distance = 1; distance < 4; ++distance) { + const int right = t + distance; + if (right < block_size) { + const int64_t src = static_cast(request) * block_size + right; + const bool src_active = valid[src] != 0; + const float value = + src_active + ? Caster::template Cast( + input[src * stride_t + ch * stride_c]) + : 0.0f; + acc += value * Caster::template Cast( + right_weight[ch * right_c_stride + + (distance - 1) * right_w_stride]); + } + } + const float sigmoid = 1.0f / (1.0f + expf(-acc)); + const float result = acc * sigmoid; + out[token * out_t_stride + ch * out_c_stride] = + Caster::template Cast(result); + } +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_block_conv1d/kernel.h b/src/native/cuda/ops/clover_block_conv1d/kernel.h new file mode 100644 index 000000000..32e36202c --- /dev/null +++ b/src/native/cuda/ops/clover_block_conv1d/kernel.h @@ -0,0 +1,73 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_H_ + +#include "base/clover_block_conv1d.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_block_conv1d/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverBlockConv1d : public CloverBlockConv1d { + public: + using CloverBlockConv1d::CloverBlockConv1d; + + void operator()(const Tensor input, const Tensor state, + const Tensor indices, const Tensor weight, + const Tensor right_weight, const Tensor valid, + size_t block_size, Tensor out) const override { + const auto tokens = input.size(0); + const auto channels = input.size(1); + const auto requests = indices.numel(); + if (tokens == 0 || channels == 0 || requests == 0) return; + + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr unsigned int kThreads = 128; + const unsigned int ch_blocks = + static_cast((channels + kThreads - 1) / kThreads); + const int tile_t = 4; + const int num_tiles = + static_cast((block_size + tile_t - 1) / tile_t); + const unsigned int programs = + static_cast(requests * num_tiles); + + DispatchFunc>( + {static_cast(input.dtype()), + static_cast(indices.dtype())}, + [&](auto list_tag) { + using Data = TypeMapType(list_tag)>; + using Index = TypeMapType(list_tag)>; + CloverBlockConv1dKernel + <<>>( + reinterpret_cast(input.data()), + reinterpret_cast(state.data()), + reinterpret_cast(indices.data()), + reinterpret_cast(weight.data()), + reinterpret_cast(right_weight.data()), + reinterpret_cast(valid.data()), + reinterpret_cast(out.data()), + static_cast(channels), + static_cast(block_size), + static_cast(state.size(0)), + static_cast(input.stride(0)), + static_cast(input.stride(1)), + static_cast(state.stride(0)), + static_cast(state.stride(1)), + static_cast(state.stride(2)), + static_cast(weight.stride(0)), + static_cast(weight.stride(2)), + static_cast(right_weight.stride(0)), + static_cast(right_weight.stride(2)), + static_cast(out.stride(0)), + static_cast(out.stride(1)), tile_t, num_tiles); + }, + "CudaCloverBlockConv1d::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_BLOCK_CONV1D_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_coreset/kernel.cuh b/src/native/cuda/ops/clover_coreset/kernel.cuh new file mode 100644 index 000000000..982e141f0 --- /dev/null +++ b/src/native/cuda/ops/clover_coreset/kernel.cuh @@ -0,0 +1,136 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_CUH_ + +#include +#include +#include + +namespace infini::ops { + +// Shared-layer SIMT kernels for CloverCoreset, mirroring the three Triton +// stages: (0) per block, accumulate routing mass per expert over the union +// of each row's top-8 experts; (1) per block, keep experts by descending +// mass until the cumulative NORMALIZED probability reaches top_p (mass is +// normalized by the block total, matching the Triton reference); (2) per +// token, apply the block's allow-list to its logits row. Layout [tokens,256]. + +constexpr int kCloverCoresetExperts = 256; + +template +__global__ void CloverCoresetMassKernel(const Data* __restrict__ logits, + float* __restrict__ mass, + int width, int64_t blocks) { + // One thread per expert (blockDim = 256), iterating the block's rows + // serially. Deterministic by construction: no atomics, fixed order. + const int64_t block = static_cast(blockIdx.x); + if (block >= blocks) return; + const int expert = static_cast(threadIdx.x); // 0..255 + + float total = 0.0f; + // A row counts this expert only if the expert is in that row's top-8 + // (ties resolved by ascending expert id, like the Triton tie_break_left). + for (int t = 0; t < width; ++t) { + const Data* row = + logits + (block * width + t) * kCloverCoresetExperts; + const float my_logit = static_cast(row[expert]); + // Rank of my expert in this row: count strictly-greater logits plus + // equal logits with a smaller id. + int stronger = 0; + float row_max = -INFINITY; + float denom = 0.0f; + for (int e = 0; e < kCloverCoresetExperts; ++e) { + const float v = static_cast(row[e]); + if (v > row_max) row_max = v; + } + for (int e = 0; e < kCloverCoresetExperts; ++e) { + const float v = static_cast(row[e]); + denom += expf(v - row_max); + if (v > my_logit || (v == my_logit && e < expert)) ++stronger; + } + if (stronger < 8) { + total += expf(my_logit - row_max) / denom; + } + } + mass[block * kCloverCoresetExperts + expert] = total; +} + +template +__global__ void CloverCoresetAllowKernel(const float* __restrict__ mass, + int8_t* __restrict__ allow, + int64_t blocks, float top_p) { + const int64_t block = static_cast(blockIdx.x); + if (block >= blocks) return; + const float* block_mass = mass + block * kCloverCoresetExperts; + + extern __shared__ float sorted[]; // [256] + __shared__ float total[1]; + __shared__ int allow_shared[kCloverCoresetExperts]; + for (int e = threadIdx.x; e < kCloverCoresetExperts; e += blockDim.x) + sorted[e] = block_mass[e]; + __syncthreads(); + if (threadIdx.x == 0) { + float sum = 0.0f; + for (int e = 0; e < kCloverCoresetExperts; ++e) sum += sorted[e]; + total[0] = sum > 1.0e-9f ? sum : 1.0e-9f; + } + for (int e = threadIdx.x; e < kCloverCoresetExperts; e += blockDim.x) + allow_shared[e] = 0; + __syncthreads(); + + // Serial selection by thread 0, matching the Triton reference: keep + // experts in descending-mass order while the cumulative probability + // (mass normalized by the block total) BEFORE each element stays below + // top_p — i.e., include the first element that reaches the threshold. + if (threadIdx.x == 0) { + const bool keep_positive_only = top_p >= 1.0f; + float kept = 0.0f; + for (int picked = 0; picked < kCloverCoresetExperts; ++picked) { + int best = -1; + float best_v = 0.0f; // ties keep expert id ascending, like Triton's key + for (int e = 0; e < kCloverCoresetExperts; ++e) { + if (sorted[e] > best_v) { + best_v = sorted[e]; + best = e; + } + } + if (best < 0) break; + sorted[best] = -1.0f; + if (keep_positive_only) { + allow_shared[best] = 1; // top_p >= 1: keep every positive-mass expert + } else { + allow_shared[best] = 1; + kept += best_v / total[0]; + if (kept >= top_p) break; + } + } + // The Triton reference ORs `keep` with (position == 0) in the + // sorted coordinate system, which keeps the highest-mass expert — + // already guaranteed by the first pick above. No extra action. + } + __syncthreads(); + + for (int e = threadIdx.x; e < kCloverCoresetExperts; e += blockDim.x) + allow[block * kCloverCoresetExperts + e] = + static_cast(allow_shared[e]); +} + +template +__global__ void CloverCoresetApplyKernel(const Data* __restrict__ logits, + const int8_t* __restrict__ allow, + Data* __restrict__ out, int width, + int64_t tokens) { + const int64_t token = static_cast(blockIdx.x); + if (token >= tokens) return; + const int64_t block = token / width; + const Data* row = logits + token * kCloverCoresetExperts; + Data* dst = out + token * kCloverCoresetExperts; + const int8_t* block_allow = allow + block * kCloverCoresetExperts; + for (int e = threadIdx.x; e < kCloverCoresetExperts; e += blockDim.x) + dst[e] = block_allow[e] != 0 + ? row[e] + : static_cast(-INFINITY); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_coreset/kernel.h b/src/native/cuda/ops/clover_coreset/kernel.h new file mode 100644 index 000000000..f8e705635 --- /dev/null +++ b/src/native/cuda/ops/clover_coreset/kernel.h @@ -0,0 +1,71 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_H_ + +#include "base/clover_coreset.h" +#include "data_type.h" +#include "dispatcher.h" +#include "common/generic_utils.h" +#include "native/cuda/ops/clover_coreset/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverCoreset : public CloverCoreset { + public: + // Workspace: mass [blocks,256] fp32 + allow [blocks,256] int8, laid out + // exactly like the Triton provider declares. + CudaCloverCoreset(const Tensor logits, size_t block_size, double top_p, + Tensor out) + : CloverCoreset(logits, block_size, top_p, out), + blocks_(logits.size(0) / block_size) {} + + size_t workspace_size_in_bytes() const override { + return blocks_ * kCloverCoresetExperts * 5; + } + + void operator()(const Tensor logits, size_t block_size, double top_p, + Tensor out) const override { + if (blocks_ == 0) return; + if (!workspace_ || workspace_size_in_bytes_ < workspace_size_in_bytes()) + throw std::invalid_argument("CudaCloverCoreset requires its workspace."); + const auto tokens = logits.size(0); + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + + float* mass = reinterpret_cast(workspace_); + int8_t* allow = + reinterpret_cast(workspace_) + blocks_ * kCloverCoresetExperts * 4; + + DispatchFunc, ReducedFloatTypes>>( + {static_cast(logits.dtype())}, + [&](auto list_tag) { + using Data = TypeMapType(list_tag)>; + + CloverCoresetMassKernel + <<(blocks_), + kCloverCoresetExperts, 0, cuda_stream>>>( + reinterpret_cast(logits.data()), mass, + static_cast(block_size), static_cast(blocks_)); + + CloverCoresetAllowKernel + <<(blocks_), 32, + kCloverCoresetExperts * sizeof(float), cuda_stream>>>( + mass, allow, static_cast(blocks_), + static_cast(top_p)); + + CloverCoresetApplyKernel + <<(tokens), 256, 0, cuda_stream>>>( + reinterpret_cast(logits.data()), allow, + reinterpret_cast(out.data()), + static_cast(block_size), static_cast(tokens)); + }, + "CudaCloverCoreset::operator()"); + } + + private: + int64_t blocks_; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_CORESET_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_live_mask/kernel.cuh b/src/native/cuda/ops/clover_live_mask/kernel.cuh new file mode 100644 index 000000000..cccb3d1c4 --- /dev/null +++ b/src/native/cuda/ops/clover_live_mask/kernel.cuh @@ -0,0 +1,31 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_CUH_ + +#include + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverLiveMask, mirroring stage 0 of the +// Triton `clover_live_rows` kernel: a row is live when it is a mask now, was +// a mask in the previous snapshot, or the caller forces a refresh. The +// current ids are written back into the snapshot for the next call. +template +__global__ void CloverLiveMaskKernel(const Index* __restrict__ ids, + Index* __restrict__ previous, + int64_t mask_id, int refresh, + int8_t* __restrict__ live, int64_t n) { + const int64_t i = static_cast(blockIdx.x) * blockDim.x + + static_cast(threadIdx.x); + if (i >= n) return; + const Index current = ids[i]; + const Index prev = refresh ? static_cast(mask_id) : previous[i]; + live[i] = (current == static_cast(mask_id)) || + (prev == static_cast(mask_id)) || refresh + ? 1 + : 0; + previous[i] = current; +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_live_mask/kernel.h b/src/native/cuda/ops/clover_live_mask/kernel.h new file mode 100644 index 000000000..5dc5454d9 --- /dev/null +++ b/src/native/cuda/ops/clover_live_mask/kernel.h @@ -0,0 +1,43 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_H_ + +#include "base/clover_live_mask.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_live_mask/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverLiveMask : public CloverLiveMask { + public: + using CloverLiveMask::CloverLiveMask; + + void operator()(const Tensor ids, Tensor previous, int64_t mask_id, + bool refresh, Tensor live) const override { + const auto n = ids.numel(); + if (n == 0) return; + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr unsigned int kThreads = 256; + const auto blocks = utils::CeilDiv(n, static_cast(kThreads)); + + DispatchFunc>( + {static_cast(ids.dtype())}, + [&](auto list_tag) { + using Index = TypeMapType(list_tag)>; + CloverLiveMaskKernel + <<(blocks), kThreads, 0, cuda_stream>>>( + reinterpret_cast(ids.data()), + reinterpret_cast(previous.data()), + static_cast(mask_id), refresh ? 1 : 0, + reinterpret_cast(live.data()), n); + }, + "CudaCloverLiveMask::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_LIVE_MASK_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_local_memory/kernel.cuh b/src/native/cuda/ops/clover_local_memory/kernel.cuh new file mode 100644 index 000000000..a8ea282be --- /dev/null +++ b/src/native/cuda/ops/clover_local_memory/kernel.cuh @@ -0,0 +1,185 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_CUH_ + +#include + +#include "native/cuda/caster.cuh" + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverLocalMemory, mirroring the Triton +// reference (read-only BD-GDN preview). Per (request, value head): +// 1. normalize q rows and block-local k columns (L2, eps 1e-6, fp32) +// 2. scores = (q . k) * logit_scale; masked by local validity (-inf) +// 3. history = q . clean_state[v0:v0+VEC] (state layout [slot,H,V,K], fp32) +// 4. softmax over {local scores} + one history bucket whose logit is +// history_bias + history_c * log(valid_block_size / reference) +// 5. out = hist * scale * exp(g) * history_weight + softmax . values +// Deterministic: one thread per (query row, value slice), serial loops. +// Grid: x = value tiles (VALUE_DIM/VEC), y = requests * query tiles, z = value heads. +template +__global__ void CloverLocalMemoryKernel( + const Data* __restrict__ q, const Data* __restrict__ k, + const Data* __restrict__ v, const float* __restrict__ g, + const float* __restrict__ state, const Index* __restrict__ indices, + const int8_t* __restrict__ valid, const Data* __restrict__ logit_scale, + const Data* __restrict__ history_bias_in, + const Data* __restrict__ history_c_in, Data* __restrict__ out, + int64_t q_stride_t, int64_t q_stride_h, int64_t q_stride_k, + int64_t k_stride_t, int64_t k_stride_h, int64_t k_stride_k, + int64_t v_stride_t, int64_t v_stride_h, int64_t v_stride_v, + int64_t g_stride_t, int64_t g_stride_h, int64_t state_slot_stride, + int64_t state_h_stride, int64_t state_v_stride, int64_t state_k_stride, + int64_t out_t_stride, int64_t out_h_stride, int64_t out_v_stride, + float scale, float reference_block_size, int block_size, int num_slots, + int num_q_heads, int num_k_heads, int num_v_heads) { + constexpr int kDim = 128; // KEY_DIM == VALUE_DIM == 128 (validated) + constexpr int kVec = 8; // value dims per thread + + const int value_tile = static_cast(blockIdx.x); + const int query_program = static_cast(blockIdx.y); + const int num_query_tiles = gridDim.y / gridDim.z >= 1 + ? (gridDim.y + 0) / 1 / 1 + : 1; // placeholder; host passes 1 tile + (void)num_query_tiles; + const int request = query_program; // host: y = requests * 1 tile + const int value_head = static_cast(blockIdx.z); + + const int t = static_cast(threadIdx.x); // query row in block + if (t >= block_size) return; + const int v0 = value_tile * kVec; + + const auto cache_index = indices[request]; + const bool request_active = cache_index >= 0 && cache_index < num_slots; + const auto safe_slot = cache_index >= 0 ? cache_index : 0; + const int token = request * block_size + t; + const bool row_active = request_active && (valid[token] != 0); + + const int q_head = num_v_heads > num_q_heads + ? value_head / (num_v_heads / num_q_heads) : value_head; + const int k_head = num_v_heads > num_k_heads + ? value_head / (num_v_heads / num_k_heads) : value_head; + + const Data* q_row = q + static_cast(token) * q_stride_t + + q_head * q_stride_h; + const Data* k_base = k + static_cast(request) * block_size * k_stride_t + + k_head * k_stride_h; + const Data* v_base = v + static_cast(request) * block_size * v_stride_t + + value_head * v_stride_h; + const float* state_base = + state + static_cast(safe_slot) * state_slot_stride + + value_head * state_h_stride; + + // 1. q row L2-normalized into registers (fp32). + float qv[kDim]; + float q_sq = 0.0f; +#pragma unroll 8 + for (int kk = 0; kk < kDim; ++kk) { + qv[kk] = Caster::template Cast(q_row[kk * q_stride_k]); + q_sq += qv[kk] * qv[kk]; + } + const float q_norm = rsqrtf(q_sq + 1.0e-6f); +#pragma unroll 8 + for (int kk = 0; kk < kDim; ++kk) qv[kk] *= q_norm; + + // 2. k column norms for the whole block (shared across threads). + extern __shared__ float k_norm_s[]; // [block_size] + if (t < block_size) { + float ks = 0.0f; + const Data* k_col = k_base + t * k_stride_t; + for (int kk = 0; kk < kDim; ++kk) { + const float kv = Caster::template Cast(k_col[kk * k_stride_k]); + ks += kv * kv; + } + k_norm_s[t] = rsqrtf(ks + 1.0e-6f); + } + __syncthreads(); + + // 3. scores for this row against each local key; track validity stats. + float scores[32]; // block_size <= 32 + float logits[32]; + const float ls = Caster::template Cast(logit_scale[value_head]); + for (int j = 0; j < block_size; ++j) { + const Data* k_col = k_base + j * k_stride_t; + float dot = 0.0f; + for (int kk = 0; kk < kDim; ++kk) { + const float kv = Caster::template Cast(k_col[kk * k_stride_k]) * + k_norm_s[j]; + dot += qv[kk] * kv; + } + scores[j] = dot; + logits[j] = valid[request * block_size + j] != 0 && request_active + ? dot * ls + : -INFINITY; + } + + // 4. history readout for this thread's v0..v0+7 slice. + float hist[kVec]; +#pragma unroll + for (int d = 0; d < kVec; ++d) hist[d] = 0.0f; + if (request_active) { + for (int kk = 0; kk < kDim; ++kk) { + const float* st = state_base + kk * state_k_stride; +#pragma unroll + for (int d = 0; d < kVec; ++d) + hist[d] += qv[kk] * st[(v0 + d) * state_v_stride]; + } + } + + // 5. combined softmax with the history bucket. + float hb = Caster::template Cast(history_bias_in[value_head]); + const float hc = Caster::template Cast(history_c_in[value_head]); + int valid_count = 0; + for (int j = 0; j < block_size; ++j) + if (valid[request * block_size + j] != 0 && request_active) ++valid_count; + const float vbs = valid_count > 0 ? static_cast(valid_count) : 1.0f; + hb += hc * __logf(vbs / reference_block_size); + + float row_max = hb; + for (int j = 0; j < block_size; ++j) + if (logits[j] > row_max) row_max = logits[j]; + + float denom = expf(hb - row_max); + float exps[32]; + for (int j = 0; j < block_size; ++j) { + exps[j] = logits[j] == -INFINITY ? 0.0f : expf(logits[j] - row_max); + denom += exps[j]; + } + const float inv_denom = 1.0f / denom; + const float history_weight = expf(hb - row_max) * inv_denom; + + // 6. output = hist * scale * exp(g) * hw + Σ w_j * values_j + const float gate = scale * expf(g[token * g_stride_t + value_head * g_stride_h]); + const float hist_gain = gate * history_weight; + float result[kVec]; +#pragma unroll + for (int d = 0; d < kVec; ++d) result[d] = hist[d] * hist_gain; + + for (int j = 0; j < block_size; ++j) { + if (exps[j] == 0.0f) continue; + const float w = exps[j] * inv_denom; + const Data* v_row = + v_base + static_cast(j) * v_stride_t + v0 * v_stride_v; +#pragma unroll + for (int d = 0; d < kVec; ++d) + result[d] += w * Caster::template Cast(v_row[d * v_stride_v]); + } + + Data* out_row = out + static_cast(token) * out_t_stride + + value_head * out_h_stride + v0 * out_v_stride; + if (row_active) { +#pragma unroll + for (int d = 0; d < kVec; ++d) + out_row[d * out_v_stride] = + Caster::template Cast(result[d]); + } else { +#pragma unroll + for (int d = 0; d < kVec; ++d) + out_row[d * out_v_stride] = Data(0); + } + (void)scores; +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_local_memory/kernel.h b/src/native/cuda/ops/clover_local_memory/kernel.h new file mode 100644 index 000000000..c9f63f178 --- /dev/null +++ b/src/native/cuda/ops/clover_local_memory/kernel.h @@ -0,0 +1,89 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_H_ + +#include "base/clover_local_memory.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_local_memory/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverLocalMemory : public CloverLocalMemory { + public: + using CloverLocalMemory::CloverLocalMemory; + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor state, const Tensor indices, + const Tensor valid, const Tensor logit_scale, + const Tensor history_bias, const Tensor history_c, + size_t block_size, Tensor out) const override { + const auto tokens = q.size(1); + const auto requests = indices.numel(); + const auto num_v_heads = v.size(2); + const auto value_dim = v.size(3); + if (tokens == 0 || requests == 0 || num_v_heads == 0) return; + + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr int kVec = 8; + dim3 grid(static_cast(value_dim / kVec), + static_cast(requests), + static_cast(num_v_heads)); + dim3 block(static_cast(block_size)); + const size_t shared = block_size * sizeof(float); + + const float scale = 1.0f / sqrtf(128.0f); + const float reference = static_cast(block_size); + + DispatchFunc>( + {static_cast(q.dtype()), + static_cast(indices.dtype())}, + [&](auto list_tag) { + using Data = TypeMapType(list_tag)>; + using Index = TypeMapType(list_tag)>; + CloverLocalMemoryKernel + <<>>( + reinterpret_cast(q.data()), + reinterpret_cast(k.data()), + reinterpret_cast(v.data()), + reinterpret_cast(g.data()), + reinterpret_cast(state.data()), + reinterpret_cast(indices.data()), + reinterpret_cast(valid.data()), + reinterpret_cast(logit_scale.data()), + reinterpret_cast(history_bias.data()), + reinterpret_cast(history_c.data()), + reinterpret_cast(out.data()), + static_cast(q.stride(1)), // t (dim0 is batch=1) + static_cast(q.stride(2)), + static_cast(q.stride(3)), + static_cast(k.stride(1)), + static_cast(k.stride(2)), + static_cast(k.stride(3)), + static_cast(v.stride(1)), + static_cast(v.stride(2)), + static_cast(v.stride(3)), + static_cast(g.stride(1)), + static_cast(g.stride(2)), + static_cast(state.stride(0)), + static_cast(state.stride(1)), + static_cast(state.stride(2)), + static_cast(state.stride(3)), + static_cast(out.stride(1)), + static_cast(out.stride(2)), + static_cast(out.stride(3)), + scale, reference, static_cast(block_size), + static_cast(state.size(0)), + static_cast(q.size(2)), + static_cast(k.size(2)), + static_cast(num_v_heads)); + }, + "CudaCloverLocalMemory::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_LOCAL_MEMORY_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_mask_routes/kernel.cuh b/src/native/cuda/ops/clover_mask_routes/kernel.cuh new file mode 100644 index 000000000..237baea63 --- /dev/null +++ b/src/native/cuda/ops/clover_mask_routes/kernel.cuh @@ -0,0 +1,27 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_CUH_ + +#include + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverMaskRoutes, mirroring stage 1 of the +// Triton `clover_live_rows` kernel: live rows pass their route through +// unchanged (negative/dead ids included), non-live rows are replaced by +// `num_experts` (the routing mask value). `ids` layout is [rows, topk] and +// `live` is per row. +template +__global__ void CloverMaskRoutesKernel(const Index* __restrict__ ids, + const int8_t* __restrict__ live, + int64_t num_experts, int64_t width, + Index* __restrict__ out, int64_t n) { + const int64_t i = static_cast(blockIdx.x) * blockDim.x + + static_cast(threadIdx.x); + if (i >= n) return; + const int64_t row = i / width; + out[i] = live[row] != 0 ? ids[i] : static_cast(num_experts); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_mask_routes/kernel.h b/src/native/cuda/ops/clover_mask_routes/kernel.h new file mode 100644 index 000000000..b3ae40d99 --- /dev/null +++ b/src/native/cuda/ops/clover_mask_routes/kernel.h @@ -0,0 +1,44 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_H_ + +#include "base/clover_mask_routes.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_mask_routes/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverMaskRoutes : public CloverMaskRoutes { + public: + using CloverMaskRoutes::CloverMaskRoutes; + + void operator()(const Tensor ids, const Tensor live, int64_t num_experts, + Tensor out) const override { + const auto n = ids.numel(); + if (n == 0) return; + const auto width = ids.size(1); + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr unsigned int kThreads = 256; + const auto blocks = utils::CeilDiv(n, static_cast(kThreads)); + + DispatchFunc>( + {static_cast(ids.dtype())}, + [&](auto list_tag) { + using Index = TypeMapType(list_tag)>; + CloverMaskRoutesKernel + <<(blocks), kThreads, 0, cuda_stream>>>( + reinterpret_cast(ids.data()), + reinterpret_cast(live.data()), + num_experts, static_cast(width), + reinterpret_cast(out.data()), n); + }, + "CudaCloverMaskRoutes::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_MASK_ROUTES_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_merge_moe/kernel.cuh b/src/native/cuda/ops/clover_merge_moe/kernel.cuh new file mode 100644 index 000000000..7e5695d18 --- /dev/null +++ b/src/native/cuda/ops/clover_merge_moe/kernel.cuh @@ -0,0 +1,28 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_CUH_ + +#include + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverMergeMoe, mirroring stage 2 of the +// Triton `clover_live_rows` kernel: freshly computed values replace the +// cache on live rows and are propagated to the output; cached values are +// reused on non-live rows. `fresh`/`cache`/`out` share [rows, width]. +template +__global__ void CloverMergeMoeKernel(const Data* __restrict__ fresh, + const int8_t* __restrict__ live, + Data* __restrict__ cache, Data* __restrict__ out, + int64_t width, int64_t n) { + const int64_t i = static_cast(blockIdx.x) * blockDim.x + + static_cast(threadIdx.x); + if (i >= n) return; + const int64_t row = i / width; + const Data value = live[row] != 0 ? fresh[i] : cache[i]; + out[i] = value; + cache[i] = value; +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_merge_moe/kernel.h b/src/native/cuda/ops/clover_merge_moe/kernel.h new file mode 100644 index 000000000..08c4ab79f --- /dev/null +++ b/src/native/cuda/ops/clover_merge_moe/kernel.h @@ -0,0 +1,45 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_H_ + +#include "base/clover_merge_moe.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_merge_moe/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverMergeMoe : public CloverMergeMoe { + public: + using CloverMergeMoe::CloverMergeMoe; + + void operator()(const Tensor fresh, const Tensor live, Tensor cache, + Tensor out) const override { + const auto n = fresh.numel(); + if (n == 0) return; + const auto width = fresh.size(1); + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr unsigned int kThreads = 256; + const auto blocks = utils::CeilDiv(n, static_cast(kThreads)); + + DispatchFunc, ReducedFloatTypes>>( + {static_cast(fresh.dtype())}, + [&](auto list_tag) { + using Data = TypeMapType(list_tag)>; + CloverMergeMoeKernel + <<(blocks), kThreads, 0, cuda_stream>>>( + reinterpret_cast(fresh.data()), + reinterpret_cast(live.data()), + reinterpret_cast(cache.data()), + reinterpret_cast(out.data()), + static_cast(width), n); + }, + "CudaCloverMergeMoe::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_MERGE_MOE_KERNEL_H_ diff --git a/src/native/cuda/ops/clover_recurrent_gdn/kernel.cuh b/src/native/cuda/ops/clover_recurrent_gdn/kernel.cuh new file mode 100644 index 000000000..c95945097 --- /dev/null +++ b/src/native/cuda/ops/clover_recurrent_gdn/kernel.cuh @@ -0,0 +1,104 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_CUH_ +#define INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_CUH_ + +#include + +#include "native/cuda/caster.cuh" + +namespace infini::ops { + +// Shared-layer SIMT kernel for CloverRecurrentGdn, mirroring the Triton +// reference: per (request, head, value tile), sequentially update the GDN +// state [V, K] from the initial slot and write the final state back: +// q,k L2-normalized (eps 1e-6), q scaled by 0.08838834764831845 (=1/sqrt(128)) +// state *= exp(g); delta = (v - state·k) * beta; state += delta⊗k; y = state·q +// Deterministic: serial token loop, one thread per value row. +// Grid: x = requests, y = heads, z = value tiles (128 / BV). +template +__global__ void CloverRecurrentGdnKernel( + const Data* __restrict__ q, const Data* __restrict__ k, + const Data* __restrict__ v, const float* __restrict__ g, + const float* __restrict__ beta, const int32_t* __restrict__ offsets, + const int32_t* __restrict__ initial, const int32_t* __restrict__ final_idx, + float* __restrict__ state, Data* __restrict__ out, int64_t qs, int64_t qh, + int64_t ks, int64_t kh, int64_t vs, int64_t vh, int64_t gs, int64_t gh, + int64_t bs, int64_t bh, int64_t ss, int64_t sh, int64_t sv, int64_t sk, + int64_t os, int64_t oh, int64_t tokens, int64_t pool, int num_k_heads, + int value_dim, int bv) { + const int request = static_cast(blockIdx.x); + const int head = static_cast(blockIdx.y); + const int value_tile = static_cast(blockIdx.z); + + const int begin = offsets[request]; + const int end = offsets[request + 1]; + const int init = initial[request]; + const int fin = final_idx[request]; + if (!(begin >= 0 && end >= begin && end <= tokens && init >= 0 && + init < pool && fin >= 0 && fin < pool)) + return; + + const int value_row = value_tile * bv + static_cast(threadIdx.x); + if (value_row >= value_dim) return; + + const int q_head = head / (gridDim.y / num_k_heads); // v-heads share q/k heads + const int k_head = q_head; + + float* state_row = + state + static_cast(init) * ss + head * sh + + static_cast(value_row) * sv; + + for (int token = begin; token < end; ++token) { + // k row L2 norm (fp32, over 128 keys) — every thread recomputes it; kept + // simple and deterministic. + float k_vec[128]; + float k_sq = 0.0f; + const Data* k_row = k + static_cast(token) * ks + k_head * kh; + for (int kk = 0; kk < 128; ++kk) { + k_vec[kk] = Caster::template Cast(k_row[kk]); + k_sq += k_vec[kk] * k_vec[kk]; + } + const float k_norm = rsqrtf(k_sq + 1.0e-6f); + for (int kk = 0; kk < 128; ++kk) k_vec[kk] *= k_norm; + + const float decay = expf(g[token * gs + head * gh]); + const float b = beta[token * bs + head * bh]; + const float v_row = Caster::template Cast( + v[static_cast(token) * vs + head * vh + value_row]); + + float remembered = 0.0f; + for (int kk = 0; kk < 128; ++kk) { + state_row[kk * sk] *= decay; + remembered += state_row[kk * sk] * k_vec[kk]; + } + const float delta = (v_row - remembered) * b; + float y = 0.0f; + const Data* q_row = + q + static_cast(token) * qs + q_head * qh; + float q_sq = 0.0f; + float q_vec[128]; + for (int kk = 0; kk < 128; ++kk) { + q_vec[kk] = Caster::template Cast(q_row[kk]); + q_sq += q_vec[kk] * q_vec[kk]; + } + const float q_norm = rsqrtf(q_sq + 1.0e-6f) * 0.08838834764831845f; + for (int kk = 0; kk < 128; ++kk) { + q_vec[kk] *= q_norm; + state_row[kk * sk] += delta * k_vec[kk]; + y += state_row[kk * sk] * q_vec[kk]; + } + out[static_cast(token) * os + head * oh + value_row] = + Caster::template Cast(y); + } + + // Write the final state back to the final slot (may equal the initial one). + if (fin != init) { + float* dst_row = state + static_cast(fin) * ss + head * sh + + static_cast(value_row) * sv; + for (int kk = 0; kk < 128; ++kk) + dst_row[kk * sk] = state_row[kk * sk]; + } +} + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_CUH_ diff --git a/src/native/cuda/ops/clover_recurrent_gdn/kernel.h b/src/native/cuda/ops/clover_recurrent_gdn/kernel.h new file mode 100644 index 000000000..5ef208534 --- /dev/null +++ b/src/native/cuda/ops/clover_recurrent_gdn/kernel.h @@ -0,0 +1,80 @@ +#ifndef INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_H_ +#define INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_H_ + +#include "base/clover_recurrent_gdn.h" +#include "common/generic_utils.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/ops/clover_recurrent_gdn/kernel.cuh" + +namespace infini::ops { + +template +class CudaCloverRecurrentGdn : public CloverRecurrentGdn { + public: + using CloverRecurrentGdn::CloverRecurrentGdn; + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, const Tensor offsets, + const Tensor initial_indices, const Tensor final_indices, + Tensor state, Tensor out) const override { + // Layout [1, tokens, heads, dim]; strides skip the batch dim. + const auto tokens = q.size(1); + const auto num_heads = v.size(2); + const auto value_dim = v.size(3); + const auto requests = offsets.numel() - 1; + if (tokens == 0 || requests <= 0) return; + + auto cuda_stream = + static_cast(stream_ ? stream_ : 0); + constexpr int kBV = 16; + dim3 grid(static_cast(requests), + static_cast(num_heads), + static_cast((value_dim + kBV - 1) / kBV)); + dim3 block(static_cast(kBV)); + + const int num_k_heads = static_cast(k.size(2)); + + DispatchFunc( + {static_cast(q.dtype())}, + [&](auto list_tag) { + using Data = TypeMapType(list_tag)>; + CloverRecurrentGdnKernel + <<>>( + reinterpret_cast(q.data()), + reinterpret_cast(k.data()), + reinterpret_cast(v.data()), + reinterpret_cast(g.data()), + reinterpret_cast(beta.data()), + reinterpret_cast(offsets.data()), + reinterpret_cast(initial_indices.data()), + reinterpret_cast(final_indices.data()), + reinterpret_cast(state.data()), + reinterpret_cast(out.data()), + static_cast(q.stride(1)), + static_cast(q.stride(2)), + static_cast(k.stride(1)), + static_cast(k.stride(2)), + static_cast(v.stride(1)), + static_cast(v.stride(2)), + static_cast(g.stride(1)), + static_cast(g.stride(2)), + static_cast(beta.stride(1)), + static_cast(beta.stride(2)), + static_cast(state.stride(0)), + static_cast(state.stride(1)), + static_cast(state.stride(2)), + static_cast(state.stride(3)), + static_cast(out.stride(1)), + static_cast(out.stride(2)), + static_cast(tokens), + static_cast(state.size(0)), num_k_heads, + static_cast(value_dim), kBV); + }, + "CudaCloverRecurrentGdn::operator()"); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CLOVER_RECURRENT_GDN_KERNEL_H_ diff --git a/src/triton/jit/backend.h b/src/triton/jit/backend.h index ee080734f..21cf6ee62 100644 --- a/src/triton/jit/backend.h +++ b/src/triton/jit/backend.h @@ -31,6 +31,8 @@ struct Target { int warp_size{0}; }; +Target HygonTarget(int device_id); + enum class DataType { kPointer, kInt8, @@ -111,6 +113,11 @@ struct BackendTraits { static constexpr const char* kName = "cuda"; }; +template <> +struct BackendTraits { + static constexpr const char* kName = "hip"; +}; + template ::kName)>> class Backend { @@ -124,6 +131,9 @@ class Backend { static Target CurrentTarget() { const int device_id = CurrentDevice(); if (device_id < 0) return {}; + if constexpr (kDev == Device::Type::kHygon) { + return HygonTarget(device_id); + } int major = 0; auto status = Runtime::DeviceGetAttribute( @@ -179,12 +189,12 @@ class Backend { constexpr unsigned kDefaultSharedMemoryLimit = 48 * 1024; if (metadata.shared_memory_size > kDefaultSharedMemoryLimit) { int maximum_shared_memory = 0; - status = Runtime::DeviceGetAttribute( + const auto runtime_status = Runtime::DeviceGetAttribute( &maximum_shared_memory, Runtime::kDevAttrMaxSharedMemoryPerBlockOptin, CurrentDevice()); - assert(status == Runtime::kSuccess && + assert(runtime_status == Runtime::kSuccess && "Triton JIT failed to get the shared memory limit."); - if (status != Runtime::kSuccess) { + if (runtime_status != Runtime::kSuccess) { Driver::ModuleUnload(module); return nullptr; } diff --git a/src/triton/jit/compile.py b/src/triton/jit/compile.py index 8b49f8e94..6439d9bce 100644 --- a/src/triton/jit/compile.py +++ b/src/triton/jit/compile.py @@ -14,9 +14,28 @@ _DRIVER_BACKENDS = { "cuda": "nvidia", + "hip": "amd", } +def get_hygon_target(device_id): + """Read the compiler target from the active DTK Triton driver.""" + driver = triton.runtime.driver.active + previous = driver.get_current_device() + try: + driver.set_current_device(device_id) + target = driver.get_current_target() + if target.backend != "hip": + raise RuntimeError("Hygon JIT requires a HIP Triton target.") + return { + "backend": target.backend, + "architecture": target.arch, + "warp_size": target.warp_size, + } + finally: + driver.set_current_device(previous) + + def _cache_directory(): configured_path = os.environ.get("INFINI_OPS_TRITON_CACHE_DIR") if configured_path: @@ -211,7 +230,13 @@ def _use_device(target, device_id, stream): previous_driver = triton.runtime.driver.active previous_device = previous_driver.get_current_device() driver_backend = _DRIVER_BACKENDS[target["backend"]] - active_driver = triton.backends.backends[driver_backend].driver() + if target["backend"] == "hip": + # DTK supplies a vendor driver with its own device-property ABI. + active_driver = previous_driver + if active_driver.get_current_target().backend != "hip": + raise RuntimeError("HIP auto-tuning requires the active HIP driver.") + else: + active_driver = triton.backends.backends[driver_backend].driver() try: triton.runtime.driver.set_active(active_driver) @@ -312,8 +337,12 @@ def _write_artifacts(compiled_kernel, target, output_prefix, cache_identity): "num_warps": compiled_kernel.metadata.num_warps, "binary_ext": binary_extension, "shared": compiled_kernel.metadata.shared, - "global_scratch_size": compiled_kernel.metadata.global_scratch_size, - "profile_scratch_size": compiled_kernel.metadata.profile_scratch_size, + "global_scratch_size": getattr( + compiled_kernel.metadata, "global_scratch_size", 0 + ), + "profile_scratch_size": getattr( + compiled_kernel.metadata, "profile_scratch_size", 0 + ), "cache_identity": cache_identity, } temporary_paths = [] diff --git a/src/triton/jit/compiler.cc b/src/triton/jit/compiler.cc index d22323661..966a4a788 100644 --- a/src/triton/jit/compiler.cc +++ b/src/triton/jit/compiler.cc @@ -142,6 +142,23 @@ py::list BuildArguments(const std::vector& arguments) { } // namespace +Target HygonTarget(int device_id) { + static thread_local std::unordered_map targets; + const auto cached = targets.find(device_id); + if (cached != targets.end()) return cached->second; + const auto target = RunPython("Hygon target lookup", [&] { + const auto value = CompilerFunction("get_hygon_target")(device_id); + return Target{value["backend"].cast(), + value["architecture"].cast(), + value["warp_size"].cast()}; + }); + assert(target.has_value() && + "Triton JIT failed to identify the Hygon target."); + if (!target) return {}; + targets.emplace(device_id, *target); + return *target; +} + std::string CompilationFingerprint(const std::string& op_name) { static std::mutex cache_mutex; static std::unordered_map cache; diff --git a/src/triton/ops/add/jit.h b/src/triton/ops/add/jit.h index da3274c5c..821dbe1fb 100644 --- a/src/triton/ops/add/jit.h +++ b/src/triton/ops/add/jit.h @@ -95,6 +95,13 @@ class Operator using triton::jit::Add::Add; }; +template <> +class Operator + : public triton::jit::Add { + public: + using triton::jit::Add::Add; +}; + } // namespace infini::ops #endif diff --git a/src/triton/ops/clover_block_conv1d/jit.h b/src/triton/ops/clover_block_conv1d/jit.h new file mode 100644 index 000000000..39f0857ee --- /dev/null +++ b/src/triton/ops/clover_block_conv1d/jit.h @@ -0,0 +1,63 @@ +#pragma once + +#include "base/clover_block_conv1d.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverBlockConv1d : public ::infini::ops::CloverBlockConv1d { + public: + using ::infini::ops::CloverBlockConv1d::CloverBlockConv1d; + + void operator()(const Tensor input, const Tensor state, const Tensor indices, + const Tensor weight, const Tensor right_weight, + const Tensor valid, size_t block_size, + Tensor out) const override { + if (input.numel() == 0) return; + const Config config{4u, + 1u, + {{"NUM_CHANNELS", input.size(1)}, + {"BLOCK_SIZE", block_size}, + {"NUM_SLOTS", state.size(0)}, + {"WIDTH", 4}, + {"HAS_BIAS", 0}, + {"INDEPENDENT_SYMMETRIC", 1}, + {"BLOCK_T", 4}, + {"NUM_TOKEN_TILES", (block_size + 3) / 4}, + {"BLOCK_C", 128}}}; + Launch( + "clover_block_conv1d", input.device().index(), this->stream_, + this->config_ptr_.get(), config, {}, + [&](const Config&) -> std::optional { + return Grid{ + static_cast((input.size(1) + 127) / 128), + static_cast(indices.numel() * ((block_size + 3) / 4))}; + }, + input, state, indices, weight, weight, right_weight, valid, out, + input.stride(0), input.stride(1), state.stride(0), state.stride(1), + state.stride(2), weight.stride(0), weight.stride(2), + right_weight.stride(0), right_weight.stride(2), out.stride(0), + out.stride(1)); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverBlockConv1d { + public: + using triton::jit::CloverBlockConv1d::CloverBlockConv1d; +}; + +template <> +class Operator + : public triton::jit::CloverBlockConv1d { + public: + using triton::jit::CloverBlockConv1d::CloverBlockConv1d; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_block_conv1d/jit.py b/src/triton/ops/clover_block_conv1d/jit.py new file mode 100644 index 000000000..898fa496b --- /dev/null +++ b/src/triton/ops/clover_block_conv1d/jit.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from SGLang PR #46927, commit 058939af24d5cdc44bb4b8852d0ddfab555ed71e. +# Changes: InfiniOps JIT entry point, Hygon launch geometry and pool bounds. +import triton +import triton.language as tl + + +@triton.jit +def clover_block_conv1d_kernel( + input_ptr, + clean_state_ptr, + cache_indices_ptr, + weight_ptr, + bias_ptr, + right_weight_ptr, + active_valid_ptr, + output_ptr, + input_stride_t, + input_stride_c, + state_stride_slot, + state_stride_c, + state_stride_w, + weight_stride_c, + weight_stride_w, + right_weight_stride_c, + right_weight_stride_w, + output_stride_t, + output_stride_c, + NUM_CHANNELS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + NUM_SLOTS: tl.constexpr, + WIDTH: tl.constexpr, + HAS_BIAS: tl.constexpr, + INDEPENDENT_SYMMETRIC: tl.constexpr, + BLOCK_T: tl.constexpr, + NUM_TOKEN_TILES: tl.constexpr, + BLOCK_C: tl.constexpr, +): + channel_offsets = tl.program_id(0) * BLOCK_C + tl.arange(0, BLOCK_C) + token_program = tl.program_id(1) + request_idx = token_program // NUM_TOKEN_TILES + token_tile = token_program % NUM_TOKEN_TILES + token_offsets = token_tile * BLOCK_T + tl.arange(0, BLOCK_T) + channel_mask = channel_offsets < NUM_CHANNELS + token_mask = token_offsets < BLOCK_SIZE + token_indices = request_idx * BLOCK_SIZE + token_offsets + + cache_index = tl.load(cache_indices_ptr + request_idx) + request_active = (cache_index >= 0) & (cache_index < NUM_SLOTS) + safe_cache_index = tl.maximum(cache_index, 0) + output_active = ( + tl.load(active_valid_ptr + token_indices, mask=token_mask, other=0) != 0 + ) & request_active + + if HAS_BIAS: + bias = tl.load(bias_ptr + channel_offsets, mask=channel_mask, other=0.0).to( + tl.float32 + ) + else: + bias = tl.zeros((BLOCK_C,), dtype=tl.float32) + accumulator = tl.broadcast_to(bias[None, :], (BLOCK_T, BLOCK_C)) + + for tap in tl.static_range(WIDTH): + relative_source = token_offsets + tap - (WIDTH - 1) + from_current = relative_source >= 0 + current_token = request_idx * BLOCK_SIZE + tl.maximum(relative_source, 0) + current_valid = ( + tl.load( + active_valid_ptr + current_token, + mask=token_mask & from_current, + other=0, + ) + != 0 + ) & request_active + current_value = tl.load( + input_ptr + + current_token[:, None] * input_stride_t + + channel_offsets[None, :] * input_stride_c, + mask=(token_mask & from_current & current_valid)[:, None] + & channel_mask[None, :], + other=0.0, + ).to(tl.float32) + + history_offset = relative_source + (WIDTH - 1) + history_value = tl.load( + clean_state_ptr + + safe_cache_index * state_stride_slot + + channel_offsets[None, :] * state_stride_c + + tl.maximum(history_offset, 0)[:, None] * state_stride_w, + mask=(token_mask & ~from_current & request_active)[:, None] + & channel_mask[None, :], + other=0.0, + ).to(tl.float32) + source_value = tl.where(from_current[:, None], current_value, history_value) + weight = tl.load( + weight_ptr + channel_offsets * weight_stride_c + tap * weight_stride_w, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + accumulator += source_value * weight[None, :] + + if INDEPENDENT_SYMMETRIC: + right_accumulator = tl.zeros((BLOCK_T, BLOCK_C), dtype=tl.float32) + for tap in tl.static_range(WIDTH - 1): + distance = WIDTH - 1 - tap + right_offset = token_offsets + distance + right_token = request_idx * BLOCK_SIZE + right_offset + right_valid = ( + tl.load( + active_valid_ptr + right_token, + mask=token_mask & (right_offset < BLOCK_SIZE), + other=0, + ) + != 0 + ) & request_active + right_value = tl.load( + input_ptr + + right_token[:, None] * input_stride_t + + channel_offsets[None, :] * input_stride_c, + mask=(token_mask & (right_offset < BLOCK_SIZE) & right_valid)[:, None] + & channel_mask[None, :], + other=0.0, + ).to(tl.float32) + weight = tl.load( + right_weight_ptr + + channel_offsets * right_weight_stride_c + + (distance - 1) * right_weight_stride_w, + mask=channel_mask, + other=0.0, + ).to(tl.float32) + right_accumulator += right_value * weight[None, :] + accumulator += right_accumulator + + output = accumulator * tl.sigmoid(accumulator) + output = tl.where(output_active[:, None], output, 0.0) + tl.store( + output_ptr + + token_indices[:, None] * output_stride_t + + channel_offsets[None, :] * output_stride_c, + output, + mask=token_mask[:, None] & channel_mask[None, :], + ) + + +kernel = clover_block_conv1d_kernel diff --git a/src/triton/ops/clover_coreset/jit.h b/src/triton/ops/clover_coreset/jit.h new file mode 100644 index 000000000..b1a369956 --- /dev/null +++ b/src/triton/ops/clover_coreset/jit.h @@ -0,0 +1,65 @@ +#pragma once + +#include "base/clover_coreset.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverCoreset : public ::infini::ops::CloverCoreset { + public: + CloverCoreset(const Tensor logits, size_t block_size, double top_p, Tensor out) + : ::infini::ops::CloverCoreset(logits, block_size, top_p, out), + blocks_(logits.size(0) / block_size) {} + + size_t workspace_size_in_bytes() const override { return blocks_ * 256 * 5; } + + void operator()(const Tensor logits, size_t block_size, double top_p, + Tensor out) const override { + if (!blocks_) return; + if (!workspace_ || workspace_size_in_bytes_ < workspace_size_in_bytes()) + throw std::invalid_argument("CloverCoreset: missing workspace."); + Tensor mass{ + workspace_, {blocks_, 256}, DataType::kFloat32, logits.device()}; + Tensor allow{static_cast(workspace_) + blocks_ * 256 * 4, + {blocks_, 256}, + DataType::kInt8, + logits.device()}; + for (int stage = 0; stage < 3; ++stage) { + const Config config{ + 4u, 1u, {{"WIDTH", block_size}, {"STAGE", stage}}}; + Launch( + "clover_coreset", logits.device().index(), this->stream_, + this->config_ptr_.get(), config, + {}, + [&](const Config&) -> std::optional { + return Grid{ + static_cast(stage == 2 ? logits.size(0) : blocks_)}; + }, + logits, mass, allow, out, static_cast(top_p)); + } + } + + private: + size_t blocks_; +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverCoreset { + public: + using triton::jit::CloverCoreset::CloverCoreset; +}; + +template <> +class Operator + : public triton::jit::CloverCoreset { + public: + using triton::jit::CloverCoreset::CloverCoreset; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_coreset/jit.py b/src/triton/ops/clover_coreset/jit.py new file mode 100644 index 000000000..08dc7d469 --- /dev/null +++ b/src/triton/ops/clover_coreset/jit.py @@ -0,0 +1,52 @@ +import triton +import triton.language as tl + + +@triton.jit +def clover_coreset_kernel( + Logits, Mass, Allow, Out, top_p, WIDTH: tl.constexpr, STAGE: tl.constexpr +): + expert = tl.arange(0, 256) + if STAGE == 0: + block = tl.program_id(0) + tokens = tl.arange(0, 32) + logits = tl.load( + Logits + (block * WIDTH + tokens[:, None]) * 256 + expert[None, :], + tokens[:, None] < WIDTH, + -float("inf"), + ).to(tl.float32) + row_max = tl.max(logits, 1) + probability = tl.exp(logits - row_max[:, None]) + probability = probability / tl.sum(probability, 1)[:, None] + selected = tl.full((32, 256), False, tl.int1) + remaining = logits + for rank in tl.static_range(8): + best = tl.argmax(remaining, 1, tie_break_left=True) + chosen = expert[None, :] == best[:, None] + selected |= chosen + remaining = tl.where(chosen, -float("inf"), remaining) + mass = tl.sum( + tl.where(selected & (tokens[:, None] < WIDTH), probability, 0.0), 0 + ) + tl.store(Mass + block * 256 + expert, mass) + elif STAGE == 1: + block = tl.program_id(0) + mass = tl.load(Mass + block * 256 + expert) + # Positive float bits sort by magnitude; low bits break ties by expert ID. + bits = mass.to(tl.int32, bitcast=True).to(tl.int64) + key = ((0x7FFFFFFF - bits) << 32) | expert.to(tl.int64) + order = (tl.sort(key, descending=False) & 0xFFFFFFFF).to(tl.int32) + sorted_mass = tl.load(Mass + block * 256 + order) + probabilities = sorted_mass / tl.maximum(tl.sum(mass, 0), 1.0e-9) + keep = tl.cumsum(probabilities, 0) - probabilities < top_p + keep = tl.where(top_p >= 1.0, sorted_mass > 0.0, keep) + keep = keep | (expert == 0) + tl.store(Allow + block * 256 + order, keep) + else: + token = tl.program_id(0) + logits = tl.load(Logits + token * 256 + expert) + keep = tl.load(Allow + (token // WIDTH) * 256 + expert) + tl.store(Out + token * 256 + expert, tl.where(keep != 0, logits, -float("inf"))) + + +kernel = clover_coreset_kernel diff --git a/src/triton/ops/clover_live_mask/jit.h b/src/triton/ops/clover_live_mask/jit.h new file mode 100644 index 000000000..64af98358 --- /dev/null +++ b/src/triton/ops/clover_live_mask/jit.h @@ -0,0 +1,52 @@ +#pragma once +#include "base/clover_live_mask.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverLiveMask : public ::infini::ops::CloverLiveMask { + public: + using ::infini::ops::CloverLiveMask::CloverLiveMask; + + void operator()(const Tensor ids, Tensor previous, int64_t mask_id, + bool refresh, Tensor live) const override { + if (ids.numel() == 0) return; + const Config config{4u, + 1u, + {{"N", ids.numel()}, + {"WIDTH", 1}, + {"MASK", mask_id}, + {"REFRESH", refresh}, + {"STAGE", 0}, + {"BLOCK", 256}}}; + Launch( + "clover_live_rows", ids.device().index(), this->stream_, + this->config_ptr_.get(), config, {}, + [&](const Config&) -> std::optional { + return Grid{ + static_cast((ids.numel() + 255) / 256)}; + }, + ids, previous, live, live); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverLiveMask { + public: + using triton::jit::CloverLiveMask::CloverLiveMask; +}; + +template <> +class Operator + : public triton::jit::CloverLiveMask { + public: + using triton::jit::CloverLiveMask::CloverLiveMask; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_live_rows/jit.py b/src/triton/ops/clover_live_rows/jit.py new file mode 100644 index 000000000..230ba3a84 --- /dev/null +++ b/src/triton/ops/clover_live_rows/jit.py @@ -0,0 +1,39 @@ +import triton +import triton.language as tl + + +@triton.jit +def clover_live_rows_kernel( + A, + B, + C, + D, + N: tl.constexpr, + WIDTH: tl.constexpr, + MASK: tl.constexpr, + REFRESH: tl.constexpr, + STAGE: tl.constexpr, + BLOCK: tl.constexpr, +): + offset = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) + valid = offset < N + if STAGE == 0: + current = tl.load(A + offset, valid, 0) + previous = tl.load(B + offset, valid & (not REFRESH), MASK) + live = (current == MASK) | (previous == MASK) | REFRESH + tl.store(C + offset, live, valid) + tl.store(B + offset, current, valid) + elif STAGE == 1: + route = tl.load(A + offset, valid, MASK) + live = tl.load(B + offset // WIDTH, valid, 0) + tl.store(C + offset, tl.where(live != 0, route, MASK), valid) + else: + fresh = tl.load(A + offset, valid, 0) + live = tl.load(B + offset // WIDTH, valid, 0) + cached = tl.load(C + offset, valid & (live == 0), 0) + output = tl.where(live != 0, fresh, cached) + tl.store(D + offset, output, valid) + tl.store(C + offset, output, valid) + + +kernel = clover_live_rows_kernel diff --git a/src/triton/ops/clover_local_memory/jit.h b/src/triton/ops/clover_local_memory/jit.h new file mode 100644 index 000000000..1fc42a934 --- /dev/null +++ b/src/triton/ops/clover_local_memory/jit.h @@ -0,0 +1,69 @@ +#pragma once + +#include + +#include "base/clover_local_memory.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverLocalMemory : public ::infini::ops::CloverLocalMemory { + public: + using ::infini::ops::CloverLocalMemory::CloverLocalMemory; + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor state, const Tensor indices, + const Tensor valid, const Tensor logit_scale, + const Tensor history_bias, const Tensor history_c, + size_t block_size, Tensor out) const override { + if (q.numel() == 0) return; + const Config config{4u, + 1u, + {{"NUM_Q_HEADS", q.size(2)}, + {"NUM_K_HEADS", k.size(2)}, + {"NUM_V_HEADS", v.size(2)}, + {"BLOCK_SIZE", block_size}, + {"NUM_SLOTS", state.size(0)}, + {"KEY_DIM", 128}, + {"VALUE_DIM", 128}, + {"BLOCK_T", 32}, + {"BLOCK_Q", 32}, + {"NUM_QUERY_TILES", 1}, + {"BLOCK_K", 128}, + {"BLOCK_V", 64}}}; + Launch( + "clover_local_memory", q.device().index(), this->stream_, + this->config_ptr_.get(), config, {}, + [&](const Config&) -> std::optional { + return Grid{2, static_cast(indices.numel()), + static_cast(v.size(2))}; + }, + q, k, v, g, state, indices, valid, logit_scale, history_bias, history_c, + out, q.stride(1), q.stride(2), q.stride(3), k.stride(1), k.stride(2), + k.stride(3), v.stride(1), v.stride(2), v.stride(3), g.stride(1), + g.stride(2), state.stride(0), state.stride(1), state.stride(2), + state.stride(3), out.stride(1), out.stride(2), out.stride(3), + 1.0f / std::sqrt(128.0f), 32.0f); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverLocalMemory { + public: + using triton::jit::CloverLocalMemory::CloverLocalMemory; +}; + +template <> +class Operator + : public triton::jit::CloverLocalMemory { + public: + using triton::jit::CloverLocalMemory::CloverLocalMemory; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_local_memory/jit.py b/src/triton/ops/clover_local_memory/jit.py new file mode 100644 index 000000000..8595de10c --- /dev/null +++ b/src/triton/ops/clover_local_memory/jit.py @@ -0,0 +1,188 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from SGLang PR #46927, commit 058939af24d5cdc44bb4b8852d0ddfab555ed71e. +# Changes: InfiniOps JIT entry point, Hygon launch geometry and pool bounds. +import triton +import triton.language as tl + + +@triton.jit +def clover_local_memory_kernel( + query_ptr, + key_ptr, + value_ptr, + g_ptr, + clean_state_ptr, + cache_indices_ptr, + active_valid_ptr, + softmax_logit_scale_ptr, + softmax_history_bias_ptr, + softmax_history_c_ptr, + output_ptr, + query_stride_t, + query_stride_h, + query_stride_k, + key_stride_t, + key_stride_h, + key_stride_k, + value_stride_t, + value_stride_h, + value_stride_v, + g_stride_t, + g_stride_h, + state_stride_slot, + state_stride_h, + state_stride_v, + state_stride_k, + output_stride_t, + output_stride_h, + output_stride_v, + scale, + softmax_reference_block_size, + NUM_Q_HEADS: tl.constexpr, + NUM_K_HEADS: tl.constexpr, + NUM_V_HEADS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + NUM_SLOTS: tl.constexpr, + KEY_DIM: tl.constexpr, + VALUE_DIM: tl.constexpr, + BLOCK_T: tl.constexpr, + BLOCK_Q: tl.constexpr, + NUM_QUERY_TILES: tl.constexpr, + BLOCK_K: tl.constexpr, + BLOCK_V: tl.constexpr, +): + """Compute the read-only BD-GDN preview output for one fixed-size block. + + Block-local softmax includes the gate-decayed clean readout as one + history bucket. Its logit is history_bias plus + history_c * log(valid_block_size / reference_block_size). + + One program handles one request, one value head, and one value-dimension + tile. Persistent state is read directly in the [slot, H, V, K] + layout and is never written. + """ + value_tile = tl.program_id(0) + query_program = tl.program_id(1) + request_idx = query_program // NUM_QUERY_TILES + query_tile = query_program % NUM_QUERY_TILES + value_head = tl.program_id(2) + + token_offsets = query_tile * BLOCK_Q + tl.arange(0, BLOCK_Q) + local_key_offsets = tl.arange(0, BLOCK_T) + key_offsets = tl.arange(0, BLOCK_K) + value_offsets = value_tile * BLOCK_V + tl.arange(0, BLOCK_V) + token_mask = token_offsets < BLOCK_SIZE + local_key_mask = local_key_offsets < BLOCK_SIZE + key_mask = key_offsets < KEY_DIM + value_mask = value_offsets < VALUE_DIM + token_indices = request_idx * BLOCK_SIZE + token_offsets + + cache_index = tl.load(cache_indices_ptr + request_idx) + request_active = (cache_index >= 0) & (cache_index < NUM_SLOTS) + safe_cache_index = tl.maximum(cache_index, 0) + active = ( + tl.load(active_valid_ptr + token_indices, mask=token_mask, other=0) != 0 + ) & request_active + + # Value heads may share Q/K heads. Clover maps pairs of value heads to + # one of 16 Q/K heads without materializing repeat_interleave. + query_head = value_head // (NUM_V_HEADS // NUM_Q_HEADS) + key_head = value_head // (NUM_V_HEADS // NUM_K_HEADS) + + query_offsets = ( + token_indices[:, None] * query_stride_t + + query_head * query_stride_h + + key_offsets[None, :] * query_stride_k + ) + key_offsets_transposed = ( + (request_idx * BLOCK_SIZE + local_key_offsets)[None, :] * key_stride_t + + key_head * key_stride_h + + key_offsets[:, None] * key_stride_k + ) + value_matrix_offsets = ( + (request_idx * BLOCK_SIZE + local_key_offsets)[:, None] * value_stride_t + + value_head * value_stride_h + + value_offsets[None, :] * value_stride_v + ) + + query = tl.load( + query_ptr + query_offsets, + mask=token_mask[:, None] & key_mask[None, :] & request_active, + other=0.0, + ) + key_transposed = tl.load( + key_ptr + key_offsets_transposed, + mask=key_mask[:, None] & local_key_mask[None, :] & request_active, + other=0.0, + ) + values = tl.load( + value_ptr + value_matrix_offsets, + mask=local_key_mask[:, None] & value_mask[None, :] & request_active, + other=0.0, + ) + + # Match the GDN epsilon and normalize before Tensor Core dot products. + query_f32 = query.to(tl.float32) + query_norm = tl.rsqrt(tl.sum(query_f32 * query_f32, axis=1) + 1.0e-6) + query = (query_f32 * query_norm[:, None]).to(query.dtype) + key_f32 = key_transposed.to(tl.float32) + key_norm = tl.rsqrt(tl.sum(key_f32 * key_f32, axis=0) + 1.0e-6) + key_transposed = (key_f32 * key_norm[None, :]).to(key_transposed.dtype) + + state_offsets = ( + safe_cache_index * state_stride_slot + + value_head * state_stride_h + + value_offsets[None, :] * state_stride_v + + key_offsets[:, None] * state_stride_k + ) + clean_state = tl.load( + clean_state_ptr + state_offsets, + mask=key_mask[:, None] & value_mask[None, :] & request_active, + other=0.0, + ) + + scalar_offsets = token_indices * g_stride_t + value_head * g_stride_h + g = tl.load(g_ptr + scalar_offsets, mask=token_mask & request_active, other=0.0).to( + tl.float32 + ) + local_active = ( + tl.load( + active_valid_ptr + request_idx * BLOCK_SIZE + local_key_offsets, + mask=local_key_mask, + other=0, + ) + != 0 + ) & request_active + + history = tl.dot(query, clean_state.to(query.dtype)) + scores = tl.dot(query, key_transposed) + logit_scale = tl.load(softmax_logit_scale_ptr + value_head).to(tl.float32) + history_bias = tl.load(softmax_history_bias_ptr + value_head).to(tl.float32) + history_c = tl.load(softmax_history_c_ptr + value_head).to(tl.float32) + valid_block_size = tl.maximum(tl.sum(local_active, axis=0), 1.0) + history_bias += history_c * tl.log(valid_block_size / softmax_reference_block_size) + logits = scores * logit_scale + logits = tl.where(local_active[None, :], logits, float("-inf")) + row_max = tl.maximum(tl.max(logits, axis=1), history_bias) + exp_local = tl.exp(logits - row_max[:, None]) + exp_history = tl.exp(history_bias - row_max) + denominator = tl.sum(exp_local, axis=1) + exp_history + local_weights = exp_local / denominator[:, None] + history_weight = exp_history / denominator + local = tl.dot(local_weights.to(values.dtype), values) + output = history * (scale * tl.exp(g) * history_weight)[:, None] + local + output = tl.where(active[:, None], output, 0.0) + + output_offsets = ( + token_indices[:, None] * output_stride_t + + value_head * output_stride_h + + value_offsets[None, :] * output_stride_v + ) + tl.store( + output_ptr + output_offsets, + output, + mask=token_mask[:, None] & value_mask[None, :], + ) + + +kernel = clover_local_memory_kernel diff --git a/src/triton/ops/clover_mask_routes/jit.h b/src/triton/ops/clover_mask_routes/jit.h new file mode 100644 index 000000000..ee21594ab --- /dev/null +++ b/src/triton/ops/clover_mask_routes/jit.h @@ -0,0 +1,52 @@ +#pragma once +#include "base/clover_mask_routes.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverMaskRoutes : public ::infini::ops::CloverMaskRoutes { + public: + using ::infini::ops::CloverMaskRoutes::CloverMaskRoutes; + + void operator()(const Tensor ids, const Tensor live, int64_t num_experts, + Tensor out) const override { + if (ids.numel() == 0) return; + const Config config{4u, + 1u, + {{"N", ids.numel()}, + {"WIDTH", ids.size(1)}, + {"MASK", num_experts}, + {"REFRESH", false}, + {"STAGE", 1}, + {"BLOCK", 256}}}; + Launch( + "clover_live_rows", ids.device().index(), this->stream_, + this->config_ptr_.get(), config, {}, + [&](const Config&) -> std::optional { + return Grid{ + static_cast((ids.numel() + 255) / 256)}; + }, + ids, live, out, out); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverMaskRoutes { + public: + using triton::jit::CloverMaskRoutes::CloverMaskRoutes; +}; + +template <> +class Operator + : public triton::jit::CloverMaskRoutes { + public: + using triton::jit::CloverMaskRoutes::CloverMaskRoutes; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_merge_moe/jit.h b/src/triton/ops/clover_merge_moe/jit.h new file mode 100644 index 000000000..8b3e50bd6 --- /dev/null +++ b/src/triton/ops/clover_merge_moe/jit.h @@ -0,0 +1,53 @@ +#pragma once +#include "base/clover_merge_moe.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverMergeMoe : public ::infini::ops::CloverMergeMoe { + public: + using ::infini::ops::CloverMergeMoe::CloverMergeMoe; + + void operator()(const Tensor fresh, const Tensor live, Tensor cache, + Tensor out) const override { + if (fresh.numel() == 0) return; + const Config config{4u, + 1u, + {{"N", fresh.numel()}, + {"WIDTH", fresh.size(1)}, + {"MASK", 0}, + {"REFRESH", false}, + {"STAGE", 2}, + {"BLOCK", 256}}}; + Launch( + "clover_live_rows", fresh.device().index(), this->stream_, + this->config_ptr_.get(), config, + {}, + [&](const Config&) -> std::optional { + return Grid{ + static_cast((fresh.numel() + 255) / 256)}; + }, + fresh, live, cache, out); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverMergeMoe { + public: + using triton::jit::CloverMergeMoe::CloverMergeMoe; +}; + +template <> +class Operator + : public triton::jit::CloverMergeMoe { + public: + using triton::jit::CloverMergeMoe::CloverMergeMoe; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_recurrent_gdn/jit.h b/src/triton/ops/clover_recurrent_gdn/jit.h new file mode 100644 index 000000000..25d07afb9 --- /dev/null +++ b/src/triton/ops/clover_recurrent_gdn/jit.h @@ -0,0 +1,59 @@ +#pragma once + +#include "base/clover_recurrent_gdn.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class CloverRecurrentGdn : public ::infini::ops::CloverRecurrentGdn { + public: + using ::infini::ops::CloverRecurrentGdn::CloverRecurrentGdn; + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, const Tensor offsets, + const Tensor initial_indices, const Tensor final_indices, + Tensor state, Tensor out) const override { + const Config config{4u, + 1u, + {{"TOKENS", q.size(1)}, + {"POOL", state.size(0)}, + {"HK", q.size(2)}, + {"HV", v.size(2)}, + {"BV", 16}}}; + Launch( + "clover_recurrent_gdn", q.device().index(), this->stream_, + this->config_ptr_.get(), config, {}, + [&](const Config&) -> std::optional { + return Grid{static_cast(offsets.numel() - 1), + static_cast(v.size(2)), 8}; + }, + q, k, v, g, beta, offsets, initial_indices, final_indices, state, out, + q.stride(1), q.stride(2), k.stride(1), k.stride(2), v.stride(1), + v.stride(2), g.stride(1), g.stride(2), beta.stride(1), beta.stride(2), + state.stride(0), state.stride(1), state.stride(2), state.stride(3), + out.stride(1), out.stride(2)); + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::CloverRecurrentGdn { + public: + using triton::jit::CloverRecurrentGdn< + Device::Type::kNvidia>::CloverRecurrentGdn; +}; + +template <> +class Operator + : public triton::jit::CloverRecurrentGdn { + public: + using triton::jit::CloverRecurrentGdn< + Device::Type::kHygon>::CloverRecurrentGdn; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/clover_recurrent_gdn/jit.py b/src/triton/ops/clover_recurrent_gdn/jit.py new file mode 100644 index 000000000..1019f1d7f --- /dev/null +++ b/src/triton/ops/clover_recurrent_gdn/jit.py @@ -0,0 +1,77 @@ +import triton +import triton.language as tl + + +@triton.jit +def clover_recurrent_gdn_kernel( + Q, + K, + V, + G, + Beta, + Offsets, + Initial, + Final, + State, + Out, + qs, + qh, + ks, + kh, + vs, + vh, + gs, + gh, + bs, + bh, + ss, + sh, + sv, + sk, + os, + oh, + TOKENS: tl.constexpr, + POOL: tl.constexpr, + HK: tl.constexpr, + HV: tl.constexpr, + BV: tl.constexpr, +): + request, head, tile = tl.program_id(0), tl.program_id(1), tl.program_id(2) + begin, end = tl.load(Offsets + request), tl.load(Offsets + request + 1) + initial, final = tl.load(Initial + request), tl.load(Final + request) + if ( + begin >= 0 + and end >= begin + and end <= TOKENS + and initial >= 0 + and initial < POOL + and final >= 0 + and final < POOL + ): + key = tl.arange(0, 128) + value = tile * BV + tl.arange(0, BV) + state = tl.load( + State + initial * ss + head * sh + value[:, None] * sv + key[None, :] * sk + ) + key_head = head // (HV // HK) + for token in range(begin, end): + q = tl.load(Q + token * qs + key_head * qh + key).to(tl.float32) + k = tl.load(K + token * ks + key_head * kh + key).to(tl.float32) + v = tl.load(V + token * vs + head * vh + value).to(tl.float32) + q = q * tl.rsqrt(tl.sum(q * q, 0) + 1.0e-6) * 0.08838834764831845 + k = k * tl.rsqrt(tl.sum(k * k, 0) + 1.0e-6) + decay = tl.exp(tl.load(G + token * gs + head * gh)) + beta = tl.load(Beta + token * bs + head * bh) + state = state * decay + remembered = tl.sum(state * k[None, :], 1) + delta = (v - remembered) * beta + state += delta[:, None] * k[None, :] + y = tl.sum(state * q[None, :], 1) + tl.store(Out + token * os + head * oh + value, y) + tl.store( + State + final * ss + head * sh + value[:, None] * sv + key[None, :] * sk, + state, + ) + + +kernel = clover_recurrent_gdn_kernel diff --git a/src/triton/ops/moe_fused_dense/jit.h b/src/triton/ops/moe_fused_dense/jit.h new file mode 100644 index 000000000..b38bf80c9 --- /dev/null +++ b/src/triton/ops/moe_fused_dense/jit.h @@ -0,0 +1,100 @@ +#pragma once + +#include "base/moe_fused_dense.h" +#include "triton/jit/jit.h" + +namespace infini::ops::triton::jit { + +template +class MoeFusedDense : public ::infini::ops::MoeFusedDense { + public: + using ::infini::ops::MoeFusedDense::MoeFusedDense; + + static bool supports(Tensor output, const Tensor hidden, const Tensor w13, + const Tensor w2, const Tensor weights, const Tensor ids, + const Tensor sorted, const Tensor experts, + const Tensor total, size_t block_size, + const std::optional bias1 = std::nullopt, + const std::optional bias2 = std::nullopt) { + return block_size == 16 && !bias1 && !bias2 && + ::infini::ops::MoeFusedDense::supports( + output, hidden, w13, w2, weights, ids, sorted, experts, total, + block_size, bias1, bias2); + } + + size_t workspace_size_in_bytes() const override { + const auto pairs = metadata_[1].size(0) * metadata_[5].size(1); + return pairs * (metadata_[3].size(2) * 2 + metadata_[1].size(1)) * + metadata_[1].element_size(); + } + + void operator()(Tensor output, const Tensor hidden, const Tensor w13, + const Tensor w2, const Tensor weights, const Tensor ids, + const Tensor sorted, const Tensor experts, const Tensor total, + size_t block_size, const std::optional bias1, + const std::optional bias2) const override { + Invocation(output, hidden, w13, w2, weights, ids, sorted, experts, total, + block_size, bias1, bias2); + if (bias1 || bias2) + throw std::invalid_argument( + "Hygon Triton MoE does not support expert biases."); + if (!workspace_ || workspace_size_in_bytes_ < workspace_size_in_bytes()) + throw std::invalid_argument( + "Hygon Triton MoE requires its declared workspace."); + const auto n = hidden.size(0), h = hidden.size(1), i = w2.size(2), + topk = ids.size(1); + const auto pairs = n * topk; + Tensor mid{workspace_, {pairs, 2 * i}, hidden.dtype(), hidden.device()}; + Tensor pair_output{ + static_cast(workspace_) + pairs * 2 * i * hidden.element_size(), + {pairs, h}, + hidden.dtype(), + hidden.device()}; + for (int stage = 0; stage < 3; ++stage) { + const Config config{4u, + 2u, + {{"TOKENS", n}, + {"E", w13.size(0)}, + {"H", h}, + {"I", i}, + {"TOPK", topk}, + {"BM", block_size}, + {"BN", 64}, + {"BK", 64}, + {"STAGE", stage}}}; + Launch( + "moe_fused_dense", hidden.device().index(), this->stream_, nullptr, + config, {}, + [&](const Config&) -> std::optional { + if (stage == 2) + return Grid{static_cast(n), + static_cast((h + 63) / 64)}; + const auto cols = stage == 0 ? 2 * i : h; + return Grid{static_cast(sorted.numel() / block_size), + static_cast((cols + 63) / 64)}; + }, + hidden, w13, w2, weights, ids, sorted, experts, total, mid, + pair_output, output); + } + } +}; + +} // namespace infini::ops::triton::jit + +namespace infini::ops { + +template <> +class Operator + : public triton::jit::MoeFusedDense { + public: + using triton::jit::MoeFusedDense::MoeFusedDense; +}; + +template <> +class Operator + : public triton::jit::MoeFusedDense { + public: + using triton::jit::MoeFusedDense::MoeFusedDense; +}; + +} // namespace infini::ops diff --git a/src/triton/ops/moe_fused_dense/jit.py b/src/triton/ops/moe_fused_dense/jit.py new file mode 100644 index 000000000..e06e63d7e --- /dev/null +++ b/src/triton/ops/moe_fused_dense/jit.py @@ -0,0 +1,103 @@ +import triton +import triton.language as tl + + +@triton.jit +def moe_fused_dense_kernel( + X, + W13, + W2, + Weights, + Ids, + Sorted, + Experts, + Total, + Mid, + Pairs, + Out, + TOKENS: tl.constexpr, + E: tl.constexpr, + H: tl.constexpr, + I: tl.constexpr, # noqa: E741 + TOPK: tl.constexpr, + BM: tl.constexpr, + BN: tl.constexpr, + BK: tl.constexpr, + STAGE: tl.constexpr, +): + if STAGE == 2: + token = tl.program_id(0) + h = tl.program_id(1) * BN + tl.arange(0, BN) + ranks = tl.arange(0, triton.next_power_of_2(TOPK)) + route = tl.load(Ids + token * TOPK + ranks, ranks < TOPK, E) + active = (route >= 0) & (route < E) + values = tl.load( + Pairs + (token * TOPK + ranks[:, None]) * H + h[None, :], + active[:, None] & (h[None, :] < H), + 0, + ).to(tl.float32) + weights = tl.load(Weights + token * TOPK + ranks, ranks < TOPK, 0) + tl.store(Out + token * H + h, tl.sum(values * weights[:, None], 0), h < H) + else: + block = tl.program_id(0) + n = tl.program_id(1) * BN + tl.arange(0, BN) + if block * BM < tl.load(Total): + expert = tl.load(Experts + block) + rows = tl.load(Sorted + block * BM + tl.arange(0, BM)) + valid = rows < TOKENS * TOPK + kk = tl.arange(0, BK) + acc = tl.zeros((BM, BN), tl.float32) + if expert >= 0: + if STAGE == 0: + for start in range(tl.cdiv(H, BK)): + k = start * BK + kk + x = tl.load( + X + (rows[:, None] // TOPK) * H + k[None, :], + valid[:, None] & (k[None, :] < H), + 0, + ) + w = tl.load( + W13 + expert * (2 * I * H) + n[None, :] * H + k[:, None], + (n[None, :] < 2 * I) & (k[:, None] < H), + 0, + ) + acc += tl.dot(x, w) + tl.store( + Mid + rows[:, None] * (2 * I) + n[None, :], + acc, + valid[:, None] & (n[None, :] < 2 * I), + ) + else: + for start in range(tl.cdiv(I, BK)): + k = start * BK + kk + gate = tl.load( + Mid + rows[:, None] * (2 * I) + k[None, :], + valid[:, None] & (k[None, :] < I), + 0, + ).to(tl.float32) + up = tl.load( + Mid + rows[:, None] * (2 * I) + I + k[None, :], + valid[:, None] & (k[None, :] < I), + 0, + ).to(tl.float32) + x = (gate * tl.sigmoid(gate) * up).to(Mid.dtype.element_ty) + w = tl.load( + W2 + expert * (H * I) + n[None, :] * I + k[:, None], + (n[None, :] < H) & (k[:, None] < I), + 0, + ) + acc += tl.dot(x, w) + tl.store( + Pairs + rows[:, None] * H + n[None, :], + acc, + valid[:, None] & (n[None, :] < H), + ) + elif STAGE == 1: + tl.store( + Pairs + rows[:, None] * H + n[None, :], + acc, + valid[:, None] & (n[None, :] < H), + ) + + +kernel = moe_fused_dense_kernel diff --git a/tests/conftest.py b/tests/conftest.py index cfc2691c1..8e9ef0628 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,7 @@ def _host_range_profile_compiled(): def pytest_configure(config): - torch.backends.fp32_precision = "tf32" + torch.backends.fp32_precision = "ieee" if torch.version.hip else "tf32" profile_output = config.getoption("--host-range-profile") config._infini_host_range_profile_output = None diff --git a/tests/native/build_clover_workspace.py b/tests/native/build_clover_workspace.py new file mode 100644 index 000000000..56f261634 --- /dev/null +++ b/tests/native/build_clover_workspace.py @@ -0,0 +1,37 @@ +"""Build the native workspace test adapter against an installed InfiniOps SDK.""" + +import os +import pathlib +import subprocess +import sysconfig + +import pybind11 + +root = pathlib.Path(__file__).resolve().parents[2] +prefix = pathlib.Path(os.environ["INFINI_ROOT"]) +cuda = pathlib.Path(os.environ.get("CUDA_HOME", "/opt/dtk/cuda/cuda")) +output = root / "build" / "clover-tests" +output.mkdir(parents=True, exist_ok=True) +subprocess.run( + [ + os.environ.get("CXX", "g++"), + "-std=c++17", + "-O2", + "-shared", + "-fPIC", + str(pathlib.Path(__file__).with_name("clover_workspace_smoke.cpp")), + "-o", + str(output / ("workspace_ops" + sysconfig.get_config_var("EXT_SUFFIX"))), + "-I" + pybind11.get_include(), + "-I" + sysconfig.get_path("include"), + "-I" + str(prefix / "include"), + "-I" + str(root / "src"), + "-I" + str(cuda / "include"), + "-L" + str(prefix / "lib"), + "-linfiniops", + "-linfinirt", + "-Wl,-rpath," + str(prefix / "lib"), + ], + check=True, +) +print(output) diff --git a/tests/native/check_clover_workspace.py b/tests/native/check_clover_workspace.py new file mode 100644 index 000000000..65235b89c --- /dev/null +++ b/tests/native/check_clover_workspace.py @@ -0,0 +1,68 @@ +import infini.ops as ops +import torch +import workspace_ops + +torch.manual_seed(29) +s = torch.cuda.current_stream().cuda_stream +d = "cuda" +dtype = torch.bfloat16 +for n, k in ((1, 8), (32, 8), (64, 8), (32, 3)): + e, h, i, bm = 16, 128, 64, 16 + x = torch.randn(n, h, device=d, dtype=dtype) * 0.2 + w13 = torch.randn(e, 2 * i, h, device=d, dtype=dtype) * 0.1 + w2 = torch.randn(e, h, i, device=d, dtype=dtype) * 0.1 + scores = torch.randn(n, e, device=d).softmax(-1) + weights, ids = scores.topk(k, dim=-1) + weights = weights.contiguous() + ids = ids.int().contiguous() + cap = max(n * k * bm if n * k < e else n * k + e * (bm - 1), bm) + sorted_ids = torch.empty(cap, device=d, dtype=torch.int32) + experts = torch.empty((cap + bm - 1) // bm, device=d, dtype=torch.int32) + total = torch.empty(1, device=d, dtype=torch.int32) + ops.moe_align_block_size(ids, None, e, bm, sorted_ids, experts, total, stream=s) + y = torch.full_like(x, float("nan")) + scratch = torch.empty(n * k * (2 * i + h) * 2, device=d, dtype=torch.uint8) + workspace_ops.moe( + [y, x, w13, w2, weights, ids, sorted_ids, experts, total], bm, scratch, s + ) + ref = torch.zeros_like(x, dtype=torch.float32) + for token in range(n): + for r in range(k): + expert = int(ids[token, r]) + mid = x[token] @ w13[expert].T + act = (torch.nn.functional.silu(mid[:i].float()) * mid[i:].float()).to( + dtype + ) + value = act @ w2[expert].T + ref[token] += value.float() * weights[token, r] + torch.testing.assert_close(y, ref.to(dtype), rtol=0.025, atol=0.003) + print("PASS MoE", n, float((y - ref).abs().max()), flush=True) +q = torch.randn(1, 11, 2, 128, device=d, dtype=dtype) +k = torch.randn_like(q) +v = torch.randn(1, 11, 4, 128, device=d, dtype=dtype) +g = -torch.rand(1, 11, 4, device=d) +beta = torch.rand_like(g) +state = torch.randn(3, 4, 128, 128, device=d) +old = state.clone() +out = torch.empty_like(v) +offsets = torch.tensor([0, 4, 11], device=d, dtype=torch.int32) +init = torch.tensor([0, 1], device=d, dtype=torch.int32) +final = init.clone() +ops.clover_recurrent_gdn(q, k, v, g, beta, offsets, init, final, state, out, stream=s) +qn = q.float() * torch.rsqrt(q.float().square().sum(-1, keepdim=True) + 1e-6) +kn = k.float() * torch.rsqrt(k.float().square().sum(-1, keepdim=True) + 1e-6) +qn = qn.repeat_interleave(2, 2) / 128**0.5 +kn = kn.repeat_interleave(2, 2) +ref = torch.empty_like(v) +for b, (start, end) in enumerate(((0, 4), (4, 11))): + st = old[b].clone() + for t in range(start, end): + st = st * g[0, t].exp()[:, None, None] + remembered = (st * kn[0, t, :, None, :]).sum(-1) + delta = (v[0, t].float() - remembered) * beta[0, t, :, None] + st += delta[:, :, None] * kn[0, t, :, None, :] + ref[0, t] = (st * qn[0, t, :, None, :]).sum(-1) + torch.testing.assert_close(state[b], st, rtol=0.001, atol=0.001) +torch.testing.assert_close(out, ref, rtol=0.02, atol=0.01) +assert torch.equal(state[2], old[2]) +print("PASS recurrent GDN", float((out - ref).abs().max()), flush=True) diff --git a/tests/native/clover_native_adapter.cpp b/tests/native/clover_native_adapter.cpp new file mode 100644 index 000000000..c7461fca7 --- /dev/null +++ b/tests/native/clover_native_adapter.cpp @@ -0,0 +1,25 @@ +// Test adapter for workspace-taking Clover operators with explicit +// implementation index selection, mirroring clover_workspace_smoke.cpp. +#include +#include + +#include "pybind11_utils.h" + +namespace py = pybind11; +using namespace infini::ops; + +PYBIND11_MODULE(clover_native_adapter, m) { + m.def("coreset", [](py::object logits, size_t width, double top_p, + py::object out, py::object workspace, uintptr_t stream, + size_t implementation_index) { + Handle handle; + handle.set_stream(reinterpret_cast(stream)); + auto w = TensorFromPybind11Handle(workspace); + handle.set_workspace(w.data()); + handle.set_workspace_size_in_bytes(w.numel() * w.element_size()); + Config config; + config.set_implementation_index(implementation_index); + CloverCoreset::Call(handle, config, TensorFromPybind11Handle(logits), + width, top_p, TensorFromPybind11Handle(out)); + }); +} diff --git a/tests/native/clover_workspace_smoke.cpp b/tests/native/clover_workspace_smoke.cpp new file mode 100644 index 000000000..4b98d5571 --- /dev/null +++ b/tests/native/clover_workspace_smoke.cpp @@ -0,0 +1,41 @@ +#include +#include + +#include "pybind11_utils.h" + +namespace py = pybind11; +using namespace infini::ops; + +PYBIND11_MODULE(workspace_ops, m) { + m.def("moe", [](py::list tensors, size_t block_size, py::object workspace, + uintptr_t stream) { + if (tensors.size() != 9) + throw py::value_error("Expected nine MoE tensors."); + Handle handle; + handle.set_stream(reinterpret_cast(stream)); + auto scratch = TensorFromPybind11Handle(workspace); + handle.set_workspace(scratch.data()); + handle.set_workspace_size_in_bytes(scratch.numel() * + scratch.element_size()); + const auto tensor = [&](int i) { + return TensorFromPybind11Handle(tensors[i]); + }; + MoeFusedDense::Call(handle, + MoeFusedDense::DefaultConfig(Device::Type::kHygon), + tensor(0), tensor(1), tensor(2), tensor(3), tensor(4), + tensor(5), tensor(6), tensor(7), tensor(8), block_size, + std::optional{}, std::optional{}); + }); + m.def("coreset", [](py::object logits, size_t width, double top_p, + py::object out, py::object workspace, uintptr_t stream) { + Handle handle; + handle.set_stream(reinterpret_cast(stream)); + auto w = TensorFromPybind11Handle(workspace); + handle.set_workspace(w.data()); + handle.set_workspace_size_in_bytes(w.numel()); + CloverCoreset::Call(handle, + CloverCoreset::DefaultConfig(Device::Type::kHygon), + TensorFromPybind11Handle(logits), width, top_p, + TensorFromPybind11Handle(out)); + }); +} diff --git a/tests/test_clover_block_conv1d.py b/tests/test_clover_block_conv1d.py new file mode 100644 index 000000000..04946ce4a --- /dev/null +++ b/tests/test_clover_block_conv1d.py @@ -0,0 +1,87 @@ +"""Correctness tests for the native CloverBlockConv1d provider. + +Golden model replicates the Triton reference: per (request, token, channel), +silu(left-history 4-tap conv + right-symmetric 3-tap conv), zeroed on invalid +rows. Cross-checked against the Triton provider on identical inputs, covering +bf16/fp16, block sizes 1/8/32, dead slots (-1) and per-row validity masks. +""" + +import infini.ops +import pytest +import torch + +if not hasattr(infini.ops, "CloverBlockConv1d"): + pytest.skip("`CloverBlockConv1d` is not available", allow_module_level=True) + +NATIVE, TRITON = 0, 10 + + +def torch_golden(x, state, indices, weight, right_weight, valid, block_size): + tokens, channels = x.shape + requests = indices.numel() + out = torch.zeros_like(x, dtype=torch.float32) + xf, wf, rwf = x.float(), weight.float(), right_weight.float() + for r in range(requests): + slot = int(indices[r]) + active_request = 0 <= slot < state.size(0) + for t in range(block_size): + token = r * block_size + t + if not active_request or int(valid[token]) == 0: + continue + acc = 0.0 + for tap in range(4): + rel = t + tap - 3 + if rel >= 0: + src = r * block_size + rel + value = xf[src] if int(valid[src]) != 0 else 0.0 + else: + value = state[slot, :, rel + 3].float() + acc = acc + value * wf[:, 0, tap] + for d in range(1, 4): + right = t + d + if right < block_size: + src = r * block_size + right + value = xf[src] if int(valid[src]) != 0 else 0.0 + acc = acc + value * rwf[:, 0, d - 1] + out[token] = acc * torch.sigmoid(acc) + return out + + +@pytest.mark.parametrize("block_size", (1, 8, 32)) +@pytest.mark.parametrize("dtype,rtol", ((torch.bfloat16, 3e-2), (torch.float16, 3e-2))) +@pytest.mark.parametrize("id_dtype", (torch.int32, torch.int64)) +def test_block_conv1d(device, block_size, dtype, rtol, id_dtype): + if device != "cuda": + pytest.skip("requires the CUDA backend") + impls = infini.ops.CloverBlockConv1d.active_implementation_indices("nvidia") + if NATIVE not in impls or TRITON not in impls: + pytest.skip(f"requires impls 0 and 10, active={impls}") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(31) + requests, channels, slots = 6, 256, 8 + + x = torch.randn(requests * block_size, channels, device=device, dtype=dtype) + state = torch.randn(slots, channels, 3, device=device, dtype=dtype) + # One dead slot and one out-of-range slot exercise the guard paths. + indices = torch.arange(requests, device=device, dtype=id_dtype) % slots + indices[4] = -1 + indices[5] = slots + 3 + weight = torch.randn(channels, 1, 4, device=device, dtype=dtype) * 0.2 + right_weight = torch.randn(channels, 1, 3, device=device, dtype=dtype) * 0.2 + valid = torch.ones(requests * block_size, device=device, dtype=torch.int8) + valid[::5] = 0 # scattered invalid rows inside blocks + out_n = torch.empty_like(x) + out_t = torch.empty_like(x) + + infini.ops.clover_block_conv1d(x, state, indices, weight, right_weight, + valid, block_size, out_n, + stream=stream, implementation_index=NATIVE) + infini.ops.clover_block_conv1d(x, state, indices, weight, right_weight, + valid, block_size, out_t, + stream=stream, implementation_index=TRITON) + torch.cuda.synchronize() + + golden = torch_golden(x, state, indices, weight, right_weight, valid, + block_size).to(device) + torch.testing.assert_close(out_n.float(), golden, rtol=rtol, atol=rtol) + torch.testing.assert_close(out_n.float(), out_t.float(), rtol=rtol, atol=rtol) diff --git a/tests/test_clover_live_mask.py b/tests/test_clover_live_mask.py new file mode 100644 index 000000000..7a259ea07 --- /dev/null +++ b/tests/test_clover_live_mask.py @@ -0,0 +1,161 @@ +"""Correctness tests for the shared-layer native Clover metadata operators. + +Providers: native SIMT (impl 0, this PR) cross-checked against the Triton +reference (impl 10) on identical inputs, plus explicit torch golden models. +The first three ops share the `clover_live_rows` semantics; CloverCoreset +recomputes the Triton three-stage pipeline in torch. +""" + +import infini.ops +import pytest +import torch + +if not hasattr(infini.ops, "CloverLiveMask"): + pytest.skip("`CloverLiveMask` is not available", allow_module_level=True) + +NATIVE, TRITON = 0, 10 + + +def _impls_ok(op): + impls = getattr(infini.ops, op).active_implementation_indices("nvidia") + return NATIVE in impls and TRITON in impls + + +def test_live_mask(device): + if device != "cuda" or not _impls_ok("CloverLiveMask"): + pytest.skip("requires CUDA with native and triton providers") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(3) + mask_id, n = 151669, 4096 + ids = torch.randint(0, 200, (n,), device=device, dtype=torch.int32) + ids[::7] = mask_id + previous = torch.randint(0, 200, (n,), device=device, dtype=torch.int32) + previous[::5] = mask_id + live = torch.empty(n, device=device, dtype=torch.int8) + + outs = {} + for impl, prev in ((NATIVE, previous.clone()), (TRITON, previous.clone())): + infini.ops.clover_live_mask(ids, prev, mask_id, False, live, + stream=stream, implementation_index=impl) + outs[impl] = (live.clone(), prev.clone()) + + ids_now_mask = ids == mask_id + prev_was_mask = previous == mask_id + golden_live = (ids_now_mask | prev_was_mask).to(torch.int8) + torch.testing.assert_close(outs[NATIVE][0], golden_live) + torch.testing.assert_close(outs[NATIVE][1], ids) # snapshot updated + torch.testing.assert_close(outs[NATIVE][0], outs[TRITON][0]) + torch.testing.assert_close(outs[NATIVE][1], outs[TRITON][1]) + + # refresh=True: everything live regardless of history. + for impl, prev in ((NATIVE, previous.clone()), (TRITON, previous.clone())): + infini.ops.clover_live_mask(ids, prev, mask_id, True, live, + stream=stream, implementation_index=impl) + assert torch.count_nonzero(live) == n, f"impl {impl} refresh" + + +def test_mask_routes(device): + if device != "cuda" or not _impls_ok("CloverMaskRoutes"): + pytest.skip("requires CUDA with native and triton providers") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(4) + rows, topk, experts = 512, 8, 256 + ids = torch.randint(-1, experts, (rows, topk), device=device, dtype=torch.int32) + live = (torch.rand(rows, device=device) < 0.3).to(torch.int8) + out_n = torch.empty_like(ids) + out_t = torch.empty_like(ids) + infini.ops.clover_mask_routes(ids, live, experts, out_n, + stream=stream, implementation_index=NATIVE) + infini.ops.clover_mask_routes(ids, live, experts, out_t, + stream=stream, implementation_index=TRITON) + + golden = torch.where(live.unsqueeze(1) != 0, ids, + torch.full_like(ids, experts)) + torch.testing.assert_close(out_n, golden) + torch.testing.assert_close(out_n, out_t) + + +def test_merge_moe(device): + if device != "cuda" or not _impls_ok("CloverMergeMoe"): + pytest.skip("requires CUDA with native and triton providers") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(5) + rows, width = 256, 320 + fresh = torch.randn(rows, width, device=device, dtype=torch.bfloat16) + cache0 = torch.randn(rows, width, device=device, dtype=torch.bfloat16) + live = (torch.rand(rows, device=device) < 0.4).to(torch.int8) + out = torch.empty_like(fresh) + + cache_n, cache_t = cache0.clone(), cache0.clone() + infini.ops.clover_merge_moe(fresh, live, cache_n, out, + stream=stream, implementation_index=NATIVE) + out_n = out.clone() + infini.ops.clover_merge_moe(fresh, live, cache_t, out, + stream=stream, implementation_index=TRITON) + + pick = torch.where(live.unsqueeze(1) != 0, fresh, cache0) + torch.testing.assert_close(out_n, pick) + torch.testing.assert_close(cache_n, pick) # cache also updated + torch.testing.assert_close(out_n, out) + torch.testing.assert_close(cache_n, cache_t) + + +def torch_coreset(logits, width, top_p): + """Golden model replicating the Triton three-stage pipeline.""" + tokens, experts = logits.shape + assert experts == 256 + blocks = tokens // width + out = torch.empty_like(logits) + for b in range(blocks): + block = logits[b * width:(b + 1) * width].float() + prob = torch.softmax(block, dim=1) + # top-8 union per row + sel = torch.zeros_like(prob, dtype=torch.bool) + rem = block.clone() + for _ in range(8): + best = rem.argmax(dim=1) + sel.scatter_(1, best.unsqueeze(1), True) + rem.scatter_(1, best.unsqueeze(1), float("-inf")) + mass = (prob * sel).sum(0) + order = torch.argsort(mass, descending=True, stable=True) + p = mass[order] / max(mass.sum(), 1e-9) + keep_cum = (p.cumsum(0) - p) < top_p + keep = torch.zeros_like(keep_cum) + keep[order[keep_cum]] = True + out[b * width:(b + 1) * width] = torch.where( + keep.unsqueeze(0), logits[b * width:(b + 1) * width], + torch.full_like(logits[b * width:(b + 1) * width], float("-inf"))) + return out + + +def test_coreset(device): + if device != "cuda" or not _impls_ok("CloverCoreset"): + pytest.skip("requires CUDA with native and triton providers") + import importlib.util + import pathlib + import sys + adapter_path = ( + pathlib.Path(__file__).resolve().parents[1] + / "build" / "moe-simt-tests" / "clover_native_adapter.so" + ) + if not adapter_path.exists(): + pytest.skip("clover_native_adapter is not built; see tests/native/") + spec = importlib.util.spec_from_file_location("clover_native_adapter", adapter_path) + adapter = importlib.util.module_from_spec(spec) + sys.modules["clover_native_adapter"] = adapter + spec.loader.exec_module(adapter) + + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(6) + width = 32 + for blocks, top_p in ((4, 0.9), (2, 0.5), (3, 1.0)): + logits = torch.randn(blocks * width, 256, device=device, dtype=torch.float32) + out_n = torch.empty_like(logits) + out_t = torch.empty_like(logits) + scratch = torch.empty(blocks * 256 * 8, device=device, dtype=torch.uint8) + # Workspace-taking providers are fed via the pybind adapter. + adapter.coreset(logits, width, top_p, out_n, scratch, stream, NATIVE) + adapter.coreset(logits, width, top_p, out_t, scratch, stream, TRITON) + golden = torch_coreset(logits.float().cpu(), width, top_p).to(logits.device) + torch.testing.assert_close(out_n, golden, equal_nan=True) + torch.testing.assert_close(out_n, out_t, equal_nan=True) diff --git a/tests/test_clover_live_rows.py b/tests/test_clover_live_rows.py new file mode 100644 index 000000000..2abe04381 --- /dev/null +++ b/tests/test_clover_live_rows.py @@ -0,0 +1,30 @@ +import infini.ops as ops +import pytest +import torch + + +def test_clover_live_rows(): + if not torch.cuda.is_available() or not torch.version.hip: + pytest.skip("requires Hygon") + if not hasattr(ops, "CloverLiveMask"): + pytest.skip("requires Clover providers") + s = torch.cuda.current_stream().cuda_stream + ids = torch.tensor([151669, 3, 4, 151669], device="cuda", dtype=torch.int64) + previous = torch.empty_like(ids) + live = torch.empty(4, device="cuda", dtype=torch.int8) + ops.clover_live_mask(ids, previous, 151669, True, live, stream=s) + assert live.tolist() == [1, 1, 1, 1] + ids[:] = torch.tensor([5, 3, 4, 151669], device="cuda") + ops.clover_live_mask(ids, previous, 151669, False, live, stream=s) + assert live.tolist() == [1, 0, 0, 1] + routes = torch.arange(16, device="cuda", dtype=torch.int32).view(4, 4) + ops.clover_mask_routes(routes, live, 256, routes, stream=s) + assert routes[1:3].eq(256).all() + fresh = torch.randn(4, 128, device="cuda", dtype=torch.bfloat16) + cache = torch.randn_like(fresh) + old = cache.clone() + out = torch.empty_like(fresh) + ops.clover_merge_moe(fresh, live, cache, out, stream=s) + ref = torch.where(live.bool()[:, None], fresh, old) + torch.testing.assert_close(out, ref, rtol=0, atol=0) + assert torch.equal(cache, ref) diff --git a/tests/test_clover_local_memory.py b/tests/test_clover_local_memory.py new file mode 100644 index 000000000..b518bdb7c --- /dev/null +++ b/tests/test_clover_local_memory.py @@ -0,0 +1,106 @@ +"""Correctness tests for the native CloverLocalMemory provider. + +Golden model replicates the Triton reference semantics: L2-normalized q/k, +block-local softmax with the gate-decayed clean-state history bucket +(history logit = bias + c * log(valid_count / block_size)), per-head +scalars, dead slots and per-row validity. Cross-checked against Triton. +""" + +import infini.ops +import pytest +import torch + +if not hasattr(infini.ops, "CloverLocalMemory"): + pytest.skip("`CloverLocalMemory` is not available", allow_module_level=True) + +NATIVE, TRITON = 0, 10 + + +def golden(q, k, v, g, state, indices, valid, ls, hb, hc, block, scale=1 / 128**0.5): + _, tokens, nqh, kd = q.shape + _, _, nvh, vd = v.shape + nkh = k.shape[2] + requests = indices.numel() + out = torch.zeros_like(v, dtype=torch.float32) + qf, kf, vf = q.float(), k.float(), v.float() + for r in range(requests): + slot = int(indices[r]) + ra = 0 <= slot < state.size(0) + ss = max(slot, 0) + for h in range(nvh): + qh = h // (nvh // nqh) + kh = h // (nvh // nkh) + blk_q = qf[0, r * block:(r + 1) * block, qh] # [T,K] + blk_k = kf[0, r * block:(r + 1) * block, kh] # [T,K] + blk_v = vf[0, r * block:(r + 1) * block, h] # [T,V] + qn = torch.nn.functional.normalize(blk_q, dim=1, eps=1e-6**0.5 * 1e-3) + qn = blk_q / (blk_q.pow(2).sum(-1, keepdim=True) + 1e-6).sqrt() + kn = blk_k / (blk_k.pow(2).sum(-1, keepdim=True) + 1e-6).sqrt() + scores = (qn @ kn.T) * float(ls[h]) + local_valid = torch.tensor( + [1.0 if (ra and int(valid[r * block + j]) != 0) else 0.0 + for j in range(block)], device=q.device) + logits = scores.masked_fill(local_valid[None, :] == 0, float("-inf")) + vbs = max(local_valid.sum().item(), 1.0) + bias = float(hb[h]) + float(hc[h]) * torch.log( + torch.tensor(vbs / block, device=q.device)) + hist = qn @ state[ss, h].float() # [T,V] + row_max = torch.maximum(logits.max(-1).values, + torch.full_like(logits.max(-1).values, bias)) + el = torch.exp(logits - row_max[:, None]) + eh = torch.exp(bias - row_max) + denom = el.sum(-1) + eh + w = el / denom[:, None] + hw = eh / denom + gate = scale * torch.exp(g[0, r * block:(r + 1) * block, h].float()) + local = w @ blk_v + o = hist * (gate * hw)[:, None] + local + active = torch.tensor( + [1.0 if (ra and int(valid[r * block + t]) != 0) else 0.0 + for t in range(block)], device=q.device) + out[0, r * block:(r + 1) * block, h] = o * active[:, None] + return out + + +@pytest.mark.parametrize("block", (8, 32)) +@pytest.mark.parametrize("dtype", (torch.bfloat16, torch.float16)) +@pytest.mark.parametrize("heads", ((4, 4, 8), (2, 2, 2))) # (nq, nk, nv) +def test_local_memory(device, block, dtype, heads): + if device != "cuda": + pytest.skip("requires the CUDA backend") + impls = infini.ops.CloverLocalMemory.active_implementation_indices("nvidia") + if NATIVE not in impls or TRITON not in impls: + pytest.skip(f"requires impls 0 and 10, active={impls}") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(41) + requests, slots, dim = 3, 4, 128 + nqh, nkh, nvh = heads + tokens = requests * block + + q = torch.randn(1, tokens, nqh, dim, device=device, dtype=dtype) * 0.5 + k = torch.randn(1, tokens, nkh, dim, device=device, dtype=dtype) * 0.5 + v = torch.randn(1, tokens, nvh, dim, device=device, dtype=dtype) * 0.5 + g = torch.randn(1, tokens, nvh, device=device, dtype=torch.float32) * 0.1 - 1.0 + state = torch.randn(slots, nvh, dim, dim, device=device, dtype=torch.float32) * 0.1 + indices = torch.tensor([0, 1, -1], device=device, dtype=torch.int32) + valid = torch.ones(tokens, device=device, dtype=torch.int8) + valid[::4] = 0 # scattered inactive rows + ls = torch.randn(nvh, device=device, dtype=dtype) * 0.3 + 1.0 + hb = torch.randn(nvh, device=device, dtype=dtype) * 0.2 + hc = torch.randn(nvh, device=device, dtype=dtype) * 0.1 + out_n = torch.empty_like(v) + out_t = torch.empty_like(v) + + infini.ops.clover_local_memory(q, k, v, g, state, indices, valid, ls, hb, hc, + block, out_n, stream=stream, + implementation_index=NATIVE) + infini.ops.clover_local_memory(q, k, v, g, state, indices, valid, ls, hb, hc, + block, out_t, stream=stream, + implementation_index=TRITON) + torch.cuda.synchronize() + + ref = golden(q, k, v, g, state, indices, valid, ls, hb, hc, block).to(v.device) + scale = max(ref.abs().max().item(), 1.0) + torch.testing.assert_close(out_n.float(), ref, rtol=3e-2, atol=scale * 3e-2) + torch.testing.assert_close(out_n.float(), out_t.float(), rtol=3e-2, + atol=scale * 3e-2) diff --git a/tests/test_clover_preview.py b/tests/test_clover_preview.py new file mode 100644 index 000000000..c249215d0 --- /dev/null +++ b/tests/test_clover_preview.py @@ -0,0 +1,126 @@ +"""Validate Clover preview outputs and read-only clean-state behavior.""" + +import infini.ops as ops +import pytest +import torch + + +@pytest.fixture(autouse=True) +def _ieee_reference(): + previous = torch.backends.fp32_precision + torch.backends.fp32_precision = "ieee" + try: + yield + finally: + torch.backends.fp32_precision = previous + + +def test_clover_preview_reference_and_state(): + if not torch.cuda.is_available() or not torch.version.hip: + pytest.skip("requires a Hygon Torch environment") + if not hasattr( + ops, "CloverLocalMemory" + ) or not ops.CloverLocalMemory.active_implementation_indices("hygon"): + pytest.skip("requires Hygon Triton Clover providers") + torch.manual_seed(17) + stream = torch.cuda.current_stream().cuda_stream + for width in (1, 8, 32): + for batch in (1, 3): + dtype = torch.bfloat16 + channels = 128 + heads = 4 + x = torch.randn(batch * width, channels, device="cuda", dtype=dtype) + state = torch.randn(batch + 1, channels, 3, device="cuda", dtype=dtype) + state0 = state.clone() + indices = torch.arange(batch, device="cuda", dtype=torch.int32) + weight = torch.randn(channels, 1, 4, device="cuda", dtype=dtype) + right = torch.randn(channels, 1, 3, device="cuda", dtype=dtype) + valid = torch.ones(batch * width, device="cuda", dtype=torch.int8) + out = torch.empty_like(x) + ops.clover_block_conv1d( + x, state, indices, weight, right, valid, width, out, stream=stream + ) + ref = [] + for b in range(batch): + row = x.view(batch, width, channels)[b] + joined = torch.cat((state[b].T, row), 0).float() + y = torch.zeros_like(row, dtype=torch.float32) + for t in range(width): + y[t] = (joined[t : t + 4] * weight[:, 0, :].T.float()).sum(0) + for j in range(1, 4): + if t + j < width: + y[t] += row[t + j].float() * right[:, 0, j - 1].float() + ref.append(torch.nn.functional.silu(y).to(dtype)) + torch.testing.assert_close( + out, torch.stack(ref).view_as(out), rtol=0.02, atol=0.04 + ) + assert torch.equal(state, state0) + q = torch.randn(1, batch * width, 2, 128, device="cuda", dtype=dtype) + k = torch.randn_like(q) + v = torch.randn(1, batch * width, heads, 128, device="cuda", dtype=dtype) + g = -torch.rand(1, batch * width, heads, device="cuda") + s = torch.randn(batch + 1, heads, 128, 128, device="cuda") + s0 = s.clone() + scale = torch.full((heads,), 10.0, device="cuda") + bias = torch.full_like(scale, 5.0) + hc = torch.ones_like(scale) + y = torch.empty_like(v) + ops.clover_local_memory( + q, k, v, g, s, indices, valid, scale, bias, hc, width, y, stream=stream + ) + qn = ( + ( + q.float() + * torch.rsqrt(q.float().square().sum(-1, keepdim=True) + 1e-6) + ) + .to(dtype) + .float() + ) + kn = ( + ( + k.float() + * torch.rsqrt(k.float().square().sum(-1, keepdim=True) + 1e-6) + ) + .to(dtype) + .float() + ) + refs = [] + for b in range(batch): + qi = ( + qn[0, b * width : (b + 1) * width] + .repeat_interleave(2, 1) + .permute(1, 0, 2) + ) + ki = ( + kn[0, b * width : (b + 1) * width] + .repeat_interleave(2, 1) + .permute(1, 2, 0) + ) + vi = v[0, b * width : (b + 1) * width].permute(1, 0, 2).float() + scores = qi @ ki * 10 + history_logit = 5 + torch.log(torch.tensor(width / 32.0)) + weights = torch.softmax( + torch.cat( + ( + scores, + torch.full((heads, width, 1), history_logit, device="cuda"), + ), + -1, + ), + -1, + ) + history = qi @ s[b].to(dtype).float().transpose(-1, -2) + local = weights[:, :, :width].to(dtype).float() @ vi + oi = ( + history + * ( + g[0, b * width : (b + 1) * width].T.exp() + * weights[:, :, -1] + / 128**0.5 + ).unsqueeze(-1) + + local + ) + refs.append(oi.permute(1, 0, 2)) + ref = torch.cat(refs).unsqueeze(0).to(dtype) + torch.testing.assert_close(y, ref, rtol=0.035, atol=0.04) + assert torch.equal(s, s0) diff --git a/tests/test_clover_recurrent_gdn.py b/tests/test_clover_recurrent_gdn.py new file mode 100644 index 000000000..4acf231c2 --- /dev/null +++ b/tests/test_clover_recurrent_gdn.py @@ -0,0 +1,125 @@ +"""Correctness tests for the native CloverRecurrentGdn provider. + +Golden model replicates the Triton reference: sequential GDN state update +per (request, head), L2-normalized q/k with the 1/sqrt(128) q scale, decay +from g, delta-rule state write, final-state copy to the final slot. +Cross-checked against the Triton provider, covering dead initial/final +slots, empty request ranges and head-count mapping. +""" + +import infini.ops +import pytest +import torch + +if not hasattr(infini.ops, "CloverRecurrentGdn"): + pytest.skip("`CloverRecurrentGdn` is not available", allow_module_level=True) + +NATIVE, TRITON = 0, 10 +QSCALE = 0.08838834764831845 + + +def golden(q, k, v, g, beta, offsets, init, fin, state): + q, k, v, g, beta = (t.float().cpu() for t in (q, k, v, g, beta)) + _, tokens, nv, dim = v.shape + nk = k.shape[2] + out = torch.zeros_like(v, dtype=torch.float32) + state_out = state.clone() + for r in range(offsets.numel() - 1): + begin, end = int(offsets[r]), int(offsets[r + 1]) + i0, i1 = int(init[r]), int(fin[r]) + if not (0 <= begin <= end <= tokens and 0 <= i0 < state.size(0) + and 0 <= i1 < state.size(0)): + continue + for h in range(nv): + kh = h // (nv // nk) + s = state[i0, h].clone() + for t in range(begin, end): + qv = q[0, t, kh].float() + kv = k[0, t, kh].float() + vv = v[0, t, h].float() + qn = qv / (qv @ qv + 1e-6).sqrt() * QSCALE + kn = kv / (kv @ kv + 1e-6).sqrt() + d = torch.exp(g[0, t, h]) + b = beta[0, t, h] + s = s * d + rem = s @ kn + s += ((vv - rem) * b)[:, None] * kn[None, :] + out[0, t, h] = s @ qn + state_out[i1, h] = s + return out, state_out + + +@pytest.mark.parametrize("dtype", (torch.bfloat16, torch.float16)) +@pytest.mark.parametrize("heads", ((4, 4), (2, 8))) # (nk, nv) +def test_recurrent_gdn(device, dtype, heads): + if device != "cuda": + pytest.skip("requires the CUDA backend") + impls = infini.ops.CloverRecurrentGdn.active_implementation_indices("nvidia") + if NATIVE not in impls or TRITON not in impls: + pytest.skip(f"requires impls 0 and 10, active={impls}") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(51) + nk, nv = heads + tokens, dim, pool = 96, 128, 4 + # Three requests: normal, touching an empty range, one with dead slots. + offsets = torch.tensor([0, 40, 40, 96], device=device, dtype=torch.int32) + init = torch.tensor([0, 1, 2], device=device, dtype=torch.int32) + fin = torch.tensor([0, 3, 2], device=device, dtype=torch.int32) + + q = torch.randn(1, tokens, nk, dim, device=device, dtype=dtype) * 0.5 + k = torch.randn(1, tokens, nk, dim, device=device, dtype=dtype) * 0.5 + v = torch.randn(1, tokens, nv, dim, device=device, dtype=dtype) * 0.5 + g = torch.randn(1, tokens, nv, device=device, dtype=torch.float32) * 0.2 - 0.5 + beta = torch.rand(1, tokens, nv, device=device, dtype=torch.float32) + state = torch.randn(pool, nv, dim, dim, device=device, + dtype=torch.float32) * 0.1 + out_n = torch.empty_like(v) + out_t = torch.empty_like(v) + + infini.ops.clover_recurrent_gdn(q, k, v, g, beta, offsets, init, fin, + state, out_n, stream=stream, + implementation_index=NATIVE) + infini.ops.clover_recurrent_gdn(q, k, v, g, beta, offsets, init, fin, + state, out_t, stream=stream, + implementation_index=TRITON) + torch.cuda.synchronize() + + ref_out, ref_state = golden(q, k, v, g, beta, offsets.cpu(), init.cpu(), + fin.cpu(), state.cpu()) + scale = max(ref_out.abs().max().item(), 1.0) + torch.testing.assert_close(out_n.float(), ref_out.to(v.device), + rtol=3e-2, atol=scale * 3e-2) + torch.testing.assert_close(out_n.float(), out_t.float(), rtol=3e-2, + atol=scale * 3e-2) + + +def test_recurrent_gdn_dead_slots(device): + """All requests have invalid initial slots: no output or state writes.""" + if device != "cuda": + pytest.skip("requires the CUDA backend") + impls = infini.ops.CloverRecurrentGdn.active_implementation_indices("nvidia") + if NATIVE not in impls or TRITON not in impls: + pytest.skip(f"requires impls 0 and 10, active={impls}") + stream = torch.cuda.current_stream().cuda_stream + torch.manual_seed(52) + dtype, nk, nv = torch.bfloat16, 2, 2 + tokens, dim, pool = 16, 128, 2 + offsets = torch.tensor([0, 16], device=device, dtype=torch.int32) + init = torch.tensor([-1], device=device, dtype=torch.int32) + fin = torch.tensor([0], device=device, dtype=torch.int32) + q = torch.randn(1, tokens, nk, dim, device=device, dtype=dtype) + k = torch.randn_like(q) + v = torch.randn(1, tokens, nv, dim, device=device, dtype=dtype) + g = torch.randn(1, tokens, nv, device=device, dtype=torch.float32) * 0.1 + beta = torch.rand(1, tokens, nv, device=device, dtype=torch.float32) + state = torch.randn(pool, nv, dim, dim, device=device, + dtype=torch.float32) * 0.1 + state0 = state.clone() + out = torch.full_like(v, float("nan")) + infini.ops.clover_recurrent_gdn(q, k, v, g, beta, offsets, init, fin, + state, out, stream=stream, + implementation_index=NATIVE) + torch.cuda.synchronize() + # Both providers leave the output untouched for dead initial slots. + assert torch.isnan(out.float()).all(), "dead slot must not write output" + assert torch.equal(state, state0) diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py index 922bc9851..3ad43e93f 100644 --- a/tests/test_generate_wrappers.py +++ b/tests/test_generate_wrappers.py @@ -1358,3 +1358,15 @@ def test_w4_public_api_selection_is_explicit(tmp_path): module._select_ops_from_config( {}, {name: {"headers": None, "implementations": (0,)}}, "ops.json" ) + + +def test_default_argument_scan_ignores_comment_examples(monkeypatch, tmp_path): + module = _load_generator_module() + header = tmp_path / "sample.h" + header.write_text( + "// bool is_neox=false is GPT-J, true is GPT-NeoX.\n" + "/* float alpha = invalid example, */\n" + "class Sample { public: Sample(bool is_neox, float alpha = 1.0f); };\n" + ) + monkeypatch.setattr(module, "_find_base_header", lambda name: header) + assert module._find_params_with_defaults("sample") == {"alpha": "1.0f"} diff --git a/tests/test_triton_jit_compile.py b/tests/test_triton_jit_compile.py index 4b9b186e0..2ecf00faf 100644 --- a/tests/test_triton_jit_compile.py +++ b/tests/test_triton_jit_compile.py @@ -360,3 +360,27 @@ def test_auto_tuning_cache_rejects_zero_compile_options(monkeypatch, tmp_path, f module.write_auto_tuning_config(identity, config) assert module.read_auto_tuning_config(identity) is None + + +def test_hip_target_keeps_gfx_architecture(monkeypatch): + module, _ = _load_compile_module(monkeypatch) + target = module._create_target( + {"backend": "hip", "architecture": "gfx936", "warp_size": 64} + ) + assert (target.backend, target.arch, target.warp_size) == ("hip", "gfx936", 64) + + +def test_hip_artifacts_without_cuda_scratch_fields(monkeypatch, tmp_path): + module, fake_triton = _load_compile_module(monkeypatch) + fake_triton.compiler.make_backend = lambda target: types.SimpleNamespace( + binary_ext="hsaco" + ) + compiled = types.SimpleNamespace( + metadata=types.SimpleNamespace(name="hip_kernel", num_warps=4, shared=0), + asm={"hsaco": b"hip-binary"}, + ) + prefix = tmp_path / "kernel" + module._write_artifacts(compiled, object(), prefix, "hip-identity") + artifact = module.read_kernel_artifact(prefix, "hip-identity") + assert artifact["binary"] == b"hip-binary" + assert artifact["global_scratch_size"] == artifact["profile_scratch_size"] == 0