From 75f4feece70b05cc15d004a92ace50061e2d0e4b Mon Sep 17 00:00:00 2001 From: rubik Date: Tue, 10 Nov 2026 17:34:52 +0800 Subject: [PATCH] feat(metax): add MoE, GDN, quantization and attention operators for MetaX GPUs Add native MetaX (MACA) device kernels and linked torch operator providers so InfiniLM can run on MetaX C550 GPUs: - Native ops (src/native/cuda/metax): moe_fused_gate, moe_fused_dense, deepseek_moe, chunked/recurrent gated delta rule (linear attention), mamba_selective_scan, causal_conv1d, w4_marlin, per-tensor/per-channel int8 quant kernels - Linked torch ops (src/linked/torch/metax): vLLM providers for moe_align_block_size, topk_sigmoid, quick_gelu; ATen provider for pad; flash_attn_varlen_func, rms_norm and silu_and_mul updates - Shared CUDA reference kernels for moe_fused_dense (src/native/cuda/ops) - Metax specializations of argmax/layer_norm/mrope include caster.cuh so TypeMap is defined and the kernels compile for MetaX Validated end-to-end with qwen3.8-27B (TP2, batch size 1) on C550 / MACA 3.5.3: TTFT 285.01 ms, decode 33.54 tok/s. --- src/linked/torch/metax/mcoplib.yaml | 2 + src/linked/torch/metax/mcoplib_moe.yaml | 2 + .../ops/flash_attn_varlen_func/flash_attn.cc | 5 +- .../flash_attn_varlen_func/flash_attn.yaml | 2 +- .../metax/ops/moe_align_block_size/vllm.cc | 8 + .../metax/ops/moe_align_block_size/vllm.h | 66 ++++ .../metax/ops/moe_align_block_size/vllm.yaml | 5 + src/linked/torch/metax/ops/pad/aten.cc | 7 + src/linked/torch/metax/ops/pad/aten.h | 53 +++ src/linked/torch/metax/ops/pad/aten.yaml | 4 + src/linked/torch/metax/ops/quick_gelu/vllm.cc | 19 + src/linked/torch/metax/ops/quick_gelu/vllm.h | 37 ++ .../torch/metax/ops/quick_gelu/vllm.yaml | 3 + src/linked/torch/metax/ops/rms_norm/vllm.yaml | 2 +- .../torch/metax/ops/silu_and_mul/vllm.yaml | 2 +- .../torch/metax/ops/topk_sigmoid/vllm.cc | 50 +++ .../torch/metax/ops/topk_sigmoid/vllm.h | 43 +++ .../torch/metax/ops/topk_sigmoid/vllm.yaml | 6 + src/linked/torch/metax/vllm.yaml.disabled | 2 + src/linked/torch/ops/moe_align_block_size.h | 70 ++++ src/linked/torch/ops/pad.h | 56 +++ src/linked/torch/ops/quick_gelu.h | 59 +++ src/native/cuda/device_buffer.h | 7 +- src/native/cuda/metax/ops/argmax/kernel.h | 1 + .../cuda/metax/ops/causal_conv1d/kernel.h | 196 ++++++++++ .../metax/ops/chunk_gated_delta_rule/kernel.h | 24 ++ .../ops/fused_gated_delta_net_gating/kernel.h | 115 ++++++ src/native/cuda/metax/ops/layer_norm/kernel.h | 1 + .../metax/ops/mamba_selective_scan/kernel.h | 15 + .../cuda/metax/ops/moe_fused_dense/kernel.h | 23 ++ src/native/cuda/metax/ops/mrope/kernel.h | 1 + .../cuda/metax/ops/per_tensor_i8/kernel.h | 242 ++++++++++++ .../ops/recurrent_gated_delta_rule/kernel.h | 24 ++ .../ops/chunk_gated_delta_rule/kernel.h | 145 +------- .../ops/recurrent_gated_delta_rule/kernel.h | 125 +------ .../chunk_gated_delta_rule_legacy_kernel.cuh | 11 +- .../cuda/ops/chunk_gated_delta_rule/kernel.h | 213 +++++++++++ .../cuda/ops/moe_fused_dense/kernel.cuh | 158 ++++++++ src/native/cuda/ops/moe_fused_dense/kernel.h | 350 ++++++++++++++++++ .../ops/recurrent_gated_delta_rule/kernel.h | 172 +++++++++ .../recurrent_delta_rule_common.cuh | 9 +- ...current_gated_delta_rule_legacy_kernel.cuh | 27 +- tests/test_chunk_gated_delta_rule.py | 204 ++++++++++ tests/test_moe_fused_dense.py | 138 +++++++ tests/test_per_tensor_quant_i8.py | 95 +++++ tests/test_recurrent_gated_delta_rule.py | 182 +++++++++ 46 files changed, 2718 insertions(+), 263 deletions(-) create mode 100644 src/linked/torch/metax/mcoplib.yaml create mode 100644 src/linked/torch/metax/mcoplib_moe.yaml create mode 100644 src/linked/torch/metax/ops/moe_align_block_size/vllm.cc create mode 100644 src/linked/torch/metax/ops/moe_align_block_size/vllm.h create mode 100644 src/linked/torch/metax/ops/moe_align_block_size/vllm.yaml create mode 100644 src/linked/torch/metax/ops/pad/aten.cc create mode 100644 src/linked/torch/metax/ops/pad/aten.h create mode 100644 src/linked/torch/metax/ops/pad/aten.yaml create mode 100644 src/linked/torch/metax/ops/quick_gelu/vllm.cc create mode 100644 src/linked/torch/metax/ops/quick_gelu/vllm.h create mode 100644 src/linked/torch/metax/ops/quick_gelu/vllm.yaml create mode 100644 src/linked/torch/metax/ops/topk_sigmoid/vllm.cc create mode 100644 src/linked/torch/metax/ops/topk_sigmoid/vllm.h create mode 100644 src/linked/torch/metax/ops/topk_sigmoid/vllm.yaml create mode 100644 src/linked/torch/metax/vllm.yaml.disabled create mode 100644 src/linked/torch/ops/moe_align_block_size.h create mode 100644 src/linked/torch/ops/pad.h create mode 100644 src/linked/torch/ops/quick_gelu.h create mode 100644 src/native/cuda/metax/ops/causal_conv1d/kernel.h create mode 100644 src/native/cuda/metax/ops/chunk_gated_delta_rule/kernel.h create mode 100644 src/native/cuda/metax/ops/fused_gated_delta_net_gating/kernel.h create mode 100644 src/native/cuda/metax/ops/mamba_selective_scan/kernel.h create mode 100644 src/native/cuda/metax/ops/moe_fused_dense/kernel.h create mode 100644 src/native/cuda/metax/ops/per_tensor_i8/kernel.h create mode 100644 src/native/cuda/metax/ops/recurrent_gated_delta_rule/kernel.h rename src/native/cuda/{nvidia => }/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh (97%) create mode 100644 src/native/cuda/ops/chunk_gated_delta_rule/kernel.h create mode 100644 src/native/cuda/ops/moe_fused_dense/kernel.cuh create mode 100644 src/native/cuda/ops/moe_fused_dense/kernel.h create mode 100644 src/native/cuda/ops/recurrent_gated_delta_rule/kernel.h rename src/native/cuda/{nvidia => }/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh (97%) rename src/native/cuda/{nvidia => }/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh (76%) create mode 100644 tests/test_chunk_gated_delta_rule.py create mode 100644 tests/test_moe_fused_dense.py create mode 100644 tests/test_per_tensor_quant_i8.py create mode 100644 tests/test_recurrent_gated_delta_rule.py diff --git a/src/linked/torch/metax/mcoplib.yaml b/src/linked/torch/metax/mcoplib.yaml new file mode 100644 index 000000000..4247fea78 --- /dev/null +++ b/src/linked/torch/metax/mcoplib.yaml @@ -0,0 +1,2 @@ +python_distribution_package: mcoplib +library_glob: mcoplib/_C*.so diff --git a/src/linked/torch/metax/mcoplib_moe.yaml b/src/linked/torch/metax/mcoplib_moe.yaml new file mode 100644 index 000000000..c9f6a4203 --- /dev/null +++ b/src/linked/torch/metax/mcoplib_moe.yaml @@ -0,0 +1,2 @@ +python_distribution_package: mcoplib +library_glob: mcoplib/_moe_C*.so diff --git a/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.cc b/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.cc index 27b62f01c..32ca75de9 100644 --- a/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.cc +++ b/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.cc @@ -15,7 +15,7 @@ std::vector mha_varlen_fwd( float dropout_p, float softmax_scale, bool zero_tensors, bool causal, int window_size_left, int window_size_right, float softcap, bool return_softmax, std::optional generator, - std::optional& flash_attn_mars_ext); + std::optional& flash_attn_mars_ext, bool return_max_logit); namespace infini::ops::linked::torch::metax { @@ -36,7 +36,8 @@ struct FlashAttnVarlen : C10 { max_seqlen_k, dropout_p, softmax_scale, zero_tensors, causal, window_size_left, window_size_right, softcap, return_softmax, - generator, flash_attn_mars_ext); + generator, flash_attn_mars_ext, + /*return_max_logit=*/false); } }; diff --git a/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.yaml b/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.yaml index c692e1abd..874420dc4 100644 --- a/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.yaml +++ b/src/linked/torch/metax/ops/flash_attn_varlen_func/flash_attn.yaml @@ -1,4 +1,4 @@ library: flash_attn required_symbols: - >- - mha_varlen_fwd(at::Tensor&, at::Tensor const&, at::Tensor const&, std::optional&, at::Tensor const&, at::Tensor const&, std::optional&, std::optional&, std::optional&, std::optional&, int, int, float, float, bool, bool, int, int, float, bool, std::optional, std::optional&) + mha_varlen_fwd(at::Tensor&, at::Tensor const&, at::Tensor const&, std::optional&, at::Tensor const&, at::Tensor const&, std::optional&, std::optional&, std::optional&, std::optional&, int, int, float, float, bool, bool, int, int, float, bool, std::optional, std::optional&, bool) diff --git a/src/linked/torch/metax/ops/moe_align_block_size/vllm.cc b/src/linked/torch/metax/ops/moe_align_block_size/vllm.cc new file mode 100644 index 000000000..4d2f7172c --- /dev/null +++ b/src/linked/torch/metax/ops/moe_align_block_size/vllm.cc @@ -0,0 +1,8 @@ +#include "linked/torch/metax/ops/moe_align_block_size/vllm.h" + +namespace infini::ops::linked::torch { + +template class TorchMoeAlignBlockSize< + ::infini::ops::linked::torch::metax::VllmMoeAlignBlockSize>; + +} // namespace infini::ops::linked::torch diff --git a/src/linked/torch/metax/ops/moe_align_block_size/vllm.h b/src/linked/torch/metax/ops/moe_align_block_size/vllm.h new file mode 100644 index 000000000..2fb7ac468 --- /dev/null +++ b/src/linked/torch/metax/ops/moe_align_block_size/vllm.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_ +#define INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_ + +#include +#include + +#include +#include + +#include "linked/torch/ops/moe_align_block_size.h" +#include "torch/metax/c10.h" + +namespace infini::ops::linked::torch::metax { + +struct VllmMoeAlignBlockSize : C10 { + static void Call(at::Tensor topk_ids, int64_t num_experts, int64_t block_size, + at::Tensor sorted_token_ids, at::Tensor experts_ids, + at::Tensor num_tokens_post_pad, + std::optional maybe_expert_map) { + static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow( + "_moe_C::moe_align_block_size", ""); + c10::Stack stack; + stack.reserve(7); + stack.emplace_back(std::move(topk_ids)); + stack.emplace_back(num_experts); + stack.emplace_back(block_size); + stack.emplace_back(std::move(sorted_token_ids)); + stack.emplace_back(std::move(experts_ids)); + stack.emplace_back(std::move(num_tokens_post_pad)); + if (maybe_expert_map.has_value()) { + stack.emplace_back(std::move(*maybe_expert_map)); + } else { + stack.emplace_back(); + } + op.callBoxed(&stack); + + assert(stack.empty() && + "`moe_align_block_size` returned an unexpected number of values"); + } +}; + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +extern template class TorchMoeAlignBlockSize< + ::infini::ops::linked::torch::metax::VllmMoeAlignBlockSize>; + +} // namespace infini::ops::linked::torch + +namespace infini::ops { + +template <> +class Operator + : public linked::torch::TorchMoeAlignBlockSize< + linked::torch::metax::VllmMoeAlignBlockSize> { + public: + using linked::torch::TorchMoeAlignBlockSize< + linked::torch::metax::VllmMoeAlignBlockSize>::TorchMoeAlignBlockSize; + using linked::torch::TorchMoeAlignBlockSize< + linked::torch::metax::VllmMoeAlignBlockSize>::operator(); +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_ diff --git a/src/linked/torch/metax/ops/moe_align_block_size/vllm.yaml b/src/linked/torch/metax/ops/moe_align_block_size/vllm.yaml new file mode 100644 index 000000000..d94b147a6 --- /dev/null +++ b/src/linked/torch/metax/ops/moe_align_block_size/vllm.yaml @@ -0,0 +1,5 @@ +library: mcoplib_moe +required_symbols: + - >- + moe_align_block_size(at::Tensor, long, long, at::Tensor, at::Tensor, + at::Tensor, std::optional) diff --git a/src/linked/torch/metax/ops/pad/aten.cc b/src/linked/torch/metax/ops/pad/aten.cc new file mode 100644 index 000000000..1426d3163 --- /dev/null +++ b/src/linked/torch/metax/ops/pad/aten.cc @@ -0,0 +1,7 @@ +#include "linked/torch/metax/ops/pad/aten.h" + +namespace infini::ops::linked::torch { + +template class TorchPad<::infini::ops::linked::torch::metax::AtenPad>; + +} // namespace infini::ops::linked::torch diff --git a/src/linked/torch/metax/ops/pad/aten.h b/src/linked/torch/metax/ops/pad/aten.h new file mode 100644 index 000000000..c2cd94789 --- /dev/null +++ b/src/linked/torch/metax/ops/pad/aten.h @@ -0,0 +1,53 @@ +#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_ +#define INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_ + +#include +#include + +#include +#include +#include + +#include "linked/torch/ops/pad.h" +#include "torch/metax/c10.h" + +namespace infini::ops::linked::torch::metax { + +struct AtenPad : C10 { + static void Call(at::Tensor out, at::Tensor input, std::vector pad, + double value) { + static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow( + "aten::constant_pad_nd", ""); + c10::Stack stack; + stack.reserve(4); + stack.emplace_back(std::move(input)); + stack.emplace_back(c10::List(pad)); + stack.emplace_back(value); + op.callBoxed(&stack); + TORCH_CHECK(stack.size() == 1, + "`constant_pad_nd` returned unexpected values."); + out.copy_(stack.front().toTensor()); + } +}; + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +extern template class TorchPad< ::infini::ops::linked::torch::metax::AtenPad>; + +} // namespace infini::ops::linked::torch + +namespace infini::ops { + +template <> +class Operator + : public linked::torch::TorchPad { + public: + using linked::torch::TorchPad::TorchPad; + using linked::torch::TorchPad::operator(); +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_ diff --git a/src/linked/torch/metax/ops/pad/aten.yaml b/src/linked/torch/metax/ops/pad/aten.yaml new file mode 100644 index 000000000..6da0c6054 --- /dev/null +++ b/src/linked/torch/metax/ops/pad/aten.yaml @@ -0,0 +1,4 @@ +library: mcoplib +operator_schema: >- + aten::constant_pad_nd(Tensor self, SymInt[] pad, Scalar value=0) -> Tensor +dispatch_key: CompositeExplicitAutograd diff --git a/src/linked/torch/metax/ops/quick_gelu/vllm.cc b/src/linked/torch/metax/ops/quick_gelu/vllm.cc new file mode 100644 index 000000000..4b4e19a5b --- /dev/null +++ b/src/linked/torch/metax/ops/quick_gelu/vllm.cc @@ -0,0 +1,19 @@ +#include "linked/torch/metax/ops/quick_gelu/vllm.h" + +// MetaX vLLM `_C` exports this global C++ symbol. +void gelu_quick(at::Tensor& out, at::Tensor& input); + +namespace infini::ops::linked::torch::metax { + +void VllmQuickGelu::Call(at::Tensor& out, at::Tensor& input) { + ::gelu_quick(out, input); +} + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +template class TorchQuickGelu< + ::infini::ops::linked::torch::metax::VllmQuickGelu>; + +} // namespace infini::ops::linked::torch diff --git a/src/linked/torch/metax/ops/quick_gelu/vllm.h b/src/linked/torch/metax/ops/quick_gelu/vllm.h new file mode 100644 index 000000000..ae3c4ddbc --- /dev/null +++ b/src/linked/torch/metax/ops/quick_gelu/vllm.h @@ -0,0 +1,37 @@ +#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_ +#define INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_ + +#include "linked/torch/ops/quick_gelu.h" +#include "torch/metax/c10.h" + +namespace infini::ops::linked::torch::metax { + +struct VllmQuickGelu : C10 { + static void Call(at::Tensor& out, at::Tensor& input); +}; + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +extern template class TorchQuickGelu< + ::infini::ops::linked::torch::metax::VllmQuickGelu>; + +} // namespace infini::ops::linked::torch + +namespace infini::ops { + +template <> +class Operator + : public linked::torch::TorchQuickGelu< + linked::torch::metax::VllmQuickGelu> { + public: + using linked::torch::TorchQuickGelu< + linked::torch::metax::VllmQuickGelu>::TorchQuickGelu; + using linked::torch::TorchQuickGelu< + linked::torch::metax::VllmQuickGelu>::operator(); +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_ diff --git a/src/linked/torch/metax/ops/quick_gelu/vllm.yaml b/src/linked/torch/metax/ops/quick_gelu/vllm.yaml new file mode 100644 index 000000000..c026ad592 --- /dev/null +++ b/src/linked/torch/metax/ops/quick_gelu/vllm.yaml @@ -0,0 +1,3 @@ +library: mcoplib +required_symbols: + - gelu_quick(at::Tensor&, at::Tensor&) diff --git a/src/linked/torch/metax/ops/rms_norm/vllm.yaml b/src/linked/torch/metax/ops/rms_norm/vllm.yaml index cfed9bbc1..40648c257 100644 --- a/src/linked/torch/metax/ops/rms_norm/vllm.yaml +++ b/src/linked/torch/metax/ops/rms_norm/vllm.yaml @@ -1,3 +1,3 @@ -library: vllm +library: mcoplib required_symbols: - rms_norm(at::Tensor&, at::Tensor&, at::Tensor&, double) diff --git a/src/linked/torch/metax/ops/silu_and_mul/vllm.yaml b/src/linked/torch/metax/ops/silu_and_mul/vllm.yaml index 56fafbd6e..dd2d6616d 100644 --- a/src/linked/torch/metax/ops/silu_and_mul/vllm.yaml +++ b/src/linked/torch/metax/ops/silu_and_mul/vllm.yaml @@ -1,3 +1,3 @@ -library: vllm +library: mcoplib required_symbols: - silu_and_mul(at::Tensor&, at::Tensor&) diff --git a/src/linked/torch/metax/ops/topk_sigmoid/vllm.cc b/src/linked/torch/metax/ops/topk_sigmoid/vllm.cc new file mode 100644 index 000000000..eb8aaa1df --- /dev/null +++ b/src/linked/torch/metax/ops/topk_sigmoid/vllm.cc @@ -0,0 +1,50 @@ +#include "linked/torch/metax/ops/topk_sigmoid/vllm.h" + +#include +#include +#include + +#include + +namespace infini::ops::linked::torch::metax { + +void VllmTopkSigmoid::Validate(bool has_is_padding, + double routed_scaling_factor) { + TORCH_CHECK(!has_is_padding, + "Linked `topk_sigmoid` does not support `is_padding`."); + TORCH_CHECK(routed_scaling_factor == 1.0, + "Linked `topk_sigmoid` requires " + "`routed_scaling_factor == 1.0`."); +} + +void VllmTopkSigmoid::Call(at::Tensor topk_weights, at::Tensor topk_indices, + at::Tensor token_expert_indices, + at::Tensor gating_output, bool renormalize, + std::optional bias) { + static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow( + "_moe_C::topk_sigmoid", ""); + c10::Stack stack; + stack.emplace_back(std::move(topk_weights)); + stack.emplace_back(std::move(topk_indices)); + stack.emplace_back(std::move(token_expert_indices)); + stack.emplace_back(std::move(gating_output)); + stack.emplace_back(renormalize); + if (bias.has_value()) { + stack.emplace_back(std::move(*bias)); + } else { + stack.emplace_back(); + } + op.callBoxed(&stack); + + TORCH_CHECK(stack.empty(), + "Linked `topk_sigmoid` returned unexpected values."); +} + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +template class TorchTopkSigmoid< + ::infini::ops::linked::torch::metax::VllmTopkSigmoid>; + +} // namespace infini::ops::linked::torch diff --git a/src/linked/torch/metax/ops/topk_sigmoid/vllm.h b/src/linked/torch/metax/ops/topk_sigmoid/vllm.h new file mode 100644 index 000000000..42705a4ad --- /dev/null +++ b/src/linked/torch/metax/ops/topk_sigmoid/vllm.h @@ -0,0 +1,43 @@ +#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_ +#define INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_ + +#include + +#include "linked/torch/ops/topk_sigmoid.h" +#include "torch/metax/c10.h" + +namespace infini::ops::linked::torch::metax { + +struct VllmTopkSigmoid : C10 { + static void Validate(bool has_is_padding, double routed_scaling_factor); + + static void Call(at::Tensor topk_weights, at::Tensor topk_indices, + at::Tensor token_expert_indices, at::Tensor gating_output, + bool renormalize, std::optional bias); +}; + +} // namespace infini::ops::linked::torch::metax + +namespace infini::ops::linked::torch { + +extern template class TorchTopkSigmoid< + ::infini::ops::linked::torch::metax::VllmTopkSigmoid>; + +} // namespace infini::ops::linked::torch + +namespace infini::ops { + +template <> +class Operator + : public linked::torch::TorchTopkSigmoid< + linked::torch::metax::VllmTopkSigmoid> { + public: + using linked::torch::TorchTopkSigmoid< + linked::torch::metax::VllmTopkSigmoid>::TorchTopkSigmoid; + using linked::torch::TorchTopkSigmoid< + linked::torch::metax::VllmTopkSigmoid>::operator(); +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_ diff --git a/src/linked/torch/metax/ops/topk_sigmoid/vllm.yaml b/src/linked/torch/metax/ops/topk_sigmoid/vllm.yaml new file mode 100644 index 000000000..70d5f22c6 --- /dev/null +++ b/src/linked/torch/metax/ops/topk_sigmoid/vllm.yaml @@ -0,0 +1,6 @@ +library: mcoplib_moe +operator_schema: >- + _moe_C::topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices, + Tensor! token_expert_indices, Tensor gating_output, bool renormalize, + Tensor? bias) -> () +dispatch_key: CUDA diff --git a/src/linked/torch/metax/vllm.yaml.disabled b/src/linked/torch/metax/vllm.yaml.disabled new file mode 100644 index 000000000..7f9882f27 --- /dev/null +++ b/src/linked/torch/metax/vllm.yaml.disabled @@ -0,0 +1,2 @@ +python_distribution_package: vllm +library_glob: vllm/_C*.so diff --git a/src/linked/torch/ops/moe_align_block_size.h b/src/linked/torch/ops/moe_align_block_size.h new file mode 100644 index 000000000..f6e705f0a --- /dev/null +++ b/src/linked/torch/ops/moe_align_block_size.h @@ -0,0 +1,70 @@ +#ifndef INFINI_OPS_LINKED_TORCH_OPS_MOE_ALIGN_BLOCK_SIZE_H_ +#define INFINI_OPS_LINKED_TORCH_OPS_MOE_ALIGN_BLOCK_SIZE_H_ + +#include + +#include "base/moe_align_block_size.h" +#include "torch/tensor_.h" + +namespace infini::ops::linked::torch { + +template +class TorchMoeAlignBlockSize : public ::infini::ops::MoeAlignBlockSize { + public: + TorchMoeAlignBlockSize(const Tensor topk_ids, + const std::optional expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, + Tensor num_tokens_post_pad) + : ::infini::ops::MoeAlignBlockSize{topk_ids, expert_map, + num_experts, block_size, + sorted_token_ids, experts_ids, + num_tokens_post_pad}, + device_index_{topk_ids.device().index()} {} + + using ::infini::ops::MoeAlignBlockSize::operator(); + + void Run(const Tensor topk_ids, const std::optional expert_map, + const int64_t num_experts, const int64_t block_size, + Tensor sorted_token_ids, Tensor experts_ids, + Tensor num_tokens_post_pad) const override { + const typename Backend::StreamGuard stream_guard{ + Backend::GetStreamFromExternal(stream_, device_index_)}; + + auto at_topk_ids = ToAtenTensor( + const_cast(topk_ids.data()), topk_ids_metadata_.shape(), + topk_ids_metadata_.strides(), topk_ids_metadata_.dtype(), + device_index_); + auto at_sorted_token_ids = ToAtenTensor( + sorted_token_ids.data(), sorted_token_ids_metadata_.shape(), + sorted_token_ids_metadata_.strides(), + sorted_token_ids_metadata_.dtype(), device_index_); + auto at_experts_ids = ToAtenTensor( + experts_ids.data(), experts_ids_metadata_.shape(), + experts_ids_metadata_.strides(), experts_ids_metadata_.dtype(), + device_index_); + auto at_num_tokens_post_pad = ToAtenTensor( + num_tokens_post_pad.data(), num_tokens_post_pad_metadata_.shape(), + num_tokens_post_pad_metadata_.strides(), + num_tokens_post_pad_metadata_.dtype(), device_index_); + std::optional at_expert_map; + if (expert_map.has_value()) { + at_expert_map.emplace(ToAtenTensor( + const_cast(expert_map->data()), expert_map_metadata_->shape(), + expert_map_metadata_->strides(), expert_map_metadata_->dtype(), + device_index_)); + } + + Backend::Call(std::move(at_topk_ids), static_cast(num_experts_), + static_cast(block_size_), + std::move(at_sorted_token_ids), std::move(at_experts_ids), + std::move(at_num_tokens_post_pad), std::move(at_expert_map)); + } + + private: + int device_index_{0}; +}; + +} // namespace infini::ops::linked::torch + +#endif // INFINI_OPS_LINKED_TORCH_OPS_MOE_ALIGN_BLOCK_SIZE_H_ diff --git a/src/linked/torch/ops/pad.h b/src/linked/torch/ops/pad.h new file mode 100644 index 000000000..893eb06d6 --- /dev/null +++ b/src/linked/torch/ops/pad.h @@ -0,0 +1,56 @@ +#ifndef INFINI_OPS_LINKED_TORCH_OPS_PAD_H_ +#define INFINI_OPS_LINKED_TORCH_OPS_PAD_H_ + +#include +#include + +#include "base/pad.h" +#include "torch/tensor_.h" + +namespace infini::ops::linked::torch { + +template +class TorchPad : public ::infini::ops::Pad { + public: + TorchPad(const Tensor input, const std::vector pad, + const std::string mode, const std::optional value, + Tensor out) + : ::infini::ops::Pad{input, pad, mode, value, out}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + input_type_{input.dtype()}, + out_type_{out.dtype()}, + device_index_{out.device().index()} {} + + using ::infini::ops::Pad::operator(); + + void operator()(const Tensor input, const std::vector pad, + const std::string mode, const std::optional value, + Tensor out) const override { + const typename Backend::StreamGuard stream_guard{ + Backend::GetStreamFromExternal(stream_, device_index_)}; + auto at_input = ToAtenTensor( + const_cast(input.data()), input_shape_, input_strides_, + input_type_, device_index_); + auto at_out = ToAtenTensor( + out.data(), out_shape_, out_strides_, out_type_, device_index_); + (void)mode; + Backend::Call(std::move(at_out), std::move(at_input), pad, + value.value_or(0.0)); + } + + private: + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType input_type_; + DataType out_type_; + int device_index_{0}; +}; + +} // namespace infini::ops::linked::torch + +#endif // INFINI_OPS_LINKED_TORCH_OPS_PAD_H_ diff --git a/src/linked/torch/ops/quick_gelu.h b/src/linked/torch/ops/quick_gelu.h new file mode 100644 index 000000000..6a2c88de3 --- /dev/null +++ b/src/linked/torch/ops/quick_gelu.h @@ -0,0 +1,59 @@ +#ifndef INFINI_OPS_LINKED_TORCH_OPS_QUICK_GELU_H_ +#define INFINI_OPS_LINKED_TORCH_OPS_QUICK_GELU_H_ + +#include "base/quick_gelu.h" +#include "torch/tensor_.h" + +namespace infini::ops::linked::torch { + +template +class TorchQuickGelu : public ::infini::ops::QuickGelu { + public: + TorchQuickGelu(const Tensor input, Tensor out) + : ::infini::ops::QuickGelu{input, out}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + input_type_{input.dtype()}, + out_type_{out.dtype()}, + device_index_{out.device().index()} {} + + using ::infini::ops::QuickGelu::operator(); + + void operator()(const Tensor input, Tensor out) const override { + const typename Backend::StreamGuard stream_guard{ + Backend::GetStreamFromExternal(stream_, device_index_)}; + auto at_input = ToAtenTensor( + const_cast(input.data()), input_shape_, input_strides_, + input_type_, device_index_); + auto at_out = ToAtenTensor( + out.data(), out_shape_, out_strides_, out_type_, device_index_); + + auto backend_input = + is_input_contiguous_ ? at_input : at_input.contiguous(); + auto backend_out = is_out_contiguous_ + ? at_out + : at::empty(at_out.sizes(), at_out.options()); + + Backend::Call(backend_out, backend_input); + if (!is_out_contiguous_) { + at_out.copy_(backend_out); + } + } + + private: + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType input_type_; + DataType out_type_; + bool is_input_contiguous_{false}; + bool is_out_contiguous_{false}; + int device_index_{0}; +}; + +} // namespace infini::ops::linked::torch + +#endif // INFINI_OPS_LINKED_TORCH_OPS_QUICK_GELU_H_ diff --git a/src/native/cuda/device_buffer.h b/src/native/cuda/device_buffer.h index 58134b03c..3d1ed68fd 100644 --- a/src/native/cuda/device_buffer.h +++ b/src/native/cuda/device_buffer.h @@ -19,7 +19,10 @@ class CudaDeviceBuffer { std::byte* Allocate(std::size_t bytes) { if (!bytes) return nullptr; - if (data_) throw std::logic_error("device buffer already allocated"); + if (data_) { + if (bytes <= bytes_) return static_cast(data_); + throw std::logic_error("device buffer already allocated"); + } if (Backend::GetDevice(&owner_device_) != Backend::kSuccess) throw std::runtime_error("cannot query allocation owner device"); void* allocation = nullptr; @@ -30,6 +33,7 @@ class CudaDeviceBuffer { throw std::bad_alloc(); } data_ = allocation; + bytes_ = bytes; return static_cast(data_); } @@ -48,6 +52,7 @@ class CudaDeviceBuffer { private: void* data_{nullptr}; + std::size_t bytes_{0}; int owner_device_{-1}; }; diff --git a/src/native/cuda/metax/ops/argmax/kernel.h b/src/native/cuda/metax/ops/argmax/kernel.h index a802f6713..4219b58b2 100644 --- a/src/native/cuda/metax/ops/argmax/kernel.h +++ b/src/native/cuda/metax/ops/argmax/kernel.h @@ -3,6 +3,7 @@ #include +#include "native/cuda/metax/caster.cuh" #include "native/cuda/metax/runtime_.h" #include "native/cuda/ops/argmax/kernel.h" diff --git a/src/native/cuda/metax/ops/causal_conv1d/kernel.h b/src/native/cuda/metax/ops/causal_conv1d/kernel.h new file mode 100644 index 000000000..7b0e069a6 --- /dev/null +++ b/src/native/cuda/metax/ops/causal_conv1d/kernel.h @@ -0,0 +1,196 @@ +#ifndef INFINI_OPS_METAX_CAUSAL_CONV1D_KERNEL_H_ +#define INFINI_OPS_METAX_CAUSAL_CONV1D_KERNEL_H_ + +#include +#include + +#include "base/causal_conv1d.h" +#include "native/cuda/metax/runtime_.h" +// cuda_fp16.h / cuda_bf16.h come from the MACA cu-bridge, +// where __nv_bfloat16 aliases __maca_bfloat16. +#include +#include + +namespace infini::ops { +namespace metax_causal_conv1d_detail { +struct Layout { + ptrdiff_t strides[9][3]{}; + size_t tokens, channels, pool; + bool cu_i64, initial_i64, final_i64; +}; +__device__ inline int64_t index(const void* p, bool i64, int i, int fallback) { + return p ? (i64 ? static_cast(p)[i] + : static_cast(static_cast(p)[i])) + : fallback; +} +template +__device__ inline float history(const T* state, const T* qkv, int64_t pos, + int64_t begin, int batch, int channel, + ptrdiff_t base, const Layout& p) { + if (pos < 3) return static_cast(state[base + pos * p.strides[2][2]]); + const auto off = static_cast(batch) * p.strides[0][0] + + static_cast(begin + pos - 3) * p.strides[0][1] + + static_cast(channel) * p.strides[0][2]; + return static_cast(qkv[off]); +} +// Port of baseline InfiniCore-0.2.9 causal_conv1d_k4_kernel. One thread per +// request/channel reads the entire sequence before committing its final state. +template +__global__ void kernel(const T* qkv, const T* weight, T* state, const T* bias, + const void* cu, const void* initial, + const void* final_indices, T* out, T* final_state, + Layout p) { + const int request = blockIdx.x, channel = blockIdx.y; + if (static_cast(channel) >= p.channels || threadIdx.x != 0) return; + int64_t begin = 0, end = static_cast(p.tokens); + int batch = request; + if (cu) { + begin = index(cu, p.cu_i64, request, 0); + end = index(cu, p.cu_i64, request + 1, 0); + batch = 0; + // Preserve baseline device-value behavior: malformed range leaves the + // corresponding output/state untouched. This is not host validation. + if (begin < 0 || end < begin || end > static_cast(p.tokens)) + return; + } + const int64_t length = end - begin; + const int64_t read = index(initial, p.initial_i64, request, request); + const int64_t write = index(final_indices, p.final_i64, request, request); + if (read < 0 || write < 0 || read >= static_cast(p.pool) || + (final_indices && write >= static_cast(p.pool))) + return; + const auto base = static_cast(read) * p.strides[2][0] + + static_cast(channel) * p.strides[2][1]; + const auto wb = static_cast(channel) * p.strides[1][0]; + for (int64_t t = 0; t < length; ++t) { + float acc = static_cast(weight[wb]) * + history(state, qkv, t, begin, batch, channel, base, p) + + static_cast(weight[wb + p.strides[1][2]]) * + history(state, qkv, t + 1, begin, batch, channel, base, p) + + static_cast(weight[wb + 2 * p.strides[1][2]]) * + history(state, qkv, t + 2, begin, batch, channel, base, p) + + static_cast(weight[wb + 3 * p.strides[1][2]]) * + history(state, qkv, t + 3, begin, batch, channel, base, p); + if (bias) + acc += static_cast( + bias[static_cast(channel) * p.strides[3][0]]); + const auto off = static_cast(batch) * p.strides[7][0] + + static_cast(begin + t) * p.strides[7][1] + + static_cast(channel) * p.strides[7][2]; + out[off] = static_cast(acc); + } + T* target = final_indices ? state : final_state; + const int stride_index = final_indices ? 2 : 8; + const auto target_base = + static_cast(final_indices ? write : request) * + p.strides[stride_index][0] + + static_cast(channel) * p.strides[stride_index][1]; + // Same ordered commit as baseline. Exact initial/final alias is valid for + // short sequences because copied history source positions lie ahead of + // writes. + target[target_base] = static_cast( + history(state, qkv, length, begin, batch, channel, base, p)); + target[target_base + p.strides[stride_index][2]] = static_cast( + history(state, qkv, length + 1, begin, batch, channel, base, p)); + target[target_base + 2 * p.strides[stride_index][2]] = static_cast( + history(state, qkv, length + 2, begin, batch, channel, base, p)); +} +} // namespace metax_causal_conv1d_detail + +template <> +class Operator : public CausalConv1d { + public: + using CausalConv1d::CausalConv1d; + using CausalConv1d::operator(); + + Operator(const Tensor qkv, const Tensor weight, Tensor conv_state, + const std::optional bias, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_conv_state) + : CausalConv1d(qkv, weight, conv_state, bias, cu_seqlens, + initial_state_indices, final_state_indices, out, + final_conv_state) { + if (weight.shape()[2] != 4 || conv_state.shape()[2] != 3) + throw std::invalid_argument( + "MetaX causal conv kernel requires K4 and state_len3"); + if (tokens_ > static_cast(std::numeric_limits::max() - 3)) + throw std::overflow_error( + "MetaX causal conv history index exceeds int64"); + if (requests_ >= static_cast(INT_MAX) || channels_ > 65535) + throw std::invalid_argument( + "causal conv request/channel launch extent unsupported"); + p_.tokens = tokens_; + p_.channels = channels_; + p_.pool = pool_; + for (size_t i = 0; i < metadata_.size(); ++i) + if (metadata_[i]) + for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) + p_.strides[i][j] = metadata_[i]->strides()[j]; + p_.cu_i64 = metadata_[4] && metadata_[4]->dtype() == DataType::kInt64; + p_.initial_i64 = metadata_[5] && metadata_[5]->dtype() == DataType::kInt64; + p_.final_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; + } + + void operator()(const Tensor qkv, const Tensor weight, Tensor conv_state, + const std::optional bias, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_conv_state) const override { + const Views views{qkv, + weight, + conv_state, + bias, + cu_seqlens, + initial_state_indices, + final_state_indices, + out, + final_conv_state}; + for (size_t i = 0; i < views.size(); ++i) { + const auto& t = views[i]; + if (t.has_value() != metadata_[i].has_value() || + (t && (!std::equal_to{}(*t, *metadata_[i]) || + (t->numel() && !t->data())))) + throw std::invalid_argument( + "causal conv metadata changed or nonempty storage is null"); + } + if (!requests_ || !channels_) return; + switch (dtype_) { + case DataType::kFloat32: + launch(views); + break; + case DataType::kFloat16: + launch(views); + break; + case DataType::kBFloat16: + launch<__nv_bfloat16>(views); + break; + default: + throw std::invalid_argument("causal conv dtype unsupported"); + } + } + + private: + metax_causal_conv1d_detail::Layout p_{}; + template + void launch(const Views& v) const { + auto pointer = [&](size_t i) -> void* { + return v[i] ? const_cast(v[i]->data()) : nullptr; + }; + const dim3 grid(static_cast(requests_), + static_cast(channels_), 1); + const auto stream = + static_cast::Stream>( + stream_ ? stream_ : nullptr); + metax_causal_conv1d_detail::kernel<<>>( + static_cast(pointer(0)), static_cast(pointer(1)), + static_cast(pointer(2)), static_cast(pointer(3)), + pointer(4), pointer(5), pointer(6), static_cast(pointer(7)), + static_cast(pointer(8)), p_); + } +}; +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/metax/ops/chunk_gated_delta_rule/kernel.h b/src/native/cuda/metax/ops/chunk_gated_delta_rule/kernel.h new file mode 100644 index 000000000..6c40a4b82 --- /dev/null +++ b/src/native/cuda/metax/ops/chunk_gated_delta_rule/kernel.h @@ -0,0 +1,24 @@ +#ifndef INFINI_OPS_METAX_CHUNK_GATED_DELTA_RULE_KERNEL_H_ +#define INFINI_OPS_METAX_CHUNK_GATED_DELTA_RULE_KERNEL_H_ + +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +#include "native/cuda/ops/chunk_gated_delta_rule/kernel.h" + +namespace infini::ops { + +// MetaX provider for the shared CUDA chunked-prefill gated-delta-rule +// kernel. The shared host path performs all shape/extent/workspace +// validation and drives the same device code the NVIDIA provider runs; +// MetaX adds no extra checks (no CUDA device query is available here). +template <> +class Operator + : public CudaChunkGatedDeltaRule> { + public: + using CudaChunkGatedDeltaRule< + Runtime>::CudaChunkGatedDeltaRule; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/metax/ops/fused_gated_delta_net_gating/kernel.h b/src/native/cuda/metax/ops/fused_gated_delta_net_gating/kernel.h new file mode 100644 index 000000000..afa7a3be4 --- /dev/null +++ b/src/native/cuda/metax/ops/fused_gated_delta_net_gating/kernel.h @@ -0,0 +1,115 @@ +#ifndef INFINI_OPS_METAX_FUSED_GATED_DELTA_NET_GATING_KERNEL_H_ +#define INFINI_OPS_METAX_FUSED_GATED_DELTA_NET_GATING_KERNEL_H_ + +#include +#include +#include +#include +#include + +#include "base/fused_gated_delta_net_gating.h" +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +// cuda_fp16.h / cuda_bf16.h come from the MACA cu-bridge, +// where __nv_bfloat16 aliases __maca_bfloat16. +#include +#include + +namespace infini::ops { +namespace metax_gating_detail { +struct Layout { + ptrdiff_t strides[6][3]{}; + size_t total, seq, hidden; + float beta, threshold; +}; +template +__global__ void kernel(const T* A, const T* a, const T* b, const T* dt, + float* g, float* betao, Layout p) { + const size_t linear = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (linear >= p.total) return; + const size_t h = linear % p.hidden, s = (linear / p.hidden) % p.seq; + const size_t batch = linear / p.hidden / p.seq; + ptrdiff_t offsets[6]; + for (int i = 0; i < 6; ++i) + offsets[i] = static_cast(batch) * p.strides[i][0] + + static_cast(s) * p.strides[i][1] + + static_cast(h) * p.strides[i][2]; + const float x = static_cast(a[offsets[1]]) + + static_cast(dt[h * p.strides[3][0]]); + const float bx = p.beta * x; + const float softplus = bx <= p.threshold ? log1pf(expf(bx)) / p.beta : x; + g[offsets[4]] = -expf(static_cast(A[h * p.strides[0][0]])) * softplus; + const float bv = static_cast(b[offsets[2]]); + const float z = expf(bv >= 0 ? -bv : bv); + betao[offsets[5]] = bv >= 0 ? 1.0f / (1.0f + z) : z / (1.0f + z); +} +} // namespace metax_gating_detail + +template <> +class Operator + : public FusedGatedDeltaNetGating { + public: + using FusedGatedDeltaNetGating::FusedGatedDeltaNetGating; + using FusedGatedDeltaNetGating::operator(); + + bool capture_safe() const override { return true; } + + void operator()(Tensor A, Tensor a, Tensor b, Tensor dt, float beta, + float threshold, Tensor g, Tensor betao) const override { + auto& p = const_cast(p_); + std::array tensors{A, a, b, dt, g, betao}; + for (size_t i = 0; i < 6; ++i) + if (!std::equal_to{}(tensors[i], metadata_[i])) + throw std::invalid_argument("gating metadata changed after Make"); + if (std::memcmp(&beta, &beta_, sizeof(float)) || + std::memcmp(&threshold, &threshold_, sizeof(float))) + throw std::invalid_argument("gating attributes changed after Make"); + if (total_ / 256 + (total_ % 256 != 0) > + std::numeric_limits::max()) + throw std::overflow_error("gating grid exceeds CUDA limit"); + p.total = total_; + p.seq = seq_; + p.hidden = hidden_; + p.beta = beta_; + p.threshold = threshold_; + for (size_t i = 0; i < 6; ++i) + for (size_t j = 0; j < metadata_[i].strides().size(); ++j) + p.strides[i][j] = metadata_[i].strides()[j]; + if (!total_) return; + const dim3 grid( + static_cast(total_ / 256 + (total_ % 256 != 0))); + const auto stream = + static_cast::Stream>( + stream_ ? stream_ : nullptr); + switch (metadata_[1].dtype()) { + case DataType::kFloat32: + launch(A, a, b, dt, g, betao, grid, stream); + break; + case DataType::kFloat16: + launch(A, a, b, dt, g, betao, grid, stream); + break; + case DataType::kBFloat16: + launch<__nv_bfloat16>(A, a, b, dt, g, betao, grid, stream); + break; + default: + throw std::invalid_argument("gating dtype unsupported"); + } + } + + private: + mutable metax_gating_detail::Layout p_{}; + template + void launch(Tensor A, Tensor a, Tensor b, Tensor dt, Tensor g, Tensor betao, + dim3 grid, + typename Runtime::Stream stream) const { + metax_gating_detail::kernel<<>>( + static_cast(A.data()), static_cast(a.data()), + static_cast(b.data()), static_cast(dt.data()), + static_cast(g.data()), static_cast(betao.data()), p_); + } +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/metax/ops/layer_norm/kernel.h b/src/native/cuda/metax/ops/layer_norm/kernel.h index 8932ea146..92a63ceca 100644 --- a/src/native/cuda/metax/ops/layer_norm/kernel.h +++ b/src/native/cuda/metax/ops/layer_norm/kernel.h @@ -3,6 +3,7 @@ #include +#include "native/cuda/metax/caster.cuh" #include "native/cuda/metax/runtime_.h" #include "native/cuda/ops/layer_norm/kernel.h" diff --git a/src/native/cuda/metax/ops/mamba_selective_scan/kernel.h b/src/native/cuda/metax/ops/mamba_selective_scan/kernel.h new file mode 100644 index 000000000..61947fc9c --- /dev/null +++ b/src/native/cuda/metax/ops/mamba_selective_scan/kernel.h @@ -0,0 +1,15 @@ +#pragma once +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +#include "native/cuda/ops/mamba_selective_scan/kernel.h" +namespace infini::ops { +struct MambaScanMetaxBackend : Runtime { + static void CheckLaunch() {} +}; +template <> +class Operator + : public CudaMambaSelectiveScan { + public: + using CudaMambaSelectiveScan::CudaMambaSelectiveScan; +}; +} // namespace infini::ops diff --git a/src/native/cuda/metax/ops/moe_fused_dense/kernel.h b/src/native/cuda/metax/ops/moe_fused_dense/kernel.h new file mode 100644 index 000000000..8cedf837e --- /dev/null +++ b/src/native/cuda/metax/ops/moe_fused_dense/kernel.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_METAX_MOE_FUSED_DENSE_KERNEL_H_ +#define INFINI_OPS_METAX_MOE_FUSED_DENSE_KERNEL_H_ + +#include + +#include "native/cuda/blas.h" +#include "native/cuda/metax/blas.h" +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +#include "native/cuda/ops/moe_fused_dense/kernel.h" + +namespace infini::ops { + +template <> +class Operator + : public CudaMoeFusedDense> { + public: + using CudaMoeFusedDense>::CudaMoeFusedDense; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_METAX_MOE_FUSED_DENSE_KERNEL_H_ diff --git a/src/native/cuda/metax/ops/mrope/kernel.h b/src/native/cuda/metax/ops/mrope/kernel.h index c097668c5..54ca2e41f 100644 --- a/src/native/cuda/metax/ops/mrope/kernel.h +++ b/src/native/cuda/metax/ops/mrope/kernel.h @@ -3,6 +3,7 @@ #include +#include "native/cuda/metax/caster.cuh" #include "native/cuda/metax/runtime_.h" #include "native/cuda/ops/mrope/kernel.h" diff --git a/src/native/cuda/metax/ops/per_tensor_i8/kernel.h b/src/native/cuda/metax/ops/per_tensor_i8/kernel.h new file mode 100644 index 000000000..e3826a0b4 --- /dev/null +++ b/src/native/cuda/metax/ops/per_tensor_i8/kernel.h @@ -0,0 +1,242 @@ +#ifndef INFINI_OPS_METAX_PER_TENSOR_I8_KERNEL_H_ +#define INFINI_OPS_METAX_PER_TENSOR_I8_KERNEL_H_ + +#include +#include +#include +#include + +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" + +namespace infini::ops { +namespace metax_per_tensor_i8_detail { + +struct Layout { + std::int64_t ndim = 0, shape[4]{}, x_stride[4]{}, packed_stride[4]{}; +}; + +__device__ std::int64_t Offset(std::size_t linear, const Layout& l, + bool packed) { + std::int64_t offset = 0; + for (std::int64_t d = l.ndim; d-- > 0;) { + auto coord = linear % l.shape[d]; + linear /= l.shape[d]; + offset += coord * (packed ? l.packed_stride[d] : l.x_stride[d]); + } + return offset; +} + +__device__ void AtomicMax(float* ptr, float value) { + auto bits = reinterpret_cast(ptr); + int old = *bits, assumed; + do { + assumed = old; + old = atomicCAS(bits, assumed, + __float_as_int(fmaxf(value, __int_as_float(assumed)))); + } while (old != assumed); +} + +template +__global__ void Absmax(const T* x, float* scale, Layout l, std::size_t count) { + float maximum = 0; + for (std::size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < count; + i += std::size_t(gridDim.x) * blockDim.x) + maximum = fmaxf(maximum, fabsf(float(x[Offset(i, l, false)]))); + AtomicMax(scale, maximum / 127.0f); +} + +template +__global__ void Quant(const T* x, std::int8_t* packed, const float* scale, + Layout l, std::size_t count) { + const float inverse = 1.0f / scale[0]; + for (std::size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < count; + i += std::size_t(gridDim.x) * blockDim.x) { + const float value = float(x[Offset(i, l, false)]) * inverse; + const float rounded = floorf(fabsf(value) + 0.5f); + int q = value >= 0 ? int(rounded) : -int(rounded); + q = q > 127 ? 127 : (q < -127 ? -127 : q); + packed[Offset(i, l, true)] = std::int8_t(q); + } +} + +template +__global__ void Dequant(T* x, const std::int8_t* packed, const float* scale, + Layout l, std::size_t count) { + for (std::size_t i = blockIdx.x * blockDim.x + threadIdx.x; i < count; + i += std::size_t(gridDim.x) * blockDim.x) + x[Offset(i, l, false)] = T(float(packed[Offset(i, l, true)]) * scale[0]); +} + +inline bool Supports(Tensor x, Tensor packed, Tensor scale, + const std::optional zero, bool quant, + bool is_static = true) { + if (zero || + (x.dtype() != DataType::kFloat16 && x.dtype() != DataType::kFloat32 && + x.dtype() != DataType::kBFloat16) || + packed.dtype() != DataType::kInt8 || + scale.dtype() != DataType::kFloat32 || scale.numel() != 1 || + x.device() != packed.device() || x.device() != scale.device() || + x.shape() != packed.shape() || x.ndim() > 4 || + x.strides().size() != x.ndim() || + packed.strides().size() != packed.ndim() || + !(quant ? x.IsContiguous() : packed.IsContiguous()) || + (quant && !is_static && !scale.IsContiguous())) + return false; + for (std::size_t d = 0; d < x.ndim(); ++d) + if (x.strides()[d] < 0 || packed.strides()[d] < 0 || + x.shape()[d] > static_cast(INT64_MAX)) + return false; + return true; +} + +struct Metadata { + DataType dtype; + Layout layout{}; + std::size_t count; + Device device; + Metadata(Tensor x, Tensor packed) + : dtype(x.dtype()), count(x.numel()), device(x.device()) { + layout.ndim = x.ndim(); + for (std::size_t d = 0; d < x.ndim(); ++d) { + layout.shape[d] = x.shape()[d]; + layout.x_stride[d] = x.strides()[d]; + layout.packed_stride[d] = packed.strides()[d]; + } + } + bool Matches(Tensor x, Tensor packed) const { + if (x.dtype() != dtype || x.device() != device || + x.ndim() != static_cast(layout.ndim)) + return false; + for (std::size_t d = 0; d < x.ndim(); ++d) + if (x.shape()[d] != static_cast(layout.shape[d]) || + x.strides()[d] != layout.x_stride[d] || + packed.strides()[d] != layout.packed_stride[d]) + return false; + return true; + } +}; + +template +void SubmitQuant(const void* x, std::int8_t* packed, float* scale, Layout l, + std::size_t count, bool is_static, + Runtime::Stream stream) { + const unsigned blocks = + unsigned(std::min((count - 1) / 256 + 1, 65535)); + if (!is_static) { + // The Absmax reduction folds its per-block maxima into `scale` with an + // atomic maximum, so the buffer must start at zero; the caller is free + // to pass an uninitialized buffer. + Runtime::Memset(scale, 0, sizeof(float)); + Absmax + <<>>(static_cast(x), scale, l, count); + } + Quant<<>>(static_cast(x), packed, scale, + l, count); +} + +template +void Dequantize(void* x, const std::int8_t* packed, const float* scale, + Layout l, std::size_t count, + Runtime::Stream stream) { + const unsigned blocks = + unsigned(std::min((count - 1) / 256 + 1, 65535)); + Dequant + <<>>(static_cast(x), packed, scale, l, count); +} + +inline Tensor Checked(Tensor x, Tensor packed, Tensor scale, + const std::optional zero, bool quant, + bool is_static = true) { + if (!Supports(x, packed, scale, zero, quant, is_static)) + throw std::invalid_argument("unsupported per-tensor INT8 operands"); + return x; +} + +} // namespace metax_per_tensor_i8_detail + +template <> +class Operator + : public PerTensorQuantI8 { + public: + static bool supports(const Tensor x, Tensor packed, Tensor scale, + const std::optional zero, const bool is_static) { + return metax_per_tensor_i8_detail::Supports(x, packed, scale, zero, true, + is_static); + } + Operator(const Tensor x, Tensor packed, Tensor scale, + const std::optional zero, const bool is_static) + : PerTensorQuantI8(x, packed, scale, zero, is_static), + meta_(metax_per_tensor_i8_detail::Checked(x, packed, scale, zero, true, + is_static), + packed), + is_static_(is_static) {} + bool capture_safe() const override { return true; } + void operator()(const Tensor x, Tensor packed, Tensor scale, + const std::optional zero, + const bool is_static) const override { + if (!supports(x, packed, scale, zero, is_static) || + !meta_.Matches(x, packed) || is_static != is_static_) + throw std::invalid_argument("unsupported per-tensor INT8 operands"); + auto stream = static_cast::Stream>( + stream_ ? stream_ : 0); + DispatchFunc< + List>( + {static_cast(meta_.dtype)}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + metax_per_tensor_i8_detail::template SubmitQuant( + x.data(), static_cast(packed.data()), + static_cast(scale.data()), meta_.layout, meta_.count, + is_static, stream); + }, + "MetaxPerTensorQuantI8"); + } + + private: + metax_per_tensor_i8_detail::Metadata meta_; + bool is_static_; +}; + +template <> +class Operator + : public PerTensorDequantI8 { + public: + static bool supports(Tensor x, const Tensor packed, const Tensor scale, + const std::optional zero) { + return metax_per_tensor_i8_detail::Supports(x, packed, scale, zero, false); + } + Operator(Tensor x, const Tensor packed, const Tensor scale, + const std::optional zero) + : PerTensorDequantI8(x, packed, scale, zero), + meta_( + metax_per_tensor_i8_detail::Checked(x, packed, scale, zero, false), + packed) {} + bool capture_safe() const override { return true; } + void operator()(Tensor x, const Tensor packed, const Tensor scale, + const std::optional zero) const override { + if (!supports(x, packed, scale, zero) || !meta_.Matches(x, packed)) + throw std::invalid_argument("unsupported per-tensor INT8 operands"); + auto stream = static_cast::Stream>( + stream_ ? stream_ : 0); + DispatchFunc< + List>( + {static_cast(meta_.dtype)}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + metax_per_tensor_i8_detail::template Dequantize( + x.data(), static_cast(packed.data()), + static_cast(scale.data()), meta_.layout, + meta_.count, stream); + }, + "MetaxPerTensorDequantI8"); + } + + private: + metax_per_tensor_i8_detail::Metadata meta_; +}; +} // namespace infini::ops + +#endif // INFINI_OPS_METAX_PER_TENSOR_I8_KERNEL_H_ diff --git a/src/native/cuda/metax/ops/recurrent_gated_delta_rule/kernel.h b/src/native/cuda/metax/ops/recurrent_gated_delta_rule/kernel.h new file mode 100644 index 000000000..d647b589b --- /dev/null +++ b/src/native/cuda/metax/ops/recurrent_gated_delta_rule/kernel.h @@ -0,0 +1,24 @@ +#ifndef INFINI_OPS_METAX_RECURRENT_GATED_DELTA_RULE_KERNEL_H_ +#define INFINI_OPS_METAX_RECURRENT_GATED_DELTA_RULE_KERNEL_H_ + +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +#include "native/cuda/ops/recurrent_gated_delta_rule/kernel.h" + +namespace infini::ops { + +// MetaX provider for the shared CUDA single-token decode gated-delta-rule +// kernel. The shared host path performs all shape/extent validation and +// drives the same device code the NVIDIA provider runs; MetaX adds no +// extra checks (no CUDA device query is available here). +template <> +class Operator + : public CudaRecurrentGatedDeltaRule> { + public: + using CudaRecurrentGatedDeltaRule< + Runtime>::CudaRecurrentGatedDeltaRule; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/kernel.h b/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/kernel.h index 000fa36ca..70e5dd51b 100644 --- a/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/kernel.h +++ b/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/kernel.h @@ -6,40 +6,19 @@ #include #include "base/chunk_gated_delta_rule.h" -#include "chunk_gated_delta_rule_legacy_kernel.cuh" #include "native/cuda/nvidia/caster.cuh" #include "native/cuda/nvidia/runtime_.h" +#include "native/cuda/ops/chunk_gated_delta_rule/kernel.h" + namespace infini::ops { -namespace chunk_native_detail { -struct Layout { - // Rows follow Views: q, k, v, g, beta, initial_state, cu_seqlens, - // initial_state_indices, final_state_indices, out, final_state (0..10). - // Each row stores that tensor's axis strides, up to the four state axes. - ptrdiff_t s[11][4]{}; - bool cu_i64, initial_i64, final_i64, norm, indexed, packed; - size_t T, pool, Hk, ratio; -}; -template -__global__ void kernel(float* workspace, const Data* q, const Data* k, - const Data* v, const Gate* g, const Gate* beta, - Data* state, const void* cu, const void* initial_indices, - const void* final_indices, Data* out, Data* final_state, - Layout p) { - // q/k/v/out have a contiguous last axis, so only their first three strides - // are passed. Initial/final state uses all axes of [pool,Hv,Dv,Dk]. - chunkGatedDeltaRuleRecurrentKernel( - workspace, out, state, final_state, q, k, v, g, beta, cu, initial_indices, - final_indices, p.cu_i64, p.initial_i64, p.final_i64, p.norm, p.packed, - p.indexed, p.T, p.pool, p.Hk, p.ratio, p.s[9][0], p.s[9][1], p.s[9][2], - p.s[5][0], p.s[5][1], p.s[5][2], p.s[5][3], p.s[10][0], p.s[10][1], - p.s[10][2], p.s[10][3], p.s[0][0], p.s[0][1], p.s[0][2], p.s[1][0], - p.s[1][1], p.s[1][2], p.s[2][0], p.s[2][1], p.s[2][2], p.s[3][0], - p.s[3][1], p.s[3][2], p.s[4][0], p.s[4][1], p.s[4][2]); -} -} // namespace chunk_native_detail + +// NVIDIA provider for the shared CUDA chunked-prefill gated-delta-rule +// kernel. The shared host path performs the shape/extent/workspace +// validation; NVIDIA adds the device guard, the per-device block-size +// precheck, and the post-launch error check. template <> class Operator - : public ChunkGatedDeltaRule { + : public CudaChunkGatedDeltaRule> { public: Operator(const Tensor q, const Tensor k, const Tensor v, const Tensor g, const Tensor beta, Tensor initial_state, @@ -48,47 +27,18 @@ class Operator const std::optional final_state_indices, Tensor out, std::optional final_state, bool use_qk_l2norm, size_t chunk_size) - : ChunkGatedDeltaRule(q, k, v, g, beta, initial_state, cu_seqlens, - initial_state_indices, final_state_indices, out, - final_state, use_qk_l2norm, chunk_size) { + : CudaChunkGatedDeltaRule>( + q, k, v, g, beta, initial_state, cu_seqlens, initial_state_indices, + final_state_indices, out, final_state, use_qk_l2norm, chunk_size) { if (device_.type() != Device::Type::kNvidia) throw std::invalid_argument("chunk delta requires NVIDIA"); - if (v.shape()[3] != D_ || (D_ != 64 && D_ != 128)) - throw std::invalid_argument( - "NVIDIA chunk delta kernels require Dk=Dv64/128"); CheckDevice(); cudaDeviceProp property{}; if (cudaGetDeviceProperties(&property, device_.index()) != cudaSuccess || property.maxThreadsPerBlock < int(D_)) throw std::runtime_error("chunk delta CUDA block size unsupported"); - if (B_ >= static_cast(INT_MAX) || Hv_ > 65535 || - Hk_ > static_cast(INT_MAX)) - throw std::invalid_argument("chunk delta launch extent unsupported"); - const auto maximum = std::numeric_limits::max(); - const size_t per_head = D_ * D_ * sizeof(float); - if (B_ > maximum / Hv_ || B_ * Hv_ > maximum / per_head) - throw std::overflow_error("chunk delta workspace size overflows"); - bytes_ = B_ * Hv_ * per_head; - if (bytes_ / sizeof(float) > - static_cast(std::numeric_limits::max())) - throw std::overflow_error( - "chunk delta workspace element offset exceeds ptrdiff"); - p_.pool = pool_; - p_.Hk = Hk_; - p_.ratio = Hv_ / Hk_; - p_.norm = norm_; - p_.T = T_; - p_.packed = metadata_[6].has_value(); - p_.indexed = metadata_[7].has_value() || metadata_[8].has_value(); - p_.cu_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; - p_.initial_i64 = metadata_[7] && metadata_[7]->dtype() == DataType::kInt64; - p_.final_i64 = metadata_[8] && metadata_[8]->dtype() == DataType::kInt64; - for (size_t i = 0; i < metadata_.size(); ++i) - if (metadata_[i]) - for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) - p_.s[i][j] = metadata_[i]->strides()[j]; } - size_t workspace_size_in_bytes() const override { return bytes_; } + void operator()(const Tensor q, const Tensor k, const Tensor v, const Tensor g, const Tensor beta, Tensor initial_state, const std::optional cu_seqlens, @@ -96,42 +46,15 @@ class Operator const std::optional final_state_indices, Tensor out, std::optional final_state, bool use_qk_l2norm, size_t chunk_size) const override { - const Views views{q, - k, - v, - g, - beta, - initial_state, - cu_seqlens, - initial_state_indices, - final_state_indices, - out, - final_state}; - CheckInvocation(views, use_qk_l2norm, chunk_size); CheckDevice(); - if (!B_) return; - if (!workspace_ || workspace_size_in_bytes_ < bytes_) - throw std::invalid_argument("chunk delta workspace missing or too small"); - switch (data_type_) { - case DataType::kFloat16: - gate(views); - break; - case DataType::kBFloat16: - gate<__nv_bfloat16>(views); - break; - case DataType::kFloat32: - gate(views); - break; - default: - throw std::invalid_argument("chunk delta data dtype unsupported"); - } + CudaChunkGatedDeltaRule>::operator()( + q, k, v, g, beta, initial_state, cu_seqlens, initial_state_indices, + final_state_indices, out, final_state, use_qk_l2norm, chunk_size); if (cudaPeekAtLastError() != cudaSuccess) throw std::runtime_error("chunk delta CUDA launch failed"); } private: - size_t bytes_{0}; - chunk_native_detail::Layout p_{}; void CheckDevice() const { int device = -1; if (cudaGetDevice(&device) != cudaSuccess) @@ -139,42 +62,6 @@ class Operator if (device != device_.index()) throw std::invalid_argument("chunk delta owner device mismatch"); } - template - void gate(const Views& v) const { - switch (gate_type_) { - case DataType::kFloat16: - dimension(v); - break; - case DataType::kBFloat16: - dimension(v); - break; - case DataType::kFloat32: - dimension(v); - break; - default: - throw std::invalid_argument("chunk delta gate dtype unsupported"); - } - } - template - void dimension(const Views& v) const { - if (D_ == 64) - launch(v); - else - launch(v); - } - template - void launch(const Views& v) const { - auto ptr = [&](size_t i) -> void* { - return v[i] ? const_cast(v[i]->data()) : nullptr; - }; - dim3 grid(static_cast(B_), static_cast(Hv_), 1); - const auto stream = static_cast(stream_ ? stream_ : nullptr); - chunk_native_detail::kernel<<>>( - static_cast(workspace_), static_cast(ptr(0)), - static_cast(ptr(1)), static_cast(ptr(2)), - static_cast(ptr(3)), static_cast(ptr(4)), - static_cast(ptr(5)), ptr(6), ptr(7), ptr(8), - static_cast(ptr(9)), static_cast(ptr(10)), p_); - } }; + } // namespace infini::ops diff --git a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/kernel.h b/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/kernel.h index ddd5c8c71..823db8be2 100644 --- a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/kernel.h +++ b/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/kernel.h @@ -8,106 +8,51 @@ #include "base/recurrent_gated_delta_rule.h" #include "native/cuda/nvidia/caster.cuh" #include "native/cuda/nvidia/runtime_.h" -#include "recurrent_gated_delta_rule_legacy_kernel.cuh" +#include "native/cuda/ops/recurrent_gated_delta_rule/kernel.h" + namespace infini::ops { -namespace recurrent_native_detail { -struct Layout { - ptrdiff_t s[10][4]{}; - bool initial_i64, final_i64, norm, indexed; - size_t pool, Hk, ratio; -}; -template -__global__ void kernel(const Data* q, const Data* k, const Data* v, - const Gate* g, const Gate* beta, Data* state, - const void* initial_indices, const void* final_indices, - Data* out, Data* final_state, Layout p) { - extern __shared__ float shared[]; - recurrentGatedDeltaRuleIndexedPoolWarpKernel( - out, state, final_state, q, k, v, g, beta, initial_indices, final_indices, - p.initial_i64, p.final_i64, p.norm, p.indexed, p.pool, p.Hk, p.ratio, - p.s[8][0], p.s[8][1], p.s[8][2], p.s[5][0], p.s[5][1], p.s[5][2], - p.s[5][3], p.s[9][0], p.s[9][1], p.s[9][2], p.s[9][3], p.s[0][0], - p.s[0][1], p.s[0][2], p.s[1][0], p.s[1][1], p.s[1][2], p.s[2][0], - p.s[2][1], p.s[2][2], p.s[3][0], p.s[3][1], p.s[3][2], p.s[4][0], - p.s[4][1], p.s[4][2], shared); -} -} // namespace recurrent_native_detail + +// NVIDIA provider for the shared CUDA single-token decode gated-delta-rule +// kernel. The shared host path performs the shape/extent validation; +// NVIDIA adds the device guard, the per-device block-size precheck, and +// the post-launch error check. template <> class Operator - : public RecurrentGatedDeltaRule { + : public CudaRecurrentGatedDeltaRule> { public: Operator(const Tensor q, const Tensor k, const Tensor v, const Tensor g, const Tensor beta, Tensor initial_state, const std::optional initial_state_indices, const std::optional final_state_indices, Tensor out, std::optional final_state, bool use_qk_l2norm) - : RecurrentGatedDeltaRule(q, k, v, g, beta, initial_state, - initial_state_indices, final_state_indices, out, - final_state, use_qk_l2norm) { + : CudaRecurrentGatedDeltaRule>( + q, k, v, g, beta, initial_state, initial_state_indices, + final_state_indices, out, final_state, use_qk_l2norm) { if (device_.type() != Device::Type::kNvidia) throw std::invalid_argument("recurrent delta requires NVIDIA"); - if (q.shape()[1] != 1 || v.shape()[3] != D_ || (D_ != 64 && D_ != 128)) - throw std::invalid_argument( - "NVIDIA recurrent delta kernels require T1 and Dk=Dv64/128"); CheckDevice(); cudaDeviceProp property{}; if (cudaGetDeviceProperties(&property, device_.index()) != cudaSuccess || property.maxThreadsPerBlock < 256) throw std::runtime_error( "recurrent delta requires 256-thread CUDA blocks"); - if (B_ >= static_cast(INT_MAX) || Hv_ > 65535 || - Hk_ > static_cast(INT_MAX)) - throw std::invalid_argument("recurrent delta launch extent unsupported"); - p_.pool = pool_; - p_.Hk = Hk_; - p_.ratio = Hv_ / Hk_; - p_.norm = norm_; - p_.indexed = metadata_[6].has_value() || metadata_[7].has_value(); - p_.initial_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; - p_.final_i64 = metadata_[7] && metadata_[7]->dtype() == DataType::kInt64; - for (size_t i = 0; i < metadata_.size(); ++i) - if (metadata_[i]) - for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) - p_.s[i][j] = metadata_[i]->strides()[j]; } + void operator()(const Tensor q, const Tensor k, const Tensor v, const Tensor g, const Tensor beta, Tensor initial_state, const std::optional initial_state_indices, const std::optional final_state_indices, Tensor out, std::optional final_state, bool use_qk_l2norm) const override { - const Views views{q, - k, - v, - g, - beta, - initial_state, - initial_state_indices, - final_state_indices, - out, - final_state}; - CheckInvocation(views, use_qk_l2norm); CheckDevice(); - if (!B_) return; - switch (data_type_) { - case DataType::kFloat16: - gate(views); - break; - case DataType::kBFloat16: - gate<__nv_bfloat16>(views); - break; - case DataType::kFloat32: - gate(views); - break; - default: - throw std::invalid_argument("recurrent delta data dtype unsupported"); - } + CudaRecurrentGatedDeltaRule>::operator()( + q, k, v, g, beta, initial_state, initial_state_indices, + final_state_indices, out, final_state, use_qk_l2norm); if (cudaPeekAtLastError() != cudaSuccess) throw std::runtime_error("recurrent delta CUDA launch failed"); } private: - recurrent_native_detail::Layout p_{}; void CheckDevice() const { int device = -1; if (cudaGetDevice(&device) != cudaSuccess) @@ -115,44 +60,6 @@ class Operator if (device != device_.index()) throw std::invalid_argument("recurrent delta owner device mismatch"); } - template - void gate(const Views& v) const { - switch (gate_type_) { - case DataType::kFloat16: - dimension(v); - break; - case DataType::kBFloat16: - dimension(v); - break; - case DataType::kFloat32: - dimension(v); - break; - default: - throw std::invalid_argument("recurrent delta gate dtype unsupported"); - } - } - template - void dimension(const Views& v) const { - if (D_ == 64) - launch(v); - else - launch(v); - } - template - void launch(const Views& v) const { - auto ptr = [&](size_t i) -> void* { - return v[i] ? const_cast(v[i]->data()) : nullptr; - }; - dim3 grid(static_cast(B_), static_cast(Hv_), - D / 8); - const auto stream = static_cast(stream_ ? stream_ : nullptr); - recurrent_native_detail::kernel - <<>>( - static_cast(ptr(0)), static_cast(ptr(1)), - static_cast(ptr(2)), static_cast(ptr(3)), - static_cast(ptr(4)), static_cast(ptr(5)), - ptr(6), ptr(7), static_cast(ptr(8)), - static_cast(ptr(9)), p_); - } }; + } // namespace infini::ops diff --git a/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh b/src/native/cuda/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh similarity index 97% rename from src/native/cuda/nvidia/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh rename to src/native/cuda/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh index 8e24924c4..79d23d108 100644 --- a/src/native/cuda/nvidia/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh +++ b/src/native/cuda/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh @@ -1,15 +1,14 @@ +#include +#include // Ported from InfiniCore 1ab85ef1d3c2bce49edf1fc73a5b48becaf7552e: // src/infiniop/ops/chunk_gated_delta_rule/cuda/kernel.cuh. // Modified for InfiniOps: active helpers/state kernel and namespace adaptation. // Copyright 2025 InfiniTensor; MIT notice: LICENSES/InfiniCore-MIT.txt. #pragma once #include - -#include "native/cuda/nvidia/data_type_.h" - // Exact baseline helpers and active state-workspace kernel. The baseline // chunked-matrix branch is hard-disabled and is not copied as a second path. -namespace infini::ops::chunk_native_detail { +namespace infini::ops::chunk_gated_delta_rule_detail { __device__ inline int64_t loadOptionalIndex(const void* indices, bool is_i64, int idx, int fallback) { if (indices == nullptr) { @@ -31,7 +30,7 @@ __device__ inline float loadAsFloat(const half* ptr, ptrdiff_t offset) { } template <> -__device__ inline float loadAsFloat(const cuda_bfloat16* ptr, +__device__ inline float loadAsFloat<__nv_bfloat16>(const __nv_bfloat16* ptr, ptrdiff_t offset) { return __bfloat162float(ptr[offset]); } @@ -259,4 +258,4 @@ __device__ void chunkGatedDeltaRuleRecurrentKernel( } } #undef CGDR_FOR -} // namespace infini::ops::chunk_native_detail +} // namespace infini::ops::chunk_gated_delta_rule_detail diff --git a/src/native/cuda/ops/chunk_gated_delta_rule/kernel.h b/src/native/cuda/ops/chunk_gated_delta_rule/kernel.h new file mode 100644 index 000000000..ca219c551 --- /dev/null +++ b/src/native/cuda/ops/chunk_gated_delta_rule/kernel.h @@ -0,0 +1,213 @@ +#ifndef INFINI_OPS_CUDA_CHUNK_GATED_DELTA_RULE_KERNEL_H_ +#define INFINI_OPS_CUDA_CHUNK_GATED_DELTA_RULE_KERNEL_H_ + +#include +#include + +#include "base/chunk_gated_delta_rule.h" +#include "native/cuda/device_buffer.h" +#include "native/cuda/ops/chunk_gated_delta_rule/chunk_gated_delta_rule_legacy_kernel.cuh" + +// cuda_fp16.h / cuda_bf16.h come from the CUDA runtime on NVIDIA and from +// the MACA cu-bridge on MetaX, where __nv_bfloat16 aliases __maca_bfloat16. +#include +#include + +namespace infini::ops { +namespace chunk_gated_delta_rule_detail { +struct Layout { + // Rows follow Views: q, k, v, g, beta, initial_state, cu_seqlens, + // initial_state_indices, final_state_indices, out, final_state (0..10). + // Each row stores that tensor axis strides, up to the four state axes. + ptrdiff_t s[11][4]{}; + bool cu_i64, initial_i64, final_i64, norm, indexed, packed; + size_t T, pool, Hk, ratio; +}; +template +__global__ void kernel(float* workspace, const Data* q, const Data* k, + const Data* v, const Gate* g, const Gate* beta, + Data* state, const void* cu, const void* initial_indices, + const void* final_indices, Data* out, Data* final_state, + Layout p) { + // q/k/v/out have a contiguous last axis, so only their first three strides + // are passed. Initial/final state uses all axes of [pool,Hv,Dv,Dk]. + chunkGatedDeltaRuleRecurrentKernel( + workspace, out, state, final_state, q, k, v, g, beta, cu, initial_indices, + final_indices, p.cu_i64, p.initial_i64, p.final_i64, p.norm, p.packed, + p.indexed, p.T, p.pool, p.Hk, p.ratio, p.s[9][0], p.s[9][1], p.s[9][2], + p.s[5][0], p.s[5][1], p.s[5][2], p.s[5][3], p.s[10][0], p.s[10][1], + p.s[10][2], p.s[10][3], p.s[0][0], p.s[0][1], p.s[0][2], p.s[1][0], + p.s[1][1], p.s[1][2], p.s[2][0], p.s[2][1], p.s[2][2], p.s[3][0], + p.s[3][1], p.s[3][2], p.s[4][0], p.s[4][1], p.s[4][2]); +} +} // namespace chunk_gated_delta_rule_detail + +// Chunked prefill kernel for gated delta rule, shared by the CUDA-like +// backends. The `Backend` type must expose `kDeviceType` and `Stream` +// (the metax and nvidia runtimes both do). Backend-specific host-side +// checks stay in the per-device providers. +template +class CudaChunkGatedDeltaRule : public ChunkGatedDeltaRule { + public: + CudaChunkGatedDeltaRule(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, + Tensor initial_state, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, + Tensor out, std::optional final_state, + bool use_qk_l2norm, size_t chunk_size) + : ChunkGatedDeltaRule(q, k, v, g, beta, initial_state, cu_seqlens, + initial_state_indices, final_state_indices, out, + final_state, use_qk_l2norm, chunk_size) { + if (v.shape()[3] != D_ || (D_ != 64 && D_ != 128)) + throw std::invalid_argument("chunk delta kernels require Dk=Dv64/128"); + if (B_ >= static_cast(INT_MAX) || Hv_ > 65535 || + Hk_ > static_cast(INT_MAX)) + throw std::invalid_argument("chunk delta launch extent unsupported"); + const auto maximum = std::numeric_limits::max(); + const size_t per_head = D_ * D_ * sizeof(float); + if (B_ > maximum / Hv_ || B_ * Hv_ > maximum / per_head) + throw std::overflow_error("chunk delta workspace size overflows"); + bytes_ = B_ * Hv_ * per_head; + if (bytes_ / sizeof(float) > + static_cast(std::numeric_limits::max())) + throw std::overflow_error( + "chunk delta workspace element offset exceeds ptrdiff"); + p_.pool = pool_; + p_.Hk = Hk_; + p_.ratio = Hv_ / Hk_; + p_.norm = norm_; + p_.T = T_; + p_.packed = metadata_[6].has_value(); + p_.indexed = metadata_[7].has_value() || metadata_[8].has_value(); + p_.cu_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; + p_.initial_i64 = metadata_[7] && metadata_[7]->dtype() == DataType::kInt64; + p_.final_i64 = metadata_[8] && metadata_[8]->dtype() == DataType::kInt64; + // The kernel reads the last data axis with an implicit stride of 1 and + // consumes only the first three strides of each 4-D tensor, so the + // innermost axis must be contiguous. This also rejects channels-last + // tensors (torch 4-D copies can be channels-last by default), which + // would otherwise be read misaligned and silently produce wrong + // results; the base-class check for this is a debug-only assert. + for (const auto i : {size_t{0}, size_t{1}, size_t{2}, size_t{9}}) { + if (metadata_[i] && metadata_[i]->numel() && + metadata_[i]->strides()[3] != 1) + throw std::invalid_argument( + "chunk delta innermost data dimension must be contiguous"); + } + for (size_t i = 0; i < metadata_.size(); ++i) + if (metadata_[i]) + for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) + p_.s[i][j] = metadata_[i]->strides()[j]; + } + + size_t workspace_size_in_bytes() const override { return bytes_; } + + // The dispatch Call path passes an explicit workspace when the caller + // provides one (C API / engine). Callers without workspace acquisition + // (python bindings) fall back to a buffer owned by this instance. + // `Operator::operator()` resets `workspace_` from the handle on every + // call, so the fallback allocation is cached and reused across + // invocations (CudaDeviceBuffer::Allocate is idempotent for equal or + // smaller sizes). + void UseWorkspace() const { + if (workspace_ && workspace_size_in_bytes_ >= bytes_) return; + auto* self = const_cast(this); + self->workspace_ = self->scratch_.Allocate(bytes_); + self->workspace_size_in_bytes_ = bytes_; + } + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, Tensor initial_state, + const std::optional cu_seqlens, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_state, bool use_qk_l2norm, + size_t chunk_size) const override { + const Views views{q, + k, + v, + g, + beta, + initial_state, + cu_seqlens, + initial_state_indices, + final_state_indices, + out, + final_state}; + CheckInvocation(views, use_qk_l2norm, chunk_size); + if (!B_) return; + UseWorkspace(); + switch (data_type_) { + case DataType::kFloat16: + gate(views); + break; + case DataType::kBFloat16: + gate<__nv_bfloat16>(views); + break; + case DataType::kFloat32: + gate(views); + break; + default: + throw std::invalid_argument("chunk delta data dtype unsupported"); + } + } + + protected: + ~CudaChunkGatedDeltaRule() = default; + + private: + size_t bytes_{0}; + chunk_gated_delta_rule_detail::Layout p_{}; + // Fallback scratch for dispatch paths that provide no workspace. The + // device memory outlives every asynchronous launch of this instance. + mutable CudaDeviceBuffer> scratch_; + + protected: + template + void gate(const Views& v) const { + switch (gate_type_) { + case DataType::kFloat16: + dimension(v); + break; + case DataType::kBFloat16: + dimension(v); + break; + case DataType::kFloat32: + dimension(v); + break; + default: + throw std::invalid_argument("chunk delta gate dtype unsupported"); + } + } + + template + void dimension(const Views& v) const { + if (D_ == 64) + launch(v); + else + launch(v); + } + + template + void launch(const Views& v) const { + auto ptr = [&](size_t i) -> void* { + return v[i] ? const_cast(v[i]->data()) : nullptr; + }; + dim3 grid(static_cast(B_), static_cast(Hv_), 1); + const auto stream = + static_cast(stream_ ? stream_ : nullptr); + chunk_gated_delta_rule_detail::kernel + <<>>( + static_cast(workspace_), static_cast(ptr(0)), + static_cast(ptr(1)), static_cast(ptr(2)), + static_cast(ptr(3)), static_cast(ptr(4)), + static_cast(ptr(5)), ptr(6), ptr(7), ptr(8), + static_cast(ptr(9)), static_cast(ptr(10)), p_); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_CHUNK_GATED_DELTA_RULE_KERNEL_H_ diff --git a/src/native/cuda/ops/moe_fused_dense/kernel.cuh b/src/native/cuda/ops/moe_fused_dense/kernel.cuh new file mode 100644 index 000000000..97848ea9b --- /dev/null +++ b/src/native/cuda/ops/moe_fused_dense/kernel.cuh @@ -0,0 +1,158 @@ +#ifndef INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_CUH_ +#define INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_CUH_ + +#include + +#include "native/cuda/kernel_commons.cuh" + +namespace infini::ops { +namespace moe_fused_dense_detail { + +// Computes the inclusive prefix sum of `counts[0..num_experts-1]`, +// storing the exclusive prefix in `offsets[0..num_experts-1]` and the +// total in `offsets[num_experts]`. Only one thread (thread 0) is used. +template +__global__ void ExclusivePrefixCountsKernel(const int* counts, int* offsets, + int num_experts) { + if (threadIdx.x == 0) { + offsets[0] = 0; + int sum = 0; + for (int i = 0; i < num_experts; ++i) { + sum += counts[i]; + offsets[i + 1] = sum; + } + } +} + +// For each aligned block of `block_size` tokens, atomically adds `block_size` +// to the count of the expert that owns the block. Rows whose expert id is +// out of range (padding rows) are skipped. `num_tokens_post_padded` is read +// on the host before launch. +__global__ void CountAlignedExpertsKernel(const int* expert_ids, + int num_tokens_post_padded, + int* counts, int num_experts, + int block_size) { + int block = blockIdx.x * blockDim.x + threadIdx.x; + int num_blocks = (num_tokens_post_padded + block_size - 1) / block_size; + if (block >= num_blocks) { + return; + } + int expert = expert_ids[block]; + if (expert >= 0 && expert < num_experts) { + atomicAdd(counts + expert, block_size); + } +} + +// Gathers hidden states into a packed, expert-bucketed buffer. Each output +// row `row` maps to `pair = sorted_token_ids[row]`, and its hidden state is +// copied from the source token `pair / topk`. Rows pointing past the valid +// pair range (padding) are zero-filled. +template +__global__ void PackHiddenAlignedKernel(const T* hidden, + const int* sorted_token_ids, + int* output_permutation, + T* packed_hidden, int pairs, int topk, + int hidden_size, + int max_num_tokens_padded) { + int row = blockIdx.x; + int tid = threadIdx.x; + if (row >= max_num_tokens_padded) { + return; + } + int pair = sorted_token_ids[row]; + if (pair >= 0 && pair < pairs) { + if (tid == 0) { + output_permutation[pair] = row; + } + int token = pair / topk; + for (int h = tid; h < hidden_size; h += blockDim.x) { + packed_hidden[static_cast(row) * hidden_size + h] = + hidden[static_cast(token) * hidden_size + h]; + } + } else { + for (int h = tid; h < hidden_size; h += blockDim.x) { + packed_hidden[static_cast(row) * hidden_size + h] = + Caster::template Cast(0.0f); + } + } +} + +// Fills the packed gate_up / expert_out rows routed to `expert` with the +// per-expert bias rows, so GEMM with beta=1 incorporates the bias. +template +__global__ void FillPrefillBiasKernel(T* gate_up, T* expert_out, const T* bias1, + const T* bias2, const int* counts, + const int* offsets, int intermediate_size, + int hidden_size) { + const int expert = blockIdx.x; + const int rows = counts[expert]; + const int offset = offsets[expert]; + const int gate_columns = intermediate_size * 2; + if (bias1) { + for (size_t i = threadIdx.x; i < static_cast(rows) * gate_columns; + i += blockDim.x) { + gate_up[static_cast(offset) * gate_columns + i] = + bias1[static_cast(expert) * gate_columns + i % gate_columns]; + } + } + if (bias2) { + for (size_t i = threadIdx.x; i < static_cast(rows) * hidden_size; + i += blockDim.x) { + expert_out[static_cast(offset) * hidden_size + i] = + bias2[static_cast(expert) * hidden_size + i % hidden_size]; + } + } +} + +// SwiGLU activation: out = up * silu(gate), where silu(x) = x / (1 + exp(-x)). +template +__global__ void SwigluKernel(const T* gate_up, T* activated, int rows, + int intermediate_size) { + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int total = rows * intermediate_size; + if (idx >= total) { + return; + } + int row = idx / intermediate_size; + int col = idx - row * intermediate_size; + const T* base = gate_up + static_cast(row) * intermediate_size * 2; + float gate = Caster::template Cast(base[col]); + float up = Caster::template Cast(base[intermediate_size + col]); + float silu = gate / (1.0f + expf(-gate)); + activated[idx] = Caster::template Cast(up * silu); +} + +// Scatters the weighted expert outputs back to the original token rows. For +// each token `token`, sums over its `topk` pairs, gathering rows through +// `output_permutation` and weighting by `topk_weights`. +template +__global__ void ApplyShuffleMulSumKernel( + const T* __restrict__ expert_out, T* __restrict__ out, + const int* __restrict__ output_permutation, + const float* __restrict__ topk_weights, int num_tokens, int topk, + int hidden_size) { + int token = blockIdx.x; + if (token >= num_tokens) { + return; + } + + for (int h = threadIdx.x; h < hidden_size; h += blockDim.x) { + float sum = 0.0f; + for (int k = 0; k < topk; ++k) { + int pair = token * topk + k; + int src_row = output_permutation[pair]; + if (src_row >= 0) { + sum += Caster::template Cast( + expert_out[static_cast(src_row) * hidden_size + h]) * + topk_weights[pair]; + } + } + out[static_cast(token) * hidden_size + h] = + Caster::template Cast(sum); + } +} + +} // namespace moe_fused_dense_detail +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_CUH_ diff --git a/src/native/cuda/ops/moe_fused_dense/kernel.h b/src/native/cuda/ops/moe_fused_dense/kernel.h new file mode 100644 index 000000000..e61a461e3 --- /dev/null +++ b/src/native/cuda/ops/moe_fused_dense/kernel.h @@ -0,0 +1,350 @@ +#ifndef INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_H_ +#define INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_H_ + +#include +#include +#include +#include +#include + +#include "base/moe_fused_dense.h" +#include "data_type.h" +#include "dispatcher.h" +#include "native/cuda/device_buffer.h" +#include "native/cuda/ops/moe_fused_dense/kernel.cuh" +#include "native/cuda/runtime_utils.h" + +namespace infini::ops { + +// MoE fused-dense reference for CUDA-like backends without a +// grouped GEMM (e.g. Metax): sorted tokens are packed into +// per-expert contiguous buckets, one GEMM runs per expert, SwiGLU +// + the down-projection run, and the weighted results are +// scattered back to the original tokens. +// +// The `Backend` type must expose the CUDA-like runtime members +// (`Stream`, `Malloc`, `Free`, `Memcpy`, `Memset`, +// `DeviceSynchronize`, `kMemcpyDeviceToHost`) and the BLAS +// members used here (`BlasHandle`, `BlasCreate`, `BlasSetStream`, +// `BlasGemmStridedBatchedEx`, `BLAS_OP_T`, `BLAS_OP_N`, `R_16F`, +// `R_16BF`, `BLAS_COMPUTE_32F`, `BLAS_GEMM_DEFAULT`). +template +class CudaMoeFusedDense : public MoeFusedDense { + public: + CudaMoeFusedDense(Tensor output, const Tensor hidden_states, const Tensor w13, + const Tensor w2, const Tensor topk_weights, + const Tensor topk_ids, const Tensor sorted_token_ids, + const Tensor expert_ids, + const Tensor num_tokens_post_padded, size_t block_size, + const std::optional bias1 = std::nullopt, + const std::optional bias2 = std::nullopt) + : MoeFusedDense{output, + hidden_states, + w13, + w2, + topk_weights, + topk_ids, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + block_size, + bias1, + bias2} { + moe_detail::Require( + hidden_states.IsContiguous() && w13.IsContiguous() && + w2.IsContiguous() && topk_weights.IsContiguous() && + topk_ids.IsContiguous() && sorted_token_ids.IsContiguous() && + expert_ids.IsContiguous() && num_tokens_post_padded.IsContiguous() && + output.IsContiguous()); + moe_detail::Require(hidden_states.dtype() == DataType::kFloat16 || + hidden_states.dtype() == DataType::kBFloat16); + if (bias1) { + moe_detail::Require(bias1->IsContiguous()); + } + if (bias2) { + moe_detail::Require(bias2->IsContiguous()); + } + } + + ~CudaMoeFusedDense() override = default; + + std::size_t workspace_size_in_bytes() const override { + const auto& metadata_ = MoeFusedDense::metadata_; + const auto dtype = metadata_[1].dtype(); + const std::size_t dtype_size = kDataTypeToSize.at(dtype); + const std::size_t num_tokens = metadata_[1].size(0); + const std::size_t hidden_size = metadata_[1].size(1); + const std::size_t intermediate_size = metadata_[3].size(2); + const std::size_t topk = metadata_[5].size(1); + const std::size_t max_num_tokens_padded = metadata_[6].numel(); + const std::size_t pairs = num_tokens * topk; + std::size_t bytes = 0; + bytes += AlignUp((num_experts() + 1) * sizeof(int)); + bytes += AlignUp((num_experts() + 1) * sizeof(int)); + bytes += AlignUp(pairs * sizeof(int)); + bytes += AlignUp(max_num_tokens_padded * hidden_size * dtype_size); + bytes += + AlignUp(max_num_tokens_padded * intermediate_size * 2 * dtype_size); + bytes += AlignUp(max_num_tokens_padded * intermediate_size * dtype_size); + bytes += AlignUp(max_num_tokens_padded * hidden_size * dtype_size); + return bytes; + } + + bool capture_safe() const override { return false; } + + void operator()(Tensor output, const Tensor hidden_states, const Tensor w13, + const Tensor w2, const Tensor weights, const Tensor ids, + const Tensor sorted_token_ids, const Tensor expert_ids, + const Tensor num_tokens_post_padded, size_t block_size, + const std::optional bias1, + const std::optional bias2) const override { + Invocation(output, hidden_states, w13, w2, weights, ids, sorted_token_ids, + expert_ids, num_tokens_post_padded, block_size, bias1, bias2); + + const int num_tokens = static_cast(hidden_states.size(0)); + const int hidden_size = static_cast(hidden_states.size(1)); + const int num_experts = static_cast(w13.size(0)); + const int intermediate_size = static_cast(w2.size(2)); + const int topk = static_cast(ids.size(1)); + const int num_tokens_padded = static_cast(sorted_token_ids.numel()); + const int max_num_tokens_padded = + static_cast(sorted_token_ids.numel()); + const int pairs = num_tokens * topk; + const int block_size_i = static_cast(block_size); + const std::size_t dtype_size = kDataTypeToSize.at(hidden_states.dtype()); + + auto stream = static_cast(stream_); + + void* workspace_ptr = workspace_; + if (workspace_ptr == nullptr || + workspace_size_in_bytes_ < workspace_size_in_bytes()) { + // The dispatch Call path passes an explicit workspace when the caller + // provides one (C API / engine). Callers without workspace acquisition + // (python bindings) fall back to a buffer owned by this instance; + // CudaDeviceBuffer::Allocate is idempotent for equal or smaller + // sizes, and `Operator::operator()` resets `workspace_` from the + // handle on every call, so the allocation is reused across them. + auto* self = const_cast(this); + workspace_ptr = self->fallback_.Allocate(workspace_size_in_bytes()); + self->workspace_ = workspace_ptr; + self->workspace_size_in_bytes_ = workspace_size_in_bytes(); + } + + // Workspace layout: counts, offsets, permutation, then data buffers. + std::uint8_t* ptr = reinterpret_cast(workspace_ptr); + int* counts = Advance(ptr, num_experts + 1); + int* offsets = Advance(ptr, num_experts + 1); + int* output_permutation = Advance(ptr, pairs); + void* packed_hidden = + AdvanceBytes(ptr, static_cast(num_tokens_padded) * + hidden_size * dtype_size); + void* gate_up = + AdvanceBytes(ptr, static_cast(num_tokens_padded) * + intermediate_size * 2 * dtype_size); + void* activated = + AdvanceBytes(ptr, static_cast(num_tokens_padded) * + intermediate_size * dtype_size); + void* expert_out = + AdvanceBytes(ptr, static_cast(num_tokens_padded) * + hidden_size * dtype_size); + + auto& blas_handle = GetHandle(); + Backend::BlasSetStream(blas_handle, stream); + + const bool has_bias1 = bias1.has_value(); + const bool has_bias2 = bias2.has_value(); + + Backend::Memset(output_permutation, 0xFF, pairs * sizeof(int)); + Backend::Memset(counts, 0, (num_experts + 1) * sizeof(int)); + Backend::Memset( + expert_out, 0, + static_cast(num_tokens_padded) * hidden_size * dtype_size); + + // Read num_tokens_post_padded on the host before launching the + // counting / packing kernels. + int num_tokens_post_padded_host = 0; + Backend::Memcpy(&num_tokens_post_padded_host, num_tokens_post_padded.data(), + sizeof(int), Backend::kMemcpyDeviceToHost); + int num_blocks_aligned = + (num_tokens_post_padded_host + block_size_i - 1) / block_size_i; + + moe_fused_dense_detail::CountAlignedExpertsKernel<<< + (num_blocks_aligned + 255) / 256, 256, 0, stream>>>( + reinterpret_cast(expert_ids.data()), + num_tokens_post_padded_host, counts, num_experts, block_size_i); + moe_fused_dense_detail::ExclusivePrefixCountsKernel + <<<1, 1, 0, stream>>>(counts, offsets, num_experts); + + DispatchFunc>( + {static_cast(hidden_states.dtype())}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + moe_fused_dense_detail::PackHiddenAlignedKernel + <<>>( + reinterpret_cast(hidden_states.data()), + reinterpret_cast(sorted_token_ids.data()), + output_permutation, reinterpret_cast(packed_hidden), + pairs, topk, hidden_size, num_tokens_padded); + }, + "CudaMoeFusedDense::PackHiddenAligned"); + + // Per-expert GEMMs launch from the host, so copy counts/offsets + // back and wait for the counting / packing kernels as well as the + // D2H copies. + std::vector host_counts(num_experts + 1); + std::vector host_offsets(num_experts + 1); + Backend::Memcpy(host_counts.data(), counts, (num_experts + 1) * sizeof(int), + Backend::kMemcpyDeviceToHost); + Backend::Memcpy(host_offsets.data(), offsets, + (num_experts + 1) * sizeof(int), + Backend::kMemcpyDeviceToHost); + Backend::DeviceSynchronize(); + + const auto blas_dtype = (hidden_states.dtype() == DataType::kFloat16) + ? Backend::R_16F + : Backend::R_16BF; + + // Bias rows are written into the packed buffers before each GEMM + // so rows the GEMM never touches still receive the bias. + if (has_bias1 || has_bias2) { + DispatchFunc>( + {static_cast(hidden_states.dtype())}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + moe_fused_dense_detail::FillPrefillBiasKernel< + Backend::kDeviceType, T><<>>( + has_bias1 ? reinterpret_cast(gate_up) : nullptr, + has_bias2 ? reinterpret_cast(expert_out) : nullptr, + has_bias1 ? reinterpret_cast(bias1->data()) : nullptr, + has_bias2 ? reinterpret_cast(bias2->data()) : nullptr, + host_counts.data(), host_offsets.data(), intermediate_size, + hidden_size); + }, + "CudaMoeFusedDense::FillBias"); + } + + // GEMM1: gate_up (m x 2*I) = w13^T (2*I x H) @ packed_hidden^T. + // w13 is col-major, dimensions (H, 2*I) -> OP_T with lda = H. + { + const float alpha = 1.0f; + const float beta = 0.0f; + for (int e = 0; e < num_experts; ++e) { + const int m = host_counts[e]; + if (m <= 0) { + continue; + } + const int off = host_offsets[e]; + Backend::BlasGemmStridedBatchedEx( + blas_handle, Backend::BLAS_OP_T, Backend::BLAS_OP_N, + intermediate_size * 2, m, hidden_size, &alpha, + reinterpret_cast(w13.data()) + + static_cast(e) * intermediate_size * 2 * + hidden_size * dtype_size, + blas_dtype, hidden_size, 0, + reinterpret_cast(packed_hidden) + + static_cast(off) * hidden_size * dtype_size, + blas_dtype, hidden_size, 0, &beta, + reinterpret_cast(gate_up) + + static_cast(off) * intermediate_size * 2 * + dtype_size, + blas_dtype, intermediate_size * 2, 0, /*batch_count=*/1, + Backend::BLAS_COMPUTE_32F, Backend::BLAS_GEMM_DEFAULT); + } + } + + DispatchFunc>( + {static_cast(hidden_states.dtype())}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + moe_fused_dense_detail::SwigluKernel + <<>>(reinterpret_cast(gate_up), + reinterpret_cast(activated), num_tokens_padded, + intermediate_size); + }, + "CudaMoeFusedDense::Swiglu"); + + // GEMM2: expert_out (m x H) = w2^T (H x I) @ activated^T. + // w2 is col-major, dimensions (I, H) -> OP_T with lda = I. + { + const float alpha = 1.0f; + const float beta = 0.0f; + for (int e = 0; e < num_experts; ++e) { + const int m = host_counts[e]; + if (m <= 0) { + continue; + } + const int off = host_offsets[e]; + Backend::BlasGemmStridedBatchedEx( + blas_handle, Backend::BLAS_OP_T, Backend::BLAS_OP_N, hidden_size, m, + intermediate_size, &alpha, + reinterpret_cast(w2.data()) + + static_cast(e) * hidden_size * intermediate_size * + dtype_size, + blas_dtype, intermediate_size, 0, + reinterpret_cast(activated) + + static_cast(off) * intermediate_size * dtype_size, + blas_dtype, intermediate_size, 0, &beta, + reinterpret_cast(expert_out) + + static_cast(off) * hidden_size * dtype_size, + blas_dtype, hidden_size, 0, /*batch_count=*/1, + Backend::BLAS_COMPUTE_32F, Backend::BLAS_GEMM_DEFAULT); + } + } + + DispatchFunc>( + {static_cast(hidden_states.dtype())}, + [&](auto type_tag) { + using T = TypeMapType(type_tag)>; + moe_fused_dense_detail::ApplyShuffleMulSumKernel + <<>>( + reinterpret_cast(expert_out), + reinterpret_cast(output.data()), output_permutation, + reinterpret_cast(weights.data()), num_tokens, + topk, hidden_size); + }, + "CudaMoeFusedDense::ApplyShuffleMulSum"); + } + + protected: + static std::size_t AlignUp(std::size_t value, std::size_t alignment = 16) { + return (value + alignment - 1) / alignment * alignment; + } + + template + static T* Advance(std::uint8_t*& ptr, std::size_t count) { + T* out = reinterpret_cast(ptr); + ptr += AlignUp(count * sizeof(T)); + return out; + } + + static void* AdvanceBytes(std::uint8_t*& ptr, std::size_t bytes) { + void* out = reinterpret_cast(ptr); + ptr += AlignUp(bytes); + return out; + } + + static typename Backend::BlasHandle& GetHandle() { + thread_local typename Backend::BlasHandle handle = []() { + typename Backend::BlasHandle h; + Backend::BlasCreate(&h); + return h; + }(); + return handle; + } + + private: + std::size_t num_experts() const { + return MoeFusedDense::metadata_[2].size(0); + } + + // Fallback scratch for dispatch paths that provide no workspace. The + // device memory outlives every asynchronous launch of this instance. + mutable CudaDeviceBuffer fallback_; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_MOE_FUSED_DENSE_KERNEL_H_ diff --git a/src/native/cuda/ops/recurrent_gated_delta_rule/kernel.h b/src/native/cuda/ops/recurrent_gated_delta_rule/kernel.h new file mode 100644 index 000000000..94a169cc7 --- /dev/null +++ b/src/native/cuda/ops/recurrent_gated_delta_rule/kernel.h @@ -0,0 +1,172 @@ +#ifndef INFINI_OPS_CUDA_RECURRENT_GATED_DELTA_RULE_KERNEL_H_ +#define INFINI_OPS_CUDA_RECURRENT_GATED_DELTA_RULE_KERNEL_H_ + +#include +#include + +#include "base/recurrent_gated_delta_rule.h" +#include "native/cuda/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh" +#include "native/cuda/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh" + +// cuda_fp16.h / cuda_bf16.h come from the CUDA runtime on NVIDIA and from +// the MACA cu-bridge on MetaX, where __nv_bfloat16 aliases __maca_bfloat16. +#include +#include + +namespace infini::ops { +namespace recurrent_gated_delta_rule_detail { +struct Layout { + ptrdiff_t s[10][4]{}; + bool initial_i64, final_i64, norm, indexed; + size_t pool, Hk, ratio; +}; +template +__global__ void kernel(const Data* q, const Data* k, const Data* v, + const Gate* g, const Gate* beta, Data* state, + const void* initial_indices, const void* final_indices, + Data* out, Data* final_state, Layout p) { + extern __shared__ float shared[]; + recurrentGatedDeltaRuleIndexedPoolWarpKernel( + out, state, final_state, q, k, v, g, beta, initial_indices, final_indices, + p.initial_i64, p.final_i64, p.norm, p.indexed, p.pool, p.Hk, p.ratio, + p.s[8][0], p.s[8][1], p.s[8][2], p.s[5][0], p.s[5][1], p.s[5][2], + p.s[5][3], p.s[9][0], p.s[9][1], p.s[9][2], p.s[9][3], p.s[0][0], + p.s[0][1], p.s[0][2], p.s[1][0], p.s[1][1], p.s[1][2], p.s[2][0], + p.s[2][1], p.s[2][2], p.s[3][0], p.s[3][1], p.s[3][2], p.s[4][0], + p.s[4][1], p.s[4][2], shared); +} +} // namespace recurrent_gated_delta_rule_detail + +// Single-token decode kernel for gated delta rule, shared by the CUDA-like +// backends. The `Backend` type must expose `kDeviceType` and `Stream` +// (the metax and nvidia runtimes both do). Backend-specific host-side +// checks stay in the per-device providers. +template +class CudaRecurrentGatedDeltaRule : public RecurrentGatedDeltaRule { + public: + CudaRecurrentGatedDeltaRule(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, + Tensor initial_state, + const std::optional initial_state_indices, + const std::optional final_state_indices, + Tensor out, std::optional final_state, + bool use_qk_l2norm) + : RecurrentGatedDeltaRule(q, k, v, g, beta, initial_state, + initial_state_indices, final_state_indices, out, + final_state, use_qk_l2norm) { + if (q.shape()[1] != 1 || v.shape()[3] != D_ || (D_ != 64 && D_ != 128)) + throw std::invalid_argument( + "recurrent delta kernels require T1 and Dk=Dv64/128"); + if (B_ >= static_cast(INT_MAX) || Hv_ > 65535 || + Hk_ > static_cast(INT_MAX)) + throw std::invalid_argument("recurrent delta launch extent unsupported"); + p_.pool = pool_; + p_.Hk = Hk_; + p_.ratio = Hv_ / Hk_; + p_.norm = norm_; + p_.indexed = metadata_[6].has_value() || metadata_[7].has_value(); + p_.initial_i64 = metadata_[6] && metadata_[6]->dtype() == DataType::kInt64; + p_.final_i64 = metadata_[7] && metadata_[7]->dtype() == DataType::kInt64; + // The kernel reads the last data axis with an implicit stride of 1 and + // consumes only the first three strides of each 4-D tensor, so the + // innermost axis must be contiguous. This also rejects channels-last + // tensors (torch 4-D copies can be channels-last by default), which + // would otherwise be read misaligned and silently produce wrong + // results; the base-class check for this is a debug-only assert. + for (const auto i : {size_t{0}, size_t{1}, size_t{2}, size_t{8}}) { + if (metadata_[i] && metadata_[i]->numel() && + metadata_[i]->strides()[3] != 1) + throw std::invalid_argument( + "recurrent delta innermost data dimension must be contiguous"); + } + for (size_t i = 0; i < metadata_.size(); ++i) + if (metadata_[i]) + for (size_t j = 0; j < metadata_[i]->strides().size(); ++j) + p_.s[i][j] = metadata_[i]->strides()[j]; + } + + void operator()(const Tensor q, const Tensor k, const Tensor v, + const Tensor g, const Tensor beta, Tensor initial_state, + const std::optional initial_state_indices, + const std::optional final_state_indices, Tensor out, + std::optional final_state, + bool use_qk_l2norm) const override { + const Views views{q, + k, + v, + g, + beta, + initial_state, + initial_state_indices, + final_state_indices, + out, + final_state}; + CheckInvocation(views, use_qk_l2norm); + if (!B_) return; + switch (data_type_) { + case DataType::kFloat16: + gate(views); + break; + case DataType::kBFloat16: + gate<__nv_bfloat16>(views); + break; + case DataType::kFloat32: + gate(views); + break; + default: + throw std::invalid_argument("recurrent delta data dtype unsupported"); + } + } + + protected: + ~CudaRecurrentGatedDeltaRule() = default; + + private: + recurrent_gated_delta_rule_detail::Layout p_{}; + + protected: + template + void gate(const Views& v) const { + switch (gate_type_) { + case DataType::kFloat16: + dimension(v); + break; + case DataType::kBFloat16: + dimension(v); + break; + case DataType::kFloat32: + dimension(v); + break; + default: + throw std::invalid_argument("recurrent delta gate dtype unsupported"); + } + } + template + void dimension(const Views& v) const { + if (D_ == 64) + launch(v); + else + launch(v); + } + template + void launch(const Views& v) const { + auto ptr = [&](size_t i) -> void* { + return v[i] ? const_cast(v[i]->data()) : nullptr; + }; + dim3 grid(static_cast(B_), static_cast(Hv_), + D / 8); + const auto stream = + static_cast(stream_ ? stream_ : nullptr); + recurrent_gated_delta_rule_detail::kernel + <<>>( + static_cast(ptr(0)), static_cast(ptr(1)), + static_cast(ptr(2)), static_cast(ptr(3)), + static_cast(ptr(4)), static_cast(ptr(5)), + ptr(6), ptr(7), static_cast(ptr(8)), + static_cast(ptr(9)), p_); + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_CUDA_RECURRENT_GATED_DELTA_RULE_KERNEL_H_ diff --git a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh b/src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh similarity index 97% rename from src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh rename to src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh index 9a715fdc7..9380a723f 100644 --- a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh +++ b/src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_delta_rule_common.cuh @@ -8,11 +8,16 @@ #include #include +// cuda_fp16.h / cuda_bf16.h come from the CUDA runtime on NVIDIA and from +// the MACA cu-bridge on MetaX, where __nv_bfloat16 aliases __maca_bfloat16. +#include +#include + // Retain the original NVIDIA warp width. #define INFINIOP_RECURRENT_DELTA_RULE_WARP_SIZE 32 #define INFINIOP_RECURRENT_DELTA_RULE_FULL_MASK 0xffffffffU -namespace op::recurrent_gated_delta_rule::cuda { +namespace infini::ops::recurrent_gated_delta_rule_detail { template __device__ inline float loadAsFloat(const T* ptr, ptrdiff_t offset) { @@ -260,6 +265,6 @@ __device__ void recurrentDeltaRuleWarpSequence( } } -} // namespace op::recurrent_gated_delta_rule::cuda +} // namespace infini::ops::recurrent_gated_delta_rule_detail #endif // __RECURRENT_DELTA_RULE_COMMON_CUH__ diff --git a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh b/src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh similarity index 76% rename from src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh rename to src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh index d5dd6e14b..44bd1e948 100644 --- a/src/native/cuda/nvidia/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh +++ b/src/native/cuda/ops/recurrent_gated_delta_rule/recurrent_gated_delta_rule_legacy_kernel.cuh @@ -1,3 +1,5 @@ +#include +#include // Copied from InfiniCore 1ab85ef1d3c2bce49edf1fc73a5b48becaf7552e: // src/infiniop/ops/recurrent_gated_delta_rule/cuda/kernel.cuh. // Modified for InfiniOps: retain the scalar, single-token decode path. @@ -31,9 +33,11 @@ struct ScalarGatedDeltaRulePolicy { static_cast(token_idx) * beta_s1 + static_cast(value_head_idx) * beta_s2; decay[0] = expf(static_cast( - op::recurrent_gated_delta_rule::cuda::loadAsFloat(g, gate_offset))); + infini::ops::recurrent_gated_delta_rule_detail::loadAsFloat( + g, gate_offset))); beta_out[0] = static_cast( - op::recurrent_gated_delta_rule::cuda::loadAsFloat(beta, beta_offset)); + infini::ops::recurrent_gated_delta_rule_detail::loadAsFloat( + beta, beta_offset)); } __syncthreads(); const Tcompute scalar_decay = decay[0]; @@ -64,15 +68,16 @@ __device__ void recurrentGatedDeltaRuleIndexedPoolWarpKernel( const ScalarGatedDeltaRulePolicy gate_policy{ g, beta, g_s0, g_s1, g_s2, beta_s0, beta_s1, beta_s2, }; - op::recurrent_gated_delta_rule::cuda::recurrentDeltaRuleWarpSequence< - Tdata, Tcompute, Dk, Dv, WARPS_PER_BLOCK>( - out, initial_state, final_state, q, k, v, initial_state_indices, - final_state_indices, initial_state_indices_i64, final_state_indices_i64, - use_qk_l2norm, indexed_state_pool, pool_size, num_key_heads, - value_heads_per_key_head, rsqrtf(static_cast(Dk)), out_s0, - out_s1, out_s2, initial_s0, initial_s1, initial_s2, initial_s3, final_s0, - final_s1, final_s2, final_s3, q_s0, q_s1, q_s2, k_s0, k_s1, k_s2, v_s0, - v_s1, v_s2, gate_policy, shared); + infini::ops::recurrent_gated_delta_rule_detail:: + recurrentDeltaRuleWarpSequence( + out, initial_state, final_state, q, k, v, initial_state_indices, + final_state_indices, initial_state_indices_i64, + final_state_indices_i64, use_qk_l2norm, indexed_state_pool, pool_size, + num_key_heads, value_heads_per_key_head, + rsqrtf(static_cast(Dk)), out_s0, out_s1, out_s2, initial_s0, + initial_s1, initial_s2, initial_s3, final_s0, final_s1, final_s2, + final_s3, q_s0, q_s1, q_s2, k_s0, k_s1, k_s2, v_s0, v_s1, v_s2, + gate_policy, shared); } #endif // __RECURRENT_GATED_DELTA_RULE_KERNEL_CUH__ diff --git a/tests/test_chunk_gated_delta_rule.py b/tests/test_chunk_gated_delta_rule.py new file mode 100644 index 000000000..a5e293e68 --- /dev/null +++ b/tests/test_chunk_gated_delta_rule.py @@ -0,0 +1,204 @@ +"""Metax unit tests for the chunked-prefill gated delta rule operator. + +The reference below mirrors the device kernel math: per (batch, value +head) a running state s[Dv, Dk] (one row per value dim) evolves as + k_hat = l2norm(k) (optional) + q_hat = l2norm(q) * rsqrt(Dk) (when l2norm, else rsqrt(Dk)) + decay = exp(g) + v_new = beta * (v - decay * (s @ k_hat)) + s = decay * s + outer(v_new, k_hat) + out = s @ q_hat +""" + +import torch +import pytest + +import infini.ops + +from tests.utils import get_stream + + +if not hasattr(infini.ops, "chunk_gated_delta_rule"): + pytest.skip( + "`chunk_gated_delta_rule` is not available on this platform", + allow_module_level=True, + ) + + +def _sanitize(*tensors): + """MACA's host-to-device copy reports wrong stride metadata (the + data itself is contiguous); the operator kernels index through the + reported strides, so normalize every device tensor here.""" + return tuple(t.contiguous() if t.is_cuda else t for t in tensors) + + +def _reference(q, k, v, g, beta, initial_state, use_qk_l2norm): + """q/k/v: [B, T, H, D]; g/beta: [B, T, H]; initial_state: [pool/B, H, Dv, Dk]. + + Computed in float64 so the reference stays exact regardless of the + session-wide TF32 matmul setting (`tests/conftest.py`) and does not + itself become the source of float32 divergence. + """ + B, T, Hk, Dk = q.shape + Hv = v.shape[2] + ratio = Hv // Hk + state = initial_state.to(torch.float64).clone() + out = torch.empty_like(v, dtype=torch.float64) + final = torch.empty_like(initial_state, dtype=torch.float64) + + scale = Dk**-0.5 + for b in range(B): + for hv in range(Hv): + hk = hv // ratio + s = state[b, hv].clone() + for t in range(T): + qr = q[b, t, hk].to(torch.float64) + kr = k[b, t, hk].to(torch.float64) + decay = torch.exp(g[b, t, hv].to(torch.float64)) + beta_t = beta[b, t, hv].to(torch.float64) + if use_qk_l2norm: + q_hat = qr * (torch.rsqrt(qr.square().sum() + 1e-6) * scale) + k_hat = kr * torch.rsqrt(kr.square().sum() + 1e-6) + else: + q_hat = qr * scale + k_hat = kr + projected = s @ k_hat + v_new = beta_t * (v[b, t, hv].to(torch.float64) - decay * projected) + s = decay * s + torch.outer(v_new, k_hat) + out[b, t, hv] = s @ q_hat + final[b, hv] = s + return out, final + + +def _make_inputs(B, T, Hk, Hv, Dk, Dv, dtype, gate_dtype, device, seed=0): + """Inputs follow the operator's production contract: q/k are l2-normalized + per head unless `use_qk_l2norm` is set (then the kernel normalizes them + itself). Feeding raw gaussian k would make the state recursion expand by + a factor ~|k|^2 per token, which is numerically unstable for any + float32 comparison. Scale keeps the state magnitude in a testable range. + """ + gen = torch.Generator(device="cpu").manual_seed(seed) + q = torch.randn(B, T, Hk, Dk, generator=gen) + k = torch.randn(B, T, Hk, Dk, generator=gen) + v = torch.randn(B, T, Hv, Dv, generator=gen) + g = -torch.rand(B, T, Hv, generator=gen) * 0.1 + beta = torch.sigmoid(torch.randn(B, T, Hv, generator=gen)) + initial = torch.randn(B, Hv, Dv, Dk, generator=gen) + q = q / q.square().sum(-1, keepdim=True).sqrt() + k = k / k.square().sum(-1, keepdim=True).sqrt() + initial = initial * 0.5 + return ( + q.to(dtype).to(device).contiguous(), + k.to(dtype).to(device).contiguous(), + v.to(dtype).to(device).contiguous(), + g.to(gate_dtype).to(device).contiguous(), + beta.to(gate_dtype).to(device).contiguous(), + initial.to(dtype).to(device).contiguous(), + ) + + +@pytest.mark.parametrize("use_qk_l2norm", (False, True)) +@pytest.mark.parametrize( + "dtype, gate_dtype, rtol, atol", + ( + # The kernel reduces the state matmuls in a different order than the + # sequential reference; float32 recursion accumulates that rounding + # difference, so the tolerance is a few ulps above the fp32 noise + # floor rather than the tight per-op default. + (torch.float32, torch.float32, 1e-4, 1e-4), + (torch.float16, torch.float16, 2e-2, 2e-2), + (torch.bfloat16, torch.bfloat16, 4e-2, 4e-2), + ), +) +def test_chunk_gated_delta_rule(dtype, gate_dtype, use_qk_l2norm, rtol, atol, device): + B, T, Hk, Hv, Dk, Dv = 2, 7, 1, 2, 64, 64 + q, k, v, g, beta, initial = _make_inputs( + B, T, Hk, Hv, Dk, Dv, dtype, gate_dtype, device + ) + out = torch.empty(B, T, Hv, Dv, dtype=dtype, device=device) + final = torch.empty(B, Hv, Dv, Dk, dtype=dtype, device=device) + + infini.ops.chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + initial, + out=out, + final_state=final, + use_qk_l2norm=use_qk_l2norm, + chunk_size=64, + stream=get_stream(device), + ) + + ref_out, ref_final = _reference(q, k, v, g, beta, initial, use_qk_l2norm) + torch.testing.assert_close(out, ref_out.to(dtype), rtol=rtol, atol=atol) + torch.testing.assert_close(final, ref_final.to(dtype), rtol=rtol, atol=atol) + + +@pytest.mark.parametrize( + "dtype, gate_dtype, rtol, atol", + ( + (torch.float32, torch.float32, 1e-4, 1e-4), + (torch.bfloat16, torch.bfloat16, 4e-2, 4e-2), + ), +) +def test_chunk_gated_delta_rule_state_pool(dtype, gate_dtype, rtol, atol, device): + """Indexed state pool: two sequences read distinct slots and write + their final states to other slots; untouched slots keep their values.""" + B, T, Hk, Hv, Dk, Dv = 2, 5, 1, 1, 64, 64 + q, k, v, g, beta, _ = _make_inputs( + B, T, Hk, Hv, Dk, Dv, dtype, gate_dtype, device, seed=3 + ) + pool = 4 + state_pool = torch.randn(pool, Hv, Dv, Dk).to(dtype).to(device) + initial_indices = torch.tensor((1, 3), dtype=torch.int32, device=device) + final_indices = torch.tensor((2, 0), dtype=torch.int32, device=device) + out = torch.empty(B, T, Hv, Dv, dtype=dtype, device=device) + before = state_pool.clone() + + infini.ops.chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + state_pool, + initial_state_indices=initial_indices, + final_state_indices=final_indices, + out=out, + use_qk_l2norm=False, + chunk_size=64, + stream=get_stream(device), + ) + + ref_out, _ = _reference( + q, k, v, g, beta, state_pool[initial_indices.long()].clone(), False + ) + torch.testing.assert_close(out, ref_out.to(dtype), rtol=rtol, atol=atol) + ref0, final0 = _reference( + q[0:1], + k[0:1], + v[0:1], + g[0:1], + beta[0:1], + state_pool[1:2].clone(), + False, + ) + ref1, final1 = _reference( + q[1:2], + k[1:2], + v[1:2], + g[1:2], + beta[1:2], + state_pool[3:4].clone(), + False, + ) + # Seq0 read slot 1 and wrote its final state to slot 2. + torch.testing.assert_close(state_pool[2], final0[0].to(dtype), rtol=rtol, atol=atol) + # Seq1 read slot 3 and wrote its final state to slot 0. + torch.testing.assert_close(state_pool[0], final1[0].to(dtype), rtol=rtol, atol=atol) + # Untouched slots 1 and 3 keep their original values. + torch.testing.assert_close(state_pool[1], before[1]) + torch.testing.assert_close(state_pool[3], before[3]) diff --git a/tests/test_moe_fused_dense.py b/tests/test_moe_fused_dense.py new file mode 100644 index 000000000..a2b2b03a2 --- /dev/null +++ b/tests/test_moe_fused_dense.py @@ -0,0 +1,138 @@ +"""Metax unit tests for the MoE fused-dense operator. + +Builds the packed-routing inputs the same way `moe_align_block_size` +produces them (sequential expert buckets, aligned to `block_size`), +then checks the fused gate/up/down projection against a per-expert +torch reference. +""" + +import torch +import pytest + +import infini.ops + +from tests.utils import get_stream + + +if not hasattr(infini.ops, "moe_fused_dense"): + pytest.skip( + "`moe_fused_dense` is not available on this platform", + allow_module_level=True, + ) + + +def _make_routing(topk_ids, num_experts, block_size, device): + """Produce sorted_token_ids/expert_ids/num_tokens_post_pad matching + the moe_align_block_size sequential-bucket layout.""" + n, topk = topk_ids.shape + pairs = n * topk + capacity = pairs + num_experts * (block_size - 1) + counts = torch.bincount(topk_ids.reshape(-1), minlength=num_experts) + sorted_token_ids = torch.full((capacity,), pairs, dtype=torch.int32, device=device) + # Sequential buckets: expert e occupies rows [offset_e, offset_e + count_e). + row = 0 + token_rows = [] + for e in range(num_experts): + token_rows.append((e, counts[e].item())) + row += counts[e].item() + pad = (-row) % block_size + row += pad + flat = topk_ids.reshape(-1) + row_cursor = 0 + for e in range(num_experts): + cnt = counts[e].item() + for i in range(cnt): + pass + row_cursor += cnt + pad = (-row_cursor) % block_size + row_cursor += pad + # simpler: compute per-expert rows then scatter + sorted_token_ids = torch.full((capacity,), pairs, dtype=torch.int32, device=device) + expert_ids = torch.full( + ((capacity + block_size - 1) // block_size,), + num_experts, + dtype=torch.int32, + device=device, + ) + offsets_i = 0 + for e in range(num_experts): + cnt = int(counts[e]) + block_start = offsets_i + for j in range(cnt): + # find (j+1)-th token routed to expert e + idx = (flat == e).nonzero()[j].item() + sorted_token_ids[block_start + j] = idx + offsets_i += cnt + offsets_i = (offsets_i + block_size - 1) // block_size * block_size + num_post = torch.tensor([offsets_i], dtype=torch.int32, device=device) + n_blocks_used = offsets_i // block_size + expert_ids[:n_blocks_used] = 0 + block_cursor = 0 + for e in range(num_experts): + cnt = int(counts[e]) + if cnt == 0: + continue + start_block = block_cursor + end_block = (block_cursor * block_size + cnt + block_size - 1) // block_size + expert_ids[start_block:end_block] = e + block_cursor = end_block + return sorted_token_ids, expert_ids, num_post + + +def _reference(hidden, w13, w2, topk_weights, topk_ids, block_size): + n, topk = topk_ids.shape + out = torch.zeros_like(hidden, dtype=torch.float32) + flat_ids = topk_ids.reshape(-1) + flat_w = topk_weights.reshape(-1) + m = w2.shape[2] + for route in range(n * topk): + e = int(flat_ids[route]) + token = route // topk + gate_w, up_w = w13[e][:m].float(), w13[e][m:].float() + hidden_f = hidden[token].float() + activated = torch.nn.functional.silu(hidden_f @ gate_w.T) * (hidden_f @ up_w.T) + out[token] += flat_w[route] * (activated @ w2[e].float().T) + return out + + +@pytest.mark.parametrize( + "dtype, rtol, atol", + ( + # The fused-dense implementation computes through the reduced + # (fp16/bf16) BLAS path; fp32 is rejected at Make time. + (torch.float16, 2e-2, 2e-2), + (torch.bfloat16, 4e-2, 4e-2), + ), +) +def test_moe_fused_dense(dtype, rtol, atol, device): + if device != "cuda": + pytest.skip("`moe_fused_dense` numeric test targets the metax/cuda backend") + torch.manual_seed(0) + n, h, m, e, topk, block_size = 4, 8, 6, 3, 2, 4 + hidden = torch.randn(n, h, dtype=dtype, device=device).contiguous() + w13 = torch.randn(e, 2 * m, h, dtype=dtype, device=device).contiguous() + w2 = torch.randn(e, h, m, dtype=dtype, device=device).contiguous() + logits = torch.randn(n, e, device=device) + topk_weights, topk_ids = torch.topk(torch.softmax(logits, -1), topk, dim=-1) + topk_weights = topk_weights.contiguous() + topk_ids = topk_ids.to(torch.int32).contiguous() + + sorted_ids, expert_ids, num_post = _make_routing(topk_ids, e, block_size, device) + + output = torch.empty(n, h, dtype=dtype, device=device) + infini.ops.moe_fused_dense( + output, + hidden, + w13, + w2, + topk_weights, + topk_ids, + sorted_ids, + expert_ids, + num_post, + block_size, + stream=get_stream(device), + ) + + expected = _reference(hidden, w13, w2, topk_weights, topk_ids, block_size) + torch.testing.assert_close(output, expected.to(dtype), rtol=rtol, atol=atol) diff --git a/tests/test_per_tensor_quant_i8.py b/tests/test_per_tensor_quant_i8.py new file mode 100644 index 000000000..00b962e3e --- /dev/null +++ b/tests/test_per_tensor_quant_i8.py @@ -0,0 +1,95 @@ +"""Metax unit tests for the per-tensor INT8 quantization operators. + +Covers the metax implementations in `per_tensor_i8`: +- PerTensorQuantI8: dynamic (scale = absmax / 127) and static (caller + provided scale) quantization, round-half-away-from-zero, saturation + to [-127, 127]. +- PerTensorDequantI8: packed int8 * scale -> floating point. +- Quantize -> dequantize round-trip within 1 ulp of the step size. +""" + +import torch +import pytest + +import infini.ops + +from tests.utils import get_stream + + +if not hasattr(infini.ops, "per_tensor_quant_i8"): + pytest.skip( + "`per_tensor_quant_i8` is not available on this platform", + allow_module_level=True, + ) + + +@pytest.mark.parametrize( + "dtype", + (torch.float32, torch.float16, torch.bfloat16), +) +def test_per_tensor_quant_i8_dynamic(dtype, device): + x = (torch.randn(3, 5) * 4.0).to(dtype).to(device) + packed = torch.empty(x.shape, dtype=torch.int8, device=device) + scale = torch.empty(1, dtype=torch.float32, device=device) + + infini.ops.per_tensor_quant_i8( + x, packed, scale, None, False, stream=get_stream(device) + ) + + expected_scale = x.abs().max().float() / 127.0 + torch.testing.assert_close(scale[0], expected_scale, rtol=1e-5, atol=1e-8) + q = (x.float() / expected_scale).round().clamp(-127, 127).to(torch.int8) + torch.testing.assert_close(packed, q, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "dtype", + (torch.float32, torch.float16), +) +def test_per_tensor_quant_i8_static(dtype, device): + x = (torch.randn(2, 3, 4) * 2.0).to(dtype).to(device) + packed = torch.empty(x.shape, dtype=torch.int8, device=device) + scale = torch.tensor([0.05], dtype=torch.float32, device=device) + + infini.ops.per_tensor_quant_i8( + x, packed, scale, None, True, stream=get_stream(device) + ) + + q = (x.float() / 0.05).round().clamp(-127, 127).to(torch.int8) + torch.testing.assert_close(packed, q, rtol=0, atol=0) + + +def test_per_tensor_quant_i8_saturation(device): + """Values beyond scale*127 saturate instead of wrapping.""" + x = torch.tensor((200.0, -200.0, 1.0), dtype=torch.float32, device=device) + packed = torch.empty(x.shape, dtype=torch.int8, device=device) + scale = torch.tensor([1.0], dtype=torch.float32, device=device) + + infini.ops.per_tensor_quant_i8( + x, packed, scale, None, True, stream=get_stream(device) + ) + + assert packed[0].item() == 127 + assert packed[1].item() == -127 + assert packed[2].item() == 1 + + +@pytest.mark.parametrize( + "dtype", + (torch.float32, torch.float16, torch.bfloat16), +) +def test_per_tensor_quant_dequant_roundtrip(dtype, device): + x = (torch.randn(4, 6) * 3.0).to(dtype).to(device) + packed = torch.empty(x.shape, dtype=torch.int8, device=device) + scale = torch.empty(1, dtype=torch.float32, device=device) + restored = torch.empty(x.shape, dtype=dtype, device=device) + + infini.ops.per_tensor_quant_i8( + x, packed, scale, None, False, stream=get_stream(device) + ) + infini.ops.per_tensor_dequant_i8( + restored, packed, scale, None, stream=get_stream(device) + ) + + step = scale[0].float() + torch.testing.assert_close(restored.float(), x.float(), rtol=0, atol=step) diff --git a/tests/test_recurrent_gated_delta_rule.py b/tests/test_recurrent_gated_delta_rule.py new file mode 100644 index 000000000..877c79ffc --- /dev/null +++ b/tests/test_recurrent_gated_delta_rule.py @@ -0,0 +1,182 @@ +"""Metax unit tests for the single-token (decode) gated delta rule operator. + +The reference mirrors the device kernel math. Per (batch, value head), +for the single token t=0 with state row s[Dv, Dk] (one row per value +dim): + k_hat = l2norm(k) (optional); q_hat = l2norm(q) * rsqrt(Dk) or rsqrt(Dk) + decay = exp(g); beta_t = beta + delta_v = (v - (decay * s_row) . k_hat) * beta_t (per value dim) + out = (decay * s_row) . q_hat + delta_v * (k_hat . q_hat) + s_row = decay * s_row + delta_v * k_hat +""" + +import torch +import pytest + +import infini.ops + +from tests.utils import get_stream + + +if not hasattr(infini.ops, "recurrent_gated_delta_rule"): + pytest.skip( + "`recurrent_gated_delta_rule` is not available on this platform", + allow_module_level=True, + ) + + +def _sanitize(*tensors): + """MACA's host-to-device copy reports wrong stride metadata (the + data itself is contiguous); the operator kernels index through the + reported strides, so normalize every device tensor here.""" + return tuple(t.contiguous() if t.is_cuda else t for t in tensors) + + +def _reference(q, k, v, g, beta, initial_state, use_qk_l2norm): + """q/k/v: [B, 1, H, D]; g/beta: [B, 1, H]; initial_state: [pool/B, H, Dv, Dk]. + + Computed in float64 so the reference stays exact regardless of the + session-wide TF32 matmul setting (`tests/conftest.py`). + """ + B, _, Hk, Dk = q.shape + Hv = v.shape[2] + ratio = Hv // Hk + state = initial_state.to(torch.float64).clone() + out = torch.empty_like(v, dtype=torch.float64) + final = torch.empty_like(initial_state, dtype=torch.float64) + + query_scale = Dk**-0.5 + for b in range(B): + for hv in range(Hv): + hk = hv // ratio + qr = q[b, 0, hk].to(torch.float64) + kr = k[b, 0, hk].to(torch.float64) + decay = torch.exp(g[b, 0, hv].to(torch.float64)) + beta_t = beta[b, 0, hv].to(torch.float64) + if use_qk_l2norm: + q_hat = qr * torch.rsqrt(qr.square().sum() + 1e-6) * query_scale + k_hat = kr * torch.rsqrt(kr.square().sum() + 1e-6) + else: + q_hat = qr * query_scale + k_hat = kr + s = state[b, hv].to(torch.float64) # [Dv, Dk] + decayed = s * decay # [Dv, Dk] + kv = decayed @ k_hat # [Dv] + delta = (v[b, 0, hv].to(torch.float64) - kv) * beta_t + out[b, 0, hv] = decayed @ q_hat + delta * (k_hat @ q_hat) + final[b, hv] = s * decay + torch.outer(delta, k_hat) + return out, final + + +def _make_inputs(B, Hk, Hv, Dk, Dv, dtype, gate_dtype, device, seed=0): + gen = torch.Generator(device="cpu").manual_seed(seed) + q = torch.randn(B, 1, Hk, Dk, generator=gen).to(dtype).to(device).contiguous() + k = torch.randn(B, 1, Hk, Dk, generator=gen).to(dtype).to(device).contiguous() + v = torch.randn(B, 1, Hv, Dv, generator=gen).to(dtype).to(device).contiguous() + g = ( + (-torch.rand(B, 1, Hv, generator=gen) * 0.1) + .to(gate_dtype) + .to(device) + .contiguous() + ) + beta = ( + torch.sigmoid(torch.randn(B, 1, Hv, generator=gen)) + .to(gate_dtype) + .to(device) + .contiguous() + ) + initial = ( + torch.randn(B, Hv, Dv, Dk, generator=gen).to(dtype).to(device).contiguous() + ) + return _sanitize(q, k, v, g, beta, initial) + + +@pytest.mark.parametrize("use_qk_l2norm", (False, True)) +@pytest.mark.parametrize( + "dtype, gate_dtype, rtol, atol", + ( + (torch.float32, torch.float32, 2e-5, 2e-5), + (torch.float16, torch.float16, 2e-2, 2e-2), + (torch.bfloat16, torch.bfloat16, 4e-2, 4e-2), + ), +) +def test_recurrent_gated_delta_rule( + dtype, gate_dtype, use_qk_l2norm, rtol, atol, device +): + B, Hk, Hv, Dk, Dv = 2, 1, 2, 64, 64 + q, k, v, g, beta, initial = _make_inputs( + B, Hk, Hv, Dk, Dv, dtype, gate_dtype, device + ) + out = torch.empty(B, 1, Hv, Dv, dtype=dtype, device=device) + final = torch.empty(B, Hv, Dv, Dk, dtype=dtype, device=device) + + infini.ops.recurrent_gated_delta_rule( + q, + k, + v, + g, + beta, + initial, + out=out, + final_state=final, + use_qk_l2norm=use_qk_l2norm, + stream=get_stream(device), + ) + + ref_out, ref_final = _reference(q, k, v, g, beta, initial, use_qk_l2norm) + torch.testing.assert_close(out, ref_out.to(dtype), rtol=rtol, atol=atol) + torch.testing.assert_close(final, ref_final.to(dtype), rtol=rtol, atol=atol) + + +@pytest.mark.parametrize( + "dtype, gate_dtype, rtol, atol", + ( + (torch.float32, torch.float32, 2e-5, 2e-5), + (torch.bfloat16, torch.bfloat16, 4e-2, 4e-2), + ), +) +def test_recurrent_gated_delta_rule_state_pool(dtype, gate_dtype, rtol, atol, device): + """Indexed state pool: two sequences read distinct slots and write + their final states to other slots; untouched slots keep their values.""" + B, Hk, Hv, Dk, Dv = 2, 1, 1, 64, 64 + q, k, v, g, beta, _ = _make_inputs( + B, Hk, Hv, Dk, Dv, dtype, gate_dtype, device, seed=5 + ) + pool = 4 + state_pool = torch.randn(pool, Hv, Dv, Dk).to(dtype).to(device) + initial_indices = torch.tensor((1, 3), dtype=torch.int32, device=device) + final_indices = torch.tensor((2, 0), dtype=torch.int32, device=device) + out = torch.empty(B, 1, Hv, Dv, dtype=dtype, device=device) + before = state_pool.clone() + + infini.ops.recurrent_gated_delta_rule( + q, + k, + v, + g, + beta, + state_pool, + initial_state_indices=initial_indices, + final_state_indices=final_indices, + out=out, + use_qk_l2norm=False, + stream=get_stream(device), + ) + + ref_out, _ = _reference( + q, k, v, g, beta, state_pool[initial_indices.long()].clone(), False + ) + torch.testing.assert_close(out, ref_out.to(dtype), rtol=rtol, atol=atol) + _, final0 = _reference( + q[0:1], k[0:1], v[0:1], g[0:1], beta[0:1], state_pool[1:2].clone(), False + ) + _, final1 = _reference( + q[1:2], k[1:2], v[1:2], g[1:2], beta[1:2], state_pool[3:4].clone(), False + ) + # Seq0 read slot 1 and wrote its final state to slot 2. + torch.testing.assert_close(state_pool[2], final0[0].to(dtype), rtol=rtol, atol=atol) + # Seq1 read slot 3 and wrote its final state to slot 0. + torch.testing.assert_close(state_pool[0], final1[0].to(dtype), rtol=rtol, atol=atol) + # Untouched slots 1 and 3 keep their original values. + torch.testing.assert_close(state_pool[1], before[1]) + torch.testing.assert_close(state_pool[3], before[3])