From 5ac9c1e41e42d7682b5f10da7fed4486f810dcc3 Mon Sep 17 00:00:00 2001 From: rubik Date: Thu, 19 Nov 2026 10:47:17 +0800 Subject: [PATCH] feat(metax): add scalar Mul and native DeepseekMoe for DeepSeek-style MoE Add the MetaX operators the 021 model (DeepseekV2-style MoE with 80 experts, top-7 routing) needs beyond the core inference set: - Scalar Mul: the shared CudaMul only prepares tensor x tensor multiplication; its scalar constructor is protected and its scalar operator() rejects scalar mode, so dispatching Mul(tensor, double, tensor) found no MetaX provider. Add a native scalar kernel with contiguous and strided paths over fp16/bf16/fp32/fp64; the strided metadata is allocated at prepare time so the buffer outlives the asynchronous kernel launch. - DeepseekMoe: replace the throw stub with a native expert-grouped implementation. Routing runs on device (expert counting, prefix scan, pack-by-expert), per-expert gate/up/down projections go through mcblasGemmStridedBatchedEx with batch=1 (the pattern moe_fused_dense already exercises on MetaX), SiLU*up*topk_weight applies in place, and a final kernel sums the top-k contributions per token. Large prefills are chunked to bound the workspace. Together these run the fused MoE path end to end on MetaX with no engine-side fallback: on 021-32B TP2 (C550, MACA 3.5.3) prefill TTFT drops from 2340 ms to 207 ms (54.7 to 618 tok/s) and decode improves from 46.7 ms to 42.0 ms per token versus the CPU-routed per-expert Gemm fallback. --- .../cuda/metax/ops/deepseek_moe/kernel.h | 398 ++++++++++++++++++ src/native/cuda/metax/ops/mul/kernel.h | 28 ++ src/native/cuda/metax/ops/mul/scalar.h | 170 ++++++++ 3 files changed, 596 insertions(+) create mode 100644 src/native/cuda/metax/ops/deepseek_moe/kernel.h create mode 100644 src/native/cuda/metax/ops/mul/scalar.h diff --git a/src/native/cuda/metax/ops/deepseek_moe/kernel.h b/src/native/cuda/metax/ops/deepseek_moe/kernel.h new file mode 100644 index 000000000..8012f4e8d --- /dev/null +++ b/src/native/cuda/metax/ops/deepseek_moe/kernel.h @@ -0,0 +1,398 @@ +#ifndef INFINI_OPS_METAX_DEEPSEEK_MOE_KERNEL_H_ +#define INFINI_OPS_METAX_DEEPSEEK_MOE_KERNEL_H_ + +#include +#include +#include +#include +#include +#include +#include + +// MACA's cu-bridge provides the CUDA-style half/bfloat16 headers; +// __nv_bfloat16 aliases __maca_bfloat16 (same include pattern as +// native/cuda/metax/ops/chunk_gated_delta_rule). +#include +#include + +#include "base/deepseek_moe.h" +#include "native/cuda/metax/blas.h" +#include "native/cuda/metax/runtime_.h" + +namespace infini::ops { +namespace metax_deepseek_moe_detail { + +constexpr int kThreads = 256; +// The prefix-scan kernel runs in a single block of kThreads threads. +// 021 routes over 80 experts; nvidia's grouped path also caps at 256. +constexpr int kMaxExperts = 256; +// Tokens processed per pass so the intermediate workspace stays bounded +// for long-context prefill (021 supports 256k tokens). +constexpr size_t kChunkTokens = 4096; + +inline size_t AlignUp(size_t value, size_t alignment = 256) { + return (value + alignment - 1) / alignment * alignment; +} + +inline void CheckMc(mcError_t status, const char* what) { + if (status != mcSuccess) { + throw std::runtime_error(std::string("MetaX DeepseekMoe ") + what + + " failed: " + std::to_string(int(status))); + } +} + +inline mcblasHandle_t GetHandle() { + static thread_local mcblasHandle_t handle = []() { + mcblasHandle_t h{}; + if (Blas::BlasCreate(&h) != MCBLAS_STATUS_SUCCESS) + throw std::runtime_error( + "MetaX DeepseekMoe: creating mcBLAS handle failed"); + return h; + }(); + return handle; +} + +struct WorkspaceLayout { + size_t counts, offsets, positions, row_to_route, route_to_row, packed_hidden, + gate_buf, up_buf, total; +}; + +inline WorkspaceLayout Layout(size_t num_experts, size_t routes, + size_t hidden_size, size_t intermediate_size, + size_t element_size) { + WorkspaceLayout l{}; + l.counts = 0; + // Counts and offsets stay adjacent so one D2H copy fetches both. + l.offsets = l.counts + (num_experts + 1) * sizeof(int); + l.positions = AlignUp(l.offsets + (num_experts + 1) * sizeof(int)); + l.row_to_route = AlignUp(l.positions + (num_experts + 1) * sizeof(int)); + l.route_to_row = l.row_to_route + routes * sizeof(int); + l.packed_hidden = AlignUp(l.route_to_row + routes * sizeof(int)); + l.gate_buf = AlignUp(l.packed_hidden + routes * hidden_size * element_size); + l.up_buf = AlignUp(l.gate_buf + routes * intermediate_size * element_size); + l.total = AlignUp(l.up_buf + routes * intermediate_size * element_size); + return l; +} + +__global__ void CountExpertsKernel(const int* topk_indices, int* counts, + int routes, int num_experts) { + const int route = blockIdx.x * blockDim.x + threadIdx.x; + if (route >= routes) { + return; + } + const int expert = topk_indices[route]; + if (expert >= 0 && expert < num_experts) { + atomicAdd(counts + expert, 1); + } +} + +__global__ void PrefixCountsKernel(const int* counts, int* offsets, + int num_experts) { + extern __shared__ int scan[]; + const int tid = threadIdx.x; + scan[tid] = tid < num_experts ? counts[tid] : 0; + __syncthreads(); + for (int stride = 1; stride < blockDim.x; stride <<= 1) { + int value = 0; + if (tid >= stride) { + value = scan[tid - stride]; + } + __syncthreads(); + scan[tid] += value; + __syncthreads(); + } + if (tid == 0) { + offsets[0] = 0; + } + if (tid < num_experts) { + offsets[tid + 1] = scan[tid]; + } +} + +template +__global__ void PackHiddenKernel(const T* hidden, const int* topk_indices, + const int* offsets, int* positions, + int* row_to_route, int* route_to_row, + T* packed_hidden, int routes, int hidden_size, + int topk, int num_experts) { + const int route = blockIdx.x; + if (route >= routes) { + return; + } + const int expert = topk_indices[route]; + if (expert < 0 || expert >= num_experts) { + return; + } + const int tid = threadIdx.x; + __shared__ int row; + if (tid == 0) { + const int local = atomicAdd(positions + expert, 1); + row = offsets[expert] + local; + row_to_route[row] = route; + route_to_row[route] = row; + } + __syncthreads(); + const int token = route / topk; + for (int h = tid; h < hidden_size; h += blockDim.x) { + packed_hidden[static_cast(row) * hidden_size + h] = + hidden[static_cast(token) * hidden_size + h]; + } +} + +template +__global__ void SwigluWeightKernel(T* activated, const T* gate, const T* up, + const float* topk_weights, + const int* row_to_route, int rows, + int intermediate_size) { + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + const int total = rows * intermediate_size; + if (idx >= total) { + return; + } + const int row = idx / intermediate_size; + const int route = row_to_route[row]; + const float g = static_cast(gate[idx]); + const float u = static_cast(up[idx]); + const float silu = g / (1.0f + expf(-g)); + activated[idx] = static_cast(silu * u * topk_weights[route]); +} + +template +__global__ void SumGroupedOutKernel(T* out, const T* packed_out, + const int* route_to_row, int ntokens, + int hidden_size, int topk) { + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + const int total = ntokens * hidden_size; + if (idx >= total) { + return; + } + const int token = idx / hidden_size; + const int h = idx - token * hidden_size; + float acc = 0.0f; + for (int k = 0; k < topk; ++k) { + const int route = token * topk + k; + const int row = route_to_row[route]; + acc += static_cast( + packed_out[static_cast(row) * hidden_size + h]); + } + out[idx] = static_cast(acc); +} + +// Expert-grouped MoE: pack routed tokens by expert, run gate/up/down GEMMs +// per expert (batch=1 strided GEMM, the pattern moe_fused_dense already +// exercises on MetaX), apply SiLU*up*weight in place, then sum the topk +// contributions per token. Large prefills are chunked so the workspace +// stays bounded. +template +void RunChunks(std::uint8_t* base, const WorkspaceLayout& layout, + size_t ntokens, size_t hidden_size, size_t topk, + size_t intermediate_size, size_t num_experts, const T* hidden, + const int* topk_indices, const float* topk_weights, + const std::vector& gate_ptrs, + const std::vector& up_ptrs, + const std::vector& down_ptrs, T* out, + Runtime::Stream stream, + mcblasHandle_t handle) { + using MetaxBlas = Blas; + using MetaxRuntime = Runtime; + const auto blas_dtype = std::is_same_v ? macaDataType(MACA_R_16F) + : macaDataType(MACA_R_16BF); + const int hidden_i = static_cast(hidden_size); + const int inter_i = static_cast(intermediate_size); + const int topk_i = static_cast(topk); + const int experts_i = static_cast(num_experts); + + auto* counts = reinterpret_cast(base + layout.counts); + auto* offsets = reinterpret_cast(base + layout.offsets); + auto* positions = reinterpret_cast(base + layout.positions); + auto* row_to_route = reinterpret_cast(base + layout.row_to_route); + auto* route_to_row = reinterpret_cast(base + layout.route_to_row); + auto* packed = reinterpret_cast(base + layout.packed_hidden); + auto* gate_buf = reinterpret_cast(base + layout.gate_buf); + auto* up_buf = reinterpret_cast(base + layout.up_buf); + + std::vector host_counts(num_experts + 1); + std::vector host_offsets(num_experts + 1); + + // Weights are row-major [out_features, in_features]; OP_T turns them + // into column-major (in_features, out_features), matching the nvidia + // run_gemm convention (m=out, n=tokens, k=in). + auto gemm = [&](const void* a, const void* b, void* c, int m, int n, int k, + int lda, int ldb, int ldc) { + const float alpha = 1.0f; + const float beta = 0.0f; + const auto status = MetaxBlas::BlasGemmStridedBatchedEx( + handle, MetaxBlas::BLAS_OP_T, MetaxBlas::BLAS_OP_N, m, n, k, &alpha, a, + blas_dtype, lda, 0, b, blas_dtype, ldb, 0, &beta, c, blas_dtype, ldc, 0, + 1, MetaxBlas::BLAS_COMPUTE_32F, MetaxBlas::BLAS_GEMM_DEFAULT); + if (status != MCBLAS_STATUS_SUCCESS) { + throw std::runtime_error("MetaX DeepseekMoe GEMM failed: " + + std::to_string(int(status))); + } + }; + + for (size_t begin = 0; begin < ntokens; begin += kChunkTokens) { + const int tokens = + static_cast(std::min(kChunkTokens, ntokens - begin)); + const int routes = tokens * topk_i; + const T* hidden_chunk = hidden + begin * hidden_size; + const int* ids_chunk = topk_indices + begin * topk_i; + const float* weights_chunk = topk_weights + begin * topk_i; + T* out_chunk = out + begin * hidden_size; + + CheckMc(MetaxRuntime::MemsetAsync(counts, 0, + (num_experts + 1) * sizeof(int), stream), + "memset"); + CheckMc(MetaxRuntime::MemsetAsync(positions, 0, + (num_experts + 1) * sizeof(int), stream), + "memset"); + CountExpertsKernel<<<(routes + kThreads - 1) / kThreads, kThreads, 0, + stream>>>(ids_chunk, counts, routes, experts_i); + PrefixCountsKernel<<<1, kThreads, kThreads * sizeof(int), stream>>>( + counts, offsets, experts_i); + PackHiddenKernel<<>>( + hidden_chunk, ids_chunk, offsets, positions, row_to_route, route_to_row, + packed, routes, hidden_i, topk_i, experts_i); + CheckMc(MetaxRuntime::MemcpyAsync( + host_counts.data(), counts, (num_experts + 1) * sizeof(int), + MetaxRuntime::kMemcpyDeviceToHost, stream), + "counts copy"); + CheckMc(MetaxRuntime::MemcpyAsync( + host_offsets.data(), offsets, (num_experts + 1) * sizeof(int), + MetaxRuntime::kMemcpyDeviceToHost, stream), + "offsets copy"); + CheckMc(MetaxRuntime::StreamSynchronize(stream), "route sync"); + + for (int e = 0; e < experts_i; ++e) { + const int count = host_counts[e]; + if (count <= 0) { + continue; + } + const size_t off = static_cast(host_offsets[e]); + gemm(gate_ptrs[e], packed + off * hidden_size, + gate_buf + off * intermediate_size, inter_i, count, hidden_i, + hidden_i, hidden_i, inter_i); + gemm(up_ptrs[e], packed + off * hidden_size, + up_buf + off * intermediate_size, inter_i, count, hidden_i, hidden_i, + hidden_i, inter_i); + } + SwigluWeightKernel + <<<(static_cast(routes) * intermediate_size + kThreads - 1) / + kThreads, + kThreads, 0, stream>>>(gate_buf, gate_buf, up_buf, weights_chunk, + row_to_route, routes, inter_i); + for (int e = 0; e < experts_i; ++e) { + const int count = host_counts[e]; + if (count <= 0) { + continue; + } + const size_t off = static_cast(host_offsets[e]); + gemm(down_ptrs[e], gate_buf + off * intermediate_size, + packed + off * hidden_size, hidden_i, count, inter_i, inter_i, + inter_i, hidden_i); + } + SumGroupedOutKernel + <<<(static_cast(tokens) * hidden_size + kThreads - 1) / + kThreads, + kThreads, 0, stream>>>(out_chunk, packed, route_to_row, tokens, + hidden_i, topk_i); + } + // Keep workspace reuse safe across chunks / subsequent layers. + CheckMc(MetaxRuntime::StreamSynchronize(stream), "final sync"); +} + +} // namespace metax_deepseek_moe_detail + +// Native MetaX DeepseekMoe: expert-grouped GEMM path. The routing +// (count/scan/pack), activation and reduction kernels run on device and +// the per-expert projections go through mcBLAS, so no per-expert operator +// submits cross the dispatcher. +template <> +class Operator : public DeepseekMoe { + public: + using DeepseekMoe::DeepseekMoe; + + bool capture_safe() const override { return false; } + + size_t workspace_size_in_bytes() const override { + const auto& hidden = metadata_[1]; + const size_t esz = hidden.element_size(); + const size_t ntokens = std::min( + hidden.size(0), metax_deepseek_moe_detail::kChunkTokens); + const size_t routes = ntokens * metadata_[2].size(1); + return metax_deepseek_moe_detail::Layout(num_experts_, routes, + hidden.size(1), intermediate_, esz) + .total; + } + + void operator()(Tensor output, const Tensor hidden, const Tensor ids, + const Tensor weights, const std::vector gate, + const std::vector up, const std::vector down, + size_t intermediate, size_t num_experts) const override { + namespace detail = metax_deepseek_moe_detail; + using MetaxBlas = Blas; + using MetaxRuntime = Runtime; + + Invocation(output, hidden, ids, weights, gate, up, down, intermediate, + num_experts); + if (hidden.size(0) == 0) { + return; + } + moe_detail::Require(num_experts <= size_t(detail::kMaxExperts)); + moe_detail::Require(hidden.dtype() == DataType::kFloat16 || + hidden.dtype() == DataType::kBFloat16); + + const size_t hidden_size = hidden.size(1); + const size_t topk = ids.size(1); + const size_t ntokens = hidden.size(0); + const size_t esz = hidden.element_size(); + + moe_detail::Workspace( + workspace_, workspace_size_in_bytes_, + detail::Layout(num_experts, + std::min(ntokens, detail::kChunkTokens) * topk, + hidden_size, intermediate, esz) + .total, + DeepseekMoe::Collect(output, hidden, ids, weights, gate, up, down)); + + const auto stream = + static_cast(stream_ ? stream_ : nullptr); + auto handle = detail::GetHandle(); + MetaxBlas::BlasSetStream(handle, stream); + + std::vector gate_ptrs(num_experts); + std::vector up_ptrs(num_experts); + std::vector down_ptrs(num_experts); + for (size_t e = 0; e < num_experts; ++e) { + gate_ptrs[e] = gate[e].data(); + up_ptrs[e] = up[e].data(); + down_ptrs[e] = down[e].data(); + } + + const auto layout = detail::Layout( + num_experts, std::min(ntokens, detail::kChunkTokens) * topk, + hidden_size, intermediate, esz); + auto* base = static_cast(workspace_); + + if (hidden.dtype() == DataType::kFloat16) { + detail::RunChunks<__half>( + base, layout, ntokens, hidden_size, topk, intermediate, num_experts, + reinterpret_cast(hidden.data()), + reinterpret_cast(ids.data()), + reinterpret_cast(weights.data()), gate_ptrs, up_ptrs, + down_ptrs, reinterpret_cast<__half*>(output.data()), stream, handle); + } else { + detail::RunChunks<__nv_bfloat16>( + base, layout, ntokens, hidden_size, topk, intermediate, num_experts, + reinterpret_cast(hidden.data()), + reinterpret_cast(ids.data()), + reinterpret_cast(weights.data()), gate_ptrs, up_ptrs, + down_ptrs, reinterpret_cast<__nv_bfloat16*>(output.data()), stream, + handle); + } + } +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_METAX_DEEPSEEK_MOE_KERNEL_H_ diff --git a/src/native/cuda/metax/ops/mul/kernel.h b/src/native/cuda/metax/ops/mul/kernel.h index b80e96843..aac4f22ec 100644 --- a/src/native/cuda/metax/ops/mul/kernel.h +++ b/src/native/cuda/metax/ops/mul/kernel.h @@ -1,9 +1,11 @@ #ifndef INFINI_OPS_METAX_MUL_KERNEL_H_ #define INFINI_OPS_METAX_MUL_KERNEL_H_ +#include #include #include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/ops/mul/scalar.h" #include "native/cuda/metax/runtime_.h" #include "native/cuda/ops/mul/kernel.h" @@ -14,6 +16,32 @@ class Operator : public CudaMul> { public: using CudaMul>::CudaMul; + using CudaMul>::operator(); + + // The shared `CudaMul` scalar constructor is protected and its + // scalar `operator()` throws, so the inherited `using` declarations + // never satisfy the scalar `Mul(tensor, double, tensor)` dispatch. + // The DeepSeek-V2 MoE router scale needs that overload; serve it + // with a MetaX scalar kernel. + Operator(const Tensor input, const double alpha, Tensor out) + : CudaMul>{input, alpha, out}, + scalar_(std::in_place, input, alpha, out) {} + + static bool supports(const Tensor input, const double alpha, Tensor out) { + return MetaxScalarMul::supports(input, alpha, out); + } + + void operator()(const Tensor input, const double alpha, + Tensor out) const override { + if (!scalar_) + throw std::invalid_argument("MetaX Mul prepared for tensor operands"); + scalar_->Run(input, alpha, out, + static_cast::Stream>( + stream_ ? stream_ : nullptr)); + } + + private: + std::optional scalar_; }; } // namespace infini::ops diff --git a/src/native/cuda/metax/ops/mul/scalar.h b/src/native/cuda/metax/ops/mul/scalar.h new file mode 100644 index 000000000..1d1a8b7b0 --- /dev/null +++ b/src/native/cuda/metax/ops/mul/scalar.h @@ -0,0 +1,170 @@ +#ifndef INFINI_OPS_METAX_MUL_SCALAR_H_ +#define INFINI_OPS_METAX_MUL_SCALAR_H_ + +#include +#include +#include +#include + +#include "native/cuda/caster.cuh" +#include "native/cuda/device_buffer.h" +#include "native/cuda/metax/caster.cuh" +#include "native/cuda/metax/runtime_.h" +#include "native/cuda/ops/mul/kernel.cuh" + +namespace infini::ops::metax_mul_scalar_detail { + +template +__global__ void MulScalarContiguousKernel(T* __restrict__ out, + const T* __restrict__ input, T alpha, + std::size_t count) { + const auto start = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const auto step = static_cast(gridDim.x) * blockDim.x; + for (std::size_t linear = start; linear < count; linear += step) { + out[linear] = MulOp{}(input[linear], alpha); + } +} + +template +__global__ void MulScalarStridedKernel( + T* __restrict__ out, const T* __restrict__ input, T alpha, + std::size_t count, std::int64_t ndim, const std::size_t* __restrict__ shape, + const std::ptrdiff_t* __restrict__ input_strides, + const std::ptrdiff_t* __restrict__ out_strides) { + const auto start = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const auto step = static_cast(gridDim.x) * blockDim.x; + for (std::size_t linear = start; linear < count; linear += step) { + std::size_t remaining = linear; + std::ptrdiff_t input_offset = 0, out_offset = 0; + for (std::int64_t d = ndim; d-- > 0;) { + const auto coord = static_cast( + remaining % static_cast(shape[d])); + remaining /= static_cast(shape[d]); + input_offset += coord * input_strides[d]; + out_offset += coord * out_strides[d]; + } + out[out_offset] = MulOp{}(input[input_offset], alpha); + } +} + +} // namespace infini::ops::metax_mul_scalar_detail + +namespace infini::ops { + +// Scalar-multiplier provider for MetaX: `Mul(tensor, double, tensor)`. +// The shared `CudaMul` only implements the tensor x tensor ctor; the +// scalar ctor there is protected and its `operator()` rejects scalar +// mode, so dispatch of the scalar overload finds no MetaX provider. +// The DeepSeek-V2 MoE router scale (routed_scaling_factor) needs it. +class MetaxScalarMul { + public: + static bool supports(const Tensor input, const double, Tensor out) { + if (input.device().type() != Device::Type::kMetax || + input.device() != out.device() || input.dtype() != out.dtype()) + return false; + switch (input.dtype()) { + case DataType::kFloat16: + case DataType::kBFloat16: + case DataType::kFloat32: + case DataType::kFloat64: + break; + default: + return false; + } + const auto ndim = input.ndim(); + if (out.shape() != input.shape() || out.strides().size() != ndim || + input.strides().size() != ndim) + return false; + // The strided kernel indexes linearly over the output; require a + // contiguous output so the linear index maps directly. + return out.IsContiguous(); + } + + MetaxScalarMul(const Tensor input, const double alpha, Tensor out) + : input_meta_(input), out_meta_(out), alpha_(alpha) { + count_ = out.numel(); + contiguous_ = input.IsContiguous() && out.IsContiguous(); + if (contiguous_) { + return; + } + shape_.assign(input.shape().begin(), input.shape().end()); + input_strides_.assign(input.strides().begin(), input.strides().end()); + out_strides_.assign(out.strides().begin(), out.strides().end()); + // The strided kernel reads this metadata asynchronously, so the + // allocation must outlive the launch: keep it as a member instead + // of a Run-scoped buffer whose Free would race the kernel. + const std::size_t shape_bytes = shape_.size() * sizeof(Tensor::Size); + strides_bytes_ = shape_.size() * sizeof(Tensor::Stride); + const std::size_t metadata_bytes = shape_bytes + 2 * strides_bytes_; + std::vector host(metadata_bytes); + std::memcpy(host.data(), shape_.data(), shape_bytes); + std::memcpy(host.data() + shape_bytes, input_strides_.data(), + strides_bytes_); + std::memcpy(host.data() + shape_bytes + strides_bytes_, out_strides_.data(), + strides_bytes_); + metadata_ptr_ = metadata_.Allocate(metadata_bytes); + if (Runtime::Memcpy( + const_cast(metadata_ptr_), host.data(), metadata_bytes, + Runtime::kMemcpyHostToDevice) != + Runtime::kSuccess) { + throw std::runtime_error("MetaX scalar Mul metadata copy failed"); + } + } + + void Run(const Tensor input, const double alpha, Tensor out, + typename Runtime::Stream stream) const { + if (!std::equal_to{}(input, input_meta_) || + !std::equal_to{}(out, out_meta_) || alpha != alpha_) + throw std::invalid_argument( + "MetaX scalar Mul metadata or alpha changed after Make"); + if (!count_) return; + DispatchFunc( + {static_cast(input.dtype())}, + [&](auto list_tag) { + using T = TypeMapType(list_tag)>; + const auto scalar = Caster::Cast(alpha_); + auto* d_out = reinterpret_cast(out.data()); + const T* d_input = reinterpret_cast(input.data()); + constexpr unsigned kThreads = 256; + const auto blocks = static_cast( + std::min((count_ - 1) / kThreads + 1, 65535)); + if (contiguous_) { + metax_mul_scalar_detail::MulScalarContiguousKernel + <<>>(d_out, d_input, scalar, + count_); + } else { + const std::size_t shape_bytes = + shape_.size() * sizeof(Tensor::Size); + metax_mul_scalar_detail::MulScalarStridedKernel + <<>>( + d_out, d_input, scalar, count_, + static_cast(shape_.size()), + reinterpret_cast(metadata_ptr_), + reinterpret_cast(metadata_ptr_ + + shape_bytes), + reinterpret_cast( + metadata_ptr_ + shape_bytes + strides_bytes_)); + } + }, + "MetaxScalarMul::Run"); + } + + private: + Tensor input_meta_; + Tensor out_meta_; + double alpha_; + std::size_t count_{0}; + bool contiguous_{false}; + std::size_t strides_bytes_{0}; + std::vector shape_{}; + std::vector input_strides_{}; + std::vector out_strides_{}; + CudaDeviceBuffer> metadata_; + std::byte* metadata_ptr_{nullptr}; +}; + +} // namespace infini::ops + +#endif