From 98b063854d68a68ee124404eaf60a138190e48b5 Mon Sep 17 00:00:00 2001 From: jingbo Date: Thu, 24 Sep 2026 10:33:09 +0800 Subject: [PATCH 1/3] feat(ascend): add FRACTAL_NZ weight layout support Convert eligible weights with aclnnTransMatmulWeight and execute them through aclnnMatmulWeightNz. --- src/base/matmul_weight_nz.h | 53 +++++++++++++ src/base/trans_matmul_weight_nz.h | 41 ++++++++++ .../ascend/ops/matmul_weight_nz/kernel.h | 78 +++++++++++++++++++ .../ops/trans_matmul_weight_nz/kernel.h | 66 ++++++++++++++++ tests/test_matmul_weight_nz.py | 46 +++++++++++ 5 files changed, 284 insertions(+) create mode 100644 src/base/matmul_weight_nz.h create mode 100644 src/base/trans_matmul_weight_nz.h create mode 100644 src/native/ascend/ops/matmul_weight_nz/kernel.h create mode 100644 src/native/ascend/ops/trans_matmul_weight_nz/kernel.h create mode 100644 tests/test_matmul_weight_nz.py diff --git a/src/base/matmul_weight_nz.h b/src/base/matmul_weight_nz.h new file mode 100644 index 000000000..9610a86dd --- /dev/null +++ b/src/base/matmul_weight_nz.h @@ -0,0 +1,53 @@ +#ifndef INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_ +#define INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class MatmulWeightNz : public Operator { + public: + MatmulWeightNz(const Tensor input, const Tensor weight, Tensor out) + : rows_{input.size(0)}, + channels_{input.size(1)}, + outputs_{weight.size(1)} { + assert(input.ndim() == 2 && weight.ndim() == 2 && out.ndim() == 2 && + "operator `MatmulWeightNz` requires 2D tensors"); + assert(input.size(1) == weight.size(0) && + "operator `MatmulWeightNz` input channels must match weight rows"); + assert(out.size(0) == input.size(0) && + "operator `MatmulWeightNz` output rows must match input"); + assert(out.size(1) == weight.size(1) && + "operator `MatmulWeightNz` output columns must match weight"); + assert(input.dtype() == weight.dtype() && input.dtype() == out.dtype() && + "operator `MatmulWeightNz` requires matching dtypes"); + assert((input.dtype() == DataType::kFloat16 || + input.dtype() == DataType::kBFloat16) && + "operator `MatmulWeightNz` supports fp16 or bf16 only"); + assert(weight.stride(0) == static_cast(weight.size(1)) && + weight.stride(1) == 1 && + "operator `MatmulWeightNz` weight must be contiguous"); + assert(input.stride(0) == static_cast(input.size(1)) && + input.stride(1) == 1 && + "operator `MatmulWeightNz` input must be contiguous"); + assert(out.stride(0) == static_cast(out.size(1)) && + out.stride(1) == 1 && + "operator `MatmulWeightNz` output must be contiguous"); + assert(weight.size(0) % 16 == 0 && weight.size(1) % 16 == 0 && + "operator `MatmulWeightNz` requires 16-aligned weight shape"); + } + + virtual void operator()(const Tensor input, const Tensor weight, + Tensor out) const = 0; + + protected: + Tensor::Size rows_; + Tensor::Size channels_; + Tensor::Size outputs_; +}; + +} // namespace infini::ops + +#endif diff --git a/src/base/trans_matmul_weight_nz.h b/src/base/trans_matmul_weight_nz.h new file mode 100644 index 000000000..bb9329993 --- /dev/null +++ b/src/base/trans_matmul_weight_nz.h @@ -0,0 +1,41 @@ +#ifndef INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_ +#define INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class TransMatmulWeightNz : public Operator { + public: + TransMatmulWeightNz(const Tensor input, Tensor output) + : rows_{input.size(0)}, columns_{input.size(1)} { + assert(input.ndim() == 2 && output.ndim() == 2 && + "operator `TransMatmulWeightNz` requires 2D tensors"); + assert(input.shape() == output.shape() && + "operator `TransMatmulWeightNz` input and output shapes differ"); + assert(input.dtype() == output.dtype() && + (input.dtype() == DataType::kFloat16 || + input.dtype() == DataType::kBFloat16) && + "operator `TransMatmulWeightNz` supports fp16 or bf16 only"); + assert(input.stride(0) == static_cast(input.size(1)) && + input.stride(1) == 1 && + "operator `TransMatmulWeightNz` input must be contiguous"); + assert(output.stride(0) == static_cast(output.size(1)) && + output.stride(1) == 1 && + "operator `TransMatmulWeightNz` output must be contiguous"); + assert(rows_ % 16 == 0 && columns_ % 16 == 0 && + "operator `TransMatmulWeightNz` requires a 16-aligned shape"); + } + + virtual void operator()(const Tensor input, Tensor output) const = 0; + + protected: + Tensor::Size rows_; + Tensor::Size columns_; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/ascend/ops/matmul_weight_nz/kernel.h b/src/native/ascend/ops/matmul_weight_nz/kernel.h new file mode 100644 index 000000000..6774a5659 --- /dev/null +++ b/src/native/ascend/ops/matmul_weight_nz/kernel.h @@ -0,0 +1,78 @@ +#ifndef INFINI_OPS_ASCEND_MATMUL_WEIGHT_NZ_KERNEL_H_ +#define INFINI_OPS_ASCEND_MATMUL_WEIGHT_NZ_KERNEL_H_ + +#include +#include + +#include "acl/acl.h" +#include "aclnn/aclnn_base.h" +#include "aclnnop/aclnn_matmul.h" +#include "base/matmul_weight_nz.h" +#include "native/ascend/common.h" +#include "native/ascend/error_.h" +#include "native/ascend/workspace_pool_.h" +#include "operator.h" + +namespace infini::ops { +namespace ascend_matmul_weight_nz_detail { + +inline aclTensor* MakeContiguousTensor(const Tensor& tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides(tensor.strides().begin(), + tensor.strides().end()); + std::vector storage{static_cast(tensor.numel())}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(tensor.dtype()), strides.data(), 0, + ACL_FORMAT_ND, storage.data(), + static_cast(storage.size()), + const_cast(tensor.data())); +} + +inline aclTensor* MakeNzWeight(const Tensor& weight) { + const int64_t rows = static_cast(weight.size(0)); + const int64_t columns = static_cast(weight.size(1)); + std::vector shape{rows, columns}; + std::vector strides{columns, 1}; + std::vector storage{columns / 16, rows / 16, 16, 16}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(weight.dtype()), strides.data(), 0, + ACL_FORMAT_FRACTAL_NZ, storage.data(), + static_cast(storage.size()), + const_cast(weight.data())); +} + +} // namespace ascend_matmul_weight_nz_detail + +template <> +class Operator : public MatmulWeightNz { + public: + Operator(const Tensor input, const Tensor weight, Tensor out) + : MatmulWeightNz(input, weight, out) {} + + void operator()(const Tensor input, const Tensor weight, + Tensor out) const override { + using ascend_matmul_weight_nz_detail::MakeContiguousTensor; + using ascend_matmul_weight_nz_detail::MakeNzWeight; + + auto input_tensor = ascend::OwnAclTensor(MakeContiguousTensor(input)); + auto weight_tensor = ascend::OwnAclTensor(MakeNzWeight(weight)); + auto out_tensor = ascend::OwnAclTensor(MakeContiguousTensor(out)); + + uint64_t workspace_size = 0; + aclOpExecutor* executor = nullptr; + ascend::CheckAcl(aclnnMatmulWeightNzGetWorkspaceSize( + input_tensor.get(), weight_tensor.get(), + out_tensor.get(), 1, &workspace_size, &executor), + "MatmulWeightNz workspace query"); + + auto stream = static_cast(stream_); + auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size); + ascend::CheckAcl( + aclnnMatmulWeightNz(arena.buf, workspace_size, executor, stream), + "MatmulWeightNz execution"); + } +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h new file mode 100644 index 000000000..9afe50030 --- /dev/null +++ b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_ASCEND_TRANS_MATMUL_WEIGHT_NZ_KERNEL_H_ +#define INFINI_OPS_ASCEND_TRANS_MATMUL_WEIGHT_NZ_KERNEL_H_ + +#include +#include + +#include "acl/acl.h" +#include "aclnn/aclnn_base.h" +#include "aclnnop/aclnn_trans_matmul_weight.h" +#include "base/trans_matmul_weight_nz.h" +#include "native/ascend/common.h" +#include "native/ascend/error_.h" +#include "native/ascend/workspace_pool_.h" +#include "operator.h" + +namespace infini::ops { +namespace ascend_trans_matmul_weight_nz_detail { + +inline aclTensor* MakeContiguousTensor(const Tensor& tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides(tensor.strides().begin(), + tensor.strides().end()); + std::vector storage{static_cast(tensor.numel())}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(tensor.dtype()), strides.data(), 0, + ACL_FORMAT_ND, storage.data(), + static_cast(storage.size()), + const_cast(tensor.data())); +} + +} // namespace ascend_trans_matmul_weight_nz_detail + +template <> +class Operator + : public TransMatmulWeightNz { + public: + Operator(const Tensor input, Tensor output) + : TransMatmulWeightNz(input, output) {} + + void operator()(const Tensor input, Tensor output) const override { + auto stream = static_cast(stream_); + const auto bytes = input.numel() * kDataTypeToSize.at(input.dtype()); + ascend::CheckAcl(aclrtMemcpyAsync(output.data(), bytes, input.data(), bytes, + ACL_MEMCPY_DEVICE_TO_DEVICE, stream), + "TransMatmulWeightNz source copy"); + + auto weight = ascend::OwnAclTensor( + ascend_trans_matmul_weight_nz_detail::MakeContiguousTensor(output)); + uint64_t workspace_size = 0; + aclOpExecutor* executor = nullptr; + ascend::CheckAcl(aclnnTransMatmulWeightGetWorkspaceSize( + weight.get(), &workspace_size, &executor), + "TransMatmulWeightNz workspace query"); + + auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size); + ascend::CheckAcl( + aclnnTransMatmulWeight(arena.buf, workspace_size, executor, stream), + "TransMatmulWeightNz execution"); + ascend::CheckAcl(aclrtSynchronizeStream(stream), + "TransMatmulWeightNz synchronize"); + } +}; + +} // namespace infini::ops + +#endif diff --git a/tests/test_matmul_weight_nz.py b/tests/test_matmul_weight_nz.py new file mode 100644 index 000000000..c06fc6cab --- /dev/null +++ b/tests/test_matmul_weight_nz.py @@ -0,0 +1,46 @@ +import infini.ops +import pytest +import torch + +from tests.utils import empty_strided, get_stream, randn_strided + + +@pytest.mark.parametrize( + "input_shape, weight_shape", + ( + ((1, 16), (16, 16)), + ((3, 32), (32, 48)), + ((17, 64), (64, 80)), + ), +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float16, 1e-2, 1e-2), + (torch.bfloat16, 2e-2, 1e-2), + ), +) +def test_trans_matmul_weight_nz( + input_shape, + weight_shape, + dtype, + device, + rtol, + atol, +): + if device != "npu": + pytest.skip("ACLNN MatmulWeightNz is an Ascend-only path") + + input = randn_strided(input_shape, None, dtype=dtype, device=device) + weight = randn_strided(weight_shape, None, dtype=dtype, device=device) + converted = empty_strided(weight.shape, None, dtype=dtype, device=device) + output = empty_strided( + (input.shape[0], weight.shape[1]), None, dtype=dtype, device=device + ) + stream = get_stream(input.device) + + infini.ops.trans_matmul_weight_nz(weight, converted, stream=stream) + infini.ops.matmul_weight_nz(input, converted, output, stream=stream) + + reference = torch.matmul(input.float(), weight.float()).to(dtype) + torch.testing.assert_close(output, reference, rtol=rtol, atol=atol) From ea74120149716cfd55b74f863fed65dca48826d4 Mon Sep 17 00:00:00 2001 From: wooway777 Date: Tue, 10 Nov 2026 13:00:14 +0800 Subject: [PATCH 2/3] perf(ascend): mark FM9G hot kernels graph-capture safe Cache the repeatable MatmulWeightNz executor and qualify the hot Ascend decode kernels for device graph capture. This ports the capture behavior used by the FM9G70B Ascend stack to the current operator API. --- src/native/ascend/ops/add/kernel.h | 2 + src/native/ascend/ops/argmax/kernel.h | 2 + src/native/ascend/ops/copy/kernel.h | 2 + src/native/ascend/ops/embedding/kernel.h | 8 ++-- src/native/ascend/ops/fill/kernel.h | 2 + .../ops/flash_attn_varlen_func/kernel.h | 2 + .../ops/flash_attn_with_kvcache/kernel.h | 2 + .../ascend/ops/fused_add_rms_norm/kernel.h | 2 + src/native/ascend/ops/gemm/kernel.h | 2 + src/native/ascend/ops/linear/kernel.h | 2 + src/native/ascend/ops/matmul/kernel.h | 2 + .../ascend/ops/matmul_weight_nz/kernel.h | 41 ++++++++++++++----- src/native/ascend/ops/mul/kernel.h | 2 + .../ops/reshape_and_cache_flash/kernel.h | 2 + src/native/ascend/ops/rms_norm/kernel.h | 2 + .../ascend/ops/rotary_embedding/kernel.h | 2 + src/native/ascend/ops/silu_and_mul/kernel.h | 2 + 17 files changed, 65 insertions(+), 14 deletions(-) diff --git a/src/native/ascend/ops/add/kernel.h b/src/native/ascend/ops/add/kernel.h index 84aa2f4b0..60f337835 100644 --- a/src/native/ascend/ops/add/kernel.h +++ b/src/native/ascend/ops/add/kernel.h @@ -53,6 +53,8 @@ class Operator : public Add { if (alpha_) aclDestroyScalar(alpha_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, const double /*alpha*/, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/argmax/kernel.h b/src/native/ascend/ops/argmax/kernel.h index 9db23e0fb..25dbee16b 100644 --- a/src/native/ascend/ops/argmax/kernel.h +++ b/src/native/ascend/ops/argmax/kernel.h @@ -56,6 +56,8 @@ class Operator : public Argmax { } } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const std::optional dim, const bool keepdim, Tensor out) const override { (void)dim; diff --git a/src/native/ascend/ops/copy/kernel.h b/src/native/ascend/ops/copy/kernel.h index 7660a637b..24aa09176 100644 --- a/src/native/ascend/ops/copy/kernel.h +++ b/src/native/ascend/ops/copy/kernel.h @@ -19,6 +19,8 @@ class Operator : public Copy { in_cache_(BroadcastView(src, out)), out_cache_(out) {} + bool capture_safe() const override { return true; } + void operator()(const Tensor src, const bool /*non_blocking*/, Tensor out) const override { if (output_size_ == 0) return; diff --git a/src/native/ascend/ops/embedding/kernel.h b/src/native/ascend/ops/embedding/kernel.h index 0b6e0ae09..0291b32b9 100644 --- a/src/native/ascend/ops/embedding/kernel.h +++ b/src/native/ascend/ops/embedding/kernel.h @@ -40,9 +40,9 @@ class Operator : public Embedding { /// \deprecated Use the overload that also accepts `max_norm` and /// `norm_type` instead. - [[deprecated("Use the PyTorch-compatible overload instead.")]] - Operator(const Tensor input, const Tensor weight, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, Tensor out) + [[deprecated("Use the PyTorch-compatible overload instead.")]] Operator( + const Tensor input, const Tensor weight, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, Tensor out) : Operator(input, weight, padding_idx, std::nullopt, 2.0, scale_grad_by_freq, sparse, out) {} @@ -54,6 +54,8 @@ class Operator : public Embedding { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, const std::optional /*padding_idx*/, const std::optional max_norm, const double norm_type, diff --git a/src/native/ascend/ops/fill/kernel.h b/src/native/ascend/ops/fill/kernel.h index c067ab4a7..5c4343553 100644 --- a/src/native/ascend/ops/fill/kernel.h +++ b/src/native/ascend/ops/fill/kernel.h @@ -43,6 +43,8 @@ class Operator : public Fill { } } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const double /*value*/, Tensor out) const override { (void)input; diff --git a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h index ef09039f1..ac5ba8acd 100644 --- a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h +++ b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h @@ -127,6 +127,8 @@ class Operator const_cast(tensor.data())); } + bool capture_safe() const override { return true; } + void operator()(const Tensor q, const Tensor k, const Tensor v, const Tensor cu_seqlens_q, const Tensor cu_seqlens_k, const std::optional alibi_slopes, diff --git a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h index dbda76793..be92dbb1c 100644 --- a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h +++ b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h @@ -80,6 +80,8 @@ class Operator using FlashAttnWithKvcache::operator(); + bool capture_safe() const override { return true; } + void operator()(const Tensor q, Tensor k_cache, Tensor v_cache, const std::optional k, const std::optional v, const std::optional rotary_cos, diff --git a/src/native/ascend/ops/fused_add_rms_norm/kernel.h b/src/native/ascend/ops/fused_add_rms_norm/kernel.h index 01ef9d8ce..8909a9ca6 100644 --- a/src/native/ascend/ops/fused_add_rms_norm/kernel.h +++ b/src/native/ascend/ops/fused_add_rms_norm/kernel.h @@ -105,6 +105,8 @@ class Operator // `rstd_tensor_` remains owned by the repeatable ACLNN executor. } + bool capture_safe() const override { return true; } + void operator()(Tensor input, Tensor residual, const std::optional weight, float epsilon) const override { diff --git a/src/native/ascend/ops/gemm/kernel.h b/src/native/ascend/ops/gemm/kernel.h index 983042ca0..4cc19a593 100644 --- a/src/native/ascend/ops/gemm/kernel.h +++ b/src/native/ascend/ops/gemm/kernel.h @@ -55,6 +55,8 @@ class Operator : public Gemm { if (beta_scalar_) aclDestroyScalar(beta_scalar_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor a, const Tensor b, const std::optional c, std::optional alpha, std::optional beta, std::optional trans_a, std::optional trans_b, diff --git a/src/native/ascend/ops/linear/kernel.h b/src/native/ascend/ops/linear/kernel.h index faa995dd8..1aab679af 100644 --- a/src/native/ascend/ops/linear/kernel.h +++ b/src/native/ascend/ops/linear/kernel.h @@ -45,6 +45,8 @@ class Operator : public Linear { if (beta_scalar_) aclDestroyScalar(beta_scalar_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, std::optional bias, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/matmul/kernel.h b/src/native/ascend/ops/matmul/kernel.h index bcf2d40aa..bd52002be 100644 --- a/src/native/ascend/ops/matmul/kernel.h +++ b/src/native/ascend/ops/matmul/kernel.h @@ -29,6 +29,8 @@ class Operator : public Matmul { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/matmul_weight_nz/kernel.h b/src/native/ascend/ops/matmul_weight_nz/kernel.h index 6774a5659..0ea8f0d8a 100644 --- a/src/native/ascend/ops/matmul_weight_nz/kernel.h +++ b/src/native/ascend/ops/matmul_weight_nz/kernel.h @@ -49,28 +49,47 @@ class Operator : public MatmulWeightNz { Operator(const Tensor input, const Tensor weight, Tensor out) : MatmulWeightNz(input, weight, out) {} + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, Tensor out) const override { using ascend_matmul_weight_nz_detail::MakeContiguousTensor; using ascend_matmul_weight_nz_detail::MakeNzWeight; - auto input_tensor = ascend::OwnAclTensor(MakeContiguousTensor(input)); - auto weight_tensor = ascend::OwnAclTensor(MakeNzWeight(weight)); - auto out_tensor = ascend::OwnAclTensor(MakeContiguousTensor(out)); + if (input_tensor_ == nullptr) { + input_tensor_ = MakeContiguousTensor(input); + weight_tensor_ = MakeNzWeight(weight); + out_tensor_ = MakeContiguousTensor(out); + } else { + aclSetRawTensorAddr(input_tensor_, const_cast(input.data())); + aclSetRawTensorAddr(weight_tensor_, const_cast(weight.data())); + aclSetRawTensorAddr(out_tensor_, out.data()); + } - uint64_t workspace_size = 0; - aclOpExecutor* executor = nullptr; - ascend::CheckAcl(aclnnMatmulWeightNzGetWorkspaceSize( - input_tensor.get(), weight_tensor.get(), - out_tensor.get(), 1, &workspace_size, &executor), - "MatmulWeightNz workspace query"); + if (executor_ == nullptr) { + ascend::CheckAcl(aclnnMatmulWeightNzGetWorkspaceSize( + input_tensor_, weight_tensor_, out_tensor_, 1, + &workspace_size_, &executor_), + "MatmulWeightNz workspace query"); + ascend::CheckAcl(aclSetAclOpExecutorRepeatable(executor_), + "MatmulWeightNz repeatable executor"); + } auto stream = static_cast(stream_); - auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size); + auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size_); ascend::CheckAcl( - aclnnMatmulWeightNz(arena.buf, workspace_size, executor, stream), + aclnnMatmulWeightNz(arena.buf, workspace_size_, executor_, stream), "MatmulWeightNz execution"); } + + private: + // Repeatable executors retain ACL descriptors at shutdown; follow the + // descriptor-release convention used by AclTensorCache in other providers. + mutable aclTensor* input_tensor_ = nullptr; + mutable aclTensor* weight_tensor_ = nullptr; + mutable aclTensor* out_tensor_ = nullptr; + mutable aclOpExecutor* executor_ = nullptr; + mutable uint64_t workspace_size_ = 0; }; } // namespace infini::ops diff --git a/src/native/ascend/ops/mul/kernel.h b/src/native/ascend/ops/mul/kernel.h index 311b85de8..172815a40 100644 --- a/src/native/ascend/ops/mul/kernel.h +++ b/src/native/ascend/ops/mul/kernel.h @@ -29,6 +29,8 @@ class Operator : public Mul { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h index eb05890c9..939c25fc0 100644 --- a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h +++ b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h @@ -65,6 +65,8 @@ class Operator if (indices_device_) aclrtFree(indices_device_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor key, const Tensor value, const Tensor slot_mapping, const Tensor /*k_scale*/, const Tensor /*v_scale*/, diff --git a/src/native/ascend/ops/rms_norm/kernel.h b/src/native/ascend/ops/rms_norm/kernel.h index 42ffeb803..eb093264e 100644 --- a/src/native/ascend/ops/rms_norm/kernel.h +++ b/src/native/ascend/ops/rms_norm/kernel.h @@ -123,6 +123,8 @@ class Operator : public RmsNorm { // `rstd_tensor_` leaks with the executor at shutdown (see `64c367c`). } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, float eps, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/rotary_embedding/kernel.h b/src/native/ascend/ops/rotary_embedding/kernel.h index 9c1cf6adf..eda894be9 100644 --- a/src/native/ascend/ops/rotary_embedding/kernel.h +++ b/src/native/ascend/ops/rotary_embedding/kernel.h @@ -128,6 +128,8 @@ class Operator ascend::CheckAcl(aclrtFree(key_out_data_), "RoPE key output release"); } + bool capture_safe() const override { return true; } + void operator()(const Tensor positions, Tensor query, std::optional key, const Tensor cos_sin_cache, int64_t head_size, bool is_neox, int64_t rope_dim_offset = 0, diff --git a/src/native/ascend/ops/silu_and_mul/kernel.h b/src/native/ascend/ops/silu_and_mul/kernel.h index f8715cb4b..b5cecfaa2 100644 --- a/src/native/ascend/ops/silu_and_mul/kernel.h +++ b/src/native/ascend/ops/silu_and_mul/kernel.h @@ -40,6 +40,8 @@ class Operator : public SiluAndMul { if (out_staging_cache_) out_staging_cache_->release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, Tensor out) const override { auto t_input = input_cache_.get(const_cast(input.data())); auto t_out = out_cache_.get(out.data()); From 385f1dee2a2b279e23a5d5b6a8bec20f4210ab40 Mon Sep 17 00:00:00 2001 From: wooway777 Date: Fri, 13 Nov 2026 11:02:39 +0800 Subject: [PATCH 3/3] perf(ascend): accelerate FM9G70 paged serving Port the CANN FAI decode path, parallelize and chunk paged cache writes, and retain host-side uniform prefill sequence metadata to reduce synchronous D2H copies. --- .gitmodules | 4 + scripts/fetch-ops-transformer.sh | 19 + scripts/generate_wrappers.py | 44 ++- src/CMakeLists.txt | 55 ++- src/base/matmul_weight_nz.h | 4 + src/base/trans_matmul_weight_nz.h | 4 + src/native/ascend/custom/CMakeLists.txt | 19 +- src/native/ascend/custom/build.sh | 1 + .../op_kernel/paged_attention.cpp | 338 ++++++++++++++++++ .../op_kernel/paged_attention_fai.cpp | 139 +++++++ .../op_kernel/reshape_and_cache_flash.cpp | 65 ++-- .../ops/flash_attn_varlen_func/kernel.h | 28 +- .../ops/flash_attn_with_kvcache/kernel.h | 221 ++++++++++++ .../ascend/ops/matmul_weight_nz/kernel.h | 2 + .../ops/reshape_and_cache_flash/kernel.h | 58 ++- .../ops/trans_matmul_weight_nz/kernel.h | 4 +- tests/test_flash_attn_with_kvcache.py | 56 ++- tests/test_generate_wrappers.py | 8 + tests/test_matmul_weight_nz.py | 46 --- third_party/ops-transformer | 1 + third_party/ops-transformer-fm9g70.patch | 51 +++ 21 files changed, 1036 insertions(+), 131 deletions(-) create mode 100755 scripts/fetch-ops-transformer.sh create mode 100644 src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp create mode 100644 src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp delete mode 100644 tests/test_matmul_weight_nz.py create mode 160000 third_party/ops-transformer create mode 100644 third_party/ops-transformer-fm9g70.patch diff --git a/.gitmodules b/.gitmodules index be99e8a8f..902143ba0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule ".ci"] path = .ci url = https://github.com/InfiniTensor/ci.git +[submodule "third_party/ops-transformer"] + path = third_party/ops-transformer + url = https://gitcode.com/cann/ops-transformer.git + update = none diff --git a/scripts/fetch-ops-transformer.sh b/scripts/fetch-ops-transformer.sh new file mode 100755 index 000000000..3be63c91a --- /dev/null +++ b/scripts/fetch-ops-transformer.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +SUBMODULE_REL=third_party/ops-transformer +SUBMODULE="$ROOT/$SUBMODULE_REL" +GIT="git -c safe.directory=*" +EXPECTED=$($GIT -C "$ROOT" ls-tree HEAD "$SUBMODULE_REL" | awk '{print $3}') + +if ! $GIT -C "$SUBMODULE" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + $GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL" +else + ACTUAL=$($GIT -C "$SUBMODULE" rev-parse HEAD) + if [ "$ACTUAL" != "$EXPECTED" ]; then + $GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL" + fi +fi + +printf 'ops-transformer: %s\n' "$($GIT -C "$SUBMODULE" rev-parse HEAD)" diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py index bb03d2b57..dc9f7bb87 100644 --- a/scripts/generate_wrappers.py +++ b/scripts/generate_wrappers.py @@ -41,6 +41,14 @@ _OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") +# Temporary compatibility operators used directly by InfiniLM. Keep their +# generated C++ dispatch available, but do not expose legacy C or Python APIs. +_CPP_ONLY_OPS = frozenset(("matmul_weight_nz", "trans_matmul_weight_nz")) + + +def _generates_public_api(op_name): + return op_name not in _CPP_ONLY_OPS + def _op_namespace_parts(op_name): parts = op_name.split("_") @@ -2120,6 +2128,7 @@ def _generate_op_artifacts(item): "op_name": op_name, "header_name": header_name, "bind_func_name": f"Bind{_op_symbol_name(op_name)}", + "public_api": _generates_public_api(op_name), "pybind11": _generate_pybind11(operator), "binding_source": _generate_binding_source(op_name), "legacy_c_source": legacy_c_source, @@ -2356,28 +2365,29 @@ def _dispatch_gen_batch_size(): source_path = _GENERATED_SRC_DIR / op_name header_name = artifact["header_name"] bind_func_name = artifact["bind_func_name"] - binding_header_path = _BINDINGS_DIR / header_name + if artifact["public_api"]: + binding_header_path = _BINDINGS_DIR / header_name - _write_text_if_changed(binding_header_path, artifact["pybind11"]) - expected_binding_files.add(binding_header_path) + _write_text_if_changed(binding_header_path, artifact["pybind11"]) + expected_binding_files.add(binding_header_path) - if use_monolithic_bindings: - op_includes.append(f'#include "{header_name}"') - else: - binding_source_path = _BINDINGS_DIR / f"{op_name}.cc" - _write_text_if_changed(binding_source_path, artifact["binding_source"]) - expected_binding_files.add(binding_source_path) + if use_monolithic_bindings: + op_includes.append(f'#include "{header_name}"') + else: + binding_source_path = _BINDINGS_DIR / f"{op_name}.cc" + _write_text_if_changed(binding_source_path, artifact["binding_source"]) + expected_binding_files.add(binding_source_path) - source_path.mkdir(exist_ok=True) - legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc" - _write_text_if_changed(legacy_source_path, artifact["legacy_c_source"]) - expected_generated_src_files.add(legacy_source_path) + source_path.mkdir(exist_ok=True) + legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc" + _write_text_if_changed(legacy_source_path, artifact["legacy_c_source"]) + expected_generated_src_files.add(legacy_source_path) - include_path = _INCLUDE_DIR / header_name - _write_text_if_changed(include_path, artifact["legacy_c_header"]) - expected_include_files.add(include_path) + include_path = _INCLUDE_DIR / header_name + _write_text_if_changed(include_path, artifact["legacy_c_header"]) + expected_include_files.add(include_path) - bind_func_names.append(bind_func_name) + bind_func_names.append(bind_func_name) dispatch_header = _generate_generated_dispatch_header( op_names, args.devices, dispatch_declarations diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e685b4f12..2765ab70c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -551,6 +551,51 @@ if(WITH_ASCEND) # Custom `AscendC` kernels (PyTorch extension, requires `torch_npu`). if(BUILD_ASCEND_CUSTOM) + set(_ops_transformer_root + "${PROJECT_SOURCE_DIR}/third_party/ops-transformer") + option(INFINIOPS_FETCH_OPS_TRANSFORMER + "Fetch and pin the official Ascend ops-transformer submodule" ON) + if(INFINIOPS_FETCH_OPS_TRANSFORMER) + execute_process( + COMMAND bash "${PROJECT_SOURCE_DIR}/scripts/fetch-ops-transformer.sh" + RESULT_VARIABLE _ops_transformer_fetch_result) + if(NOT _ops_transformer_fetch_result EQUAL 0) + message(FATAL_ERROR "Failed to fetch the pinned ops-transformer submodule") + endif() + endif() + + if(NOT EXISTS "${_ops_transformer_root}/CMakeLists.txt") + message(FATAL_ERROR + "ops-transformer is missing; run scripts/fetch-ops-transformer.sh") + endif() + + # Keep the submodule pinned to an official commit and apply only the + # small FM9G70 decode ABI patch after checkout. + find_package(Git REQUIRED) + set(_ops_transformer_patch + "${PROJECT_SOURCE_DIR}/third_party/ops-transformer-fm9g70.patch") + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + --check -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_patch_status) + if(_ops_transformer_patch_status EQUAL 0) + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_patch_result) + if(NOT _ops_transformer_patch_result EQUAL 0) + message(FATAL_ERROR "Failed to patch ops-transformer for FM9G70") + endif() + else() + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + --check --reverse -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_reverse_status) + if(NOT _ops_transformer_reverse_status EQUAL 0) + message(FATAL_ERROR + "ops-transformer has incompatible local changes") + endif() + endif() # In-tree `ascendc_library()` trips the `CANN` `extract_host_stub.py` # path-handling bug under `scikit-build-core`'s temp-dir builds # (`KeyError` on `/./workspace/...` paths in `$`). @@ -579,6 +624,12 @@ if(WITH_ASCEND) "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/*.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/CMakeLists.txt" "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/build.sh") + set(_ops_transformer_fai_dir + "${_ops_transformer_root}/attention/fused_infer_attention_score/op_kernel") + file(GLOB_RECURSE _ops_transformer_fai_srcs CONFIGURE_DEPENDS + "${_ops_transformer_fai_dir}/*.h" + "${_ops_transformer_fai_dir}/*.hpp" + "${_ops_transformer_fai_dir}/*.cpp") # Scrub env inherited from the outer `scikit-build-core` invocation # before handing control to `build.sh`: `CMAKE_GENERATOR` / @@ -604,9 +655,10 @@ if(WITH_ASCEND) --unset=CMAKE_BUILD_PARALLEL_LEVEL "BUILD_DIR=${_custom_build_dir}" "MAIN_SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}" + "OPS_TRANSFORMER_ROOT=${_ops_transformer_root}" bash ${_custom_source_dir}/build.sh ${SOC_VERSION} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom - DEPENDS ${_custom_srcs} + DEPENDS ${_custom_srcs} ${_ops_transformer_fai_srcs} COMMENT "Building custom AscendC kernels (SOC_VERSION=${SOC_VERSION})" VERBATIM) @@ -621,6 +673,7 @@ if(WITH_ASCEND) # custom kernel implementations (e.g. `RmsNorm` index 1) can call # them via the generated launch functions. target_compile_definitions(infiniops PUBLIC INFINI_HAS_CUSTOM_KERNELS=1) + target_compile_definitions(infiniops PUBLIC INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION=1) target_include_directories(infiniops PRIVATE "${_custom_build_dir}/include/no_workspace_kernel") target_link_libraries(infiniops PRIVATE diff --git a/src/base/matmul_weight_nz.h b/src/base/matmul_weight_nz.h index 9610a86dd..7ff95eaaa 100644 --- a/src/base/matmul_weight_nz.h +++ b/src/base/matmul_weight_nz.h @@ -7,6 +7,10 @@ namespace infini::ops { +// Temporary compatibility operator for InfiniLM's Ascend NZ weight path. It is +// intentionally C++-only. Once TensorView distinguishes ND and NZ +// distributions, migrate this behavior to the general Matmul operator and +// remove this class. class MatmulWeightNz : public Operator { public: MatmulWeightNz(const Tensor input, const Tensor weight, Tensor out) diff --git a/src/base/trans_matmul_weight_nz.h b/src/base/trans_matmul_weight_nz.h index bb9329993..126df569b 100644 --- a/src/base/trans_matmul_weight_nz.h +++ b/src/base/trans_matmul_weight_nz.h @@ -7,6 +7,10 @@ namespace infini::ops { +// Temporary compatibility operator for InfiniLM's Ascend NZ weight conversion. +// It is intentionally C++-only. Once TensorView distinguishes ND and NZ +// distributions, migrate this behavior to the general transpose-matmul path and +// remove this class. class TransMatmulWeightNz : public Operator { public: TransMatmulWeightNz(const Tensor input, Tensor output) diff --git a/src/native/ascend/custom/CMakeLists.txt b/src/native/ascend/custom/CMakeLists.txt index 94f19850a..5547a21bc 100644 --- a/src/native/ascend/custom/CMakeLists.txt +++ b/src/native/ascend/custom/CMakeLists.txt @@ -32,6 +32,8 @@ endif() set(PROJECT_OP_SRC_BASE ${PROJECT_SOURCE_DIR}) set(MAIN_SRC_DIR "${PROJECT_OP_SRC_BASE}/../.." CACHE PATH "Main InfiniOps source directory.") +set(OPS_TRANSFORMER_ROOT "" CACHE PATH + "Pinned ops-transformer source root supplied by the outer build") include(cmake/config_envs.cmake) include(cmake/config_ascend.cmake) @@ -63,6 +65,8 @@ ascendc_library(no_workspace_kernel STATIC rms_norm/op_kernel/rms_norm.cpp add_rms_norm/op_kernel/add_rms_norm.cpp reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp + paged_attention/op_kernel/paged_attention.cpp + paged_attention/op_kernel/paged_attention_fai.cpp ) # `CANN 8.5.1` installs host objects under `objects-/`, while @@ -80,9 +84,18 @@ endif() # enum. `ascendc_library` forwards the interface target's `INCLUDES` # property to the nested `ExternalProject_Add` (see # `${ASCEND_HOME_PATH}/tools/tikcpp/ascendc_kernel_cmake/legacy_modules/function.cmake`), -# so append the main `src/` dir here. -set_property(TARGET no_workspace_kernel_interface APPEND PROPERTY - INCLUDES ${MAIN_SRC_DIR}) +# so register the main `src/` dir and repository root here. Resolve the +# repository root through the symlinked source directory used by the outer +# build. Resolve the symlink before appending `..`; CMake otherwise +# normalizes those components lexically. +get_filename_component(CUSTOM_SOURCE_DIR "${PROJECT_SOURCE_DIR}" REALPATH) +get_filename_component(INFINIOPS_ROOT + "${CUSTOM_SOURCE_DIR}/../../../.." REALPATH) +ascendc_include_directories(no_workspace_kernel INTERFACE + ${MAIN_SRC_DIR} + ${INFINIOPS_ROOT} + ${OPS_TRANSFORMER_ROOT} +) # Create the shared library `libascend_kernel.so`. add_library(${OP_PLUGIN_NAME} SHARED ${OP_SRCS}) diff --git a/src/native/ascend/custom/build.sh b/src/native/ascend/custom/build.sh index f7e1cf684..25ee14410 100755 --- a/src/native/ascend/custom/build.sh +++ b/src/native/ascend/custom/build.sh @@ -33,6 +33,7 @@ cmake \ -DSOC_VERSION="${SOC_VERSION}" \ -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="${OUTPUT_DIR}" \ -DMAIN_SRC_DIR="${MAIN_SRC_DIR}" \ + -DOPS_TRANSFORMER_ROOT="${OPS_TRANSFORMER_ROOT}" \ -B "${BUILD_DIR}" \ -S "${SCRIPT_DIR}" diff --git a/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp new file mode 100644 index 000000000..58817b51a --- /dev/null +++ b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp @@ -0,0 +1,338 @@ +#include "kernel_operator.h" + +constexpr size_t PAGED_ATTENTION_TILE = 128; +constexpr size_t PAGED_ATTENTION_COPY_TILE = 32; +constexpr size_t BYTE_ALIGN = 32; + +template +__aicore__ inline size_t alignTileLen(size_t tile_len, size_t byte_align) { + size_t bytes = tile_len * sizeof(T); + size_t aligned_bytes = (bytes % byte_align == 0) + ? bytes + : (bytes + byte_align - bytes % byte_align); + return aligned_bytes / sizeof(T); +} + +#include + +using namespace AscendC; + +template +__aicore__ inline int64_t loadIndex(GlobalTensor& tensor, + ptrdiff_t offset) { + return static_cast(tensor.GetValue(offset)); +} + +template +class PagedAttentionKernel { + public: + __aicore__ inline PagedAttentionKernel() {} + + __aicore__ inline void init( + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, + GM_ADDR block_tables, GM_ADDR cache_lens, GM_ADDR alibi_slopes, + bool has_alibi, size_t num_heads, size_t num_seqs, size_t num_kv_heads, + size_t head_size, float scale, size_t max_num_blocks_per_seq, + size_t page_block_size, ptrdiff_t q_stride, ptrdiff_t k_batch_stride, + ptrdiff_t k_row_stride, ptrdiff_t k_head_stride, ptrdiff_t v_batch_stride, + ptrdiff_t v_row_stride, ptrdiff_t v_head_stride, ptrdiff_t o_stride, + ptrdiff_t block_table_batch_stride, ptrdiff_t cache_lens_stride) { + _num_heads = num_heads; + _num_seqs = num_seqs; + _num_kv_heads = num_kv_heads; + _head_size = head_size; + _scale = scale; + _max_num_blocks_per_seq = max_num_blocks_per_seq; + _page_block_size = page_block_size; + _q_stride = q_stride; + _k_batch_stride = k_batch_stride; + _k_row_stride = k_row_stride; + _k_head_stride = k_head_stride; + _v_batch_stride = v_batch_stride; + _v_row_stride = v_row_stride; + _v_head_stride = v_head_stride; + _o_stride = o_stride; + _block_table_batch_stride = block_table_batch_stride; + _cache_lens_stride = cache_lens_stride; + _has_alibi = has_alibi; + + _out_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(out)); + _q_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(q)); + _k_cache_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(k_cache)); + _v_cache_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(v_cache)); + _block_tables_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ Tindex*>(block_tables)); + _cache_lens_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ Tindex*>(cache_lens)); + if (_has_alibi) { + _alibi_slopes_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ float*>(alibi_slopes)); + } + + _pipe.InitBuffer( + _q_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(float)); + _pipe.InitBuffer( + _acc_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(float)); + _pipe.InitBuffer(_score_buf, PAGED_ATTENTION_TILE * sizeof(float)); + _pipe.InitBuffer( + _out_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(Tdata)); + + // Shared float scratchpad for K/V type-casting (used by computeScore and V + // accumulation) + _pipe.InitBuffer(_v_f32_buf, alignTileLen(_head_size, BYTE_ALIGN) * + sizeof(float)); + _pipe.InitBuffer(_reduce_buf, alignTileLen(_head_size, BYTE_ALIGN) * + sizeof(float)); + // Double-buffered VECIN queue for K prefetch + _head_stride = alignTileLen(_head_size, BYTE_ALIGN); + _pipe.InitBuffer(_k_queue, 2, + PAGED_ATTENTION_COPY_TILE * _head_stride * sizeof(Tdata)); + // Double-buffered VECIN queue for V prefetch (overlaps with Score VEC + // computation) + _pipe.InitBuffer(_v_queue, 2, + PAGED_ATTENTION_COPY_TILE * _head_stride * sizeof(Tdata)); + } + + __aicore__ inline void process() { + const size_t work_idx = GetBlockIdx(); + const size_t head_idx = work_idx % _num_heads; + const size_t seq_idx = work_idx / _num_heads; + if (seq_idx >= _num_seqs) { + return; + } + + const int64_t seq_len_i64 = loadIndex( + _cache_lens_gm, static_cast(seq_idx) * _cache_lens_stride); + if (seq_len_i64 <= 0) { + return; + } + const size_t seq_len = static_cast(seq_len_i64); + const size_t num_queries_per_kv = _num_heads / _num_kv_heads; + const size_t kv_head_idx = head_idx / num_queries_per_kv; + const float alibi_slope = + _has_alibi ? _alibi_slopes_gm.GetValue(head_idx) : 0.0f; + + LocalTensor q_local = _q_buf.Get(); + LocalTensor acc_local = _acc_buf.Get(); + LocalTensor score_local = _score_buf.Get(); + + // Load Q from global memory + const ptrdiff_t q_base = static_cast(seq_idx) * _q_stride + + static_cast(head_idx * _head_size); + + LocalTensor q_raw_local = _out_buf.Get(); + DataCopy(q_raw_local, _q_gm[q_base], _head_size); + PipeBarrier(); + + Cast(q_local, q_raw_local, AscendC::RoundMode::CAST_NONE, _head_size); + Duplicate(acc_local, 0.0f, _head_size); + + // Online softmax with burst K/V copies. + float max_score = -3.4028234663852886e38f; + float sum_exp = 0.0f; + const uint16_t copy_len = + static_cast(_head_stride * sizeof(Tdata) / ONE_BLK_SIZE); + const auto gap32 = [](ptrdiff_t row, size_t head) { + const ptrdiff_t value = row - static_cast(head); + if (value <= 0) return static_cast(0); + return static_cast(value * sizeof(Tdata) / ONE_BLK_SIZE - 1); + }; + const uint16_t k_gap = gap32(_k_row_stride, _head_stride); + const uint16_t v_gap = gap32(_v_row_stride, _head_stride); + for (size_t block_idx = 0; block_idx < _max_num_blocks_per_seq; + ++block_idx) { + const size_t token_start = block_idx * _page_block_size; + if (token_start >= seq_len) break; + const int64_t block_id = + loadIndex(_block_tables_gm, static_cast(seq_idx) * + _block_table_batch_stride + + static_cast(block_idx)); + size_t token_end = token_start + _page_block_size; + if (token_end > seq_len) token_end = seq_len; + const size_t tile_count = token_end - token_start; + const ptrdiff_t k_base = + static_cast(block_id) * _k_batch_stride + + static_cast(kv_head_idx) * _k_head_stride; + const ptrdiff_t v_base = + static_cast(block_id) * _v_batch_stride + + static_cast(kv_head_idx) * _v_head_stride; + size_t offset = 0; + while (offset < tile_count) { + const size_t count = (tile_count - offset < PAGED_ATTENTION_COPY_TILE + ? tile_count - offset + : PAGED_ATTENTION_COPY_TILE); + if (offset == 0) { + LocalTensor k0 = _k_queue.AllocTensor(); + DataCopy( + k0, _k_cache_gm[k_base], + DataCopyParams(static_cast(count), copy_len, k_gap, 0)); + _k_queue.EnQue(k0); + LocalTensor v0 = _v_queue.AllocTensor(); + DataCopy( + v0, _v_cache_gm[v_base], + DataCopyParams(static_cast(count), copy_len, v_gap, 0)); + _v_queue.EnQue(v0); + } + LocalTensor k_tile = _k_queue.DeQue(); + LocalTensor v_tile = _v_queue.DeQue(); + const size_t next = offset + count; + if (next < tile_count) { + const size_t nc = (tile_count - next < PAGED_ATTENTION_COPY_TILE + ? tile_count - next + : PAGED_ATTENTION_COPY_TILE); + LocalTensor kn = _k_queue.AllocTensor(); + DataCopy( + kn, _k_cache_gm[k_base + next * _k_row_stride], + DataCopyParams(static_cast(nc), copy_len, k_gap, 0)); + _k_queue.EnQue(kn); + LocalTensor vn = _v_queue.AllocTensor(); + DataCopy( + vn, _v_cache_gm[v_base + next * _v_row_stride], + DataCopyParams(static_cast(nc), copy_len, v_gap, 0)); + _v_queue.EnQue(vn); + } + for (size_t i = 0; i < count; ++i) { + LocalTensor k_raw = k_tile[i * _head_stride]; + LocalTensor v_raw = v_tile[i * _head_stride]; + const size_t t = token_start + offset + i; + const float score = + computeScore(q_local, k_raw, t, alibi_slope, seq_len); + if (score > max_score) { + score_local.SetValue(0, max_score - score); + Exp(score_local, score_local, 1); + const float rescale = score_local.GetValue(0); + Muls(acc_local, acc_local, rescale, _head_size); + sum_exp *= rescale; + max_score = score; + } + score_local.SetValue(0, score - max_score); + Exp(score_local, score_local, 1); + const float prob = score_local.GetValue(0); + sum_exp += prob; + LocalTensor v_f32 = _v_f32_buf.Get(); + Cast(v_f32, v_raw, AscendC::RoundMode::CAST_NONE, _head_size); + Muls(v_f32, v_f32, prob, _head_size); + Add(acc_local, acc_local, v_f32, _head_size); + } + _k_queue.FreeTensor(k_tile); + _v_queue.FreeTensor(v_tile); + offset = next; + } + } + PipeBarrier(); + // Normalize accumulator and write back output + const float inv_sum = 1.0f / (sum_exp + 1e-6f); + LocalTensor out_local = _out_buf.Get(); + + Muls(acc_local, acc_local, inv_sum, _head_size); + PipeBarrier(); + + Cast(out_local, acc_local, AscendC::RoundMode::CAST_RINT, + static_cast(_head_size)); + PipeBarrier(); + + const ptrdiff_t out_base = static_cast(seq_idx) * _o_stride + + static_cast(head_idx * _head_size); + + DataCopy(_out_gm[out_base], out_local, _head_size); + } + + private: + __aicore__ inline float computeScore(LocalTensor& q_local, + LocalTensor& k_raw_local, + size_t token_idx, float alibi_slope, + size_t seq_len) { + LocalTensor k_f32_local = _v_f32_buf.Get(); + + Cast(k_f32_local, k_raw_local, AscendC::RoundMode::CAST_NONE, _head_size); + Mul(k_f32_local, q_local, k_f32_local, _head_size); + LocalTensor temp_workspace = _reduce_buf.Get(); + ReduceSum(k_f32_local, k_f32_local, temp_workspace, _head_size); + PipeBarrier(); + + float score = k_f32_local.GetValue(0); + + score *= _scale; + if (_has_alibi) { + score += + alibi_slope * static_cast(static_cast(token_idx) - + static_cast(seq_len) + 1); + } + return score; + } + + GlobalTensor _out_gm; + GlobalTensor _q_gm; + GlobalTensor _k_cache_gm; + GlobalTensor _v_cache_gm; + GlobalTensor _block_tables_gm; + GlobalTensor _cache_lens_gm; + GlobalTensor _alibi_slopes_gm; + + TPipe _pipe; + TBuf _q_buf; + TBuf _acc_buf; + TBuf _score_buf; + TBuf _out_buf; + // Shared float scratchpad for K/V type-casting (used by computeScore and V + // accumulation) + TBuf _v_f32_buf; + TBuf _reduce_buf; + // Double-buffered VECIN queue for K prefetch + TQue _k_queue; + // Double-buffered VECIN queue for V prefetch (overlaps with Score compute) + TQue _v_queue; + + size_t _num_heads; + size_t _num_seqs; + size_t _num_kv_heads; + size_t _head_size; + size_t _head_stride; + float _scale; + size_t _max_num_blocks_per_seq; + size_t _page_block_size; + ptrdiff_t _q_stride; + ptrdiff_t _k_batch_stride; + ptrdiff_t _k_row_stride; + ptrdiff_t _k_head_stride; + ptrdiff_t _v_batch_stride; + ptrdiff_t _v_row_stride; + ptrdiff_t _v_head_stride; + ptrdiff_t _o_stride; + ptrdiff_t _block_table_batch_stride; + ptrdiff_t _cache_lens_stride; + bool _has_alibi; +}; + +#define DEFINE_PAGED_ATTENTION_KERNEL(KERNEL_NAME, DATA_TYPE, INDEX_TYPE) \ + extern "C" __global__ __aicore__ void KERNEL_NAME( \ + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, \ + GM_ADDR block_tables, GM_ADDR cache_lens, GM_ADDR alibi_slopes, \ + bool has_alibi, size_t num_heads, size_t num_seqs, size_t num_kv_heads, \ + size_t head_size, float scale, size_t max_num_blocks_per_seq, \ + size_t page_block_size, ptrdiff_t q_stride, ptrdiff_t k_batch_stride, \ + ptrdiff_t k_row_stride, ptrdiff_t k_head_stride, \ + ptrdiff_t v_batch_stride, ptrdiff_t v_row_stride, \ + ptrdiff_t v_head_stride, ptrdiff_t o_stride, \ + ptrdiff_t block_table_batch_stride, ptrdiff_t cache_lens_stride) { \ + PagedAttentionKernel op; \ + op.init(out, q, k_cache, v_cache, block_tables, cache_lens, alibi_slopes, \ + has_alibi, num_heads, num_seqs, num_kv_heads, head_size, scale, \ + max_num_blocks_per_seq, page_block_size, q_stride, k_batch_stride, \ + k_row_stride, k_head_stride, v_batch_stride, v_row_stride, \ + v_head_stride, o_stride, block_table_batch_stride, \ + cache_lens_stride); \ + op.process(); \ + } + +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_i64, half, int64_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_i32, half, int32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_u32, half, uint32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_i64, bfloat16_t, + int64_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_i32, bfloat16_t, + int32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_u32, bfloat16_t, + uint32_t) + +#undef DEFINE_PAGED_ATTENTION_KERNEL diff --git a/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp new file mode 100644 index 000000000..51e3a44da --- /dev/null +++ b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp @@ -0,0 +1,139 @@ +#include "attention/fused_infer_attention_score/op_kernel/attn_infra/base_defs.hpp" +#include "attention/fused_infer_attention_score/op_kernel/attn_infra/detail/alignment.hpp" +namespace NpuArch { +template +__aicore__ inline auto CeilDiv(T dividend, U divisor) { + return Detail::Alignment::CeilDiv(dividend, divisor); +} +template +__aicore__ inline auto RoundDown(T value, U align) { + return Detail::Alignment::RoundDown(value, align); +} +template +__aicore__ inline auto RoundUp(T value, U align) { + return Detail::Alignment::RoundUp(value, align); +} +struct splitNode { + uint32_t batchIdx[64]; + uint32_t headStartIdx[64]; + uint32_t headEndIdx[64]; + uint32_t qStartIdx[64]; + uint32_t qEndIdx[64]; + uint32_t lseTaskOffset[64]; + uint32_t oTaskOffset[64]; + uint32_t splitNum[64]; +}; +} // namespace NpuArch +#include "attention/fused_infer_attention_score/op_kernel/kernel_common.hpp" + +struct FAInferTilingData { + uint64_t mm1OutSize; + uint64_t smOnlineOutSize; + uint64_t mm2OutSize; + uint32_t batch; + uint32_t numHeads; + uint32_t kvHeads; + uint32_t embeddingSize; + uint32_t embeddingSizeV; + uint32_t blockSize; + uint32_t maxNumBlocksPerBatch; + uint32_t firstBatchTaskNum; + uint32_t totalTaskNum; + uint32_t mainLoopTaskNum; + uint32_t tailLoopTaskNum; + uint32_t tailStartBatch; + uint32_t tailStartN2; + uint32_t tailKvNBlockTile; + uint32_t maskType; + float scaleValue; + uint32_t sparseMode; + int64_t preToken; + int64_t nextToken; + uint64_t qStride; + uint64_t oStride; +}; + +#include "attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h" + +extern "C" __global__ __aicore__ void infinilm_paged_attention_fai_bf16_i32( + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, + GM_ADDR block_tables, GM_ADDR cache_lens, float scale, size_t num_heads, + size_t num_seqs, size_t num_kv_heads, size_t head_size, + size_t max_num_blocks_per_seq, size_t page_block_size, ptrdiff_t q_stride, + ptrdiff_t o_stride, GM_ADDR workspace, GM_ADDR tiling) { + using namespace NpuArch; + using ArchTag = Arch::AtlasA2; + using ElementQ = bfloat16_t; + using LayoutQ = layout::RowMajor; + using ElementK = bfloat16_t; + using LayoutK = layout::ColumnMajor; + using ElementV = bfloat16_t; + using LayoutV = layout::RowMajor; + using ElementS = float; + using LayoutS = layout::RowMajor; + using ElementSink = bfloat16_t; + using LayoutSink = layout::RowMajor; + using ElementP = bfloat16_t; + using LayoutP = layout::RowMajor; + using ElementO = bfloat16_t; + using LayoutO = layout::RowMajor; + using ElementLse = float; + using LayoutLse = layout::RowMajor; + using ElementMask = int8_t; + using LayoutMask = layout::RowMajor; + using ElementOTmp = float; + using LayoutOTmp = layout::RowMajor; + using ElementUpdate = float; + using LayoutUpdate = layout::RowMajor; + using L1TileShapeQK = GemmShape; + using L0TileShapeQK = GemmShape<128, 128, 128>; + using DispatchPolicyQK = Gemm::MmadAtlasA2FAIQKDecode; + using QType = Gemm::GemmType; + using KType = Gemm::GemmType; + using SType = Gemm::GemmType; + using SinkType = Gemm::GemmType; + using BlockMmadQK = + Gemm::Block::BlockMmad; + using DispatchPolicyOnlineSoftmax = Epilogue::EpilogueAtlasA2OnlineSoftmax< + Epilogue::LseMode::NONE, Epilogue::SinkMode::DISABLE, + Epilogue::MaskMode::NO_MASK, float>; + using maskType = Gemm::GemmType; + using pseShiftType = Gemm::GemmType; + using PType = Gemm::GemmType; + using EpilogueOnlineSoftmax = + Epilogue::Block::BlockEpilogue; + using L1TileShapePV = GemmShape<128, 128, 256>; + using L0TileShapePV = GemmShape<128, 128, 128>; + using DispatchPolicyPV = Gemm::MmadAtlasA2FAIPVDecode; + using VType = Gemm::GemmType; + using OTmpType = Gemm::GemmType; + using BlockMmadPV = + Gemm::Block::BlockMmad; + using DispatchPolicyRescaleO = + Epilogue::EpilogueAtlasA2RescaleO; + using OType = Gemm::GemmType; + using OUpdateType = Gemm::GemmType; + using LseType = Gemm::GemmType; + using EpilogueRescaleO = + Epilogue::Block::BlockEpilogue; + using DispatchPolicyInitOutWhenZero = + Epilogue::EpilogueAtlasA2InitOutWhenZero; + using EpilogueInitOut = + Epilogue::Block::BlockEpilogue; + using SplitFuse::FAInferKernelDecoding; + using FAI = + FAInferKernelDecoding; + FAIKernelParams params{ + q, k_cache, v_cache, nullptr, nullptr, block_tables, nullptr, + cache_lens, out, nullptr, workspace, tiling, nullptr}; + FAI kernel; + kernel(params); +} diff --git a/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp b/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp index 7d1069856..6aa93d35a 100644 --- a/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp +++ b/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp @@ -43,52 +43,41 @@ class KernelReshapeAndCacheFlashStrided { } __aicore__ inline void Process() { -#if defined(__CCE_AICORE__) && __CCE_AICORE__ == 200 - AscendC::DataCacheCleanAndInvalid( - slot_mapping_); -#endif + const int64_t block_index = AscendC::GetBlockIdx(); + const int64_t block_count = AscendC::GetBlockNum(); + const int64_t work_count = num_tokens_ * num_heads_; auto local = copy_buffer_.Get(); AscendC::DataCopyExtParams copy_params{ 1, static_cast(head_bytes_), 0, 0, 0}; AscendC::DataCopyPadExtParams pad_params{false, 0, 0, 0}; - for (int64_t token = 0; token < num_tokens_; ++token) { - int64_t slot = slot_mapping_.GetValue(token); - if (slot < 0 || slot >= num_blocks_ * block_size_) { - continue; - } - - int64_t block = slot / block_size_; - int64_t page = slot % block_size_; + for (int64_t work = block_index; work < work_count; work += block_count) { + const int64_t token = work / num_heads_; + const int64_t head = work % num_heads_; + const int64_t slot = slot_mapping_.GetValue(token); + if (slot < 0 || slot >= num_blocks_ * block_size_) continue; - for (int64_t head = 0; head < num_heads_; ++head) { - int64_t key_offset = - (token * key_token_stride_ + head * key_head_stride_) * - element_size_; - int64_t key_cache_offset = - (block * key_cache_block_stride_ + page * key_cache_page_stride_ + - head * key_cache_head_stride_) * - element_size_; - Copy(key_, key_offset, key_cache_, key_cache_offset, local, copy_params, - pad_params); + const int64_t block = slot / block_size_; + const int64_t page = slot % block_size_; + const int64_t key_offset = + (token * key_token_stride_ + head * key_head_stride_) * element_size_; + const int64_t key_cache_offset = + (block * key_cache_block_stride_ + page * key_cache_page_stride_ + + head * key_cache_head_stride_) * + element_size_; + Copy(key_, key_offset, key_cache_, key_cache_offset, local, copy_params, + pad_params); - int64_t value_offset = - (token * value_token_stride_ + head * value_head_stride_) * - element_size_; - int64_t value_cache_offset = (block * value_cache_block_stride_ + - page * value_cache_page_stride_ + - head * value_cache_head_stride_) * - element_size_; - Copy(value_, value_offset, value_cache_, value_cache_offset, local, - copy_params, pad_params); - } + const int64_t value_offset = + (token * value_token_stride_ + head * value_head_stride_) * + element_size_; + const int64_t value_cache_offset = + (block * value_cache_block_stride_ + page * value_cache_page_stride_ + + head * value_cache_head_stride_) * + element_size_; + Copy(value_, value_offset, value_cache_, value_cache_offset, local, + copy_params, pad_params); } -#if defined(__CCE_AICORE__) && __CCE_AICORE__ == 200 - AscendC::DataCacheCleanAndInvalid( - key_cache_); -#endif } private: diff --git a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h index ac5ba8acd..daa14a352 100644 --- a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h +++ b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h @@ -60,6 +60,20 @@ class Operator q_lengths_i64_.resize(sequence_count_); k_lengths_i64_.resize(sequence_count_); + // Equal-length prompts can be proven from the caller's maxima and total + // token count. Avoid a per-layer D2H copy and stream synchronization in + // the common fixed-shape serving path. + host_lengths_ready_ = sequence_count_ > 0 && max_seqlen_q > 0 && + max_seqlen_q == max_seqlen_k && + q.size(0) == sequence_count_ * max_seqlen_q; + if (host_lengths_ready_) { + for (size_t sequence = 0; sequence < sequence_count_; ++sequence) { + q_lengths_i64_[sequence] = + static_cast((sequence + 1) * max_seqlen_q); + k_lengths_i64_[sequence] = static_cast(max_seqlen_q); + } + } + use_prompt_310p_ = ascend::IsAscend310P() && !block_table.has_value() && sequence_count_ == 1 && q.size(0) > 0 && q.dtype() == DataType::kFloat16 && @@ -193,8 +207,7 @@ class Operator if (causal || window_size != std::vector({-1, -1})) { sparse_mode = window_size == std::vector({-1, -1}) ? 3 : 4; if (window_size[0] >= 0) pre_tokens = window_size[0]; - next_tokens = causal ? 0 : window_size[1]; - if (next_tokens < 0) next_tokens = max_token_count; + if (window_size[1] >= 0) next_tokens = window_size[1]; } auto scale = softmax_scale.value_or(1.0 / std::sqrt(q.size(2))); @@ -229,9 +242,9 @@ class Operator /*dequantScaleQuery=*/nullptr, /*learnableSink=*/nullptr, q.size(1), scale, pre_tokens, next_tokens, input_layout_.data(), num_key_value_heads, sparse_mode, - // Correct fully masked rows when right-down causal has Q longer than - // KV. - /*innerPrecise=*/2, + // Jingbo's validated Ascend TND paged prefill path uses the + // high-throughput innerPrecise=0 mode. + /*innerPrecise=*/0, /*blockSize=*/block_size, /*antiquantMode=*/0, /*softmaxLseFlag=*/false, @@ -549,6 +562,10 @@ class Operator aclIntArray* MakeSequenceLengths(const Tensor cu_seqlens, std::vector& lengths, aclrtStream stream, bool cumulative) const { + if (host_lengths_ready_) { + return aclCreateIntArray(lengths.data(), lengths.size()); + } + const auto bytes = cu_i32_host_.size() * sizeof(cu_i32_host_[0]); auto ret = aclrtMemcpyAsync(cu_i32_host_.data(), bytes, cu_seqlens.data(), bytes, ACL_MEMCPY_DEVICE_TO_HOST, stream); @@ -574,6 +591,7 @@ class Operator mutable std::vector cu_i32_host_; mutable std::vector q_lengths_i64_; mutable std::vector k_lengths_i64_; + bool host_lengths_ready_{false}; void* attention_mask_data_{nullptr}; bool use_prompt_310p_{false}; bool use_prompt_fallback_310p_{false}; diff --git a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h index be92dbb1c..08f06e74a 100644 --- a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h +++ b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h @@ -17,7 +17,41 @@ #include "native/ascend/workspace_pool_.h" #include "operator.h" +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION +#include "aclrtlaunch_infinilm_paged_attention_bf16_i32.h" +#include "aclrtlaunch_infinilm_paged_attention_f16_i32.h" +#include "aclrtlaunch_infinilm_paged_attention_fai_bf16_i32.h" +#endif + namespace infini::ops { +namespace { +struct InfinilmFaiTiling { + uint64_t mm1OutSize = 0; + uint64_t smOnlineOutSize = 0; + uint64_t mm2OutSize = 0; + uint32_t batch = 0; + uint32_t numHeads = 0; + uint32_t kvHeads = 0; + uint32_t embeddingSize = 0; + uint32_t embeddingSizeV = 0; + uint32_t blockSize = 0; + uint32_t maxNumBlocksPerBatch = 0; + uint32_t firstBatchTaskNum = 0; + uint32_t totalTaskNum = 0; + uint32_t mainLoopTaskNum = 0; + uint32_t tailLoopTaskNum = 0; + uint32_t tailStartBatch = 0; + uint32_t tailStartN2 = 0; + uint32_t tailKvNBlockTile = 0; + uint32_t maskType = 0; + float scaleValue = 0.0F; + uint32_t sparseMode = 0; + int64_t preToken = 0; + int64_t nextToken = 0; + uint64_t qStride = 0; + uint64_t oStride = 0; +}; +} // namespace template <> class Operator @@ -82,6 +116,13 @@ class Operator bool capture_safe() const override { return true; } + ~Operator() override { + if (fai_tiling_ != nullptr && ascend::IsAclRuntimeAlive()) { + (void)aclrtFree(fai_tiling_); + fai_tiling_ = nullptr; + } + } + void operator()(const Tensor q, Tensor k_cache, Tensor v_cache, const std::optional k, const std::optional v, const std::optional rotary_cos, @@ -138,6 +179,15 @@ class Operator tensor.device(), strides}; } + static Tensor SqueezeSequenceView(const Tensor tensor) { + auto shape = Tensor::Shape{tensor.shape()}; + auto strides = Tensor::Strides{tensor.strides()}; + Tensor::Shape squeezed_shape{shape[0], shape[2], shape[3]}; + Tensor::Strides squeezed_strides{strides[0], strides[2], strides[3]}; + return Tensor{const_cast(tensor.data()), squeezed_shape, + tensor.dtype(), tensor.device(), squeezed_strides}; + } + void Initialize(const Tensor q, const Tensor k_cache, const Tensor v_cache, const std::optional& k, const std::optional& v, @@ -173,6 +223,14 @@ class Operator if (paged_) block_table_cache_ = ascend::AclTensorCache(*block_table); seq_i32_host_.resize(batch_size_); seq_i64_host_.resize(batch_size_); + if (paged_ && q.dtype() == DataType::kBFloat16 && + q.size(2) == k_cache.size(2) && q.size(3) == k_cache.size(3) && + block_table->IsContiguous() && k_cache.IsContiguous() && + v_cache.IsContiguous()) { + ascend::CheckAcl(aclrtMalloc(&fai_tiling_, sizeof(InfinilmFaiTiling), + ACL_MEM_MALLOC_HUGE_FIRST), + "FAI paged attention tiling allocation"); + } } void Run(const Tensor q, const Tensor k_cache, const Tensor v_cache, @@ -183,6 +241,26 @@ class Operator auto stream = static_cast(stream_); assert(block_table.has_value() == paged_ && "Ascend `FlashAttnWithKvcache` cache mode changed between calls"); +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION + const Tensor custom_q = + paged_ && q.ndim() == 4 && q.size(1) == 1 ? SqueezeSequenceView(q) : q; + const Tensor custom_out = paged_ && out.ndim() == 4 && out.size(1) == 1 + ? SqueezeSequenceView(out) + : out; + if (fai_tiling_ != nullptr && + CanRunFai(custom_q, k_cache, v_cache, cache_seqlens, + scalar_cache_seqlens, block_table, custom_out)) { + RunFai(custom_q, k_cache, v_cache, *cache_seqlens, *block_table, + softmax_scale, custom_out, stream); + return; + } + if (CanRunPagedCustom(custom_q, k_cache, v_cache, cache_seqlens, + scalar_cache_seqlens, block_table, custom_out)) { + RunPagedCustom(custom_q, k_cache, v_cache, *cache_seqlens, *block_table, + softmax_scale, custom_out, stream); + return; + } +#endif auto actual_seq_lengths = MakeSeqLengths(cache_seqlens, scalar_cache_seqlens, stream); assert(!paged_ || actual_seq_lengths != nullptr && @@ -245,6 +323,148 @@ class Operator if (actual_seq_lengths) aclDestroyIntArray(actual_seq_lengths); } +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION + bool CanRunFai(const Tensor q, const Tensor k_cache, const Tensor v_cache, + const std::optional& cache_seqlens, + const std::optional& scalar_cache_seqlens, + const std::optional& block_table, + const Tensor out) const { + return !scalar_cache_seqlens.has_value() && cache_seqlens.has_value() && + block_table.has_value() && q.dtype() == DataType::kBFloat16 && + q.size(1) == k_cache.size(2) && + cache_seqlens->dtype() == DataType::kInt32 && + cache_seqlens->IsContiguous() && block_table->IsContiguous() && + q.stride(1) == q.size(2) && q.stride(2) == 1 && + out.stride(1) == out.size(2) && out.stride(2) == 1 && + k_cache.IsContiguous() && v_cache.IsContiguous() && + q.dtype() == k_cache.dtype() && q.dtype() == v_cache.dtype(); + } + + void RunFai(const Tensor q, const Tensor k_cache, const Tensor v_cache, + const Tensor cache_seqlens, const Tensor block_table, + const std::optional softmax_scale, Tensor out, + aclrtStream stream) const { + constexpr uint64_t kSegmentBytes = uint64_t{64} << 20; + constexpr uint32_t kAtlasA2BlockCount = 20; + InfinilmFaiTiling tiling{}; + tiling.mm1OutSize = kSegmentBytes; + tiling.smOnlineOutSize = kSegmentBytes; + tiling.mm2OutSize = kSegmentBytes; + tiling.batch = static_cast(q.size(0)); + tiling.numHeads = static_cast(q.size(1)); + tiling.kvHeads = static_cast(k_cache.size(2)); + tiling.embeddingSize = static_cast(q.size(2)); + tiling.embeddingSizeV = static_cast(out.size(2)); + tiling.blockSize = static_cast(k_cache.size(1)); + tiling.maxNumBlocksPerBatch = static_cast(block_table.size(1)); + tiling.firstBatchTaskNum = tiling.batch <= 4 ? tiling.kvHeads : 1; + tiling.totalTaskNum = tiling.batch * tiling.firstBatchTaskNum; + tiling.scaleValue = static_cast( + softmax_scale.value_or(1.0 / std::sqrt(double(q.size(2))))); + tiling.preToken = std::numeric_limits::max(); + tiling.nextToken = std::numeric_limits::max(); + tiling.qStride = static_cast(q.stride(0)); + tiling.oStride = static_cast(out.stride(0)); + ascend::CheckAcl(aclrtMemcpy(fai_tiling_, sizeof(tiling), &tiling, + sizeof(tiling), ACL_MEMCPY_HOST_TO_DEVICE), + "FAI paged attention tiling upload"); + auto& arena = ascend::GetWorkspacePool().Ensure(stream, kSegmentBytes * 4, + "fai_paged_decode"); + const auto ret = aclrtlaunch_infinilm_paged_attention_fai_bf16_i32( + kAtlasA2BlockCount, stream, out.data(), const_cast(q.data()), + const_cast(k_cache.data()), const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), tiling.scaleValue, + size_t(q.size(1)), size_t(q.size(0)), size_t(k_cache.size(2)), + size_t(q.size(2)), size_t(block_table.size(1)), size_t(k_cache.size(1)), + q.stride(0), out.stride(0), arena.buf, fai_tiling_); + ascend::CheckAcl(ret, "FAI paged attention execution"); + } + + bool CanRunPagedCustom(const Tensor q, const Tensor k_cache, + const Tensor v_cache, + const std::optional& cache_seqlens, + const std::optional& scalar_cache_seqlens, + const std::optional& block_table, + const Tensor out) const { + return !scalar_cache_seqlens.has_value() && cache_seqlens.has_value() && + block_table.has_value() && !has_alibi_slopes_ && + cache_seqlens->dtype() == DataType::kInt32 && + cache_seqlens->IsContiguous() && q.ndim() == 3 && out.ndim() == 3 && + q.stride(1) == q.size(2) && q.stride(2) == 1 && + out.stride(1) == out.size(2) && out.stride(2) == 1 && + k_cache.IsContiguous() && v_cache.IsContiguous() && + (q.dtype() == DataType::kFloat16 || + q.dtype() == DataType::kBFloat16) && + q.dtype() == k_cache.dtype() && q.dtype() == v_cache.dtype(); + } + + static void RunPagedCustom(const Tensor q, const Tensor k_cache, + const Tensor v_cache, const Tensor cache_seqlens, + const Tensor block_table, + const std::optional softmax_scale, + Tensor out, aclrtStream stream) { + const int64_t num_seqs = static_cast(q.size(0)); + const int64_t num_heads = static_cast(q.size(1)); + const int64_t num_kv_heads = static_cast(k_cache.size(2)); + const int64_t head_size = static_cast(q.size(2)); + const int64_t max_blocks = static_cast(block_table.size(1)); + const int64_t page_size = static_cast(k_cache.size(1)); + const double default_scale = + 1.0 / std::sqrt(static_cast(head_size)); + const auto scale = + static_cast(softmax_scale.value_or(default_scale)); + const int64_t block_num = num_seqs * num_heads; + const auto ret = + q.dtype() == DataType::kBFloat16 + ? aclrtlaunch_infinilm_paged_attention_bf16_i32( + static_cast(block_num), stream, out.data(), + const_cast(q.data()), + const_cast(k_cache.data()), + const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), nullptr, false, + static_cast(num_heads), static_cast(num_seqs), + static_cast(num_kv_heads), + static_cast(head_size), scale, + static_cast(max_blocks), + static_cast(page_size), + static_cast(q.stride(0)), + static_cast(k_cache.stride(0)), + static_cast(k_cache.stride(1)), + static_cast(k_cache.stride(2)), + static_cast(v_cache.stride(0)), + static_cast(v_cache.stride(1)), + static_cast(v_cache.stride(2)), + static_cast(out.stride(0)), + static_cast(block_table.stride(0)), + static_cast(cache_seqlens.stride(0))) + : aclrtlaunch_infinilm_paged_attention_f16_i32( + static_cast(block_num), stream, out.data(), + const_cast(q.data()), + const_cast(k_cache.data()), + const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), nullptr, false, + static_cast(num_heads), static_cast(num_seqs), + static_cast(num_kv_heads), + static_cast(head_size), scale, + static_cast(max_blocks), + static_cast(page_size), + static_cast(q.stride(0)), + static_cast(k_cache.stride(0)), + static_cast(k_cache.stride(1)), + static_cast(k_cache.stride(2)), + static_cast(v_cache.stride(0)), + static_cast(v_cache.stride(1)), + static_cast(v_cache.stride(2)), + static_cast(out.stride(0)), + static_cast(block_table.stride(0)), + static_cast(cache_seqlens.stride(0))); + ascend::CheckAcl(ret, "Ascend custom paged attention execution"); + } +#endif + aclIntArray* MakeSeqLengths( const std::optional& cache_seqlens, const std::optional& scalar_cache_seqlens, @@ -296,6 +516,7 @@ class Operator mutable ascend::AclTensorCache query_cache_; mutable ascend::AclTensorCache out_cache_; mutable ascend::AclTensorCache block_table_cache_; + void* fai_tiling_ = nullptr; mutable std::vector seq_i32_host_; mutable std::vector seq_i64_host_; mutable std::array bsnd_layout_{'B', 'S', 'N', 'D', '\0'}; diff --git a/src/native/ascend/ops/matmul_weight_nz/kernel.h b/src/native/ascend/ops/matmul_weight_nz/kernel.h index 0ea8f0d8a..dbc4a43fb 100644 --- a/src/native/ascend/ops/matmul_weight_nz/kernel.h +++ b/src/native/ascend/ops/matmul_weight_nz/kernel.h @@ -43,6 +43,8 @@ inline aclTensor* MakeNzWeight(const Tensor& weight) { } // namespace ascend_matmul_weight_nz_detail +// Short-term C++-only operator; migrate to Matmul after TensorView models NZ +// as a distinct distribution. template <> class Operator : public MatmulWeightNz { public: diff --git a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h index 939c25fc0..092ad030a 100644 --- a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h +++ b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h @@ -128,24 +128,46 @@ class Operator void RunStridedKernel(const Tensor key, const Tensor value, const Tensor slot_mapping, Tensor key_cache, Tensor value_cache, aclrtStream stream) const { - auto ret = aclrtlaunch_reshape_and_cache_flash_strided( - 1, stream, const_cast(key.data()), - const_cast(value.data()), const_cast(slot_mapping.data()), - key_cache.data(), value_cache.data(), static_cast(num_tokens_), - static_cast(num_heads_), static_cast(head_size_), - static_cast(block_size_), static_cast(num_blocks_), - static_cast(key_token_stride_), - static_cast(key_head_stride_), - static_cast(value_token_stride_), - static_cast(value_head_stride_), - static_cast(key_cache_block_stride_), - static_cast(key_cache_page_stride_), - static_cast(key_cache_head_stride_), - static_cast(value_cache_block_stride_), - static_cast(value_cache_page_stride_), - static_cast(value_cache_head_stride_), - static_cast(element_size_)); - ascend::CheckAcl(ret, "ReshapeAndCacheFlash strided execution"); + constexpr uint32_t kMaxCacheWriteBlocks = 64; + constexpr int64_t kMaxCacheWriteTokensPerLaunch = 4096; + auto* key_bytes = static_cast(const_cast(key.data())); + auto* value_bytes = static_cast(const_cast(value.data())); + auto* slot_bytes = + static_cast(const_cast(slot_mapping.data())); + + for (int64_t token_offset = 0; + token_offset < static_cast(num_tokens_); + token_offset += kMaxCacheWriteTokensPerLaunch) { + const auto remaining = static_cast(num_tokens_) - token_offset; + const auto token_count = remaining < kMaxCacheWriteTokensPerLaunch + ? remaining + : kMaxCacheWriteTokensPerLaunch; + const auto requested_work_items = + static_cast(token_count * num_heads_); + const auto work_items = requested_work_items < kMaxCacheWriteBlocks + ? requested_work_items + : kMaxCacheWriteBlocks; + auto ret = aclrtlaunch_reshape_and_cache_flash_strided( + work_items, stream, + key_bytes + token_offset * key_token_stride_ * element_size_, + value_bytes + token_offset * value_token_stride_ * element_size_, + slot_bytes + token_offset, key_cache.data(), value_cache.data(), + token_count, static_cast(num_heads_), + static_cast(head_size_), static_cast(block_size_), + static_cast(num_blocks_), + static_cast(key_token_stride_), + static_cast(key_head_stride_), + static_cast(value_token_stride_), + static_cast(value_head_stride_), + static_cast(key_cache_block_stride_), + static_cast(key_cache_page_stride_), + static_cast(key_cache_head_stride_), + static_cast(value_cache_block_stride_), + static_cast(value_cache_page_stride_), + static_cast(value_cache_head_stride_), + static_cast(element_size_)); + ascend::CheckAcl(ret, "ReshapeAndCacheFlash strided execution"); + } } #endif diff --git a/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h index 9afe50030..36cdfe173 100644 --- a/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h +++ b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h @@ -30,6 +30,8 @@ inline aclTensor* MakeContiguousTensor(const Tensor& tensor) { } // namespace ascend_trans_matmul_weight_nz_detail +// Short-term C++-only operator; migrate to the general transpose-matmul path +// after TensorView models NZ as a distinct distribution. template <> class Operator : public TransMatmulWeightNz { @@ -56,8 +58,6 @@ class Operator ascend::CheckAcl( aclnnTransMatmulWeight(arena.buf, workspace_size, executor, stream), "TransMatmulWeightNz execution"); - ascend::CheckAcl(aclrtSynchronizeStream(stream), - "TransMatmulWeightNz synchronize"); } }; diff --git a/tests/test_flash_attn_with_kvcache.py b/tests/test_flash_attn_with_kvcache.py index 4bf20a328..60bc3806a 100644 --- a/tests/test_flash_attn_with_kvcache.py +++ b/tests/test_flash_attn_with_kvcache.py @@ -6,7 +6,6 @@ from tests.utils import get_stream - if not hasattr(infini.ops, "FlashAttnWithKvcache"): pytest.skip( "`FlashAttnWithKvcache` is not available on this platform", @@ -154,6 +153,61 @@ def test_flash_attn_with_kvcache_ascend_paged(device, implementation_index): torch.testing.assert_close(actual, expected, rtol=2e-3, atol=2e-3) +def test_flash_attn_with_kvcache_ascend_paged_bf16_mha(device, implementation_index): + if device != "npu": + pytest.skip("this test covers the Ascend provider") + + batch_size, page_size = 2, 256 + num_heads = num_kv_heads = 4 + head_size = 64 + q = torch.randn( + (batch_size, 1, num_heads, head_size), + dtype=torch.bfloat16, + device=device, + ) + k_cache = torch.randn( + (4, page_size, num_kv_heads, head_size), + dtype=torch.bfloat16, + device=device, + ) + v_cache = torch.randn_like(k_cache) + lengths = (130, 300) + cache_seqlens = torch.tensor(lengths, dtype=torch.int32, device=device) + block_table = torch.tensor(((0, 1), (2, 3)), dtype=torch.int32, device=device) + expected = _reference_decode( + q, k_cache, v_cache, lengths, block_table=block_table, scale=0.125 + ) + actual = torch.empty_like(q) + + infini.ops.flash_attn_with_kvcache( + q, + k_cache, + v_cache, + None, + None, + None, + None, + cache_seqlens, + None, + None, + block_table, + None, + 0.125, + True, + (-1, -1), + 0.0, + True, + 0, + False, + actual, + None, + stream=get_stream(q.device), + implementation_index=implementation_index, + ) + + torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2) + + def test_flash_attn_with_kvcache_ascend_non_default_stream( device, implementation_index ): diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py index 922bc9851..2afce1fb1 100644 --- a/tests/test_generate_wrappers.py +++ b/tests/test_generate_wrappers.py @@ -19,6 +19,14 @@ def _load_generator_module(): return module +def test_temporary_nz_matmul_operators_are_cpp_only(): + module = _load_generator_module() + + assert not module._generates_public_api("matmul_weight_nz") + assert not module._generates_public_api("trans_matmul_weight_nz") + assert module._generates_public_api("matmul") + + def test_generated_dispatch_keeps_scalar_and_optional_tensor_overloads_distinct( monkeypatch, tmp_path ): diff --git a/tests/test_matmul_weight_nz.py b/tests/test_matmul_weight_nz.py deleted file mode 100644 index c06fc6cab..000000000 --- a/tests/test_matmul_weight_nz.py +++ /dev/null @@ -1,46 +0,0 @@ -import infini.ops -import pytest -import torch - -from tests.utils import empty_strided, get_stream, randn_strided - - -@pytest.mark.parametrize( - "input_shape, weight_shape", - ( - ((1, 16), (16, 16)), - ((3, 32), (32, 48)), - ((17, 64), (64, 80)), - ), -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float16, 1e-2, 1e-2), - (torch.bfloat16, 2e-2, 1e-2), - ), -) -def test_trans_matmul_weight_nz( - input_shape, - weight_shape, - dtype, - device, - rtol, - atol, -): - if device != "npu": - pytest.skip("ACLNN MatmulWeightNz is an Ascend-only path") - - input = randn_strided(input_shape, None, dtype=dtype, device=device) - weight = randn_strided(weight_shape, None, dtype=dtype, device=device) - converted = empty_strided(weight.shape, None, dtype=dtype, device=device) - output = empty_strided( - (input.shape[0], weight.shape[1]), None, dtype=dtype, device=device - ) - stream = get_stream(input.device) - - infini.ops.trans_matmul_weight_nz(weight, converted, stream=stream) - infini.ops.matmul_weight_nz(input, converted, output, stream=stream) - - reference = torch.matmul(input.float(), weight.float()).to(dtype) - torch.testing.assert_close(output, reference, rtol=rtol, atol=atol) diff --git a/third_party/ops-transformer b/third_party/ops-transformer new file mode 160000 index 000000000..e1e99f483 --- /dev/null +++ b/third_party/ops-transformer @@ -0,0 +1 @@ +Subproject commit e1e99f4832cf10d9af870d29ee4135f2e749f898 diff --git a/third_party/ops-transformer-fm9g70.patch b/third_party/ops-transformer-fm9g70.patch new file mode 100644 index 000000000..92571ad9a --- /dev/null +++ b/third_party/ops-transformer-fm9g70.patch @@ -0,0 +1,51 @@ +--- a/attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h ++++ b/attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h +@@ -97,10 +97,8 @@ + int64_t nextToken = fATilingData->nextToken; + AscendC::GlobalTensor gQ; + gQ.SetGlobalBuffer((__gm__ ElementQ *)params.q); +- AscendC::ListTensorDesc keyListTensorDescInit((__gm__ void*)params.k); +- AscendC::ListTensorDesc valueListTensorDescInit((__gm__ void*)params.v); +- __gm__ uint8_t* currentKey = (__gm__ uint8_t*)keyListTensorDescInit.GetDataPtr<__gm__ uint8_t>(0); +- __gm__ uint8_t* currentValue = (__gm__ uint8_t*)valueListTensorDescInit.GetDataPtr<__gm__ uint8_t>(0); ++ __gm__ uint8_t* currentKey = (__gm__ uint8_t*)params.k; ++ __gm__ uint8_t* currentValue = (__gm__ uint8_t*)params.v; + AscendC::GlobalTensor gK; + gK.SetGlobalBuffer((__gm__ ElementK *)currentKey); + AscendC::GlobalTensor gV; +@@ -109,10 +107,10 @@ + gMask.SetGlobalBuffer((__gm__ ElementMask *)params.mask); + AscendC::GlobalTensor gBlockTable; + gBlockTable.SetGlobalBuffer((__gm__ int32_t *)(params.blockTables)); +- AscendC::GlobalTensor gActualQseqlen; +- gActualQseqlen.SetGlobalBuffer((__gm__ int64_t *)params.actualQseqlen); +- AscendC::GlobalTensor gActualKvseqlen; +- gActualKvseqlen.SetGlobalBuffer((__gm__ int64_t *)params.actualKvseqlen); ++ AscendC::GlobalTensor gActualQseqlen; ++ gActualQseqlen.SetGlobalBuffer((__gm__ int32_t *)params.actualQseqlen); ++ AscendC::GlobalTensor gActualKvseqlen; ++ gActualKvseqlen.SetGlobalBuffer((__gm__ int32_t *)params.actualKvseqlen); + AscendC::GlobalTensor gO; + gO.SetGlobalBuffer((__gm__ ElementO *)params.o); + AscendC::GlobalTensor gLse; +@@ -192,16 +190,16 @@ + + coreIdx = AscendC::GetBlockIdx() / AscendC::GetSubBlockNum(); + #endif +- uint64_t strideQ = static_cast(qHeads * embed); +- uint64_t strideO = static_cast(qHeads * embedV); ++ uint64_t strideQ = fATilingData->qStride; ++ uint64_t strideO = fATilingData->oStride; + uint64_t strideK = static_cast(kvHeads * embed); + uint64_t strideV = static_cast(kvHeads * embedV); + uint32_t embedRound = NpuArch::Detail::Alignment::RoundUp(embed, FaiKernel::BLOCK_SIZE); + uint32_t embedRoundV = NpuArch::Detail::Alignment::RoundUp(embedV, FaiKernel::BLOCK_SIZE); + uint32_t groupSize = qHeads / kvHeads; + +- uint32_t totalQTokens = static_cast(gActualQseqlen.GetValue(batch - 1)); +- uint32_t curKvNBlockTile = GetKvNBlockTile(groupSize, kvHeads); ++ uint32_t totalQTokens = batch; ++ uint32_t curKvNBlockTile = batch <= 4 ? 1 : GetKvNBlockTile(groupSize, kvHeads); + uint32_t curKvNBlockNum = firstBatchTaskNum; + uint32_t rowNumConst = groupSize; + uint32_t rowNumRoundConst = NpuArch::Detail::Alignment::RoundUp(rowNumConst, FaiKernel::BLOCK_SIZE);