diff --git a/.gitmodules b/.gitmodules index be99e8a8f..902143ba0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,7 @@ [submodule ".ci"] path = .ci url = https://github.com/InfiniTensor/ci.git +[submodule "third_party/ops-transformer"] + path = third_party/ops-transformer + url = https://gitcode.com/cann/ops-transformer.git + update = none diff --git a/scripts/fetch-ops-transformer.sh b/scripts/fetch-ops-transformer.sh new file mode 100755 index 000000000..3be63c91a --- /dev/null +++ b/scripts/fetch-ops-transformer.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +SUBMODULE_REL=third_party/ops-transformer +SUBMODULE="$ROOT/$SUBMODULE_REL" +GIT="git -c safe.directory=*" +EXPECTED=$($GIT -C "$ROOT" ls-tree HEAD "$SUBMODULE_REL" | awk '{print $3}') + +if ! $GIT -C "$SUBMODULE" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + $GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL" +else + ACTUAL=$($GIT -C "$SUBMODULE" rev-parse HEAD) + if [ "$ACTUAL" != "$EXPECTED" ]; then + $GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL" + fi +fi + +printf 'ops-transformer: %s\n' "$($GIT -C "$SUBMODULE" rev-parse HEAD)" diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py index bb03d2b57..dc9f7bb87 100644 --- a/scripts/generate_wrappers.py +++ b/scripts/generate_wrappers.py @@ -41,6 +41,14 @@ _OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") +# Temporary compatibility operators used directly by InfiniLM. Keep their +# generated C++ dispatch available, but do not expose legacy C or Python APIs. +_CPP_ONLY_OPS = frozenset(("matmul_weight_nz", "trans_matmul_weight_nz")) + + +def _generates_public_api(op_name): + return op_name not in _CPP_ONLY_OPS + def _op_namespace_parts(op_name): parts = op_name.split("_") @@ -2120,6 +2128,7 @@ def _generate_op_artifacts(item): "op_name": op_name, "header_name": header_name, "bind_func_name": f"Bind{_op_symbol_name(op_name)}", + "public_api": _generates_public_api(op_name), "pybind11": _generate_pybind11(operator), "binding_source": _generate_binding_source(op_name), "legacy_c_source": legacy_c_source, @@ -2356,28 +2365,29 @@ def _dispatch_gen_batch_size(): source_path = _GENERATED_SRC_DIR / op_name header_name = artifact["header_name"] bind_func_name = artifact["bind_func_name"] - binding_header_path = _BINDINGS_DIR / header_name + if artifact["public_api"]: + binding_header_path = _BINDINGS_DIR / header_name - _write_text_if_changed(binding_header_path, artifact["pybind11"]) - expected_binding_files.add(binding_header_path) + _write_text_if_changed(binding_header_path, artifact["pybind11"]) + expected_binding_files.add(binding_header_path) - if use_monolithic_bindings: - op_includes.append(f'#include "{header_name}"') - else: - binding_source_path = _BINDINGS_DIR / f"{op_name}.cc" - _write_text_if_changed(binding_source_path, artifact["binding_source"]) - expected_binding_files.add(binding_source_path) + if use_monolithic_bindings: + op_includes.append(f'#include "{header_name}"') + else: + binding_source_path = _BINDINGS_DIR / f"{op_name}.cc" + _write_text_if_changed(binding_source_path, artifact["binding_source"]) + expected_binding_files.add(binding_source_path) - source_path.mkdir(exist_ok=True) - legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc" - _write_text_if_changed(legacy_source_path, artifact["legacy_c_source"]) - expected_generated_src_files.add(legacy_source_path) + source_path.mkdir(exist_ok=True) + legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc" + _write_text_if_changed(legacy_source_path, artifact["legacy_c_source"]) + expected_generated_src_files.add(legacy_source_path) - include_path = _INCLUDE_DIR / header_name - _write_text_if_changed(include_path, artifact["legacy_c_header"]) - expected_include_files.add(include_path) + include_path = _INCLUDE_DIR / header_name + _write_text_if_changed(include_path, artifact["legacy_c_header"]) + expected_include_files.add(include_path) - bind_func_names.append(bind_func_name) + bind_func_names.append(bind_func_name) dispatch_header = _generate_generated_dispatch_header( op_names, args.devices, dispatch_declarations diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e685b4f12..2765ab70c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -551,6 +551,51 @@ if(WITH_ASCEND) # Custom `AscendC` kernels (PyTorch extension, requires `torch_npu`). if(BUILD_ASCEND_CUSTOM) + set(_ops_transformer_root + "${PROJECT_SOURCE_DIR}/third_party/ops-transformer") + option(INFINIOPS_FETCH_OPS_TRANSFORMER + "Fetch and pin the official Ascend ops-transformer submodule" ON) + if(INFINIOPS_FETCH_OPS_TRANSFORMER) + execute_process( + COMMAND bash "${PROJECT_SOURCE_DIR}/scripts/fetch-ops-transformer.sh" + RESULT_VARIABLE _ops_transformer_fetch_result) + if(NOT _ops_transformer_fetch_result EQUAL 0) + message(FATAL_ERROR "Failed to fetch the pinned ops-transformer submodule") + endif() + endif() + + if(NOT EXISTS "${_ops_transformer_root}/CMakeLists.txt") + message(FATAL_ERROR + "ops-transformer is missing; run scripts/fetch-ops-transformer.sh") + endif() + + # Keep the submodule pinned to an official commit and apply only the + # small FM9G70 decode ABI patch after checkout. + find_package(Git REQUIRED) + set(_ops_transformer_patch + "${PROJECT_SOURCE_DIR}/third_party/ops-transformer-fm9g70.patch") + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + --check -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_patch_status) + if(_ops_transformer_patch_status EQUAL 0) + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_patch_result) + if(NOT _ops_transformer_patch_result EQUAL 0) + message(FATAL_ERROR "Failed to patch ops-transformer for FM9G70") + endif() + else() + execute_process( + COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply + --check --reverse -p1 "${_ops_transformer_patch}" + RESULT_VARIABLE _ops_transformer_reverse_status) + if(NOT _ops_transformer_reverse_status EQUAL 0) + message(FATAL_ERROR + "ops-transformer has incompatible local changes") + endif() + endif() # In-tree `ascendc_library()` trips the `CANN` `extract_host_stub.py` # path-handling bug under `scikit-build-core`'s temp-dir builds # (`KeyError` on `/./workspace/...` paths in `$`). @@ -579,6 +624,12 @@ if(WITH_ASCEND) "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/*.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/CMakeLists.txt" "${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/build.sh") + set(_ops_transformer_fai_dir + "${_ops_transformer_root}/attention/fused_infer_attention_score/op_kernel") + file(GLOB_RECURSE _ops_transformer_fai_srcs CONFIGURE_DEPENDS + "${_ops_transformer_fai_dir}/*.h" + "${_ops_transformer_fai_dir}/*.hpp" + "${_ops_transformer_fai_dir}/*.cpp") # Scrub env inherited from the outer `scikit-build-core` invocation # before handing control to `build.sh`: `CMAKE_GENERATOR` / @@ -604,9 +655,10 @@ if(WITH_ASCEND) --unset=CMAKE_BUILD_PARALLEL_LEVEL "BUILD_DIR=${_custom_build_dir}" "MAIN_SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}" + "OPS_TRANSFORMER_ROOT=${_ops_transformer_root}" bash ${_custom_source_dir}/build.sh ${SOC_VERSION} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom - DEPENDS ${_custom_srcs} + DEPENDS ${_custom_srcs} ${_ops_transformer_fai_srcs} COMMENT "Building custom AscendC kernels (SOC_VERSION=${SOC_VERSION})" VERBATIM) @@ -621,6 +673,7 @@ if(WITH_ASCEND) # custom kernel implementations (e.g. `RmsNorm` index 1) can call # them via the generated launch functions. target_compile_definitions(infiniops PUBLIC INFINI_HAS_CUSTOM_KERNELS=1) + target_compile_definitions(infiniops PUBLIC INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION=1) target_include_directories(infiniops PRIVATE "${_custom_build_dir}/include/no_workspace_kernel") target_link_libraries(infiniops PRIVATE diff --git a/src/base/matmul_weight_nz.h b/src/base/matmul_weight_nz.h new file mode 100644 index 000000000..7ff95eaaa --- /dev/null +++ b/src/base/matmul_weight_nz.h @@ -0,0 +1,57 @@ +#ifndef INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_ +#define INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +// Temporary compatibility operator for InfiniLM's Ascend NZ weight path. It is +// intentionally C++-only. Once TensorView distinguishes ND and NZ +// distributions, migrate this behavior to the general Matmul operator and +// remove this class. +class MatmulWeightNz : public Operator { + public: + MatmulWeightNz(const Tensor input, const Tensor weight, Tensor out) + : rows_{input.size(0)}, + channels_{input.size(1)}, + outputs_{weight.size(1)} { + assert(input.ndim() == 2 && weight.ndim() == 2 && out.ndim() == 2 && + "operator `MatmulWeightNz` requires 2D tensors"); + assert(input.size(1) == weight.size(0) && + "operator `MatmulWeightNz` input channels must match weight rows"); + assert(out.size(0) == input.size(0) && + "operator `MatmulWeightNz` output rows must match input"); + assert(out.size(1) == weight.size(1) && + "operator `MatmulWeightNz` output columns must match weight"); + assert(input.dtype() == weight.dtype() && input.dtype() == out.dtype() && + "operator `MatmulWeightNz` requires matching dtypes"); + assert((input.dtype() == DataType::kFloat16 || + input.dtype() == DataType::kBFloat16) && + "operator `MatmulWeightNz` supports fp16 or bf16 only"); + assert(weight.stride(0) == static_cast(weight.size(1)) && + weight.stride(1) == 1 && + "operator `MatmulWeightNz` weight must be contiguous"); + assert(input.stride(0) == static_cast(input.size(1)) && + input.stride(1) == 1 && + "operator `MatmulWeightNz` input must be contiguous"); + assert(out.stride(0) == static_cast(out.size(1)) && + out.stride(1) == 1 && + "operator `MatmulWeightNz` output must be contiguous"); + assert(weight.size(0) % 16 == 0 && weight.size(1) % 16 == 0 && + "operator `MatmulWeightNz` requires 16-aligned weight shape"); + } + + virtual void operator()(const Tensor input, const Tensor weight, + Tensor out) const = 0; + + protected: + Tensor::Size rows_; + Tensor::Size channels_; + Tensor::Size outputs_; +}; + +} // namespace infini::ops + +#endif diff --git a/src/base/trans_matmul_weight_nz.h b/src/base/trans_matmul_weight_nz.h new file mode 100644 index 000000000..126df569b --- /dev/null +++ b/src/base/trans_matmul_weight_nz.h @@ -0,0 +1,45 @@ +#ifndef INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_ +#define INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +// Temporary compatibility operator for InfiniLM's Ascend NZ weight conversion. +// It is intentionally C++-only. Once TensorView distinguishes ND and NZ +// distributions, migrate this behavior to the general transpose-matmul path and +// remove this class. +class TransMatmulWeightNz : public Operator { + public: + TransMatmulWeightNz(const Tensor input, Tensor output) + : rows_{input.size(0)}, columns_{input.size(1)} { + assert(input.ndim() == 2 && output.ndim() == 2 && + "operator `TransMatmulWeightNz` requires 2D tensors"); + assert(input.shape() == output.shape() && + "operator `TransMatmulWeightNz` input and output shapes differ"); + assert(input.dtype() == output.dtype() && + (input.dtype() == DataType::kFloat16 || + input.dtype() == DataType::kBFloat16) && + "operator `TransMatmulWeightNz` supports fp16 or bf16 only"); + assert(input.stride(0) == static_cast(input.size(1)) && + input.stride(1) == 1 && + "operator `TransMatmulWeightNz` input must be contiguous"); + assert(output.stride(0) == static_cast(output.size(1)) && + output.stride(1) == 1 && + "operator `TransMatmulWeightNz` output must be contiguous"); + assert(rows_ % 16 == 0 && columns_ % 16 == 0 && + "operator `TransMatmulWeightNz` requires a 16-aligned shape"); + } + + virtual void operator()(const Tensor input, Tensor output) const = 0; + + protected: + Tensor::Size rows_; + Tensor::Size columns_; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/ascend/custom/CMakeLists.txt b/src/native/ascend/custom/CMakeLists.txt index 94f19850a..5547a21bc 100644 --- a/src/native/ascend/custom/CMakeLists.txt +++ b/src/native/ascend/custom/CMakeLists.txt @@ -32,6 +32,8 @@ endif() set(PROJECT_OP_SRC_BASE ${PROJECT_SOURCE_DIR}) set(MAIN_SRC_DIR "${PROJECT_OP_SRC_BASE}/../.." CACHE PATH "Main InfiniOps source directory.") +set(OPS_TRANSFORMER_ROOT "" CACHE PATH + "Pinned ops-transformer source root supplied by the outer build") include(cmake/config_envs.cmake) include(cmake/config_ascend.cmake) @@ -63,6 +65,8 @@ ascendc_library(no_workspace_kernel STATIC rms_norm/op_kernel/rms_norm.cpp add_rms_norm/op_kernel/add_rms_norm.cpp reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp + paged_attention/op_kernel/paged_attention.cpp + paged_attention/op_kernel/paged_attention_fai.cpp ) # `CANN 8.5.1` installs host objects under `objects-/`, while @@ -80,9 +84,18 @@ endif() # enum. `ascendc_library` forwards the interface target's `INCLUDES` # property to the nested `ExternalProject_Add` (see # `${ASCEND_HOME_PATH}/tools/tikcpp/ascendc_kernel_cmake/legacy_modules/function.cmake`), -# so append the main `src/` dir here. -set_property(TARGET no_workspace_kernel_interface APPEND PROPERTY - INCLUDES ${MAIN_SRC_DIR}) +# so register the main `src/` dir and repository root here. Resolve the +# repository root through the symlinked source directory used by the outer +# build. Resolve the symlink before appending `..`; CMake otherwise +# normalizes those components lexically. +get_filename_component(CUSTOM_SOURCE_DIR "${PROJECT_SOURCE_DIR}" REALPATH) +get_filename_component(INFINIOPS_ROOT + "${CUSTOM_SOURCE_DIR}/../../../.." REALPATH) +ascendc_include_directories(no_workspace_kernel INTERFACE + ${MAIN_SRC_DIR} + ${INFINIOPS_ROOT} + ${OPS_TRANSFORMER_ROOT} +) # Create the shared library `libascend_kernel.so`. add_library(${OP_PLUGIN_NAME} SHARED ${OP_SRCS}) diff --git a/src/native/ascend/custom/build.sh b/src/native/ascend/custom/build.sh index f7e1cf684..25ee14410 100755 --- a/src/native/ascend/custom/build.sh +++ b/src/native/ascend/custom/build.sh @@ -33,6 +33,7 @@ cmake \ -DSOC_VERSION="${SOC_VERSION}" \ -DCMAKE_LIBRARY_OUTPUT_DIRECTORY="${OUTPUT_DIR}" \ -DMAIN_SRC_DIR="${MAIN_SRC_DIR}" \ + -DOPS_TRANSFORMER_ROOT="${OPS_TRANSFORMER_ROOT}" \ -B "${BUILD_DIR}" \ -S "${SCRIPT_DIR}" diff --git a/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp new file mode 100644 index 000000000..58817b51a --- /dev/null +++ b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention.cpp @@ -0,0 +1,338 @@ +#include "kernel_operator.h" + +constexpr size_t PAGED_ATTENTION_TILE = 128; +constexpr size_t PAGED_ATTENTION_COPY_TILE = 32; +constexpr size_t BYTE_ALIGN = 32; + +template +__aicore__ inline size_t alignTileLen(size_t tile_len, size_t byte_align) { + size_t bytes = tile_len * sizeof(T); + size_t aligned_bytes = (bytes % byte_align == 0) + ? bytes + : (bytes + byte_align - bytes % byte_align); + return aligned_bytes / sizeof(T); +} + +#include + +using namespace AscendC; + +template +__aicore__ inline int64_t loadIndex(GlobalTensor& tensor, + ptrdiff_t offset) { + return static_cast(tensor.GetValue(offset)); +} + +template +class PagedAttentionKernel { + public: + __aicore__ inline PagedAttentionKernel() {} + + __aicore__ inline void init( + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, + GM_ADDR block_tables, GM_ADDR cache_lens, GM_ADDR alibi_slopes, + bool has_alibi, size_t num_heads, size_t num_seqs, size_t num_kv_heads, + size_t head_size, float scale, size_t max_num_blocks_per_seq, + size_t page_block_size, ptrdiff_t q_stride, ptrdiff_t k_batch_stride, + ptrdiff_t k_row_stride, ptrdiff_t k_head_stride, ptrdiff_t v_batch_stride, + ptrdiff_t v_row_stride, ptrdiff_t v_head_stride, ptrdiff_t o_stride, + ptrdiff_t block_table_batch_stride, ptrdiff_t cache_lens_stride) { + _num_heads = num_heads; + _num_seqs = num_seqs; + _num_kv_heads = num_kv_heads; + _head_size = head_size; + _scale = scale; + _max_num_blocks_per_seq = max_num_blocks_per_seq; + _page_block_size = page_block_size; + _q_stride = q_stride; + _k_batch_stride = k_batch_stride; + _k_row_stride = k_row_stride; + _k_head_stride = k_head_stride; + _v_batch_stride = v_batch_stride; + _v_row_stride = v_row_stride; + _v_head_stride = v_head_stride; + _o_stride = o_stride; + _block_table_batch_stride = block_table_batch_stride; + _cache_lens_stride = cache_lens_stride; + _has_alibi = has_alibi; + + _out_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(out)); + _q_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(q)); + _k_cache_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(k_cache)); + _v_cache_gm.SetGlobalBuffer(reinterpret_cast<__gm__ Tdata*>(v_cache)); + _block_tables_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ Tindex*>(block_tables)); + _cache_lens_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ Tindex*>(cache_lens)); + if (_has_alibi) { + _alibi_slopes_gm.SetGlobalBuffer( + reinterpret_cast<__gm__ float*>(alibi_slopes)); + } + + _pipe.InitBuffer( + _q_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(float)); + _pipe.InitBuffer( + _acc_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(float)); + _pipe.InitBuffer(_score_buf, PAGED_ATTENTION_TILE * sizeof(float)); + _pipe.InitBuffer( + _out_buf, alignTileLen(_head_size, BYTE_ALIGN) * sizeof(Tdata)); + + // Shared float scratchpad for K/V type-casting (used by computeScore and V + // accumulation) + _pipe.InitBuffer(_v_f32_buf, alignTileLen(_head_size, BYTE_ALIGN) * + sizeof(float)); + _pipe.InitBuffer(_reduce_buf, alignTileLen(_head_size, BYTE_ALIGN) * + sizeof(float)); + // Double-buffered VECIN queue for K prefetch + _head_stride = alignTileLen(_head_size, BYTE_ALIGN); + _pipe.InitBuffer(_k_queue, 2, + PAGED_ATTENTION_COPY_TILE * _head_stride * sizeof(Tdata)); + // Double-buffered VECIN queue for V prefetch (overlaps with Score VEC + // computation) + _pipe.InitBuffer(_v_queue, 2, + PAGED_ATTENTION_COPY_TILE * _head_stride * sizeof(Tdata)); + } + + __aicore__ inline void process() { + const size_t work_idx = GetBlockIdx(); + const size_t head_idx = work_idx % _num_heads; + const size_t seq_idx = work_idx / _num_heads; + if (seq_idx >= _num_seqs) { + return; + } + + const int64_t seq_len_i64 = loadIndex( + _cache_lens_gm, static_cast(seq_idx) * _cache_lens_stride); + if (seq_len_i64 <= 0) { + return; + } + const size_t seq_len = static_cast(seq_len_i64); + const size_t num_queries_per_kv = _num_heads / _num_kv_heads; + const size_t kv_head_idx = head_idx / num_queries_per_kv; + const float alibi_slope = + _has_alibi ? _alibi_slopes_gm.GetValue(head_idx) : 0.0f; + + LocalTensor q_local = _q_buf.Get(); + LocalTensor acc_local = _acc_buf.Get(); + LocalTensor score_local = _score_buf.Get(); + + // Load Q from global memory + const ptrdiff_t q_base = static_cast(seq_idx) * _q_stride + + static_cast(head_idx * _head_size); + + LocalTensor q_raw_local = _out_buf.Get(); + DataCopy(q_raw_local, _q_gm[q_base], _head_size); + PipeBarrier(); + + Cast(q_local, q_raw_local, AscendC::RoundMode::CAST_NONE, _head_size); + Duplicate(acc_local, 0.0f, _head_size); + + // Online softmax with burst K/V copies. + float max_score = -3.4028234663852886e38f; + float sum_exp = 0.0f; + const uint16_t copy_len = + static_cast(_head_stride * sizeof(Tdata) / ONE_BLK_SIZE); + const auto gap32 = [](ptrdiff_t row, size_t head) { + const ptrdiff_t value = row - static_cast(head); + if (value <= 0) return static_cast(0); + return static_cast(value * sizeof(Tdata) / ONE_BLK_SIZE - 1); + }; + const uint16_t k_gap = gap32(_k_row_stride, _head_stride); + const uint16_t v_gap = gap32(_v_row_stride, _head_stride); + for (size_t block_idx = 0; block_idx < _max_num_blocks_per_seq; + ++block_idx) { + const size_t token_start = block_idx * _page_block_size; + if (token_start >= seq_len) break; + const int64_t block_id = + loadIndex(_block_tables_gm, static_cast(seq_idx) * + _block_table_batch_stride + + static_cast(block_idx)); + size_t token_end = token_start + _page_block_size; + if (token_end > seq_len) token_end = seq_len; + const size_t tile_count = token_end - token_start; + const ptrdiff_t k_base = + static_cast(block_id) * _k_batch_stride + + static_cast(kv_head_idx) * _k_head_stride; + const ptrdiff_t v_base = + static_cast(block_id) * _v_batch_stride + + static_cast(kv_head_idx) * _v_head_stride; + size_t offset = 0; + while (offset < tile_count) { + const size_t count = (tile_count - offset < PAGED_ATTENTION_COPY_TILE + ? tile_count - offset + : PAGED_ATTENTION_COPY_TILE); + if (offset == 0) { + LocalTensor k0 = _k_queue.AllocTensor(); + DataCopy( + k0, _k_cache_gm[k_base], + DataCopyParams(static_cast(count), copy_len, k_gap, 0)); + _k_queue.EnQue(k0); + LocalTensor v0 = _v_queue.AllocTensor(); + DataCopy( + v0, _v_cache_gm[v_base], + DataCopyParams(static_cast(count), copy_len, v_gap, 0)); + _v_queue.EnQue(v0); + } + LocalTensor k_tile = _k_queue.DeQue(); + LocalTensor v_tile = _v_queue.DeQue(); + const size_t next = offset + count; + if (next < tile_count) { + const size_t nc = (tile_count - next < PAGED_ATTENTION_COPY_TILE + ? tile_count - next + : PAGED_ATTENTION_COPY_TILE); + LocalTensor kn = _k_queue.AllocTensor(); + DataCopy( + kn, _k_cache_gm[k_base + next * _k_row_stride], + DataCopyParams(static_cast(nc), copy_len, k_gap, 0)); + _k_queue.EnQue(kn); + LocalTensor vn = _v_queue.AllocTensor(); + DataCopy( + vn, _v_cache_gm[v_base + next * _v_row_stride], + DataCopyParams(static_cast(nc), copy_len, v_gap, 0)); + _v_queue.EnQue(vn); + } + for (size_t i = 0; i < count; ++i) { + LocalTensor k_raw = k_tile[i * _head_stride]; + LocalTensor v_raw = v_tile[i * _head_stride]; + const size_t t = token_start + offset + i; + const float score = + computeScore(q_local, k_raw, t, alibi_slope, seq_len); + if (score > max_score) { + score_local.SetValue(0, max_score - score); + Exp(score_local, score_local, 1); + const float rescale = score_local.GetValue(0); + Muls(acc_local, acc_local, rescale, _head_size); + sum_exp *= rescale; + max_score = score; + } + score_local.SetValue(0, score - max_score); + Exp(score_local, score_local, 1); + const float prob = score_local.GetValue(0); + sum_exp += prob; + LocalTensor v_f32 = _v_f32_buf.Get(); + Cast(v_f32, v_raw, AscendC::RoundMode::CAST_NONE, _head_size); + Muls(v_f32, v_f32, prob, _head_size); + Add(acc_local, acc_local, v_f32, _head_size); + } + _k_queue.FreeTensor(k_tile); + _v_queue.FreeTensor(v_tile); + offset = next; + } + } + PipeBarrier(); + // Normalize accumulator and write back output + const float inv_sum = 1.0f / (sum_exp + 1e-6f); + LocalTensor out_local = _out_buf.Get(); + + Muls(acc_local, acc_local, inv_sum, _head_size); + PipeBarrier(); + + Cast(out_local, acc_local, AscendC::RoundMode::CAST_RINT, + static_cast(_head_size)); + PipeBarrier(); + + const ptrdiff_t out_base = static_cast(seq_idx) * _o_stride + + static_cast(head_idx * _head_size); + + DataCopy(_out_gm[out_base], out_local, _head_size); + } + + private: + __aicore__ inline float computeScore(LocalTensor& q_local, + LocalTensor& k_raw_local, + size_t token_idx, float alibi_slope, + size_t seq_len) { + LocalTensor k_f32_local = _v_f32_buf.Get(); + + Cast(k_f32_local, k_raw_local, AscendC::RoundMode::CAST_NONE, _head_size); + Mul(k_f32_local, q_local, k_f32_local, _head_size); + LocalTensor temp_workspace = _reduce_buf.Get(); + ReduceSum(k_f32_local, k_f32_local, temp_workspace, _head_size); + PipeBarrier(); + + float score = k_f32_local.GetValue(0); + + score *= _scale; + if (_has_alibi) { + score += + alibi_slope * static_cast(static_cast(token_idx) - + static_cast(seq_len) + 1); + } + return score; + } + + GlobalTensor _out_gm; + GlobalTensor _q_gm; + GlobalTensor _k_cache_gm; + GlobalTensor _v_cache_gm; + GlobalTensor _block_tables_gm; + GlobalTensor _cache_lens_gm; + GlobalTensor _alibi_slopes_gm; + + TPipe _pipe; + TBuf _q_buf; + TBuf _acc_buf; + TBuf _score_buf; + TBuf _out_buf; + // Shared float scratchpad for K/V type-casting (used by computeScore and V + // accumulation) + TBuf _v_f32_buf; + TBuf _reduce_buf; + // Double-buffered VECIN queue for K prefetch + TQue _k_queue; + // Double-buffered VECIN queue for V prefetch (overlaps with Score compute) + TQue _v_queue; + + size_t _num_heads; + size_t _num_seqs; + size_t _num_kv_heads; + size_t _head_size; + size_t _head_stride; + float _scale; + size_t _max_num_blocks_per_seq; + size_t _page_block_size; + ptrdiff_t _q_stride; + ptrdiff_t _k_batch_stride; + ptrdiff_t _k_row_stride; + ptrdiff_t _k_head_stride; + ptrdiff_t _v_batch_stride; + ptrdiff_t _v_row_stride; + ptrdiff_t _v_head_stride; + ptrdiff_t _o_stride; + ptrdiff_t _block_table_batch_stride; + ptrdiff_t _cache_lens_stride; + bool _has_alibi; +}; + +#define DEFINE_PAGED_ATTENTION_KERNEL(KERNEL_NAME, DATA_TYPE, INDEX_TYPE) \ + extern "C" __global__ __aicore__ void KERNEL_NAME( \ + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, \ + GM_ADDR block_tables, GM_ADDR cache_lens, GM_ADDR alibi_slopes, \ + bool has_alibi, size_t num_heads, size_t num_seqs, size_t num_kv_heads, \ + size_t head_size, float scale, size_t max_num_blocks_per_seq, \ + size_t page_block_size, ptrdiff_t q_stride, ptrdiff_t k_batch_stride, \ + ptrdiff_t k_row_stride, ptrdiff_t k_head_stride, \ + ptrdiff_t v_batch_stride, ptrdiff_t v_row_stride, \ + ptrdiff_t v_head_stride, ptrdiff_t o_stride, \ + ptrdiff_t block_table_batch_stride, ptrdiff_t cache_lens_stride) { \ + PagedAttentionKernel op; \ + op.init(out, q, k_cache, v_cache, block_tables, cache_lens, alibi_slopes, \ + has_alibi, num_heads, num_seqs, num_kv_heads, head_size, scale, \ + max_num_blocks_per_seq, page_block_size, q_stride, k_batch_stride, \ + k_row_stride, k_head_stride, v_batch_stride, v_row_stride, \ + v_head_stride, o_stride, block_table_batch_stride, \ + cache_lens_stride); \ + op.process(); \ + } + +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_i64, half, int64_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_i32, half, int32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_f16_u32, half, uint32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_i64, bfloat16_t, + int64_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_i32, bfloat16_t, + int32_t) +DEFINE_PAGED_ATTENTION_KERNEL(infinilm_paged_attention_bf16_u32, bfloat16_t, + uint32_t) + +#undef DEFINE_PAGED_ATTENTION_KERNEL diff --git a/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp new file mode 100644 index 000000000..51e3a44da --- /dev/null +++ b/src/native/ascend/custom/paged_attention/op_kernel/paged_attention_fai.cpp @@ -0,0 +1,139 @@ +#include "attention/fused_infer_attention_score/op_kernel/attn_infra/base_defs.hpp" +#include "attention/fused_infer_attention_score/op_kernel/attn_infra/detail/alignment.hpp" +namespace NpuArch { +template +__aicore__ inline auto CeilDiv(T dividend, U divisor) { + return Detail::Alignment::CeilDiv(dividend, divisor); +} +template +__aicore__ inline auto RoundDown(T value, U align) { + return Detail::Alignment::RoundDown(value, align); +} +template +__aicore__ inline auto RoundUp(T value, U align) { + return Detail::Alignment::RoundUp(value, align); +} +struct splitNode { + uint32_t batchIdx[64]; + uint32_t headStartIdx[64]; + uint32_t headEndIdx[64]; + uint32_t qStartIdx[64]; + uint32_t qEndIdx[64]; + uint32_t lseTaskOffset[64]; + uint32_t oTaskOffset[64]; + uint32_t splitNum[64]; +}; +} // namespace NpuArch +#include "attention/fused_infer_attention_score/op_kernel/kernel_common.hpp" + +struct FAInferTilingData { + uint64_t mm1OutSize; + uint64_t smOnlineOutSize; + uint64_t mm2OutSize; + uint32_t batch; + uint32_t numHeads; + uint32_t kvHeads; + uint32_t embeddingSize; + uint32_t embeddingSizeV; + uint32_t blockSize; + uint32_t maxNumBlocksPerBatch; + uint32_t firstBatchTaskNum; + uint32_t totalTaskNum; + uint32_t mainLoopTaskNum; + uint32_t tailLoopTaskNum; + uint32_t tailStartBatch; + uint32_t tailStartN2; + uint32_t tailKvNBlockTile; + uint32_t maskType; + float scaleValue; + uint32_t sparseMode; + int64_t preToken; + int64_t nextToken; + uint64_t qStride; + uint64_t oStride; +}; + +#include "attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h" + +extern "C" __global__ __aicore__ void infinilm_paged_attention_fai_bf16_i32( + GM_ADDR out, GM_ADDR q, GM_ADDR k_cache, GM_ADDR v_cache, + GM_ADDR block_tables, GM_ADDR cache_lens, float scale, size_t num_heads, + size_t num_seqs, size_t num_kv_heads, size_t head_size, + size_t max_num_blocks_per_seq, size_t page_block_size, ptrdiff_t q_stride, + ptrdiff_t o_stride, GM_ADDR workspace, GM_ADDR tiling) { + using namespace NpuArch; + using ArchTag = Arch::AtlasA2; + using ElementQ = bfloat16_t; + using LayoutQ = layout::RowMajor; + using ElementK = bfloat16_t; + using LayoutK = layout::ColumnMajor; + using ElementV = bfloat16_t; + using LayoutV = layout::RowMajor; + using ElementS = float; + using LayoutS = layout::RowMajor; + using ElementSink = bfloat16_t; + using LayoutSink = layout::RowMajor; + using ElementP = bfloat16_t; + using LayoutP = layout::RowMajor; + using ElementO = bfloat16_t; + using LayoutO = layout::RowMajor; + using ElementLse = float; + using LayoutLse = layout::RowMajor; + using ElementMask = int8_t; + using LayoutMask = layout::RowMajor; + using ElementOTmp = float; + using LayoutOTmp = layout::RowMajor; + using ElementUpdate = float; + using LayoutUpdate = layout::RowMajor; + using L1TileShapeQK = GemmShape; + using L0TileShapeQK = GemmShape<128, 128, 128>; + using DispatchPolicyQK = Gemm::MmadAtlasA2FAIQKDecode; + using QType = Gemm::GemmType; + using KType = Gemm::GemmType; + using SType = Gemm::GemmType; + using SinkType = Gemm::GemmType; + using BlockMmadQK = + Gemm::Block::BlockMmad; + using DispatchPolicyOnlineSoftmax = Epilogue::EpilogueAtlasA2OnlineSoftmax< + Epilogue::LseMode::NONE, Epilogue::SinkMode::DISABLE, + Epilogue::MaskMode::NO_MASK, float>; + using maskType = Gemm::GemmType; + using pseShiftType = Gemm::GemmType; + using PType = Gemm::GemmType; + using EpilogueOnlineSoftmax = + Epilogue::Block::BlockEpilogue; + using L1TileShapePV = GemmShape<128, 128, 256>; + using L0TileShapePV = GemmShape<128, 128, 128>; + using DispatchPolicyPV = Gemm::MmadAtlasA2FAIPVDecode; + using VType = Gemm::GemmType; + using OTmpType = Gemm::GemmType; + using BlockMmadPV = + Gemm::Block::BlockMmad; + using DispatchPolicyRescaleO = + Epilogue::EpilogueAtlasA2RescaleO; + using OType = Gemm::GemmType; + using OUpdateType = Gemm::GemmType; + using LseType = Gemm::GemmType; + using EpilogueRescaleO = + Epilogue::Block::BlockEpilogue; + using DispatchPolicyInitOutWhenZero = + Epilogue::EpilogueAtlasA2InitOutWhenZero; + using EpilogueInitOut = + Epilogue::Block::BlockEpilogue; + using SplitFuse::FAInferKernelDecoding; + using FAI = + FAInferKernelDecoding; + FAIKernelParams params{ + q, k_cache, v_cache, nullptr, nullptr, block_tables, nullptr, + cache_lens, out, nullptr, workspace, tiling, nullptr}; + FAI kernel; + kernel(params); +} diff --git a/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp b/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp index 7d1069856..6aa93d35a 100644 --- a/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp +++ b/src/native/ascend/custom/reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp @@ -43,52 +43,41 @@ class KernelReshapeAndCacheFlashStrided { } __aicore__ inline void Process() { -#if defined(__CCE_AICORE__) && __CCE_AICORE__ == 200 - AscendC::DataCacheCleanAndInvalid( - slot_mapping_); -#endif + const int64_t block_index = AscendC::GetBlockIdx(); + const int64_t block_count = AscendC::GetBlockNum(); + const int64_t work_count = num_tokens_ * num_heads_; auto local = copy_buffer_.Get(); AscendC::DataCopyExtParams copy_params{ 1, static_cast(head_bytes_), 0, 0, 0}; AscendC::DataCopyPadExtParams pad_params{false, 0, 0, 0}; - for (int64_t token = 0; token < num_tokens_; ++token) { - int64_t slot = slot_mapping_.GetValue(token); - if (slot < 0 || slot >= num_blocks_ * block_size_) { - continue; - } - - int64_t block = slot / block_size_; - int64_t page = slot % block_size_; + for (int64_t work = block_index; work < work_count; work += block_count) { + const int64_t token = work / num_heads_; + const int64_t head = work % num_heads_; + const int64_t slot = slot_mapping_.GetValue(token); + if (slot < 0 || slot >= num_blocks_ * block_size_) continue; - for (int64_t head = 0; head < num_heads_; ++head) { - int64_t key_offset = - (token * key_token_stride_ + head * key_head_stride_) * - element_size_; - int64_t key_cache_offset = - (block * key_cache_block_stride_ + page * key_cache_page_stride_ + - head * key_cache_head_stride_) * - element_size_; - Copy(key_, key_offset, key_cache_, key_cache_offset, local, copy_params, - pad_params); + const int64_t block = slot / block_size_; + const int64_t page = slot % block_size_; + const int64_t key_offset = + (token * key_token_stride_ + head * key_head_stride_) * element_size_; + const int64_t key_cache_offset = + (block * key_cache_block_stride_ + page * key_cache_page_stride_ + + head * key_cache_head_stride_) * + element_size_; + Copy(key_, key_offset, key_cache_, key_cache_offset, local, copy_params, + pad_params); - int64_t value_offset = - (token * value_token_stride_ + head * value_head_stride_) * - element_size_; - int64_t value_cache_offset = (block * value_cache_block_stride_ + - page * value_cache_page_stride_ + - head * value_cache_head_stride_) * - element_size_; - Copy(value_, value_offset, value_cache_, value_cache_offset, local, - copy_params, pad_params); - } + const int64_t value_offset = + (token * value_token_stride_ + head * value_head_stride_) * + element_size_; + const int64_t value_cache_offset = + (block * value_cache_block_stride_ + page * value_cache_page_stride_ + + head * value_cache_head_stride_) * + element_size_; + Copy(value_, value_offset, value_cache_, value_cache_offset, local, + copy_params, pad_params); } -#if defined(__CCE_AICORE__) && __CCE_AICORE__ == 200 - AscendC::DataCacheCleanAndInvalid( - key_cache_); -#endif } private: diff --git a/src/native/ascend/ops/add/kernel.h b/src/native/ascend/ops/add/kernel.h index 84aa2f4b0..60f337835 100644 --- a/src/native/ascend/ops/add/kernel.h +++ b/src/native/ascend/ops/add/kernel.h @@ -53,6 +53,8 @@ class Operator : public Add { if (alpha_) aclDestroyScalar(alpha_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, const double /*alpha*/, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/argmax/kernel.h b/src/native/ascend/ops/argmax/kernel.h index 9db23e0fb..25dbee16b 100644 --- a/src/native/ascend/ops/argmax/kernel.h +++ b/src/native/ascend/ops/argmax/kernel.h @@ -56,6 +56,8 @@ class Operator : public Argmax { } } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const std::optional dim, const bool keepdim, Tensor out) const override { (void)dim; diff --git a/src/native/ascend/ops/copy/kernel.h b/src/native/ascend/ops/copy/kernel.h index 7660a637b..24aa09176 100644 --- a/src/native/ascend/ops/copy/kernel.h +++ b/src/native/ascend/ops/copy/kernel.h @@ -19,6 +19,8 @@ class Operator : public Copy { in_cache_(BroadcastView(src, out)), out_cache_(out) {} + bool capture_safe() const override { return true; } + void operator()(const Tensor src, const bool /*non_blocking*/, Tensor out) const override { if (output_size_ == 0) return; diff --git a/src/native/ascend/ops/embedding/kernel.h b/src/native/ascend/ops/embedding/kernel.h index 0b6e0ae09..0291b32b9 100644 --- a/src/native/ascend/ops/embedding/kernel.h +++ b/src/native/ascend/ops/embedding/kernel.h @@ -40,9 +40,9 @@ class Operator : public Embedding { /// \deprecated Use the overload that also accepts `max_norm` and /// `norm_type` instead. - [[deprecated("Use the PyTorch-compatible overload instead.")]] - Operator(const Tensor input, const Tensor weight, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, Tensor out) + [[deprecated("Use the PyTorch-compatible overload instead.")]] Operator( + const Tensor input, const Tensor weight, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, Tensor out) : Operator(input, weight, padding_idx, std::nullopt, 2.0, scale_grad_by_freq, sparse, out) {} @@ -54,6 +54,8 @@ class Operator : public Embedding { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, const std::optional /*padding_idx*/, const std::optional max_norm, const double norm_type, diff --git a/src/native/ascend/ops/fill/kernel.h b/src/native/ascend/ops/fill/kernel.h index c067ab4a7..5c4343553 100644 --- a/src/native/ascend/ops/fill/kernel.h +++ b/src/native/ascend/ops/fill/kernel.h @@ -43,6 +43,8 @@ class Operator : public Fill { } } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const double /*value*/, Tensor out) const override { (void)input; diff --git a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h index ef09039f1..daa14a352 100644 --- a/src/native/ascend/ops/flash_attn_varlen_func/kernel.h +++ b/src/native/ascend/ops/flash_attn_varlen_func/kernel.h @@ -60,6 +60,20 @@ class Operator q_lengths_i64_.resize(sequence_count_); k_lengths_i64_.resize(sequence_count_); + // Equal-length prompts can be proven from the caller's maxima and total + // token count. Avoid a per-layer D2H copy and stream synchronization in + // the common fixed-shape serving path. + host_lengths_ready_ = sequence_count_ > 0 && max_seqlen_q > 0 && + max_seqlen_q == max_seqlen_k && + q.size(0) == sequence_count_ * max_seqlen_q; + if (host_lengths_ready_) { + for (size_t sequence = 0; sequence < sequence_count_; ++sequence) { + q_lengths_i64_[sequence] = + static_cast((sequence + 1) * max_seqlen_q); + k_lengths_i64_[sequence] = static_cast(max_seqlen_q); + } + } + use_prompt_310p_ = ascend::IsAscend310P() && !block_table.has_value() && sequence_count_ == 1 && q.size(0) > 0 && q.dtype() == DataType::kFloat16 && @@ -127,6 +141,8 @@ class Operator const_cast(tensor.data())); } + bool capture_safe() const override { return true; } + void operator()(const Tensor q, const Tensor k, const Tensor v, const Tensor cu_seqlens_q, const Tensor cu_seqlens_k, const std::optional alibi_slopes, @@ -191,8 +207,7 @@ class Operator if (causal || window_size != std::vector({-1, -1})) { sparse_mode = window_size == std::vector({-1, -1}) ? 3 : 4; if (window_size[0] >= 0) pre_tokens = window_size[0]; - next_tokens = causal ? 0 : window_size[1]; - if (next_tokens < 0) next_tokens = max_token_count; + if (window_size[1] >= 0) next_tokens = window_size[1]; } auto scale = softmax_scale.value_or(1.0 / std::sqrt(q.size(2))); @@ -227,9 +242,9 @@ class Operator /*dequantScaleQuery=*/nullptr, /*learnableSink=*/nullptr, q.size(1), scale, pre_tokens, next_tokens, input_layout_.data(), num_key_value_heads, sparse_mode, - // Correct fully masked rows when right-down causal has Q longer than - // KV. - /*innerPrecise=*/2, + // Jingbo's validated Ascend TND paged prefill path uses the + // high-throughput innerPrecise=0 mode. + /*innerPrecise=*/0, /*blockSize=*/block_size, /*antiquantMode=*/0, /*softmaxLseFlag=*/false, @@ -547,6 +562,10 @@ class Operator aclIntArray* MakeSequenceLengths(const Tensor cu_seqlens, std::vector& lengths, aclrtStream stream, bool cumulative) const { + if (host_lengths_ready_) { + return aclCreateIntArray(lengths.data(), lengths.size()); + } + const auto bytes = cu_i32_host_.size() * sizeof(cu_i32_host_[0]); auto ret = aclrtMemcpyAsync(cu_i32_host_.data(), bytes, cu_seqlens.data(), bytes, ACL_MEMCPY_DEVICE_TO_HOST, stream); @@ -572,6 +591,7 @@ class Operator mutable std::vector cu_i32_host_; mutable std::vector q_lengths_i64_; mutable std::vector k_lengths_i64_; + bool host_lengths_ready_{false}; void* attention_mask_data_{nullptr}; bool use_prompt_310p_{false}; bool use_prompt_fallback_310p_{false}; diff --git a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h index dbda76793..08f06e74a 100644 --- a/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h +++ b/src/native/ascend/ops/flash_attn_with_kvcache/kernel.h @@ -17,7 +17,41 @@ #include "native/ascend/workspace_pool_.h" #include "operator.h" +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION +#include "aclrtlaunch_infinilm_paged_attention_bf16_i32.h" +#include "aclrtlaunch_infinilm_paged_attention_f16_i32.h" +#include "aclrtlaunch_infinilm_paged_attention_fai_bf16_i32.h" +#endif + namespace infini::ops { +namespace { +struct InfinilmFaiTiling { + uint64_t mm1OutSize = 0; + uint64_t smOnlineOutSize = 0; + uint64_t mm2OutSize = 0; + uint32_t batch = 0; + uint32_t numHeads = 0; + uint32_t kvHeads = 0; + uint32_t embeddingSize = 0; + uint32_t embeddingSizeV = 0; + uint32_t blockSize = 0; + uint32_t maxNumBlocksPerBatch = 0; + uint32_t firstBatchTaskNum = 0; + uint32_t totalTaskNum = 0; + uint32_t mainLoopTaskNum = 0; + uint32_t tailLoopTaskNum = 0; + uint32_t tailStartBatch = 0; + uint32_t tailStartN2 = 0; + uint32_t tailKvNBlockTile = 0; + uint32_t maskType = 0; + float scaleValue = 0.0F; + uint32_t sparseMode = 0; + int64_t preToken = 0; + int64_t nextToken = 0; + uint64_t qStride = 0; + uint64_t oStride = 0; +}; +} // namespace template <> class Operator @@ -80,6 +114,15 @@ class Operator using FlashAttnWithKvcache::operator(); + bool capture_safe() const override { return true; } + + ~Operator() override { + if (fai_tiling_ != nullptr && ascend::IsAclRuntimeAlive()) { + (void)aclrtFree(fai_tiling_); + fai_tiling_ = nullptr; + } + } + void operator()(const Tensor q, Tensor k_cache, Tensor v_cache, const std::optional k, const std::optional v, const std::optional rotary_cos, @@ -136,6 +179,15 @@ class Operator tensor.device(), strides}; } + static Tensor SqueezeSequenceView(const Tensor tensor) { + auto shape = Tensor::Shape{tensor.shape()}; + auto strides = Tensor::Strides{tensor.strides()}; + Tensor::Shape squeezed_shape{shape[0], shape[2], shape[3]}; + Tensor::Strides squeezed_strides{strides[0], strides[2], strides[3]}; + return Tensor{const_cast(tensor.data()), squeezed_shape, + tensor.dtype(), tensor.device(), squeezed_strides}; + } + void Initialize(const Tensor q, const Tensor k_cache, const Tensor v_cache, const std::optional& k, const std::optional& v, @@ -171,6 +223,14 @@ class Operator if (paged_) block_table_cache_ = ascend::AclTensorCache(*block_table); seq_i32_host_.resize(batch_size_); seq_i64_host_.resize(batch_size_); + if (paged_ && q.dtype() == DataType::kBFloat16 && + q.size(2) == k_cache.size(2) && q.size(3) == k_cache.size(3) && + block_table->IsContiguous() && k_cache.IsContiguous() && + v_cache.IsContiguous()) { + ascend::CheckAcl(aclrtMalloc(&fai_tiling_, sizeof(InfinilmFaiTiling), + ACL_MEM_MALLOC_HUGE_FIRST), + "FAI paged attention tiling allocation"); + } } void Run(const Tensor q, const Tensor k_cache, const Tensor v_cache, @@ -181,6 +241,26 @@ class Operator auto stream = static_cast(stream_); assert(block_table.has_value() == paged_ && "Ascend `FlashAttnWithKvcache` cache mode changed between calls"); +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION + const Tensor custom_q = + paged_ && q.ndim() == 4 && q.size(1) == 1 ? SqueezeSequenceView(q) : q; + const Tensor custom_out = paged_ && out.ndim() == 4 && out.size(1) == 1 + ? SqueezeSequenceView(out) + : out; + if (fai_tiling_ != nullptr && + CanRunFai(custom_q, k_cache, v_cache, cache_seqlens, + scalar_cache_seqlens, block_table, custom_out)) { + RunFai(custom_q, k_cache, v_cache, *cache_seqlens, *block_table, + softmax_scale, custom_out, stream); + return; + } + if (CanRunPagedCustom(custom_q, k_cache, v_cache, cache_seqlens, + scalar_cache_seqlens, block_table, custom_out)) { + RunPagedCustom(custom_q, k_cache, v_cache, *cache_seqlens, *block_table, + softmax_scale, custom_out, stream); + return; + } +#endif auto actual_seq_lengths = MakeSeqLengths(cache_seqlens, scalar_cache_seqlens, stream); assert(!paged_ || actual_seq_lengths != nullptr && @@ -243,6 +323,148 @@ class Operator if (actual_seq_lengths) aclDestroyIntArray(actual_seq_lengths); } +#ifdef INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION + bool CanRunFai(const Tensor q, const Tensor k_cache, const Tensor v_cache, + const std::optional& cache_seqlens, + const std::optional& scalar_cache_seqlens, + const std::optional& block_table, + const Tensor out) const { + return !scalar_cache_seqlens.has_value() && cache_seqlens.has_value() && + block_table.has_value() && q.dtype() == DataType::kBFloat16 && + q.size(1) == k_cache.size(2) && + cache_seqlens->dtype() == DataType::kInt32 && + cache_seqlens->IsContiguous() && block_table->IsContiguous() && + q.stride(1) == q.size(2) && q.stride(2) == 1 && + out.stride(1) == out.size(2) && out.stride(2) == 1 && + k_cache.IsContiguous() && v_cache.IsContiguous() && + q.dtype() == k_cache.dtype() && q.dtype() == v_cache.dtype(); + } + + void RunFai(const Tensor q, const Tensor k_cache, const Tensor v_cache, + const Tensor cache_seqlens, const Tensor block_table, + const std::optional softmax_scale, Tensor out, + aclrtStream stream) const { + constexpr uint64_t kSegmentBytes = uint64_t{64} << 20; + constexpr uint32_t kAtlasA2BlockCount = 20; + InfinilmFaiTiling tiling{}; + tiling.mm1OutSize = kSegmentBytes; + tiling.smOnlineOutSize = kSegmentBytes; + tiling.mm2OutSize = kSegmentBytes; + tiling.batch = static_cast(q.size(0)); + tiling.numHeads = static_cast(q.size(1)); + tiling.kvHeads = static_cast(k_cache.size(2)); + tiling.embeddingSize = static_cast(q.size(2)); + tiling.embeddingSizeV = static_cast(out.size(2)); + tiling.blockSize = static_cast(k_cache.size(1)); + tiling.maxNumBlocksPerBatch = static_cast(block_table.size(1)); + tiling.firstBatchTaskNum = tiling.batch <= 4 ? tiling.kvHeads : 1; + tiling.totalTaskNum = tiling.batch * tiling.firstBatchTaskNum; + tiling.scaleValue = static_cast( + softmax_scale.value_or(1.0 / std::sqrt(double(q.size(2))))); + tiling.preToken = std::numeric_limits::max(); + tiling.nextToken = std::numeric_limits::max(); + tiling.qStride = static_cast(q.stride(0)); + tiling.oStride = static_cast(out.stride(0)); + ascend::CheckAcl(aclrtMemcpy(fai_tiling_, sizeof(tiling), &tiling, + sizeof(tiling), ACL_MEMCPY_HOST_TO_DEVICE), + "FAI paged attention tiling upload"); + auto& arena = ascend::GetWorkspacePool().Ensure(stream, kSegmentBytes * 4, + "fai_paged_decode"); + const auto ret = aclrtlaunch_infinilm_paged_attention_fai_bf16_i32( + kAtlasA2BlockCount, stream, out.data(), const_cast(q.data()), + const_cast(k_cache.data()), const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), tiling.scaleValue, + size_t(q.size(1)), size_t(q.size(0)), size_t(k_cache.size(2)), + size_t(q.size(2)), size_t(block_table.size(1)), size_t(k_cache.size(1)), + q.stride(0), out.stride(0), arena.buf, fai_tiling_); + ascend::CheckAcl(ret, "FAI paged attention execution"); + } + + bool CanRunPagedCustom(const Tensor q, const Tensor k_cache, + const Tensor v_cache, + const std::optional& cache_seqlens, + const std::optional& scalar_cache_seqlens, + const std::optional& block_table, + const Tensor out) const { + return !scalar_cache_seqlens.has_value() && cache_seqlens.has_value() && + block_table.has_value() && !has_alibi_slopes_ && + cache_seqlens->dtype() == DataType::kInt32 && + cache_seqlens->IsContiguous() && q.ndim() == 3 && out.ndim() == 3 && + q.stride(1) == q.size(2) && q.stride(2) == 1 && + out.stride(1) == out.size(2) && out.stride(2) == 1 && + k_cache.IsContiguous() && v_cache.IsContiguous() && + (q.dtype() == DataType::kFloat16 || + q.dtype() == DataType::kBFloat16) && + q.dtype() == k_cache.dtype() && q.dtype() == v_cache.dtype(); + } + + static void RunPagedCustom(const Tensor q, const Tensor k_cache, + const Tensor v_cache, const Tensor cache_seqlens, + const Tensor block_table, + const std::optional softmax_scale, + Tensor out, aclrtStream stream) { + const int64_t num_seqs = static_cast(q.size(0)); + const int64_t num_heads = static_cast(q.size(1)); + const int64_t num_kv_heads = static_cast(k_cache.size(2)); + const int64_t head_size = static_cast(q.size(2)); + const int64_t max_blocks = static_cast(block_table.size(1)); + const int64_t page_size = static_cast(k_cache.size(1)); + const double default_scale = + 1.0 / std::sqrt(static_cast(head_size)); + const auto scale = + static_cast(softmax_scale.value_or(default_scale)); + const int64_t block_num = num_seqs * num_heads; + const auto ret = + q.dtype() == DataType::kBFloat16 + ? aclrtlaunch_infinilm_paged_attention_bf16_i32( + static_cast(block_num), stream, out.data(), + const_cast(q.data()), + const_cast(k_cache.data()), + const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), nullptr, false, + static_cast(num_heads), static_cast(num_seqs), + static_cast(num_kv_heads), + static_cast(head_size), scale, + static_cast(max_blocks), + static_cast(page_size), + static_cast(q.stride(0)), + static_cast(k_cache.stride(0)), + static_cast(k_cache.stride(1)), + static_cast(k_cache.stride(2)), + static_cast(v_cache.stride(0)), + static_cast(v_cache.stride(1)), + static_cast(v_cache.stride(2)), + static_cast(out.stride(0)), + static_cast(block_table.stride(0)), + static_cast(cache_seqlens.stride(0))) + : aclrtlaunch_infinilm_paged_attention_f16_i32( + static_cast(block_num), stream, out.data(), + const_cast(q.data()), + const_cast(k_cache.data()), + const_cast(v_cache.data()), + const_cast(block_table.data()), + const_cast(cache_seqlens.data()), nullptr, false, + static_cast(num_heads), static_cast(num_seqs), + static_cast(num_kv_heads), + static_cast(head_size), scale, + static_cast(max_blocks), + static_cast(page_size), + static_cast(q.stride(0)), + static_cast(k_cache.stride(0)), + static_cast(k_cache.stride(1)), + static_cast(k_cache.stride(2)), + static_cast(v_cache.stride(0)), + static_cast(v_cache.stride(1)), + static_cast(v_cache.stride(2)), + static_cast(out.stride(0)), + static_cast(block_table.stride(0)), + static_cast(cache_seqlens.stride(0))); + ascend::CheckAcl(ret, "Ascend custom paged attention execution"); + } +#endif + aclIntArray* MakeSeqLengths( const std::optional& cache_seqlens, const std::optional& scalar_cache_seqlens, @@ -294,6 +516,7 @@ class Operator mutable ascend::AclTensorCache query_cache_; mutable ascend::AclTensorCache out_cache_; mutable ascend::AclTensorCache block_table_cache_; + void* fai_tiling_ = nullptr; mutable std::vector seq_i32_host_; mutable std::vector seq_i64_host_; mutable std::array bsnd_layout_{'B', 'S', 'N', 'D', '\0'}; diff --git a/src/native/ascend/ops/fused_add_rms_norm/kernel.h b/src/native/ascend/ops/fused_add_rms_norm/kernel.h index 01ef9d8ce..8909a9ca6 100644 --- a/src/native/ascend/ops/fused_add_rms_norm/kernel.h +++ b/src/native/ascend/ops/fused_add_rms_norm/kernel.h @@ -105,6 +105,8 @@ class Operator // `rstd_tensor_` remains owned by the repeatable ACLNN executor. } + bool capture_safe() const override { return true; } + void operator()(Tensor input, Tensor residual, const std::optional weight, float epsilon) const override { diff --git a/src/native/ascend/ops/gemm/kernel.h b/src/native/ascend/ops/gemm/kernel.h index 983042ca0..4cc19a593 100644 --- a/src/native/ascend/ops/gemm/kernel.h +++ b/src/native/ascend/ops/gemm/kernel.h @@ -55,6 +55,8 @@ class Operator : public Gemm { if (beta_scalar_) aclDestroyScalar(beta_scalar_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor a, const Tensor b, const std::optional c, std::optional alpha, std::optional beta, std::optional trans_a, std::optional trans_b, diff --git a/src/native/ascend/ops/linear/kernel.h b/src/native/ascend/ops/linear/kernel.h index faa995dd8..1aab679af 100644 --- a/src/native/ascend/ops/linear/kernel.h +++ b/src/native/ascend/ops/linear/kernel.h @@ -45,6 +45,8 @@ class Operator : public Linear { if (beta_scalar_) aclDestroyScalar(beta_scalar_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, std::optional bias, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/matmul/kernel.h b/src/native/ascend/ops/matmul/kernel.h index bcf2d40aa..bd52002be 100644 --- a/src/native/ascend/ops/matmul/kernel.h +++ b/src/native/ascend/ops/matmul/kernel.h @@ -29,6 +29,8 @@ class Operator : public Matmul { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/matmul_weight_nz/kernel.h b/src/native/ascend/ops/matmul_weight_nz/kernel.h new file mode 100644 index 000000000..dbc4a43fb --- /dev/null +++ b/src/native/ascend/ops/matmul_weight_nz/kernel.h @@ -0,0 +1,99 @@ +#ifndef INFINI_OPS_ASCEND_MATMUL_WEIGHT_NZ_KERNEL_H_ +#define INFINI_OPS_ASCEND_MATMUL_WEIGHT_NZ_KERNEL_H_ + +#include +#include + +#include "acl/acl.h" +#include "aclnn/aclnn_base.h" +#include "aclnnop/aclnn_matmul.h" +#include "base/matmul_weight_nz.h" +#include "native/ascend/common.h" +#include "native/ascend/error_.h" +#include "native/ascend/workspace_pool_.h" +#include "operator.h" + +namespace infini::ops { +namespace ascend_matmul_weight_nz_detail { + +inline aclTensor* MakeContiguousTensor(const Tensor& tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides(tensor.strides().begin(), + tensor.strides().end()); + std::vector storage{static_cast(tensor.numel())}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(tensor.dtype()), strides.data(), 0, + ACL_FORMAT_ND, storage.data(), + static_cast(storage.size()), + const_cast(tensor.data())); +} + +inline aclTensor* MakeNzWeight(const Tensor& weight) { + const int64_t rows = static_cast(weight.size(0)); + const int64_t columns = static_cast(weight.size(1)); + std::vector shape{rows, columns}; + std::vector strides{columns, 1}; + std::vector storage{columns / 16, rows / 16, 16, 16}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(weight.dtype()), strides.data(), 0, + ACL_FORMAT_FRACTAL_NZ, storage.data(), + static_cast(storage.size()), + const_cast(weight.data())); +} + +} // namespace ascend_matmul_weight_nz_detail + +// Short-term C++-only operator; migrate to Matmul after TensorView models NZ +// as a distinct distribution. +template <> +class Operator : public MatmulWeightNz { + public: + Operator(const Tensor input, const Tensor weight, Tensor out) + : MatmulWeightNz(input, weight, out) {} + + bool capture_safe() const override { return true; } + + void operator()(const Tensor input, const Tensor weight, + Tensor out) const override { + using ascend_matmul_weight_nz_detail::MakeContiguousTensor; + using ascend_matmul_weight_nz_detail::MakeNzWeight; + + if (input_tensor_ == nullptr) { + input_tensor_ = MakeContiguousTensor(input); + weight_tensor_ = MakeNzWeight(weight); + out_tensor_ = MakeContiguousTensor(out); + } else { + aclSetRawTensorAddr(input_tensor_, const_cast(input.data())); + aclSetRawTensorAddr(weight_tensor_, const_cast(weight.data())); + aclSetRawTensorAddr(out_tensor_, out.data()); + } + + if (executor_ == nullptr) { + ascend::CheckAcl(aclnnMatmulWeightNzGetWorkspaceSize( + input_tensor_, weight_tensor_, out_tensor_, 1, + &workspace_size_, &executor_), + "MatmulWeightNz workspace query"); + ascend::CheckAcl(aclSetAclOpExecutorRepeatable(executor_), + "MatmulWeightNz repeatable executor"); + } + + auto stream = static_cast(stream_); + auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size_); + ascend::CheckAcl( + aclnnMatmulWeightNz(arena.buf, workspace_size_, executor_, stream), + "MatmulWeightNz execution"); + } + + private: + // Repeatable executors retain ACL descriptors at shutdown; follow the + // descriptor-release convention used by AclTensorCache in other providers. + mutable aclTensor* input_tensor_ = nullptr; + mutable aclTensor* weight_tensor_ = nullptr; + mutable aclTensor* out_tensor_ = nullptr; + mutable aclOpExecutor* executor_ = nullptr; + mutable uint64_t workspace_size_ = 0; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/ascend/ops/mul/kernel.h b/src/native/ascend/ops/mul/kernel.h index 311b85de8..172815a40 100644 --- a/src/native/ascend/ops/mul/kernel.h +++ b/src/native/ascend/ops/mul/kernel.h @@ -29,6 +29,8 @@ class Operator : public Mul { out_cache_.release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor other, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h index eb05890c9..092ad030a 100644 --- a/src/native/ascend/ops/reshape_and_cache_flash/kernel.h +++ b/src/native/ascend/ops/reshape_and_cache_flash/kernel.h @@ -65,6 +65,8 @@ class Operator if (indices_device_) aclrtFree(indices_device_); } + bool capture_safe() const override { return true; } + void operator()(const Tensor key, const Tensor value, const Tensor slot_mapping, const Tensor /*k_scale*/, const Tensor /*v_scale*/, @@ -126,24 +128,46 @@ class Operator void RunStridedKernel(const Tensor key, const Tensor value, const Tensor slot_mapping, Tensor key_cache, Tensor value_cache, aclrtStream stream) const { - auto ret = aclrtlaunch_reshape_and_cache_flash_strided( - 1, stream, const_cast(key.data()), - const_cast(value.data()), const_cast(slot_mapping.data()), - key_cache.data(), value_cache.data(), static_cast(num_tokens_), - static_cast(num_heads_), static_cast(head_size_), - static_cast(block_size_), static_cast(num_blocks_), - static_cast(key_token_stride_), - static_cast(key_head_stride_), - static_cast(value_token_stride_), - static_cast(value_head_stride_), - static_cast(key_cache_block_stride_), - static_cast(key_cache_page_stride_), - static_cast(key_cache_head_stride_), - static_cast(value_cache_block_stride_), - static_cast(value_cache_page_stride_), - static_cast(value_cache_head_stride_), - static_cast(element_size_)); - ascend::CheckAcl(ret, "ReshapeAndCacheFlash strided execution"); + constexpr uint32_t kMaxCacheWriteBlocks = 64; + constexpr int64_t kMaxCacheWriteTokensPerLaunch = 4096; + auto* key_bytes = static_cast(const_cast(key.data())); + auto* value_bytes = static_cast(const_cast(value.data())); + auto* slot_bytes = + static_cast(const_cast(slot_mapping.data())); + + for (int64_t token_offset = 0; + token_offset < static_cast(num_tokens_); + token_offset += kMaxCacheWriteTokensPerLaunch) { + const auto remaining = static_cast(num_tokens_) - token_offset; + const auto token_count = remaining < kMaxCacheWriteTokensPerLaunch + ? remaining + : kMaxCacheWriteTokensPerLaunch; + const auto requested_work_items = + static_cast(token_count * num_heads_); + const auto work_items = requested_work_items < kMaxCacheWriteBlocks + ? requested_work_items + : kMaxCacheWriteBlocks; + auto ret = aclrtlaunch_reshape_and_cache_flash_strided( + work_items, stream, + key_bytes + token_offset * key_token_stride_ * element_size_, + value_bytes + token_offset * value_token_stride_ * element_size_, + slot_bytes + token_offset, key_cache.data(), value_cache.data(), + token_count, static_cast(num_heads_), + static_cast(head_size_), static_cast(block_size_), + static_cast(num_blocks_), + static_cast(key_token_stride_), + static_cast(key_head_stride_), + static_cast(value_token_stride_), + static_cast(value_head_stride_), + static_cast(key_cache_block_stride_), + static_cast(key_cache_page_stride_), + static_cast(key_cache_head_stride_), + static_cast(value_cache_block_stride_), + static_cast(value_cache_page_stride_), + static_cast(value_cache_head_stride_), + static_cast(element_size_)); + ascend::CheckAcl(ret, "ReshapeAndCacheFlash strided execution"); + } } #endif diff --git a/src/native/ascend/ops/rms_norm/kernel.h b/src/native/ascend/ops/rms_norm/kernel.h index 42ffeb803..eb093264e 100644 --- a/src/native/ascend/ops/rms_norm/kernel.h +++ b/src/native/ascend/ops/rms_norm/kernel.h @@ -123,6 +123,8 @@ class Operator : public RmsNorm { // `rstd_tensor_` leaks with the executor at shutdown (see `64c367c`). } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, const Tensor weight, float eps, Tensor out) const override { auto stream = static_cast(stream_); diff --git a/src/native/ascend/ops/rotary_embedding/kernel.h b/src/native/ascend/ops/rotary_embedding/kernel.h index 9c1cf6adf..eda894be9 100644 --- a/src/native/ascend/ops/rotary_embedding/kernel.h +++ b/src/native/ascend/ops/rotary_embedding/kernel.h @@ -128,6 +128,8 @@ class Operator ascend::CheckAcl(aclrtFree(key_out_data_), "RoPE key output release"); } + bool capture_safe() const override { return true; } + void operator()(const Tensor positions, Tensor query, std::optional key, const Tensor cos_sin_cache, int64_t head_size, bool is_neox, int64_t rope_dim_offset = 0, diff --git a/src/native/ascend/ops/silu_and_mul/kernel.h b/src/native/ascend/ops/silu_and_mul/kernel.h index f8715cb4b..b5cecfaa2 100644 --- a/src/native/ascend/ops/silu_and_mul/kernel.h +++ b/src/native/ascend/ops/silu_and_mul/kernel.h @@ -40,6 +40,8 @@ class Operator : public SiluAndMul { if (out_staging_cache_) out_staging_cache_->release(); } + bool capture_safe() const override { return true; } + void operator()(const Tensor input, Tensor out) const override { auto t_input = input_cache_.get(const_cast(input.data())); auto t_out = out_cache_.get(out.data()); diff --git a/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h new file mode 100644 index 000000000..36cdfe173 --- /dev/null +++ b/src/native/ascend/ops/trans_matmul_weight_nz/kernel.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_ASCEND_TRANS_MATMUL_WEIGHT_NZ_KERNEL_H_ +#define INFINI_OPS_ASCEND_TRANS_MATMUL_WEIGHT_NZ_KERNEL_H_ + +#include +#include + +#include "acl/acl.h" +#include "aclnn/aclnn_base.h" +#include "aclnnop/aclnn_trans_matmul_weight.h" +#include "base/trans_matmul_weight_nz.h" +#include "native/ascend/common.h" +#include "native/ascend/error_.h" +#include "native/ascend/workspace_pool_.h" +#include "operator.h" + +namespace infini::ops { +namespace ascend_trans_matmul_weight_nz_detail { + +inline aclTensor* MakeContiguousTensor(const Tensor& tensor) { + std::vector shape(tensor.shape().begin(), tensor.shape().end()); + std::vector strides(tensor.strides().begin(), + tensor.strides().end()); + std::vector storage{static_cast(tensor.numel())}; + return aclCreateTensor(shape.data(), static_cast(shape.size()), + ascend::ToAclDtype(tensor.dtype()), strides.data(), 0, + ACL_FORMAT_ND, storage.data(), + static_cast(storage.size()), + const_cast(tensor.data())); +} + +} // namespace ascend_trans_matmul_weight_nz_detail + +// Short-term C++-only operator; migrate to the general transpose-matmul path +// after TensorView models NZ as a distinct distribution. +template <> +class Operator + : public TransMatmulWeightNz { + public: + Operator(const Tensor input, Tensor output) + : TransMatmulWeightNz(input, output) {} + + void operator()(const Tensor input, Tensor output) const override { + auto stream = static_cast(stream_); + const auto bytes = input.numel() * kDataTypeToSize.at(input.dtype()); + ascend::CheckAcl(aclrtMemcpyAsync(output.data(), bytes, input.data(), bytes, + ACL_MEMCPY_DEVICE_TO_DEVICE, stream), + "TransMatmulWeightNz source copy"); + + auto weight = ascend::OwnAclTensor( + ascend_trans_matmul_weight_nz_detail::MakeContiguousTensor(output)); + uint64_t workspace_size = 0; + aclOpExecutor* executor = nullptr; + ascend::CheckAcl(aclnnTransMatmulWeightGetWorkspaceSize( + weight.get(), &workspace_size, &executor), + "TransMatmulWeightNz workspace query"); + + auto& arena = ascend::GetWorkspacePool().Ensure(stream, workspace_size); + ascend::CheckAcl( + aclnnTransMatmulWeight(arena.buf, workspace_size, executor, stream), + "TransMatmulWeightNz execution"); + } +}; + +} // namespace infini::ops + +#endif diff --git a/tests/test_flash_attn_with_kvcache.py b/tests/test_flash_attn_with_kvcache.py index 4bf20a328..60bc3806a 100644 --- a/tests/test_flash_attn_with_kvcache.py +++ b/tests/test_flash_attn_with_kvcache.py @@ -6,7 +6,6 @@ from tests.utils import get_stream - if not hasattr(infini.ops, "FlashAttnWithKvcache"): pytest.skip( "`FlashAttnWithKvcache` is not available on this platform", @@ -154,6 +153,61 @@ def test_flash_attn_with_kvcache_ascend_paged(device, implementation_index): torch.testing.assert_close(actual, expected, rtol=2e-3, atol=2e-3) +def test_flash_attn_with_kvcache_ascend_paged_bf16_mha(device, implementation_index): + if device != "npu": + pytest.skip("this test covers the Ascend provider") + + batch_size, page_size = 2, 256 + num_heads = num_kv_heads = 4 + head_size = 64 + q = torch.randn( + (batch_size, 1, num_heads, head_size), + dtype=torch.bfloat16, + device=device, + ) + k_cache = torch.randn( + (4, page_size, num_kv_heads, head_size), + dtype=torch.bfloat16, + device=device, + ) + v_cache = torch.randn_like(k_cache) + lengths = (130, 300) + cache_seqlens = torch.tensor(lengths, dtype=torch.int32, device=device) + block_table = torch.tensor(((0, 1), (2, 3)), dtype=torch.int32, device=device) + expected = _reference_decode( + q, k_cache, v_cache, lengths, block_table=block_table, scale=0.125 + ) + actual = torch.empty_like(q) + + infini.ops.flash_attn_with_kvcache( + q, + k_cache, + v_cache, + None, + None, + None, + None, + cache_seqlens, + None, + None, + block_table, + None, + 0.125, + True, + (-1, -1), + 0.0, + True, + 0, + False, + actual, + None, + stream=get_stream(q.device), + implementation_index=implementation_index, + ) + + torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2) + + def test_flash_attn_with_kvcache_ascend_non_default_stream( device, implementation_index ): diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py index 922bc9851..2afce1fb1 100644 --- a/tests/test_generate_wrappers.py +++ b/tests/test_generate_wrappers.py @@ -19,6 +19,14 @@ def _load_generator_module(): return module +def test_temporary_nz_matmul_operators_are_cpp_only(): + module = _load_generator_module() + + assert not module._generates_public_api("matmul_weight_nz") + assert not module._generates_public_api("trans_matmul_weight_nz") + assert module._generates_public_api("matmul") + + def test_generated_dispatch_keeps_scalar_and_optional_tensor_overloads_distinct( monkeypatch, tmp_path ): diff --git a/third_party/ops-transformer b/third_party/ops-transformer new file mode 160000 index 000000000..e1e99f483 --- /dev/null +++ b/third_party/ops-transformer @@ -0,0 +1 @@ +Subproject commit e1e99f4832cf10d9af870d29ee4135f2e749f898 diff --git a/third_party/ops-transformer-fm9g70.patch b/third_party/ops-transformer-fm9g70.patch new file mode 100644 index 000000000..92571ad9a --- /dev/null +++ b/third_party/ops-transformer-fm9g70.patch @@ -0,0 +1,51 @@ +--- a/attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h ++++ b/attention/fused_infer_attention_score/op_kernel/flash_attention_regular_decode.h +@@ -97,10 +97,8 @@ + int64_t nextToken = fATilingData->nextToken; + AscendC::GlobalTensor gQ; + gQ.SetGlobalBuffer((__gm__ ElementQ *)params.q); +- AscendC::ListTensorDesc keyListTensorDescInit((__gm__ void*)params.k); +- AscendC::ListTensorDesc valueListTensorDescInit((__gm__ void*)params.v); +- __gm__ uint8_t* currentKey = (__gm__ uint8_t*)keyListTensorDescInit.GetDataPtr<__gm__ uint8_t>(0); +- __gm__ uint8_t* currentValue = (__gm__ uint8_t*)valueListTensorDescInit.GetDataPtr<__gm__ uint8_t>(0); ++ __gm__ uint8_t* currentKey = (__gm__ uint8_t*)params.k; ++ __gm__ uint8_t* currentValue = (__gm__ uint8_t*)params.v; + AscendC::GlobalTensor gK; + gK.SetGlobalBuffer((__gm__ ElementK *)currentKey); + AscendC::GlobalTensor gV; +@@ -109,10 +107,10 @@ + gMask.SetGlobalBuffer((__gm__ ElementMask *)params.mask); + AscendC::GlobalTensor gBlockTable; + gBlockTable.SetGlobalBuffer((__gm__ int32_t *)(params.blockTables)); +- AscendC::GlobalTensor gActualQseqlen; +- gActualQseqlen.SetGlobalBuffer((__gm__ int64_t *)params.actualQseqlen); +- AscendC::GlobalTensor gActualKvseqlen; +- gActualKvseqlen.SetGlobalBuffer((__gm__ int64_t *)params.actualKvseqlen); ++ AscendC::GlobalTensor gActualQseqlen; ++ gActualQseqlen.SetGlobalBuffer((__gm__ int32_t *)params.actualQseqlen); ++ AscendC::GlobalTensor gActualKvseqlen; ++ gActualKvseqlen.SetGlobalBuffer((__gm__ int32_t *)params.actualKvseqlen); + AscendC::GlobalTensor gO; + gO.SetGlobalBuffer((__gm__ ElementO *)params.o); + AscendC::GlobalTensor gLse; +@@ -192,16 +190,16 @@ + + coreIdx = AscendC::GetBlockIdx() / AscendC::GetSubBlockNum(); + #endif +- uint64_t strideQ = static_cast(qHeads * embed); +- uint64_t strideO = static_cast(qHeads * embedV); ++ uint64_t strideQ = fATilingData->qStride; ++ uint64_t strideO = fATilingData->oStride; + uint64_t strideK = static_cast(kvHeads * embed); + uint64_t strideV = static_cast(kvHeads * embedV); + uint32_t embedRound = NpuArch::Detail::Alignment::RoundUp(embed, FaiKernel::BLOCK_SIZE); + uint32_t embedRoundV = NpuArch::Detail::Alignment::RoundUp(embedV, FaiKernel::BLOCK_SIZE); + uint32_t groupSize = qHeads / kvHeads; + +- uint32_t totalQTokens = static_cast(gActualQseqlen.GetValue(batch - 1)); +- uint32_t curKvNBlockTile = GetKvNBlockTile(groupSize, kvHeads); ++ uint32_t totalQTokens = batch; ++ uint32_t curKvNBlockTile = batch <= 4 ? 1 : GetKvNBlockTile(groupSize, kvHeads); + uint32_t curKvNBlockNum = firstBatchTaskNum; + uint32_t rowNumConst = groupSize; + uint32_t rowNumRoundConst = NpuArch::Detail::Alignment::RoundUp(rowNumConst, FaiKernel::BLOCK_SIZE);