Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
[submodule ".ci"]
path = .ci
url = https://github.com/InfiniTensor/ci.git
[submodule "third_party/ops-transformer"]
path = third_party/ops-transformer
url = https://gitcode.com/cann/ops-transformer.git
update = none
19 changes: 19 additions & 0 deletions scripts/fetch-ops-transformer.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -euo pipefail

ROOT=$(cd "$(dirname "$0")/.." && pwd)
SUBMODULE_REL=third_party/ops-transformer
SUBMODULE="$ROOT/$SUBMODULE_REL"
GIT="git -c safe.directory=*"
EXPECTED=$($GIT -C "$ROOT" ls-tree HEAD "$SUBMODULE_REL" | awk '{print $3}')

if ! $GIT -C "$SUBMODULE" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
$GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL"
else
ACTUAL=$($GIT -C "$SUBMODULE" rev-parse HEAD)
if [ "$ACTUAL" != "$EXPECTED" ]; then
$GIT -C "$ROOT" submodule update --init --checkout --force "$SUBMODULE_REL"
fi
fi

printf 'ops-transformer: %s\n' "$($GIT -C "$SUBMODULE" rev-parse HEAD)"
44 changes: 27 additions & 17 deletions scripts/generate_wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,14 @@

_OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft")

# Temporary compatibility operators used directly by InfiniLM. Keep their
# generated C++ dispatch available, but do not expose legacy C or Python APIs.
_CPP_ONLY_OPS = frozenset(("matmul_weight_nz", "trans_matmul_weight_nz"))


def _generates_public_api(op_name):
return op_name not in _CPP_ONLY_OPS


def _op_namespace_parts(op_name):
parts = op_name.split("_")
Expand Down Expand Up @@ -2120,6 +2128,7 @@ def _generate_op_artifacts(item):
"op_name": op_name,
"header_name": header_name,
"bind_func_name": f"Bind{_op_symbol_name(op_name)}",
"public_api": _generates_public_api(op_name),
"pybind11": _generate_pybind11(operator),
"binding_source": _generate_binding_source(op_name),
"legacy_c_source": legacy_c_source,
Expand Down Expand Up @@ -2356,28 +2365,29 @@ def _dispatch_gen_batch_size():
source_path = _GENERATED_SRC_DIR / op_name
header_name = artifact["header_name"]
bind_func_name = artifact["bind_func_name"]
binding_header_path = _BINDINGS_DIR / header_name
if artifact["public_api"]:
binding_header_path = _BINDINGS_DIR / header_name

_write_text_if_changed(binding_header_path, artifact["pybind11"])
expected_binding_files.add(binding_header_path)
_write_text_if_changed(binding_header_path, artifact["pybind11"])
expected_binding_files.add(binding_header_path)

if use_monolithic_bindings:
op_includes.append(f'#include "{header_name}"')
else:
binding_source_path = _BINDINGS_DIR / f"{op_name}.cc"
_write_text_if_changed(binding_source_path, artifact["binding_source"])
expected_binding_files.add(binding_source_path)
if use_monolithic_bindings:
op_includes.append(f'#include "{header_name}"')
else:
binding_source_path = _BINDINGS_DIR / f"{op_name}.cc"
_write_text_if_changed(binding_source_path, artifact["binding_source"])
expected_binding_files.add(binding_source_path)

source_path.mkdir(exist_ok=True)
legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc"
_write_text_if_changed(legacy_source_path, artifact["legacy_c_source"])
expected_generated_src_files.add(legacy_source_path)
source_path.mkdir(exist_ok=True)
legacy_source_path = _GENERATED_SRC_DIR / op_name / "operator.cc"
_write_text_if_changed(legacy_source_path, artifact["legacy_c_source"])
expected_generated_src_files.add(legacy_source_path)

include_path = _INCLUDE_DIR / header_name
_write_text_if_changed(include_path, artifact["legacy_c_header"])
expected_include_files.add(include_path)
include_path = _INCLUDE_DIR / header_name
_write_text_if_changed(include_path, artifact["legacy_c_header"])
expected_include_files.add(include_path)

bind_func_names.append(bind_func_name)
bind_func_names.append(bind_func_name)

dispatch_header = _generate_generated_dispatch_header(
op_names, args.devices, dispatch_declarations
Expand Down
55 changes: 54 additions & 1 deletion src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,51 @@ if(WITH_ASCEND)

# Custom `AscendC` kernels (PyTorch extension, requires `torch_npu`).
if(BUILD_ASCEND_CUSTOM)
set(_ops_transformer_root
"${PROJECT_SOURCE_DIR}/third_party/ops-transformer")
option(INFINIOPS_FETCH_OPS_TRANSFORMER
"Fetch and pin the official Ascend ops-transformer submodule" ON)
if(INFINIOPS_FETCH_OPS_TRANSFORMER)
execute_process(
COMMAND bash "${PROJECT_SOURCE_DIR}/scripts/fetch-ops-transformer.sh"
RESULT_VARIABLE _ops_transformer_fetch_result)
if(NOT _ops_transformer_fetch_result EQUAL 0)
message(FATAL_ERROR "Failed to fetch the pinned ops-transformer submodule")
endif()
endif()

if(NOT EXISTS "${_ops_transformer_root}/CMakeLists.txt")
message(FATAL_ERROR
"ops-transformer is missing; run scripts/fetch-ops-transformer.sh")
endif()

# Keep the submodule pinned to an official commit and apply only the
# small FM9G70 decode ABI patch after checkout.
find_package(Git REQUIRED)
set(_ops_transformer_patch
"${PROJECT_SOURCE_DIR}/third_party/ops-transformer-fm9g70.patch")
execute_process(
COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply
--check -p1 "${_ops_transformer_patch}"
RESULT_VARIABLE _ops_transformer_patch_status)
if(_ops_transformer_patch_status EQUAL 0)
execute_process(
COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply
-p1 "${_ops_transformer_patch}"
RESULT_VARIABLE _ops_transformer_patch_result)
if(NOT _ops_transformer_patch_result EQUAL 0)
message(FATAL_ERROR "Failed to patch ops-transformer for FM9G70")
endif()
else()
execute_process(
COMMAND ${GIT_EXECUTABLE} -c safe.directory=* -C "${_ops_transformer_root}" apply
--check --reverse -p1 "${_ops_transformer_patch}"
RESULT_VARIABLE _ops_transformer_reverse_status)
if(NOT _ops_transformer_reverse_status EQUAL 0)
message(FATAL_ERROR
"ops-transformer has incompatible local changes")
endif()
endif()
# In-tree `ascendc_library()` trips the `CANN` `extract_host_stub.py`
# path-handling bug under `scikit-build-core`'s temp-dir builds
# (`KeyError` on `/./workspace/...` paths in `$<TARGET_OBJECTS>`).
Expand Down Expand Up @@ -579,6 +624,12 @@ if(WITH_ASCEND)
"${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/*.cmake"
"${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/CMakeLists.txt"
"${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom/build.sh")
set(_ops_transformer_fai_dir
"${_ops_transformer_root}/attention/fused_infer_attention_score/op_kernel")
file(GLOB_RECURSE _ops_transformer_fai_srcs CONFIGURE_DEPENDS
"${_ops_transformer_fai_dir}/*.h"
"${_ops_transformer_fai_dir}/*.hpp"
"${_ops_transformer_fai_dir}/*.cpp")

# Scrub env inherited from the outer `scikit-build-core` invocation
# before handing control to `build.sh`: `CMAKE_GENERATOR` /
Expand All @@ -604,9 +655,10 @@ if(WITH_ASCEND)
--unset=CMAKE_BUILD_PARALLEL_LEVEL
"BUILD_DIR=${_custom_build_dir}"
"MAIN_SRC_DIR=${CMAKE_CURRENT_SOURCE_DIR}"
"OPS_TRANSFORMER_ROOT=${_ops_transformer_root}"
bash ${_custom_source_dir}/build.sh ${SOC_VERSION}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/native/ascend/custom
DEPENDS ${_custom_srcs}
DEPENDS ${_custom_srcs} ${_ops_transformer_fai_srcs}
COMMENT "Building custom AscendC kernels (SOC_VERSION=${SOC_VERSION})"
VERBATIM)

Expand All @@ -621,6 +673,7 @@ if(WITH_ASCEND)
# custom kernel implementations (e.g. `RmsNorm` index 1) can call
# them via the generated launch functions.
target_compile_definitions(infiniops PUBLIC INFINI_HAS_CUSTOM_KERNELS=1)
target_compile_definitions(infiniops PUBLIC INFINILM_ENABLE_CUSTOM_PAGED_ATTENTION=1)
target_include_directories(infiniops PRIVATE
"${_custom_build_dir}/include/no_workspace_kernel")
target_link_libraries(infiniops PRIVATE
Expand Down
57 changes: 57 additions & 0 deletions src/base/matmul_weight_nz.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#ifndef INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_
#define INFINI_OPS_BASE_MATMUL_WEIGHT_NZ_H_

#include <cassert>

#include "operator.h"

namespace infini::ops {

// Temporary compatibility operator for InfiniLM's Ascend NZ weight path. It is
// intentionally C++-only. Once TensorView distinguishes ND and NZ
// distributions, migrate this behavior to the general Matmul operator and
// remove this class.
class MatmulWeightNz : public Operator<MatmulWeightNz> {
public:
MatmulWeightNz(const Tensor input, const Tensor weight, Tensor out)
: rows_{input.size(0)},
channels_{input.size(1)},
outputs_{weight.size(1)} {
assert(input.ndim() == 2 && weight.ndim() == 2 && out.ndim() == 2 &&
"operator `MatmulWeightNz` requires 2D tensors");
assert(input.size(1) == weight.size(0) &&
"operator `MatmulWeightNz` input channels must match weight rows");
assert(out.size(0) == input.size(0) &&
"operator `MatmulWeightNz` output rows must match input");
assert(out.size(1) == weight.size(1) &&
"operator `MatmulWeightNz` output columns must match weight");
assert(input.dtype() == weight.dtype() && input.dtype() == out.dtype() &&
"operator `MatmulWeightNz` requires matching dtypes");
assert((input.dtype() == DataType::kFloat16 ||
input.dtype() == DataType::kBFloat16) &&
"operator `MatmulWeightNz` supports fp16 or bf16 only");
assert(weight.stride(0) == static_cast<Tensor::Stride>(weight.size(1)) &&
weight.stride(1) == 1 &&
"operator `MatmulWeightNz` weight must be contiguous");
assert(input.stride(0) == static_cast<Tensor::Stride>(input.size(1)) &&
input.stride(1) == 1 &&
"operator `MatmulWeightNz` input must be contiguous");
assert(out.stride(0) == static_cast<Tensor::Stride>(out.size(1)) &&
out.stride(1) == 1 &&
"operator `MatmulWeightNz` output must be contiguous");
assert(weight.size(0) % 16 == 0 && weight.size(1) % 16 == 0 &&
"operator `MatmulWeightNz` requires 16-aligned weight shape");
}

virtual void operator()(const Tensor input, const Tensor weight,
Tensor out) const = 0;

protected:
Tensor::Size rows_;
Tensor::Size channels_;
Tensor::Size outputs_;
};

} // namespace infini::ops

#endif
45 changes: 45 additions & 0 deletions src/base/trans_matmul_weight_nz.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#ifndef INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_
#define INFINI_OPS_BASE_TRANS_MATMUL_WEIGHT_NZ_H_

#include <cassert>

#include "operator.h"

namespace infini::ops {

// Temporary compatibility operator for InfiniLM's Ascend NZ weight conversion.
// It is intentionally C++-only. Once TensorView distinguishes ND and NZ
// distributions, migrate this behavior to the general transpose-matmul path and
// remove this class.
class TransMatmulWeightNz : public Operator<TransMatmulWeightNz> {
public:
TransMatmulWeightNz(const Tensor input, Tensor output)
: rows_{input.size(0)}, columns_{input.size(1)} {
assert(input.ndim() == 2 && output.ndim() == 2 &&
"operator `TransMatmulWeightNz` requires 2D tensors");
assert(input.shape() == output.shape() &&
"operator `TransMatmulWeightNz` input and output shapes differ");
assert(input.dtype() == output.dtype() &&
(input.dtype() == DataType::kFloat16 ||
input.dtype() == DataType::kBFloat16) &&
"operator `TransMatmulWeightNz` supports fp16 or bf16 only");
assert(input.stride(0) == static_cast<Tensor::Stride>(input.size(1)) &&
input.stride(1) == 1 &&
"operator `TransMatmulWeightNz` input must be contiguous");
assert(output.stride(0) == static_cast<Tensor::Stride>(output.size(1)) &&
output.stride(1) == 1 &&
"operator `TransMatmulWeightNz` output must be contiguous");
assert(rows_ % 16 == 0 && columns_ % 16 == 0 &&
"operator `TransMatmulWeightNz` requires a 16-aligned shape");
}

virtual void operator()(const Tensor input, Tensor output) const = 0;

protected:
Tensor::Size rows_;
Tensor::Size columns_;
};

} // namespace infini::ops

#endif
19 changes: 16 additions & 3 deletions src/native/ascend/custom/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ endif()
set(PROJECT_OP_SRC_BASE ${PROJECT_SOURCE_DIR})
set(MAIN_SRC_DIR "${PROJECT_OP_SRC_BASE}/../.." CACHE PATH
"Main InfiniOps source directory.")
set(OPS_TRANSFORMER_ROOT "" CACHE PATH
"Pinned ops-transformer source root supplied by the outer build")

include(cmake/config_envs.cmake)
include(cmake/config_ascend.cmake)
Expand Down Expand Up @@ -63,6 +65,8 @@ ascendc_library(no_workspace_kernel STATIC
rms_norm/op_kernel/rms_norm.cpp
add_rms_norm/op_kernel/add_rms_norm.cpp
reshape_and_cache_flash/op_kernel/reshape_and_cache_flash.cpp
paged_attention/op_kernel/paged_attention.cpp
paged_attention/op_kernel/paged_attention_fai.cpp
)

# `CANN 8.5.1` installs host objects under `objects-<CONFIG>/`, while
Expand All @@ -80,9 +84,18 @@ endif()
# enum. `ascendc_library` forwards the interface target's `INCLUDES`
# property to the nested `ExternalProject_Add` (see
# `${ASCEND_HOME_PATH}/tools/tikcpp/ascendc_kernel_cmake/legacy_modules/function.cmake`),
# so append the main `src/` dir here.
set_property(TARGET no_workspace_kernel_interface APPEND PROPERTY
INCLUDES ${MAIN_SRC_DIR})
# so register the main `src/` dir and repository root here. Resolve the
# repository root through the symlinked source directory used by the outer
# build. Resolve the symlink before appending `..`; CMake otherwise
# normalizes those components lexically.
get_filename_component(CUSTOM_SOURCE_DIR "${PROJECT_SOURCE_DIR}" REALPATH)
get_filename_component(INFINIOPS_ROOT
"${CUSTOM_SOURCE_DIR}/../../../.." REALPATH)
ascendc_include_directories(no_workspace_kernel INTERFACE
${MAIN_SRC_DIR}
${INFINIOPS_ROOT}
${OPS_TRANSFORMER_ROOT}
)

# Create the shared library `libascend_kernel.so`.
add_library(${OP_PLUGIN_NAME} SHARED ${OP_SRCS})
Expand Down
1 change: 1 addition & 0 deletions src/native/ascend/custom/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ cmake \
-DSOC_VERSION="${SOC_VERSION}" \
-DCMAKE_LIBRARY_OUTPUT_DIRECTORY="${OUTPUT_DIR}" \
-DMAIN_SRC_DIR="${MAIN_SRC_DIR}" \
-DOPS_TRANSFORMER_ROOT="${OPS_TRANSFORMER_ROOT}" \
-B "${BUILD_DIR}" \
-S "${SCRIPT_DIR}"

Expand Down
Loading
Loading