From 6b90de1c99be4fb98df9143b797cafac04a6ee95 Mon Sep 17 00:00:00 2001 From: rubik Date: Wed, 18 Nov 2026 17:31:21 +0800 Subject: [PATCH] feat(kunlun): add XPU backend with 45 operators (22 implemented, 23 link stubs) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register Kunlun as device type 10 and implement the operator set over the vendor kernel libraries shipped in the Kunlun SDK / vLLM-XPU wheel: - src/native/kunlun: 22 implemented operators (Qwen2-7B default path and the paged-attention path fully verified). 23 additional stubs keep the InfiniLM link intact: InfiniLM's csrc references their factory symbols unconditionally (MoE, w4/int8 quantization, mamba, qwen3_next, vision), so a missing symbol breaks every model's import at dlopen. Stubs fail fast via supports() — link scaffolding, not delivery promises. Operators with no InfiniLM references (awq/gptq marlin repack, fused qk-norm, upsample) are not stubbed; they land with their implementations when a model needs them. - flash-attention family reuses vendor kernels instead of hand-written device code: reshape_and_cache (libapiinfer.so) for the KV scatter, paged_attention_xft (libxpu_flash_attention.so) for paged decode, flash_attention_context_vllm for cache-only paged prefill. The wrappers absorb the vendor contracts (head-major cache blocks, int32 slots narrowed from int64, alpha = softmax_scale * sqrt(head_dim), total-KV prefix sums for kvlod, non-dense fused-QKV views compacted via host staging) and bind the vendor context to the caller's InfiniRT stream (set_stream + xpu_wait before every D2H read) - GEMM derives op_a from the b operand's layout (fixes a transposition bug) - dispatcher: Kunlun exposes exactly one implementation slot; the device-generic ATen templates never instantiate for Kunlun - torch device facade: kKunlun maps onto the "cuda" torch device so the pybind bindings accept Kunlun tensors --- CMakeLists.txt | 47 +- scripts/generate_torch_ops.py | 3 + src/CMakeLists.txt | 100 + src/device.h | 2 +- src/native/cuda/ops/gemm/blas.h | 4 + src/native/kunlun/blas.h | 62 + src/native/kunlun/blas_utils.h | 39 + src/native/kunlun/caster_.h | 122 + src/native/kunlun/elementwise_common.h | 99 + src/native/kunlun/kernel_commons.h | 42 + src/native/kunlun/kernels.cu | 1552 +++++ src/native/kunlun/launch2_declarations.h | 53 + src/native/kunlun/ops/add/kernel.h | 98 + src/native/kunlun/ops/argmax/kernel.h | 59 + src/native/kunlun/ops/cast/kernel.h | 49 + src/native/kunlun/ops/causal_conv1d/kernel.h | 27 + .../ops/chunk_gated_delta_rule/kernel.h | 34 + src/native/kunlun/ops/convolution/kernel.h | 30 + src/native/kunlun/ops/copy/kernel.h | 72 + src/native/kunlun/ops/deepseek_moe/kernel.h | 31 + src/native/kunlun/ops/embedding/kernel.h | 87 + src/native/kunlun/ops/fill/kernel.h | 68 + .../ops/flash_attn_varlen_func/kernel.h | 317 + .../ops/flash_attn_with_kvcache/kernel.h | 420 ++ .../ops/fused_gated_delta_net_gating/kernel.h | 29 + src/native/kunlun/ops/gelu/kernel.h | 64 + src/native/kunlun/ops/gemm/kernel.h | 376 ++ src/native/kunlun/ops/i8_gemm/kernel.h | 24 + src/native/kunlun/ops/layer_norm/kernel.h | 27 + .../kunlun/ops/mamba_selective_scan/kernel.h | 30 + .../kunlun/ops/moe_align_block_size/kernel.h | 29 + .../kunlun/ops/moe_fused_dense/kernel.h | 28 + src/native/kunlun/ops/moe_fused_gate/kernel.h | 29 + src/native/kunlun/ops/mrope/kernel.h | 29 + src/native/kunlun/ops/mul/kernel.h | 97 + src/native/kunlun/ops/pad/kernel.h | 29 + .../kunlun/ops/per_channel_quant_i8/kernel.h | 24 + .../kunlun/ops/per_tensor_dequant_i8/kernel.h | 25 + .../kunlun/ops/per_tensor_quant_i8/kernel.h | 25 + src/native/kunlun/ops/quick_gelu/kernel.h | 25 + .../ops/recurrent_gated_delta_rule/kernel.h | 32 + src/native/kunlun/ops/relu/kernel.h | 199 + .../ops/reshape_and_cache_flash/kernel.h | 230 + src/native/kunlun/ops/rms_norm/kernel.h | 58 + .../kunlun/ops/separate_table_rope/kernel.h | 62 + src/native/kunlun/ops/sigmoid/kernel.h | 41 + src/native/kunlun/ops/silu/kernel.h | 53 + src/native/kunlun/ops/silu_and_mul/kernel.h | 93 + src/native/kunlun/ops/softmax/kernel.h | 67 + src/native/kunlun/ops/sub/kernel.h | 100 + src/native/kunlun/ops/topk_sigmoid/kernel.h | 31 + src/native/kunlun/ops/topk_softmax/kernel.h | 30 + src/native/kunlun/ops/tril/kernel.h | 31 + src/native/kunlun/ops/triu/kernel.h | 32 + src/native/kunlun/ops/w4_marlin_gemm/kernel.h | 27 + .../kunlun/ops/w4_marlin_parameters/kernel.h | 38 + .../kunlun/ops/w4_marlin_repack/kernel.h | 25 + src/native/kunlun/runtime_.h | 8 + src/native/kunlun/runtime_utils.h | 17 + src/native/kunlun/torch_backend_stubs.cc | 5402 +++++++++++++++++ src/native/kunlun/triangular_detail.h | 44 + src/operator.h | 12 + src/pybind11_utils.h | 1 + src/torch/device_.h | 6 + src/torch/ops/add/add.h | 4 +- tests/conftest.py | 1 + 66 files changed, 10945 insertions(+), 6 deletions(-) create mode 100644 src/native/kunlun/blas.h create mode 100644 src/native/kunlun/blas_utils.h create mode 100644 src/native/kunlun/caster_.h create mode 100644 src/native/kunlun/elementwise_common.h create mode 100644 src/native/kunlun/kernel_commons.h create mode 100644 src/native/kunlun/kernels.cu create mode 100644 src/native/kunlun/launch2_declarations.h create mode 100644 src/native/kunlun/ops/add/kernel.h create mode 100644 src/native/kunlun/ops/argmax/kernel.h create mode 100644 src/native/kunlun/ops/cast/kernel.h create mode 100644 src/native/kunlun/ops/causal_conv1d/kernel.h create mode 100644 src/native/kunlun/ops/chunk_gated_delta_rule/kernel.h create mode 100644 src/native/kunlun/ops/convolution/kernel.h create mode 100644 src/native/kunlun/ops/copy/kernel.h create mode 100644 src/native/kunlun/ops/deepseek_moe/kernel.h create mode 100644 src/native/kunlun/ops/embedding/kernel.h create mode 100644 src/native/kunlun/ops/fill/kernel.h create mode 100644 src/native/kunlun/ops/flash_attn_varlen_func/kernel.h create mode 100644 src/native/kunlun/ops/flash_attn_with_kvcache/kernel.h create mode 100644 src/native/kunlun/ops/fused_gated_delta_net_gating/kernel.h create mode 100644 src/native/kunlun/ops/gelu/kernel.h create mode 100644 src/native/kunlun/ops/gemm/kernel.h create mode 100644 src/native/kunlun/ops/i8_gemm/kernel.h create mode 100644 src/native/kunlun/ops/layer_norm/kernel.h create mode 100644 src/native/kunlun/ops/mamba_selective_scan/kernel.h create mode 100644 src/native/kunlun/ops/moe_align_block_size/kernel.h create mode 100644 src/native/kunlun/ops/moe_fused_dense/kernel.h create mode 100644 src/native/kunlun/ops/moe_fused_gate/kernel.h create mode 100644 src/native/kunlun/ops/mrope/kernel.h create mode 100644 src/native/kunlun/ops/mul/kernel.h create mode 100644 src/native/kunlun/ops/pad/kernel.h create mode 100644 src/native/kunlun/ops/per_channel_quant_i8/kernel.h create mode 100644 src/native/kunlun/ops/per_tensor_dequant_i8/kernel.h create mode 100644 src/native/kunlun/ops/per_tensor_quant_i8/kernel.h create mode 100644 src/native/kunlun/ops/quick_gelu/kernel.h create mode 100644 src/native/kunlun/ops/recurrent_gated_delta_rule/kernel.h create mode 100644 src/native/kunlun/ops/relu/kernel.h create mode 100644 src/native/kunlun/ops/reshape_and_cache_flash/kernel.h create mode 100644 src/native/kunlun/ops/rms_norm/kernel.h create mode 100644 src/native/kunlun/ops/separate_table_rope/kernel.h create mode 100644 src/native/kunlun/ops/sigmoid/kernel.h create mode 100644 src/native/kunlun/ops/silu/kernel.h create mode 100644 src/native/kunlun/ops/silu_and_mul/kernel.h create mode 100644 src/native/kunlun/ops/softmax/kernel.h create mode 100644 src/native/kunlun/ops/sub/kernel.h create mode 100644 src/native/kunlun/ops/topk_sigmoid/kernel.h create mode 100644 src/native/kunlun/ops/topk_softmax/kernel.h create mode 100644 src/native/kunlun/ops/tril/kernel.h create mode 100644 src/native/kunlun/ops/triu/kernel.h create mode 100644 src/native/kunlun/ops/w4_marlin_gemm/kernel.h create mode 100644 src/native/kunlun/ops/w4_marlin_parameters/kernel.h create mode 100644 src/native/kunlun/ops/w4_marlin_repack/kernel.h create mode 100644 src/native/kunlun/runtime_.h create mode 100644 src/native/kunlun/runtime_utils.h create mode 100644 src/native/kunlun/torch_backend_stubs.cc create mode 100644 src/native/kunlun/triangular_detail.h diff --git a/CMakeLists.txt b/CMakeLists.txt index bf0cb2f4e..e68a3bdfe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,6 +41,7 @@ option(WITH_MARS "Enable Mars backend" OFF) option(WITH_CAMBRICON "Enable Cambricon backend" OFF) option(WITH_MOORE "Enable Moore backend" OFF) option(WITH_ASCEND "Enable Ascend backend" OFF) +option(WITH_KUNLUN "Enable Kunlun XPU backend" OFF) option(WITH_TORCH "Enable PyTorch C++ backend" OFF) option(WITH_LINKED "Enable linked third-party operator backends" OFF) @@ -195,9 +196,15 @@ if(AUTO_DETECT_DEVICES) message(STATUS "Auto-detected Ascend environment.") endif() + file(GLOB KUNLUN_DEV_FILES "/dev/xpu[0-9]*") + if(KUNLUN_DEV_FILES AND NOT DEFINED ENV{WITH_KUNLUN_DISABLE_AUTODETECT}) + set(WITH_KUNLUN ON) + message(STATUS "Auto-detected Kunlun environment.") + endif() + if(WITH_NVIDIA) set(_non_nvidia_gpu_detected FALSE) - foreach(_gpu_backend WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_CAMBRICON WITH_MOORE WITH_ASCEND) + foreach(_gpu_backend WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_CAMBRICON WITH_MOORE WITH_ASCEND WITH_KUNLUN) if(${_gpu_backend}) set(_non_nvidia_gpu_detected TRUE) endif() @@ -411,7 +418,7 @@ endif() # Only one CUDA-like GPU backend can be enabled at a time. set(_gpu_backend_count 0) -foreach(_gpu_backend WITH_NVIDIA WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_MOORE WITH_ASCEND) +foreach(_gpu_backend WITH_NVIDIA WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_MOORE WITH_ASCEND WITH_KUNLUN) if(${_gpu_backend}) math(EXPR _gpu_backend_count "${_gpu_backend_count} + 1") endif() @@ -675,8 +682,42 @@ if(WITH_ASCEND) message(STATUS "Using Ascend from `${ASCEND_HOME}`.") endif() +if(WITH_KUNLUN) + add_compile_definitions(WITH_KUNLUN=1) + if(NOT DEFINED KUNLUN_HOME) + if(DEFINED ENV{KUNLUN_HOME} AND NOT "$ENV{KUNLUN_HOME}" STREQUAL "") + set(KUNLUN_HOME "$ENV{KUNLUN_HOME}" CACHE PATH "Kunlun XPU SDK root") + elseif(EXISTS "/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/torch_xmlir/xre/include/xpu/runtime.h") + # torch_xmlir package root (contains xre/ + xtdk/ + libxpu_blas.so). + set(KUNLUN_HOME "/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/torch_xmlir" CACHE PATH "Kunlun XPU SDK root") + else() + set(KUNLUN_HOME "/usr/local/xpu" CACHE PATH "Kunlun XPU SDK root") + endif() + endif() + # XRE runtime tree either directly under KUNLUN_HOME or under xre/. + if(NOT EXISTS "${KUNLUN_HOME}/include/xpu/runtime.h" AND EXISTS "${KUNLUN_HOME}/xre/include/xpu/runtime.h") + set(KUNLUN_XRE_ROOT "${KUNLUN_HOME}/xre") + else() + set(KUNLUN_XRE_ROOT "${KUNLUN_HOME}") + endif() + if(NOT EXISTS "${KUNLUN_XRE_ROOT}/include/xpu/runtime.h") + message(FATAL_ERROR "WITH_KUNLUN is ON but XRE runtime headers were not found under `${KUNLUN_XRE_ROOT}`. Set KUNLUN_HOME.") + endif() + # XTDK: device-code compiler (clang++ with --xpu-arch). Prefer the + # KUNLUN_HOME/xtdk tree, fall back to /../xtdk. + if(EXISTS "${KUNLUN_HOME}/xtdk/bin/clang++") + set(KUNLUN_XTDK_ROOT "${KUNLUN_HOME}/xtdk") + elseif(EXISTS "${KUNLUN_XRE_ROOT}/../xtdk/bin/clang++") + set(KUNLUN_XTDK_ROOT "${KUNLUN_XRE_ROOT}/../xtdk") + else() + message(FATAL_ERROR "WITH_KUNLUN is ON but the XTDK clang++ was not found. Set KUNLUN_HOME to the SDK root containing xtdk/.") + endif() + set(KUNLUN_XPU_ARCH "xpu3" CACHE STRING "Kunlun device architecture") + message(STATUS "Using Kunlun XRE from `${KUNLUN_XRE_ROOT}`, XTDK from `${KUNLUN_XTDK_ROOT}` (arch ${KUNLUN_XPU_ARCH}).") +endif() + # If all other platforms are not enabled, CPU is enabled by default. -if(NOT WITH_NVIDIA AND NOT WITH_ILUVATAR AND NOT WITH_HYGON AND NOT WITH_THEAD AND NOT WITH_METAX AND NOT WITH_MARS AND NOT WITH_MOORE AND NOT WITH_CAMBRICON AND NOT WITH_ASCEND) +if(NOT WITH_NVIDIA AND NOT WITH_ILUVATAR AND NOT WITH_HYGON AND NOT WITH_THEAD AND NOT WITH_METAX AND NOT WITH_MARS AND NOT WITH_MOORE AND NOT WITH_CAMBRICON AND NOT WITH_ASCEND AND NOT WITH_KUNLUN) set(WITH_CPU ON) endif() diff --git a/scripts/generate_torch_ops.py b/scripts/generate_torch_ops.py index 4fd92024b..a5e496ab5 100644 --- a/scripts/generate_torch_ops.py +++ b/scripts/generate_torch_ops.py @@ -1483,6 +1483,9 @@ class {pascal} : public Operator<{pascal}> {{ #define INFINI_OPS_TORCH_{name_uc}_H_ #include "base/{name}.h" +#include "torch/device_.h" + +#include namespace infini::ops {{ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e685b4f12..81da13dfb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,6 +109,106 @@ if(WITH_CPU) list(APPEND DEVICE_LIST "cpu") endif() +if(WITH_KUNLUN) + file(GLOB_RECURSE KUNLUN_SOURCES CONFIGURE_DEPENDS + "native/kunlun/*.cc" + "native/kunlun/*.cpp" + ) + target_compile_definitions(infiniops PUBLIC WITH_KUNLUN=1) + target_sources(infiniops PRIVATE ${KUNLUN_SOURCES}) + + # XRE runtime (host side) and xBLAS. + if(NOT DEFINED KUNLUN_XBLAS_ROOT) + if(EXISTS "${KUNLUN_HOME}/xhpc/xblas/include/cublas_v2.h") + set(KUNLUN_XBLAS_ROOT "${KUNLUN_HOME}/xhpc/xblas") + else() + set(KUNLUN_XBLAS_ROOT "") + endif() + endif() + find_library(KUNLUN_RUNTIME_LIB NAMES xpurt + HINTS "${KUNLUN_XRE_ROOT}/so" "${KUNLUN_XRE_ROOT}/lib" "${KUNLUN_XRE_ROOT}/lib64" + REQUIRED) + find_library(KUNLUN_BLAS_LIB NAMES xpu_blas + HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so" + "${KUNLUN_XBLAS_ROOT}/so" "${KUNLUN_XBLAS_ROOT}/cmake" + REQUIRED) + # Vendor attention/KV-cache kernel libraries. `libapiinfer.so` (paged + # KV-cache scatter) ships in the vLLM-XPU wheel (`vllm_xpu/so/`); the + # flash-attention kernels live in `libxpu_flash_attention.so` and the + # vendor runtime context in `libxpuapi.so`, both in the torch_xmlir + # package. All are optional: without them the kunlun backend builds with + # the static-attention path only. + # `xpu/refactor/context/context.h` (baidu::xpu::api::Context) ships under + # the XHPC subtree, one level above `xre/include`. PUBLIC: the + # Python-extension `ops` target (generated dispatch/bindings) also + # compiles these wrapper headers and inherits includes only through the + # `infiniops` link. + target_include_directories(infiniops PUBLIC "${KUNLUN_HOME}/include" + "${KUNLUN_HOME}/xhpc/xdnn/include") + # vLLM-XPU wheel root (sibling of torch_xmlir) for `libapiinfer.so`. + get_filename_component(KUNLUN_SITE_PKGS "${KUNLUN_HOME}" DIRECTORY) + find_library(KUNLUN_APIINFER_LIB NAMES apiinfer + HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/so" "${KUNLUN_HOME}/lib" + "${KUNLUN_HOME}/xre/so" "${KUNLUN_SITE_PKGS}/vllm_xpu/so") + find_library(KUNLUN_FLASHATTN_LIB NAMES xpu_flash_attention + HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so") + find_library(KUNLUN_XPUAPI_LIB NAMES xpuapi + HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so") + target_link_libraries(infiniops PUBLIC + "${KUNLUN_RUNTIME_LIB}" + "${KUNLUN_BLAS_LIB}") + if(KUNLUN_APIINFER_LIB) + target_link_libraries(infiniops PUBLIC "${KUNLUN_APIINFER_LIB}") + endif() + if(KUNLUN_FLASHATTN_LIB) + target_link_libraries(infiniops PUBLIC "${KUNLUN_FLASHATTN_LIB}") + endif() + if(KUNLUN_XPUAPI_LIB) + target_link_libraries(infiniops PUBLIC "${KUNLUN_XPUAPI_LIB}") + endif() + + # Device code: `native/kunlun/kernels.cu` is compiled with the XTDK + # clang++ (`--xpu-arch=xpu3`, driver mode via the `.xpu` extension) into + # a static library linked into `infiniops`. The generated dispatch TUs + # for kunlun are plain C++ because the kunlun `Operator` specializations + # only launch the pre-compiled kernels through host launchers. + # The XTDK driver enables XPU mode based on the source-file extension, + # so stage the kernel source as `kernels.xpu` before compiling. + set(KUNLUN_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/native/kunlun/kernels.cu") + set(KUNLUN_KERNEL_XPU "${CMAKE_CURRENT_BINARY_DIR}/kunlun_kernels.xpu") + set(KUNLUN_KERNEL_OBJ "${CMAKE_CURRENT_BINARY_DIR}/kunlun_kernels.o") + add_custom_command( + OUTPUT "${KUNLUN_KERNEL_OBJ}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${KUNLUN_KERNEL_SRC}" "${KUNLUN_KERNEL_XPU}" + COMMAND "${KUNLUN_XTDK_ROOT}/bin/clang++" + --sysroot=/ + --target=${CMAKE_SYSTEM_PROCESSOR}-linux-gnu + -fPIC + --xpu-arch=${KUNLUN_XPU_ARCH} + # XTDK's own numeric_limits<__half> uses narrowing brace init that + # trips -Wc++11-narrowing as an error under -O2; silence it - the + # constants are correct _Float16 bit patterns. + -Wno-narrowing -Wno-c++11-narrowing + -std=c++17 -O2 -fno-builtin + -I${PROJECT_SOURCE_DIR} + -I${KUNLUN_XRE_ROOT}/include + -c ${KUNLUN_KERNEL_XPU} + -o ${KUNLUN_KERNEL_OBJ} + DEPENDS "${KUNLUN_KERNEL_SRC}" + COMMENT "Compiling Kunlun XPU kernels (${KUNLUN_XPU_ARCH})" + VERBATIM + ) + set_source_files_properties("${KUNLUN_KERNEL_OBJ}" PROPERTIES + EXTERNAL_OBJECT TRUE GENERATED TRUE) + target_sources(infiniops PRIVATE "${KUNLUN_KERNEL_OBJ}") + target_include_directories(infiniops PUBLIC + "${KUNLUN_XRE_ROOT}/include" + ${KUNLUN_XBLAS_ROOT}/include) + + list(APPEND DEVICE_LIST "kunlun") +endif() + if(WITH_NVIDIA) if(NOT TARGET nvidia::cutlass::cutlass) find_package(NvidiaCutlass 4.4.1 EXACT CONFIG QUIET) diff --git a/src/device.h b/src/device.h index 6773b77ad..4e05bd207 100644 --- a/src/device.h +++ b/src/device.h @@ -16,7 +16,7 @@ using AllDeviceTypes = List; + Device::Type::kThead, Device::Type::kKunlun>; template struct ActiveDevicesImpl { diff --git a/src/native/cuda/ops/gemm/blas.h b/src/native/cuda/ops/gemm/blas.h index 1f56f32b3..1a1c99618 100644 --- a/src/native/cuda/ops/gemm/blas.h +++ b/src/native/cuda/ops/gemm/blas.h @@ -109,6 +109,10 @@ class BlasGemm : public Gemm { } protected: + bool a_is_col_major() const { return a_is_col_major_; } + bool b_is_col_major() const { return b_is_col_major_; } + bool swaps_a_and_b() const { return swap_a_and_b_; } + virtual typename Backend::BlasHandle GetHandle() const { thread_local typename Backend::BlasHandle handle = []() { typename Backend::BlasHandle h{}; diff --git a/src/native/kunlun/blas.h b/src/native/kunlun/blas.h new file mode 100644 index 000000000..100fd2aa9 --- /dev/null +++ b/src/native/kunlun/blas.h @@ -0,0 +1,62 @@ +#ifndef INFINI_OPS_KUNLUN_BLAS_H_ +#define INFINI_OPS_KUNLUN_BLAS_H_ + +#include + +// clang-format off +// XBLAS exposes the cublas API via `#define cublas* xblas*` macros; including +// `cublas_api.h` brings `cublasHandle_t`, `cublasCreate`, and the GemmEx +// entry points, all resolved at link time to `libxpu_blas.so` symbols. +// `library_types.h` (XRE) provides the `cudaDataType_t` enum values. +#include "cublas_api.h" +#include "library_types.h" +// clang-format on + +#include "data_type.h" +#include "native/cuda/blas.h" +#include "native/kunlun/runtime_.h" + +namespace infini::ops { + +template <> +struct Blas : public Runtime { + using BlasHandle = cublasHandle_t; + + static constexpr auto BLAS_OP_N = CUBLAS_OP_N; + + static constexpr auto BLAS_OP_T = CUBLAS_OP_T; + + static constexpr auto R_16F = CUDA_R_16F; + + static constexpr auto R_16BF = CUDA_R_16BF; + + static constexpr auto R_32F = CUDA_R_32F; + + // XBLAS has no TF32; plain 32F compute. + static constexpr auto BLAS_COMPUTE_32F = CUBLAS_COMPUTE_32F; + + static constexpr auto BLAS_COMPUTE_32F_FAST_TF32 = CUBLAS_COMPUTE_32F; + + static constexpr auto BLAS_GEMM_DEFAULT = CUBLAS_GEMM_DEFAULT; + + static constexpr auto BlasCreate = cublasCreate_v2; + + static constexpr auto BlasSetStream = cublasSetStream_v2; + + static constexpr auto BlasDestroy = cublasDestroy_v2; + + static constexpr auto BlasGemmStridedBatchedEx = [](auto&&... args) { + return cublasGemmStridedBatchedEx(std::forward(args)...); + }; + + // xBLAS's strided-batched GEMM path rejects bf16 inputs entirely (status + // 15, NOT_SUPPORTED) while single cublasGemmEx accepts AB=bf16 with C=f32. + // The bf16 wrapper loops over the batch with GemmEx instead. + static constexpr auto BlasGemmEx = [](auto&&... args) { + return cublasGemmEx(std::forward(args)...); + }; +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/kunlun/blas_utils.h b/src/native/kunlun/blas_utils.h new file mode 100644 index 000000000..08ad021e8 --- /dev/null +++ b/src/native/kunlun/blas_utils.h @@ -0,0 +1,39 @@ +#ifndef INFINI_OPS_KUNLUN_BLAS_UTILS_H_ +#define INFINI_OPS_KUNLUN_BLAS_UTILS_H_ + +#include "data_type.h" +#include "native/cuda/blas_utils.h" +#include "native/kunlun/blas.h" + +namespace infini::ops { + +template <> +struct BlasUtils { + static auto GetDataType(DataType dtype) { + switch (dtype) { + case DataType::kFloat16: + return Blas::R_16F; + case DataType::kBFloat16: + return Blas::R_16BF; + case DataType::kFloat32: + return Blas::R_32F; + default: + throw std::runtime_error("Unsupported data type for Kunlun BLAS"); + } + } + + static auto GetComputeType(DataType dtype) { + switch (dtype) { + case DataType::kFloat16: + case DataType::kBFloat16: + case DataType::kFloat32: + return Blas::BLAS_COMPUTE_32F; + default: + throw std::runtime_error("Unsupported compute type for Kunlun BLAS"); + } + } +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/kunlun/caster_.h b/src/native/kunlun/caster_.h new file mode 100644 index 000000000..054d7c088 --- /dev/null +++ b/src/native/kunlun/caster_.h @@ -0,0 +1,122 @@ +#ifndef INFINI_OPS_KUNLUN_CASTER__H_ +#define INFINI_OPS_KUNLUN_CASTER__H_ + +#include + +#include + +#include "caster.h" +#include "data_type.h" + +namespace infini::ops { + +// Kunlun XPU3 has native fp16 (`half` from the XTDK) but no bf16 type: bf16 +// payloads are `uint16_t` in memory and are computed in float with explicit +// bit manipulation on device. +namespace detail { + +template +struct IsKunlunFp16 : std::is_same {}; + +inline float Fp16BitsToFloat(std::uint16_t h) { + const std::uint32_t sign = (h & 0x8000u) << 16; + const std::uint32_t exp = (h & 0x7C00u) >> 10; + const std::uint32_t man = (h & 0x03FFu); + std::uint32_t bits; + if (exp == 0) { + if (man == 0) { + bits = sign; + } else { + // Subnormal: normalize. + std::uint32_t e = 127 - 15 + 1; + while ((man & 0x0400u) == 0) { + man <<= 1; + e--; + } + man &= 0x03FFu; + bits = sign | ((e) << 23) | (man << 13); + } + } else if (exp == 0x1F) { + bits = sign | 0x7F800000u | (man << 13); + } else { + bits = sign | ((exp + 127 - 15) << 23) | (man << 13); + } + float out; + std::memcpy(&out, &bits, sizeof(out)); + return out; +} + +inline std::uint16_t FloatToFp16Bits(float f) { + std::uint32_t bits; + std::memcpy(&bits, &f, sizeof(bits)); + const std::uint32_t sign = (bits >> 16) & 0x8000u; + const std::int32_t exp = + static_cast((bits >> 23) & 0xFFu) - 127 + 15; + const std::uint32_t man = bits & 0x007FFFFFu; + if ((bits & 0x7FFFFFFFu) == 0) { + return static_cast(sign); + } + if (exp >= 0x1F) { + // Overflow -> inf. + return static_cast(sign | 0x7C00u); + } + if (exp <= 0) { + if (exp < -10) { + // Underflow -> zero. + return static_cast(sign); + } + // Subnormal. + const std::uint32_t man_out = (man | 0x00800000u) >> (1 - exp); + return static_cast(sign | (man_out >> 13)); + } + return static_cast(sign | (exp << 10) | (man >> 13)); +} + +inline std::uint16_t Bf16BitsToFloatBits(std::uint16_t b) { + return static_cast(0); +} + +} // namespace detail + +template <> +struct Caster { + template + static Dst Cast(Src&& x) { + static_assert(!std::is_reference_v, + "`Cast` cannot return reference types"); + + using PureDst = std::remove_cv_t>; + using PureSrc = std::remove_cv_t>; + + if constexpr (std::is_same_v) { + return std::forward(x); + } else { + return FromFloatHelper(ToFloatHelper(std::forward(x))); + } + } + + private: + template + static constexpr float ToFloatHelper(T&& x) { + using PureSrc = std::remove_cv_t>; + if constexpr (std::is_same_v) { + return detail::Fp16BitsToFloat(x); + } else { + return static_cast(std::forward(x)); + } + } + + template + static constexpr Dst FromFloatHelper(float f) { + using PureDst = std::remove_cv_t>; + if constexpr (std::is_same_v) { + return detail::FloatToFp16Bits(f); + } else { + return static_cast(f); + } + } +}; + +} // namespace infini::ops + +#endif diff --git a/src/native/kunlun/elementwise_common.h b/src/native/kunlun/elementwise_common.h new file mode 100644 index 000000000..938b9834b --- /dev/null +++ b/src/native/kunlun/elementwise_common.h @@ -0,0 +1,99 @@ +#ifndef INFINI_OPS_KUNLUN_ELEMENTWISE_COMMON_H_ +#define INFINI_OPS_KUNLUN_ELEMENTWISE_COMMON_H_ + +// Host-side shared infrastructure for the Kunlun backend. +// +// The device kernels live in (compiled by the XTDK clang++); +// this header only declares their host launch entry points. + +#include +#include +#include + +#include "native/kunlun/runtime_.h" +#include "native/kunlun/runtime_utils.h" + +namespace infini::ops { + +namespace detail { + +// dtype tags shared with kernels.cu. +inline int KunlunDtypeTag(DataType dtype) { + switch (dtype) { + case DataType::kFloat32: + return 0; + case DataType::kFloat16: + return 1; + case DataType::kBFloat16: + return 2; + case DataType::kInt16: + return 5; + case DataType::kInt32: + return 3; + case DataType::kInt64: + return 4; + default: + throw std::runtime_error("kunlun: unsupported dtype " + + std::to_string(static_cast(dtype))); + } +} + +} // namespace detail + +namespace kunlun_launch { + +// dtype tag: 0=f32, 1=f16, 2=bf16. +void LaunchRelu(int tag, void* out, const void* in, + const unsigned long long* out_shape, + const unsigned long long* in_shape, + const long long* out_strides, const long long* in_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool in_c, void* stream); +void LaunchGelu(int tag, void* out, const void* in, + const unsigned long long* out_shape, + const unsigned long long* in_shape, + const long long* out_strides, const long long* in_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool in_c, int approximate_tanh, void* stream); +void LaunchSilu(int tag, void* out, const void* in, + const unsigned long long* out_shape, + const unsigned long long* in_shape, + const long long* out_strides, const long long* in_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool in_c, void* stream); +void LaunchAdd(int tag, void* out, const void* a, const void* b, + const unsigned long long* out_shape, + const unsigned long long* a_shape, + const unsigned long long* b_shape, const long long* out_strides, + const long long* a_strides, const long long* b_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool a_c, bool b_c, double alpha, void* stream); +void LaunchSub(int tag, void* out, const void* a, const void* b, + const unsigned long long* out_shape, + const unsigned long long* a_shape, + const unsigned long long* b_shape, const long long* out_strides, + const long long* a_strides, const long long* b_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool a_c, bool b_c, void* stream); +void LaunchMul(int tag, void* out, const void* a, const void* b, + const unsigned long long* out_shape, + const unsigned long long* a_shape, + const unsigned long long* b_shape, const long long* out_strides, + const long long* a_strides, const long long* b_strides, + unsigned long long output_size, unsigned long long ndim, + bool out_c, bool a_c, bool b_c, void* stream); +void LaunchSiluAndMul(int tag, void* out, const void* in, + unsigned long long rows, unsigned long long cols, + long long in_row_stride, long long out_row_stride, + bool in_c, bool out_c, void* stream); +void LaunchSiluAndMulNd(int tag, void* out, const void* in, + const unsigned long long* out_shape, + const long long* out_strides, + const long long* in_strides, unsigned long long ndim, + unsigned long long cols, void* stream); + +} // namespace kunlun_launch + +} // namespace infini::ops + +#endif diff --git a/src/native/kunlun/kernel_commons.h b/src/native/kunlun/kernel_commons.h new file mode 100644 index 000000000..e93092b3e --- /dev/null +++ b/src/native/kunlun/kernel_commons.h @@ -0,0 +1,42 @@ +#ifndef INFINI_OPS_KUNLUN_KERNEL_COMMONS_H_ +#define INFINI_OPS_KUNLUN_KERNEL_COMMONS_H_ + +// Device-side common helpers for Kunlun XPU3 kernels. This header is only +// included by translation units compiled with the XTDK clang++ (`--xpu-arch +// =xpu3`). + +#include +#include +#include + +// clang-format off +#include +#include +#include +#include +#include +#include +#include +// clang-format on + +namespace infini::ops { +namespace kunlun { + +// XPU infers plain pointer kernel parameters into global address space 1; +// templating the pointer type lets the helper accept any address space. +template +inline __device__ std::size_t IndexToOffset(std::size_t flat_index, + std::size_t ndim, SizePtr shape, + StridePtr strides) { + std::size_t res = 0; + for (std::size_t i = ndim; i-- > 0;) { + res += (flat_index % shape[i]) * static_cast(strides[i]); + flat_index /= shape[i]; + } + return res; +} + +} // namespace kunlun +} // namespace infini::ops + +#endif diff --git a/src/native/kunlun/kernels.cu b/src/native/kunlun/kernels.cu new file mode 100644 index 000000000..9df6e96fd --- /dev/null +++ b/src/native/kunlun/kernels.cu @@ -0,0 +1,1552 @@ +// Kunlun XPU device kernels + their host launchers. +// +// The `*Kernel` templates below are compiled by the XTDK clang++ into XPU +// machine code; the `Launch*` functions at the bottom are host C++ and +// submit them through the `<<>>` +// launch syntax, which the XTDK compiles to `xpu_launch_config`. +// +// Design note: XPU infers plain pointer parameters as global-address-space-1 +// qualified pointers, and the XTDK `half`/`bfloat16_t` types cannot be +// copied between address spaces implicitly. All half/bf16 data therefore +// flows as `unsigned short` payloads with explicit bit conversion to float +// for compute (the same approach the legacy backend used for bf16). + +#include +#include +#include +#include + +// clang-format off +#include +#include +#include +#include +#include +#include +#include +#include +// clang-format on + +namespace infini::ops { +namespace { + +constexpr int kTagF32 = 0; +constexpr int kTagF16 = 1; +constexpr int kTagBF16 = 2; +constexpr int kTagI32 = 3; +constexpr int kTagI64 = 4; +constexpr int kTagI16 = 5; + +// Host-side bit converters (bit-exact round-to-nearest-even; used by the +// binary launchers for alpha and by LaunchFill). Mirrors the kl2 ones. +inline unsigned short FloatToF16BitsHost(float f) { + unsigned int bits; + std::memcpy(&bits, &f, 4); + const unsigned int sign = (bits >> 16) & 0x8000; + int exponent = (int)((bits >> 23) & 0xFF) - 127; + unsigned int mantissa = bits & 0x7FFFFF; + if (exponent >= 16) { + return (unsigned short)(sign | 0x7C00 | + ((exponent == 128 && mantissa) ? 0x200 : 0)); + } + if (exponent >= -14) { + unsigned int m = mantissa + 0x1000; + if (m & 0x800000) { + m >>= 1; + ++exponent; + if (exponent >= 16) return (unsigned short)(sign | 0x7C00); + } + return (unsigned short)(sign | ((exponent + 15) << 10) | (m >> 13)); + } + if (exponent >= -25) { + unsigned int m = (mantissa | 0x800000) >> (-14 - exponent); + m = (m + 0xFFF) >> 13; + return (unsigned short)(sign | (m < 0x400 ? m : 0x3FF)); + } + return (unsigned short)sign; +} + +inline unsigned short FloatToBF16BitsHost(float f) { + unsigned int bits; + std::memcpy(&bits, &f, 4); + return (unsigned short)((bits + 0x7FFF + ((bits >> 16) & 1)) >> 16); +} + +// --------------------------------------------------------------------------- +// dtype views: fp16/bf16 as raw 16-bit payloads. +// --------------------------------------------------------------------------- + +__device__ __forceinline__ float F16ToFloat(unsigned short bits) { + __half_raw hr; + hr.x = bits; + return __half2float(__half(hr)); +} + +__device__ __forceinline__ unsigned short FloatToF16Bits(float f) { + __half h = __float2half(f); + __half_raw hr = static_cast<__half_raw>(h); + return hr.x; +} + +__device__ __forceinline__ float BF16ToFloat(unsigned short bits) { + __xpu_bfloat16_raw br; + br.x = bits; + return static_cast(__xpu_bfloat16(br)); +} + +__device__ __forceinline__ unsigned short FloatToBF16Bits(float f) { + __xpu_bfloat16 h = __float2bfloat16(f); + __xpu_bfloat16_raw br = static_cast<__xpu_bfloat16_raw>(h); + return br.x; +} + +// --------------------------------------------------------------------------- +// Elementwise op functors over the three compute types: +// float (kTagF32) +// F16View (kTagF16) -- 16-bit payload wrapper +// BF16View (kTagBF16) +// --------------------------------------------------------------------------- + +template +struct DtTraits; + +template <> +struct DtTraits { + using Storage = float; + static __device__ __forceinline__ float ToFloat(Storage v) { return v; } + static __device__ __forceinline__ Storage FromFloat(float f) { return f; } +}; + +template <> +struct DtTraits { + using Storage = unsigned short; + static __device__ __forceinline__ float ToFloat(Storage v) { + return F16ToFloat(v); + } + static __device__ __forceinline__ Storage FromFloat(float f) { + return FloatToF16Bits(f); + } +}; + +template <> +struct DtTraits { + using Storage = unsigned short; + static __device__ __forceinline__ float ToFloat(Storage v) { + return BF16ToFloat(v); + } + static __device__ __forceinline__ Storage FromFloat(float f) { + return FloatToBF16Bits(f); + } +}; + +// Integer payloads pass through unchanged; ToFloat/FromFloat are unused for +// the pure pass-through ops (Add) but must exist for the templates. +template <> +struct DtTraits { + using Storage = short; + static __device__ __forceinline__ float ToFloat(Storage v) { + return static_cast(v); + } + static __device__ __forceinline__ Storage FromFloat(float f) { + return static_cast(f); + } +}; + +template <> +struct DtTraits { + using Storage = int; + static __device__ __forceinline__ float ToFloat(Storage v) { + return static_cast(v); + } + static __device__ __forceinline__ Storage FromFloat(float f) { + return static_cast(f); + } +}; + +template <> +struct DtTraits { + using Storage = long long; + static __device__ __forceinline__ float ToFloat(Storage v) { + return static_cast(v); + } + static __device__ __forceinline__ Storage FromFloat(float f) { + return static_cast(f); + } +}; + +// Per-element unary transforms expressed in float. +template