Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 44 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ option(WITH_MARS "Enable Mars backend" OFF)
option(WITH_CAMBRICON "Enable Cambricon backend" OFF)
option(WITH_MOORE "Enable Moore backend" OFF)
option(WITH_ASCEND "Enable Ascend backend" OFF)
option(WITH_KUNLUN "Enable Kunlun XPU backend" OFF)

option(WITH_TORCH "Enable PyTorch C++ backend" OFF)
option(WITH_LINKED "Enable linked third-party operator backends" OFF)
Expand Down Expand Up @@ -195,9 +196,15 @@ if(AUTO_DETECT_DEVICES)
message(STATUS "Auto-detected Ascend environment.")
endif()

file(GLOB KUNLUN_DEV_FILES "/dev/xpu[0-9]*")
if(KUNLUN_DEV_FILES AND NOT DEFINED ENV{WITH_KUNLUN_DISABLE_AUTODETECT})
set(WITH_KUNLUN ON)
message(STATUS "Auto-detected Kunlun environment.")
endif()

if(WITH_NVIDIA)
set(_non_nvidia_gpu_detected FALSE)
foreach(_gpu_backend WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_CAMBRICON WITH_MOORE WITH_ASCEND)
foreach(_gpu_backend WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_CAMBRICON WITH_MOORE WITH_ASCEND WITH_KUNLUN)
if(${_gpu_backend})
set(_non_nvidia_gpu_detected TRUE)
endif()
Expand Down Expand Up @@ -411,7 +418,7 @@ endif()

# Only one CUDA-like GPU backend can be enabled at a time.
set(_gpu_backend_count 0)
foreach(_gpu_backend WITH_NVIDIA WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_MOORE WITH_ASCEND)
foreach(_gpu_backend WITH_NVIDIA WITH_ILUVATAR WITH_HYGON WITH_THEAD WITH_METAX WITH_MARS WITH_MOORE WITH_ASCEND WITH_KUNLUN)
if(${_gpu_backend})
math(EXPR _gpu_backend_count "${_gpu_backend_count} + 1")
endif()
Expand Down Expand Up @@ -675,8 +682,42 @@ if(WITH_ASCEND)
message(STATUS "Using Ascend from `${ASCEND_HOME}`.")
endif()

if(WITH_KUNLUN)
add_compile_definitions(WITH_KUNLUN=1)
if(NOT DEFINED KUNLUN_HOME)
if(DEFINED ENV{KUNLUN_HOME} AND NOT "$ENV{KUNLUN_HOME}" STREQUAL "")
set(KUNLUN_HOME "$ENV{KUNLUN_HOME}" CACHE PATH "Kunlun XPU SDK root")
elseif(EXISTS "/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/torch_xmlir/xre/include/xpu/runtime.h")
# torch_xmlir package root (contains xre/ + xtdk/ + libxpu_blas.so).
set(KUNLUN_HOME "/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/torch_xmlir" CACHE PATH "Kunlun XPU SDK root")
else()
set(KUNLUN_HOME "/usr/local/xpu" CACHE PATH "Kunlun XPU SDK root")
endif()
endif()
# XRE runtime tree either directly under KUNLUN_HOME or under xre/.
if(NOT EXISTS "${KUNLUN_HOME}/include/xpu/runtime.h" AND EXISTS "${KUNLUN_HOME}/xre/include/xpu/runtime.h")
set(KUNLUN_XRE_ROOT "${KUNLUN_HOME}/xre")
else()
set(KUNLUN_XRE_ROOT "${KUNLUN_HOME}")
endif()
if(NOT EXISTS "${KUNLUN_XRE_ROOT}/include/xpu/runtime.h")
message(FATAL_ERROR "WITH_KUNLUN is ON but XRE runtime headers were not found under `${KUNLUN_XRE_ROOT}`. Set KUNLUN_HOME.")
endif()
# XTDK: device-code compiler (clang++ with --xpu-arch). Prefer the
# KUNLUN_HOME/xtdk tree, fall back to <xre>/../xtdk.
if(EXISTS "${KUNLUN_HOME}/xtdk/bin/clang++")
set(KUNLUN_XTDK_ROOT "${KUNLUN_HOME}/xtdk")
elseif(EXISTS "${KUNLUN_XRE_ROOT}/../xtdk/bin/clang++")
set(KUNLUN_XTDK_ROOT "${KUNLUN_XRE_ROOT}/../xtdk")
else()
message(FATAL_ERROR "WITH_KUNLUN is ON but the XTDK clang++ was not found. Set KUNLUN_HOME to the SDK root containing xtdk/.")
endif()
set(KUNLUN_XPU_ARCH "xpu3" CACHE STRING "Kunlun device architecture")
message(STATUS "Using Kunlun XRE from `${KUNLUN_XRE_ROOT}`, XTDK from `${KUNLUN_XTDK_ROOT}` (arch ${KUNLUN_XPU_ARCH}).")
endif()

# If all other platforms are not enabled, CPU is enabled by default.
if(NOT WITH_NVIDIA AND NOT WITH_ILUVATAR AND NOT WITH_HYGON AND NOT WITH_THEAD AND NOT WITH_METAX AND NOT WITH_MARS AND NOT WITH_MOORE AND NOT WITH_CAMBRICON AND NOT WITH_ASCEND)
if(NOT WITH_NVIDIA AND NOT WITH_ILUVATAR AND NOT WITH_HYGON AND NOT WITH_THEAD AND NOT WITH_METAX AND NOT WITH_MARS AND NOT WITH_MOORE AND NOT WITH_CAMBRICON AND NOT WITH_ASCEND AND NOT WITH_KUNLUN)
set(WITH_CPU ON)
endif()

Expand Down
3 changes: 3 additions & 0 deletions scripts/generate_torch_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -1483,6 +1483,9 @@ class {pascal} : public Operator<{pascal}> {{
#define INFINI_OPS_TORCH_{name_uc}_H_

#include "base/{name}.h"
#include "torch/device_.h"

#include <type_traits>

namespace infini::ops {{

Expand Down
100 changes: 100 additions & 0 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,106 @@ if(WITH_CPU)
list(APPEND DEVICE_LIST "cpu")
endif()

if(WITH_KUNLUN)
file(GLOB_RECURSE KUNLUN_SOURCES CONFIGURE_DEPENDS
"native/kunlun/*.cc"
"native/kunlun/*.cpp"
)
target_compile_definitions(infiniops PUBLIC WITH_KUNLUN=1)
target_sources(infiniops PRIVATE ${KUNLUN_SOURCES})

# XRE runtime (host side) and xBLAS.
if(NOT DEFINED KUNLUN_XBLAS_ROOT)
if(EXISTS "${KUNLUN_HOME}/xhpc/xblas/include/cublas_v2.h")
set(KUNLUN_XBLAS_ROOT "${KUNLUN_HOME}/xhpc/xblas")
else()
set(KUNLUN_XBLAS_ROOT "")
endif()
endif()
find_library(KUNLUN_RUNTIME_LIB NAMES xpurt
HINTS "${KUNLUN_XRE_ROOT}/so" "${KUNLUN_XRE_ROOT}/lib" "${KUNLUN_XRE_ROOT}/lib64"
REQUIRED)
find_library(KUNLUN_BLAS_LIB NAMES xpu_blas
HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so"
"${KUNLUN_XBLAS_ROOT}/so" "${KUNLUN_XBLAS_ROOT}/cmake"
REQUIRED)
# Vendor attention/KV-cache kernel libraries. `libapiinfer.so` (paged
# KV-cache scatter) ships in the vLLM-XPU wheel (`vllm_xpu/so/`); the
# flash-attention kernels live in `libxpu_flash_attention.so` and the
# vendor runtime context in `libxpuapi.so`, both in the torch_xmlir
# package. All are optional: without them the kunlun backend builds with
# the static-attention path only.
# `xpu/refactor/context/context.h` (baidu::xpu::api::Context) ships under
# the XHPC subtree, one level above `xre/include`. PUBLIC: the
# Python-extension `ops` target (generated dispatch/bindings) also
# compiles these wrapper headers and inherits includes only through the
# `infiniops` link.
target_include_directories(infiniops PUBLIC "${KUNLUN_HOME}/include"
"${KUNLUN_HOME}/xhpc/xdnn/include")
# vLLM-XPU wheel root (sibling of torch_xmlir) for `libapiinfer.so`.
get_filename_component(KUNLUN_SITE_PKGS "${KUNLUN_HOME}" DIRECTORY)
find_library(KUNLUN_APIINFER_LIB NAMES apiinfer
HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/so" "${KUNLUN_HOME}/lib"
"${KUNLUN_HOME}/xre/so" "${KUNLUN_SITE_PKGS}/vllm_xpu/so")
find_library(KUNLUN_FLASHATTN_LIB NAMES xpu_flash_attention
HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so")
find_library(KUNLUN_XPUAPI_LIB NAMES xpuapi
HINTS "${KUNLUN_HOME}" "${KUNLUN_HOME}/lib" "${KUNLUN_HOME}/so")
target_link_libraries(infiniops PUBLIC
"${KUNLUN_RUNTIME_LIB}"
"${KUNLUN_BLAS_LIB}")
if(KUNLUN_APIINFER_LIB)
target_link_libraries(infiniops PUBLIC "${KUNLUN_APIINFER_LIB}")
endif()
if(KUNLUN_FLASHATTN_LIB)
target_link_libraries(infiniops PUBLIC "${KUNLUN_FLASHATTN_LIB}")
endif()
if(KUNLUN_XPUAPI_LIB)
target_link_libraries(infiniops PUBLIC "${KUNLUN_XPUAPI_LIB}")
endif()

# Device code: `native/kunlun/kernels.cu` is compiled with the XTDK
# clang++ (`--xpu-arch=xpu3`, driver mode via the `.xpu` extension) into
# a static library linked into `infiniops`. The generated dispatch TUs
# for kunlun are plain C++ because the kunlun `Operator` specializations
# only launch the pre-compiled kernels through host launchers.
# The XTDK driver enables XPU mode based on the source-file extension,
# so stage the kernel source as `kernels.xpu` before compiling.
set(KUNLUN_KERNEL_SRC "${CMAKE_CURRENT_SOURCE_DIR}/native/kunlun/kernels.cu")
set(KUNLUN_KERNEL_XPU "${CMAKE_CURRENT_BINARY_DIR}/kunlun_kernels.xpu")
set(KUNLUN_KERNEL_OBJ "${CMAKE_CURRENT_BINARY_DIR}/kunlun_kernels.o")
add_custom_command(
OUTPUT "${KUNLUN_KERNEL_OBJ}"
COMMAND ${CMAKE_COMMAND} -E copy_if_different
"${KUNLUN_KERNEL_SRC}" "${KUNLUN_KERNEL_XPU}"
COMMAND "${KUNLUN_XTDK_ROOT}/bin/clang++"
--sysroot=/
--target=${CMAKE_SYSTEM_PROCESSOR}-linux-gnu
-fPIC
--xpu-arch=${KUNLUN_XPU_ARCH}
# XTDK's own numeric_limits<__half> uses narrowing brace init that
# trips -Wc++11-narrowing as an error under -O2; silence it - the
# constants are correct _Float16 bit patterns.
-Wno-narrowing -Wno-c++11-narrowing
-std=c++17 -O2 -fno-builtin
-I${PROJECT_SOURCE_DIR}
-I${KUNLUN_XRE_ROOT}/include
-c ${KUNLUN_KERNEL_XPU}
-o ${KUNLUN_KERNEL_OBJ}
DEPENDS "${KUNLUN_KERNEL_SRC}"
COMMENT "Compiling Kunlun XPU kernels (${KUNLUN_XPU_ARCH})"
VERBATIM
)
set_source_files_properties("${KUNLUN_KERNEL_OBJ}" PROPERTIES
EXTERNAL_OBJECT TRUE GENERATED TRUE)
target_sources(infiniops PRIVATE "${KUNLUN_KERNEL_OBJ}")
target_include_directories(infiniops PUBLIC
"${KUNLUN_XRE_ROOT}/include"
${KUNLUN_XBLAS_ROOT}/include)

list(APPEND DEVICE_LIST "kunlun")
endif()

if(WITH_NVIDIA)
if(NOT TARGET nvidia::cutlass::cutlass)
find_package(NvidiaCutlass 4.4.1 EXACT CONFIG QUIET)
Expand Down
2 changes: 1 addition & 1 deletion src/device.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ using AllDeviceTypes =
List<Device::Type::kCpu, Device::Type::kNvidia, Device::Type::kCambricon,
Device::Type::kAscend, Device::Type::kMetax, Device::Type::kMoore,
Device::Type::kIluvatar, Device::Type::kHygon, Device::Type::kMars,
Device::Type::kThead>;
Device::Type::kThead, Device::Type::kKunlun>;

template <typename>
struct ActiveDevicesImpl {
Expand Down
4 changes: 4 additions & 0 deletions src/native/cuda/ops/gemm/blas.h
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,10 @@ class BlasGemm : public Gemm {
}

protected:
bool a_is_col_major() const { return a_is_col_major_; }
bool b_is_col_major() const { return b_is_col_major_; }
bool swaps_a_and_b() const { return swap_a_and_b_; }

virtual typename Backend::BlasHandle GetHandle() const {
thread_local typename Backend::BlasHandle handle = []() {
typename Backend::BlasHandle h{};
Expand Down
62 changes: 62 additions & 0 deletions src/native/kunlun/blas.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#ifndef INFINI_OPS_KUNLUN_BLAS_H_
#define INFINI_OPS_KUNLUN_BLAS_H_

#include <utility>

// clang-format off
// XBLAS exposes the cublas API via `#define cublas* xblas*` macros; including
// `cublas_api.h` brings `cublasHandle_t`, `cublasCreate`, and the GemmEx
// entry points, all resolved at link time to `libxpu_blas.so` symbols.
// `library_types.h` (XRE) provides the `cudaDataType_t` enum values.
#include "cublas_api.h"
#include "library_types.h"
// clang-format on

#include "data_type.h"
#include "native/cuda/blas.h"
#include "native/kunlun/runtime_.h"

namespace infini::ops {

template <>
struct Blas<Device::Type::kKunlun> : public Runtime<Device::Type::kKunlun> {
using BlasHandle = cublasHandle_t;

static constexpr auto BLAS_OP_N = CUBLAS_OP_N;

static constexpr auto BLAS_OP_T = CUBLAS_OP_T;

static constexpr auto R_16F = CUDA_R_16F;

static constexpr auto R_16BF = CUDA_R_16BF;

static constexpr auto R_32F = CUDA_R_32F;

// XBLAS has no TF32; plain 32F compute.
static constexpr auto BLAS_COMPUTE_32F = CUBLAS_COMPUTE_32F;

static constexpr auto BLAS_COMPUTE_32F_FAST_TF32 = CUBLAS_COMPUTE_32F;

static constexpr auto BLAS_GEMM_DEFAULT = CUBLAS_GEMM_DEFAULT;

static constexpr auto BlasCreate = cublasCreate_v2;

static constexpr auto BlasSetStream = cublasSetStream_v2;

static constexpr auto BlasDestroy = cublasDestroy_v2;

static constexpr auto BlasGemmStridedBatchedEx = [](auto&&... args) {
return cublasGemmStridedBatchedEx(std::forward<decltype(args)>(args)...);
};

// xBLAS's strided-batched GEMM path rejects bf16 inputs entirely (status
// 15, NOT_SUPPORTED) while single cublasGemmEx accepts AB=bf16 with C=f32.
// The bf16 wrapper loops over the batch with GemmEx instead.
static constexpr auto BlasGemmEx = [](auto&&... args) {
return cublasGemmEx(std::forward<decltype(args)>(args)...);
};
};

} // namespace infini::ops

#endif
39 changes: 39 additions & 0 deletions src/native/kunlun/blas_utils.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
#ifndef INFINI_OPS_KUNLUN_BLAS_UTILS_H_
#define INFINI_OPS_KUNLUN_BLAS_UTILS_H_

#include "data_type.h"
#include "native/cuda/blas_utils.h"
#include "native/kunlun/blas.h"

namespace infini::ops {

template <>
struct BlasUtils<Device::Type::kKunlun> {
static auto GetDataType(DataType dtype) {
switch (dtype) {
case DataType::kFloat16:
return Blas<Device::Type::kKunlun>::R_16F;
case DataType::kBFloat16:
return Blas<Device::Type::kKunlun>::R_16BF;
case DataType::kFloat32:
return Blas<Device::Type::kKunlun>::R_32F;
default:
throw std::runtime_error("Unsupported data type for Kunlun BLAS");
}
}

static auto GetComputeType(DataType dtype) {
switch (dtype) {
case DataType::kFloat16:
case DataType::kBFloat16:
case DataType::kFloat32:
return Blas<Device::Type::kKunlun>::BLAS_COMPUTE_32F;
default:
throw std::runtime_error("Unsupported compute type for Kunlun BLAS");
}
}
};

} // namespace infini::ops

#endif
Loading
Loading