From af68f746a3693dbb3b9e053aea04e11019e2fdcb Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Thu, 19 Nov 2026 12:10:19 +0000 Subject: [PATCH 1/3] feat(ccl): auto-select `AllToAll` native vs grouped API implementations At configure time, `infiniccl_configure_ccl_capabilities()` probes each registered operation spelling (`ncclAlltoAll` vs `ncclAllToAll`) and generates `capabilities.h`; `api.h` then dispatches to the native API when the SDK provides it, else falls back to grouped `Send`/`Recv`. Probe results are cached by SDK fingerprint; registering another API is one registry line per spelling plus its dispatch branch. Also fix the hybrid `AllToAll` example timer to exclude the validation copy. --- CMakeLists.txt | 1 + cmake/CCLCapabilities.cmake | 218 ++++++++++++++++++++++ cmake/backends/MCCL.cmake | 12 ++ cmake/backends/NCCL.cmake | 14 ++ examples/ccl_mpi_hybrid/all_to_all.cc | 3 +- src/backends/ccl/common/impl/all_to_all.h | 82 +++++--- src/backends/ccl/mccl/api.h | 19 ++ src/backends/ccl/nccl/api.h | 19 ++ 8 files changed, 344 insertions(+), 24 deletions(-) create mode 100644 cmake/CCLCapabilities.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 918d596..900bea3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,6 +47,7 @@ include(cmake/devices/Hygon.cmake) include(cmake/devices/THead.cmake) # Backend modules provide backend probes and required dependency resolution. +include(cmake/CCLCapabilities.cmake) include(cmake/backends/MPI.cmake) include(cmake/backends/NCCL.cmake) include(cmake/backends/MCCL.cmake) diff --git a/cmake/CCLCapabilities.cmake b/cmake/CCLCapabilities.cmake new file mode 100644 index 0000000..f2aea87 --- /dev/null +++ b/cmake/CCLCapabilities.cmake @@ -0,0 +1,218 @@ +include_guard(GLOBAL) + +include(CheckCXXSourceCompiles) +include(CMakePushCheckState) + +# Probes optional CCL API operations against the selected SDK and generates a +# namespace of Boolean constants, one per vendor spelling plus one aggregated +# per-operation alias. +# +# ``` +# infiniccl_configure_ccl_capabilities( +# PROVIDER # capability namespace provider +# HEADER
# include used by every probe +# OPS |[|] ... # operations and vendor spellings +# [INCLUDES ] # probe include directories +# [LIBRARIES ] # probe link libraries +# [DEFINITIONS ] # probe compile definitions +# ) +# ``` +# +# Each `OPS` entry describes one operation: `` is its name (without the +# provider prefix) followed by one C++ call expression per vendor spelling of +# the same functionality, separated by `|` (call expressions must not contain +# `|`). Every spelling is probed independently by compiling and linking a +# call against the provider SDK. The generated +# `${CMAKE_BINARY_DIR}/src/backends/ccl//capabilities.h` defines, per +# operation: +# `kHas` first spelling is available +# `kHasAlternate` second spelling is available +# `kHasNative` any declared spelling is available +# +# Probe results are cached in +# `/src/backends/ccl//capabilities.stamp`. The stamp +# records an MD5 over everything that can change +# probe outcomes: the op registry, the probe header, include directories, +# library paths with file mtimes, and compile definitions. A configure whose +# fingerprint matches reuses the stored results and skips every probe; any +# registry or SDK change (including a library mtime bump) re-probes. The +# generated header is rewritten only when its content differs, so no-op +# reconfigures leave the file mtime — and therefore includers — untouched. +function(infiniccl_configure_ccl_capabilities) + set(_options) + set(_one_value_args PROVIDER HEADER) + set(_multi_value_args OPS INCLUDES LIBRARIES DEFINITIONS) + cmake_parse_arguments(CAP "${_options}" "${_one_value_args}" + "${_multi_value_args}" ${ARGN}) + + if(NOT CAP_PROVIDER) + message(FATAL_ERROR "CCL capabilities require a provider name.") + endif() + if(NOT CAP_HEADER) + message(FATAL_ERROR "CCL capabilities require a header.") + endif() + if(NOT CAP_OPS) + message(FATAL_ERROR "CCL capabilities require operation entries.") + endif() + + string(TOUPPER "${CAP_PROVIDER}" _provider_upper) + set(_output + "${CMAKE_BINARY_DIR}/src/backends/ccl/${CAP_PROVIDER}/capabilities.h") + set(_stamp + "${CMAKE_BINARY_DIR}/src/backends/ccl/${CAP_PROVIDER}/capabilities.stamp") + + # ---- SDK fingerprint: everything that can change probe outcomes -------- + set(_fingerprint + "gen=2.1|provider=${CAP_PROVIDER}|header=${CAP_HEADER}|ops=${CAP_OPS}|includes=${CAP_INCLUDES}|definitions=${CAP_DEFINITIONS}|libraries=${CAP_LIBRARIES}") + foreach(_lib IN LISTS CAP_LIBRARIES) + if(EXISTS "${_lib}") + file(TIMESTAMP "${_lib}" _lib_mtime "%s") + string(APPEND _fingerprint "|mtime:${_lib}=${_lib_mtime}") + endif() + endforeach() + string(MD5 _fingerprint_md5 "${_fingerprint}") + + set(_reuse FALSE) + set(_cached_results "") + if(EXISTS "${_stamp}") + file(READ "${_stamp}" _stamp_text) + string(REPLACE " +" ";" _stamp_lines "${_stamp_text}") + list(GET _stamp_lines 0 _stamp_md5) + list(LENGTH _stamp_lines _stamp_line_count) + if(_stamp_line_count GREATER 1 AND "${_stamp_md5}" STREQUAL "${_fingerprint_md5}") + list(GET _stamp_lines 1 _cached_csv) + string(REPLACE "," ";" _cached_results "${_cached_csv}") + set(_reuse TRUE) + endif() + endif() + + if(_reuse) + message(STATUS + "CCL[${CAP_PROVIDER}] capabilities: SDK fingerprint unchanged, reusing cached probe results") + else() + list(LENGTH CAP_OPS _op_count) + message(STATUS + "CCL[${CAP_PROVIDER}] capabilities: probing ${_op_count} operation(s) against the selected SDK") + endif() + + string(CONCAT _content + "#ifndef INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n" + "#define INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n" + "\n" + "// Generated by infiniccl_configure_ccl_capabilities() at configure\n" + "// time from compile-link probes against the selected SDK. Do not edit.\n" + "\n" + "#include \"device.h\"\n" + "\n" + "namespace infini::ccl::${CAP_PROVIDER}::capabilities {\n" + ) + + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_INCLUDES ${CAP_INCLUDES}) + set(CMAKE_REQUIRED_LIBRARIES ${CAP_LIBRARIES}) + set(CMAKE_REQUIRED_DEFINITIONS ${CAP_DEFINITIONS}) + list(TRANSFORM CMAKE_REQUIRED_DEFINITIONS PREPEND "-D") + + set(_probe_index 0) + set(_results_csv "") + set(_declared_ops "") + foreach(_op_entry IN LISTS CAP_OPS) + string(REPLACE "|" ";" _op_parts "${_op_entry}") + list(GET _op_parts 0 _op_name) + if(_op_name IN_LIST _declared_ops) + message(FATAL_ERROR + "Operation ${_op_name} is declared more than once.") + endif() + list(APPEND _declared_ops "${_op_name}") + + list(LENGTH _op_parts _spelling_count) + if(_spelling_count LESS 2 OR _spelling_count GREATER 3) + message(FATAL_ERROR + "Operation ${_op_name} needs one or two vendor spellings.") + endif() + + set(_alias_terms "") + set(_native_spelling 0) + math(EXPR _last_spelling_index "${_spelling_count} - 1") + foreach(_spelling_index RANGE 1 "${_last_spelling_index}") + list(GET _op_parts "${_spelling_index}" _call) + if(_spelling_index EQUAL 1) + set(_constant "kHas${_op_name}") + else() + set(_constant "kHas${_op_name}Alternate") + endif() + + if(_reuse) + list(GET _cached_results ${_probe_index} _available) + else() + set(_result_var "_INFINICCL_${_provider_upper}_${_constant}") + unset("${_result_var}" CACHE) + unset("${_result_var}") + set(_source "\n#include \"${CAP_HEADER}\"\nint main() {\n ${_call};\n}\n") + check_cxx_source_compiles("${_source}" "${_result_var}") + + set(_available 0) + if(${_result_var}) + set(_available 1) + endif() + endif() + math(EXPR _probe_index "${_probe_index} + 1") + string(APPEND _results_csv "${_available},") + + if(_available AND _native_spelling EQUAL 0) + set(_native_spelling ${_spelling_index}) + endif() + + string(APPEND _content + "template \n" + "inline constexpr bool ${_constant} = ${_available};\n") + list(APPEND _alias_terms "${_constant}") + endforeach() + + math(EXPR _spelling_total "${_spelling_count} - 1") + if(_native_spelling GREATER 0) + message(STATUS + "CCL[${CAP_PROVIDER}] ${_op_name}: native (spelling ${_native_spelling} of ${_spelling_total})") + else() + message(STATUS + "CCL[${CAP_PROVIDER}] ${_op_name}: no native API in this SDK (fallback)") + endif() + + string(REPLACE ";" " || " _alias_expr "${_alias_terms}") + string(APPEND _content + "template \n" + "inline constexpr bool kHasNative${_op_name} = ${_alias_expr};\n") + endforeach() + + cmake_pop_check_state() + + string(APPEND _content + "\n" + "} // namespace infini::ccl::${CAP_PROVIDER}::capabilities\n" + "\n" + "#endif // INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n" + ) + + if(NOT _reuse) + string(REGEX REPLACE ",$" "" _results_csv "${_results_csv}") + file(WRITE "${_stamp}" "${_fingerprint_md5}\n${_results_csv}") + endif() + + # ---- Emit the header only when its content actually changes ------------ + set(_header_changed TRUE) + if(EXISTS "${_output}") + file(READ "${_output}" _existing_content) + if(_existing_content STREQUAL "${_content}") + set(_header_changed FALSE) + endif() + endif() + if(_header_changed) + file(WRITE "${_output}" "${_content}") + message(STATUS + "CCL[${CAP_PROVIDER}] capabilities: wrote ${_output}") + else() + message(STATUS + "CCL[${CAP_PROVIDER}] capabilities: ${_output} unchanged") + endif() +endfunction() diff --git a/cmake/backends/MCCL.cmake b/cmake/backends/MCCL.cmake index 4402311..6f61e3a 100644 --- a/cmake/backends/MCCL.cmake +++ b/cmake/backends/MCCL.cmake @@ -58,6 +58,18 @@ function(infiniccl_configure_mccl include_output library_output) set(_provider METAX) endif() _infiniccl_find_mccl(_INFINICCL_MCCL ${_provider} TRUE _include _library) + + set(_api_operations + "AllToAll|mcclAllToAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)|mcclAlltoAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)" + ) + infiniccl_configure_ccl_capabilities( + PROVIDER mccl + HEADER "mccl.h" + OPS ${_api_operations} + INCLUDES "${_include}" + LIBRARIES "${_library}" + ) + set(${include_output} "${_include}" PARENT_SCOPE) set(${library_output} "${_library}" PARENT_SCOPE) endfunction() diff --git a/cmake/backends/NCCL.cmake b/cmake/backends/NCCL.cmake index 905a60e..2a8c9f1 100644 --- a/cmake/backends/NCCL.cmake +++ b/cmake/backends/NCCL.cmake @@ -81,6 +81,20 @@ function(infiniccl_configure_nccl include_output library_output definitions_outp endif() _infiniccl_find_nccl(_INFINICCL_NCCL TRUE _include _library _definitions) + + set(_api_operations + "AllToAll|ncclAlltoAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)|ncclAllToAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)" + ) + infiniccl_configure_ccl_capabilities( + PROVIDER nccl + HEADER "backends/ccl/nccl/compat.h" + OPS ${_api_operations} + INCLUDES "${PROJECT_SOURCE_DIR}/src" "${_include}" + ${CUDAToolkit_INCLUDE_DIRS} ${HYGON_HIP_INCLUDE_DIR} + LIBRARIES "${_library}" + DEFINITIONS ${_definitions} ${HYGON_HIP_DEFINITIONS} + ) + set(${include_output} "${_include}" PARENT_SCOPE) set(${library_output} "${_library}" PARENT_SCOPE) set(${definitions_output} "${_definitions}" PARENT_SCOPE) diff --git a/examples/ccl_mpi_hybrid/all_to_all.cc b/examples/ccl_mpi_hybrid/all_to_all.cc index 149a2bf..707894b 100644 --- a/examples/ccl_mpi_hybrid/all_to_all.cc +++ b/examples/ccl_mpi_hybrid/all_to_all.cc @@ -130,9 +130,10 @@ bool RunAllToAllExample(int argc, char **argv, int warmup_iter, } CHECK_RT(Rt, Rt::StreamSynchronize(nullptr)); + double elapsed = timer.ElapsedMs() / static_cast(profile_iter); + CHECK_RT(Rt, Rt::Memcpy(h_recv.data(), d_recv, total_bytes, Rt::MemcpyDeviceToHost)); - double elapsed = timer.ElapsedMs() / static_cast(profile_iter); // Result Validation // recv `block[src]` on rank r should come from `src`'s send `block[dst=r]`. diff --git a/src/backends/ccl/common/impl/all_to_all.h b/src/backends/ccl/common/impl/all_to_all.h index 4d39b80..cbf58cb 100644 --- a/src/backends/ccl/common/impl/all_to_all.h +++ b/src/backends/ccl/common/impl/all_to_all.h @@ -15,26 +15,35 @@ namespace infini::ccl { template class CclAllToAllImpl { - public: - static ReturnStatus Apply(const void *send_buff, void *recv_buff, - size_t count, DataType data_type, - Communicator *comm, void *stream) { - using Api = CclApi; - using TypeMap = CclTypeMap; - using CommInstance = CclCommInstance; + private: + using Api = CclApi; + using TypeMap = CclTypeMap; + using CommInstance = CclCommInstance; + + struct Arguments { + typename Api::Comm handle{}; + typename Api::DataType data_type{}; + typename Api::Stream stream{}; + int nranks = 0; + size_t peer_bytes = 0; + }; + static ReturnStatus Prepare(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, + Communicator *comm, void *stream, + Arguments *args) { if (!comm || !comm->intra_comm() || comm->intra_comm_backend() != backend || comm->device_type() != device) { return ReturnStatus::kInternalError; } if (send_buff == recv_buff) { - LOG("In-place buffers are not supported by the native CCL `AllToAll` " + LOG("In-place buffers are not supported by the CCL `AllToAll` " "implementation."); return ReturnStatus::kNotSupported; } if (comm->size() <= 0 || comm->rank() < 0 || comm->rank() >= comm->size()) { - LOG("Invalid rank or world size for native CCL `AllToAll`."); + LOG("Invalid rank or world size for CCL `AllToAll`."); return ReturnStatus::kInternalError; } @@ -50,48 +59,75 @@ class CclAllToAllImpl { const size_t type_size = kDataTypeToSize.at(data_type); if (count > std::numeric_limits::max() / type_size) { - LOG("Per-peer byte size overflows `size_t` for native CCL " - "`AllToAll`."); + LOG("Per-peer byte size overflows `size_t` for CCL `AllToAll`."); return ReturnStatus::kInvalidArgument; } const size_t peer_bytes = count * type_size; - const size_t world_size = static_cast(comm->size()); - if (peer_bytes > std::numeric_limits::max() / world_size) { - LOG("Total byte size overflows `size_t` for native CCL `AllToAll`."); + if (peer_bytes > std::numeric_limits::max() / + static_cast(comm->size())) { + LOG("Total byte size overflows `size_t` for CCL `AllToAll`."); return ReturnStatus::kInvalidArgument; } - auto native_stream = reinterpret_cast(stream); + args->handle = instance->handle; + args->data_type = native_type; + args->stream = reinterpret_cast(stream); + args->nranks = comm->size(); + args->peer_bytes = peer_bytes; + return ReturnStatus::kSuccess; + } + + static ReturnStatus ApplyGrouped(const void *send_buff, void *recv_buff, + size_t count, const Arguments &args) { ReturnStatus status = Api::Check(Api::GroupStart()); if (status != ReturnStatus::kSuccess) { return status; } - // Each peer receives its send block and supplies its receive block. ReturnStatus first_error = ReturnStatus::kSuccess; const auto *send_bytes = static_cast(send_buff); auto *recv_bytes = static_cast(recv_buff); - for (int peer = 0; peer < comm->size(); ++peer) { - const size_t offset = static_cast(peer) * peer_bytes; - status = Api::Check(Api::Send(send_bytes + offset, count, native_type, - peer, instance->handle, native_stream)); + + for (int peer = 0; peer < args.nranks; ++peer) { + const size_t offset = static_cast(peer) * args.peer_bytes; + status = Api::Check(Api::Send(send_bytes + offset, count, args.data_type, + peer, args.handle, args.stream)); if (status != ReturnStatus::kSuccess && first_error == ReturnStatus::kSuccess) { first_error = status; } - status = Api::Check(Api::Recv(recv_bytes + offset, count, native_type, - peer, instance->handle, native_stream)); + status = Api::Check(Api::Recv(recv_bytes + offset, count, args.data_type, + peer, args.handle, args.stream)); if (status != ReturnStatus::kSuccess && first_error == ReturnStatus::kSuccess) { first_error = status; } } - // Always close a successfully started group, even after a send/recv error. + // Always close a successfully opened group, even after an enqueue error. const ReturnStatus group_end_status = Api::Check(Api::GroupEnd()); return first_error != ReturnStatus::kSuccess ? first_error : group_end_status; } + + public: + static ReturnStatus Apply(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, + Communicator *comm, void *stream) { + Arguments args; + const ReturnStatus status = + Prepare(send_buff, recv_buff, count, data_type, comm, stream, &args); + if (status != ReturnStatus::kSuccess) { + return status; + } + + if constexpr (Api::kHasNativeAllToAll) { + return Api::AllToAll(send_buff, recv_buff, count, args.data_type, + args.handle, args.stream); + } else { + return ApplyGrouped(send_buff, recv_buff, count, args); + } + } }; } // namespace infini::ccl diff --git a/src/backends/ccl/mccl/api.h b/src/backends/ccl/mccl/api.h index cfbb610..de611aa 100644 --- a/src/backends/ccl/mccl/api.h +++ b/src/backends/ccl/mccl/api.h @@ -6,6 +6,7 @@ #include #include "backends/ccl/common/api.h" +#include "backends/ccl/mccl/capabilities.h" #include "logging.h" #include "return_status_impl.h" #include "runtime.h" @@ -20,6 +21,9 @@ struct McclApi { static constexpr BackendType kBackendType = BackendType::kMccl; static constexpr Device::Type kDeviceType = device; + static constexpr bool kHasNativeAllToAll = + mccl::capabilities::kHasNativeAllToAll; + using Comm = mcclComm_t; using UniqueId = mcclUniqueId; using Result = mcclResult_t; @@ -70,6 +74,21 @@ struct McclApi { stream); } + static ReturnStatus AllToAll(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, Comm comm, + Stream stream) { + if constexpr (mccl::capabilities::kHasAllToAll) { + return Check( + mcclAllToAll(send_buff, recv_buff, count, data_type, comm, stream)); + } else if constexpr (mccl::capabilities::kHasAllToAllAlternate) { + return Check( + mcclAlltoAll(send_buff, recv_buff, count, data_type, comm, stream)); + } else { + LOG("Native MCCL `AllToAll` is unavailable."); + return ReturnStatus::kNotSupported; + } + } + static Result ReduceScatter(const void *send_buff, void *recv_buff, size_t recv_count, DataType data_type, RedOp op, Comm comm, Stream stream) { diff --git a/src/backends/ccl/nccl/api.h b/src/backends/ccl/nccl/api.h index 4c71fd8..f6916c5 100644 --- a/src/backends/ccl/nccl/api.h +++ b/src/backends/ccl/nccl/api.h @@ -4,6 +4,7 @@ #include #include "backends/ccl/common/api.h" +#include "backends/ccl/nccl/capabilities.h" #include "backends/ccl/nccl/compat.h" #include "logging.h" #include "return_status_impl.h" @@ -19,6 +20,9 @@ struct NcclApi { static constexpr BackendType kBackendType = BackendType::kNccl; static constexpr Device::Type kDeviceType = device; + static constexpr bool kHasNativeAllToAll = + nccl::capabilities::kHasNativeAllToAll; + using Comm = ncclComm_t; using UniqueId = ncclUniqueId; using Result = ncclResult_t; @@ -76,6 +80,21 @@ struct NcclApi { stream); } + static ReturnStatus AllToAll(const void* send_buff, void* recv_buff, + size_t count, DataType data_type, Comm comm, + Stream stream) { + if constexpr (nccl::capabilities::kHasAllToAll) { + return Check( + ncclAlltoAll(send_buff, recv_buff, count, data_type, comm, stream)); + } else if constexpr (nccl::capabilities::kHasAllToAllAlternate) { + return Check( + ncclAllToAll(send_buff, recv_buff, count, data_type, comm, stream)); + } else { + LOG("Native NCCL `AllToAll` is unavailable."); + return ReturnStatus::kNotSupported; + } + } + static Result ReduceScatter(const void* send_buff, void* recv_buff, size_t recv_count, DataType data_type, RedOp op, Comm comm, Stream stream) { From 3341812b933db2a117b9bba1e50357feeaf65262 Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Fri, 20 Nov 2026 01:26:14 +0000 Subject: [PATCH 2/3] feat(ccl): auto-select `Scatter` and `Gather` native vs grouped APIs Register both operations with the configure-time capability probe so `capabilities.h` selects `ncclScatter`/`ncclGather` where the SDK provides them and grouped `Send`/`Recv` fallback elsewhere (e.g. NCCL 2.22.3), mirroring `AllToAll`. --- cmake/backends/MCCL.cmake | 2 + cmake/backends/NCCL.cmake | 2 + src/backends/ccl/common/impl/gather.h | 97 ++++++++++++++++++++++---- src/backends/ccl/common/impl/scatter.h | 97 ++++++++++++++++++++++---- src/backends/ccl/mccl/api.h | 33 ++++++--- src/backends/ccl/nccl/api.h | 33 ++++++--- 6 files changed, 220 insertions(+), 44 deletions(-) diff --git a/cmake/backends/MCCL.cmake b/cmake/backends/MCCL.cmake index 6f61e3a..38e3df4 100644 --- a/cmake/backends/MCCL.cmake +++ b/cmake/backends/MCCL.cmake @@ -61,6 +61,8 @@ function(infiniccl_configure_mccl include_output library_output) set(_api_operations "AllToAll|mcclAllToAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)|mcclAlltoAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)" + "Scatter|mcclScatter(nullptr, nullptr, 0, mcclFloat32, 0, nullptr, nullptr)" + "Gather|mcclGather(nullptr, nullptr, 0, mcclFloat32, 0, nullptr, nullptr)" ) infiniccl_configure_ccl_capabilities( PROVIDER mccl diff --git a/cmake/backends/NCCL.cmake b/cmake/backends/NCCL.cmake index 2a8c9f1..75268ec 100644 --- a/cmake/backends/NCCL.cmake +++ b/cmake/backends/NCCL.cmake @@ -84,6 +84,8 @@ function(infiniccl_configure_nccl include_output library_output definitions_outp set(_api_operations "AllToAll|ncclAlltoAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)|ncclAllToAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)" + "Scatter|ncclScatter(nullptr, nullptr, 0, ncclFloat32, 0, nullptr, nullptr)" + "Gather|ncclGather(nullptr, nullptr, 0, ncclFloat32, 0, nullptr, nullptr)" ) infiniccl_configure_ccl_capabilities( PROVIDER nccl diff --git a/src/backends/ccl/common/impl/gather.h b/src/backends/ccl/common/impl/gather.h index 7068c54..c106211 100644 --- a/src/backends/ccl/common/impl/gather.h +++ b/src/backends/ccl/common/impl/gather.h @@ -15,14 +15,25 @@ namespace infini::ccl { template class CclGatherImpl { - public: - static ReturnStatus Apply(const void *send_buff, void *recv_buff, - size_t count, DataType data_type, int root, - Communicator *comm, void *stream) { - using Api = CclApi; - using TypeMap = CclTypeMap; - using CommInstance = CclCommInstance; + private: + using Api = CclApi; + using TypeMap = CclTypeMap; + using CommInstance = CclCommInstance; + + struct Arguments { + typename Api::Comm handle{}; + typename Api::DataType data_type{}; + typename Api::Stream stream{}; + int nranks = 0; + int rank = 0; + int root = 0; + size_t rank_bytes = 0; + }; + static ReturnStatus Prepare(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Communicator *comm, void *stream, + Arguments *args) { if (!comm || !comm->intra_comm() || comm->intra_comm_backend() != backend || comm->device_type() != device) { return ReturnStatus::kInternalError; @@ -30,7 +41,7 @@ class CclGatherImpl { if (comm->size() <= 0 || comm->rank() < 0 || comm->rank() >= comm->size() || root < 0 || root >= comm->size()) { - LOG("Invalid rank, root, or world size for native CCL `Gather`."); + LOG("Invalid rank, root, or world size for CCL `Gather`."); return ReturnStatus::kInternalError; } @@ -46,19 +57,79 @@ class CclGatherImpl { const size_t type_size = kDataTypeToSize.at(data_type); if (count > std::numeric_limits::max() / type_size) { - LOG("Per-rank byte size overflows `size_t` for native CCL `Gather`."); + LOG("Per-rank byte size overflows `size_t` for CCL `Gather`."); return ReturnStatus::kInvalidArgument; } const size_t rank_bytes = count * type_size; const size_t world_size = static_cast(comm->size()); if (rank_bytes > std::numeric_limits::max() / world_size) { - LOG("Total byte size overflows `size_t` for native CCL `Gather`."); + LOG("Total byte size overflows `size_t` for CCL `Gather`."); return ReturnStatus::kInvalidArgument; } - return Api::Check(Api::Gather( - send_buff, recv_buff, count, native_type, root, instance->handle, - reinterpret_cast(stream))); + args->handle = instance->handle; + args->data_type = native_type; + args->stream = reinterpret_cast(stream); + args->nranks = comm->size(); + args->rank = comm->rank(); + args->root = root; + args->rank_bytes = rank_bytes; + return ReturnStatus::kSuccess; + } + + static ReturnStatus ApplyGrouped(const void *send_buff, void *recv_buff, + size_t count, const Arguments &args) { + ReturnStatus status = Api::Check(Api::GroupStart()); + if (status != ReturnStatus::kSuccess) { + return status; + } + + ReturnStatus first_error = ReturnStatus::kSuccess; + auto *recv_bytes = static_cast(recv_buff); + + status = Api::Check(Api::Send(send_buff, count, args.data_type, args.root, + args.handle, args.stream)); + if (status != ReturnStatus::kSuccess && + first_error == ReturnStatus::kSuccess) { + first_error = status; + } + + if (args.rank == args.root) { + for (int peer = 0; peer < args.nranks; ++peer) { + const size_t offset = static_cast(peer) * args.rank_bytes; + status = + Api::Check(Api::Recv(recv_bytes + offset, count, args.data_type, + peer, args.handle, args.stream)); + if (status != ReturnStatus::kSuccess && + first_error == ReturnStatus::kSuccess) { + first_error = status; + } + } + } + + // Always close a successfully opened group, even after an enqueue error. + const ReturnStatus group_end_status = Api::Check(Api::GroupEnd()); + return first_error != ReturnStatus::kSuccess ? first_error + : group_end_status; + } + + public: + static ReturnStatus Apply(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Communicator *comm, void *stream) { + Arguments args; + const ReturnStatus status = Prepare(send_buff, recv_buff, count, data_type, + root, comm, stream, &args); + if (status != ReturnStatus::kSuccess) { + return status; + } + + if constexpr (Api::kHasNativeGather) { + return Api::Gather(send_buff, recv_buff, count, args.data_type, args.root, + args.handle, args.stream); + } else { + return ApplyGrouped(send_buff, recv_buff, count, args); + } } }; diff --git a/src/backends/ccl/common/impl/scatter.h b/src/backends/ccl/common/impl/scatter.h index 07e7699..da5a25a 100644 --- a/src/backends/ccl/common/impl/scatter.h +++ b/src/backends/ccl/common/impl/scatter.h @@ -15,14 +15,25 @@ namespace infini::ccl { template class CclScatterImpl { - public: - static ReturnStatus Apply(const void *send_buff, void *recv_buff, - size_t count, DataType data_type, int root, - Communicator *comm, void *stream) { - using Api = CclApi; - using TypeMap = CclTypeMap; - using CommInstance = CclCommInstance; + private: + using Api = CclApi; + using TypeMap = CclTypeMap; + using CommInstance = CclCommInstance; + + struct Arguments { + typename Api::Comm handle{}; + typename Api::DataType data_type{}; + typename Api::Stream stream{}; + int nranks = 0; + int rank = 0; + int root = 0; + size_t rank_bytes = 0; + }; + static ReturnStatus Prepare(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Communicator *comm, void *stream, + Arguments *args) { if (!comm || !comm->intra_comm() || comm->intra_comm_backend() != backend || comm->device_type() != device) { return ReturnStatus::kInternalError; @@ -30,7 +41,7 @@ class CclScatterImpl { if (comm->size() <= 0 || comm->rank() < 0 || comm->rank() >= comm->size() || root < 0 || root >= comm->size()) { - LOG("Invalid rank, root, or world size for native CCL `Scatter`."); + LOG("Invalid rank, root, or world size for CCL `Scatter`."); return ReturnStatus::kInternalError; } @@ -46,19 +57,79 @@ class CclScatterImpl { const size_t type_size = kDataTypeToSize.at(data_type); if (count > std::numeric_limits::max() / type_size) { - LOG("Per-rank byte size overflows `size_t` for native CCL `Scatter`."); + LOG("Per-rank byte size overflows `size_t` for CCL `Scatter`."); return ReturnStatus::kInvalidArgument; } const size_t rank_bytes = count * type_size; const size_t world_size = static_cast(comm->size()); if (rank_bytes > std::numeric_limits::max() / world_size) { - LOG("Total byte size overflows `size_t` for native CCL `Scatter`."); + LOG("Total byte size overflows `size_t` for CCL `Scatter`."); return ReturnStatus::kInvalidArgument; } - return Api::Check(Api::Scatter( - send_buff, recv_buff, count, native_type, root, instance->handle, - reinterpret_cast(stream))); + args->handle = instance->handle; + args->data_type = native_type; + args->stream = reinterpret_cast(stream); + args->nranks = comm->size(); + args->rank = comm->rank(); + args->root = root; + args->rank_bytes = rank_bytes; + return ReturnStatus::kSuccess; + } + + static ReturnStatus ApplyGrouped(const void *send_buff, void *recv_buff, + size_t count, const Arguments &args) { + ReturnStatus status = Api::Check(Api::GroupStart()); + if (status != ReturnStatus::kSuccess) { + return status; + } + + ReturnStatus first_error = ReturnStatus::kSuccess; + const auto *send_bytes = static_cast(send_buff); + + if (args.rank == args.root) { + for (int peer = 0; peer < args.nranks; ++peer) { + const size_t offset = static_cast(peer) * args.rank_bytes; + status = + Api::Check(Api::Send(send_bytes + offset, count, args.data_type, + peer, args.handle, args.stream)); + if (status != ReturnStatus::kSuccess && + first_error == ReturnStatus::kSuccess) { + first_error = status; + } + } + } + + status = Api::Check(Api::Recv(recv_buff, count, args.data_type, args.root, + args.handle, args.stream)); + if (status != ReturnStatus::kSuccess && + first_error == ReturnStatus::kSuccess) { + first_error = status; + } + + // Always close a successfully opened group, even after an enqueue error. + const ReturnStatus group_end_status = Api::Check(Api::GroupEnd()); + return first_error != ReturnStatus::kSuccess ? first_error + : group_end_status; + } + + public: + static ReturnStatus Apply(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Communicator *comm, void *stream) { + Arguments args; + const ReturnStatus status = Prepare(send_buff, recv_buff, count, data_type, + root, comm, stream, &args); + if (status != ReturnStatus::kSuccess) { + return status; + } + + if constexpr (Api::kHasNativeScatter) { + return Api::Scatter(send_buff, recv_buff, count, args.data_type, + args.root, args.handle, args.stream); + } else { + return ApplyGrouped(send_buff, recv_buff, count, args); + } } }; diff --git a/src/backends/ccl/mccl/api.h b/src/backends/ccl/mccl/api.h index de611aa..6e03722 100644 --- a/src/backends/ccl/mccl/api.h +++ b/src/backends/ccl/mccl/api.h @@ -23,6 +23,10 @@ struct McclApi { static constexpr bool kHasNativeAllToAll = mccl::capabilities::kHasNativeAllToAll; + static constexpr bool kHasNativeScatter = + mccl::capabilities::kHasNativeScatter; + static constexpr bool kHasNativeGather = + mccl::capabilities::kHasNativeGather; using Comm = mcclComm_t; using UniqueId = mcclUniqueId; @@ -47,11 +51,16 @@ struct McclApi { static Result CommDestroy(Comm comm) { return mcclCommDestroy(comm); } - static Result Scatter(const void *send_buff, void *recv_buff, size_t count, - DataType data_type, int root, Comm comm, - Stream stream) { - return mcclScatter(send_buff, recv_buff, count, data_type, root, comm, - stream); + static ReturnStatus Scatter(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Comm comm, Stream stream) { + if constexpr (mccl::capabilities::kHasScatter) { + return Check(mcclScatter(send_buff, recv_buff, count, data_type, root, + comm, stream)); + } else { + LOG("Native MCCL `Scatter` is unavailable."); + return ReturnStatus::kNotSupported; + } } static Result AllReduce(const void *send_buff, void *recv_buff, size_t count, @@ -61,10 +70,16 @@ struct McclApi { stream); } - static Result Gather(const void *send_buff, void *recv_buff, size_t count, - DataType data_type, int root, Comm comm, Stream stream) { - return mcclGather(send_buff, recv_buff, count, data_type, root, comm, - stream); + static ReturnStatus Gather(const void *send_buff, void *recv_buff, + size_t count, DataType data_type, int root, + Comm comm, Stream stream) { + if constexpr (mccl::capabilities::kHasGather) { + return Check(mcclGather(send_buff, recv_buff, count, data_type, root, + comm, stream)); + } else { + LOG("Native MCCL `Gather` is unavailable."); + return ReturnStatus::kNotSupported; + } } static Result AllGather(const void *send_buff, void *recv_buff, diff --git a/src/backends/ccl/nccl/api.h b/src/backends/ccl/nccl/api.h index f6916c5..af393b8 100644 --- a/src/backends/ccl/nccl/api.h +++ b/src/backends/ccl/nccl/api.h @@ -22,6 +22,10 @@ struct NcclApi { static constexpr bool kHasNativeAllToAll = nccl::capabilities::kHasNativeAllToAll; + static constexpr bool kHasNativeScatter = + nccl::capabilities::kHasNativeScatter; + static constexpr bool kHasNativeGather = + nccl::capabilities::kHasNativeGather; using Comm = ncclComm_t; using UniqueId = ncclUniqueId; @@ -46,11 +50,16 @@ struct NcclApi { static Result CommDestroy(Comm comm) { return ncclCommDestroy(comm); } - static Result Scatter(const void* send_buff, void* recv_buff, size_t count, - DataType data_type, int root, Comm comm, - Stream stream) { - return ncclScatter(send_buff, recv_buff, count, data_type, root, comm, - stream); + static ReturnStatus Scatter(const void* send_buff, void* recv_buff, + size_t count, DataType data_type, int root, + Comm comm, Stream stream) { + if constexpr (nccl::capabilities::kHasScatter) { + return Check(ncclScatter(send_buff, recv_buff, count, data_type, root, + comm, stream)); + } else { + LOG("Native NCCL `Scatter` is unavailable."); + return ReturnStatus::kNotSupported; + } } static Result AllReduce(const void* send_buff, void* recv_buff, size_t count, @@ -67,10 +76,16 @@ struct NcclApi { stream); } - static Result Gather(const void* send_buff, void* recv_buff, size_t count, - DataType data_type, int root, Comm comm, Stream stream) { - return ncclGather(send_buff, recv_buff, count, data_type, root, comm, - stream); + static ReturnStatus Gather(const void* send_buff, void* recv_buff, + size_t count, DataType data_type, int root, + Comm comm, Stream stream) { + if constexpr (nccl::capabilities::kHasGather) { + return Check(ncclGather(send_buff, recv_buff, count, data_type, root, + comm, stream)); + } else { + LOG("Native NCCL `Gather` is unavailable."); + return ReturnStatus::kNotSupported; + } } static Result AllGather(const void* send_buff, void* recv_buff, From 07a86d4a2eeca713bc58f5be61b3b9a89bc8c4c8 Mon Sep 17 00:00:00 2001 From: Zimin Li Date: Fri, 20 Nov 2026 03:09:12 +0000 Subject: [PATCH 3/3] style: add backticks around `infiniccl_configure_ccl_capabilities()` for the generated capability headers --- cmake/CCLCapabilities.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/CCLCapabilities.cmake b/cmake/CCLCapabilities.cmake index f2aea87..c69d8d9 100644 --- a/cmake/CCLCapabilities.cmake +++ b/cmake/CCLCapabilities.cmake @@ -100,7 +100,7 @@ function(infiniccl_configure_ccl_capabilities) "#ifndef INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n" "#define INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n" "\n" - "// Generated by infiniccl_configure_ccl_capabilities() at configure\n" + "// Generated by `infiniccl_configure_ccl_capabilities()` at configure\n" "// time from compile-link probes against the selected SDK. Do not edit.\n" "\n" "#include \"device.h\"\n"