Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ include(cmake/devices/Hygon.cmake)
include(cmake/devices/THead.cmake)

# Backend modules provide backend probes and required dependency resolution.
include(cmake/CCLCapabilities.cmake)
include(cmake/backends/MPI.cmake)
include(cmake/backends/NCCL.cmake)
include(cmake/backends/MCCL.cmake)
Expand Down
218 changes: 218 additions & 0 deletions cmake/CCLCapabilities.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
include_guard(GLOBAL)

include(CheckCXXSourceCompiles)
include(CMakePushCheckState)

# Probes optional CCL API operations against the selected SDK and generates a
# namespace of Boolean constants, one per vendor spelling plus one aggregated
# per-operation alias.
#
# ```
# infiniccl_configure_ccl_capabilities(
# PROVIDER <provider> # capability namespace provider
# HEADER <header> # include used by every probe
# OPS <Op>|<call>[|<call>] ... # operations and vendor spellings
# [INCLUDES <dirs...>] # probe include directories
# [LIBRARIES <libs...>] # probe link libraries
# [DEFINITIONS <defs...>] # probe compile definitions
# )
# ```
#
# Each `OPS` entry describes one operation: `<Op>` is its name (without the
# provider prefix) followed by one C++ call expression per vendor spelling of
# the same functionality, separated by `|` (call expressions must not contain
# `|`). Every spelling is probed independently by compiling and linking a
# call against the provider SDK. The generated
# `${CMAKE_BINARY_DIR}/src/backends/ccl/<provider>/capabilities.h` defines, per
# operation:
# `kHas<Op>` first spelling is available
# `kHas<Op>Alternate` second spelling is available
# `kHasNative<Op>` any declared spelling is available
#
# Probe results are cached in
# `<build>/src/backends/ccl/<provider>/capabilities.stamp`. The stamp
# records an MD5 over everything that can change
# probe outcomes: the op registry, the probe header, include directories,
# library paths with file mtimes, and compile definitions. A configure whose
# fingerprint matches reuses the stored results and skips every probe; any
# registry or SDK change (including a library mtime bump) re-probes. The
# generated header is rewritten only when its content differs, so no-op
# reconfigures leave the file mtime — and therefore includers — untouched.
function(infiniccl_configure_ccl_capabilities)
set(_options)
set(_one_value_args PROVIDER HEADER)
set(_multi_value_args OPS INCLUDES LIBRARIES DEFINITIONS)
cmake_parse_arguments(CAP "${_options}" "${_one_value_args}"
"${_multi_value_args}" ${ARGN})

if(NOT CAP_PROVIDER)
message(FATAL_ERROR "CCL capabilities require a provider name.")
endif()
if(NOT CAP_HEADER)
message(FATAL_ERROR "CCL capabilities require a header.")
endif()
if(NOT CAP_OPS)
message(FATAL_ERROR "CCL capabilities require operation entries.")
endif()

string(TOUPPER "${CAP_PROVIDER}" _provider_upper)
set(_output
"${CMAKE_BINARY_DIR}/src/backends/ccl/${CAP_PROVIDER}/capabilities.h")
set(_stamp
"${CMAKE_BINARY_DIR}/src/backends/ccl/${CAP_PROVIDER}/capabilities.stamp")

# ---- SDK fingerprint: everything that can change probe outcomes --------
set(_fingerprint
"gen=2.1|provider=${CAP_PROVIDER}|header=${CAP_HEADER}|ops=${CAP_OPS}|includes=${CAP_INCLUDES}|definitions=${CAP_DEFINITIONS}|libraries=${CAP_LIBRARIES}")
foreach(_lib IN LISTS CAP_LIBRARIES)
if(EXISTS "${_lib}")
file(TIMESTAMP "${_lib}" _lib_mtime "%s")
string(APPEND _fingerprint "|mtime:${_lib}=${_lib_mtime}")
endif()
endforeach()
string(MD5 _fingerprint_md5 "${_fingerprint}")

set(_reuse FALSE)
set(_cached_results "")
if(EXISTS "${_stamp}")
file(READ "${_stamp}" _stamp_text)
string(REPLACE "
" ";" _stamp_lines "${_stamp_text}")
list(GET _stamp_lines 0 _stamp_md5)
list(LENGTH _stamp_lines _stamp_line_count)
if(_stamp_line_count GREATER 1 AND "${_stamp_md5}" STREQUAL "${_fingerprint_md5}")
list(GET _stamp_lines 1 _cached_csv)
string(REPLACE "," ";" _cached_results "${_cached_csv}")
set(_reuse TRUE)
endif()
endif()

if(_reuse)
message(STATUS
"CCL[${CAP_PROVIDER}] capabilities: SDK fingerprint unchanged, reusing cached probe results")
else()
list(LENGTH CAP_OPS _op_count)
message(STATUS
"CCL[${CAP_PROVIDER}] capabilities: probing ${_op_count} operation(s) against the selected SDK")
endif()

string(CONCAT _content
"#ifndef INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n"
"#define INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n"
"\n"
"// Generated by `infiniccl_configure_ccl_capabilities()` at configure\n"
"// time from compile-link probes against the selected SDK. Do not edit.\n"
"\n"
"#include \"device.h\"\n"
"\n"
"namespace infini::ccl::${CAP_PROVIDER}::capabilities {\n"
)

cmake_push_check_state(RESET)
set(CMAKE_REQUIRED_INCLUDES ${CAP_INCLUDES})
set(CMAKE_REQUIRED_LIBRARIES ${CAP_LIBRARIES})
set(CMAKE_REQUIRED_DEFINITIONS ${CAP_DEFINITIONS})
list(TRANSFORM CMAKE_REQUIRED_DEFINITIONS PREPEND "-D")

set(_probe_index 0)
set(_results_csv "")
set(_declared_ops "")
foreach(_op_entry IN LISTS CAP_OPS)
string(REPLACE "|" ";" _op_parts "${_op_entry}")
list(GET _op_parts 0 _op_name)
if(_op_name IN_LIST _declared_ops)
message(FATAL_ERROR
"Operation ${_op_name} is declared more than once.")
endif()
list(APPEND _declared_ops "${_op_name}")

list(LENGTH _op_parts _spelling_count)
if(_spelling_count LESS 2 OR _spelling_count GREATER 3)
message(FATAL_ERROR
"Operation ${_op_name} needs one or two vendor spellings.")
endif()

set(_alias_terms "")
set(_native_spelling 0)
math(EXPR _last_spelling_index "${_spelling_count} - 1")
foreach(_spelling_index RANGE 1 "${_last_spelling_index}")
list(GET _op_parts "${_spelling_index}" _call)
if(_spelling_index EQUAL 1)
set(_constant "kHas${_op_name}")
else()
set(_constant "kHas${_op_name}Alternate")
endif()

if(_reuse)
list(GET _cached_results ${_probe_index} _available)
else()
set(_result_var "_INFINICCL_${_provider_upper}_${_constant}")
unset("${_result_var}" CACHE)
unset("${_result_var}")
set(_source "\n#include \"${CAP_HEADER}\"\nint main() {\n ${_call};\n}\n")
check_cxx_source_compiles("${_source}" "${_result_var}")

set(_available 0)
if(${_result_var})
set(_available 1)
endif()
endif()
math(EXPR _probe_index "${_probe_index} + 1")
string(APPEND _results_csv "${_available},")

if(_available AND _native_spelling EQUAL 0)
set(_native_spelling ${_spelling_index})
endif()

string(APPEND _content
"template <Device::Type device>\n"
"inline constexpr bool ${_constant} = ${_available};\n")
list(APPEND _alias_terms "${_constant}<device>")
endforeach()

math(EXPR _spelling_total "${_spelling_count} - 1")
if(_native_spelling GREATER 0)
message(STATUS
"CCL[${CAP_PROVIDER}] ${_op_name}: native (spelling ${_native_spelling} of ${_spelling_total})")
else()
message(STATUS
"CCL[${CAP_PROVIDER}] ${_op_name}: no native API in this SDK (fallback)")
endif()

string(REPLACE ";" " || " _alias_expr "${_alias_terms}")
string(APPEND _content
"template <Device::Type device>\n"
"inline constexpr bool kHasNative${_op_name} = ${_alias_expr};\n")
endforeach()

cmake_pop_check_state()

string(APPEND _content
"\n"
"} // namespace infini::ccl::${CAP_PROVIDER}::capabilities\n"
"\n"
"#endif // INFINI_CCL_BACKENDS_CCL_${_provider_upper}_CAPABILITIES_H_\n"
)

if(NOT _reuse)
string(REGEX REPLACE ",$" "" _results_csv "${_results_csv}")
file(WRITE "${_stamp}" "${_fingerprint_md5}\n${_results_csv}")
endif()

# ---- Emit the header only when its content actually changes ------------
set(_header_changed TRUE)
if(EXISTS "${_output}")
file(READ "${_output}" _existing_content)
if(_existing_content STREQUAL "${_content}")
set(_header_changed FALSE)
endif()
endif()
if(_header_changed)
file(WRITE "${_output}" "${_content}")
message(STATUS
"CCL[${CAP_PROVIDER}] capabilities: wrote ${_output}")
else()
message(STATUS
"CCL[${CAP_PROVIDER}] capabilities: ${_output} unchanged")
endif()
endfunction()
14 changes: 14 additions & 0 deletions cmake/backends/MCCL.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,20 @@ function(infiniccl_configure_mccl include_output library_output)
set(_provider METAX)
endif()
_infiniccl_find_mccl(_INFINICCL_MCCL ${_provider} TRUE _include _library)

set(_api_operations
"AllToAll|mcclAllToAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)|mcclAlltoAll(nullptr, nullptr, 0, mcclFloat32, nullptr, nullptr)"
"Scatter|mcclScatter(nullptr, nullptr, 0, mcclFloat32, 0, nullptr, nullptr)"
"Gather|mcclGather(nullptr, nullptr, 0, mcclFloat32, 0, nullptr, nullptr)"
)
infiniccl_configure_ccl_capabilities(
PROVIDER mccl
HEADER "mccl.h"
OPS ${_api_operations}
INCLUDES "${_include}"
LIBRARIES "${_library}"
)

set(${include_output} "${_include}" PARENT_SCOPE)
set(${library_output} "${_library}" PARENT_SCOPE)
endfunction()
16 changes: 16 additions & 0 deletions cmake/backends/NCCL.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,22 @@ function(infiniccl_configure_nccl include_output library_output definitions_outp
endif()

_infiniccl_find_nccl(_INFINICCL_NCCL TRUE _include _library _definitions)

set(_api_operations
"AllToAll|ncclAlltoAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)|ncclAllToAll(nullptr, nullptr, 0, ncclFloat32, nullptr, nullptr)"
"Scatter|ncclScatter(nullptr, nullptr, 0, ncclFloat32, 0, nullptr, nullptr)"
"Gather|ncclGather(nullptr, nullptr, 0, ncclFloat32, 0, nullptr, nullptr)"
)
infiniccl_configure_ccl_capabilities(
PROVIDER nccl
HEADER "backends/ccl/nccl/compat.h"
OPS ${_api_operations}
INCLUDES "${PROJECT_SOURCE_DIR}/src" "${_include}"
${CUDAToolkit_INCLUDE_DIRS} ${HYGON_HIP_INCLUDE_DIR}
LIBRARIES "${_library}"
DEFINITIONS ${_definitions} ${HYGON_HIP_DEFINITIONS}
)

set(${include_output} "${_include}" PARENT_SCOPE)
set(${library_output} "${_library}" PARENT_SCOPE)
set(${definitions_output} "${_definitions}" PARENT_SCOPE)
Expand Down
3 changes: 2 additions & 1 deletion examples/ccl_mpi_hybrid/all_to_all.cc
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,10 @@ bool RunAllToAllExample(int argc, char **argv, int warmup_iter,
}

CHECK_RT(Rt, Rt::StreamSynchronize(nullptr));
double elapsed = timer.ElapsedMs() / static_cast<double>(profile_iter);

CHECK_RT(Rt, Rt::Memcpy(h_recv.data(), d_recv, total_bytes,
Rt::MemcpyDeviceToHost));
double elapsed = timer.ElapsedMs() / static_cast<double>(profile_iter);

// Result Validation
// recv `block[src]` on rank r should come from `src`'s send `block[dst=r]`.
Expand Down
Loading
Loading