Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,11 @@ endif()

if(WITH_OMPI OR WITH_MPICH)
find_package(MPI REQUIRED)
find_path(IBVERBS_INCLUDE_DIR NAMES infiniband/verbs.h REQUIRED)
find_library(IBVERBS_LIBRARY NAMES ibverbs REQUIRED)
find_path(MLX5DV_INCLUDE_DIR NAMES infiniband/mlx5dv.h REQUIRED)
find_library(MLX5DV_LIBRARY NAMES mlx5 REQUIRED)
find_package(Threads REQUIRED)
endif()

if(WITH_NCCL)
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -366,9 +366,15 @@ export LD_LIBRARY_PATH=${INFINI_INSTALL}/lib:$LD_LIBRARY_PATH
| **MCCL** | Partial | `WITH_MCCL=ON` | Requires MetaX or Moore MCCL. Currently available when `WITH_METAX=ON` or `WITH_MOORE=ON`.|
| **CNCL** | Partial | `WITH_CNCL=ON` | Requires Cambricon CNCL. Available only when `WITH_CAMBRICON=ON`.|
| **HCCL** | Partial | `WITH_HCCL=ON` | Requires Ascend CANN, the ACLNN division header and `libopapi`. Available only when `WITH_ASCEND=ON`.|
| **RDMA (transport attribute)** | Experimental | `WITH_OMPI=ON` or `WITH_MPICH=ON`, plus `INFINICCL_RDMA_MODE=auto` | Requires Linux, `libibverbs`, and `libmlx5` with Scatter-to-CQE control. The runtime mode defaults to `staging`; `auto` enables device-memory `Send`/`Recv` over verbs RC when both peers support it (NVIDIA peer-memory: CUDA ≥ 11.3 and compatible `nvidia-peermem`; MetaX DMA-BUF: MACA/mxkw and explicit opt-in). Every unsupported message automatically falls back to the existing host staging path. The existing MPI examples exercise it without changes; communicator teardown prints per-path statistics. Messages are limited to the NIC message-size limit negotiated between peers.|

</details>

`INFINICCL_RDMA_MODE` controls only the InfiniCCL verbs data path. Set it to
`auto` to enable RDMA selection or `staging` to use the MPI host-staging path;
`staging` is the current default. This variable does not control RDMA usage
inside OpenMPI or UCX.

## License
InfiniCCL is distributed under the [LICENSE](LICENSE) file in the root of the repository.

Expand Down
45 changes: 44 additions & 1 deletion scripts/gen_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@
"hccl": "backends/ccl/hccl",
}

MPI_RDMA_BACKENDS = {"ompi", "mpich"}

# =================================================================
# LOGIC
# =================================================================
Expand Down Expand Up @@ -179,7 +181,48 @@ def generate(project_root, output_dir, devices, backends):
with open(os.path.join(output_dir, "backend_manifest.h"), "w") as f:
f.write("\n".join(manifest_lines))

# 2. Generate `comm_bridge.cc`.
# 2. Generate `rdma_manifest.h`.
rdma_lines = [
AUTOGEN_HEADER,
"#ifndef INFINI_CCL_RDMA_MANIFEST_H_",
"#define INFINI_CCL_RDMA_MANIFEST_H_",
"",
'#include "traits.h"',
]

rdma_devices = []

rdma_enabled = any(bb in MPI_RDMA_BACKENDS for bb in backends)
if rdma_enabled:
rdma_lines.append("\n// --- RDMA MEMORY ADAPTERS ---")

for dev in devices:
if not dev:
continue

rel_path = f"devices/{dev}/rdma_memory.h"
if os.path.exists(os.path.join(src_dir, rel_path)):
rdma_lines.append(f'#include "{rel_path}"')
rdma_devices.append(f"Device::Type::k{dev.capitalize()}")

def device_list(values):
return ", ".join(values) if values else ""

rdma_lines.extend(
[
"\nnamespace infini::ccl {\n",
f"using EnabledRdmaAdapters = List<{device_list(rdma_devices)}>;",
"using RdmaDispatchDevices = "
f"List<{device_list(found_devices)}>;",
"\n} // namespace infini::ccl",
"\n#endif // INFINI_CCL_RDMA_MANIFEST_H_\n",
]
)

with open(os.path.join(output_dir, "rdma_manifest.h"), "w") as f:
f.write("\n".join(rdma_lines))

# 3. Generate `comm_bridge.cc`.
bridge_lines = [
AUTOGEN_HEADER,
'#include "backend_manifest.h"',
Expand Down
73 changes: 71 additions & 2 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ target_sources(infiniccl PRIVATE

# CPU (Always On as the base architecture)
file(GLOB_RECURSE CPU_SRCS "devices/cpu/*.cc")
list(FILTER CPU_SRCS EXCLUDE REGEX "rdma_memory\\.cc$")
target_sources(infiniccl PRIVATE ${CPU_SRCS})

# NVIDIA
Expand All @@ -58,6 +59,7 @@ if(WITH_NVIDIA)
"devices/nvidia/*.cu"
)
file(GLOB NVIDIA_SOURCES ${NVIDIA_PATTERNS})
list(FILTER NVIDIA_SOURCES EXCLUDE REGEX "rdma_memory\\.cc$")

target_sources(infiniccl PRIVATE ${NVIDIA_SOURCES})
target_link_libraries(infiniccl PRIVATE CUDA::cudart CUDA::cuda_driver)
Expand Down Expand Up @@ -129,6 +131,7 @@ if(WITH_METAX)
"devices/metax/*.maca"
)
file(GLOB METAX_SOURCES ${METAX_PATTERNS})
list(FILTER METAX_SOURCES EXCLUDE REGEX "rdma_memory\\.cc$")

set_source_files_properties(${METAX_SOURCES} PROPERTIES LANGUAGE CXX)

Expand Down Expand Up @@ -282,6 +285,67 @@ if(WITH_OMPI)
endif()
endif()

# RDMA (transport attribute of the MPI backends)
if(WITH_OMPI OR WITH_MPICH)
target_sources(infiniccl PRIVATE
backends/mpi/rdma/rdma_transport.cc
backends/mpi/rdma/memory_adaptor.cc
devices/cpu/rdma_memory.cc
)
target_include_directories(infiniccl PRIVATE
${IBVERBS_INCLUDE_DIR}
${MLX5DV_INCLUDE_DIR}
)
target_link_libraries(infiniccl PRIVATE
${IBVERBS_LIBRARY}
${MLX5DV_LIBRARY}
Threads::Threads
)
if(TARGET MPI::MPI_CXX)
target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX)
endif()

include(CMakePushCheckState)
include(CheckCXXSourceCompiles)
cmake_push_check_state(RESET)
set(CMAKE_REQUIRED_INCLUDES ${MLX5DV_INCLUDE_DIR} ${IBVERBS_INCLUDE_DIR})
set(CMAKE_REQUIRED_LIBRARIES ${MLX5DV_LIBRARY} ${IBVERBS_LIBRARY})
check_cxx_source_compiles("
#include <infiniband/mlx5dv.h>
int main() {
ibv_qp_init_attr_ex attr{};
mlx5dv_qp_init_attr mlx5_attr{};
attr.comp_mask = IBV_QP_INIT_ATTR_PD;
mlx5_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS;
mlx5_attr.create_flags = MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE;
auto *qp = mlx5dv_create_qp(nullptr, &attr, &mlx5_attr);
return mlx5dv_is_supported(nullptr) && qp ? 0 : 1;
}
" INFINICCL_HAS_MLX5DV_SCATTER_CONTROL)
cmake_pop_check_state()
if(NOT INFINICCL_HAS_MLX5DV_SCATTER_CONTROL)
message(FATAL_ERROR
"MPI RDMA requires mlx5dv Scatter-to-CQE control support.")
endif()

if(WITH_NVIDIA)
if(CUDAToolkit_VERSION VERSION_LESS 11.3)
message(FATAL_ERROR
"NVIDIA RDMA requires CUDA 11.3 or newer for visibility capability checks.")
endif()
target_sources(infiniccl PRIVATE devices/nvidia/rdma_memory.cc)
endif()
if(WITH_METAX)
find_path(METAX_RDMA_INCLUDE_DIR NAMES mxkw/mxkw.h
HINTS "${MACA_PATH}/include" REQUIRED)
find_library(METAX_RDMA_LIBRARY NAMES mxkw
HINTS "${MACA_PATH}/lib" "${MACA_PATH}/lib64" REQUIRED)
target_sources(infiniccl PRIVATE devices/metax/rdma_memory.cc)
target_include_directories(infiniccl PRIVATE ${METAX_RDMA_INCLUDE_DIR})
target_link_libraries(infiniccl PRIVATE ${METAX_RDMA_LIBRARY})
endif()
endif()

if(WITH_MPICH)
list(APPEND BACKEND_LIST "mpich")
# Note: MPICH shares the same internal implementation as OMPI, so we reuse the `ompi/impl` sources for both.
Expand Down Expand Up @@ -360,11 +424,13 @@ file(WRITE "${PUBLIC_HEADERS_DIR}/infiniccl.h" "${INFINICCL_H_CONTENT}")
# --- Generate the bridge files ---
set(GENERATED_BRIDGE "${CMAKE_CURRENT_BINARY_DIR}/comm_bridge.cc")
set(GENERATED_MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/backend_manifest.h")
set(GENERATED_RDMA_MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/rdma_manifest.h")
set(DEV_STR "${DEVICE_LIST}")
set(BACK_STR "${BACKEND_LIST}")

add_custom_command(
OUTPUT "${GENERATED_BRIDGE}" "${GENERATED_MANIFEST}"
"${GENERATED_RDMA_MANIFEST}"
COMMAND ${Python3_EXECUTABLE} "${PROJECT_SOURCE_DIR}/scripts/gen_bridge.py"
"${PROJECT_SOURCE_DIR}"
"${CMAKE_CURRENT_BINARY_DIR}"
Expand All @@ -376,10 +442,13 @@ add_custom_command(
${BASE_IMPL_SRCS}
${BRIDGE_DEP_HEADERS}
VERBATIM
COMMENT "Generating InfiniCCL bridge and manifest files for Devices: [${DEV_STR}] Backends: [${BACK_STR}]..."
COMMENT "Generating InfiniCCL bridge and manifests for Devices: [${DEV_STR}] Backends: [${BACK_STR}]..."
)

target_sources(infiniccl PRIVATE "${GENERATED_BRIDGE}")
target_sources(infiniccl PRIVATE
"${GENERATED_BRIDGE}"
"${GENERATED_RDMA_MANIFEST}"
)
target_include_directories(infiniccl PRIVATE "${CMAKE_CURRENT_BINARY_DIR}")

# =========================================================
Expand Down
5 changes: 5 additions & 0 deletions src/backends/mpi/ompi/comm_instance.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

#include <mpi.h>

#include "backends/mpi/rdma/rdma_transport.h"

#include "checks.h"
#include "communicator.h"

Expand All @@ -11,10 +13,13 @@ namespace infini::ccl {
struct OmpiInstance : public BackendCommInstance {
MPI_Comm handle = MPI_COMM_NULL;

RdmaTransport rdma_transport;

OmpiInstance() { type = BackendType::kOmpi; }
~OmpiInstance() override { Destroy(); }

void Destroy() {
rdma_transport.Destroy();
if (handle != MPI_COMM_WORLD && handle != MPI_COMM_NULL) {
INFINI_CHECK_MPI(MPI_Comm_free(&handle));
handle = MPI_COMM_NULL;
Expand Down
5 changes: 5 additions & 0 deletions src/backends/mpi/ompi/impl/comm_init_all.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ class CommInitAllImpl<BackendType::kOmpi, device_type> {

comm->set_world_info(rank, size);
comm->set_local_size(n_dev);

// The RDMA transport is an attribute of this backend: it initializes
// collectively and degrades to host staging when any rank does not support RDMA.
inst->rdma_transport.Initialize(rank, size, inst->handle);

comm->set_inter_comm(std::move(inst));

int local_rank = 0;
Expand Down
7 changes: 7 additions & 0 deletions src/backends/mpi/ompi/impl/recv.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ class RecvImpl<BackendType::kOmpi, device_type> {
}

size_t total_bytes = count * type_size;

if (inst->rdma_transport.SupportRdma(recv_buff, total_bytes, peer,
device_type, stream,
/*send=*/false)) {
return inst->rdma_transport.Recv(recv_buff, total_bytes, peer);
}

void *host_buf = std::malloc(total_bytes);
if (!host_buf) {
LOG("Failed to allocate host buffer for `Recv` staging.");
Expand Down
8 changes: 8 additions & 0 deletions src/backends/mpi/ompi/impl/send.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ class SendImpl<BackendType::kOmpi, device_type> {
}

size_t total_bytes = count * type_size;

auto *send_buffer = const_cast<void *>(send_buff);
if (inst->rdma_transport.SupportRdma(send_buffer, total_bytes, peer,
device_type, stream,
/*send=*/true)) {
return inst->rdma_transport.Send(send_buffer, total_bytes, peer, stream);
}

void *host_buf = std::malloc(total_bytes);
if (!host_buf) {
LOG("Failed to allocate host buffer for `Send` staging.");
Expand Down
Loading