From 9faa1ef530d22db76a8f8114870f17e499cd00e3 Mon Sep 17 00:00:00 2001 From: Jinyao618 Date: Tue, 20 Oct 2026 17:49:55 +0800 Subject: [PATCH 1/2] feat: add MPI RDMA send/recv implementation --- CMakeLists.txt | 36 + README.md | 2 + src/CMakeLists.txt | 78 ++ src/backends/mpi/ompi/comm_instance.h | 11 + src/backends/mpi/ompi/impl/comm_init_all.h | 7 + src/backends/mpi/ompi/impl/recv.h | 9 + src/backends/mpi/ompi/impl/send.h | 10 + src/backends/mpi/rdma/memory_adaptor.cc | 207 ++++++ src/backends/mpi/rdma/memory_adaptor.h | 141 ++++ src/backends/mpi/rdma/rdma_transport.cc | 825 +++++++++++++++++++++ src/backends/mpi/rdma/rdma_transport.h | 68 ++ src/devices/cpu/rdma_memory.cc | 47 ++ src/devices/cpu/rdma_memory.h | 17 + src/devices/metax/rdma_memory.cc | 256 +++++++ src/devices/metax/rdma_memory.h | 17 + src/devices/nvidia/rdma_memory.cc | 219 ++++++ src/devices/nvidia/rdma_memory.h | 17 + 17 files changed, 1967 insertions(+) create mode 100644 src/backends/mpi/rdma/memory_adaptor.cc create mode 100644 src/backends/mpi/rdma/memory_adaptor.h create mode 100644 src/backends/mpi/rdma/rdma_transport.cc create mode 100644 src/backends/mpi/rdma/rdma_transport.h create mode 100644 src/devices/cpu/rdma_memory.cc create mode 100644 src/devices/cpu/rdma_memory.h create mode 100644 src/devices/metax/rdma_memory.cc create mode 100644 src/devices/metax/rdma_memory.h create mode 100644 src/devices/nvidia/rdma_memory.cc create mode 100644 src/devices/nvidia/rdma_memory.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 918d596..30758b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ set(WITH_CPU ON CACHE INTERNAL "CPU backend is always enabled") # --- BACKEND OPTIONS (Communication Protocols) --- # ========================================================= option(WITH_OMPI "Enable OpenMPI backend" OFF) +option(WITH_RDMA "Enable RDMA backend" OFF) option(WITH_MPICH "Enable MPICH backend" OFF) option(WITH_NCCL "Enable NCCL backend" OFF) option(WITH_MCCL "Enable MCCL backend" OFF) @@ -116,6 +117,29 @@ if(AUTO_DETECT_BACKENDS) message(STATUS "Auto-detected MPICH/Hydra environment.") endif() + # Detect the RDMA transport only when it can attach to an MPI backend. + if(WITH_OMPI OR WITH_MPICH) + if(NOT WITH_RDMA) + find_program(RDMA_DEVINFO_PATH ibv_devinfo) + find_path(AUTO_IBVERBS_INCLUDE_DIR NAMES infiniband/verbs.h QUIET) + find_library(AUTO_IBVERBS_LIBRARY NAMES ibverbs QUIET) + + if(RDMA_DEVINFO_PATH AND AUTO_IBVERBS_INCLUDE_DIR AND + AUTO_IBVERBS_LIBRARY) + execute_process( + COMMAND ${RDMA_DEVINFO_PATH} + RESULT_VARIABLE RDMA_DEVINFO_RESULT + OUTPUT_QUIET + ERROR_QUIET + ) + if(RDMA_DEVINFO_RESULT EQUAL 0) + set(WITH_RDMA ON) + message(STATUS "Auto-detected RDMA environment.") + endif() + endif() + endif() + endif() + # Detect NCCL Dependencies if(WITH_NVIDIA OR WITH_THEAD OR WITH_ILUVATAR OR WITH_HYGON) infiniccl_probe_nccl(NCCL_FOUND) @@ -171,6 +195,12 @@ if(NOT WITH_OMPI AND NOT WITH_MPICH AND NOT WITH_NCCL AND NOT WITH_MCCL AND NOT message(STATUS "No backend specified or detected. Defaulting to `WITH_OMPI=ON`") endif() +# RDMA is an MPI transport attribute, not a standalone communication backend. +if(WITH_RDMA AND NOT WITH_OMPI AND NOT WITH_MPICH) + set(WITH_RDMA OFF) + message(STATUS "RDMA is ignored because no MPI backend is enabled.") +endif() + # ========================================================= # --- DEPENDENCY DISCOVERY --- # ========================================================= @@ -218,6 +248,12 @@ if(WITH_OMPI OR WITH_MPICH) find_package(MPI REQUIRED) endif() +if(WITH_RDMA) + find_path(IBVERBS_INCLUDE_DIR NAMES infiniband/verbs.h REQUIRED) + find_library(IBVERBS_LIBRARY NAMES ibverbs REQUIRED) + find_package(Threads REQUIRED) +endif() + if(WITH_NCCL) infiniccl_configure_nccl(NCCL_INC NCCL_LIB NCCL_COMPILE_DEFINITIONS) include_directories(${NCCL_INC}) diff --git a/README.md b/README.md index 0843ca3..c49fe24 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,7 @@ cmake .. -DWITH_NVIDIA=ON -DWITH_OMPI=ON | `WITH_MCCL` | Enable MCCL backend | `OFF` | | `WITH_CNCL` | Enable CNCL backend | `OFF` | | `WITH_HCCL` | Enable HCCL backend | `OFF` | +| `WITH_RDMA` | Enable the RDMA transport attribute of the MPI backends; requires `libibverbs` and one of the MPI backends (`WITH_OMPI` or `WITH_MPICH`), otherwise the option is ignored | `OFF` | | **Miscellaneous** ||| | `AUTO_DETECT_DEVICES` | Automatically detect available devices and enable corresponding support | `ON` | | `AUTO_DETECT_BACKENDS` | Automatically detect available communication backends and enable corresponding support | `OFF` | @@ -366,6 +367,7 @@ export LD_LIBRARY_PATH=${INFINI_INSTALL}/lib:$LD_LIBRARY_PATH | **MCCL** | Partial | `WITH_MCCL=ON` | Requires MetaX or Moore MCCL. Currently available when `WITH_METAX=ON` or `WITH_MOORE=ON`.| | **CNCL** | Partial | `WITH_CNCL=ON` | Requires Cambricon CNCL. Available only when `WITH_CAMBRICON=ON`.| | **HCCL** | Partial | `WITH_HCCL=ON` | Requires Ascend CANN, the ACLNN division header and `libopapi`. Available only when `WITH_ASCEND=ON`.| +| **RDMA (transport attribute)** | Experimental | `WITH_RDMA=ON` + (`WITH_OMPI=ON` or `WITH_MPICH=ON`) | Requires Linux and `libibverbs`. Device-memory `Send`/`Recv` between two ranks goes over verbs RC when both peers support it (NVIDIA peer-memory: CUDA ≥ 11.3 and compatible `nvidia-peermem`; MetaX DMA-BUF: MACA/mxkw and explicit opt-in); every other message automatically falls back to the existing host staging path. The existing MPI examples exercise it without changes; communicator teardown prints per-path statistics. Messages are limited to 1 MiB and negotiated port limits.| diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index affb03f..f2f7506 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -43,6 +43,7 @@ target_sources(infiniccl PRIVATE # CPU (Always On as the base architecture) file(GLOB_RECURSE CPU_SRCS "devices/cpu/*.cc") +list(FILTER CPU_SRCS EXCLUDE REGEX "rdma_memory\\.cc$") target_sources(infiniccl PRIVATE ${CPU_SRCS}) # NVIDIA @@ -58,6 +59,7 @@ if(WITH_NVIDIA) "devices/nvidia/*.cu" ) file(GLOB NVIDIA_SOURCES ${NVIDIA_PATTERNS}) + list(FILTER NVIDIA_SOURCES EXCLUDE REGEX "rdma_memory\\.cc$") target_sources(infiniccl PRIVATE ${NVIDIA_SOURCES}) target_link_libraries(infiniccl PRIVATE CUDA::cudart CUDA::cuda_driver) @@ -129,6 +131,7 @@ if(WITH_METAX) "devices/metax/*.maca" ) file(GLOB METAX_SOURCES ${METAX_PATTERNS}) + list(FILTER METAX_SOURCES EXCLUDE REGEX "rdma_memory\\.cc$") set_source_files_properties(${METAX_SOURCES} PROPERTIES LANGUAGE CXX) @@ -282,6 +285,81 @@ if(WITH_OMPI) endif() endif() +# RDMA (transport attribute of the MPI backends) +if(WITH_RDMA) + if(WITH_OMPI OR WITH_MPICH) + # The transport exchanges its control frames and queue-pair metadata + # over MPI, so it only builds together with an MPI backend. + target_sources(infiniccl PRIVATE + backends/mpi/rdma/rdma_transport.cc + backends/mpi/rdma/memory_adaptor.cc + devices/cpu/rdma_memory.cc + ) + target_include_directories(infiniccl PRIVATE + ${IBVERBS_INCLUDE_DIR} + ) + target_link_libraries(infiniccl PRIVATE + ${IBVERBS_LIBRARY} + Threads::Threads + ) + if(TARGET MPI::MPI_CXX) + target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX) + endif() + target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_RDMA) + find_path(MLX5DV_INCLUDE_DIR NAMES infiniband/mlx5dv.h) + find_library(MLX5DV_LIBRARY NAMES mlx5) + if(MLX5DV_INCLUDE_DIR AND MLX5DV_LIBRARY) + include(CMakePushCheckState) + include(CheckCXXSourceCompiles) + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_INCLUDES ${MLX5DV_INCLUDE_DIR} ${IBVERBS_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${MLX5DV_LIBRARY} ${IBVERBS_LIBRARY}) + check_cxx_source_compiles(" + #include + int main() { + ibv_qp_init_attr_ex attr{}; + mlx5dv_qp_init_attr mlx5_attr{}; + attr.comp_mask = IBV_QP_INIT_ATTR_PD; + mlx5_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS; + mlx5_attr.create_flags = MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; + auto *qp = mlx5dv_create_qp(nullptr, &attr, &mlx5_attr); + return mlx5dv_is_supported(nullptr) && qp ? 0 : 1; + } + " INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) + cmake_pop_check_state() + endif() + if(MLX5DV_INCLUDE_DIR AND MLX5DV_LIBRARY AND + INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) + target_include_directories(infiniccl PRIVATE ${MLX5DV_INCLUDE_DIR}) + target_link_libraries(infiniccl PRIVATE ${MLX5DV_LIBRARY}) + target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_MLX5DV) + message(STATUS "RDMA: per-QP mlx5 Scatter-to-CQE disabling enabled.") + else() + message(STATUS "RDMA: mlx5dv unavailable; Mellanox QPs will be rejected for safety.") + endif() + if(WITH_NVIDIA) + if(CUDAToolkit_VERSION VERSION_LESS 11.3) + message(FATAL_ERROR "NVIDIA RDMA requires CUDA 11.3 or newer for visibility capability checks.") + endif() + target_sources(infiniccl PRIVATE devices/nvidia/rdma_memory.cc) + target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_NVIDIA_RDMA) + endif() + if(WITH_METAX) + find_path(METAX_RDMA_INCLUDE_DIR NAMES mxkw/mxkw.h + HINTS "${MACA_PATH}/include" REQUIRED) + find_library(METAX_RDMA_LIBRARY NAMES mxkw + HINTS "${MACA_PATH}/lib" "${MACA_PATH}/lib64" REQUIRED) + target_sources(infiniccl PRIVATE devices/metax/rdma_memory.cc) + target_include_directories(infiniccl PRIVATE ${METAX_RDMA_INCLUDE_DIR}) + target_link_libraries(infiniccl PRIVATE ${METAX_RDMA_LIBRARY}) + target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_METAX_RDMA) + endif() + else() + set(WITH_RDMA OFF PARENT_SCOPE) + message(STATUS "RDMA: the transport requires an MPI backend (WITH_OMPI or WITH_MPICH); it is disabled.") + endif() +endif() + if(WITH_MPICH) list(APPEND BACKEND_LIST "mpich") # Note: MPICH shares the same internal implementation as OMPI, so we reuse the `ompi/impl` sources for both. diff --git a/src/backends/mpi/ompi/comm_instance.h b/src/backends/mpi/ompi/comm_instance.h index d0f3a9a..f2df9ab 100644 --- a/src/backends/mpi/ompi/comm_instance.h +++ b/src/backends/mpi/ompi/comm_instance.h @@ -3,6 +3,10 @@ #include +#if defined(INFINICCL_WITH_RDMA) +#include "backends/mpi/rdma/rdma_transport.h" +#endif + #include "checks.h" #include "communicator.h" @@ -11,10 +15,17 @@ namespace infini::ccl { struct OmpiInstance : public BackendCommInstance { MPI_Comm handle = MPI_COMM_NULL; +#if defined(INFINICCL_WITH_RDMA) + RdmaTransport rdma_transport; +#endif + OmpiInstance() { type = BackendType::kOmpi; } ~OmpiInstance() override { Destroy(); } void Destroy() { +#if defined(INFINICCL_WITH_RDMA) + rdma_transport.Destroy(); +#endif if (handle != MPI_COMM_WORLD && handle != MPI_COMM_NULL) { INFINI_CHECK_MPI(MPI_Comm_free(&handle)); handle = MPI_COMM_NULL; diff --git a/src/backends/mpi/ompi/impl/comm_init_all.h b/src/backends/mpi/ompi/impl/comm_init_all.h index 4812f9c..a1379d7 100644 --- a/src/backends/mpi/ompi/impl/comm_init_all.h +++ b/src/backends/mpi/ompi/impl/comm_init_all.h @@ -32,6 +32,13 @@ class CommInitAllImpl { comm->set_world_info(rank, size); comm->set_local_size(n_dev); + +#if defined(INFINICCL_WITH_RDMA) + // The RDMA transport is an attribute of this backend: it initializes + // collectively and degrades to host staging when any rank does not support RDMA. + inst->rdma_transport.Initialize(rank, size, inst->handle); +#endif + comm->set_inter_comm(std::move(inst)); int local_rank = 0; diff --git a/src/backends/mpi/ompi/impl/recv.h b/src/backends/mpi/ompi/impl/recv.h index b1422a5..321a075 100644 --- a/src/backends/mpi/ompi/impl/recv.h +++ b/src/backends/mpi/ompi/impl/recv.h @@ -35,6 +35,15 @@ class RecvImpl { } size_t total_bytes = count * type_size; + +#if defined(INFINICCL_WITH_RDMA) + if (inst->rdma_transport.SupportRdma(recv_buff, total_bytes, peer, + device_type, stream, + /*send=*/false)) { + return inst->rdma_transport.Recv(recv_buff, total_bytes, peer); + } +#endif + void *host_buf = std::malloc(total_bytes); if (!host_buf) { LOG("Failed to allocate host buffer for `Recv` staging."); diff --git a/src/backends/mpi/ompi/impl/send.h b/src/backends/mpi/ompi/impl/send.h index 16ea91d..7a6bce0 100644 --- a/src/backends/mpi/ompi/impl/send.h +++ b/src/backends/mpi/ompi/impl/send.h @@ -36,6 +36,16 @@ class SendImpl { } size_t total_bytes = count * type_size; + +#if defined(INFINICCL_WITH_RDMA) + auto *send_buffer = const_cast(send_buff); + if (inst->rdma_transport.SupportRdma(send_buffer, total_bytes, peer, + device_type, stream, + /*send=*/true)) { + return inst->rdma_transport.Send(send_buffer, total_bytes, peer, stream); + } +#endif + void *host_buf = std::malloc(total_bytes); if (!host_buf) { LOG("Failed to allocate host buffer for `Send` staging."); diff --git a/src/backends/mpi/rdma/memory_adaptor.cc b/src/backends/mpi/rdma/memory_adaptor.cc new file mode 100644 index 0000000..e49b976 --- /dev/null +++ b/src/backends/mpi/rdma/memory_adaptor.cc @@ -0,0 +1,207 @@ +#include "backends/mpi/rdma/memory_adaptor.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "devices/cpu/rdma_memory.h" + +#if defined(INFINICCL_WITH_METAX_RDMA) +#include "devices/metax/rdma_memory.h" +#endif + +#if defined(INFINICCL_WITH_NVIDIA_RDMA) +#include "devices/nvidia/rdma_memory.h" +#endif + +#include "dispatcher.h" +#include "logging.h" + +namespace infini::ccl { + +namespace { + +// Cached registrations are capped so repeated transfers over distinct large +// buffers cannot pin unbounded memory; eviction is least-recently-used. +constexpr size_t kMrCacheLimitBytes = 256 << 20; + +} // namespace + +RdmaMemoryRegion::RdmaMemoryRegion(RdmaMemoryRegion &&other) noexcept + : mr(other.mr), fd(other.fd) { + other.mr = nullptr; + other.fd = -1; +} + +RdmaMemoryRegion &RdmaMemoryRegion::operator=( + RdmaMemoryRegion &&other) noexcept { + if (this != &other) { + mr = other.mr; + fd = other.fd; + other.mr = nullptr; + other.fd = -1; + } + return *this; +} + +RdmaMemoryRegion::~RdmaMemoryRegion() { + const auto status = Reset(); + if (status != ReturnStatus::kSuccess) { + std::cerr << __FILE__ << ':' << __LINE__ << ' ' << __func__ + << ": Failed to release registered RDMA memory safely.\n"; + assert(status == ReturnStatus::kSuccess); + std::abort(); + } +} + +ReturnStatus RdmaMemoryRegion::Reset() { + if (mr) { + if (ibv_dereg_mr(mr) != 0) { + std::cerr << __FILE__ << ':' << __LINE__ << ' ' << __func__ + << ": Failed to deregister RDMA memory safely.\n"; + assert(false && "RDMA memory deregistration failed."); + std::abort(); + } + mr = nullptr; + } + if (fd >= 0) { + const int status = close(fd); + fd = -1; + if (status != 0) { + LOG("Failed to close the DMA-BUF descriptor."); + return ReturnStatus::kSystemError; + } + } + return ReturnStatus::kSuccess; +} + +ReturnStatus RegisterRdmaHostMemory(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region) { + if (!pd || !buffer || bytes == 0 || !region || region->mr || + region->fd >= 0) { + return ReturnStatus::kInvalidArgument; + } + region->mr = ibv_reg_mr(pd, buffer, bytes, IBV_ACCESS_LOCAL_WRITE); + if (!region->mr) { + const int error = errno; + std::cerr << "Host MR registration failed: " << std::strerror(error) << " (" + << error << ").\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus MrCache::Acquire(ibv_pd *pd, RdmaMemoryAdapter *adapter, + void *buffer, size_t bytes, ibv_mr **mr) { + const RdmaMemoryKey key = adapter->memory_key(); + std::lock_guard lock(mutex_); + ++acquisitions_; + auto cached = entries_.find(Range(key.base, key.bytes)); + if (cached != entries_.end()) { + if (key.tag == 0 || cached->second.tag == key.tag) { + cached->second.last_used = ++clock_; + *mr = cached->second.region.mr; + return ReturnStatus::kSuccess; + } + // The allocation behind this range was freed and its address reused. + total_bytes_ -= cached->first.second; + entries_.erase(cached); + } + auto status = InsertLocked(key, pd, adapter, buffer, bytes, mr); + if (status != ReturnStatus::kSuccess && !entries_.empty()) { + // Registration can fail once pinning resources are exhausted; free the + // least recently used entry and retry once. + EvictOldestLocked(); + status = InsertLocked(key, pd, adapter, buffer, bytes, mr); + } + if (status == ReturnStatus::kSuccess) { + EvictOverflowLocked(); + } + return status; +} + +ReturnStatus MrCache::InsertLocked(const RdmaMemoryKey &key, ibv_pd *pd, + RdmaMemoryAdapter *adapter, void *buffer, + size_t bytes, ibv_mr **mr) { + RdmaMemoryRegion region; + auto status = adapter->Register(pd, buffer, bytes, ®ion); + if (status != ReturnStatus::kSuccess) { + return status; + } + Entry entry; + entry.tag = key.tag; + entry.last_used = ++clock_; + entry.region = std::move(region); + total_bytes_ += key.bytes; + ++registrations_; + const auto inserted = + entries_.emplace(Range(key.base, key.bytes), std::move(entry)); + *mr = inserted.first->second.region.mr; + return ReturnStatus::kSuccess; +} + +void MrCache::EvictOldestLocked() { + auto oldest = entries_.begin(); + for (auto it = entries_.begin(); it != entries_.end(); ++it) { + if (it->second.last_used < oldest->second.last_used) { + oldest = it; + } + } + total_bytes_ -= oldest->first.second; + entries_.erase(oldest); +} + +void MrCache::EvictOverflowLocked() { + while (total_bytes_ > kMrCacheLimitBytes && entries_.size() > 1) { + EvictOldestLocked(); + } +} + +void MrCache::Reset() { + std::lock_guard lock(mutex_); + for (auto &item : entries_) { + const auto status = item.second.region.Reset(); + if (status != ReturnStatus::kSuccess) { + std::cerr << __FILE__ << ':' << __LINE__ << ' ' << __func__ + << ": Failed to release a cached RDMA memory region safely.\n"; + assert(status == ReturnStatus::kSuccess); + std::abort(); + } + } + entries_.clear(); + total_bytes_ = 0; +} + +MrCacheStats MrCache::Stats() const { + std::lock_guard lock(mutex_); + MrCacheStats stats; + stats.registrations = registrations_; + stats.acquisitions = acquisitions_; + stats.entries = entries_.size(); + stats.retained_bytes = total_bytes_; + return stats; +} + +ReturnStatus CreateRdmaMemoryAdapter( + Device::Type device, std::unique_ptr *adapter) { + if (!adapter) { + return ReturnStatus::kInvalidArgument; + } + if (device < Device::Type::kCpu || device >= Device::Type::kCount) { + return ReturnStatus::kNotSupported; + } + return DispatchFunc( + device, + [&](auto tag) { + constexpr Device::Type kDevice = decltype(tag)::value; + return MemoryAdapterFactory::Create(adapter); + }, + "CreateRdmaMemoryAdapter"); +} + +} // namespace infini::ccl diff --git a/src/backends/mpi/rdma/memory_adaptor.h b/src/backends/mpi/rdma/memory_adaptor.h new file mode 100644 index 0000000..5a8dd2f --- /dev/null +++ b/src/backends/mpi/rdma/memory_adaptor.h @@ -0,0 +1,141 @@ +#ifndef INFINI_CCL_BACKENDS_RDMA_MEMORY_ADAPTOR_H_ +#define INFINI_CCL_BACKENDS_RDMA_MEMORY_ADAPTOR_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "device.h" +#include "return_status_impl.h" + +namespace infini::ccl { + +struct RdmaMemoryRegion { + ~RdmaMemoryRegion(); + + RdmaMemoryRegion() = default; + + RdmaMemoryRegion(const RdmaMemoryRegion &) = delete; + + RdmaMemoryRegion &operator=(const RdmaMemoryRegion &) = delete; + + RdmaMemoryRegion(RdmaMemoryRegion &&other) noexcept; + + RdmaMemoryRegion &operator=(RdmaMemoryRegion &&other) noexcept; + + ReturnStatus Reset(); + + ibv_mr *mr = nullptr; + + int fd = -1; +}; + +// Identifies the memory allocation a buffer belongs to. Adapters fill it in +// `Prepare`; `tag` carries an allocation identity when the runtime provides +// one and stays zero when only the address range is known. +struct RdmaMemoryKey { + uintptr_t base = 0; + + size_t bytes = 0; + + uint64_t tag = 0; +}; + +class RdmaMemoryAdapter { + public: + virtual ~RdmaMemoryAdapter() = default; + + virtual ReturnStatus Prepare(void *buffer, size_t bytes, void *stream) = 0; + + virtual ReturnStatus Register(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region) = 0; + + virtual ReturnStatus Complete(bool receive) = 0; + + virtual RdmaMemoryKey memory_key() const = 0; + + // Reports whether the buffer could take the RDMA path. This is a cheap, + // side-effect free filter for the path decision; the authoritative checks + // still run in `Prepare`, so a late failure there is a hard error once the + // peer has confirmed the RDMA path. + virtual bool CanRegister(const void *buffer, size_t bytes) const = 0; +}; + +struct MrCacheStats { + uint64_t registrations = 0; + + uint64_t acquisitions = 0; + + size_t entries = 0; + + size_t retained_bytes = 0; +}; + +// Caches registered memory regions keyed by allocation range so repeated +// transfers on the same buffer reuse one registration instead of paying the +// pinning cost per message. Entries are validated against `tag` to detect +// freed-and-reused allocations and evicted least-recently-used first. +class MrCache { + public: + ReturnStatus Acquire(ibv_pd *pd, RdmaMemoryAdapter *adapter, void *buffer, + size_t bytes, ibv_mr **mr); + + // Releases every cached region. Unsafe release aborts the process, mirroring + // `RdmaMemoryRegion` teardown semantics. + void Reset(); + + MrCacheStats Stats() const; + + private: + struct Entry { + RdmaMemoryRegion region; + + uint64_t tag = 0; + + uint64_t last_used = 0; + }; + + using Range = std::pair; + + ReturnStatus InsertLocked(const RdmaMemoryKey &key, ibv_pd *pd, + RdmaMemoryAdapter *adapter, void *buffer, + size_t bytes, ibv_mr **mr); + + void EvictOldestLocked(); + + void EvictOverflowLocked(); + + mutable std::mutex mutex_; + + std::map entries_; + + size_t total_bytes_ = 0; + + uint64_t clock_ = 0; + + uint64_t registrations_ = 0; + + uint64_t acquisitions_ = 0; +}; + +template +struct MemoryAdapterFactory { + static ReturnStatus Create(std::unique_ptr * /*adapter*/) { + return ReturnStatus::kNotSupported; + } +}; + +ReturnStatus RegisterRdmaHostMemory(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region); + +ReturnStatus CreateRdmaMemoryAdapter( + Device::Type device, std::unique_ptr *adapter); + +} // namespace infini::ccl + +#endif // INFINI_CCL_BACKENDS_RDMA_MEMORY_ADAPTOR_H_ diff --git a/src/backends/mpi/rdma/rdma_transport.cc b/src/backends/mpi/rdma/rdma_transport.cc new file mode 100644 index 0000000..540ba93 --- /dev/null +++ b/src/backends/mpi/rdma/rdma_transport.cc @@ -0,0 +1,825 @@ +#include "backends/mpi/rdma/rdma_transport.h" + +#include +#include +#include +#include + +#if defined(INFINICCL_WITH_MLX5DV) +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "backends/mpi/rdma/memory_adaptor.h" +#include "logging.h" + +namespace infini::ccl { + +namespace { + +constexpr size_t kGroupIdBytes = 16; + +constexpr uint32_t kMellanoxVendorId = 0x02c9; + +// Control frames ride on a dedicated MPI tag so they never clash with the +// staged payload messages of the MPI backends (tag 0). +constexpr int kControlTag = 777; + +struct PeerMetadata { + uint32_t rank; + + uint32_t qp_num; + + uint32_t lid; + + uint32_t mtu; + + uint32_t link_layer; + + uint32_t has_gid; + + uint32_t max_bytes; + + uint32_t pkey; + + uint8_t gid[16]; + + char group_id[kGroupIdBytes]; +}; + +static_assert(sizeof(PeerMetadata) == 8 * sizeof(uint32_t) + 16 + kGroupIdBytes, + "`PeerMetadata` must not contain padding."); + +enum class StagedReason : size_t { + kNoLink = 0, + kNoAdapter, + kBuffer, + kSizeLimit, + kPeerRejected, + kCount +}; + +enum class ControlKind : uint8_t { + kStaged, + kPropose, + kAccept, + kReject +}; + +struct ControlFrame { + ControlKind kind; +}; + +ReturnStatus WaitForRdmaCompletion(ibv_cq *cq, bool receive, + size_t expected_bytes) { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(30); + while (std::chrono::steady_clock::now() < deadline) { + ibv_wc completion{}; + const int count = ibv_poll_cq(cq, 1, &completion); + if (count < 0) { + LOG("Failed to poll the RDMA completion queue."); + return ReturnStatus::kSystemError; + } + if (count == 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + if (completion.status != IBV_WC_SUCCESS) { + std::cerr << "RDMA completion failed: " + << ibv_wc_status_str(completion.status) + << ", vendor error: " << completion.vendor_err << ".\n"; + return ReturnStatus::kRemoteError; + } + if (completion.wr_id != 1 || + completion.opcode != (receive ? IBV_WC_RECV : IBV_WC_SEND) || + (receive && completion.byte_len != expected_bytes)) { + LOG("Unexpected RDMA completion or message size."); + return ReturnStatus::kRemoteError; + } + return ReturnStatus::kSuccess; + } + LOG("Timed out while waiting for an RDMA completion."); + return ReturnStatus::kSystemError; +} + +ReturnStatus ReportVerbsError(const char *operation, int error) { + std::cerr << "RDMA operation `" << operation + << "` failed: " << std::strerror(error) << " (" << error << ").\n"; + return ReturnStatus::kSystemError; +} + +bool ReadOption(const char *name, int default_value, int minimum, int maximum, + int *result) { + const char *value = std::getenv(name); + if (!value) { + *result = default_value; + return true; + } + char *end = nullptr; + errno = 0; + const long parsed = std::strtol(value, &end, 10); + if (errno || end == value || *end || parsed < minimum || parsed > maximum) { + std::cerr << "Invalid option `" << name << "`.\n"; + return false; + } + *result = static_cast(parsed); + return true; +} + +ReturnStatus SelectDevice(ibv_context **context, ibv_port_attr *port_attr, + int port) { + int count = 0; + ibv_device **devices = ibv_get_device_list(&count); + if (!devices) { + return ReportVerbsError("ibv_get_device_list", errno); + } + const char *name = std::getenv("INFINICCL_RDMA_DEVICE"); + ibv_context *found = nullptr; + for (int i = 0; i < count; ++i) { + if (name && std::strcmp(name, ibv_get_device_name(devices[i])) != 0) { + continue; + } + ibv_context *opened = ibv_open_device(devices[i]); + if (!opened) { + continue; + } + if (ibv_query_port(opened, port, port_attr) == 0 && + port_attr->state == IBV_PORT_ACTIVE) { + found = opened; + break; + } + ibv_close_device(opened); + } + ibv_free_device_list(devices); + if (!found) { + LOG("No selected RDMA device has an active port."); + return ReturnStatus::kSystemError; + } + *context = found; + return ReturnStatus::kSuccess; +} + +#if defined(INFINICCL_WITH_MLX5DV) +ReturnStatus CreateMlx5QueuePair(const ibv_qp_init_attr &base, ibv_pd *pd, + ibv_context *context, ibv_qp **qp) { + ibv_qp_init_attr_ex attr{}; + attr.qp_context = base.qp_context; + attr.send_cq = base.send_cq; + attr.recv_cq = base.recv_cq; + attr.srq = base.srq; + attr.cap = base.cap; + attr.qp_type = base.qp_type; + attr.sq_sig_all = base.sq_sig_all; + attr.comp_mask = IBV_QP_INIT_ATTR_PD; + attr.pd = pd; + mlx5dv_qp_init_attr mlx5_attr{}; + mlx5_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS; + mlx5_attr.create_flags = MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; + *qp = mlx5dv_create_qp(context, &attr, &mlx5_attr); + if (!*qp) { + return ReportVerbsError("mlx5dv_create_qp(disable Scatter-to-CQE)", errno); + } + return ReturnStatus::kSuccess; +} +#endif + +ReturnStatus CreateCompatibleQueuePair(const ibv_qp_init_attr &attr, ibv_pd *pd, + ibv_context *context, ibv_qp **qp) { +#if defined(INFINICCL_WITH_MLX5DV) + if (mlx5dv_is_supported(context->device)) { + return CreateMlx5QueuePair(attr, pd, context, qp); + } +#endif + ibv_device_attr device_attr{}; + const int query = ibv_query_device(context, &device_attr); + if (query != 0) { + return ReportVerbsError("ibv_query_device", query); + } + const char *name = ibv_get_device_name(context->device); + if (device_attr.vendor_id == kMellanoxVendorId || + std::strncmp(name, "mlx5", 4) == 0) { + LOG("InfiniCCL rejects this Mellanox QP because Scatter-to-CQE cannot " + "be disabled safely."); + return ReturnStatus::kNotSupported; + } + *qp = ibv_create_qp(pd, const_cast(&attr)); + if (!*qp) { + return ReportVerbsError("ibv_create_qp", errno); + } + return ReturnStatus::kSuccess; +} + +bool SendControl(MPI_Comm comm, int peer, ControlKind kind) { + const ControlFrame frame{kind}; + return MPI_Send(&frame, sizeof(frame), MPI_UINT8_T, peer, kControlTag, + comm) == MPI_SUCCESS; +} + +bool RecvControl(MPI_Comm comm, int peer, ControlKind *kind) { + ControlFrame frame{}; + if (MPI_Recv(&frame, sizeof(frame), MPI_UINT8_T, peer, kControlTag, comm, + MPI_STATUS_IGNORE) != MPI_SUCCESS) { + return false; + } + switch (frame.kind) { + case ControlKind::kStaged: + case ControlKind::kPropose: + case ControlKind::kAccept: + case ControlKind::kReject: + *kind = frame.kind; + return true; + } + return false; +} + +void DestroyQueuePair(ibv_qp *qp) { + if (qp) { + const int status = ibv_destroy_qp(qp); + if (status != 0) { + std::cerr << __FILE__ << ':' << __LINE__ << " " << __func__ + << ": Failed to destroy the RDMA QP safely.\n"; + assert(status == 0 && "Failed to destroy the RDMA QP safely."); + std::abort(); + } + } +} + +ReturnStatus PostReceive(ibv_qp *qp, void *buffer, size_t bytes, ibv_mr *mr) { + ibv_sge sge{}; + sge.addr = reinterpret_cast(buffer); + sge.length = static_cast(bytes); + sge.lkey = mr->lkey; + ibv_recv_wr wr{}; + wr.wr_id = 1; + wr.sg_list = &sge; + wr.num_sge = 1; + ibv_recv_wr *bad = nullptr; + const int status = ibv_post_recv(qp, &wr, &bad); + return status == 0 ? ReturnStatus::kSuccess + : ReportVerbsError("ibv_post_recv", status); +} + +ReturnStatus PostSend(ibv_qp *qp, void *buffer, size_t bytes, ibv_mr *mr) { + ibv_sge sge{}; + sge.addr = reinterpret_cast(buffer); + sge.length = static_cast(bytes); + sge.lkey = mr->lkey; + ibv_send_wr wr{}; + wr.wr_id = 1; + wr.sg_list = &sge; + wr.num_sge = 1; + wr.opcode = IBV_WR_SEND; + wr.send_flags = IBV_SEND_SIGNALED; + ibv_send_wr *bad = nullptr; + const int status = ibv_post_send(qp, &wr, &bad); + return status == 0 ? ReturnStatus::kSuccess + : ReportVerbsError("ibv_post_send", status); +} + +} // namespace + +struct RdmaTransport::Impl { + ibv_context *context = nullptr; + + ibv_pd *pd = nullptr; + + ibv_cq *send_cq = nullptr; + + ibv_cq *recv_cq = nullptr; + + ibv_qp *qp = nullptr; + + ibv_port_attr port_attr{}; + + ibv_gid gid{}; + + MPI_Comm comm = MPI_COMM_NULL; + + MrCache mr_cache; + + std::mutex mutex; + + std::unique_ptr adapter; + + ibv_mr *pending_mr = nullptr; + + ControlKind proposal = ControlKind::kStaged; + + uint8_t group_id[kGroupIdBytes] = {}; + + int rank = -1; + + int port = 1; + + int gid_index = -1; + + int pkey_index = 0; + + size_t max_bytes = 0; + + bool active = false; + + bool destroyed = false; + + uint64_t rdma_ops = 0; + + uint64_t staged_ops = 0; + + uint64_t fallbacks[static_cast(StagedReason::kCount)] = {}; + + void RecordFallback(StagedReason reason) { + ++staged_ops; + ++fallbacks[static_cast(reason)]; + } + + bool RejectSend(StagedReason reason); + + bool RejectReceive(StagedReason reason); + + bool PrepareSendPath(void *buffer, size_t bytes, Device::Type device); + + bool PrepareReceivePath(void *buffer, size_t bytes, Device::Type device, + void *stream); + + void DestroyLocked(); +}; + +namespace { + +ReturnStatus ValidatePort(RdmaTransport::Impl &impl, bool *valid) { + if (impl.port_attr.link_layer == IBV_LINK_LAYER_ETHERNET && + impl.gid_index < 0) { + LOG("RoCE requires an explicit `INFINICCL_RDMA_GID_INDEX`."); + return ReturnStatus::kInvalidArgument; + } + if ((impl.port_attr.flags & IBV_QPF_GRH_REQUIRED) && impl.gid_index < 0) { + LOG("This port requires a global route and an explicit GID index."); + return ReturnStatus::kNotSupported; + } + if (impl.pkey_index >= impl.port_attr.pkey_tbl_len) { + return ReturnStatus::kInvalidArgument; + } + if (impl.gid_index >= 0 && (impl.gid_index >= impl.port_attr.gid_tbl_len || + ibv_query_gid(impl.context, impl.port, + impl.gid_index, &impl.gid) != 0)) { + return ReturnStatus::kInvalidArgument; + } + *valid = true; + return ReturnStatus::kSuccess; +} + +void GenerateGroupId(RdmaTransport::Impl &impl) { + uint64_t local[kGroupIdBytes / sizeof(uint64_t)] = {}; + size_t offset = 0; + while (offset < sizeof(local)) { + const ssize_t count = getrandom(reinterpret_cast(local) + offset, + sizeof(local) - offset, 0); + if (count < 0 && errno == EINTR) { + continue; + } + if (count <= 0) { + local[0] ^= static_cast( + std::chrono::steady_clock::now().time_since_epoch().count()); + local[1] ^= static_cast(getpid()) << 32; + break; + } + offset += static_cast(count); + } + uint64_t shared[kGroupIdBytes / sizeof(uint64_t)] = {}; + (void)MPI_Allreduce(local, shared, 2, MPI_UINT64_T, MPI_BXOR, impl.comm); + std::memcpy(impl.group_id, shared, kGroupIdBytes); +} + +ReturnStatus BuildPeerMetadata(const RdmaTransport::Impl &impl, + PeerMetadata *local) { + uint16_t pkey = 0; + if (ibv_query_pkey(impl.context, impl.port, impl.pkey_index, &pkey) != 0 || + (ntohs(pkey) & 0x7fff) == 0) { + return ReturnStatus::kInvalidArgument; + } + local->rank = htonl(impl.rank); + local->qp_num = htonl(impl.qp->qp_num); + local->lid = htonl(impl.port_attr.lid); + local->mtu = htonl(impl.port_attr.active_mtu); + local->link_layer = htonl(impl.port_attr.link_layer); + local->has_gid = htonl(impl.gid_index >= 0); + local->max_bytes = htonl(impl.max_bytes); + local->pkey = htonl(ntohs(pkey)); + std::memcpy(local->gid, impl.gid.raw, sizeof(local->gid)); + std::memcpy(local->group_id, impl.group_id, sizeof(local->group_id)); + return ReturnStatus::kSuccess; +} + +ReturnStatus ValidatePeerMetadata(const RdmaTransport::Impl &impl, + const PeerMetadata &local, + const PeerMetadata &remote) { + if (ntohl(remote.rank) != static_cast(1 - impl.rank) || + remote.link_layer != local.link_layer || + remote.has_gid != local.has_gid || + (ntohl(remote.pkey) & 0x7fff) != (ntohl(local.pkey) & 0x7fff) || + std::memcmp(remote.group_id, local.group_id, kGroupIdBytes) != 0 || + ntohl(remote.qp_num) == 0 || ntohl(remote.qp_num) > 0xffffff || + ntohl(remote.lid) > 65535 || + (impl.gid_index < 0 && ntohl(remote.lid) == 0) || + ntohl(remote.mtu) < IBV_MTU_256 || ntohl(remote.mtu) > IBV_MTU_4096 || + ntohl(remote.max_bytes) == 0) { + LOG("Incompatible RDMA peer metadata or communication group."); + return ReturnStatus::kRemoteError; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus InitializeQueuePair(const RdmaTransport::Impl &impl) { + ibv_qp_attr attr{}; + attr.qp_state = IBV_QPS_INIT; + attr.port_num = impl.port; + attr.pkey_index = impl.pkey_index; + const int status = ibv_modify_qp( + impl.qp, &attr, + IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS); + return status == 0 ? ReturnStatus::kSuccess + : ReportVerbsError("ibv_modify_qp(INIT)", status); +} + +ReturnStatus ReadyToReceive(const RdmaTransport::Impl &impl, + const PeerMetadata &remote) { + ibv_qp_attr attr{}; + attr.qp_state = IBV_QPS_RTR; + attr.path_mtu = static_cast( + std::min(impl.port_attr.active_mtu, ntohl(remote.mtu))); + attr.dest_qp_num = ntohl(remote.qp_num); + attr.rq_psn = attr.dest_qp_num & 0xffffff; + attr.max_dest_rd_atomic = 1; + attr.min_rnr_timer = 12; + attr.ah_attr.dlid = ntohl(remote.lid); + attr.ah_attr.port_num = impl.port; + if (impl.gid_index >= 0) { + attr.ah_attr.is_global = 1; + std::memcpy(attr.ah_attr.grh.dgid.raw, remote.gid, + sizeof(attr.ah_attr.grh.dgid.raw)); + attr.ah_attr.grh.sgid_index = impl.gid_index; + attr.ah_attr.grh.hop_limit = 64; + } + const int status = ibv_modify_qp( + impl.qp, &attr, + IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | + IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER); + return status == 0 ? ReturnStatus::kSuccess + : ReportVerbsError("ibv_modify_qp(RTR)", status); +} + +ReturnStatus ReadyToSend(const RdmaTransport::Impl &impl) { + ibv_qp_attr attr{}; + attr.qp_state = IBV_QPS_RTS; + attr.sq_psn = impl.qp->qp_num & 0xffffff; + attr.timeout = 14; + attr.retry_cnt = 7; + attr.rnr_retry = 7; + attr.max_rd_atomic = 1; + const int status = ibv_modify_qp( + impl.qp, &attr, + IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | + IBV_QP_RNR_RETRY | IBV_QP_MAX_QP_RD_ATOMIC); + return status == 0 ? ReturnStatus::kSuccess + : ReportVerbsError("ibv_modify_qp(RTS)", status); +} + +bool PrepareLocalResources(RdmaTransport::Impl &impl) { + if (!ReadOption("INFINICCL_RDMA_IB_PORT", 1, 1, 255, &impl.port) || + !ReadOption("INFINICCL_RDMA_GID_INDEX", -1, -1, 255, &impl.gid_index) || + !ReadOption("INFINICCL_RDMA_PKEY_INDEX", 0, 0, 65535, &impl.pkey_index)) { + return false; + } + if (SelectDevice(&impl.context, &impl.port_attr, impl.port) != + ReturnStatus::kSuccess) { + impl.context = nullptr; + return false; + } + bool port_valid = false; + const auto port_status = ValidatePort(impl, &port_valid); + if (port_status != ReturnStatus::kSuccess || !port_valid) { + ibv_close_device(impl.context); + impl.context = nullptr; + return false; + } + impl.pd = ibv_alloc_pd(impl.context); + impl.send_cq = ibv_create_cq(impl.context, 16, nullptr, nullptr, 0); + impl.recv_cq = ibv_create_cq(impl.context, 16, nullptr, nullptr, 0); + ibv_qp_init_attr attr{}; + attr.send_cq = impl.send_cq; + attr.recv_cq = impl.recv_cq; + attr.cap.max_send_wr = 16; + attr.cap.max_recv_wr = 16; + attr.cap.max_send_sge = 1; + attr.cap.max_recv_sge = 1; + attr.qp_type = IBV_QPT_RC; + const auto qp_status = + CreateCompatibleQueuePair(attr, impl.pd, impl.context, &impl.qp); + if (!impl.pd || !impl.send_cq || !impl.recv_cq || + qp_status != ReturnStatus::kSuccess) { + return false; + } + impl.max_bytes = std::min(impl.port_attr.max_msg_sz, 1 << 20); + return true; +} + +ReturnStatus ConnectPeer(RdmaTransport::Impl &impl) { + auto status = ReturnStatus::kSuccess; + GenerateGroupId(impl); + PeerMetadata local{}; + PeerMetadata remote{}; + status = BuildPeerMetadata(impl, &local); + if (status == ReturnStatus::kSuccess && + MPI_Sendrecv(&local, sizeof(local), MPI_BYTE, 1 - impl.rank, kControlTag, + &remote, sizeof(remote), MPI_BYTE, 1 - impl.rank, + kControlTag, impl.comm, MPI_STATUS_IGNORE) != MPI_SUCCESS) { + LOG("Failed to exchange RDMA peer metadata over MPI."); + status = ReturnStatus::kSystemError; + } + if (status == ReturnStatus::kSuccess) { + status = ValidatePeerMetadata(impl, local, remote); + } + if (status == ReturnStatus::kSuccess) { + impl.max_bytes = std::min(impl.max_bytes, ntohl(remote.max_bytes)); + status = InitializeQueuePair(impl); + } + if (status == ReturnStatus::kSuccess) { + status = ReadyToReceive(impl, remote); + } + if (status == ReturnStatus::kSuccess) { + status = ReadyToSend(impl); + } + int ready = status == ReturnStatus::kSuccess ? 1 : 0; + int all_ready = 0; + if (MPI_Allreduce(&ready, &all_ready, 1, MPI_INT, MPI_MIN, impl.comm) != + MPI_SUCCESS || + all_ready != 1) { + return ReturnStatus::kSystemError; + } + return status; +} + +} // namespace + +bool RdmaTransport::Impl::RejectSend(StagedReason reason) { + RecordFallback(reason); + // The peer waits for exactly one control frame per message. + (void)SendControl(comm, 1 - rank, ControlKind::kStaged); + return false; +} + +bool RdmaTransport::Impl::RejectReceive(StagedReason reason) { + RecordFallback(reason); + (void)SendControl(comm, 1 - rank, ControlKind::kReject); + return false; +} + +bool RdmaTransport::Impl::PrepareSendPath(void *buffer, size_t bytes, + Device::Type device) { + if (bytes == 0 || bytes > max_bytes) { + return RejectSend(StagedReason::kSizeLimit); + } + if (CreateRdmaMemoryAdapter(device, &adapter) != ReturnStatus::kSuccess) { + adapter.reset(); + return RejectSend(StagedReason::kNoAdapter); + } + if (!adapter->CanRegister(buffer, bytes)) { + adapter.reset(); + return RejectSend(StagedReason::kBuffer); + } + if (!SendControl(comm, 1 - rank, ControlKind::kPropose) || + !RecvControl(comm, 1 - rank, &proposal)) { + RecordFallback(StagedReason::kNoLink); + adapter.reset(); + return false; + } + if (proposal != ControlKind::kAccept) { + RecordFallback(StagedReason::kPeerRejected); + adapter.reset(); + return false; + } + return true; +} + +bool RdmaTransport::Impl::PrepareReceivePath(void *buffer, size_t bytes, + Device::Type device, + void *stream) { + if (!RecvControl(comm, 1 - rank, &proposal)) { + RecordFallback(StagedReason::kNoLink); + return false; + } + if (proposal == ControlKind::kStaged) { + // The peer already chose host staging; mirror it without a reply. + ++staged_ops; + return false; + } + if (bytes == 0 || bytes > max_bytes) { + return RejectReceive(StagedReason::kSizeLimit); + } + if (CreateRdmaMemoryAdapter(device, &adapter) != ReturnStatus::kSuccess) { + adapter.reset(); + return RejectReceive(StagedReason::kNoAdapter); + } + if (!adapter->CanRegister(buffer, bytes)) { + adapter.reset(); + return RejectReceive(StagedReason::kBuffer); + } + auto status = adapter->Prepare(buffer, bytes, stream); + if (status == ReturnStatus::kSuccess) { + status = mr_cache.Acquire(pd, adapter.get(), buffer, bytes, &pending_mr); + } + if (status == ReturnStatus::kSuccess) { + status = PostReceive(qp, buffer, bytes, pending_mr); + } + if (status != ReturnStatus::kSuccess) { + adapter.reset(); + pending_mr = nullptr; + return RejectReceive(StagedReason::kBuffer); + } + // The receive work request is posted; the confirmation is the point of no + // return after which failures surface as errors instead of fallbacks. + if (!SendControl(comm, 1 - rank, ControlKind::kAccept)) { + RecordFallback(StagedReason::kNoLink); + adapter.reset(); + pending_mr = nullptr; + return false; + } + return true; +} + +void RdmaTransport::Impl::DestroyLocked() { + if (destroyed) { + return; + } + destroyed = true; + + DestroyQueuePair(qp); + qp = nullptr; + adapter.reset(); + pending_mr = nullptr; + const auto cache_stats = mr_cache.Stats(); + mr_cache.Reset(); + if (cache_stats.acquisitions > 0) { + std::cout << "[Rank " << rank + << "] RDMA MR cache: " << cache_stats.registrations + << " registrations for " << cache_stats.acquisitions + << " transfers." << std::endl; + } + if (rdma_ops > 0 || staged_ops > 0) { + std::cout << "[Rank " << rank << "] RDMA transport: rdma=" << rdma_ops + << " staged=" << staged_ops << " (no_link=" + << fallbacks[static_cast(StagedReason::kNoLink)] + << ", no_adapter=" + << fallbacks[static_cast(StagedReason::kNoAdapter)] + << ", buffer=" + << fallbacks[static_cast(StagedReason::kBuffer)] + << ", size=" + << fallbacks[static_cast(StagedReason::kSizeLimit)] + << ", peer_rejected=" + << fallbacks[static_cast(StagedReason::kPeerRejected)] + << ")" << std::endl; + } + rdma_ops = 0; + staged_ops = 0; + std::memset(fallbacks, 0, sizeof(fallbacks)); + if (send_cq) { + ibv_destroy_cq(send_cq); + send_cq = nullptr; + } + if (recv_cq) { + ibv_destroy_cq(recv_cq); + recv_cq = nullptr; + } + if (pd) { + ibv_dealloc_pd(pd); + pd = nullptr; + } + if (context) { + ibv_close_device(context); + context = nullptr; + } + active = false; +} + +RdmaTransport::RdmaTransport() : impl_(std::make_unique()) {} + +RdmaTransport::~RdmaTransport() { Destroy(); } + +void RdmaTransport::Initialize(int rank, int nranks, MPI_Comm comm) { + std::lock_guard lock(impl_->mutex); + auto &impl = *impl_; + impl.rank = rank; + impl.comm = comm; + impl.destroyed = false; + if (nranks != 2) { + // Temporarily supports two ranks. + return; + } + const bool local_ready = PrepareLocalResources(impl); + int ready = local_ready ? 1 : 0; + int all_ready = 0; + if (MPI_Allreduce(&ready, &all_ready, 1, MPI_INT, MPI_MIN, comm) != + MPI_SUCCESS) { + LOG("Failed to agree on RDMA transport availability."); + impl.DestroyLocked(); + return; + } + if (all_ready != 1) { + if (local_ready) { + std::cerr << "[Rank " << rank + << "] RDMA devices are unavailable on some rank; messages " + "use host staging.\n"; + } + impl.DestroyLocked(); + return; + } + const auto status = ConnectPeer(impl); + if (status != ReturnStatus::kSuccess) { + std::cerr << "[Rank " << rank + << "] RDMA transport setup failed; messages use host " + "staging.\n"; + impl.DestroyLocked(); + return; + } + std::cout << "[Rank " << rank << "] RDMA transport ready on " + << ibv_get_device_name(impl.context->device) << ", port " + << impl.port << ", LID " << impl.port_attr.lid << ", GID index " + << impl.gid_index << "; maximum message " << impl.max_bytes + << " bytes." << std::endl; + impl.active = true; +} + +bool RdmaTransport::SupportRdma(void *buffer, size_t bytes, int peer, + Device::Type device, void *stream, bool send) { + std::lock_guard lock(impl_->mutex); + impl_->adapter.reset(); + impl_->pending_mr = nullptr; + if (!impl_->active || peer != 1 - impl_->rank) { + impl_->RecordFallback(StagedReason::kNoLink); + return false; + } + if (send) { + return impl_->PrepareSendPath(buffer, bytes, device); + } + return impl_->PrepareReceivePath(buffer, bytes, device, stream); +} + +ReturnStatus RdmaTransport::Send(void *buffer, size_t bytes, int peer, + void *stream) { + (void)peer; + std::lock_guard lock(impl_->mutex); + auto &impl = *impl_; + auto status = impl.adapter->Prepare(buffer, bytes, stream); + if (status == ReturnStatus::kSuccess) { + status = impl.mr_cache.Acquire(impl.pd, impl.adapter.get(), buffer, bytes, + &impl.pending_mr); + } + if (status == ReturnStatus::kSuccess) { + status = PostSend(impl.qp, buffer, bytes, impl.pending_mr); + } + if (status == ReturnStatus::kSuccess) { + status = WaitForRdmaCompletion(impl.send_cq, false, bytes); + } + if (status == ReturnStatus::kSuccess) { + status = impl.adapter->Complete(false); + ++impl.rdma_ops; + return status; + } + impl.DestroyLocked(); + return status; +} + +ReturnStatus RdmaTransport::Recv(void *buffer, size_t bytes, int peer) { + (void)buffer; + (void)peer; + std::lock_guard lock(impl_->mutex); + auto &impl = *impl_; + const auto status = WaitForRdmaCompletion(impl.recv_cq, true, bytes); + if (status != ReturnStatus::kSuccess) { + impl.DestroyLocked(); + return status; + } + const auto complete = impl.adapter->Complete(true); + ++impl.rdma_ops; + if (complete != ReturnStatus::kSuccess) { + impl.DestroyLocked(); + } + return complete; +} + +void RdmaTransport::Destroy() { + std::lock_guard lock(impl_->mutex); + impl_->DestroyLocked(); +} + +} // namespace infini::ccl diff --git a/src/backends/mpi/rdma/rdma_transport.h b/src/backends/mpi/rdma/rdma_transport.h new file mode 100644 index 0000000..1e04218 --- /dev/null +++ b/src/backends/mpi/rdma/rdma_transport.h @@ -0,0 +1,68 @@ +#ifndef INFINI_CCL_BACKENDS_RDMA_RDMA_TRANSPORT_H_ +#define INFINI_CCL_BACKENDS_RDMA_RDMA_TRANSPORT_H_ + +#include +#include + +#include +#include + +#include "device.h" +#include "return_status_impl.h" + +namespace infini::ccl { + +// RDMA transport attribute of the MPI backend. Device-memory messages use +// a verbs RC connection directly when both peers support it; everything else +// falls back to the caller's host staging path. The class is internal to the +// library: applications keep using `infinicclSend`/`infinicclRecv`. +// +// Calls belonging to one communicator must be serialized by the caller, the +// same assumption the MPI staging path already makes. +class RdmaTransport { + public: + RdmaTransport(); + + ~RdmaTransport(); + + RdmaTransport(const RdmaTransport &) = delete; + + RdmaTransport &operator=(const RdmaTransport &) = delete; + + // The nested name is public so the implementation file can define helpers + // over it; the definition and all members stay in the `.cc` file. + struct Impl; + + // Creates the verbs resources and exchanges QP metadata with the peer over + // `comm`. The transport stays inert (`SupportRdma` always returns false) + // when the rank count is not two or no RDMA device is usable, so callers + // keep falling back to host staging without extra branching. + void Initialize(int rank, int nranks, MPI_Comm comm); + + // End-to-end path decision for one message. The send side runs the local + // pre-check, proposes the RDMA path to the peer and waits for its + // confirmation; the receive side waits for the proposal, validates its own + // buffer, posts the receive work request and only then confirms. Every + // fallback is recorded in the transport statistics. Returns false whenever + // either side cannot use RDMA; the caller must then use host staging. + bool SupportRdma(void *buffer, size_t bytes, int peer, Device::Type device, + void *stream, bool send); + + // Executes the RDMA send path after `SupportRdma` returned true for the + // same message. Failures after the peer confirmed are hard errors; the + // transport is torn down so later messages use host staging. + ReturnStatus Send(void *buffer, size_t bytes, int peer, void *stream); + + // Waits for the receive completion after `SupportRdma` returned true; the + // receive work request is already posted there. + ReturnStatus Recv(void *buffer, size_t bytes, int peer); + + // Releases all resources and prints the transport statistics. + void Destroy(); + + private: + std::unique_ptr impl_; +}; +} // namespace infini::ccl + +#endif // INFINI_CCL_BACKENDS_RDMA_RDMA_TRANSPORT_H_ diff --git a/src/devices/cpu/rdma_memory.cc b/src/devices/cpu/rdma_memory.cc new file mode 100644 index 0000000..e574dc1 --- /dev/null +++ b/src/devices/cpu/rdma_memory.cc @@ -0,0 +1,47 @@ +#include "devices/cpu/rdma_memory.h" + +namespace infini::ccl { +namespace { + +class HostMemoryAdapter : public RdmaMemoryAdapter { + public: + ReturnStatus Prepare(void *buffer, size_t bytes, void *stream) override { + if (stream) { + return ReturnStatus::kNotSupported; + } + key_ = RdmaMemoryKey{reinterpret_cast(buffer), bytes, 0}; + return ReturnStatus::kSuccess; + } + + ReturnStatus Register(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region) override { + return RegisterRdmaHostMemory(pd, buffer, bytes, region); + } + + ReturnStatus Complete(bool /*receive*/) override { + return ReturnStatus::kSuccess; + } + + RdmaMemoryKey memory_key() const override { return key_; } + + // Host memory temporarily not supported. + bool CanRegister(const void * /*buffer*/, size_t /*bytes*/) const override { + return false; + } + + private: + RdmaMemoryKey key_; +}; + +} // namespace + +ReturnStatus MemoryAdapterFactory::Create( + std::unique_ptr *adapter) { + if (!adapter) { + return ReturnStatus::kInvalidArgument; + } + *adapter = std::make_unique(); + return ReturnStatus::kSuccess; +} + +} // namespace infini::ccl diff --git a/src/devices/cpu/rdma_memory.h b/src/devices/cpu/rdma_memory.h new file mode 100644 index 0000000..16001d4 --- /dev/null +++ b/src/devices/cpu/rdma_memory.h @@ -0,0 +1,17 @@ +#ifndef INFINI_CCL_DEVICES_CPU_RDMA_MEMORY_H_ +#define INFINI_CCL_DEVICES_CPU_RDMA_MEMORY_H_ + +#include + +#include "backends/mpi/rdma/memory_adaptor.h" + +namespace infini::ccl { + +template <> +struct MemoryAdapterFactory { + static ReturnStatus Create(std::unique_ptr *adapter); +}; + +} // namespace infini::ccl + +#endif // INFINI_CCL_DEVICES_CPU_RDMA_MEMORY_H_ diff --git a/src/devices/metax/rdma_memory.cc b/src/devices/metax/rdma_memory.cc new file mode 100644 index 0000000..e741335 --- /dev/null +++ b/src/devices/metax/rdma_memory.cc @@ -0,0 +1,256 @@ +#include "devices/metax/rdma_memory.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace infini::ccl { +namespace { + +struct MetaxAllocation { + void *base = nullptr; + + size_t bytes = 0; + + size_t offset = 0; +}; + +ReturnStatus CheckMaca(mcError_t status, const char *operation) { + if (status != mcSuccess) { + std::cerr << "MACA operation `" << operation + << "` failed: " << mcGetErrorString(status) << ".\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +template +uintptr_t PointerAddress(T pointer) { + if constexpr (std::is_pointer_v) { + return reinterpret_cast(pointer); + } else { + return static_cast(pointer); + } +} + +ReturnStatus ValidateDeviceBuffer(void *buffer, size_t bytes, + int current_device, + MetaxAllocation *allocation) { + int buffer_device = -1; + const auto pointer = reinterpret_cast(buffer); + auto status = CheckMaca( + mcPointerGetAttribute(&buffer_device, mcPointerAttributeDevice, pointer), + "mcPointerGetAttribute(device)"); + if (status != ReturnStatus::kSuccess) { + return status; + } + if (buffer_device != current_device) { + std::cerr + << "The RDMA buffer does not belong to the current MACA device.\n"; + return ReturnStatus::kInvalidArgument; + } + mcDeviceptr_t base{}; + size_t allocation_bytes = 0; + status = CheckMaca(mcMemGetAddressRange(&base, &allocation_bytes, pointer), + "mcMemGetAddressRange"); + if (status != ReturnStatus::kSuccess) { + return status; + } + const auto address = reinterpret_cast(buffer); + const auto start = PointerAddress(base); + if (address < start || address - start > allocation_bytes || + bytes > allocation_bytes - (address - start)) { + return ReturnStatus::kInvalidArgument; + } + allocation->base = reinterpret_cast(start); + allocation->bytes = allocation_bytes; + allocation->offset = address - start; + return ReturnStatus::kSuccess; +} + +// Experimental RDMA enable switch for test. +bool ExperimentalDirectEnabled() { + const char *value = std::getenv("INFINICCL_RDMA_METAX_EXPERIMENTAL"); + return value && std::strcmp(value, "1") == 0; +} + +void QueryFlushPolicy(int device, bool *flush) { + static std::once_flag warning_once; + int ordering = 0; + int options = 0; + const auto ordering_status = mcDeviceGetAttribute( + &ordering, mcDeviceAttributeGPUDirectRDMAWritesOrdering, device); + const auto options_status = mcDeviceGetAttribute( + &options, mcDeviceAttributeGPUDirectRDMAFlushWritesOptions, device); + *flush = ordering_status == mcSuccess && options_status == mcSuccess && + ordering < mcFlushGPUDirectRDMAWritesToOwner && + (options & mcFlushGPUDirectRDMAWritesOptionHost); + const bool native_ordering = ordering_status == mcSuccess && + ordering >= mcFlushGPUDirectRDMAWritesToOwner; + std::call_once(warning_once, [flush, native_ordering] { + std::cerr << "Experimental MetaX DeviceDirect: DMA visibility is not yet " + "validated; no Host staging is used. Receive flush: " + << (*flush ? "enabled" + : native_ordering ? "native ordering reported" + : "not reported, visibility unverified") + << ".\n"; + }); +} + +ReturnStatus ExportDeviceMemory(void *buffer, size_t bytes, + RdmaMemoryRegion *region, MXuint64 *offset) { + *offset = std::numeric_limits::max(); + const auto status = + mxkwExportDMABufHandle(buffer, bytes, ®ion->fd, offset); + // The target MACA 3.2.1 probe returned zero on a successful export. + if (static_cast(status) != 0 || region->fd < 0 || + *offset > std::numeric_limits::max() - bytes) { + std::cerr << "`mxkwExportDMABufHandle` failed: status=" + << static_cast(status) << ", fd=" << region->fd << ".\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus RegisterDeviceMemory(ibv_pd *pd, const MetaxAllocation &allocation, + RdmaMemoryRegion *region) { + MXuint64 offset = 0; + auto status = + ExportDeviceMemory(allocation.base, allocation.bytes, region, &offset); + if (status != ReturnStatus::kSuccess) { + return status; + } + // The whole allocation is registered once so subrange transfers reuse the + // same memory region. + const auto iova = reinterpret_cast(allocation.base); + const long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0 || offset % page_size != iova % page_size) { + return ReturnStatus::kInvalidArgument; + } + region->mr = ibv_reg_dmabuf_mr(pd, offset, allocation.bytes, iova, region->fd, + IBV_ACCESS_LOCAL_WRITE); + if (!region->mr) { + const int error = errno; + std::cerr << "MetaX DMA-BUF MR registration failed: " + << std::strerror(error) << " (" << error << ").\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +class MetaxMemoryAdapter : public RdmaMemoryAdapter { + public: + ReturnStatus Prepare(void *buffer, size_t bytes, void *stream) override; + + ReturnStatus Register(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region) override { + return device_memory_ ? RegisterDeviceMemory(pd, allocation_, region) + : RegisterRdmaHostMemory(pd, buffer, bytes, region); + } + + ReturnStatus Complete(bool receive) override { + if (receive && device_memory_) { + QueryFlushPolicy(device_, &flush_); + if (flush_) { + return CheckMaca(mcDeviceFlushGPUDirectRDMAWrites( + mcFlushGPUDirectRDMAWritesTargetCurrentDevice, + mcFlushGPUDirectRDMAWritesToOwner), + "mcDeviceFlushGPUDirectRDMAWrites"); + } + } + return ReturnStatus::kSuccess; + } + + RdmaMemoryKey memory_key() const override { return key_; } + + bool CanRegister(const void *buffer, size_t bytes) const override { + (void)bytes; + if (!ExperimentalDirectEnabled()) { + return false; + } + mcPointerAttribute_t attributes{}; + const auto pointer = + reinterpret_cast(const_cast(buffer)); + if (mcPointerGetAttributes(&attributes, const_cast(buffer)) != + mcSuccess || + attributes.type != mcMemoryTypeDevice) { + return false; + } + int buffer_device = -1; + int current_device = -1; + return mcPointerGetAttribute(&buffer_device, mcPointerAttributeDevice, + pointer) == mcSuccess && + mcGetDevice(¤t_device) == mcSuccess && + buffer_device == current_device; + } + + private: + bool device_memory_ = false; + + bool flush_ = false; + + int device_ = -1; + + MetaxAllocation allocation_; + + RdmaMemoryKey key_; +}; + +ReturnStatus MetaxMemoryAdapter::Prepare(void *buffer, size_t bytes, + void *stream) { + mcPointerAttribute_t attributes{}; + auto status = CheckMaca(mcPointerGetAttributes(&attributes, buffer), + "mcPointerGetAttributes"); + if (status != ReturnStatus::kSuccess) { + return status; + } + device_memory_ = attributes.type == mcMemoryTypeDevice; + if (!device_memory_ && attributes.type != mcMemoryTypeHost && + attributes.type != mcMemoryTypeUnregistered) { + std::cerr << "Managed or array memory is not supported by MetaX RDMA.\n"; + return ReturnStatus::kNotSupported; + } + key_ = RdmaMemoryKey{reinterpret_cast(buffer), bytes, 0}; + if (device_memory_) { + if (!ExperimentalDirectEnabled()) { + std::cerr << "MetaX DeviceDirect requires explicit experimental opt-in: " + "`INFINICCL_RDMA_METAX_EXPERIMENTAL=1`.\n"; + return ReturnStatus::kNotSupported; + } + int device = 0; + status = CheckMaca(mcGetDevice(&device), "mcGetDevice"); + device_ = device; + if (status == ReturnStatus::kSuccess) { + status = ValidateDeviceBuffer(buffer, bytes, device, &allocation_); + } + if (status != ReturnStatus::kSuccess) { + return status; + } + key_ = RdmaMemoryKey{reinterpret_cast(allocation_.base), + allocation_.bytes, 0}; + } + return CheckMaca(mcStreamSynchronize(reinterpret_cast(stream)), + "mcStreamSynchronize"); +} + +} // namespace + +ReturnStatus MemoryAdapterFactory::Create( + std::unique_ptr *adapter) { + if (!adapter) { + return ReturnStatus::kInvalidArgument; + } + *adapter = std::make_unique(); + return ReturnStatus::kSuccess; +} + +} // namespace infini::ccl diff --git a/src/devices/metax/rdma_memory.h b/src/devices/metax/rdma_memory.h new file mode 100644 index 0000000..0f760e8 --- /dev/null +++ b/src/devices/metax/rdma_memory.h @@ -0,0 +1,17 @@ +#ifndef INFINI_CCL_DEVICES_METAX_RDMA_MEMORY_H_ +#define INFINI_CCL_DEVICES_METAX_RDMA_MEMORY_H_ + +#include + +#include "backends/mpi/rdma/memory_adaptor.h" + +namespace infini::ccl { + +template <> +struct MemoryAdapterFactory { + static ReturnStatus Create(std::unique_ptr *adapter); +}; + +} // namespace infini::ccl + +#endif // INFINI_CCL_DEVICES_METAX_RDMA_MEMORY_H_ diff --git a/src/devices/nvidia/rdma_memory.cc b/src/devices/nvidia/rdma_memory.cc new file mode 100644 index 0000000..13d076a --- /dev/null +++ b/src/devices/nvidia/rdma_memory.cc @@ -0,0 +1,219 @@ +#include "devices/nvidia/rdma_memory.h" + +#include +#include + +#include +#include +#include +#include + +namespace infini::ccl { +namespace { + +ReturnStatus CheckCuda(cudaError_t status, const char *operation) { + if (status != cudaSuccess) { + std::cerr << "CUDA operation `" << operation + << "` failed: " << cudaGetErrorString(status) << ".\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus CheckDriver(CUresult status, const char *operation) { + if (status != CUDA_SUCCESS) { + const char *message = "Unknown CUDA driver error"; + (void)cuGetErrorString(status, &message); + std::cerr << "CUDA operation `" << operation << "` failed: " << message + << ".\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus ValidateDeviceBuffer(void *buffer, size_t bytes, + RdmaMemoryKey *key) { + const auto pointer = reinterpret_cast(buffer); + CUdeviceptr base = 0; + size_t allocation_bytes = 0; + auto status = + CheckDriver(cuMemGetAddressRange(&base, &allocation_bytes, pointer), + "cuMemGetAddressRange"); + if (status != ReturnStatus::kSuccess) { + return status; + } + if (pointer < base || pointer - base > allocation_bytes || + bytes > allocation_bytes - (pointer - base)) { + return ReturnStatus::kInvalidArgument; + } + uint64_t buffer_id = 0; + // A failed identity query only weakens cache validation to range matching. + if (cuPointerGetAttribute(&buffer_id, CU_POINTER_ATTRIBUTE_BUFFER_ID, + pointer) != CUDA_SUCCESS) { + buffer_id = 0; + } + key->base = static_cast(base); + key->bytes = allocation_bytes; + key->tag = buffer_id; + unsigned int sync_memops = 1; + return CheckDriver( + cuPointerSetAttribute(&sync_memops, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, + pointer), + "cuPointerSetAttribute(SYNC_MEMOPS)"); +} + +ReturnStatus QueryFlushPolicy(int device, bool *flush) { + int supported = 0; + int ordering = 0; + int options = 0; + auto status = + CheckCuda(cudaDeviceGetAttribute( + &supported, cudaDevAttrGPUDirectRDMASupported, device), + "cudaDeviceGetAttribute(GPUDirectRDMASupported)"); + if (status != ReturnStatus::kSuccess) { + return status; + } + if (!supported) { + std::cerr << "The CUDA device does not report GPUDirect RDMA support.\n"; + return ReturnStatus::kNotSupported; + } + status = + CheckCuda(cudaDeviceGetAttribute( + &ordering, cudaDevAttrGPUDirectRDMAWritesOrdering, device), + "cudaDeviceGetAttribute(GPUDirectRDMAWritesOrdering)"); + if (status != ReturnStatus::kSuccess) { + return status; + } + *flush = ordering < cudaFlushGPUDirectRDMAWritesToOwner; + if (!*flush) { + return ReturnStatus::kSuccess; + } + status = CheckCuda( + cudaDeviceGetAttribute( + &options, cudaDevAttrGPUDirectRDMAFlushWritesOptions, device), + "cudaDeviceGetAttribute(GPUDirectRDMAFlushWritesOptions)"); + if (status != ReturnStatus::kSuccess) { + return status; + } + if (!(options & cudaFlushGPUDirectRDMAWritesOptionHost)) { + std::cerr << "NVIDIA RDMA requires native owner ordering or Host flush " + "support.\n"; + return ReturnStatus::kNotSupported; + } + return ReturnStatus::kSuccess; +} + +ReturnStatus RegisterDeviceMemory(ibv_pd *pd, const RdmaMemoryKey &key, + RdmaMemoryRegion *region) { + if (!pd || !region || key.base == 0 || key.bytes == 0 || region->mr || + region->fd >= 0) { + return ReturnStatus::kInvalidArgument; + } + // The whole allocation is registered once so subrange transfers reuse the + // same memory region. + region->mr = ibv_reg_mr(pd, reinterpret_cast(key.base), key.bytes, + IBV_ACCESS_LOCAL_WRITE); + if (!region->mr) { + const int error = errno; + std::cerr << "NVIDIA peer-memory MR registration failed: " + << std::strerror(error) << " (" << error + << "). Check `nvidia-peermem` and GPU/NIC topology; no Host " + "staging is used.\n"; + return ReturnStatus::kSystemError; + } + return ReturnStatus::kSuccess; +} + +class NvidiaMemoryAdapter : public RdmaMemoryAdapter { + public: + ReturnStatus Prepare(void *buffer, size_t bytes, void *stream) override; + + ReturnStatus Register(ibv_pd *pd, void *buffer, size_t bytes, + RdmaMemoryRegion *region) override { + return device_memory_ ? RegisterDeviceMemory(pd, key_, region) + : RegisterRdmaHostMemory(pd, buffer, bytes, region); + } + + ReturnStatus Complete(bool receive) override { + if (receive && device_memory_ && flush_) { + return CheckCuda(cudaDeviceFlushGPUDirectRDMAWrites( + cudaFlushGPUDirectRDMAWritesTargetCurrentDevice, + cudaFlushGPUDirectRDMAWritesToOwner), + "cudaDeviceFlushGPUDirectRDMAWrites"); + } + return ReturnStatus::kSuccess; + } + + RdmaMemoryKey memory_key() const override { return key_; } + + bool CanRegister(const void *buffer, size_t bytes) const override { + (void)bytes; + cudaPointerAttributes attributes{}; + if (cudaPointerGetAttributes(&attributes, buffer) != cudaSuccess || + attributes.type != cudaMemoryTypeDevice) { + return false; + } + int current_device = -1; + return cudaGetDevice(¤t_device) == cudaSuccess && + attributes.device == current_device; + } + + private: + bool device_memory_ = false; + + bool flush_ = false; + + RdmaMemoryKey key_; +}; + +ReturnStatus NvidiaMemoryAdapter::Prepare(void *buffer, size_t bytes, + void *stream) { + cudaPointerAttributes attributes{}; + auto status = CheckCuda(cudaPointerGetAttributes(&attributes, buffer), + "cudaPointerGetAttributes"); + if (status != ReturnStatus::kSuccess) { + return status; + } + device_memory_ = attributes.type == cudaMemoryTypeDevice; + if (!device_memory_ && attributes.type != cudaMemoryTypeHost && + attributes.type != cudaMemoryTypeUnregistered) { + std::cerr << "Managed or array memory is not supported by NVIDIA RDMA.\n"; + return ReturnStatus::kNotSupported; + } + key_ = RdmaMemoryKey{reinterpret_cast(buffer), bytes, 0}; + if (device_memory_) { + int device = -1; + status = CheckCuda(cudaGetDevice(&device), "cudaGetDevice"); + if (status != ReturnStatus::kSuccess) { + return status; + } + if (attributes.device != device) { + std::cerr + << "The RDMA buffer does not belong to the current CUDA device.\n"; + return ReturnStatus::kInvalidArgument; + } + status = ValidateDeviceBuffer(buffer, bytes, &key_); + if (status == ReturnStatus::kSuccess) { + status = QueryFlushPolicy(device, &flush_); + } + if (status != ReturnStatus::kSuccess) { + return status; + } + } + return CheckCuda( + cudaStreamSynchronize(reinterpret_cast(stream)), + "cudaStreamSynchronize"); +} + +} // namespace + +ReturnStatus MemoryAdapterFactory::Create( + std::unique_ptr *adapter) { + if (!adapter) { + return ReturnStatus::kInvalidArgument; + } + *adapter = std::make_unique(); + return ReturnStatus::kSuccess; +} + +} // namespace infini::ccl diff --git a/src/devices/nvidia/rdma_memory.h b/src/devices/nvidia/rdma_memory.h new file mode 100644 index 0000000..1be1bc6 --- /dev/null +++ b/src/devices/nvidia/rdma_memory.h @@ -0,0 +1,17 @@ +#ifndef INFINI_CCL_DEVICES_NVIDIA_RDMA_MEMORY_H_ +#define INFINI_CCL_DEVICES_NVIDIA_RDMA_MEMORY_H_ + +#include + +#include "backends/mpi/rdma/memory_adaptor.h" + +namespace infini::ccl { + +template <> +struct MemoryAdapterFactory { + static ReturnStatus Create(std::unique_ptr *adapter); +}; + +} // namespace infini::ccl + +#endif // INFINI_CCL_DEVICES_NVIDIA_RDMA_MEMORY_H_ From 33cf531b18e32525bdbaac41507442582194b868 Mon Sep 17 00:00:00 2001 From: Jinyao618 Date: Wed, 18 Nov 2026 16:59:57 +0800 Subject: [PATCH 2/2] fix: fix issues --- CMakeLists.txt | 35 +----- README.md | 8 +- scripts/gen_bridge.py | 45 ++++++- src/CMakeLists.txt | 135 ++++++++++----------- src/backends/mpi/ompi/comm_instance.h | 6 - src/backends/mpi/ompi/impl/comm_init_all.h | 2 - src/backends/mpi/ompi/impl/recv.h | 2 - src/backends/mpi/ompi/impl/send.h | 2 - src/backends/mpi/rdma/memory_adaptor.cc | 18 +-- src/backends/mpi/rdma/rdma_transport.cc | 77 ++++++++++-- src/backends/mpi/rdma/rdma_transport.h | 5 +- 11 files changed, 191 insertions(+), 144 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 30758b1..9c707a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,6 @@ set(WITH_CPU ON CACHE INTERNAL "CPU backend is always enabled") # --- BACKEND OPTIONS (Communication Protocols) --- # ========================================================= option(WITH_OMPI "Enable OpenMPI backend" OFF) -option(WITH_RDMA "Enable RDMA backend" OFF) option(WITH_MPICH "Enable MPICH backend" OFF) option(WITH_NCCL "Enable NCCL backend" OFF) option(WITH_MCCL "Enable MCCL backend" OFF) @@ -117,29 +116,6 @@ if(AUTO_DETECT_BACKENDS) message(STATUS "Auto-detected MPICH/Hydra environment.") endif() - # Detect the RDMA transport only when it can attach to an MPI backend. - if(WITH_OMPI OR WITH_MPICH) - if(NOT WITH_RDMA) - find_program(RDMA_DEVINFO_PATH ibv_devinfo) - find_path(AUTO_IBVERBS_INCLUDE_DIR NAMES infiniband/verbs.h QUIET) - find_library(AUTO_IBVERBS_LIBRARY NAMES ibverbs QUIET) - - if(RDMA_DEVINFO_PATH AND AUTO_IBVERBS_INCLUDE_DIR AND - AUTO_IBVERBS_LIBRARY) - execute_process( - COMMAND ${RDMA_DEVINFO_PATH} - RESULT_VARIABLE RDMA_DEVINFO_RESULT - OUTPUT_QUIET - ERROR_QUIET - ) - if(RDMA_DEVINFO_RESULT EQUAL 0) - set(WITH_RDMA ON) - message(STATUS "Auto-detected RDMA environment.") - endif() - endif() - endif() - endif() - # Detect NCCL Dependencies if(WITH_NVIDIA OR WITH_THEAD OR WITH_ILUVATAR OR WITH_HYGON) infiniccl_probe_nccl(NCCL_FOUND) @@ -195,12 +171,6 @@ if(NOT WITH_OMPI AND NOT WITH_MPICH AND NOT WITH_NCCL AND NOT WITH_MCCL AND NOT message(STATUS "No backend specified or detected. Defaulting to `WITH_OMPI=ON`") endif() -# RDMA is an MPI transport attribute, not a standalone communication backend. -if(WITH_RDMA AND NOT WITH_OMPI AND NOT WITH_MPICH) - set(WITH_RDMA OFF) - message(STATUS "RDMA is ignored because no MPI backend is enabled.") -endif() - # ========================================================= # --- DEPENDENCY DISCOVERY --- # ========================================================= @@ -246,11 +216,10 @@ endif() if(WITH_OMPI OR WITH_MPICH) find_package(MPI REQUIRED) -endif() - -if(WITH_RDMA) find_path(IBVERBS_INCLUDE_DIR NAMES infiniband/verbs.h REQUIRED) find_library(IBVERBS_LIBRARY NAMES ibverbs REQUIRED) + find_path(MLX5DV_INCLUDE_DIR NAMES infiniband/mlx5dv.h REQUIRED) + find_library(MLX5DV_LIBRARY NAMES mlx5 REQUIRED) find_package(Threads REQUIRED) endif() diff --git a/README.md b/README.md index c49fe24..0977cdc 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,6 @@ cmake .. -DWITH_NVIDIA=ON -DWITH_OMPI=ON | `WITH_MCCL` | Enable MCCL backend | `OFF` | | `WITH_CNCL` | Enable CNCL backend | `OFF` | | `WITH_HCCL` | Enable HCCL backend | `OFF` | -| `WITH_RDMA` | Enable the RDMA transport attribute of the MPI backends; requires `libibverbs` and one of the MPI backends (`WITH_OMPI` or `WITH_MPICH`), otherwise the option is ignored | `OFF` | | **Miscellaneous** ||| | `AUTO_DETECT_DEVICES` | Automatically detect available devices and enable corresponding support | `ON` | | `AUTO_DETECT_BACKENDS` | Automatically detect available communication backends and enable corresponding support | `OFF` | @@ -367,10 +366,15 @@ export LD_LIBRARY_PATH=${INFINI_INSTALL}/lib:$LD_LIBRARY_PATH | **MCCL** | Partial | `WITH_MCCL=ON` | Requires MetaX or Moore MCCL. Currently available when `WITH_METAX=ON` or `WITH_MOORE=ON`.| | **CNCL** | Partial | `WITH_CNCL=ON` | Requires Cambricon CNCL. Available only when `WITH_CAMBRICON=ON`.| | **HCCL** | Partial | `WITH_HCCL=ON` | Requires Ascend CANN, the ACLNN division header and `libopapi`. Available only when `WITH_ASCEND=ON`.| -| **RDMA (transport attribute)** | Experimental | `WITH_RDMA=ON` + (`WITH_OMPI=ON` or `WITH_MPICH=ON`) | Requires Linux and `libibverbs`. Device-memory `Send`/`Recv` between two ranks goes over verbs RC when both peers support it (NVIDIA peer-memory: CUDA ≥ 11.3 and compatible `nvidia-peermem`; MetaX DMA-BUF: MACA/mxkw and explicit opt-in); every other message automatically falls back to the existing host staging path. The existing MPI examples exercise it without changes; communicator teardown prints per-path statistics. Messages are limited to 1 MiB and negotiated port limits.| + | **RDMA (transport attribute)** | Experimental | `WITH_OMPI=ON` or `WITH_MPICH=ON`, plus `INFINICCL_RDMA_MODE=auto` | Requires Linux, `libibverbs`, and `libmlx5` with Scatter-to-CQE control. The runtime mode defaults to `staging`; `auto` enables device-memory `Send`/`Recv` over verbs RC when both peers support it (NVIDIA peer-memory: CUDA ≥ 11.3 and compatible `nvidia-peermem`; MetaX DMA-BUF: MACA/mxkw and explicit opt-in). Every unsupported message automatically falls back to the existing host staging path. The existing MPI examples exercise it without changes; communicator teardown prints per-path statistics. Messages are limited to the NIC message-size limit negotiated between peers.| +`INFINICCL_RDMA_MODE` controls only the InfiniCCL verbs data path. Set it to +`auto` to enable RDMA selection or `staging` to use the MPI host-staging path; +`staging` is the current default. This variable does not control RDMA usage +inside OpenMPI or UCX. + ## License InfiniCCL is distributed under the [LICENSE](LICENSE) file in the root of the repository. diff --git a/scripts/gen_bridge.py b/scripts/gen_bridge.py index a472c05..b763357 100644 --- a/scripts/gen_bridge.py +++ b/scripts/gen_bridge.py @@ -50,6 +50,8 @@ "hccl": "backends/ccl/hccl", } +MPI_RDMA_BACKENDS = {"ompi", "mpich"} + # ================================================================= # LOGIC # ================================================================= @@ -179,7 +181,48 @@ def generate(project_root, output_dir, devices, backends): with open(os.path.join(output_dir, "backend_manifest.h"), "w") as f: f.write("\n".join(manifest_lines)) - # 2. Generate `comm_bridge.cc`. + # 2. Generate `rdma_manifest.h`. + rdma_lines = [ + AUTOGEN_HEADER, + "#ifndef INFINI_CCL_RDMA_MANIFEST_H_", + "#define INFINI_CCL_RDMA_MANIFEST_H_", + "", + '#include "traits.h"', + ] + + rdma_devices = [] + + rdma_enabled = any(bb in MPI_RDMA_BACKENDS for bb in backends) + if rdma_enabled: + rdma_lines.append("\n// --- RDMA MEMORY ADAPTERS ---") + + for dev in devices: + if not dev: + continue + + rel_path = f"devices/{dev}/rdma_memory.h" + if os.path.exists(os.path.join(src_dir, rel_path)): + rdma_lines.append(f'#include "{rel_path}"') + rdma_devices.append(f"Device::Type::k{dev.capitalize()}") + + def device_list(values): + return ", ".join(values) if values else "" + + rdma_lines.extend( + [ + "\nnamespace infini::ccl {\n", + f"using EnabledRdmaAdapters = List<{device_list(rdma_devices)}>;", + "using RdmaDispatchDevices = " + f"List<{device_list(found_devices)}>;", + "\n} // namespace infini::ccl", + "\n#endif // INFINI_CCL_RDMA_MANIFEST_H_\n", + ] + ) + + with open(os.path.join(output_dir, "rdma_manifest.h"), "w") as f: + f.write("\n".join(rdma_lines)) + + # 3. Generate `comm_bridge.cc`. bridge_lines = [ AUTOGEN_HEADER, '#include "backend_manifest.h"', diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f2f7506..15b6e45 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -286,77 +286,63 @@ if(WITH_OMPI) endif() # RDMA (transport attribute of the MPI backends) -if(WITH_RDMA) - if(WITH_OMPI OR WITH_MPICH) - # The transport exchanges its control frames and queue-pair metadata - # over MPI, so it only builds together with an MPI backend. - target_sources(infiniccl PRIVATE - backends/mpi/rdma/rdma_transport.cc - backends/mpi/rdma/memory_adaptor.cc - devices/cpu/rdma_memory.cc - ) - target_include_directories(infiniccl PRIVATE - ${IBVERBS_INCLUDE_DIR} - ) - target_link_libraries(infiniccl PRIVATE - ${IBVERBS_LIBRARY} - Threads::Threads - ) - if(TARGET MPI::MPI_CXX) - target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX) - endif() - target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_RDMA) - find_path(MLX5DV_INCLUDE_DIR NAMES infiniband/mlx5dv.h) - find_library(MLX5DV_LIBRARY NAMES mlx5) - if(MLX5DV_INCLUDE_DIR AND MLX5DV_LIBRARY) - include(CMakePushCheckState) - include(CheckCXXSourceCompiles) - cmake_push_check_state(RESET) - set(CMAKE_REQUIRED_INCLUDES ${MLX5DV_INCLUDE_DIR} ${IBVERBS_INCLUDE_DIR}) - set(CMAKE_REQUIRED_LIBRARIES ${MLX5DV_LIBRARY} ${IBVERBS_LIBRARY}) - check_cxx_source_compiles(" - #include - int main() { - ibv_qp_init_attr_ex attr{}; - mlx5dv_qp_init_attr mlx5_attr{}; - attr.comp_mask = IBV_QP_INIT_ATTR_PD; - mlx5_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS; - mlx5_attr.create_flags = MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; - auto *qp = mlx5dv_create_qp(nullptr, &attr, &mlx5_attr); - return mlx5dv_is_supported(nullptr) && qp ? 0 : 1; - } - " INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) - cmake_pop_check_state() - endif() - if(MLX5DV_INCLUDE_DIR AND MLX5DV_LIBRARY AND - INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) - target_include_directories(infiniccl PRIVATE ${MLX5DV_INCLUDE_DIR}) - target_link_libraries(infiniccl PRIVATE ${MLX5DV_LIBRARY}) - target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_MLX5DV) - message(STATUS "RDMA: per-QP mlx5 Scatter-to-CQE disabling enabled.") - else() - message(STATUS "RDMA: mlx5dv unavailable; Mellanox QPs will be rejected for safety.") - endif() - if(WITH_NVIDIA) - if(CUDAToolkit_VERSION VERSION_LESS 11.3) - message(FATAL_ERROR "NVIDIA RDMA requires CUDA 11.3 or newer for visibility capability checks.") - endif() - target_sources(infiniccl PRIVATE devices/nvidia/rdma_memory.cc) - target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_NVIDIA_RDMA) - endif() - if(WITH_METAX) - find_path(METAX_RDMA_INCLUDE_DIR NAMES mxkw/mxkw.h - HINTS "${MACA_PATH}/include" REQUIRED) - find_library(METAX_RDMA_LIBRARY NAMES mxkw - HINTS "${MACA_PATH}/lib" "${MACA_PATH}/lib64" REQUIRED) - target_sources(infiniccl PRIVATE devices/metax/rdma_memory.cc) - target_include_directories(infiniccl PRIVATE ${METAX_RDMA_INCLUDE_DIR}) - target_link_libraries(infiniccl PRIVATE ${METAX_RDMA_LIBRARY}) - target_compile_definitions(infiniccl PRIVATE INFINICCL_WITH_METAX_RDMA) +if(WITH_OMPI OR WITH_MPICH) + target_sources(infiniccl PRIVATE + backends/mpi/rdma/rdma_transport.cc + backends/mpi/rdma/memory_adaptor.cc + devices/cpu/rdma_memory.cc + ) + target_include_directories(infiniccl PRIVATE + ${IBVERBS_INCLUDE_DIR} + ${MLX5DV_INCLUDE_DIR} + ) + target_link_libraries(infiniccl PRIVATE + ${IBVERBS_LIBRARY} + ${MLX5DV_LIBRARY} + Threads::Threads + ) + if(TARGET MPI::MPI_CXX) + target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX) + endif() + + include(CMakePushCheckState) + include(CheckCXXSourceCompiles) + cmake_push_check_state(RESET) + set(CMAKE_REQUIRED_INCLUDES ${MLX5DV_INCLUDE_DIR} ${IBVERBS_INCLUDE_DIR}) + set(CMAKE_REQUIRED_LIBRARIES ${MLX5DV_LIBRARY} ${IBVERBS_LIBRARY}) + check_cxx_source_compiles(" + #include + int main() { + ibv_qp_init_attr_ex attr{}; + mlx5dv_qp_init_attr mlx5_attr{}; + attr.comp_mask = IBV_QP_INIT_ATTR_PD; + mlx5_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS; + mlx5_attr.create_flags = MLX5DV_QP_CREATE_DISABLE_SCATTER_TO_CQE; + auto *qp = mlx5dv_create_qp(nullptr, &attr, &mlx5_attr); + return mlx5dv_is_supported(nullptr) && qp ? 0 : 1; + } + " INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) + cmake_pop_check_state() + if(NOT INFINICCL_HAS_MLX5DV_SCATTER_CONTROL) + message(FATAL_ERROR + "MPI RDMA requires mlx5dv Scatter-to-CQE control support.") + endif() + + if(WITH_NVIDIA) + if(CUDAToolkit_VERSION VERSION_LESS 11.3) + message(FATAL_ERROR + "NVIDIA RDMA requires CUDA 11.3 or newer for visibility capability checks.") endif() - else() - set(WITH_RDMA OFF PARENT_SCOPE) - message(STATUS "RDMA: the transport requires an MPI backend (WITH_OMPI or WITH_MPICH); it is disabled.") + target_sources(infiniccl PRIVATE devices/nvidia/rdma_memory.cc) + endif() + if(WITH_METAX) + find_path(METAX_RDMA_INCLUDE_DIR NAMES mxkw/mxkw.h + HINTS "${MACA_PATH}/include" REQUIRED) + find_library(METAX_RDMA_LIBRARY NAMES mxkw + HINTS "${MACA_PATH}/lib" "${MACA_PATH}/lib64" REQUIRED) + target_sources(infiniccl PRIVATE devices/metax/rdma_memory.cc) + target_include_directories(infiniccl PRIVATE ${METAX_RDMA_INCLUDE_DIR}) + target_link_libraries(infiniccl PRIVATE ${METAX_RDMA_LIBRARY}) endif() endif() @@ -438,11 +424,13 @@ file(WRITE "${PUBLIC_HEADERS_DIR}/infiniccl.h" "${INFINICCL_H_CONTENT}") # --- Generate the bridge files --- set(GENERATED_BRIDGE "${CMAKE_CURRENT_BINARY_DIR}/comm_bridge.cc") set(GENERATED_MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/backend_manifest.h") +set(GENERATED_RDMA_MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/rdma_manifest.h") set(DEV_STR "${DEVICE_LIST}") set(BACK_STR "${BACKEND_LIST}") add_custom_command( OUTPUT "${GENERATED_BRIDGE}" "${GENERATED_MANIFEST}" + "${GENERATED_RDMA_MANIFEST}" COMMAND ${Python3_EXECUTABLE} "${PROJECT_SOURCE_DIR}/scripts/gen_bridge.py" "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" @@ -454,10 +442,13 @@ add_custom_command( ${BASE_IMPL_SRCS} ${BRIDGE_DEP_HEADERS} VERBATIM - COMMENT "Generating InfiniCCL bridge and manifest files for Devices: [${DEV_STR}] Backends: [${BACK_STR}]..." + COMMENT "Generating InfiniCCL bridge and manifests for Devices: [${DEV_STR}] Backends: [${BACK_STR}]..." ) -target_sources(infiniccl PRIVATE "${GENERATED_BRIDGE}") +target_sources(infiniccl PRIVATE + "${GENERATED_BRIDGE}" + "${GENERATED_RDMA_MANIFEST}" +) target_include_directories(infiniccl PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") # ========================================================= diff --git a/src/backends/mpi/ompi/comm_instance.h b/src/backends/mpi/ompi/comm_instance.h index f2df9ab..b388c77 100644 --- a/src/backends/mpi/ompi/comm_instance.h +++ b/src/backends/mpi/ompi/comm_instance.h @@ -3,9 +3,7 @@ #include -#if defined(INFINICCL_WITH_RDMA) #include "backends/mpi/rdma/rdma_transport.h" -#endif #include "checks.h" #include "communicator.h" @@ -15,17 +13,13 @@ namespace infini::ccl { struct OmpiInstance : public BackendCommInstance { MPI_Comm handle = MPI_COMM_NULL; -#if defined(INFINICCL_WITH_RDMA) RdmaTransport rdma_transport; -#endif OmpiInstance() { type = BackendType::kOmpi; } ~OmpiInstance() override { Destroy(); } void Destroy() { -#if defined(INFINICCL_WITH_RDMA) rdma_transport.Destroy(); -#endif if (handle != MPI_COMM_WORLD && handle != MPI_COMM_NULL) { INFINI_CHECK_MPI(MPI_Comm_free(&handle)); handle = MPI_COMM_NULL; diff --git a/src/backends/mpi/ompi/impl/comm_init_all.h b/src/backends/mpi/ompi/impl/comm_init_all.h index a1379d7..16c34cd 100644 --- a/src/backends/mpi/ompi/impl/comm_init_all.h +++ b/src/backends/mpi/ompi/impl/comm_init_all.h @@ -33,11 +33,9 @@ class CommInitAllImpl { comm->set_world_info(rank, size); comm->set_local_size(n_dev); -#if defined(INFINICCL_WITH_RDMA) // The RDMA transport is an attribute of this backend: it initializes // collectively and degrades to host staging when any rank does not support RDMA. inst->rdma_transport.Initialize(rank, size, inst->handle); -#endif comm->set_inter_comm(std::move(inst)); diff --git a/src/backends/mpi/ompi/impl/recv.h b/src/backends/mpi/ompi/impl/recv.h index 321a075..139f38a 100644 --- a/src/backends/mpi/ompi/impl/recv.h +++ b/src/backends/mpi/ompi/impl/recv.h @@ -36,13 +36,11 @@ class RecvImpl { size_t total_bytes = count * type_size; -#if defined(INFINICCL_WITH_RDMA) if (inst->rdma_transport.SupportRdma(recv_buff, total_bytes, peer, device_type, stream, /*send=*/false)) { return inst->rdma_transport.Recv(recv_buff, total_bytes, peer); } -#endif void *host_buf = std::malloc(total_bytes); if (!host_buf) { diff --git a/src/backends/mpi/ompi/impl/send.h b/src/backends/mpi/ompi/impl/send.h index 7a6bce0..0583e28 100644 --- a/src/backends/mpi/ompi/impl/send.h +++ b/src/backends/mpi/ompi/impl/send.h @@ -37,14 +37,12 @@ class SendImpl { size_t total_bytes = count * type_size; -#if defined(INFINICCL_WITH_RDMA) auto *send_buffer = const_cast(send_buff); if (inst->rdma_transport.SupportRdma(send_buffer, total_bytes, peer, device_type, stream, /*send=*/true)) { return inst->rdma_transport.Send(send_buffer, total_bytes, peer, stream); } -#endif void *host_buf = std::malloc(total_bytes); if (!host_buf) { diff --git a/src/backends/mpi/rdma/memory_adaptor.cc b/src/backends/mpi/rdma/memory_adaptor.cc index e49b976..eb64cdd 100644 --- a/src/backends/mpi/rdma/memory_adaptor.cc +++ b/src/backends/mpi/rdma/memory_adaptor.cc @@ -9,18 +9,9 @@ #include #include -#include "devices/cpu/rdma_memory.h" - -#if defined(INFINICCL_WITH_METAX_RDMA) -#include "devices/metax/rdma_memory.h" -#endif - -#if defined(INFINICCL_WITH_NVIDIA_RDMA) -#include "devices/nvidia/rdma_memory.h" -#endif - #include "dispatcher.h" #include "logging.h" +#include "rdma_manifest.h" namespace infini::ccl { @@ -195,11 +186,14 @@ ReturnStatus CreateRdmaMemoryAdapter( if (device < Device::Type::kCpu || device >= Device::Type::kCount) { return ReturnStatus::kNotSupported; } - return DispatchFunc( + return DispatchFunc( device, [&](auto tag) { constexpr Device::Type kDevice = decltype(tag)::value; - return MemoryAdapterFactory::Create(adapter); + if constexpr (ContainsValue) { + return MemoryAdapterFactory::Create(adapter); + } + return ReturnStatus::kNotSupported; }, "CreateRdmaMemoryAdapter"); } diff --git a/src/backends/mpi/rdma/rdma_transport.cc b/src/backends/mpi/rdma/rdma_transport.cc index 540ba93..1d5df2b 100644 --- a/src/backends/mpi/rdma/rdma_transport.cc +++ b/src/backends/mpi/rdma/rdma_transport.cc @@ -5,9 +5,7 @@ #include #include -#if defined(INFINICCL_WITH_MLX5DV) #include -#endif #include #include @@ -16,6 +14,7 @@ #include #include #include +#include #include #include @@ -30,6 +29,9 @@ constexpr size_t kGroupIdBytes = 16; constexpr uint32_t kMellanoxVendorId = 0x02c9; +// A verbs scatter-gather entry carries its byte length as a 32-bit value. +constexpr size_t kMaxSgeBytes = std::numeric_limits::max(); + // Control frames ride on a dedicated MPI tag so they never clash with the // staged payload messages of the MPI backends (tag 0). constexpr int kControlTag = 777; @@ -60,7 +62,8 @@ static_assert(sizeof(PeerMetadata) == 8 * sizeof(uint32_t) + 16 + kGroupIdBytes, "`PeerMetadata` must not contain padding."); enum class StagedReason : size_t { - kNoLink = 0, + kDisabled = 0, + kNoLink, kNoAdapter, kBuffer, kSizeLimit, @@ -68,6 +71,25 @@ enum class StagedReason : size_t { kCount }; +enum class RdmaRuntimeMode { + kInvalid, + kStaging, + kAuto, +}; + +// Experimental env for example tests. +// `Staging` represents host-staging and `auto` means attempting RDMA first. +RdmaRuntimeMode ReadRuntimeMode() { + const char *const value = std::getenv("INFINICCL_RDMA_MODE"); + if (!value || std::strcmp(value, "staging") == 0) { + return RdmaRuntimeMode::kStaging; + } + if (std::strcmp(value, "auto") == 0) { + return RdmaRuntimeMode::kAuto; + } + return RdmaRuntimeMode::kInvalid; +} + enum class ControlKind : uint8_t { kStaged, kPropose, @@ -169,7 +191,6 @@ ReturnStatus SelectDevice(ibv_context **context, ibv_port_attr *port_attr, return ReturnStatus::kSuccess; } -#if defined(INFINICCL_WITH_MLX5DV) ReturnStatus CreateMlx5QueuePair(const ibv_qp_init_attr &base, ibv_pd *pd, ibv_context *context, ibv_qp **qp) { ibv_qp_init_attr_ex attr{}; @@ -191,15 +212,12 @@ ReturnStatus CreateMlx5QueuePair(const ibv_qp_init_attr &base, ibv_pd *pd, } return ReturnStatus::kSuccess; } -#endif ReturnStatus CreateCompatibleQueuePair(const ibv_qp_init_attr &attr, ibv_pd *pd, ibv_context *context, ibv_qp **qp) { -#if defined(INFINICCL_WITH_MLX5DV) if (mlx5dv_is_supported(context->device)) { return CreateMlx5QueuePair(attr, pd, context, qp); } -#endif ibv_device_attr device_attr{}; const int query = ibv_query_device(context, &device_attr); if (query != 0) { @@ -331,6 +349,8 @@ struct RdmaTransport::Impl { bool destroyed = false; + bool mode_disabled = false; + uint64_t rdma_ops = 0; uint64_t staged_ops = 0; @@ -529,7 +549,12 @@ bool PrepareLocalResources(RdmaTransport::Impl &impl) { qp_status != ReturnStatus::kSuccess) { return false; } - impl.max_bytes = std::min(impl.port_attr.max_msg_sz, 1 << 20); + const size_t max_message_bytes = + static_cast(impl.port_attr.max_msg_sz); + if (max_message_bytes == 0 || max_message_bytes > kMaxSgeBytes) { + return false; + } + impl.max_bytes = max_message_bytes; return true; } @@ -677,7 +702,9 @@ void RdmaTransport::Impl::DestroyLocked() { } if (rdma_ops > 0 || staged_ops > 0) { std::cout << "[Rank " << rank << "] RDMA transport: rdma=" << rdma_ops - << " staged=" << staged_ops << " (no_link=" + << " staged=" << staged_ops << " (disabled=" + << fallbacks[static_cast(StagedReason::kDisabled)] + << ", no_link=" << fallbacks[static_cast(StagedReason::kNoLink)] << ", no_adapter=" << fallbacks[static_cast(StagedReason::kNoAdapter)] @@ -721,6 +748,31 @@ void RdmaTransport::Initialize(int rank, int nranks, MPI_Comm comm) { impl.rank = rank; impl.comm = comm; impl.destroyed = false; + impl.mode_disabled = false; + + int local_auto = 0; + const RdmaRuntimeMode mode = ReadRuntimeMode(); + if (mode == RdmaRuntimeMode::kAuto) { + local_auto = 1; + } else if (mode == RdmaRuntimeMode::kInvalid) { + std::cerr << "Invalid `INFINICCL_RDMA_MODE`; expected `auto` or " + "`staging`.\n"; + } + int all_auto = 0; + if (MPI_Allreduce(&local_auto, &all_auto, 1, MPI_INT, MPI_MIN, comm) != + MPI_SUCCESS) { + impl.mode_disabled = true; + LOG("Failed to agree on the RDMA runtime mode."); + return; + } + if (all_auto != 1) { + impl.mode_disabled = true; + std::cerr << "[Rank " << rank + << "] RDMA transport uses MPI host staging " + "(`INFINICCL_RDMA_MODE=staging`).\n"; + return; + } + if (nranks != 2) { // Temporarily supports two ranks. return; @@ -764,7 +816,12 @@ bool RdmaTransport::SupportRdma(void *buffer, size_t bytes, int peer, std::lock_guard lock(impl_->mutex); impl_->adapter.reset(); impl_->pending_mr = nullptr; - if (!impl_->active || peer != 1 - impl_->rank) { + if (!impl_->active) { + impl_->RecordFallback(impl_->mode_disabled ? StagedReason::kDisabled + : StagedReason::kNoLink); + return false; + } + if (peer != 1 - impl_->rank) { impl_->RecordFallback(StagedReason::kNoLink); return false; } diff --git a/src/backends/mpi/rdma/rdma_transport.h b/src/backends/mpi/rdma/rdma_transport.h index 1e04218..38339f4 100644 --- a/src/backends/mpi/rdma/rdma_transport.h +++ b/src/backends/mpi/rdma/rdma_transport.h @@ -35,8 +35,9 @@ class RdmaTransport { // Creates the verbs resources and exchanges QP metadata with the peer over // `comm`. The transport stays inert (`SupportRdma` always returns false) - // when the rank count is not two or no RDMA device is usable, so callers - // keep falling back to host staging without extra branching. + // when the runtime mode is not `auto`, the rank count is not two, or no RDMA + // device is usable, callers keep falling back to host staging without + // extra branching. void Initialize(int rank, int nranks, MPI_Comm comm); // End-to-end path decision for one message. The send side runs the local