From 8c2505b1ba39eba0d02fa41b86d9ae9e56d583ef Mon Sep 17 00:00:00 2001 From: Jinyao618 Date: Fri, 20 Nov 2026 17:03:33 +0800 Subject: [PATCH] feat: add mpi device-aware send-recv implementation --- CMakeLists.txt | 5 + cmake/backends/MPI.cmake | 35 +++++ src/CMakeLists.txt | 5 + src/backends/mpi/ompi/buffer_plan.h | 173 +++++++++++++++++++++ src/backends/mpi/ompi/comm_instance.h | 3 + src/backends/mpi/ompi/device_support.h | 94 +++++++++++ src/backends/mpi/ompi/impl/comm_init_all.h | 15 +- src/backends/mpi/ompi/impl/recv.h | 27 ++-- src/backends/mpi/ompi/impl/send.h | 26 ++-- src/devices/metax/device_.h | 7 +- src/devices/nvidia/device_.h | 7 +- 11 files changed, 360 insertions(+), 37 deletions(-) create mode 100644 src/backends/mpi/ompi/buffer_plan.h create mode 100644 src/backends/mpi/ompi/device_support.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 918d596..61d45ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -218,6 +218,11 @@ if(WITH_OMPI OR WITH_MPICH) find_package(MPI REQUIRED) endif() +set(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS "") +if(WITH_OMPI) + infiniccl_configure_mpi_device_aware(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS) +endif() + if(WITH_NCCL) infiniccl_configure_nccl(NCCL_INC NCCL_LIB NCCL_COMPILE_DEFINITIONS) include_directories(${NCCL_INC}) diff --git a/cmake/backends/MPI.cmake b/cmake/backends/MPI.cmake index db6c365..920135a 100644 --- a/cmake/backends/MPI.cmake +++ b/cmake/backends/MPI.cmake @@ -27,3 +27,38 @@ function(infiniccl_detect_mpi output) set(${output} "${_implementation}" PARENT_SCOPE) endfunction() + +function(infiniccl_configure_mpi_device_aware output) + if(NOT WITH_OMPI) + set(${output} "" PARENT_SCOPE) + return() + endif() + + include(CheckCXXSourceCompiles) + include(CMakePushCheckState) + cmake_push_check_state(RESET) + + set(CMAKE_REQUIRED_INCLUDES ${MPI_CXX_INCLUDE_DIRS}) + set(CMAKE_REQUIRED_LIBRARIES ${MPI_CXX_LIBRARIES}) + if(MPI_CXX_COMPILE_FLAGS) + set(CMAKE_REQUIRED_FLAGS ${MPI_CXX_COMPILE_FLAGS}) + endif() + if(MPI_CXX_LINK_FLAGS) + set(CMAKE_REQUIRED_LINK_OPTIONS ${MPI_CXX_LINK_FLAGS}) + endif() + + check_cxx_source_compiles(" + #include + #include + int main() { + return MPIX_Query_cuda_support(); + } + " INFINICCL_MPI_HAS_OMPI_CUDA_QUERY) + cmake_pop_check_state() + + if(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY) + set(${output} "INFINICCL_MPI_HAS_OMPI_CUDA_QUERY=1" PARENT_SCOPE) + else() + set(${output} "" PARENT_SCOPE) + endif() +endfunction() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index affb03f..480fdfa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -280,6 +280,11 @@ if(WITH_OMPI) if(TARGET MPI::MPI_CXX) target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX) endif() + + if(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS) + target_compile_definitions(infiniccl PRIVATE + ${INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS}) + endif() endif() if(WITH_MPICH) diff --git a/src/backends/mpi/ompi/buffer_plan.h b/src/backends/mpi/ompi/buffer_plan.h new file mode 100644 index 0000000..5f48e29 --- /dev/null +++ b/src/backends/mpi/ompi/buffer_plan.h @@ -0,0 +1,173 @@ +#ifndef INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_ +#define INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_ + +#include +#include + +#include "runtime.h" + +namespace infini::ccl { + +namespace detail { + +using HostBuffer = std::unique_ptr; + +inline HostBuffer AllocateHostBuffer(std::size_t bytes) { + return HostBuffer(std::malloc(bytes), &std::free); +} + +template +bool IsDeviceBuffer(const void *ptr) { + return ptr != nullptr && GetMemorySpace(ptr) == MemorySpace::kDevice; +} + +} // namespace detail + +template +class MpiBufferPlan { + public: + enum class Mode { + kPair, + kAlias, + }; + + ReturnStatus Prepare(const void *send_buff, void *recv_buff, + std::size_t send_bytes, std::size_t recv_bytes, + bool allow_direct, void *stream, + Mode mode = Mode::kPair) { + const bool send_active = send_bytes != 0; + const bool recv_active = recv_bytes != 0; + if (!send_active && !recv_active) { + return ReturnStatus::kSuccess; + } + + user_recv_buff_ = recv_buff; + recv_bytes_ = recv_bytes; + + const bool send_device = + send_active && detail::IsDeviceBuffer(send_buff); + const bool recv_device = + recv_active && detail::IsDeviceBuffer(recv_buff); + recv_is_device_ = recv_device; + + if (mode == Mode::kPair) { + return PreparePair(send_buff, recv_buff, send_bytes, recv_bytes, + allow_direct, stream, send_active, recv_active, + send_device, recv_device); + } + + return PrepareAlias(recv_buff, recv_bytes, allow_direct, stream, + send_active, recv_device); + } + + ReturnStatus Commit() { + if (!host_recv_buff_) { + return ReturnStatus::kSuccess; + } + + CHECK_STATUS( + Runtime, + Runtime::Memcpy(user_recv_buff_, host_recv_buff_.get(), + recv_bytes_, Runtime::MemcpyHostToDevice)); + return ReturnStatus::kSuccess; + } + + const void *send_buff() const { return mpi_send_buff_; } + + void *recv_buff() const { return mpi_recv_buff_; } + + bool direct() const { return direct_; } + + private: + ReturnStatus PreparePair(const void *send_buff, void *recv_buff, + std::size_t send_bytes, std::size_t recv_bytes, + bool allow_direct, void *stream, bool send_active, + bool recv_active, bool send_device, + bool recv_device) { + direct_ = allow_direct && (!send_active || send_device) && + (!recv_active || recv_device); + if (direct_) { + if (send_device) { + CHECK_STATUS(Runtime, + Runtime::StreamSynchronize( + static_cast::Stream>(stream))); + } + + mpi_send_buff_ = send_buff; + mpi_recv_buff_ = recv_buff; + return ReturnStatus::kSuccess; + } + + if (send_device) { + CHECK_STATUS(Runtime, + Runtime::StreamSynchronize( + static_cast::Stream>(stream))); + host_send_buff_ = detail::AllocateHostBuffer(send_bytes); + if (!host_send_buff_) { + return ReturnStatus::kSystemError; + } + + CHECK_STATUS( + Runtime, + Runtime::Memcpy(host_send_buff_.get(), send_buff, send_bytes, + Runtime::MemcpyDeviceToHost)); + mpi_send_buff_ = host_send_buff_.get(); + } else { + mpi_send_buff_ = send_buff; + } + + if (recv_device) { + host_recv_buff_ = detail::AllocateHostBuffer(recv_bytes); + if (!host_recv_buff_) { + return ReturnStatus::kSystemError; + } + mpi_recv_buff_ = host_recv_buff_.get(); + } else { + mpi_recv_buff_ = recv_buff; + } + + return ReturnStatus::kSuccess; + } + + ReturnStatus PrepareAlias(void *recv_buff, std::size_t recv_bytes, + bool allow_direct, void *stream, bool send_active, + bool recv_device) { + direct_ = allow_direct && recv_device && !send_active; + if (direct_) { + CHECK_STATUS(Runtime, + Runtime::StreamSynchronize( + static_cast::Stream>(stream))); + mpi_recv_buff_ = recv_buff; + return ReturnStatus::kSuccess; + } + + if (recv_device) { + CHECK_STATUS(Runtime, + Runtime::StreamSynchronize( + static_cast::Stream>(stream))); + host_recv_buff_ = detail::AllocateHostBuffer(recv_bytes); + if (!host_recv_buff_) { + return ReturnStatus::kSystemError; + } + + mpi_recv_buff_ = host_recv_buff_.get(); + } else { + mpi_recv_buff_ = recv_buff; + } + + return ReturnStatus::kSuccess; + } + + void *user_recv_buff_ = nullptr; + const void *mpi_send_buff_ = nullptr; + void *mpi_recv_buff_ = nullptr; + detail::HostBuffer host_send_buff_{nullptr, &std::free}; + detail::HostBuffer host_recv_buff_{nullptr, &std::free}; + std::size_t recv_bytes_ = 0; + bool recv_is_device_ = false; + bool direct_ = false; +}; + +} // namespace infini::ccl + +#endif // INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_ diff --git a/src/backends/mpi/ompi/comm_instance.h b/src/backends/mpi/ompi/comm_instance.h index d0f3a9a..5193b4b 100644 --- a/src/backends/mpi/ompi/comm_instance.h +++ b/src/backends/mpi/ompi/comm_instance.h @@ -5,12 +5,15 @@ #include "checks.h" #include "communicator.h" +#include "device_support.h" namespace infini::ccl { struct OmpiInstance : public BackendCommInstance { MPI_Comm handle = MPI_COMM_NULL; + MpiDeviceSupport device_support = MpiDeviceSupport::kHostStaged; + OmpiInstance() { type = BackendType::kOmpi; } ~OmpiInstance() override { Destroy(); } diff --git a/src/backends/mpi/ompi/device_support.h b/src/backends/mpi/ompi/device_support.h new file mode 100644 index 0000000..0920dd4 --- /dev/null +++ b/src/backends/mpi/ompi/device_support.h @@ -0,0 +1,94 @@ +#ifndef INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_ +#define INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_ + +#include +#include +#include + +#if defined(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY) +#include +#include +#endif + +#include "device.h" + +namespace infini::ccl { + +enum class MpiDeviceSupport : std::uint8_t { + kHostStaged = 0, + kDeviceAware = 1, +}; + +enum class MpiDeviceAwareMode : std::uint8_t { + kAuto, + kOn, + kOff, +}; + +template +struct MpiDeviceTraits { + static constexpr bool kDeviceAwareCapable = false; + + static bool Query() { return false; } +}; + +template <> +struct MpiDeviceTraits { +#if defined(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY) + static constexpr bool kDeviceAwareCapable = true; + + static bool Query() { return MPIX_Query_cuda_support() != 0; } +#else + static constexpr bool kDeviceAwareCapable = false; + + static bool Query() { return false; } +#endif +}; + +template <> +struct MpiDeviceTraits { + // Add the vendor query here after MetaX MPI exposes a stable interface. + static constexpr bool kDeviceAwareCapable = false; + + static bool Query() { return false; } +}; + +inline MpiDeviceAwareMode MpiDeviceAwareModeFromEnv() { + const char *mode = std::getenv("INFINICCL_MPI_DEVICE_AWARE"); + if (mode == nullptr || std::strcmp(mode, "auto") == 0) { + return MpiDeviceAwareMode::kAuto; + } + if (std::strcmp(mode, "1") == 0) { + return MpiDeviceAwareMode::kOn; + } + if (std::strcmp(mode, "0") == 0) { + return MpiDeviceAwareMode::kOff; + } + return MpiDeviceAwareMode::kAuto; +} + +template +MpiDeviceSupport ResolveMpiDeviceSupport() { + const MpiDeviceAwareMode mode = MpiDeviceAwareModeFromEnv(); + if (mode == MpiDeviceAwareMode::kOff) { + return MpiDeviceSupport::kHostStaged; + } + if (mode == MpiDeviceAwareMode::kOn) { + return MpiDeviceSupport::kDeviceAware; + } + + if constexpr (!MpiDeviceTraits::kDeviceAwareCapable) { + return MpiDeviceSupport::kHostStaged; + } else { + return MpiDeviceTraits::Query() ? MpiDeviceSupport::kDeviceAware + : MpiDeviceSupport::kHostStaged; + } +} + +inline bool MpiDeviceAwareUsable(MpiDeviceSupport support) { + return support == MpiDeviceSupport::kDeviceAware; +} + +} // namespace infini::ccl + +#endif // INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_ diff --git a/src/backends/mpi/ompi/impl/comm_init_all.h b/src/backends/mpi/ompi/impl/comm_init_all.h index 4812f9c..abdac8a 100644 --- a/src/backends/mpi/ompi/impl/comm_init_all.h +++ b/src/backends/mpi/ompi/impl/comm_init_all.h @@ -3,6 +3,7 @@ #include "backends/mpi/ompi/checks.h" #include "backends/mpi/ompi/comm_instance.h" +#include "backends/mpi/ompi/device_support.h" #include "base/comm_init_all.h" #include "communicator.h" #include "logging.h" @@ -14,8 +15,7 @@ class CommInitAllImpl { public: static ReturnStatus Apply(Communicator *comm, int n_dev, const int *dev_list) { - constexpr Device::Type kDev = - ListGetBest(ActiveDevices{}); + constexpr Device::Type kDev = device_type; using Rt = Runtime; if (comm && comm->inter_comm()) { @@ -32,7 +32,6 @@ class CommInitAllImpl { comm->set_world_info(rank, size); comm->set_local_size(n_dev); - comm->set_inter_comm(std::move(inst)); int local_rank = 0; char *local_rank_str = getenv("OMPI_COMM_WORLD_LOCAL_RANK"); @@ -49,6 +48,16 @@ class CommInitAllImpl { CHECK_STATUS(Rt, Rt::SetDevice(comm->device_id())); + inst->device_support = ResolveMpiDeviceSupport(); + if (rank == 0) { + LOG(inst->device_support == MpiDeviceSupport::kDeviceAware + ? "MPI device-aware transport: enabled (device-direct)." + : "MPI device-aware transport: disabled (host-staged).", + Logger::LogLevel::kInfo); + } + + comm->set_inter_comm(std::move(inst)); + return ReturnStatus::kSuccess; } }; diff --git a/src/backends/mpi/ompi/impl/recv.h b/src/backends/mpi/ompi/impl/recv.h index b1422a5..a20aefb 100644 --- a/src/backends/mpi/ompi/impl/recv.h +++ b/src/backends/mpi/ompi/impl/recv.h @@ -1,9 +1,9 @@ #ifndef INFINI_CCL_BACKENDS_MPI_OMPI_IMPL_RECV_H_ #define INFINI_CCL_BACKENDS_MPI_OMPI_IMPL_RECV_H_ -#include #include +#include "backends/mpi/ompi/buffer_plan.h" #include "backends/mpi/ompi/checks.h" #include "backends/mpi/ompi/comm_instance.h" #include "base/recv.h" @@ -18,9 +18,7 @@ class RecvImpl { public: static ReturnStatus Apply(void *recv_buff, size_t count, DataType data_type, int peer, Communicator *comm, void *stream) { - constexpr Device::Type kDev = - ListGetBest(ActiveDevices{}); - using Rt = Runtime; + constexpr Device::Type kDev = device_type; auto *inst = static_cast(comm->inter_comm()); if (!inst || inst->handle == MPI_COMM_NULL) { @@ -34,14 +32,16 @@ class RecvImpl { return ReturnStatus::kInvalidArgument; } - size_t total_bytes = count * type_size; - void *host_buf = std::malloc(total_bytes); - if (!host_buf) { - LOG("Failed to allocate host buffer for `Recv` staging."); - return ReturnStatus::kSystemError; + const size_t total_bytes = count * type_size; + MpiBufferPlan plan; + ReturnStatus status = + plan.Prepare(nullptr, recv_buff, 0, total_bytes, + MpiDeviceAwareUsable(inst->device_support), stream); + if (status != ReturnStatus::kSuccess) { + return status; } - auto *bytes = static_cast(host_buf); + auto *bytes = static_cast(plan.recv_buff()); size_t offset = 0; constexpr size_t kMaxMpiCount = static_cast(std::numeric_limits::max()); @@ -51,17 +51,14 @@ class RecvImpl { if (chunk > kMaxMpiCount) { chunk = kMaxMpiCount; } + INFINI_CHECK_MPI(MPI_Recv(bytes + offset, static_cast(chunk), MPI_BYTE, peer, kTag, inst->handle, MPI_STATUS_IGNORE)); offset += chunk; } - CHECK_STATUS(Rt, Rt::Memcpy(recv_buff, host_buf, total_bytes, - Rt::MemcpyHostToDevice)); - - std::free(host_buf); - return ReturnStatus::kSuccess; + return plan.Commit(); } }; diff --git a/src/backends/mpi/ompi/impl/send.h b/src/backends/mpi/ompi/impl/send.h index 16ea91d..74f731a 100644 --- a/src/backends/mpi/ompi/impl/send.h +++ b/src/backends/mpi/ompi/impl/send.h @@ -1,9 +1,9 @@ #ifndef INFINI_CCL_BACKENDS_MPI_OMPI_IMPL_SEND_H_ #define INFINI_CCL_BACKENDS_MPI_OMPI_IMPL_SEND_H_ -#include #include +#include "backends/mpi/ompi/buffer_plan.h" #include "backends/mpi/ompi/checks.h" #include "backends/mpi/ompi/comm_instance.h" #include "base/send.h" @@ -19,9 +19,7 @@ class SendImpl { static ReturnStatus Apply(const void *send_buff, size_t count, DataType data_type, int peer, Communicator *comm, void *stream) { - constexpr Device::Type kDev = - ListGetBest(ActiveDevices{}); - using Rt = Runtime; + constexpr Device::Type kDev = device_type; auto *inst = static_cast(comm->inter_comm()); if (!inst || inst->handle == MPI_COMM_NULL) { @@ -35,18 +33,16 @@ class SendImpl { return ReturnStatus::kInvalidArgument; } - size_t total_bytes = count * type_size; - void *host_buf = std::malloc(total_bytes); - if (!host_buf) { - LOG("Failed to allocate host buffer for `Send` staging."); - return ReturnStatus::kSystemError; + const size_t total_bytes = count * type_size; + MpiBufferPlan plan; + ReturnStatus status = + plan.Prepare(send_buff, nullptr, total_bytes, 0, + MpiDeviceAwareUsable(inst->device_support), stream); + if (status != ReturnStatus::kSuccess) { + return status; } - CHECK_STATUS(Rt, Rt::Memcpy(host_buf, send_buff, total_bytes, - Rt::MemcpyDeviceToHost)); - CHECK_STATUS(Rt, Rt::StreamSynchronize(static_cast(stream))); - - auto *bytes = static_cast(host_buf); + const auto *bytes = static_cast(plan.send_buff()); size_t offset = 0; constexpr size_t kMaxMpiCount = static_cast(std::numeric_limits::max()); @@ -56,12 +52,12 @@ class SendImpl { if (chunk > kMaxMpiCount) { chunk = kMaxMpiCount; } + INFINI_CHECK_MPI(MPI_Send(bytes + offset, static_cast(chunk), MPI_BYTE, peer, kTag, inst->handle)); offset += chunk; } - std::free(host_buf); return ReturnStatus::kSuccess; } }; diff --git a/src/devices/metax/device_.h b/src/devices/metax/device_.h index 8f5e9a6..cffce64 100644 --- a/src/devices/metax/device_.h +++ b/src/devices/metax/device_.h @@ -5,7 +5,6 @@ #include // clang-format on -#include "checks.h" #include "device.h" namespace infini::ccl { @@ -20,7 +19,11 @@ MemorySpace GetMemorySpace(const void* ptr) { } mcPointerAttribute_t attr; - INFINI_CHECK_MACA(mcPointerGetAttributes(&attr, ptr)); + mcError_t status = mcPointerGetAttributes(&attr, ptr); + if (status != mcSuccess) { + mcGetLastError(); + return MemorySpace::kHost; + } if (attr.type == mcMemoryTypeDevice || attr.type == mcMemoryTypeManaged || attr.type == mcMemoryTypeArray) { diff --git a/src/devices/nvidia/device_.h b/src/devices/nvidia/device_.h index 87c6f90..e7d65b8 100644 --- a/src/devices/nvidia/device_.h +++ b/src/devices/nvidia/device_.h @@ -4,7 +4,6 @@ #include #include "device.h" -#include "devices/cuda/checks.h" namespace infini::ccl { @@ -18,7 +17,11 @@ MemorySpace GetMemorySpace(const void* ptr) { } cudaPointerAttributes attr; - INFINI_CHECK_CUDA(cudaPointerGetAttributes(&attr, ptr)); + cudaError_t status = cudaPointerGetAttributes(&attr, ptr); + if (status != cudaSuccess) { + cudaGetLastError(); + return MemorySpace::kHost; + } if (attr.type == cudaMemoryTypeDevice || attr.type == cudaMemoryTypeManaged) { return MemorySpace::kDevice;