Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,11 @@ if(WITH_OMPI OR WITH_MPICH)
find_package(MPI REQUIRED)
endif()

set(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS "")
if(WITH_OMPI)
infiniccl_configure_mpi_device_aware(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS)
endif()

if(WITH_NCCL)
infiniccl_configure_nccl(NCCL_INC NCCL_LIB NCCL_COMPILE_DEFINITIONS)
include_directories(${NCCL_INC})
Expand Down
35 changes: 35 additions & 0 deletions cmake/backends/MPI.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,38 @@ function(infiniccl_detect_mpi output)

set(${output} "${_implementation}" PARENT_SCOPE)
endfunction()

function(infiniccl_configure_mpi_device_aware output)
if(NOT WITH_OMPI)
set(${output} "" PARENT_SCOPE)
return()
endif()

include(CheckCXXSourceCompiles)
include(CMakePushCheckState)
cmake_push_check_state(RESET)

set(CMAKE_REQUIRED_INCLUDES ${MPI_CXX_INCLUDE_DIRS})
set(CMAKE_REQUIRED_LIBRARIES ${MPI_CXX_LIBRARIES})
if(MPI_CXX_COMPILE_FLAGS)
set(CMAKE_REQUIRED_FLAGS ${MPI_CXX_COMPILE_FLAGS})
endif()
if(MPI_CXX_LINK_FLAGS)
set(CMAKE_REQUIRED_LINK_OPTIONS ${MPI_CXX_LINK_FLAGS})
endif()

check_cxx_source_compiles("
#include <mpi.h>
#include <mpi-ext.h>
int main() {
return MPIX_Query_cuda_support();
}
" INFINICCL_MPI_HAS_OMPI_CUDA_QUERY)
cmake_pop_check_state()

if(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY)
set(${output} "INFINICCL_MPI_HAS_OMPI_CUDA_QUERY=1" PARENT_SCOPE)
else()
set(${output} "" PARENT_SCOPE)
endif()
endfunction()
5 changes: 5 additions & 0 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,11 @@ if(WITH_OMPI)
if(TARGET MPI::MPI_CXX)
target_link_libraries(infiniccl PRIVATE MPI::MPI_CXX)
endif()

if(INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS)
target_compile_definitions(infiniccl PRIVATE
${INFINICCL_MPI_DEVICE_AWARE_DEFINITIONS})
endif()
endif()

if(WITH_MPICH)
Expand Down
173 changes: 173 additions & 0 deletions src/backends/mpi/ompi/buffer_plan.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#ifndef INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_
#define INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_

#include <cstdlib>
#include <memory>

#include "runtime.h"

namespace infini::ccl {

namespace detail {

using HostBuffer = std::unique_ptr<void, decltype(&std::free)>;

inline HostBuffer AllocateHostBuffer(std::size_t bytes) {
return HostBuffer(std::malloc(bytes), &std::free);
}

template <Device::Type kDev>
bool IsDeviceBuffer(const void *ptr) {
return ptr != nullptr && GetMemorySpace<kDev>(ptr) == MemorySpace::kDevice;
}

} // namespace detail

template <Device::Type kDev>
class MpiBufferPlan {
public:
enum class Mode {
kPair,
kAlias,
};

ReturnStatus Prepare(const void *send_buff, void *recv_buff,
std::size_t send_bytes, std::size_t recv_bytes,
bool allow_direct, void *stream,
Mode mode = Mode::kPair) {
const bool send_active = send_bytes != 0;
const bool recv_active = recv_bytes != 0;
if (!send_active && !recv_active) {
return ReturnStatus::kSuccess;
}

user_recv_buff_ = recv_buff;
recv_bytes_ = recv_bytes;

const bool send_device =
send_active && detail::IsDeviceBuffer<kDev>(send_buff);
const bool recv_device =
recv_active && detail::IsDeviceBuffer<kDev>(recv_buff);
recv_is_device_ = recv_device;

if (mode == Mode::kPair) {
return PreparePair(send_buff, recv_buff, send_bytes, recv_bytes,
allow_direct, stream, send_active, recv_active,
send_device, recv_device);
}

return PrepareAlias(recv_buff, recv_bytes, allow_direct, stream,
send_active, recv_device);
}

ReturnStatus Commit() {
if (!host_recv_buff_) {
return ReturnStatus::kSuccess;
}

CHECK_STATUS(
Runtime<kDev>,
Runtime<kDev>::Memcpy(user_recv_buff_, host_recv_buff_.get(),
recv_bytes_, Runtime<kDev>::MemcpyHostToDevice));
return ReturnStatus::kSuccess;
}

const void *send_buff() const { return mpi_send_buff_; }

void *recv_buff() const { return mpi_recv_buff_; }

bool direct() const { return direct_; }

private:
ReturnStatus PreparePair(const void *send_buff, void *recv_buff,
std::size_t send_bytes, std::size_t recv_bytes,
bool allow_direct, void *stream, bool send_active,
bool recv_active, bool send_device,
bool recv_device) {
direct_ = allow_direct && (!send_active || send_device) &&
(!recv_active || recv_device);
if (direct_) {
if (send_device) {
CHECK_STATUS(Runtime<kDev>,
Runtime<kDev>::StreamSynchronize(
static_cast<typename Runtime<kDev>::Stream>(stream)));
}

mpi_send_buff_ = send_buff;
mpi_recv_buff_ = recv_buff;
return ReturnStatus::kSuccess;
}

if (send_device) {
CHECK_STATUS(Runtime<kDev>,
Runtime<kDev>::StreamSynchronize(
static_cast<typename Runtime<kDev>::Stream>(stream)));
host_send_buff_ = detail::AllocateHostBuffer(send_bytes);
if (!host_send_buff_) {
return ReturnStatus::kSystemError;
}

CHECK_STATUS(
Runtime<kDev>,
Runtime<kDev>::Memcpy(host_send_buff_.get(), send_buff, send_bytes,
Runtime<kDev>::MemcpyDeviceToHost));
mpi_send_buff_ = host_send_buff_.get();
} else {
mpi_send_buff_ = send_buff;
}

if (recv_device) {
host_recv_buff_ = detail::AllocateHostBuffer(recv_bytes);
if (!host_recv_buff_) {
return ReturnStatus::kSystemError;
}
mpi_recv_buff_ = host_recv_buff_.get();
} else {
mpi_recv_buff_ = recv_buff;
}

return ReturnStatus::kSuccess;
}

ReturnStatus PrepareAlias(void *recv_buff, std::size_t recv_bytes,
bool allow_direct, void *stream, bool send_active,
bool recv_device) {
direct_ = allow_direct && recv_device && !send_active;
if (direct_) {
CHECK_STATUS(Runtime<kDev>,
Runtime<kDev>::StreamSynchronize(
static_cast<typename Runtime<kDev>::Stream>(stream)));
mpi_recv_buff_ = recv_buff;
return ReturnStatus::kSuccess;
}

if (recv_device) {
CHECK_STATUS(Runtime<kDev>,
Runtime<kDev>::StreamSynchronize(
static_cast<typename Runtime<kDev>::Stream>(stream)));
host_recv_buff_ = detail::AllocateHostBuffer(recv_bytes);
if (!host_recv_buff_) {
return ReturnStatus::kSystemError;
}

mpi_recv_buff_ = host_recv_buff_.get();
} else {
mpi_recv_buff_ = recv_buff;
}

return ReturnStatus::kSuccess;
}

void *user_recv_buff_ = nullptr;
const void *mpi_send_buff_ = nullptr;
void *mpi_recv_buff_ = nullptr;
detail::HostBuffer host_send_buff_{nullptr, &std::free};
detail::HostBuffer host_recv_buff_{nullptr, &std::free};
std::size_t recv_bytes_ = 0;
bool recv_is_device_ = false;
bool direct_ = false;
};

} // namespace infini::ccl

#endif // INFINI_CCL_BACKENDS_MPI_OMPI_BUFFER_PLAN_H_
3 changes: 3 additions & 0 deletions src/backends/mpi/ompi/comm_instance.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@

#include "checks.h"
#include "communicator.h"
#include "device_support.h"

namespace infini::ccl {

struct OmpiInstance : public BackendCommInstance {
MPI_Comm handle = MPI_COMM_NULL;

MpiDeviceSupport device_support = MpiDeviceSupport::kHostStaged;

OmpiInstance() { type = BackendType::kOmpi; }
~OmpiInstance() override { Destroy(); }

Expand Down
94 changes: 94 additions & 0 deletions src/backends/mpi/ompi/device_support.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#ifndef INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_
#define INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_

#include <cstdint>
#include <cstdlib>
#include <cstring>

#if defined(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY)
#include <mpi-ext.h>
#include <mpi.h>
#endif

#include "device.h"

namespace infini::ccl {

enum class MpiDeviceSupport : std::uint8_t {
kHostStaged = 0,
kDeviceAware = 1,
};

enum class MpiDeviceAwareMode : std::uint8_t {
kAuto,
kOn,
kOff,
};

template <Device::Type kDev>
struct MpiDeviceTraits {
static constexpr bool kDeviceAwareCapable = false;

static bool Query() { return false; }
};

template <>
struct MpiDeviceTraits<Device::Type::kNvidia> {
#if defined(INFINICCL_MPI_HAS_OMPI_CUDA_QUERY)
static constexpr bool kDeviceAwareCapable = true;

static bool Query() { return MPIX_Query_cuda_support() != 0; }
#else
static constexpr bool kDeviceAwareCapable = false;

static bool Query() { return false; }
#endif
};

template <>
struct MpiDeviceTraits<Device::Type::kMetax> {
// Add the vendor query here after MetaX MPI exposes a stable interface.
static constexpr bool kDeviceAwareCapable = false;

static bool Query() { return false; }
};

inline MpiDeviceAwareMode MpiDeviceAwareModeFromEnv() {
const char *mode = std::getenv("INFINICCL_MPI_DEVICE_AWARE");
if (mode == nullptr || std::strcmp(mode, "auto") == 0) {
return MpiDeviceAwareMode::kAuto;
}
if (std::strcmp(mode, "1") == 0) {
return MpiDeviceAwareMode::kOn;
}
if (std::strcmp(mode, "0") == 0) {
return MpiDeviceAwareMode::kOff;
}
return MpiDeviceAwareMode::kAuto;
}

template <Device::Type kDev>
MpiDeviceSupport ResolveMpiDeviceSupport() {
const MpiDeviceAwareMode mode = MpiDeviceAwareModeFromEnv();
if (mode == MpiDeviceAwareMode::kOff) {
return MpiDeviceSupport::kHostStaged;
}
if (mode == MpiDeviceAwareMode::kOn) {
return MpiDeviceSupport::kDeviceAware;
}

if constexpr (!MpiDeviceTraits<kDev>::kDeviceAwareCapable) {
return MpiDeviceSupport::kHostStaged;
} else {
return MpiDeviceTraits<kDev>::Query() ? MpiDeviceSupport::kDeviceAware
: MpiDeviceSupport::kHostStaged;
}
}

inline bool MpiDeviceAwareUsable(MpiDeviceSupport support) {
return support == MpiDeviceSupport::kDeviceAware;
}

} // namespace infini::ccl

#endif // INFINI_CCL_BACKENDS_MPI_OMPI_DEVICE_SUPPORT_H_
15 changes: 12 additions & 3 deletions src/backends/mpi/ompi/impl/comm_init_all.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include "backends/mpi/ompi/checks.h"
#include "backends/mpi/ompi/comm_instance.h"
#include "backends/mpi/ompi/device_support.h"
#include "base/comm_init_all.h"
#include "communicator.h"
#include "logging.h"
Expand All @@ -14,8 +15,7 @@ class CommInitAllImpl<BackendType::kOmpi, device_type> {
public:
static ReturnStatus Apply(Communicator *comm, int n_dev,
const int *dev_list) {
constexpr Device::Type kDev =
ListGetBest<DevicePriority>(ActiveDevices<CommInitAll>{});
constexpr Device::Type kDev = device_type;
using Rt = Runtime<kDev>;

if (comm && comm->inter_comm()) {
Expand All @@ -32,7 +32,6 @@ class CommInitAllImpl<BackendType::kOmpi, device_type> {

comm->set_world_info(rank, size);
comm->set_local_size(n_dev);
comm->set_inter_comm(std::move(inst));

int local_rank = 0;
char *local_rank_str = getenv("OMPI_COMM_WORLD_LOCAL_RANK");
Expand All @@ -49,6 +48,16 @@ class CommInitAllImpl<BackendType::kOmpi, device_type> {

CHECK_STATUS(Rt, Rt::SetDevice(comm->device_id()));

inst->device_support = ResolveMpiDeviceSupport<kDev>();
if (rank == 0) {
LOG(inst->device_support == MpiDeviceSupport::kDeviceAware
? "MPI device-aware transport: enabled (device-direct)."
: "MPI device-aware transport: disabled (host-staged).",
Logger::LogLevel::kInfo);
}

comm->set_inter_comm(std::move(inst));

return ReturnStatus::kSuccess;
}
};
Expand Down
Loading