Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@

# Apply clang-format 16.0.6 to C/C++/CUDA sources and Black to Python scripts.
30732097c4ab0421939aab1d41248d28a64d4e68

# Format changes introduced by the generator/RNG branch.
58b899ec75f2b92819c5c3254566c1a553389f40
4 changes: 3 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ endif()

if(USE_CUDA)
add_compile_definitions(USE_CUDA=1)
if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
set(CMAKE_CUDA_ARCHITECTURES "75;80;90;120" CACHE STRING "CUDA architectures to build")
endif()
enable_language(CUDA)
find_package(CUDAToolkit REQUIRED)
include_directories(${CUDAToolkit_INCLUDE_DIRS})
Expand All @@ -115,7 +118,6 @@ if(USE_CUDA)
file(GLOB_RECURSE CUDA_KERNELS ${PROJECT_SOURCE_DIR}/infini_train/src/*.cu)

add_library(infini_train_cuda_kernels STATIC ${CUDA_KERNELS})
set_target_properties(infini_train_cuda_kernels PROPERTIES CUDA_ARCHITECTURES "75;80;90")

target_link_libraries(infini_train_cuda_kernels
PUBLIC
Expand Down
117 changes: 23 additions & 94 deletions example/gpt2/main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,13 @@
#include "infini_train/include/core/runtime/device_guard.h"
#include "infini_train/include/dataloader.h"
#include "infini_train/include/device.h"
#include "infini_train/include/generator.h"
#include "infini_train/include/lr_scheduler.h"
#include "infini_train/include/nn/lora/lora_utils.h"
#include "infini_train/include/nn/modules/loss.h"
#include "infini_train/include/nn/modules/module.h"
#include "infini_train/include/nn/modules/transformer/transformer.h"
#include "infini_train/include/nn/parallel/comm.h"
#include "infini_train/include/nn/parallel/context_parallel.h"
#include "infini_train/include/nn/parallel/ddp/distributed_data_parallel.h"
#include "infini_train/include/nn/parallel/ddp/distributed_optimizer.h"
#include "infini_train/include/nn/parallel/global.h"
Expand All @@ -56,6 +56,7 @@
#include "infini_train/include/nn/parallel/tensor_parallel.h"
#include "infini_train/include/nn/parallel/utils.h"
#include "infini_train/include/optimizer.h"
#include "infini_train/include/rng_tracker.h"
#ifdef PROFILE_MODE
#include "infini_train/include/profiler.h"
#endif
Expand Down Expand Up @@ -135,14 +136,6 @@ DEFINE_bool(tp_comm_overlap_disable_qkv, false,
"Disable QKV forward AG and split dgrad RS overlap");
DEFINE_bool(tp_comm_overlap_disable_fc1, false,
"Disable FC1 forward AG and split dgrad RS overlap");
DEFINE_uint32(context_parallel, 1, "Context Parallel world size");
DEFINE_string(
cp_comm_type, "p2p",
"Context Parallel communication type (all_gather|p2p|a2a|a2a+p2p)");
DEFINE_string(attention_cp_layout, "zigzag",
"Context Parallel sequence layout (contiguous|zigzag)");
DEFINE_string(hierarchical_context_parallel_sizes, "",
"A2A,P2P subgroup sizes for a2a+p2p, e.g. 2,2");
DEFINE_uint32(
pipeline_parallel, 1,
"Pipeline Parallel world size, specified the number of PP stages.");
Expand Down Expand Up @@ -235,14 +228,6 @@ DEFINE_validator(zero_stage, [](const char *, int32_t value) {
DEFINE_validator(lr_decay_style, [](const char *, const std::string &value) {
return kSupportedLRDecayStyles.contains(value);
});
DEFINE_validator(attention_cp_layout,
[](const char *, const std::string &value) {
return value == "contiguous" || value == "zigzag";
});
DEFINE_validator(cp_comm_type, [](const char *, const std::string &value) {
return value == "all_gather" || value == "p2p" || value == "a2a" ||
value == "a2a+p2p";
});

void Train(const nn::parallel::Rank &rank) {
using namespace nn::parallel;
Expand Down Expand Up @@ -270,42 +255,24 @@ void Train(const nn::parallel::Rank &rank) {
int ddp_world_size = global::GetDataParallelSize();
int tp_world_size = global::GetTensorParallelSize();
int sp_world_size = global::GetSequenceParallelEnabled() ? tp_world_size : 1;
int cp_world_size = global::GetContextParallelSize();
int dp_cp_world_size = ddp_world_size * cp_world_size;
int pp_world_size = global::GetPipelineParallelSize();
const auto cp_local_sequence_length = FLAGS_sequence_length / cp_world_size;

if (cp_world_size > 1) {
const int sequence_parts =
cp_world_size * (global::GetAttentionCPLayout() == "zigzag" ? 2 : 1);
CHECK_EQ(FLAGS_sequence_length % sequence_parts, 0)
<< "sequence_length must be divisible by CP size (2 * CP for zigzag).";
}
if (FLAGS_sequence_parallel) {
CHECK_EQ(cp_local_sequence_length % tp_world_size, 0)
<< "CP-local sequence_length must be divisible by tp_world_size when "
"SP is enabled.";
}
if (FLAGS_zero_stage >= 1) {
CHECK_GT(dp_cp_world_size, 1)
<< "ZeRO requires DP or CP world size greater than 1.";
CHECK_LE(FLAGS_zero_stage, 2) << "ZeRO-3 is not supported yet.";
CHECK_EQ(FLAGS_sequence_length % tp_world_size, 0)
<< "sequence_length must be divisible by tp_world_size when SP is "
"enabled (pad later if needed).";
}

int ddp_rank = 0;
int dp_cp_rank = 0;
int tp_rank = 0;
int cp_rank = 0;
int pp_rank = 0;

// Set thread-local global rank
// TODO(dcj): Use DeviceGuardImpl to get GlobalRank later.
nn::parallel::global::thread_global_rank = rank.GlobalRank();

const ProcessGroup *ddp_pg = nullptr;
const ProcessGroup *dp_cp_pg = nullptr;
const ProcessGroup *tp_pg = nullptr;
const ProcessGroup *cp_pg = nullptr;
const ProcessGroup *pp_pg = nullptr;

if (rank.IsParallel()) {
Expand All @@ -321,13 +288,6 @@ void Train(const nn::parallel::Rank &rank) {
ddp_rank = ddp_pg->GetGroupRank(rank.GlobalRank());
}

if (dp_cp_world_size > 1) {
dp_cp_pg = pg_factory->GetOrCreate(
GetDataParallelWithContextProcessGroupName(rank.GlobalRank()),
GetDataParallelWithContextGroupRanks(rank.GlobalRank()));
dp_cp_rank = dp_cp_pg->GetGroupRank(rank.GlobalRank());
}

if (tp_world_size > 1) {
tp_pg = pg_factory->GetOrCreate(
GetTensorParallelProcessGroupName(rank.GlobalRank()),
Expand All @@ -337,24 +297,6 @@ void Train(const nn::parallel::Rank &rank) {
nn::parallel::tp_rank = tp_rank;
}

if (cp_world_size > 1) {
cp_pg = pg_factory->GetOrCreate(
GetContextParallelProcessGroupName(rank.GlobalRank()),
GetContextParallelGroupRanks(rank.GlobalRank()));
cp_rank = cp_pg->GetGroupRank(rank.GlobalRank());
nn::parallel::cp_rank = cp_rank;
if (global::GetContextParallelCommType() ==
global::ContextParallelCommType::kA2AP2P) {
for (int level = 0; level < 2; ++level) {
pg_factory->GetOrCreate(
GetHierarchicalContextParallelProcessGroupName(rank.GlobalRank(),
level),
GetHierarchicalContextParallelGroupRanks(rank.GlobalRank(),
level));
}
}
}

if (pp_world_size > 1) {
pp_pg = pg_factory->GetOrCreate(
GetPipelineParallelProcessGroupName(rank.GlobalRank()),
Expand All @@ -367,6 +309,12 @@ void Train(const nn::parallel::Rank &rank) {
device = Device(device_type, 0);
}

RngTracker rng_tracker(42);
if (tp_world_size > 1) {
rng_tracker.Add(kTensorParallelInitRngName, 42 + 2718 + tp_rank);
}
RngScope rng_scope(rng_tracker);

// calculate gradient accumulation from the desired total batch size and the
// current run configuration
const auto tokens_per_fwdbwd =
Expand All @@ -377,9 +325,6 @@ void Train(const nn::parallel::Rank &rank) {
<< " => calculated gradient accumulation steps: "
<< grad_accum_steps;

// rng / reproducibility
// ManualSeed(42);

// init the model, either from scratch or from OpenAI pretrained checkpoint
nn::TransformerConfig model_config = gpt2::GPT2Config();
std::shared_ptr<nn::Module> model = nullptr;
Expand Down Expand Up @@ -454,16 +399,16 @@ void Train(const nn::parallel::Rank &rank) {

if (pp_world_size > 1) {
// NOTE(dcj): To ensure that the tensor shapes at the pipeline stage
// boundaries remain correct when context/sequence parallelism is enabled,
// use the CP-local and SP-local sequence length.
// boundaries remain correct when sequence parallelism (SP) is enabled, we
// need to divide by sp_world_size.
auto shapes = std::vector<std::vector<int64_t>>{
{FLAGS_batch_size, cp_local_sequence_length / sp_world_size,
{FLAGS_batch_size, FLAGS_sequence_length / sp_world_size,
model_config.n_embd}};

model = std::make_shared<nn::parallel::PipelineParallel>(
model, pp_world_size, num_micro_batches, shapes, pp_rank, device,
model_config.GetChunkSize());
if (dp_cp_world_size > 1) {
if (ddp_world_size > 1) {
auto ddp_config =
DistributedDataParallelConfig{.zero_stage = FLAGS_zero_stage};
auto *mutable_chunks =
Expand All @@ -474,7 +419,7 @@ void Train(const nn::parallel::Rank &rank) {
mutable_chunks->at(chunk_id), rank, ddp_config);
}
}
} else if (dp_cp_world_size > 1) {
} else if (ddp_world_size > 1) {
// NOTE(dcj): Complete all device (.to(device)) and dtype (.to(dtype))
// conversions before wrapping the model with DistributedDataParallel (DDP).
// Otherwise, DDP’s gradient hooks may be lost because new parameter tensors
Expand Down Expand Up @@ -534,8 +479,8 @@ void Train(const nn::parallel::Rank &rank) {
->mutable_chunks())
: std::vector<std::shared_ptr<nn::Module>>{model};
optimizer = std::make_shared<nn::parallel::DistributedOptimizer>(
optimizer_creator, named_parameters, model_chunks, dp_cp_world_size,
dp_cp_rank);
optimizer_creator, named_parameters, model_chunks, ddp_world_size,
ddp_rank);
} else {
optimizer = optimizer_creator(named_parameters);
}
Expand Down Expand Up @@ -616,7 +561,6 @@ void Train(const nn::parallel::Rank &rank) {
.ddp_size = ddp_world_size,
.tp_size = tp_world_size,
.sp_size = sp_world_size,
.cp_size = cp_world_size,
.pp_size = pp_world_size,
.vpp_size = static_cast<int>(FLAGS_virtual_pipeline_parallel),
.checkpoint_root_dir = FLAGS_save,
Expand Down Expand Up @@ -687,10 +631,6 @@ void Train(const nn::parallel::Rank &rank) {
auto [x, y] = next_train_batch();
x = std::make_shared<Tensor>(x->To(device));
y = std::make_shared<Tensor>(y->To(device));
if (cp_world_size > 1) {
x = nn::parallel::SliceAlongCPRegionFunc(x, 1);
y = nn::parallel::SliceAlongCPRegionFunc(y, 1);
}

LOG(INFO) << "Rank " << rank.GlobalRank() << ": start forward";

Expand All @@ -710,7 +650,7 @@ void Train(const nn::parallel::Rank &rank) {

LOG(INFO) << "Rank " << rank.GlobalRank() << ": start backward";
std::unique_ptr<nn::NoSyncGuard> no_sync_guard;
if (dp_cp_world_size > 1 && micro_step != grad_accum_steps - 1) {
if (ddp_world_size > 1 && micro_step != grad_accum_steps - 1) {
no_sync_guard = model->no_sync();
}
loss->Backward();
Expand All @@ -730,21 +670,17 @@ void Train(const nn::parallel::Rank &rank) {
auto [x, y] = next_train_batch();
x = std::make_shared<Tensor>(x->To(device));
y = std::make_shared<Tensor>(y->To(device));
if (cp_world_size > 1) {
x = nn::parallel::SliceAlongCPRegionFunc(x, 1);
y = nn::parallel::SliceAlongCPRegionFunc(y, 1);
}

lossf = model->TrainStep({x}, {y}, optimizer, loss_fn, dtype);
if (scheduler) {
scheduler->Step();
}
}

if (dp_cp_world_size > 1) {
if (ddp_world_size > 1) {
auto lossf_tensor = std::make_shared<Tensor>(
&lossf, std::vector<int64_t>{}, DataType::kFLOAT32, device);
comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, dp_cp_pg);
comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, ddp_pg);
lossf =
static_cast<const float *>(lossf_tensor->To(Device()).DataPtr())[0];
}
Expand All @@ -762,11 +698,10 @@ void Train(const nn::parallel::Rank &rank) {
"step {:4d}/{} | train loss {:.6f} | lr {:.2e} | ({:.2f} ms | {:.0f} "
"tok/s | "
"peak used: {:5d} MB | peak reserved: {:5d} MB, DP={}, TP={}, SP={}, "
"CP={}, "
"PP={})",
step + 1, FLAGS_num_iteration, lossf, current_lr, duration_us / 1e3f,
tps, used_mb, reserved_mb, ddp_world_size, tp_world_size,
sp_world_size, cp_world_size, pp_world_size);
sp_world_size, pp_world_size);

if ((step + 1) % FLAGS_freq_generate_txt == 0) {
if (tokenizer) {
Expand Down Expand Up @@ -846,18 +781,12 @@ int main(int argc, char *argv[]) {
FLAGS_tp_comm_overlap_disable_qkv;
model_parallel_config.tp_comm_overlap_disable_fc1 =
FLAGS_tp_comm_overlap_disable_fc1;
model_parallel_config.context_parallel.size = FLAGS_context_parallel;
model_parallel_config.context_parallel.comm_type =
nn::parallel::global::ParseContextParallelCommType(FLAGS_cp_comm_type);
model_parallel_config.context_parallel.hierarchical_sizes =
FLAGS_hierarchical_context_parallel_sizes;
model_parallel_config.context_parallel.attention_layout =
FLAGS_attention_cp_layout;
model_parallel_config.expert_tensor_parallel_size = 1;
nn::parallel::global::InitAllEnv(FLAGS_nthread_per_process,
model_parallel_config);

utils::PrecisionCheckEnv::Instance().Init(precision_config);
infini_train::ManualSeed(42);

LOG(INFO) << nn::parallel::global::ProcessGroupOverview();
if (FLAGS_nthread_per_process > 1) {
Expand Down
Loading
Loading