diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f8ddbc2f..d3f3d7c5 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -4,3 +4,6 @@ # Apply clang-format 16.0.6 to C/C++/CUDA sources and Black to Python scripts. 30732097c4ab0421939aab1d41248d28a64d4e68 + +# Format changes introduced by the generator/RNG branch. +58b899ec75f2b92819c5c3254566c1a553389f40 diff --git a/CMakeLists.txt b/CMakeLists.txt index 88911e84..43e0780b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -104,6 +104,9 @@ endif() if(USE_CUDA) add_compile_definitions(USE_CUDA=1) + if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES "75;80;90;120" CACHE STRING "CUDA architectures to build") + endif() enable_language(CUDA) find_package(CUDAToolkit REQUIRED) include_directories(${CUDAToolkit_INCLUDE_DIRS}) @@ -115,7 +118,6 @@ if(USE_CUDA) file(GLOB_RECURSE CUDA_KERNELS ${PROJECT_SOURCE_DIR}/infini_train/src/*.cu) add_library(infini_train_cuda_kernels STATIC ${CUDA_KERNELS}) - set_target_properties(infini_train_cuda_kernels PROPERTIES CUDA_ARCHITECTURES "75;80;90") target_link_libraries(infini_train_cuda_kernels PUBLIC diff --git a/example/gpt2/main.cc b/example/gpt2/main.cc index 64874b0e..c54cf5d6 100644 --- a/example/gpt2/main.cc +++ b/example/gpt2/main.cc @@ -39,13 +39,13 @@ #include "infini_train/include/core/runtime/device_guard.h" #include "infini_train/include/dataloader.h" #include "infini_train/include/device.h" +#include "infini_train/include/generator.h" #include "infini_train/include/lr_scheduler.h" #include "infini_train/include/nn/lora/lora_utils.h" #include "infini_train/include/nn/modules/loss.h" #include "infini_train/include/nn/modules/module.h" #include "infini_train/include/nn/modules/transformer/transformer.h" #include "infini_train/include/nn/parallel/comm.h" -#include "infini_train/include/nn/parallel/context_parallel.h" #include "infini_train/include/nn/parallel/ddp/distributed_data_parallel.h" #include "infini_train/include/nn/parallel/ddp/distributed_optimizer.h" #include "infini_train/include/nn/parallel/global.h" @@ -56,6 +56,7 @@ #include "infini_train/include/nn/parallel/tensor_parallel.h" #include "infini_train/include/nn/parallel/utils.h" #include "infini_train/include/optimizer.h" +#include "infini_train/include/rng_tracker.h" #ifdef PROFILE_MODE #include "infini_train/include/profiler.h" #endif @@ -135,14 +136,6 @@ DEFINE_bool(tp_comm_overlap_disable_qkv, false, "Disable QKV forward AG and split dgrad RS overlap"); DEFINE_bool(tp_comm_overlap_disable_fc1, false, "Disable FC1 forward AG and split dgrad RS overlap"); -DEFINE_uint32(context_parallel, 1, "Context Parallel world size"); -DEFINE_string( - cp_comm_type, "p2p", - "Context Parallel communication type (all_gather|p2p|a2a|a2a+p2p)"); -DEFINE_string(attention_cp_layout, "zigzag", - "Context Parallel sequence layout (contiguous|zigzag)"); -DEFINE_string(hierarchical_context_parallel_sizes, "", - "A2A,P2P subgroup sizes for a2a+p2p, e.g. 2,2"); DEFINE_uint32( pipeline_parallel, 1, "Pipeline Parallel world size, specified the number of PP stages."); @@ -235,14 +228,6 @@ DEFINE_validator(zero_stage, [](const char *, int32_t value) { DEFINE_validator(lr_decay_style, [](const char *, const std::string &value) { return kSupportedLRDecayStyles.contains(value); }); -DEFINE_validator(attention_cp_layout, - [](const char *, const std::string &value) { - return value == "contiguous" || value == "zigzag"; - }); -DEFINE_validator(cp_comm_type, [](const char *, const std::string &value) { - return value == "all_gather" || value == "p2p" || value == "a2a" || - value == "a2a+p2p"; -}); void Train(const nn::parallel::Rank &rank) { using namespace nn::parallel; @@ -270,32 +255,16 @@ void Train(const nn::parallel::Rank &rank) { int ddp_world_size = global::GetDataParallelSize(); int tp_world_size = global::GetTensorParallelSize(); int sp_world_size = global::GetSequenceParallelEnabled() ? tp_world_size : 1; - int cp_world_size = global::GetContextParallelSize(); - int dp_cp_world_size = ddp_world_size * cp_world_size; int pp_world_size = global::GetPipelineParallelSize(); - const auto cp_local_sequence_length = FLAGS_sequence_length / cp_world_size; - if (cp_world_size > 1) { - const int sequence_parts = - cp_world_size * (global::GetAttentionCPLayout() == "zigzag" ? 2 : 1); - CHECK_EQ(FLAGS_sequence_length % sequence_parts, 0) - << "sequence_length must be divisible by CP size (2 * CP for zigzag)."; - } if (FLAGS_sequence_parallel) { - CHECK_EQ(cp_local_sequence_length % tp_world_size, 0) - << "CP-local sequence_length must be divisible by tp_world_size when " - "SP is enabled."; - } - if (FLAGS_zero_stage >= 1) { - CHECK_GT(dp_cp_world_size, 1) - << "ZeRO requires DP or CP world size greater than 1."; - CHECK_LE(FLAGS_zero_stage, 2) << "ZeRO-3 is not supported yet."; + CHECK_EQ(FLAGS_sequence_length % tp_world_size, 0) + << "sequence_length must be divisible by tp_world_size when SP is " + "enabled (pad later if needed)."; } int ddp_rank = 0; - int dp_cp_rank = 0; int tp_rank = 0; - int cp_rank = 0; int pp_rank = 0; // Set thread-local global rank @@ -303,9 +272,7 @@ void Train(const nn::parallel::Rank &rank) { nn::parallel::global::thread_global_rank = rank.GlobalRank(); const ProcessGroup *ddp_pg = nullptr; - const ProcessGroup *dp_cp_pg = nullptr; const ProcessGroup *tp_pg = nullptr; - const ProcessGroup *cp_pg = nullptr; const ProcessGroup *pp_pg = nullptr; if (rank.IsParallel()) { @@ -321,13 +288,6 @@ void Train(const nn::parallel::Rank &rank) { ddp_rank = ddp_pg->GetGroupRank(rank.GlobalRank()); } - if (dp_cp_world_size > 1) { - dp_cp_pg = pg_factory->GetOrCreate( - GetDataParallelWithContextProcessGroupName(rank.GlobalRank()), - GetDataParallelWithContextGroupRanks(rank.GlobalRank())); - dp_cp_rank = dp_cp_pg->GetGroupRank(rank.GlobalRank()); - } - if (tp_world_size > 1) { tp_pg = pg_factory->GetOrCreate( GetTensorParallelProcessGroupName(rank.GlobalRank()), @@ -337,24 +297,6 @@ void Train(const nn::parallel::Rank &rank) { nn::parallel::tp_rank = tp_rank; } - if (cp_world_size > 1) { - cp_pg = pg_factory->GetOrCreate( - GetContextParallelProcessGroupName(rank.GlobalRank()), - GetContextParallelGroupRanks(rank.GlobalRank())); - cp_rank = cp_pg->GetGroupRank(rank.GlobalRank()); - nn::parallel::cp_rank = cp_rank; - if (global::GetContextParallelCommType() == - global::ContextParallelCommType::kA2AP2P) { - for (int level = 0; level < 2; ++level) { - pg_factory->GetOrCreate( - GetHierarchicalContextParallelProcessGroupName(rank.GlobalRank(), - level), - GetHierarchicalContextParallelGroupRanks(rank.GlobalRank(), - level)); - } - } - } - if (pp_world_size > 1) { pp_pg = pg_factory->GetOrCreate( GetPipelineParallelProcessGroupName(rank.GlobalRank()), @@ -367,6 +309,12 @@ void Train(const nn::parallel::Rank &rank) { device = Device(device_type, 0); } + RngTracker rng_tracker(42); + if (tp_world_size > 1) { + rng_tracker.Add(kTensorParallelInitRngName, 42 + 2718 + tp_rank); + } + RngScope rng_scope(rng_tracker); + // calculate gradient accumulation from the desired total batch size and the // current run configuration const auto tokens_per_fwdbwd = @@ -377,9 +325,6 @@ void Train(const nn::parallel::Rank &rank) { << " => calculated gradient accumulation steps: " << grad_accum_steps; - // rng / reproducibility - // ManualSeed(42); - // init the model, either from scratch or from OpenAI pretrained checkpoint nn::TransformerConfig model_config = gpt2::GPT2Config(); std::shared_ptr model = nullptr; @@ -454,16 +399,16 @@ void Train(const nn::parallel::Rank &rank) { if (pp_world_size > 1) { // NOTE(dcj): To ensure that the tensor shapes at the pipeline stage - // boundaries remain correct when context/sequence parallelism is enabled, - // use the CP-local and SP-local sequence length. + // boundaries remain correct when sequence parallelism (SP) is enabled, we + // need to divide by sp_world_size. auto shapes = std::vector>{ - {FLAGS_batch_size, cp_local_sequence_length / sp_world_size, + {FLAGS_batch_size, FLAGS_sequence_length / sp_world_size, model_config.n_embd}}; model = std::make_shared( model, pp_world_size, num_micro_batches, shapes, pp_rank, device, model_config.GetChunkSize()); - if (dp_cp_world_size > 1) { + if (ddp_world_size > 1) { auto ddp_config = DistributedDataParallelConfig{.zero_stage = FLAGS_zero_stage}; auto *mutable_chunks = @@ -474,7 +419,7 @@ void Train(const nn::parallel::Rank &rank) { mutable_chunks->at(chunk_id), rank, ddp_config); } } - } else if (dp_cp_world_size > 1) { + } else if (ddp_world_size > 1) { // NOTE(dcj): Complete all device (.to(device)) and dtype (.to(dtype)) // conversions before wrapping the model with DistributedDataParallel (DDP). // Otherwise, DDP’s gradient hooks may be lost because new parameter tensors @@ -534,8 +479,8 @@ void Train(const nn::parallel::Rank &rank) { ->mutable_chunks()) : std::vector>{model}; optimizer = std::make_shared( - optimizer_creator, named_parameters, model_chunks, dp_cp_world_size, - dp_cp_rank); + optimizer_creator, named_parameters, model_chunks, ddp_world_size, + ddp_rank); } else { optimizer = optimizer_creator(named_parameters); } @@ -616,7 +561,6 @@ void Train(const nn::parallel::Rank &rank) { .ddp_size = ddp_world_size, .tp_size = tp_world_size, .sp_size = sp_world_size, - .cp_size = cp_world_size, .pp_size = pp_world_size, .vpp_size = static_cast(FLAGS_virtual_pipeline_parallel), .checkpoint_root_dir = FLAGS_save, @@ -687,10 +631,6 @@ void Train(const nn::parallel::Rank &rank) { auto [x, y] = next_train_batch(); x = std::make_shared(x->To(device)); y = std::make_shared(y->To(device)); - if (cp_world_size > 1) { - x = nn::parallel::SliceAlongCPRegionFunc(x, 1); - y = nn::parallel::SliceAlongCPRegionFunc(y, 1); - } LOG(INFO) << "Rank " << rank.GlobalRank() << ": start forward"; @@ -710,7 +650,7 @@ void Train(const nn::parallel::Rank &rank) { LOG(INFO) << "Rank " << rank.GlobalRank() << ": start backward"; std::unique_ptr no_sync_guard; - if (dp_cp_world_size > 1 && micro_step != grad_accum_steps - 1) { + if (ddp_world_size > 1 && micro_step != grad_accum_steps - 1) { no_sync_guard = model->no_sync(); } loss->Backward(); @@ -730,10 +670,6 @@ void Train(const nn::parallel::Rank &rank) { auto [x, y] = next_train_batch(); x = std::make_shared(x->To(device)); y = std::make_shared(y->To(device)); - if (cp_world_size > 1) { - x = nn::parallel::SliceAlongCPRegionFunc(x, 1); - y = nn::parallel::SliceAlongCPRegionFunc(y, 1); - } lossf = model->TrainStep({x}, {y}, optimizer, loss_fn, dtype); if (scheduler) { @@ -741,10 +677,10 @@ void Train(const nn::parallel::Rank &rank) { } } - if (dp_cp_world_size > 1) { + if (ddp_world_size > 1) { auto lossf_tensor = std::make_shared( &lossf, std::vector{}, DataType::kFLOAT32, device); - comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, dp_cp_pg); + comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, ddp_pg); lossf = static_cast(lossf_tensor->To(Device()).DataPtr())[0]; } @@ -762,11 +698,10 @@ void Train(const nn::parallel::Rank &rank) { "step {:4d}/{} | train loss {:.6f} | lr {:.2e} | ({:.2f} ms | {:.0f} " "tok/s | " "peak used: {:5d} MB | peak reserved: {:5d} MB, DP={}, TP={}, SP={}, " - "CP={}, " "PP={})", step + 1, FLAGS_num_iteration, lossf, current_lr, duration_us / 1e3f, tps, used_mb, reserved_mb, ddp_world_size, tp_world_size, - sp_world_size, cp_world_size, pp_world_size); + sp_world_size, pp_world_size); if ((step + 1) % FLAGS_freq_generate_txt == 0) { if (tokenizer) { @@ -846,18 +781,12 @@ int main(int argc, char *argv[]) { FLAGS_tp_comm_overlap_disable_qkv; model_parallel_config.tp_comm_overlap_disable_fc1 = FLAGS_tp_comm_overlap_disable_fc1; - model_parallel_config.context_parallel.size = FLAGS_context_parallel; - model_parallel_config.context_parallel.comm_type = - nn::parallel::global::ParseContextParallelCommType(FLAGS_cp_comm_type); - model_parallel_config.context_parallel.hierarchical_sizes = - FLAGS_hierarchical_context_parallel_sizes; - model_parallel_config.context_parallel.attention_layout = - FLAGS_attention_cp_layout; model_parallel_config.expert_tensor_parallel_size = 1; nn::parallel::global::InitAllEnv(FLAGS_nthread_per_process, model_parallel_config); utils::PrecisionCheckEnv::Instance().Init(precision_config); + infini_train::ManualSeed(42); LOG(INFO) << nn::parallel::global::ProcessGroupOverview(); if (FLAGS_nthread_per_process > 1) { diff --git a/example/llama3/main.cc b/example/llama3/main.cc index 53613524..7f389f01 100644 --- a/example/llama3/main.cc +++ b/example/llama3/main.cc @@ -38,13 +38,13 @@ #include "infini_train/include/core/runtime/device_guard.h" #include "infini_train/include/dataloader.h" #include "infini_train/include/device.h" +#include "infini_train/include/generator.h" #include "infini_train/include/lr_scheduler.h" #include "infini_train/include/nn/lora/lora_utils.h" #include "infini_train/include/nn/modules/loss.h" #include "infini_train/include/nn/modules/module.h" #include "infini_train/include/nn/modules/transformer/transformer.h" #include "infini_train/include/nn/parallel/comm.h" -#include "infini_train/include/nn/parallel/context_parallel.h" #include "infini_train/include/nn/parallel/ddp/distributed_data_parallel.h" #include "infini_train/include/nn/parallel/ddp/distributed_optimizer.h" #include "infini_train/include/nn/parallel/global.h" @@ -56,6 +56,7 @@ #include "infini_train/include/nn/parallel/tensor_parallel.h" #include "infini_train/include/nn/parallel/utils.h" #include "infini_train/include/optimizer.h" +#include "infini_train/include/rng_tracker.h" #include "infini_train/include/utils/precision_check_config.h" #include "infini_train/include/utils/precision_checker.h" #ifdef PROFILE_MODE @@ -133,14 +134,6 @@ DEFINE_bool(tp_comm_overlap_disable_qkv, false, "Disable QKV forward AG and split dgrad RS overlap"); DEFINE_bool(tp_comm_overlap_disable_fc1, false, "Disable FC1 forward AG and split dgrad RS overlap"); -DEFINE_uint32(context_parallel, 1, "Context Parallel world size"); -DEFINE_string( - cp_comm_type, "p2p", - "Context Parallel communication type (all_gather|p2p|a2a|a2a+p2p)"); -DEFINE_string(attention_cp_layout, "zigzag", - "Context Parallel sequence layout (contiguous|zigzag)"); -DEFINE_string(hierarchical_context_parallel_sizes, "", - "A2A,P2P subgroup sizes for a2a+p2p, e.g. 2,2"); DEFINE_uint32( pipeline_parallel, 1, "Pipeline Parallel world size, specified the number of PP stages."); @@ -206,14 +199,6 @@ DEFINE_validator(zero_stage, [](const char *, int32_t value) { DEFINE_validator(lr_decay_style, [](const char *, const std::string &value) { return kSupportedLRDecayStyles.contains(value); }); -DEFINE_validator(attention_cp_layout, - [](const char *, const std::string &value) { - return value == "contiguous" || value == "zigzag"; - }); -DEFINE_validator(cp_comm_type, [](const char *, const std::string &value) { - return value == "all_gather" || value == "p2p" || value == "a2a" || - value == "a2a+p2p"; -}); void Train(const nn::parallel::Rank &rank) { using namespace nn::parallel; @@ -241,41 +226,23 @@ void Train(const nn::parallel::Rank &rank) { int ddp_world_size = global::GetDataParallelSize(); int tp_world_size = global::GetTensorParallelSize(); int sp_world_size = global::GetSequenceParallelEnabled() ? tp_world_size : 1; - int cp_world_size = global::GetContextParallelSize(); - int dp_cp_world_size = ddp_world_size * cp_world_size; int pp_world_size = global::GetPipelineParallelSize(); - const auto cp_local_sequence_length = FLAGS_sequence_length / cp_world_size; - if (cp_world_size > 1) { - const int sequence_parts = - cp_world_size * (global::GetAttentionCPLayout() == "zigzag" ? 2 : 1); - CHECK_EQ(FLAGS_sequence_length % sequence_parts, 0) - << "sequence_length must be divisible by CP size (2 * CP for zigzag)."; - } if (FLAGS_sequence_parallel) { - CHECK_EQ(cp_local_sequence_length % tp_world_size, 0) - << "CP-local sequence_length must be divisible by tp_world_size when " - "SP is enabled."; - } - if (FLAGS_zero_stage >= 1) { - CHECK_GT(dp_cp_world_size, 1) - << "ZeRO requires DP or CP world size greater than 1."; - CHECK_LE(FLAGS_zero_stage, 2) << "ZeRO-3 is not supported yet."; + CHECK_EQ(FLAGS_sequence_length % tp_world_size, 0) + << "sequence_length must be divisible by tp_world_size when SP is " + "enabled (pad later if needed)."; } int ddp_rank = 0; - int dp_cp_rank = 0; int tp_rank = 0; - int cp_rank = 0; int pp_rank = 0; // Set thread-local global rank nn::parallel::global::thread_global_rank = rank.GlobalRank(); const ProcessGroup *ddp_pg = nullptr; - const ProcessGroup *dp_cp_pg = nullptr; const ProcessGroup *tp_pg = nullptr; - const ProcessGroup *cp_pg = nullptr; const ProcessGroup *pp_pg = nullptr; if (rank.IsParallel()) { @@ -291,13 +258,6 @@ void Train(const nn::parallel::Rank &rank) { ddp_rank = ddp_pg->GetGroupRank(rank.GlobalRank()); } - if (dp_cp_world_size > 1) { - dp_cp_pg = pg_factory->GetOrCreate( - GetDataParallelWithContextProcessGroupName(rank.GlobalRank()), - GetDataParallelWithContextGroupRanks(rank.GlobalRank())); - dp_cp_rank = dp_cp_pg->GetGroupRank(rank.GlobalRank()); - } - if (tp_world_size > 1) { tp_pg = pg_factory->GetOrCreate( GetTensorParallelProcessGroupName(rank.GlobalRank()), @@ -307,24 +267,6 @@ void Train(const nn::parallel::Rank &rank) { nn::parallel::tp_rank = tp_rank; } - if (cp_world_size > 1) { - cp_pg = pg_factory->GetOrCreate( - GetContextParallelProcessGroupName(rank.GlobalRank()), - GetContextParallelGroupRanks(rank.GlobalRank())); - cp_rank = cp_pg->GetGroupRank(rank.GlobalRank()); - nn::parallel::cp_rank = cp_rank; - if (global::GetContextParallelCommType() == - global::ContextParallelCommType::kA2AP2P) { - for (int level = 0; level < 2; ++level) { - pg_factory->GetOrCreate( - GetHierarchicalContextParallelProcessGroupName(rank.GlobalRank(), - level), - GetHierarchicalContextParallelGroupRanks(rank.GlobalRank(), - level)); - } - } - } - if (pp_world_size > 1) { pp_pg = pg_factory->GetOrCreate( GetPipelineParallelProcessGroupName(rank.GlobalRank()), @@ -337,6 +279,12 @@ void Train(const nn::parallel::Rank &rank) { device = Device(device_type, 0); } + RngTracker rng_tracker(42); + if (tp_world_size > 1) { + rng_tracker.Add(kTensorParallelInitRngName, 42 + 2718 + tp_rank); + } + RngScope rng_scope(rng_tracker); + // calculate gradient accumulation from the desired total batch size and the // current run configuration const auto tokens_per_fwdbwd = @@ -349,9 +297,6 @@ void Train(const nn::parallel::Rank &rank) { << grad_accum_steps; } - // rng / reproducibility - // ManualSeed(42); - nn::TransformerConfig model_config = llama3::LLaMA3Config(); std::shared_ptr model = nullptr; const auto recompute_options = nn::ParseActivationRecomputeOptions( @@ -418,16 +363,16 @@ void Train(const nn::parallel::Rank &rank) { if (pp_world_size > 1) { // NOTE(dcj): To ensure that the tensor shapes at the pipeline stage - // boundaries remain correct when context/sequence parallelism is enabled, - // use the CP-local and SP-local sequence length. + // boundaries remain correct when sequence parallelism (SP) is enabled, we + // need to divide by sp_world_size. auto shapes = std::vector>{ - {FLAGS_batch_size, cp_local_sequence_length / sp_world_size, + {FLAGS_batch_size, FLAGS_sequence_length / sp_world_size, model_config.n_embd}}; model = std::make_shared( model, pp_world_size, num_micro_batches, shapes, pp_rank, device, model_config.GetChunkSize()); - if (dp_cp_world_size > 1) { + if (ddp_world_size > 1) { auto ddp_config = DistributedDataParallelConfig{.zero_stage = FLAGS_zero_stage}; auto *mutable_chunks = @@ -438,7 +383,7 @@ void Train(const nn::parallel::Rank &rank) { mutable_chunks->at(chunk_id), rank, ddp_config); } } - } else if (dp_cp_world_size > 1) { + } else if (ddp_world_size > 1) { // NOTE(dcj): Complete all device (.to(device)) and dtype (.to(dtype)) // conversions before wrapping the model with DistributedDataParallel (DDP). // Otherwise, DDP’s gradient hooks may be lost because new parameter tensors @@ -510,8 +455,8 @@ void Train(const nn::parallel::Rank &rank) { ->mutable_chunks()) : std::vector>{model}; optimizer = std::make_shared( - optimizer_creator, named_parameters, model_chunks, dp_cp_world_size, - dp_cp_rank); + optimizer_creator, named_parameters, model_chunks, ddp_world_size, + ddp_rank); } else { optimizer = optimizer_creator(named_parameters); } @@ -594,7 +539,6 @@ void Train(const nn::parallel::Rank &rank) { .ddp_size = ddp_world_size, .tp_size = tp_world_size, .sp_size = sp_world_size, - .cp_size = cp_world_size, .pp_size = pp_world_size, .vpp_size = static_cast(FLAGS_virtual_pipeline_parallel), .checkpoint_root_dir = FLAGS_save, @@ -663,10 +607,6 @@ void Train(const nn::parallel::Rank &rank) { auto [x, y] = next_train_batch(); x = std::make_shared(x->To(device)); y = std::make_shared(y->To(device)); - if (cp_world_size > 1) { - x = nn::parallel::SliceAlongCPRegionFunc(x, 1); - y = nn::parallel::SliceAlongCPRegionFunc(y, 1); - } LOG(INFO) << "Rank " << rank.GlobalRank() << ": start forward"; // (bs, seq_len, vocab_size) @@ -685,7 +625,7 @@ void Train(const nn::parallel::Rank &rank) { LOG(INFO) << "Rank " << rank.GlobalRank() << ": start backward"; std::unique_ptr no_sync_guard; - if (dp_cp_world_size > 1 && micro_step != grad_accum_steps - 1) { + if (ddp_world_size > 1 && micro_step != grad_accum_steps - 1) { no_sync_guard = model->no_sync(); } loss->Backward(); @@ -705,10 +645,6 @@ void Train(const nn::parallel::Rank &rank) { auto [x, y] = next_train_batch(); x = std::make_shared(x->To(device)); y = std::make_shared(y->To(device)); - if (cp_world_size > 1) { - x = nn::parallel::SliceAlongCPRegionFunc(x, 1); - y = nn::parallel::SliceAlongCPRegionFunc(y, 1); - } lossf = model->TrainStep({x}, {y}, optimizer, loss_fn, dtype); if (scheduler) { @@ -716,10 +652,10 @@ void Train(const nn::parallel::Rank &rank) { } } - if (dp_cp_world_size > 1) { + if (ddp_world_size > 1) { auto lossf_tensor = std::make_shared( &lossf, std::vector{}, DataType::kFLOAT32, device); - comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, dp_cp_pg); + comm::AllReduce(lossf_tensor, comm::ReduceOpType::kAvg, ddp_pg); lossf = static_cast(lossf_tensor->To(Device()).DataPtr())[0]; } @@ -737,11 +673,10 @@ void Train(const nn::parallel::Rank &rank) { "step {:4d}/{} | train loss {:.6f} | lr {:.2e} | ({:.2f} ms | {:.0f} " "tok/s | " "peak used: {:5d} MB | peak reserved: {:5d} MB, DP={}, TP={}, SP={}, " - "CP={}, " "PP={})", step + 1, FLAGS_num_iteration, lossf, current_lr, duration_us / 1e3f, tps, used_mb, reserved_mb, ddp_world_size, tp_world_size, - sp_world_size, cp_world_size, pp_world_size); + sp_world_size, pp_world_size); if ((step + 1) % FLAGS_freq_generate_txt == 0) { // FIXME(jym): to support PP @@ -821,18 +756,12 @@ int main(int argc, char *argv[]) { FLAGS_tp_comm_overlap_disable_qkv; model_parallel_config.tp_comm_overlap_disable_fc1 = FLAGS_tp_comm_overlap_disable_fc1; - model_parallel_config.context_parallel.size = FLAGS_context_parallel; - model_parallel_config.context_parallel.comm_type = - nn::parallel::global::ParseContextParallelCommType(FLAGS_cp_comm_type); - model_parallel_config.context_parallel.hierarchical_sizes = - FLAGS_hierarchical_context_parallel_sizes; - model_parallel_config.context_parallel.attention_layout = - FLAGS_attention_cp_layout; model_parallel_config.expert_tensor_parallel_size = 1; nn::parallel::global::InitAllEnv(FLAGS_nthread_per_process, model_parallel_config); utils::PrecisionCheckEnv::Instance().Init(precision_config); + infini_train::ManualSeed(42); LOG(INFO) << nn::parallel::global::ProcessGroupOverview(); if (FLAGS_nthread_per_process > 1) { diff --git a/example/mixtral/main.cc b/example/mixtral/main.cc index 5846ae4f..32355776 100644 --- a/example/mixtral/main.cc +++ b/example/mixtral/main.cc @@ -34,6 +34,7 @@ #include "infini_train/include/core/runtime/device_guard.h" #include "infini_train/include/dataloader.h" #include "infini_train/include/device.h" +#include "infini_train/include/generator.h" #include "infini_train/include/nn/modules/loss.h" #include "infini_train/include/nn/modules/transformer/transformer.h" #include "infini_train/include/nn/parallel/comm.h" @@ -404,6 +405,7 @@ int main(int argc, char *argv[]) { FLAGS_expert_tensor_parallel; nn::parallel::global::InitAllEnv(FLAGS_nthread_per_process, model_parallel_config); + infini_train::ManualSeed(42); LOG(INFO) << nn::parallel::global::ProcessGroupOverview( nn::parallel::global::GetDenseRankGenerator(), diff --git a/example/mnist/main.cc b/example/mnist/main.cc index 8fc3b78e..ddc639d5 100644 --- a/example/mnist/main.cc +++ b/example/mnist/main.cc @@ -15,6 +15,7 @@ #include "infini_train/include/dataloader.h" #include "infini_train/include/device.h" +#include "infini_train/include/generator.h" #include "infini_train/include/nn/modules/loss.h" #include "infini_train/include/nn/modules/module.h" #include "infini_train/include/optimizer.h" @@ -51,6 +52,7 @@ int main(int argc, char *argv[]) { "Model options: --help\nAll options: --helpfull"); example::ParseModelFlags(&argc, &argv, "example/mnist/"); google::InitGoogleLogging(argv[0]); + infini_train::ManualSeed(42); auto train_dataset = std::make_shared(FLAGS_dataset, true); DataLoader train_dataloader(train_dataset, FLAGS_bs); diff --git a/infini_train/include/autograd/dropout.h b/infini_train/include/autograd/dropout.h new file mode 100644 index 00000000..972129d1 --- /dev/null +++ b/infini_train/include/autograd/dropout.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include +#include + +#include "infini_train/include/autograd/function.h" +#include "infini_train/include/generator.h" + +namespace infini_train { +class Tensor; +} + +namespace infini_train::autograd { + +class Dropout final : public Function { + public: + static constexpr char kType[] = "DropoutFunction"; + + Dropout(double p, std::optional generator) + : Function(kType), p_(p), generator_(std::move(generator)) {} + + std::vector> Forward( + const std::vector> &input_tensors) override; + void SetupContext( + const std::vector> &input_tensors, + const std::vector> &output_tensors) override; + std::vector> Backward( + const std::vector> &grad_outputs) override; + + private: + double p_ = 0.0; + std::optional generator_; +}; + +} // namespace infini_train::autograd diff --git a/infini_train/include/core/runtime/generator_backend.h b/infini_train/include/core/runtime/generator_backend.h new file mode 100644 index 00000000..0546582e --- /dev/null +++ b/infini_train/include/core/runtime/generator_backend.h @@ -0,0 +1,65 @@ +#pragma once + +// Internal Generator backend registration interface. Application code should +// include generator.h. + +#include +#include +#include + +#include "infini_train/include/common/common.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" + +namespace infini_train { + +class RngTracker; + +namespace detail { +const Generator *TryGetScopedGenerator(const Device &device); +bool TryManualSeedScopedRng(uint64_t seed); +RngTracker *GetCurrentRngTracker(); +} // namespace detail + +class GeneratorBackend { + public: + virtual ~GeneratorBackend() = default; + + virtual Device::DeviceType Type() const = 0; + virtual Generator Create(const Device &device, uint64_t seed) = 0; + virtual const Generator &GetDefault(const Device &device) = 0; + virtual void ManualSeedAll(uint64_t seed) = 0; +}; + +// Registers one backend per device type during static initialization. +class GeneratorBackendRegistry { + public: + static GeneratorBackendRegistry &Instance(); + + void Register(Device::DeviceType type, + std::unique_ptr backend); + // Throws if no backend is registered for the requested device type. + GeneratorBackend &Get(Device::DeviceType type) const; + void ManualSeedAll(uint64_t seed) const; + + private: + GeneratorBackendRegistry() = default; + GeneratorBackendRegistry(const GeneratorBackendRegistry &) = delete; + GeneratorBackendRegistry &operator=(const GeneratorBackendRegistry &) = + delete; + + std::unordered_map> + backends_; +}; + +} // namespace infini_train + +// CAT expands __COUNTER__ before token pasting so each registration has a +// unique name. +#define INFINI_TRAIN_REGISTER_GENERATOR_BACKEND(device_type, class_impl) \ + [[maybe_unused]] static const bool CAT( \ + kInfiniTrainGeneratorBackendRegistered, __COUNTER__) = []() { \ + infini_train::GeneratorBackendRegistry::Instance().Register( \ + device_type, std::make_unique()); \ + return true; \ + }(); diff --git a/infini_train/include/generator.h b/infini_train/include/generator.h new file mode 100644 index 00000000..1daed9bf --- /dev/null +++ b/infini_train/include/generator.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include + +#include "infini_train/include/device.h" + +namespace infini_train { + +class Tensor; +class GeneratorImpl; + +// A lightweight handle with shared-copy semantics. Use Clone() for an +// independent state. +class Generator { + public: + static constexpr uint64_t kDefaultSeed = 67280421310721; + + Generator() = delete; + + Generator(const Generator &) = default; + Generator &operator=(const Generator &) = default; + Generator(Generator &&) = default; + Generator &operator=(Generator &&) = default; + + ~Generator() = default; + + void set_current_seed(uint64_t seed) const; + uint64_t current_seed() const; + uint64_t Seed(); + void set_state(const Tensor &state); + std::shared_ptr get_state() const; + Device device() const; + Generator Clone() const; + + friend bool operator==(const Generator &a, const Generator &b) { + return a.impl_ == b.impl_; + } + friend bool operator!=(const Generator &a, const Generator &b) { + return !(a == b); + } + + private: + friend class GeneratorAccessor; + + explicit Generator(std::shared_ptr impl); + std::shared_ptr impl_; +}; + +Generator CreateGenerator(const Device &device, + uint64_t seed = Generator::kDefaultSeed); + +// Returns the current RNG stream's generator, or the process default when no +// scope is active. Scoped references expire when the tracker is destroyed or +// restored; copy the handle to retain it. +const Generator &GetDefaultGenerator(const Device &device); + +// Resets the current RNG stream, or all enabled process defaults when no scope +// is active. +void ManualSeed(uint64_t seed); + +} // namespace infini_train diff --git a/infini_train/include/generator_impl.h b/infini_train/include/generator_impl.h new file mode 100644 index 00000000..fffe8c07 --- /dev/null +++ b/infini_train/include/generator_impl.h @@ -0,0 +1,86 @@ +#pragma once + +// Internal Generator implementation interface. Application code should include +// generator.h. + +#include +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" + +namespace infini_train { + +class Tensor; + +class GeneratorImpl { + public: + explicit GeneratorImpl(Device device) : device_(device) {} + virtual ~GeneratorImpl() = default; + + GeneratorImpl(const GeneratorImpl &other) = delete; + GeneratorImpl(GeneratorImpl &&other) = delete; + GeneratorImpl &operator=(const GeneratorImpl &other) = delete; + GeneratorImpl &operator=(GeneratorImpl &&other) = delete; + + virtual void set_current_seed(uint64_t seed) = 0; + virtual uint64_t current_seed() const = 0; + virtual uint64_t Seed() = 0; + virtual void set_state(const Tensor &state) = 0; + virtual std::shared_ptr get_state() const = 0; + + std::shared_ptr Clone() const; + Device device() const; + + // Callers must hold this mutex when accessing shared RNG state concurrently. + std::mutex mutex_; + + protected: + Device device_; + virtual GeneratorImpl *CloneImpl() const = 0; +}; + +class GeneratorAccessor { + public: + static GeneratorImpl &Get(const Generator &generator); + static std::mutex &Mutex(const Generator &generator); + static Generator FromImpl(std::shared_ptr impl); +}; + +template +T &CheckedGeneratorImpl(const Generator &generator, + const Device &expected_device) { + static_assert(std::is_base_of_v); + auto &base = GeneratorAccessor::Get(generator); + CHECK(base.device() == expected_device) << "Generator device mismatch"; + auto *typed = dynamic_cast(&base); + CHECK(typed != nullptr) << "Generator backend mismatch"; + return *typed; +} + +template +Generator MakeGenerator(Args &&...args) { + return GeneratorAccessor::FromImpl( + std::make_shared(std::forward(args)...)); +} + +template +T &GetGeneratorOrDefault(const std::optional &generator, + const Generator &default_generator, + const Device &expected_device) { + const Generator &chosen = + generator.has_value() ? *generator : default_generator; + return CheckedGeneratorImpl(chosen, expected_device); +} + +namespace detail { +void CheckRngState(const Tensor &state); +} // namespace detail + +} // namespace infini_train diff --git a/infini_train/include/nn/functional.h b/infini_train/include/nn/functional.h index da17c969..ba5fe742 100644 --- a/infini_train/include/nn/functional.h +++ b/infini_train/include/nn/functional.h @@ -2,8 +2,13 @@ #include #include +#include #include +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" + namespace infini_train { class Tensor; } @@ -51,6 +56,25 @@ std::shared_ptr Triu(const std::shared_ptr &input, // A tensor of the given shape filled with the scalar value 1. std::shared_ptr Ones(const std::vector size); +// Returns a tensor with uniformly distributed random values in [0, 1). +std::shared_ptr Rand(const std::vector &size, + DataType dtype = DataType::kFLOAT32, + Device device = Device(), + std::optional generator = std::nullopt, + bool requires_grad = false); + +// Returns a tensor with normally distributed random values with mean 0 and +// standard deviation 1. +std::shared_ptr Randn(const std::vector &size, + DataType dtype = DataType::kFLOAT32, + Device device = Device(), + std::optional generator = std::nullopt, + bool requires_grad = false); + +std::shared_ptr Dropout( + const std::shared_ptr &input, double p = 0.5, bool training = true, + std::optional generator = std::nullopt); + // Returns a new tensor with the reciprocal of the elements of input. // // Args: diff --git a/infini_train/include/nn/init.h b/infini_train/include/nn/init.h index f389d2c1..9c04afcc 100644 --- a/infini_train/include/nn/init.h +++ b/infini_train/include/nn/init.h @@ -1,13 +1,12 @@ #pragma once -#include - +#include #include #include -#include #include #include "infini_train/include/device.h" +#include "infini_train/include/generator.h" namespace infini_train { class Tensor; @@ -17,7 +16,7 @@ enum class DataType : int8_t; namespace infini_train::nn::init { std::shared_ptr Normal( const std::shared_ptr &tensor, float mean = 0.0, float std = 1.0, - std::optional generator = std::nullopt); + std::optional generator = std::nullopt); std::pair CalculateFanInAndFanOut( const std::shared_ptr &tensor); @@ -46,11 +45,11 @@ std::shared_ptr KaimingUniform( const std::shared_ptr &tensor, float a = 0.0f, KaimingMode mode = KaimingMode::kFanIn, NonLinearityType non_linearity = NonLinearityType::kLeakyReLU, - std::optional generator = std::nullopt); + std::optional generator = std::nullopt); std::shared_ptr Uniform( const std::shared_ptr &tensor, float a = 0.0f, float b = 1.0f, - std::optional generator = std::nullopt); + std::optional generator = std::nullopt); std::shared_ptr Ones(const std::shared_ptr &tensor); diff --git a/infini_train/include/nn/parallel/ddp/distributed_data_parallel.h b/infini_train/include/nn/parallel/ddp/distributed_data_parallel.h index ec3ff88f..44313648 100644 --- a/infini_train/include/nn/parallel/ddp/distributed_data_parallel.h +++ b/infini_train/include/nn/parallel/ddp/distributed_data_parallel.h @@ -67,6 +67,7 @@ class DistributedDataParallel : public nn::Module { const std::vector> ¶ms); void OnGradReady(const std::shared_ptr ¶m); void SetIsLastMicrobatch(bool is_last_microbatch); + void SynchronizeModuleState(); private: // Whether to enable grad sync on last microbatch (DDP naive path) diff --git a/infini_train/include/nn/parallel/tensor_parallel.h b/infini_train/include/nn/parallel/tensor_parallel.h index eb5ceeee..0c9f7ef5 100644 --- a/infini_train/include/nn/parallel/tensor_parallel.h +++ b/infini_train/include/nn/parallel/tensor_parallel.h @@ -15,6 +15,8 @@ class Tensor; namespace infini_train::nn::parallel { +inline constexpr char kTensorParallelInitRngName[] = "tensor-parallel-init"; + // NOTE(zbl): Reserved for VocabParallelEmbedding, since rank is needed in its // constructor before any Device exists // On other occasions, should use Device::Rank() diff --git a/infini_train/include/rng_tracker.h b/infini_train/include/rng_tracker.h new file mode 100644 index 00000000..b33bfff1 --- /dev/null +++ b/infini_train/include/rng_tracker.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" + +namespace infini_train { + +class Tensor; +class RngTracker; + +namespace detail { +const Generator *TryGetScopedGenerator(const Device &device); +bool TryManualSeedScopedRng(uint64_t seed); +RngTracker *GetCurrentRngTracker(); +} // namespace detail + +// Owns independent, named RNG streams for each device. Child threads do not +// inherit scopes. +class RngTracker { + public: + explicit RngTracker(uint64_t seed = Generator::kDefaultSeed); + ~RngTracker(); + + RngTracker(const RngTracker &) = delete; + RngTracker &operator=(const RngTracker &) = delete; + RngTracker(RngTracker &&) = delete; + RngTracker &operator=(RngTracker &&) = delete; + + void Add(const std::string &name, uint64_t seed); + bool Contains(const std::string &name) const; + + void ManualSeed(uint64_t seed); + void ManualSeed(const std::string &name, uint64_t seed); + + // Returns an independent CPU UINT8 tensor containing all streams. + std::shared_ptr GetState() const; + // Restoring invalidates references to this tracker's default generators. + void SetState(const Tensor &state); + + private: + friend class RngScope; + friend const Generator *detail::TryGetScopedGenerator(const Device &device); + + using DeviceKey = std::pair; + + struct RngStream { + uint64_t seed; + std::map generators; + }; + + using Streams = std::map; + + mutable std::recursive_mutex mutex_; + Streams streams_; +}; + +// Selects a stream on the current thread. External trackers must outlive their +// scopes. Concurrent scopes on different threads cannot share a tracker; +// nesting on one thread is supported. Owning scopes start fresh; binding an +// existing tracker retains its progress. +class RngScope { + public: + RngScope(); + explicit RngScope(uint64_t seed); + explicit RngScope(RngTracker &tracker); + RngScope(RngTracker &tracker, const std::string &name); + ~RngScope(); + + RngScope(const RngScope &) = delete; + RngScope &operator=(const RngScope &) = delete; + RngScope(RngScope &&) = delete; + RngScope &operator=(RngScope &&) = delete; + + private: + friend class RngTracker; + friend const Generator *detail::TryGetScopedGenerator(const Device &device); + friend bool detail::TryManualSeedScopedRng(uint64_t seed); + friend RngTracker *detail::GetCurrentRngTracker(); + + std::unique_ptr owned_tracker_; + RngTracker *tracker_; + std::string name_; + RngScope *previous_; + std::thread::id thread_; + std::unique_lock lock_; +}; + +} // namespace infini_train diff --git a/infini_train/include/tensor.h b/infini_train/include/tensor.h index adf4d2f1..17ab5e5c 100644 --- a/infini_train/include/tensor.h +++ b/infini_train/include/tensor.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -14,6 +13,8 @@ #include "infini_train/include/datatype.h" #include "infini_train/include/device.h" +// TODO: Reduce the Tensor header dependency on Generator. +#include "infini_train/include/generator.h" #include "infini_train/include/scalar.h" namespace infini_train { @@ -78,6 +79,8 @@ class Tensor : public std::enable_shared_from_this { void *DataPtr(); const void *DataPtr() const; + bool defined() const; + size_t SizeInBytes() const; const std::vector &Dims() const; @@ -165,7 +168,7 @@ class Tensor : public std::enable_shared_from_this { // distribution std::shared_ptr Uniform( float from = 0.0f, float to = 1.0f, - std::optional generator = std::nullopt); + std::optional generator = std::nullopt); std::shared_ptr Matmul(const std::shared_ptr &other); std::shared_ptr Outer(const std::shared_ptr &other); diff --git a/infini_train/src/autograd/dropout.cc b/infini_train/src/autograd/dropout.cc new file mode 100644 index 00000000..453c5beb --- /dev/null +++ b/infini_train/src/autograd/dropout.cc @@ -0,0 +1,54 @@ +#include "infini_train/include/autograd/dropout.h" + +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" +#include "infini_train/include/tensor.h" + +namespace infini_train::autograd { + +std::vector> Dropout::Forward( + const std::vector> &input_tensors) { + CHECK_EQ(input_tensors.size(), 1); + const auto &input = input_tensors[0]; + + auto device = input->GetDevice().type(); + auto outputs = + Dispatcher::Instance() + .Call, std::shared_ptr>>( + {device, "DropoutForward"}, input, p_, generator_); + // Keep the mask as a non-differentiable output so SetupContext can save it + // for backward. + return {std::get<0>(outputs), std::get<1>(outputs)}; +} + +void Dropout::SetupContext( + const std::vector> &, + const std::vector> &output_tensors) { + CHECK_EQ(output_tensors.size(), 2); + ctx_.MarkNonDifferentiable({output_tensors[1]}); + if (!ctx_.needs_input_grad().empty() && ctx_.needs_input_grad()[0]) { + ctx_.SaveForBackward({output_tensors[1]}); + } +} + +std::vector> Dropout::Backward( + const std::vector> &grad_outputs) { + // The mask output is non-differentiable, so its gradient slot is empty. + CHECK_EQ(grad_outputs.size(), 2); + auto saved_tensors = ctx_.GetSavedTensors(); + CHECK_EQ(saved_tensors.size(), 1); + const auto &grad_output = grad_outputs[0]; + const auto &mask = saved_tensors[0]; + + auto device = grad_output->GetDevice().type(); + return {Dispatcher::Instance().Call>( + {device, "DropoutBackward"}, grad_output, mask, p_)}; +} + +} // namespace infini_train::autograd diff --git a/infini_train/src/core/runtime/cpu/cpu_generator_impl.cc b/infini_train/src/core/runtime/cpu/cpu_generator_impl.cc new file mode 100644 index 00000000..3f7f553b --- /dev/null +++ b/infini_train/src/core/runtime/cpu/cpu_generator_impl.cc @@ -0,0 +1,307 @@ +#include "infini_train/src/core/runtime/cpu/cpu_generator_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/core/runtime/generator_backend.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +namespace infini_train::core::cpu { +namespace { + +// Backend tag used to reject states from other generator implementations. +constexpr char kCpuStateMagic[] = "ITRNGCPU"; +constexpr std::size_t kStateMagicSize = sizeof(kCpuStateMagic) - 1; + +// Fixed footer after the variable-length mt19937 stream. Cache flags are +// serialized as uint8_t; absent cache values are serialized as zero. +constexpr std::size_t kSerializedSeedSize = sizeof(uint64_t); +constexpr std::size_t kSerializedFloatCacheFlagSize = sizeof(uint8_t); +constexpr std::size_t kSerializedFloatCacheValueSize = sizeof(float); +constexpr std::size_t kSerializedDoubleCacheFlagSize = sizeof(uint8_t); +constexpr std::size_t kSerializedDoubleCacheValueSize = sizeof(double); +constexpr std::size_t kStateFooterSize = + kSerializedSeedSize + kSerializedFloatCacheFlagSize + + kSerializedFloatCacheValueSize + kSerializedDoubleCacheFlagSize + + kSerializedDoubleCacheValueSize; + +// Seed through seed_seq using both 32-bit halves of the seed. This produces a +// different sequence from direct std::mt19937(seed) initialization, even when +// the seed fits in 32 bits. +std::mt19937 MakeSeededEngine(uint64_t seed) { + std::seed_seq seq{static_cast(seed), + static_cast(seed >> 32)}; + std::mt19937 engine; + engine.seed(seq); + return engine; +} + +uint64_t GenerateNonDeterministicSeed() { + std::random_device rd; + uint64_t val = (static_cast(rd()) << 32) | rd(); + return val; +} + +// Copies state bytes and advances offset; data and value must reference valid +// buffers of data_size and value_size bytes, and offset must be a valid +// pointer. +void WriteBytes(uint8_t *data, std::size_t data_size, std::size_t *offset, + const void *value, std::size_t value_size) { + CHECK_LE(*offset, data_size) + << "CPU generator state write offset out of bounds"; + CHECK_LE(value_size, data_size - *offset) + << "CPU generator state write exceeds buffer"; + if (value_size != 0) { + std::memcpy(data + *offset, value, value_size); + } + *offset += value_size; +} + +void ReadBytes(const uint8_t *data, std::size_t data_size, std::size_t *offset, + void *value, std::size_t value_size) { + CHECK_LE(*offset, data_size) + << "CPU generator state read offset out of bounds"; + CHECK_LE(value_size, data_size - *offset) + << "CPU generator state read exceeds buffer"; + if (value_size != 0) { + std::memcpy(value, data + *offset, value_size); + } + *offset += value_size; +} + +} // namespace + +CpuGeneratorImpl::CpuGeneratorImpl(uint64_t seed) + : GeneratorImpl(Device(Device::DeviceType::kCPU, 0)), + engine_(MakeSeededEngine(seed)), + seed_(seed) {} + +void CpuGeneratorImpl::set_current_seed(uint64_t seed) { + seed_ = seed; + next_float_normal_sample_.reset(); + next_double_normal_sample_.reset(); + engine_ = MakeSeededEngine(seed); +} + +uint64_t CpuGeneratorImpl::current_seed() const { return seed_; } + +uint64_t CpuGeneratorImpl::Seed() { + uint64_t random_seed = GenerateNonDeterministicSeed(); + set_current_seed(random_seed); + return random_seed; +} + +// State layout: magic, serialized engine, seed, and cached float/double normal +// samples. The std::mt19937 stream format is only portable across compatible +// standard-library builds. + +void CpuGeneratorImpl::set_state(const Tensor &state) { + ::infini_train::detail::CheckRngState(state); + + const size_t data_size = state.SizeInBytes(); + CHECK_GT(data_size, kStateMagicSize + kStateFooterSize) + << "CPU generator state is too small"; + + const uint8_t *data = static_cast(state.DataPtr()); + CHECK_EQ(std::memcmp(data, kCpuStateMagic, kStateMagicSize), 0) + << "Invalid RNG state: not a CPU generator state (backend magic " + "mismatch)"; + + const size_t engine_size = data_size - kStateMagicSize - kStateFooterSize; + std::string engine_str(reinterpret_cast(data + kStateMagicSize), + engine_size); + + std::istringstream iss(engine_str); + std::mt19937 restored_engine; + iss >> restored_engine; + CHECK(!iss.fail()) << "Invalid CPU generator engine state"; + iss >> std::ws; + CHECK(iss.eof()) << "Invalid trailing bytes in CPU generator engine state"; + + size_t offset = kStateMagicSize + engine_size; + uint64_t restored_seed = 0; + ReadBytes(data, data_size, &offset, &restored_seed, kSerializedSeedSize); + uint8_t has_float = 0; + ReadBytes(data, data_size, &offset, &has_float, + kSerializedFloatCacheFlagSize); + CHECK_LE(has_float, 1) << "Invalid CPU generator float normal cache flag"; + float restored_float = 0.0f; + ReadBytes(data, data_size, &offset, &restored_float, + kSerializedFloatCacheValueSize); + uint8_t has_double = 0; + ReadBytes(data, data_size, &offset, &has_double, + kSerializedDoubleCacheFlagSize); + CHECK_LE(has_double, 1) << "Invalid CPU generator double normal cache flag"; + double restored_double = 0.0; + ReadBytes(data, data_size, &offset, &restored_double, + kSerializedDoubleCacheValueSize); + CHECK_EQ(offset, data_size) << "CPU generator state size mismatch"; + + // Do not change the generator until the complete state has been validated. + engine_ = restored_engine; + seed_ = restored_seed; + next_float_normal_sample_ = + has_float ? std::optional(restored_float) : std::nullopt; + next_double_normal_sample_ = + has_double ? std::optional(restored_double) : std::nullopt; +} + +std::shared_ptr CpuGeneratorImpl::get_state() const { + std::ostringstream oss; + oss << engine_; + std::string engine_str = oss.str(); + + const size_t engine_size = engine_str.size(); + const size_t total_size = kStateMagicSize + engine_size + kStateFooterSize; + + auto state_tensor = std::make_shared( + std::vector{static_cast(total_size)}, DataType::kUINT8, + Device(Device::DeviceType::kCPU, 0)); + + uint8_t *data = static_cast(state_tensor->DataPtr()); + size_t offset = 0; + + const uint8_t has_float = next_float_normal_sample_.has_value() ? 1 : 0; + const float float_val = has_float ? *next_float_normal_sample_ : 0.0f; + const uint8_t has_double = next_double_normal_sample_.has_value() ? 1 : 0; + const double double_val = has_double ? *next_double_normal_sample_ : 0.0; + + WriteBytes(data, total_size, &offset, kCpuStateMagic, kStateMagicSize); + WriteBytes(data, total_size, &offset, engine_str.data(), engine_size); + WriteBytes(data, total_size, &offset, &seed_, kSerializedSeedSize); + WriteBytes(data, total_size, &offset, &has_float, + kSerializedFloatCacheFlagSize); + WriteBytes(data, total_size, &offset, &float_val, + kSerializedFloatCacheValueSize); + WriteBytes(data, total_size, &offset, &has_double, + kSerializedDoubleCacheFlagSize); + WriteBytes(data, total_size, &offset, &double_val, + kSerializedDoubleCacheValueSize); + CHECK_EQ(offset, total_size) << "CPU generator state size mismatch"; + + return state_tensor; +} + +uint32_t CpuGeneratorImpl::Random() { return engine_(); } + +uint64_t CpuGeneratorImpl::Random64() { + uint32_t hi = engine_(); + uint32_t lo = engine_(); + return (static_cast(hi) << 32) | lo; +} + +std::optional CpuGeneratorImpl::next_float_normal_sample() const { + return next_float_normal_sample_; +} + +std::optional CpuGeneratorImpl::next_double_normal_sample() const { + return next_double_normal_sample_; +} + +void CpuGeneratorImpl::set_next_float_normal_sample( + std::optional randn) { + next_float_normal_sample_ = randn; +} + +void CpuGeneratorImpl::set_next_double_normal_sample( + std::optional randn) { + next_double_normal_sample_ = randn; +} + +std::shared_ptr CpuGeneratorImpl::Clone() const { + return std::shared_ptr(CloneImpl()); +} + +std::mt19937 CpuGeneratorImpl::engine() const { return engine_; } + +CpuGeneratorImpl *CpuGeneratorImpl::CloneImpl() const { + auto *clone = new CpuGeneratorImpl(seed_); + clone->set_engine(engine_); + clone->set_next_float_normal_sample(next_float_normal_sample_); + clone->set_next_double_normal_sample(next_double_normal_sample_); + return clone; +} + +void CpuGeneratorImpl::set_engine(std::mt19937 engine) { + engine_ = std::move(engine); +} + +namespace { + +struct DefaultCpuGeneratorState { + std::once_flag init_once; + std::optional generator; +}; + +DefaultCpuGeneratorState &GetDefaultCpuGeneratorState() { + static DefaultCpuGeneratorState state; + return state; +} + +} // namespace + +const Generator &GetDefaultCpuGenerator() { + if (const auto *generator = detail::TryGetScopedGenerator(Device())) { + return *generator; + } + auto &state = GetDefaultCpuGeneratorState(); + std::call_once(state.init_once, [&state] { + state.generator.emplace(CreateCpuGenerator(GenerateNonDeterministicSeed())); + }); + return *state.generator; +} + +Generator CreateCpuGenerator(uint64_t seed) { + return MakeGenerator(seed); +} + +void ManualSeed(uint64_t seed) { + auto &state = GetDefaultCpuGeneratorState(); + bool initialized_here = false; + std::call_once(state.init_once, [&state, seed, &initialized_here] { + state.generator.emplace(CreateCpuGenerator(seed)); + initialized_here = true; + }); + if (initialized_here) { + return; + } + const Generator &generator = *state.generator; + std::lock_guard lock(GeneratorAccessor::Mutex(generator)); + generator.set_current_seed(seed); +} + +namespace { + +class CpuGeneratorBackend final : public GeneratorBackend { + public: + Device::DeviceType Type() const override { return Device::DeviceType::kCPU; } + + Generator Create(const Device & /*device*/, uint64_t seed) override { + return CreateCpuGenerator(seed); + } + + const Generator &GetDefault(const Device & /*device*/) override { + return GetDefaultCpuGenerator(); + } + + void ManualSeedAll(uint64_t seed) override { + ::infini_train::core::cpu::ManualSeed(seed); + } +}; + +INFINI_TRAIN_REGISTER_GENERATOR_BACKEND(Device::DeviceType::kCPU, + CpuGeneratorBackend); + +} // namespace +} // namespace infini_train::core::cpu diff --git a/infini_train/src/core/runtime/cpu/cpu_generator_impl.h b/infini_train/src/core/runtime/cpu/cpu_generator_impl.h new file mode 100644 index 00000000..f08a4015 --- /dev/null +++ b/infini_train/src/core/runtime/cpu/cpu_generator_impl.h @@ -0,0 +1,52 @@ +#pragma once + +#include +#include +#include +#include + +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +namespace infini_train::core::cpu { + +// CPU generator backed by std::mt19937 with cached Box-Muller samples. +class CpuGeneratorImpl final : public GeneratorImpl { + public: + explicit CpuGeneratorImpl(uint64_t seed = Generator::kDefaultSeed); + ~CpuGeneratorImpl() override = default; + + void set_current_seed(uint64_t seed) override; + uint64_t current_seed() const override; + uint64_t Seed() override; + void set_state(const Tensor &state) override; + std::shared_ptr get_state() const override; + + std::shared_ptr Clone() const; + + uint32_t Random(); + uint64_t Random64(); + + std::optional next_float_normal_sample() const; + std::optional next_double_normal_sample() const; + void set_next_float_normal_sample(std::optional randn); + void set_next_double_normal_sample(std::optional randn); + + private: + CpuGeneratorImpl *CloneImpl() const override; + + std::mt19937 engine() const; + void set_engine(std::mt19937 engine); + + std::mt19937 engine_; + uint64_t seed_ = Generator::kDefaultSeed; + std::optional next_float_normal_sample_; + std::optional next_double_normal_sample_; +}; + +const Generator &GetDefaultCpuGenerator(); +Generator CreateCpuGenerator(uint64_t seed); +void ManualSeed(uint64_t seed); + +} // namespace infini_train::core::cpu diff --git a/infini_train/src/core/runtime/cuda/cuda_generator_impl.cc b/infini_train/src/core/runtime/cuda/cuda_generator_impl.cc new file mode 100644 index 00000000..6f96c6f5 --- /dev/null +++ b/infini_train/src/core/runtime/cuda/cuda_generator_impl.cc @@ -0,0 +1,191 @@ +#include "infini_train/src/core/runtime/cuda/cuda_generator_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "glog/logging.h" + +#include "infini_train/include/common/cuda/common_cuda.h" +#include "infini_train/include/core/runtime/generator_backend.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +namespace infini_train::core::cuda { +namespace { + +// Backend tag used to reject states from other generator implementations. +constexpr char kCudaStateMagic[] = "ITRNGCUD"; +constexpr size_t kStateMagicSize = sizeof(kCudaStateMagic) - 1; + +// [magic: 8B] + [seed_: 8B] + [next_philox_subsequence_: 8B] +constexpr size_t kStateSize = kStateMagicSize + sizeof(uint64_t) * 2; + +std::once_flag default_generators_init_flag; +std::vector> default_generators; +std::deque default_generator_init_flags; + +uint64_t GenerateNonDeterministicSeed() { + std::random_device random_device; + return (static_cast(random_device()) << 32) | random_device(); +} + +void InitDefaultGenerators() { + std::call_once(default_generators_init_flag, [] { + int device_count = 0; + const cudaError_t status = cudaGetDeviceCount(&device_count); + if (status == cudaErrorNoDevice) { + cudaGetLastError(); + return; + } + CHECK_EQ(status, cudaSuccess) + << "cudaGetDeviceCount failed: " << cudaGetErrorString(status); + default_generators.resize(device_count); + default_generator_init_flags.resize(device_count); + }); +} + +int ResolveDeviceIndex(int8_t device_index) { + InitDefaultGenerators(); + int index = device_index; + if (index == -1) { + CUDA_CHECK(cudaGetDevice(&index)); + } + int device_count = 0; + device_count = static_cast(default_generators.size()); + CHECK(index >= 0 && index < device_count) + << "Invalid CUDA device index " << index; + return index; +} + +} // namespace + +CudaGeneratorImpl::CudaGeneratorImpl(int8_t device_index, uint64_t seed) + : GeneratorImpl(Device(Device::DeviceType::kCUDA, device_index)), + seed_(seed) {} + +void CudaGeneratorImpl::set_current_seed(uint64_t seed) { + seed_ = seed; + next_philox_subsequence_ = 0; +} + +uint64_t CudaGeneratorImpl::current_seed() const { return seed_; } + +uint64_t CudaGeneratorImpl::Seed() { + const uint64_t random_seed = GenerateNonDeterministicSeed(); + set_current_seed(random_seed); + return random_seed; +} + +void CudaGeneratorImpl::set_state(const Tensor &state) { + ::infini_train::detail::CheckRngState(state); + CHECK_EQ(state.SizeInBytes(), kStateSize); + + const auto *data = static_cast(state.DataPtr()); + CHECK_EQ(std::memcmp(data, kCudaStateMagic, kStateMagicSize), 0) + << "Invalid RNG state: not a CUDA generator state (backend magic " + "mismatch)"; + std::memcpy(&seed_, data + kStateMagicSize, sizeof(seed_)); + std::memcpy(&next_philox_subsequence_, data + kStateMagicSize + sizeof(seed_), + sizeof(next_philox_subsequence_)); +} + +std::shared_ptr CudaGeneratorImpl::get_state() const { + auto state = std::make_shared( + std::vector{static_cast(kStateSize)}, DataType::kUINT8, + Device(Device::DeviceType::kCPU, 0)); + + auto *data = static_cast(state->DataPtr()); + std::memcpy(data, kCudaStateMagic, kStateMagicSize); + std::memcpy(data + kStateMagicSize, &seed_, sizeof(seed_)); + std::memcpy(data + kStateMagicSize + sizeof(seed_), &next_philox_subsequence_, + sizeof(next_philox_subsequence_)); + return state; +} + +uint64_t CudaGeneratorImpl::ReservePhiloxSubsequence(uint64_t increment) { + CHECK_LE(increment, + std::numeric_limits::max() - next_philox_subsequence_) + << "Philox subsequence counter overflow"; + const uint64_t subsequence = next_philox_subsequence_; + next_philox_subsequence_ += increment; + return subsequence; +} + +CudaGeneratorImpl *CudaGeneratorImpl::CloneImpl() const { + auto *clone = new CudaGeneratorImpl(device().index(), seed_); + clone->next_philox_subsequence_ = next_philox_subsequence_; + return clone; +} + +const Generator &GetDefaultCudaGenerator(int8_t device_index) { + const int index = ResolveDeviceIndex(device_index); + if (const auto *generator = detail::TryGetScopedGenerator( + Device(Device::DeviceType::kCUDA, index))) { + return *generator; + } + std::call_once(default_generator_init_flags[index], [index] { + default_generators[index].emplace(CreateCudaGenerator( + static_cast(index), GenerateNonDeterministicSeed())); + }); + return *default_generators[index]; +} + +Generator CreateCudaGenerator(int8_t device_index, uint64_t seed) { + const int index = ResolveDeviceIndex(device_index); + return MakeGenerator(static_cast(index), seed); +} + +void ManualSeedAll(uint64_t seed) { + InitDefaultGenerators(); + for (size_t index = 0; index < default_generators.size(); ++index) { + bool initialized_here = false; + std::call_once(default_generator_init_flags[index], + [index, seed, &initialized_here] { + default_generators[index].emplace( + CreateCudaGenerator(static_cast(index), seed)); + initialized_here = true; + }); + if (initialized_here) { + continue; + } + const Generator &generator = *default_generators[index]; + std::lock_guard lock(GeneratorAccessor::Mutex(generator)); + generator.set_current_seed(seed); + } +} + +namespace { + +class CudaGeneratorBackend final : public GeneratorBackend { + public: + Device::DeviceType Type() const override { return Device::DeviceType::kCUDA; } + + Generator Create(const Device &device, uint64_t seed) override { + return CreateCudaGenerator(device.index(), seed); + } + + const Generator &GetDefault(const Device &device) override { + return GetDefaultCudaGenerator(device.index()); + } + + void ManualSeedAll(uint64_t seed) override { + ::infini_train::core::cuda::ManualSeedAll(seed); + } +}; + +INFINI_TRAIN_REGISTER_GENERATOR_BACKEND(Device::DeviceType::kCUDA, + CudaGeneratorBackend); + +} // namespace +} // namespace infini_train::core::cuda diff --git a/infini_train/src/core/runtime/cuda/cuda_generator_impl.h b/infini_train/src/core/runtime/cuda/cuda_generator_impl.h new file mode 100644 index 00000000..d1fc350f --- /dev/null +++ b/infini_train/src/core/runtime/cuda/cuda_generator_impl.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" + +namespace infini_train::core::cuda { + +class CudaGeneratorImpl final : public GeneratorImpl { + public: + explicit CudaGeneratorImpl(int8_t device_index, + uint64_t seed = Generator::kDefaultSeed); + ~CudaGeneratorImpl() override = default; + + void set_current_seed(uint64_t seed) override; + uint64_t current_seed() const override; + uint64_t Seed() override; + void set_state(const Tensor &state) override; + std::shared_ptr get_state() const override; + + // The caller must hold mutex_ while reserving Philox subsequences. + uint64_t ReservePhiloxSubsequence(uint64_t increment); + + private: + CudaGeneratorImpl *CloneImpl() const override; + + uint64_t seed_ = Generator::kDefaultSeed; + uint64_t next_philox_subsequence_ = 0; +}; + +const Generator &GetDefaultCudaGenerator(int8_t device_index = -1); +Generator CreateCudaGenerator(int8_t device_index, + uint64_t seed = Generator::kDefaultSeed); +void ManualSeedAll(uint64_t seed); + +} // namespace infini_train::core::cuda diff --git a/infini_train/src/core/runtime/cuda/cuda_guard_impl.cc b/infini_train/src/core/runtime/cuda/cuda_guard_impl.cc index 8174b89d..98534866 100644 --- a/infini_train/src/core/runtime/cuda/cuda_guard_impl.cc +++ b/infini_train/src/core/runtime/cuda/cuda_guard_impl.cc @@ -112,7 +112,12 @@ void CudaGuardImpl::SetDevice(Device device) const { int CudaGuardImpl::DeviceCount() const { int device_count = 0; - CUDA_DRIVER_CHECK(cuDeviceGetCount(&device_count)); + const cudaError_t cuda_status = cudaGetDeviceCount(&device_count); + if (cuda_status == cudaErrorNoDevice) { + cudaGetLastError(); + return 0; + } + CUDA_CHECK(cuda_status); return device_count; } diff --git a/infini_train/src/core/runtime/generator_backend.cc b/infini_train/src/core/runtime/generator_backend.cc new file mode 100644 index 00000000..1f6949f9 --- /dev/null +++ b/infini_train/src/core/runtime/generator_backend.cc @@ -0,0 +1,45 @@ +#include "infini_train/include/core/runtime/generator_backend.h" + +#include +#include +#include + +#include "glog/logging.h" + +namespace infini_train { + +GeneratorBackendRegistry &GeneratorBackendRegistry::Instance() { + static GeneratorBackendRegistry instance; + return instance; +} + +void GeneratorBackendRegistry::Register( + Device::DeviceType type, std::unique_ptr backend) { + CHECK(backend != nullptr) << "Registering a null GeneratorBackend"; + // DeviceType has no stream insertion operator. + CHECK_EQ(static_cast(type), static_cast(backend->Type())) + << "GeneratorBackend type mismatch: registered as " + << static_cast(type) << " but Type() is " + << static_cast(backend->Type()); + CHECK(!backends_.contains(type)) + << "GeneratorBackend for device type " << static_cast(type) + << " is already registered"; + + backends_.emplace(type, std::move(backend)); +} + +GeneratorBackend &GeneratorBackendRegistry::Get(Device::DeviceType type) const { + auto it = backends_.find(type); + CHECK(it != backends_.end()) + << "No GeneratorBackend registered for device type " + << std::to_string(static_cast(type)); + return *it->second; +} + +void GeneratorBackendRegistry::ManualSeedAll(uint64_t seed) const { + for (const auto &entry : backends_) { + entry.second->ManualSeedAll(seed); + } +} + +} // namespace infini_train diff --git a/infini_train/src/generator.cc b/infini_train/src/generator.cc new file mode 100644 index 00000000..a48fbf2b --- /dev/null +++ b/infini_train/src/generator.cc @@ -0,0 +1,92 @@ +#include "infini_train/include/generator.h" + +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/core/runtime/generator_backend.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +namespace infini_train { + +Generator::Generator(std::shared_ptr impl) + : impl_(std::move(impl)) { + CHECK(impl_) << "GeneratorImpl with nullptr is not supported"; +} + +void Generator::set_current_seed(uint64_t seed) const { + impl_->set_current_seed(seed); +} + +uint64_t Generator::current_seed() const { return impl_->current_seed(); } + +uint64_t Generator::Seed() { return impl_->Seed(); } + +void Generator::set_state(const Tensor &state) { + CHECK(state.defined()) << "Undefined tensor is not allowed"; + impl_->set_state(state); +} + +std::shared_ptr Generator::get_state() const { + return impl_->get_state(); +} + +Device Generator::device() const { return impl_->device(); } + +Generator Generator::Clone() const { return Generator(impl_->Clone()); } + +std::shared_ptr GeneratorImpl::Clone() const { + return std::shared_ptr(CloneImpl()); +} + +Device GeneratorImpl::device() const { return device_; } + +GeneratorImpl &GeneratorAccessor::Get(const Generator &generator) { + CHECK(generator.impl_) << "Undefined Generator"; + return *generator.impl_; +} + +std::mutex &GeneratorAccessor::Mutex(const Generator &generator) { + return Get(generator).mutex_; +} + +Generator GeneratorAccessor::FromImpl(std::shared_ptr impl) { + return Generator(std::move(impl)); +} + +namespace detail { + +void CheckRngState(const Tensor &state) { + CHECK(state.GetDevice().IsCPU()) << "RNG state must be a CPU tensor"; + CHECK_EQ(static_cast(state.Dtype()), static_cast(DataType::kUINT8)) + << "RNG state must be a UINT8 tensor"; +} + +} // namespace detail + +Generator CreateGenerator(const Device &device, uint64_t seed) { + return GeneratorBackendRegistry::Instance() + .Get(device.type()) + .Create(device, seed); +} + +const Generator &GetDefaultGenerator(const Device &device) { + auto &backend = GeneratorBackendRegistry::Instance().Get(device.type()); + if (const auto *generator = detail::TryGetScopedGenerator(device)) { + return *generator; + } + return backend.GetDefault(device); +} + +void ManualSeed(uint64_t seed) { + if (!detail::TryManualSeedScopedRng(seed)) { + GeneratorBackendRegistry::Instance().ManualSeedAll(seed); + } +} + +} // namespace infini_train diff --git a/infini_train/src/kernels/cpu/distribution.cc b/infini_train/src/kernels/cpu/distribution.cc new file mode 100644 index 00000000..bd6a2db0 --- /dev/null +++ b/infini_train/src/kernels/cpu/distribution.cc @@ -0,0 +1,111 @@ +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +#include "infini_train/src/core/runtime/cpu/cpu_dispatch.h" +#include "infini_train/src/core/runtime/cpu/cpu_generator_impl.h" +#include "infini_train/src/kernels/cpu/distributions_helper.h" + +namespace infini_train::kernels::cpu { +namespace { + +constexpr int kMaxUniformAttempts = 3; + +template +void UniformImpl(Tensor &tensor, double from, double to, + core::cpu::CpuGeneratorImpl *generator) { + auto *buf = static_cast(tensor.DataPtr()); + detail::UniformRealDistribution dist(static_cast(from), + static_cast(to)); + const StorageT from_value = static_cast(from); + if (from == to) { + for (int64_t i = 0; i < tensor.NumElements(); ++i) { + buf[i] = from_value; + } + return; + } + for (int64_t i = 0; i < tensor.NumElements(); ++i) { + StorageT value; + int attempt = 0; + do { + value = static_cast(dist(generator)); + ++attempt; + } while (attempt < kMaxUniformAttempts && + (static_cast(value) < from || + static_cast(value) >= to)); + // Bounded retries may still leave out-of-range values in very narrow + // intervals. + buf[i] = static_cast(value) == to ? from_value : value; + } +} + +template +void NormalImpl(Tensor &tensor, double mean, double std, + core::cpu::CpuGeneratorImpl *generator) { + auto *buf = static_cast(tensor.DataPtr()); + detail::NormalDistribution dist(static_cast(mean), + static_cast(std)); + for (int64_t i = 0; i < tensor.NumElements(); ++i) { + buf[i] = static_cast(dist(generator)); + } +} + +} // namespace + +void Uniform(const std::shared_ptr tensor, double from, double to, + const std::optional gen) { + CHECK(tensor->GetDevice().IsCPU()); + auto &cpu_generator = GetGeneratorOrDefault( + gen, core::cpu::GetDefaultCpuGenerator(), tensor->GetDevice()); + + std::lock_guard lock(cpu_generator.mutex_); + core::cpu::DispatchCpuFunc( + tensor->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + UniformImpl(*tensor, from, to, &cpu_generator); + }, + "CPU uniform"); +} + +void Normal(const std::shared_ptr tensor, double mean, double std, + const std::optional gen) { + CHECK(tensor->GetDevice().IsCPU()); + auto &cpu_generator = GetGeneratorOrDefault( + gen, core::cpu::GetDefaultCpuGenerator(), tensor->GetDevice()); + + std::lock_guard lock(cpu_generator.mutex_); + core::cpu::DispatchCpuFunc( + tensor->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + NormalImpl(*tensor, mean, std, &cpu_generator); + }, + "CPU normal"); +} + +} // namespace infini_train::kernels::cpu + +#define REGISTER_CPU_DISTRIBUTION_KERNEL(kernel_name) \ + REGISTER_KERNEL(infini_train::Device::DeviceType::kCPU, kernel_name, \ + infini_train::kernels::cpu::kernel_name) + +REGISTER_CPU_DISTRIBUTION_KERNEL(Uniform) +REGISTER_CPU_DISTRIBUTION_KERNEL(Normal) + +#undef REGISTER_CPU_DISTRIBUTION_KERNEL diff --git a/infini_train/src/kernels/cpu/distributions_helper.h b/infini_train/src/kernels/cpu/distributions_helper.h new file mode 100644 index 00000000..7aee3933 --- /dev/null +++ b/infini_train/src/kernels/cpu/distributions_helper.h @@ -0,0 +1,134 @@ +#pragma once + +// Host-side uniform and normal distributions for generators exposing Random() +// and Random64(). Box-Muller's second sample is cached when supported by the +// generator. + +#include +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +namespace infini_train::kernels::cpu::detail { + +template +struct UniformRealDistribution { + UniformRealDistribution(T from, T to) : from_(from), to_(to) { + CHECK_LE(from, to); + CHECK_LE(to - from, std::numeric_limits::max()); + } + + UniformRealDistribution(const UniformRealDistribution &) = default; + UniformRealDistribution &operator=(const UniformRealDistribution &) = delete; + + template + T operator()(RNG *generator) const { + if constexpr (std::is_same_v) { + return Transform(generator->Random64()); + } else { + return Transform(generator->Random()); + } + } + + private: + T from_; + T to_; + + template + T Transform(V val) const { + constexpr auto kMask = static_cast( + (static_cast(1) << std::numeric_limits::digits) - 1); + constexpr auto kDivisor = + static_cast(1) / + (static_cast(1) << std::numeric_limits::digits); + T x = (val & kMask) * kDivisor; + return x * (to_ - from_) + from_; + } +}; + +template +bool MaybeGetNextNormalSample(RNG *generator, double *ret) { + const auto sample = generator->next_double_normal_sample(); + if (!sample.has_value()) { + return false; + } + *ret = sample.value(); + generator->set_next_double_normal_sample(std::nullopt); + return true; +} + +template +bool MaybeGetNextNormalSample(RNG *generator, float *ret) { + const auto sample = generator->next_float_normal_sample(); + if (!sample.has_value()) { + return false; + } + *ret = sample.value(); + generator->set_next_float_normal_sample(std::nullopt); + return true; +} + +// Fallback: RNG without cache support never has a cached sample. +template +bool MaybeGetNextNormalSample(RNG * /*generator*/, void * /*ret*/) { + return false; +} + +template +void MaybeSetNextNormalSample(RNG *generator, const double *cache) { + generator->set_next_double_normal_sample(*cache); +} + +template +void MaybeSetNextNormalSample(RNG *generator, const float *cache) { + generator->set_next_float_normal_sample(*cache); +} + +// Fallback: RNG without cache support discards the second sample. +template +void MaybeSetNextNormalSample(RNG * /*generator*/, const void * /*cache*/) {} + +template +struct NormalDistribution { + NormalDistribution(T mean, T stdv) : mean_(mean), stdv_(stdv) { + CHECK_GE(stdv, static_cast(0)); + } + + NormalDistribution(const NormalDistribution &) = default; + NormalDistribution &operator=(const NormalDistribution &) = delete; + + template + T operator()(RNG *generator) const { + T ret; + if (MaybeGetNextNormalSample(generator, &ret)) { + return ret * stdv_ + mean_; + } + + UniformRealDistribution uniform(static_cast(0), static_cast(1)); + const T u1 = uniform(generator); + const T u2 = uniform(generator); + + const T r = std::sqrt(static_cast(-2.0) * std::log1p(-u2)); + constexpr T kTwoPi = static_cast(2.0 * std::numbers::pi_v); + const T theta = kTwoPi * u1; + const T sample = r * std::sin(theta); + + MaybeSetNextNormalSample(generator, &sample); + + ret = r * std::cos(theta); + return ret * stdv_ + mean_; + } + + private: + T mean_; + T stdv_; +}; + +} // namespace infini_train::kernels::cpu::detail diff --git a/infini_train/src/kernels/cpu/dropout.cc b/infini_train/src/kernels/cpu/dropout.cc new file mode 100644 index 00000000..754e5c02 --- /dev/null +++ b/infini_train/src/kernels/cpu/dropout.cc @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +#include "infini_train/src/core/runtime/cpu/cpu_dispatch.h" +#include "infini_train/src/core/runtime/cpu/cpu_generator_impl.h" +#include "infini_train/src/kernels/cpu/distributions_helper.h" + +namespace infini_train::kernels::cpu { +namespace { + +template +void DropoutForwardImpl(Tensor &output, Tensor &mask, const Tensor &input, + double p, core::cpu::CpuGeneratorImpl *generator) { + auto *output_data = static_cast(output.DataPtr()); + auto *mask_data = static_cast(mask.DataPtr()); + const auto *input_data = static_cast(input.DataPtr()); + const int64_t n = input.NumElements(); + + if (p == 0.0) { + for (int64_t index = 0; index < n; ++index) { + mask_data[index] = 1; + output_data[index] = input_data[index]; + } + return; + } + if (p == 1.0) { + for (int64_t index = 0; index < n; ++index) { + mask_data[index] = 0; + output_data[index] = static_cast(0.0); + } + return; + } + + const RandomT scale = static_cast(1.0 / (1.0 - p)); + detail::UniformRealDistribution distribution( + static_cast(0), static_cast(1)); + for (int64_t index = 0; index < n; ++index) { + const bool keep = distribution(generator) >= static_cast(p); + mask_data[index] = keep ? 1 : 0; + output_data[index] = + keep ? static_cast(static_cast(input_data[index]) * + scale) + : static_cast(0.0); + } +} + +template +void DropoutBackwardImpl(Tensor &grad_input, const Tensor &grad_output, + const Tensor &mask, double p) { + auto *grad_input_data = static_cast(grad_input.DataPtr()); + const auto *grad_output_data = + static_cast(grad_output.DataPtr()); + const auto *mask_data = static_cast(mask.DataPtr()); + const RandomT scale = p == 1.0 ? static_cast(0) + : static_cast(1.0 / (1.0 - p)); + + for (int64_t index = 0; index < grad_output.NumElements(); ++index) { + grad_input_data[index] = + mask_data[index] + ? static_cast( + static_cast(grad_output_data[index]) * scale) + : static_cast(0.0); + } +} + +} // namespace + +std::tuple, std::shared_ptr> DropoutForward( + const std::shared_ptr input, double p, + const std::optional gen) { + CHECK(input->GetDevice().IsCPU()); + CHECK(IsFloatingPointDType(input->Dtype())) + << "Dropout supports floating-point tensors only"; + CHECK_GE(p, 0.0) << "dropout probability has to be between 0 and 1, but got " + << p; + CHECK_LE(p, 1.0) << "dropout probability has to be between 0 and 1, but got " + << p; + + auto output = std::make_shared(input->Dims(), input->Dtype(), + input->GetDevice()); + auto mask = std::make_shared(input->Dims(), DataType::kUINT8, + input->GetDevice()); + + if (p == 0.0 || p == 1.0) { + core::cpu::DispatchCpuFunc( + input->Dtype(), + [&]() { + using RandomT = std::conditional_t, + double, float>; + DropoutForwardImpl(*output, *mask, *input, p, + nullptr); + }, + "CPU dropout forward"); + return {output, mask}; + } + + auto &cpu_generator = GetGeneratorOrDefault( + gen, core::cpu::GetDefaultCpuGenerator(), input->GetDevice()); + std::lock_guard lock(cpu_generator.mutex_); + core::cpu::DispatchCpuFunc( + input->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + DropoutForwardImpl(*output, *mask, *input, p, + &cpu_generator); + }, + "CPU dropout forward"); + return {output, mask}; +} + +std::shared_ptr DropoutBackward( + const std::shared_ptr grad_output, + const std::shared_ptr mask, double p) { + CHECK(grad_output->GetDevice().IsCPU()); + auto grad_input = std::make_shared( + grad_output->Dims(), grad_output->Dtype(), grad_output->GetDevice()); + + core::cpu::DispatchCpuFunc( + grad_output->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + DropoutBackwardImpl(*grad_input, *grad_output, *mask, + p); + }, + "CPU dropout backward"); + return grad_input; +} + +} // namespace infini_train::kernels::cpu + +#define REGISTER_CPU_DROPOUT_KERNEL(kernel_name) \ + REGISTER_KERNEL(infini_train::Device::DeviceType::kCPU, kernel_name, \ + infini_train::kernels::cpu::kernel_name) + +REGISTER_CPU_DROPOUT_KERNEL(DropoutForward) +REGISTER_CPU_DROPOUT_KERNEL(DropoutBackward) + +#undef REGISTER_CPU_DROPOUT_KERNEL diff --git a/infini_train/src/kernels/cuda/distribution.cu b/infini_train/src/kernels/cuda/distribution.cu new file mode 100644 index 00000000..2e40bfe1 --- /dev/null +++ b/infini_train/src/kernels/cuda/distribution.cu @@ -0,0 +1,201 @@ +#include +#include +#include +#include +#include + +#include + +#include "glog/logging.h" + +#include "infini_train/include/common/cuda/common_cuda.h" +#include "infini_train/include/common/cuda/kernel_helper.cuh" +#include "infini_train/include/core/runtime/device_guard.h" +#include "infini_train/include/core/runtime/runtime_common.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +#include "infini_train/src/core/runtime/cuda/cuda_dispatch.h" +#include "infini_train/src/core/runtime/cuda/cuda_generator_impl.h" +#include "infini_train/src/core/runtime/cuda/cuda_runtime_common.h" + +#include + +namespace infini_train::kernels::cuda { +namespace { + +constexpr int kThreadsPerBlock = 256; +constexpr int kMaxUniformAttempts = 3; + +template +__device__ RandomT UniformSample(curandStatePhilox4_32_10_t *state) { + if constexpr (std::is_same_v) { + double sample; + do { + const double2 values = curand_uniform2_double(state); + sample = 1.0 - values.x; + // Subtraction can round a tiny positive draw to 1.0. + } while (sample >= 1.0); + return sample; + } else { + return (curand(state) >> 8) * 0x1p-24f; + } +} + +template +__device__ RandomT NormalSample(curandStatePhilox4_32_10_t *state) { + if constexpr (std::is_same_v) { + return curand_normal_double(state); + } else { + return curand_normal(state); + } +} + +template +__global__ void UniformKernel(StorageT *data, int64_t n, double from, double to, + uint64_t seed, uint64_t subsequence) { + const int64_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n) { + return; + } + + const StorageT from_value = common::cuda::Cast(from); + if (from == to) { + data[index] = from_value; + return; + } + curandStatePhilox4_32_10_t state; + curand_init(seed, subsequence + static_cast(index), 0, &state); + const RandomT sample_from = static_cast(from); + const RandomT sample_to = static_cast(to); + StorageT value; + int attempt = 0; + do { + value = common::cuda::Cast(sample_from + + UniformSample(&state) * + (sample_to - sample_from)); + ++attempt; + } while (attempt < kMaxUniformAttempts && + (common::cuda::Cast(value) < from || + common::cuda::Cast(value) >= to)); + // Bounded retries may still leave out-of-range values in very narrow + // intervals. + data[index] = common::cuda::Cast(value) == to ? from_value : value; +} + +template +__global__ void NormalKernel(StorageT *data, int64_t n, RandomT mean, + RandomT std, uint64_t seed, uint64_t subsequence) { + const int64_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n) { + return; + } + + curandStatePhilox4_32_10_t state; + curand_init(seed, subsequence + static_cast(index), 0, &state); + data[index] = + common::cuda::Cast(mean + NormalSample(&state) * std); +} + +const core::cuda::CudaStream *GetCudaStream(const Device &device) { + return dynamic_cast( + core::GetDeviceGuardImpl(device.type())->GetStream(device)); +} + +} // namespace + +void Uniform(const std::shared_ptr tensor, double from, double to, + const std::optional gen) { + const Device device = tensor->GetDevice(); + CHECK(device.IsCUDA()); + const int64_t n = tensor->NumElements(); + if (n == 0) { + return; + } + core::DeviceGuard guard(device); + auto &cuda_generator = GetGeneratorOrDefault( + gen, core::cuda::GetDefaultCudaGenerator(device.index()), device); + + uint64_t seed = 0; + uint64_t subsequence = 0; + if (from != to) { + std::lock_guard lock(cuda_generator.mutex_); + seed = cuda_generator.current_seed(); + subsequence = + cuda_generator.ReservePhiloxSubsequence(static_cast(n)); + } + + const int blocks = + static_cast((n + kThreadsPerBlock - 1) / kThreadsPerBlock); + const auto *stream = GetCudaStream(device); + core::cuda::DispatchCudaFunc( + tensor->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + UniformKernel + <<cuda_stream()>>>( + static_cast(tensor->DataPtr()), n, from, to, seed, + subsequence); + }, + "CUDA uniform"); + CUDA_CHECK(cudaGetLastError()); +} + +void Normal(const std::shared_ptr tensor, double mean, double std, + const std::optional gen) { + const Device device = tensor->GetDevice(); + CHECK(device.IsCUDA()); + const int64_t n = tensor->NumElements(); + if (n == 0) { + return; + } + core::DeviceGuard guard(device); + auto &cuda_generator = GetGeneratorOrDefault( + gen, core::cuda::GetDefaultCudaGenerator(device.index()), device); + + uint64_t seed = 0; + uint64_t subsequence = 0; + { + std::lock_guard lock(cuda_generator.mutex_); + seed = cuda_generator.current_seed(); + subsequence = + cuda_generator.ReservePhiloxSubsequence(static_cast(n)); + } + + const int blocks = + static_cast((n + kThreadsPerBlock - 1) / kThreadsPerBlock); + const auto *stream = GetCudaStream(device); + core::cuda::DispatchCudaFunc( + tensor->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + NormalKernel + <<cuda_stream()>>>( + static_cast(tensor->DataPtr()), n, + static_cast(mean), static_cast(std), seed, + subsequence); + }, + "CUDA normal"); + CUDA_CHECK(cudaGetLastError()); +} + +} // namespace infini_train::kernels::cuda + +#define REGISTER_CUDA_DISTRIBUTION_KERNEL(kernel_name) \ + REGISTER_KERNEL(infini_train::Device::DeviceType::kCUDA, kernel_name, \ + infini_train::kernels::cuda::kernel_name) + +REGISTER_CUDA_DISTRIBUTION_KERNEL(Uniform) +REGISTER_CUDA_DISTRIBUTION_KERNEL(Normal) + +#undef REGISTER_CUDA_DISTRIBUTION_KERNEL diff --git a/infini_train/src/kernels/cuda/dropout.cu b/infini_train/src/kernels/cuda/dropout.cu new file mode 100644 index 00000000..7dc55065 --- /dev/null +++ b/infini_train/src/kernels/cuda/dropout.cu @@ -0,0 +1,206 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "glog/logging.h" + +#include "infini_train/include/common/cuda/common_cuda.h" +#include "infini_train/include/common/cuda/kernel_helper.cuh" +#include "infini_train/include/core/runtime/device_guard.h" +#include "infini_train/include/core/runtime/runtime_common.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +#include "infini_train/src/core/runtime/cuda/cuda_dispatch.h" +#include "infini_train/src/core/runtime/cuda/cuda_generator_impl.h" +#include "infini_train/src/core/runtime/cuda/cuda_runtime_common.h" + +#include + +namespace infini_train::kernels::cuda { +namespace { + +constexpr int kThreadsPerBlock = 256; + +template +__device__ RandomT UniformSample(curandStatePhilox4_32_10_t *state) { + if constexpr (std::is_same_v) { + double sample; + do { + const double2 values = curand_uniform2_double(state); + sample = 1.0 - values.x; + // Subtraction can round a tiny positive draw to 1.0. + } while (sample >= 1.0); + return sample; + } else { + return (curand(state) >> 8) * 0x1p-24f; + } +} + +template +__global__ void DropoutForwardKernel(StorageT *output, uint8_t *mask, + const StorageT *input, int64_t n, + RandomT p, uint64_t seed, + uint64_t subsequence) { + const int64_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n) { + return; + } + if (p == static_cast(0)) { + mask[index] = 1; + output[index] = input[index]; + return; + } + if (p == static_cast(1)) { + mask[index] = 0; + output[index] = common::cuda::Cast(0.0f); + return; + } + + curandStatePhilox4_32_10_t state; + curand_init(seed, subsequence + static_cast(index), 0, &state); + const bool keep = UniformSample(&state) >= p; + const RandomT scale = static_cast(1) / (static_cast(1) - p); + mask[index] = keep ? 1 : 0; + output[index] = keep ? common::cuda::Cast( + common::cuda::Cast(input[index]) * scale) + : common::cuda::Cast(0.0f); +} + +template +__global__ void DropoutBackwardKernel(StorageT *grad_input, + const StorageT *grad_output, + const uint8_t *mask, int64_t n, + RandomT p) { + const int64_t index = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (index >= n) { + return; + } + const RandomT scale = + p == static_cast(1) + ? static_cast(0) + : static_cast(1) / (static_cast(1) - p); + grad_input[index] = + mask[index] ? common::cuda::Cast( + common::cuda::Cast(grad_output[index]) * scale) + : common::cuda::Cast(0.0f); +} + +const core::cuda::CudaStream *GetCudaStream(const Device &device) { + return dynamic_cast( + core::GetDeviceGuardImpl(device.type())->GetStream(device)); +} + +} // namespace + +std::tuple, std::shared_ptr> DropoutForward( + const std::shared_ptr input, double p, + const std::optional gen) { + const Device device = input->GetDevice(); + CHECK(device.IsCUDA()); + CHECK(IsFloatingPointDType(input->Dtype())) + << "Dropout supports floating-point tensors only"; + CHECK_GE(p, 0.0) << "dropout probability has to be between 0 and 1, but got " + << p; + CHECK_LE(p, 1.0) << "dropout probability has to be between 0 and 1, but got " + << p; + core::DeviceGuard guard(device); + + auto output = std::make_shared(input->Dims(), input->Dtype(), device); + auto mask = std::make_shared(input->Dims(), DataType::kUINT8, device); + + const int64_t n = input->NumElements(); + if (n == 0) { + return {output, mask}; + } + + uint64_t seed = 0; + uint64_t subsequence = 0; + if (p > 0.0 && p < 1.0) { + auto &cuda_generator = GetGeneratorOrDefault( + gen, core::cuda::GetDefaultCudaGenerator(device.index()), device); + std::lock_guard lock(cuda_generator.mutex_); + seed = cuda_generator.current_seed(); + subsequence = + cuda_generator.ReservePhiloxSubsequence(static_cast(n)); + } + + const int blocks = + static_cast((n + kThreadsPerBlock - 1) / kThreadsPerBlock); + const auto *stream = GetCudaStream(device); + core::cuda::DispatchCudaFunc( + input->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + DropoutForwardKernel + <<cuda_stream()>>>( + static_cast(output->DataPtr()), + static_cast(mask->DataPtr()), + static_cast(input->DataPtr()), n, + static_cast(p), seed, subsequence); + }, + "CUDA dropout forward"); + CUDA_CHECK(cudaGetLastError()); + return {output, mask}; +} + +std::shared_ptr DropoutBackward( + const std::shared_ptr grad_output, + const std::shared_ptr mask, double p) { + const Device device = grad_output->GetDevice(); + CHECK(device.IsCUDA()); + core::DeviceGuard guard(device); + + auto grad_input = std::make_shared(grad_output->Dims(), + grad_output->Dtype(), device); + + const int64_t n = grad_output->NumElements(); + if (n == 0) { + return grad_input; + } + + const int blocks = + static_cast((n + kThreadsPerBlock - 1) / kThreadsPerBlock); + const auto *stream = GetCudaStream(device); + core::cuda::DispatchCudaFunc( + grad_output->Dtype(), + [&]() { + using RandomT = + std::conditional_t, double, float>; + DropoutBackwardKernel + <<cuda_stream()>>>( + static_cast(grad_input->DataPtr()), + static_cast(grad_output->DataPtr()), + static_cast(mask->DataPtr()), n, + static_cast(p)); + }, + "CUDA dropout backward"); + CUDA_CHECK(cudaGetLastError()); + return grad_input; +} + +} // namespace infini_train::kernels::cuda + +#define REGISTER_CUDA_DROPOUT_KERNEL(kernel_name) \ + REGISTER_KERNEL(infini_train::Device::DeviceType::kCUDA, kernel_name, \ + infini_train::kernels::cuda::kernel_name) + +REGISTER_CUDA_DROPOUT_KERNEL(DropoutForward) +REGISTER_CUDA_DROPOUT_KERNEL(DropoutBackward) + +#undef REGISTER_CUDA_DROPOUT_KERNEL diff --git a/infini_train/src/kernels/cuda/elementwise.cu b/infini_train/src/kernels/cuda/elementwise.cu index 6b27f12d..220d3608 100644 --- a/infini_train/src/kernels/cuda/elementwise.cu +++ b/infini_train/src/kernels/cuda/elementwise.cu @@ -1,4 +1,5 @@ #include +#include #include diff --git a/infini_train/src/kernels/cuda/gather.cu b/infini_train/src/kernels/cuda/gather.cu index face8992..aed12a9d 100644 --- a/infini_train/src/kernels/cuda/gather.cu +++ b/infini_train/src/kernels/cuda/gather.cu @@ -1,3 +1,5 @@ +#include + #include "glog/logging.h" #include "infini_train/include/common/common.h" diff --git a/infini_train/src/kernels/cuda/no_op.cu b/infini_train/src/kernels/cuda/no_op.cu index a1024224..706a032d 100644 --- a/infini_train/src/kernels/cuda/no_op.cu +++ b/infini_train/src/kernels/cuda/no_op.cu @@ -1,3 +1,5 @@ +#include + #include "glog/logging.h" #include "infini_train/include/dispatcher.h" diff --git a/infini_train/src/kernels/cuda/reduction.cu b/infini_train/src/kernels/cuda/reduction.cu index 52ecbe52..26eb49ec 100644 --- a/infini_train/src/kernels/cuda/reduction.cu +++ b/infini_train/src/kernels/cuda/reduction.cu @@ -1,3 +1,5 @@ +#include + #include #include "infini_train/include/common/cuda/common_cuda.h" diff --git a/infini_train/src/nn/functional.cc b/infini_train/src/nn/functional.cc index 7bbd7feb..df1b7624 100644 --- a/infini_train/src/nn/functional.cc +++ b/infini_train/src/nn/functional.cc @@ -2,9 +2,14 @@ #include #include +#include +#include #include +#include "glog/logging.h" + #include "infini_train/include/autograd/activations.h" +#include "infini_train/include/autograd/dropout.h" #include "infini_train/include/autograd/elementwise.h" #include "infini_train/include/autograd/reduction.h" #include "infini_train/include/autograd/softmax.h" @@ -29,6 +34,35 @@ std::shared_ptr Ones(const std::vector size) { return init::Ones(ones); } +std::shared_ptr Rand(const std::vector &size, DataType dtype, + Device device, std::optional generator, + bool requires_grad) { + auto result = std::make_shared(size, dtype, device, requires_grad); + return init::Uniform(result, 0.0f, 1.0f, generator); +} + +std::shared_ptr Randn(const std::vector &size, DataType dtype, + Device device, std::optional generator, + bool requires_grad) { + auto result = std::make_shared(size, dtype, device, requires_grad); + return init::Normal(result, 0.0f, 1.0f, generator); +} + +std::shared_ptr Dropout(const std::shared_ptr &input, double p, + bool training, + std::optional generator) { + CHECK(input != nullptr); + CHECK_GE(p, 0.0) << "dropout probability has to be between 0 and 1, but got " + << p; + CHECK_LE(p, 1.0) << "dropout probability has to be between 0 and 1, but got " + << p; + if (!training || p == 0.0 || input->NumElements() == 0) { + return input; + } + return std::make_shared(p, std::move(generator)) + ->Apply({input})[0]; +} + std::shared_ptr Reciprocal(const std::shared_ptr &input) { return input->Reciprocal(); } diff --git a/infini_train/src/nn/init.cc b/infini_train/src/nn/init.cc index 9d2d8eb7..cb008d27 100644 --- a/infini_train/src/nn/init.cc +++ b/infini_train/src/nn/init.cc @@ -1,25 +1,23 @@ #include "infini_train/include/nn/init.h" -#include #include #include +#include #include #include #include #include -#include +#include +#include #include #include -#ifdef USE_OMP -#include -#endif - #include "glog/logging.h" #include "infini_train/include/core/runtime/device_guard.h" #include "infini_train/include/datatype.h" #include "infini_train/include/device.h" +#include "infini_train/include/dispatcher.h" #include "infini_train/include/tensor.h" namespace infini_train::core { @@ -28,55 +26,76 @@ class Stream; namespace infini_train::nn::init { namespace { -constexpr int kRandomSeed = 42; - -// FIXME: RNG design is incomplete. -// -// Current implementation lacks: -// - unified Generator abstraction -// - global default generator and seed control -// - reproducible / clonable RNG state -// -// TODO: -// - introduce Generator interface and backend impl -// - add default generator management (per device) -// - refactor random ops to consume Generator -static std::mt19937 gen(kRandomSeed); -} // namespace +struct DistributionBounds { + double lowest; + double max; +}; -std::shared_ptr Normal(const std::shared_ptr &tensor, - float mean, float std, - std::optional generator) { - const int64_t num_elements = tensor->NumElements(); - std::vector buffer(num_elements); - -#ifdef USE_OMP -#pragma omp parallel - { - std::mt19937 local_gen(kRandomSeed + omp_get_thread_num()); - std::normal_distribution local_dis(mean, std); -#pragma omp for - for (int i = 0; i < buffer.size(); ++i) { - buffer[i] = - generator ? local_dis(generator.value()) : local_dis(local_gen); +DistributionBounds GetDistributionBounds(DataType dtype) { + switch (dtype) { + case DataType::kFLOAT16: { + const double max = static_cast( + FP16(static_cast(0x7bff), FP16::from_bits())); + return {-max, max}; + } + case DataType::kBFLOAT16: { + const double max = static_cast( + BF16(static_cast(0x7f7f), BF16::from_bits())); + return {-max, max}; } + case DataType::kFLOAT32: + return {-std::numeric_limits::max(), + std::numeric_limits::max()}; + case DataType::kFLOAT64: + return {-std::numeric_limits::max(), + std::numeric_limits::max()}; + default: + LOG(FATAL) << "Unsupported distribution dtype: " + << kDataTypeToDesc.at(dtype); + return {}; } -#else - std::normal_distribution dis(mean, std); - std::generate(buffer.begin(), buffer.end(), [&]() { - return generator ? dis(generator.value()) : dis(gen); - }); -#endif +} + +void CheckDistributionTensor(const Tensor &tensor) { + CHECK(IsFloatingPointDType(tensor.Dtype())) + << "Uniform and Normal initialization support floating-point tensors " + "only"; +} + +void CheckUniformParameters(const Tensor &tensor, double from, double to) { + const auto bounds = GetDistributionBounds(tensor.Dtype()); + CHECK_GE(from, bounds.lowest) + << "uniform expects from to be within the range of " + << kDataTypeToDesc.at(tensor.Dtype()); + CHECK_LE(from, bounds.max) + << "uniform expects from to be within the range of " + << kDataTypeToDesc.at(tensor.Dtype()); + CHECK_GE(to, bounds.lowest) << "uniform expects to to be within the range of " + << kDataTypeToDesc.at(tensor.Dtype()); + CHECK_LE(to, bounds.max) << "uniform expects to to be within the range of " + << kDataTypeToDesc.at(tensor.Dtype()); + CHECK_LE(from, to) << "uniform expects a [from, to) range, but found from=" + << from << " > to=" << to; + CHECK_LE(to - from, bounds.max) << "uniform expects to - from to fit in " + << kDataTypeToDesc.at(tensor.Dtype()); +} + +void CheckNormalParameters(double std) { + CHECK_GE(std, 0.0) << "normal expects std >= 0.0, but found std=" << std; +} +} // namespace + +std::shared_ptr Normal(const std::shared_ptr &tensor, + float mean, float std, + std::optional generator) { + CheckDistributionTensor(*tensor); + CheckNormalParameters(std); auto device = tensor->GetDevice(); core::DeviceGuard guard(device); - auto impl = core::GetDeviceGuardImpl(device.type()); - - impl->MemcpyAsync( - tensor->DataPtr(), buffer.data(), num_elements * sizeof(float), - device.type() == Device::DeviceType::kCPU ? core::MemcpyKind::kD2D - : core::MemcpyKind::kH2D, - impl->GetStream(device)); + Dispatcher::Instance().Call({device.type(), "Normal"}, tensor, + static_cast(mean), + static_cast(std), generator); return tensor; } @@ -139,7 +158,7 @@ float CalculateGain(NonLinearityType nonlinearity, std::shared_ptr KaimingUniform(const std::shared_ptr &tensor, float a, KaimingMode mode, NonLinearityType nonlinearity, - std::optional generator) { + std::optional generator) { for (const auto dim : tensor->Dims()) { if (dim == 0) { LOG(WARNING) << "Initializing zero-element tensors is a no-op"; @@ -155,40 +174,14 @@ std::shared_ptr KaimingUniform(const std::shared_ptr &tensor, } std::shared_ptr Uniform(const std::shared_ptr &tensor, float a, - float b, - std::optional generator) { - const int64_t num_elements = tensor->NumElements(); - std::vector buffer(num_elements); - -#ifdef USE_OMP -#pragma omp parallel - { - std::mt19937 local_gen(kRandomSeed + omp_get_thread_num()); - std::uniform_real_distribution local_dis(a, b); -#pragma omp for - for (int i = 0; i < buffer.size(); ++i) { - buffer[i] = - generator ? local_dis(generator.value()) : local_dis(local_gen); - } - } -#else - std::uniform_real_distribution dis(a, b); - std::generate(buffer.begin(), buffer.end(), [&]() { - return generator ? dis(generator.value()) : dis(gen); - }); -#endif - + float b, std::optional generator) { + CheckDistributionTensor(*tensor); + CheckUniformParameters(*tensor, a, b); auto device = tensor->GetDevice(); - core::DeviceGuard guard(device); - auto impl = core::GetDeviceGuardImpl(device.type()); - - impl->MemcpyAsync( - tensor->DataPtr(), buffer.data(), num_elements * sizeof(float), - device.type() == Device::DeviceType::kCPU ? core::MemcpyKind::kD2D - : core::MemcpyKind::kH2D, - impl->GetStream(device)); - + Dispatcher::Instance().Call({device.type(), "Uniform"}, tensor, + static_cast(a), + static_cast(b), generator); return tensor; } diff --git a/infini_train/src/nn/parallel/ddp/distributed_data_parallel.cc b/infini_train/src/nn/parallel/ddp/distributed_data_parallel.cc index 03567b6a..3c4e3111 100644 --- a/infini_train/src/nn/parallel/ddp/distributed_data_parallel.cc +++ b/infini_train/src/nn/parallel/ddp/distributed_data_parallel.cc @@ -39,12 +39,10 @@ DistributedDataParallel::DistributedDataParallel( LOG(FATAL) << "DistributedDataParallel: ZeRO-3 is not implemented yet."; } auto *process_group_factory = ProcessGroupFactory::Instance(); - const auto dense_dp_ranks = - GetDataParallelWithContextGroupRanks(rank.GlobalRank()); + const auto dense_dp_ranks = GetDataParallelGroupRanks(rank.GlobalRank()); if (dense_dp_ranks.size() > 1) { dense_dp_pg_ = process_group_factory->GetOrCreate( - GetDataParallelWithContextProcessGroupName(rank.GlobalRank()), - dense_dp_ranks); + GetDataParallelProcessGroupName(rank.GlobalRank()), dense_dp_ranks); } const auto expert_dp_ranks = GetExpertDataParallelGroupRanks(rank.GlobalRank()); @@ -120,6 +118,7 @@ DistributedDataParallel::DistributedDataParallel( validate_device(buffer->GetDevice(), "buffer"); } modules_[kModuleName] = std::move(module); + SynchronizeModuleState(); if (ddp_config.zero_stage >= 1) { BuildParamAndGradBuffers(trainable_params); @@ -152,6 +151,33 @@ DistributedDataParallel::DistributedDataParallel( } } +void DistributedDataParallel::SynchronizeModuleState() { + auto state = modules_.at(kModuleName)->StateDict(); + std::vector names; + names.reserve(state.size()); + for (const auto &[name, _] : state) { + names.push_back(name); + } + std::sort(names.begin(), names.end()); + + std::vector> tensors; + tensors.reserve(names.size()); + for (const auto &name : names) { + auto tensor = state.at(name); + CHECK(tensor != nullptr) + << "DDP module state tensor '" << name << "' is null"; + if (!tensors.empty()) { + CHECK_EQ(tensor->GetDevice(), tensors.front()->GetDevice()) + << "DDP module state tensors must be on one device"; + } + tensors.push_back(std::move(tensor)); + } + + if (!tensors.empty() && dense_dp_pg_) { + dense_dp_pg_->Broadcast(tensors, 0); + } +} + const ProcessGroup *DistributedDataParallel::ProcessGroupForParameter( bool is_expert_parallel) const { return is_expert_parallel ? expert_dp_pg_ : dense_dp_pg_; @@ -159,8 +185,7 @@ const ProcessGroup *DistributedDataParallel::ProcessGroupForParameter( float DistributedDataParallel::GradientScalingFactorForParameter( bool is_expert_parallel) const { - const float dense_dp_size = static_cast( - global::GetDataParallelSize() * global::GetContextParallelSize()); + const float dense_dp_size = static_cast(global::GetDataParallelSize()); CHECK_GT(dense_dp_size, 0.f); if (!ddp_config_.average_in_collective) { return 1.f / dense_dp_size; diff --git a/infini_train/src/nn/parallel/tensor_parallel.cc b/infini_train/src/nn/parallel/tensor_parallel.cc index e7728c5c..dbee5f99 100644 --- a/infini_train/src/nn/parallel/tensor_parallel.cc +++ b/infini_train/src/nn/parallel/tensor_parallel.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include "infini_train/include/nn/parallel/reduce_op_type.h" #include "infini_train/include/nn/parallel/utils.h" #include "infini_train/include/nn/parallel/work.h" +#include "infini_train/include/rng_tracker.h" #include "infini_train/include/tensor.h" namespace infini_train::nn::parallel { @@ -238,8 +240,17 @@ class GatherFromSPRegion : public autograd::Function { }; void LinearResetParameters(std::shared_ptr weight, - std::shared_ptr bias = nullptr) { + std::shared_ptr bias, bool sharded_bias) { + std::optional rng_scope; + auto *tracker = detail::GetCurrentRngTracker(); + if (global::GetTensorParallelSize() > 1 && tracker && + tracker->Contains(kTensorParallelInitRngName)) { + rng_scope.emplace(*tracker, kTensorParallelInitRngName); + } init::KaimingUniform(weight, sqrt(5.0f)); + if (!sharded_bias) { + rng_scope.reset(); + } if (bias) { const auto [fan_in, _] = init::CalculateFanInAndFanOut(weight); const float bound = fan_in > 0 ? 1.0 / sqrt(fan_in) : 0.0; @@ -606,7 +617,8 @@ ColumnParallelLinear::ColumnParallelLinear( } LinearResetParameters(parameters_[kParamWeightName], - bias ? parameters_[kParamBiasName] : nullptr); + bias ? parameters_[kParamBiasName] : nullptr, + /*sharded_bias=*/true); } std::vector> ColumnParallelLinear::Forward( @@ -747,7 +759,8 @@ RowParallelLinear::RowParallelLinear(int64_t in_features, int64_t out_features, } LinearResetParameters(parameters_[kParamWeightName], - bias ? parameters_[kParamBiasName] : nullptr); + bias ? parameters_[kParamBiasName] : nullptr, + /*sharded_bias=*/false); } std::vector> RowParallelLinear::Forward( diff --git a/infini_train/src/rng_tracker.cc b/infini_train/src/rng_tracker.cc new file mode 100644 index 00000000..ee85963f --- /dev/null +++ b/infini_train/src/rng_tracker.cc @@ -0,0 +1,285 @@ +#include "infini_train/include/rng_tracker.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "glog/logging.h" + +#include "infini_train/include/core/runtime/device_guard.h" +#include "infini_train/include/core/runtime/generator_backend.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/generator_impl.h" +#include "infini_train/include/tensor.h" + +namespace infini_train { +namespace { + +constexpr char kDefaultStream[] = "default"; +thread_local RngScope *current_scope = nullptr; + +std::unique_lock LockTracker( + std::recursive_mutex &mutex) { + std::unique_lock lock(mutex, std::try_to_lock); + CHECK(lock.owns_lock()) << "RNG tracker is in use on another thread"; + return lock; +} + +Device CanonicalDevice(const Device &device) { + if (device.index() == -1) { + return core::GetDeviceGuardImpl(device.type())->GetDevice(); + } + return device; +} + +} // namespace + +RngTracker::RngTracker(uint64_t seed) { + streams_.emplace(kDefaultStream, RngStream{seed, {}}); +} + +RngTracker::~RngTracker() = default; + +void RngTracker::Add(const std::string &name, uint64_t seed) { + auto lock = LockTracker(mutex_); + CHECK(!name.empty() && !streams_.contains(name)) + << "RNG stream name must be nonempty and unique"; + streams_.emplace(name, RngStream{seed, {}}); +} + +bool RngTracker::Contains(const std::string &name) const { + auto lock = LockTracker(mutex_); + return streams_.contains(name); +} + +void RngTracker::ManualSeed(uint64_t seed) { ManualSeed(kDefaultStream, seed); } + +void RngTracker::ManualSeed(const std::string &name, uint64_t seed) { + auto lock = LockTracker(mutex_); + auto found = streams_.find(name); + CHECK(found != streams_.end()) << "Unknown RNG stream: " << name; + auto &stream = found->second; + for (const auto &[device, generator] : stream.generators) { + std::lock_guard generator_lock( + GeneratorAccessor::Mutex(generator)); + generator.set_current_seed(seed); + } + stream.seed = seed; +} + +std::shared_ptr RngTracker::GetState() const { + auto lock = LockTracker(mutex_); + constexpr char kStateMagic[] = "ITRNGTRK"; + constexpr uint64_t kStateVersion = 1; + std::vector bytes(kStateMagic, + kStateMagic + sizeof(kStateMagic) - 1); + auto append_uint64 = [&bytes](uint64_t value) { + for (int shift = 0; shift < 64; shift += 8) { + bytes.push_back(static_cast(value >> shift)); + } + }; + + // Metadata uses little-endian integers; backend state bytes remain opaque. + append_uint64(kStateVersion); + append_uint64(streams_.size()); + for (const auto &[name, stream] : streams_) { + append_uint64(name.size()); + bytes.insert(bytes.end(), name.begin(), name.end()); + append_uint64(stream.seed); + append_uint64(stream.generators.size()); + for (const auto &[device, generator] : stream.generators) { + std::lock_guard generator_lock( + GeneratorAccessor::Mutex(generator)); + const auto state = generator.get_state(); + append_uint64(static_cast(device.first)); + append_uint64(static_cast(device.second)); + append_uint64(state->SizeInBytes()); + const auto *data = static_cast(state->DataPtr()); + bytes.insert(bytes.end(), data, data + state->SizeInBytes()); + } + } + CHECK(bytes.size() <= + static_cast(std::numeric_limits::max())) + << "RNG tracker state is too large"; + auto state = std::make_shared( + std::vector{static_cast(bytes.size())}, + DataType::kUINT8, Device()); + std::memcpy(state->DataPtr(), bytes.data(), bytes.size()); + return state; +} + +void RngTracker::SetState(const Tensor &state) { + auto lock = LockTracker(mutex_); + constexpr char kStateMagic[] = "ITRNGTRK"; + constexpr uint64_t kStateVersion = 1; + CHECK(state.defined() && state.GetDevice().IsCPU() && + state.Dtype() == DataType::kUINT8 && state.Dims().size() == 1 && + state.SizeInBytes() >= sizeof(kStateMagic) - 1) + << "RNG tracker state must be a nonempty one-dimensional CPU UINT8 " + "tensor"; + const auto *data = static_cast(state.DataPtr()); + const size_t data_size = state.SizeInBytes(); + CHECK(std::memcmp(data, kStateMagic, sizeof(kStateMagic) - 1) == 0) + << "Invalid RNG tracker state magic"; + size_t offset = sizeof(kStateMagic) - 1; + auto read_uint64 = [&] { + CHECK(data_size - offset >= sizeof(uint64_t)) + << "Truncated RNG tracker state"; + uint64_t value = 0; + for (int shift = 0; shift < 64; shift += 8) { + value |= static_cast(data[offset++]) << shift; + } + return value; + }; + CHECK(read_uint64() == kStateVersion) + << "Unsupported RNG tracker state version"; + const uint64_t stream_count = read_uint64(); + CHECK(stream_count != 0 && + stream_count <= (data_size - offset) / (3 * sizeof(uint64_t) + 1)) + << "Invalid RNG tracker stream count"; + + struct StreamState { + uint64_t seed; + std::map> generator_ranges; + }; + std::map states; + for (uint64_t i = 0; i < stream_count; ++i) { + const uint64_t name_size = read_uint64(); + CHECK(name_size != 0 && name_size <= data_size - offset) + << "Invalid RNG stream name length"; + std::string name(reinterpret_cast(data + offset), + static_cast(name_size)); + offset += static_cast(name_size); + StreamState stream_state{read_uint64(), {}}; + auto [found, inserted] = + states.emplace(std::move(name), std::move(stream_state)); + CHECK(inserted) << "Duplicate RNG stream name"; + const uint64_t device_count = read_uint64(); + CHECK(device_count <= (data_size - offset) / (3 * sizeof(uint64_t) + 1)) + << "Invalid RNG stream device count"; + for (uint64_t j = 0; j < device_count; ++j) { + const uint64_t type = read_uint64(); + const uint64_t index = read_uint64(); + const uint64_t byte_count = read_uint64(); + CHECK(type < static_cast(Device::DeviceType::kCount) && + index <= static_cast(std::numeric_limits::max())) + << "Invalid RNG state device"; + const auto device_type = static_cast(type); + GeneratorBackendRegistry::Instance().Get(device_type); + CHECK(index < static_cast( + core::GetDeviceGuardImpl(device_type)->DeviceCount())) + << "RNG tracker state has an invalid device index"; + CHECK(byte_count != 0 && byte_count <= data_size - offset && + byte_count <= + static_cast(std::numeric_limits::max())) + << "Invalid RNG generator state length"; + const DeviceKey key{device_type, static_cast(index)}; + const bool inserted_device = + found->second.generator_ranges + .emplace(key, std::pair{offset, static_cast(byte_count)}) + .second; + CHECK(inserted_device) << "Duplicate RNG state device"; + offset += static_cast(byte_count); + } + } + CHECK(offset == data_size && states.contains(kDefaultStream)) + << "RNG tracker state has trailing bytes or no default stream"; + for (auto *scope = current_scope; scope; scope = scope->previous_) { + CHECK(scope->tracker_ != this || states.contains(scope->name_)) + << "Cannot restore a state that removes an active RNG stream"; + } + + Streams restored_streams; + for (const auto &[name, stream_state] : states) { + RngStream stream{stream_state.seed, {}}; + for (const auto &[device, range] : stream_state.generator_ranges) { + auto generator = CreateGenerator(Device(device.first, device.second), + stream_state.seed); + Tensor generator_state( + std::vector{static_cast(range.second)}, + DataType::kUINT8, Device()); + std::memcpy(generator_state.DataPtr(), data + range.first, range.second); + generator.set_state(generator_state); + stream.generators.emplace(device, std::move(generator)); + } + restored_streams.emplace(name, std::move(stream)); + } + streams_.swap(restored_streams); +} + +RngScope::RngScope() : RngScope(Generator::kDefaultSeed) {} + +RngScope::RngScope(uint64_t seed) + : owned_tracker_(std::make_unique(seed)), + tracker_(owned_tracker_.get()), + name_(kDefaultStream), + previous_(current_scope), + thread_(std::this_thread::get_id()), + lock_(LockTracker(tracker_->mutex_)) { + current_scope = this; +} + +RngScope::RngScope(RngTracker &tracker) : RngScope(tracker, kDefaultStream) {} + +RngScope::RngScope(RngTracker &tracker, const std::string &name) + : tracker_(&tracker), + name_(name), + previous_(current_scope), + thread_(std::this_thread::get_id()), + lock_(LockTracker(tracker_->mutex_)) { + CHECK(tracker_->streams_.contains(name_)) << "Unknown RNG stream: " << name_; + current_scope = this; +} + +RngScope::~RngScope() { + CHECK(thread_ == std::this_thread::get_id()) + << "RNG scopes must be destroyed on their creating thread"; + CHECK(current_scope == this) + << "RNG scopes must be destroyed in reverse construction order"; + current_scope = previous_; +} + +namespace detail { + +const Generator *TryGetScopedGenerator(const Device &requested_device) { + if (!current_scope) { + return nullptr; + } + auto &scope = *current_scope; + auto &stream = scope.tracker_->streams_.at(scope.name_); + const Device device = CanonicalDevice(requested_device); + const RngTracker::DeviceKey key{device.type(), device.index()}; + auto found = stream.generators.find(key); + if (found == stream.generators.end()) { + found = stream.generators.emplace(key, CreateGenerator(device, stream.seed)) + .first; + } + return &found->second; +} + +bool TryManualSeedScopedRng(uint64_t seed) { + if (!current_scope) { + return false; + } + auto &scope = *current_scope; + scope.tracker_->ManualSeed(scope.name_, seed); + return true; +} + +RngTracker *GetCurrentRngTracker() { + return current_scope ? current_scope->tracker_ : nullptr; +} + +} // namespace detail +} // namespace infini_train diff --git a/infini_train/src/tensor.cc b/infini_train/src/tensor.cc index 224c9940..63d88820 100644 --- a/infini_train/src/tensor.cc +++ b/infini_train/src/tensor.cc @@ -129,6 +129,8 @@ const void *Tensor::DataPtr() const { return reinterpret_cast(buffer_->DataPtr()) + offset_; } +bool Tensor::defined() const { return buffer_ != nullptr; } + size_t Tensor::SizeInBytes() const { return kDataTypeToSize.at(dtype_) * num_elements_; } @@ -599,7 +601,7 @@ std::shared_ptr Tensor::Outer(const std::shared_ptr &other) { // distribution std::shared_ptr Tensor::Uniform(float from, float to, - std::optional generator) { + std::optional generator) { return nn::init::Uniform(shared_from_this(), from, to, generator); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bfe52949..21d16623 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -41,6 +41,9 @@ add_subdirectory(dtype) # Transformer architecture tests add_subdirectory(transformer) +# Generator tests +add_subdirectory(generator) + # Checkpoint tests add_subdirectory(checkpoint) diff --git a/tests/common/test_utils.h b/tests/common/test_utils.h index 79e595bb..ac504ae7 100644 --- a/tests/common/test_utils.h +++ b/tests/common/test_utils.h @@ -2,11 +2,17 @@ #include +#include #include #include +#if defined(USE_CUDA) +#include +#endif + #include "gtest/gtest.h" +#include "infini_train/include/core/runtime/device_guard.h" #include "infini_train/include/datatype.h" #include "infini_train/include/device.h" #include "infini_train/include/tensor.h" @@ -102,6 +108,18 @@ inline void ExpectTensorNear(const std::shared_ptr &val1, float val2, } \ } while (0) +// Returns tensor bytes on the host for exact comparisons. +// Synchronizes the source device before reading the copied data. +inline std::vector TensorBytes( + const std::shared_ptr &tensor) { + const Tensor host = tensor->To(Device()); + const Device source_device = tensor->GetDevice(); + core::GetDeviceGuardImpl(source_device.type()) + ->SynchronizeDevice(source_device); + const auto *data = static_cast(host.DataPtr()); + return {data, data + host.SizeInBytes()}; +} + class InfiniTrainTest : public ::testing::TestWithParam { protected: Device GetDevice() const; diff --git a/tests/generator/CMakeLists.txt b/tests/generator/CMakeLists.txt new file mode 100644 index 00000000..c84b78ff --- /dev/null +++ b/tests/generator/CMakeLists.txt @@ -0,0 +1,30 @@ +# Shared CPU/CUDA tests. +# TODO: Add backend-independent DDP coverage when the test harness supports multi-node execution. +add_subdirectory(tensor_parallel) + +file(GLOB GENERATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test_*.cc) + +infini_train_add_test_suite(test_generator + SOURCES ${GENERATOR_SOURCES} + TEST_TIMEOUT 40 +) + +file(GLOB GENERATOR_CPU_ONLY_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/cpu_only/test_*.cc) +if(GENERATOR_CPU_ONLY_SOURCES) + infini_train_add_test(test_generator_cpu_only + SOURCES ${GENERATOR_CPU_ONLY_SOURCES} + LABELS cpu + TEST_TIMEOUT 40 + ) +endif() + +if(USE_CUDA) + file(GLOB GENERATOR_CUDA_ONLY_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/cuda_only/test_*.cc) + if(GENERATOR_CUDA_ONLY_SOURCES) + infini_train_add_test(test_generator_cuda_only + SOURCES ${GENERATOR_CUDA_ONLY_SOURCES} + LABELS cuda + TEST_TIMEOUT 40 + ) + endif() +endif() diff --git a/tests/generator/cpu_only/test_generator_cpu.cc b/tests/generator/cpu_only/test_generator_cpu.cc new file mode 100644 index 00000000..14578801 --- /dev/null +++ b/tests/generator/cpu_only/test_generator_cpu.cc @@ -0,0 +1,100 @@ +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/functional.h" +#include "infini_train/include/rng_tracker.h" + +#include "infini_train/src/core/runtime/cpu/cpu_generator_impl.h" +#include "tests/common/test_utils.h" + +using namespace infini_train; +using infini_train::test::TensorBytes; + +class GeneratorCpuTest : public ::testing::Test {}; + +TEST_F(GeneratorCpuTest, ScopedPublicAndBackendGettersSelectTheSameGenerator) { + const Device cpu; + const Generator original = core::cpu::GetDefaultCpuGenerator(); + { + RngScope scope(9050); + EXPECT_EQ(GetDefaultGenerator(cpu), core::cpu::GetDefaultCpuGenerator()); + EXPECT_NE(original, core::cpu::GetDefaultCpuGenerator()); + EXPECT_EQ(core::cpu::GetDefaultCpuGenerator().current_seed(), 9050U); + } + EXPECT_EQ(original, core::cpu::GetDefaultCpuGenerator()); +} + +#ifndef USE_CUDA +TEST_F(GeneratorCpuTest, MissingCudaBackendFailsInsideAndOutsideScopes) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device explicit_cuda(Device::DeviceType::kCUDA, 0); + const Device current_cuda(Device::DeviceType::kCUDA, -1); + EXPECT_DEATH(GetDefaultGenerator(explicit_cuda), + "No GeneratorBackend registered for device type"); + EXPECT_DEATH(GetDefaultGenerator(current_cuda), + "No GeneratorBackend registered for device type"); + RngScope scope(9056); + EXPECT_DEATH(GetDefaultGenerator(explicit_cuda), + "No GeneratorBackend registered for device type"); + EXPECT_DEATH(GetDefaultGenerator(current_cuda), + "No GeneratorBackend registered for device type"); + EXPECT_EQ(GetDefaultGenerator(Device()).current_seed(), 9056U); +} +#endif + +TEST_F(GeneratorCpuTest, HighBitsOfCpuSeedAffectTheSequence) { + const Device cpu; + constexpr uint64_t low_seed = 17; + constexpr uint64_t high_seed = low_seed + (uint64_t{1} << 32); + EXPECT_NE(TensorBytes(nn::function::Rand({256}, DataType::kFLOAT32, cpu, + CreateGenerator(cpu, low_seed))), + TensorBytes(nn::function::Rand({256}, DataType::kFLOAT32, cpu, + CreateGenerator(cpu, high_seed)))); +} + +TEST_F(GeneratorCpuTest, StateRestoresCachedNormalSequence) { + const Device cpu; + auto generator = CreateGenerator(cpu, 4567); + nn::function::Randn({3}, DataType::kFLOAT32, cpu, generator); + nn::function::Randn({3}, DataType::kFLOAT64, cpu, generator); + const auto state_after_odd_normal = generator.get_state(); + const auto normal_float = TensorBytes( + nn::function::Randn({127}, DataType::kFLOAT32, cpu, generator)); + const auto normal_double = TensorBytes( + nn::function::Randn({127}, DataType::kFLOAT64, cpu, generator)); + const auto state_after_replay = TensorBytes(generator.get_state()); + + generator.set_state(*state_after_odd_normal); + EXPECT_EQ(normal_float, TensorBytes(nn::function::Randn( + {127}, DataType::kFLOAT32, cpu, generator))); + EXPECT_EQ(normal_double, TensorBytes(nn::function::Randn( + {127}, DataType::kFLOAT64, cpu, generator))); + EXPECT_EQ(state_after_replay, TensorBytes(generator.get_state())); +} + +TEST_F(GeneratorCpuTest, ReseedingClearsBothNormalCaches) { + const Device cpu; + auto generator = CreateGenerator(cpu, 4567); + nn::function::Randn({3}, DataType::kFLOAT32, cpu, generator); + nn::function::Randn({3}, DataType::kFLOAT64, cpu, generator); + + generator.set_current_seed(5678); + auto fresh = CreateGenerator(cpu, 5678); + EXPECT_EQ(TensorBytes(generator.get_state()), TensorBytes(fresh.get_state())); + EXPECT_EQ( + TensorBytes( + nn::function::Randn({127}, DataType::kFLOAT32, cpu, generator)), + TensorBytes(nn::function::Randn({127}, DataType::kFLOAT32, cpu, fresh))); + EXPECT_EQ( + TensorBytes( + nn::function::Randn({127}, DataType::kFLOAT64, cpu, generator)), + TensorBytes(nn::function::Randn({127}, DataType::kFLOAT64, cpu, fresh))); + EXPECT_EQ(TensorBytes(generator.get_state()), TensorBytes(fresh.get_state())); +} diff --git a/tests/generator/cuda_only/test_generator_cuda.cc b/tests/generator/cuda_only/test_generator_cuda.cc new file mode 100644 index 00000000..ea876014 --- /dev/null +++ b/tests/generator/cuda_only/test_generator_cuda.cc @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/core/runtime/device_guard.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/functional.h" +#include "infini_train/include/nn/init.h" +#include "infini_train/include/rng_tracker.h" +#include "infini_train/include/tensor.h" + +#include "infini_train/src/core/runtime/cuda/cuda_generator_impl.h" +#include "tests/common/test_utils.h" + +#include + +using namespace infini_train; +using infini_train::test::TensorBytes; + +class GeneratorCudaTest : public ::testing::Test { + protected: + int DeviceCount() const { + return core::GetDeviceGuardImpl(cuda0_.type())->DeviceCount(); + } + + const Device cuda0_{Device::DeviceType::kCUDA, 0}; +}; + +TEST_F(GeneratorCudaTest, + ScopedCurrentDeviceAndBackendGettersSelectTheSameGenerator) { + core::DeviceGuard guard(cuda0_); + const Generator original = core::cuda::GetDefaultCudaGenerator(0); + { + RngScope scope(9051); + const Generator selected = GetDefaultGenerator(cuda0_); + EXPECT_NE(original, selected); + EXPECT_EQ(selected, + GetDefaultGenerator(Device(Device::DeviceType::kCUDA, -1))); + EXPECT_EQ(selected, core::cuda::GetDefaultCudaGenerator(-1)); + EXPECT_EQ(selected, core::cuda::GetDefaultCudaGenerator(0)); + } + EXPECT_EQ(original, core::cuda::GetDefaultCudaGenerator(-1)); +} + +TEST_F(GeneratorCudaTest, ScopedSeedAndSnapshotIncludeDevicesCreatedLater) { + const Device cpu; + RngTracker tracker(9052); + RngScope scope(tracker); + nn::function::Rand({19}, DataType::kFLOAT32, cpu); + ManualSeed(9053); + const auto snapshot = tracker.GetState(); + EXPECT_EQ(GetDefaultGenerator(cpu).current_seed(), 9053U); + EXPECT_EQ(GetDefaultGenerator(cuda0_).current_seed(), 9053U); + const auto expected = + TensorBytes(nn::function::Rand({19}, DataType::kFLOAT32, cuda0_)); + EXPECT_EQ(expected, + TensorBytes(nn::function::Rand({19}, DataType::kFLOAT32, cuda0_, + CreateGenerator(cuda0_, 9053)))); + ManualSeed(9054); + nn::function::Rand({19}, DataType::kFLOAT32, cuda0_); + tracker.SetState(*snapshot); + EXPECT_EQ(expected, + TensorBytes(nn::function::Rand({19}, DataType::kFLOAT32, cuda0_))); +} + +TEST_F(GeneratorCudaTest, + TrackerRejectsCudaStateWithoutChangingLiveGenerators) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + RngTracker tracker(9033); + RngScope scope(tracker); + nn::function::Rand({19}, DataType::kFLOAT32, cuda0_); + const Generator selected = GetDefaultGenerator(cuda0_); + const auto before = TensorBytes(selected.get_state()); + const auto snapshot = tracker.GetState(); + const auto bytes = TensorBytes(snapshot); + auto reference = selected.Clone(); + const auto cuda_state = snapshot->To(cuda0_); + + EXPECT_DEATH( + tracker.SetState(cuda_state), + "RNG tracker state must be a nonempty one-dimensional CPU UINT8 tensor"); + EXPECT_EQ(selected, GetDefaultGenerator(cuda0_)); + EXPECT_EQ(before, TensorBytes(selected.get_state())); + EXPECT_EQ(bytes, TensorBytes(tracker.GetState())); + EXPECT_EQ(TensorBytes(nn::function::Rand({19}, DataType::kFLOAT32, cuda0_, + reference)), + TensorBytes(nn::function::Rand({19}, DataType::kFLOAT32, cuda0_))); +} + +TEST_F(GeneratorCudaTest, ScopedGeneratorsRemainIndependentAcrossDevices) { + REQUIRE_MIN_DEVICES(2); + const Device cuda1(Device::DeviceType::kCUDA, 1); + RngScope scope(9055); + const auto untouched = TensorBytes(GetDefaultGenerator(cuda1).get_state()); + nn::function::Rand({19}, DataType::kFLOAT32, cuda0_); + EXPECT_EQ(untouched, TensorBytes(GetDefaultGenerator(cuda1).get_state())); + { + core::DeviceGuard guard(cuda1); + EXPECT_EQ(GetDefaultGenerator(cuda1), + GetDefaultGenerator(Device(Device::DeviceType::kCUDA, -1))); + EXPECT_EQ(GetDefaultGenerator(cuda1), + core::cuda::GetDefaultCudaGenerator(-1)); + } +} + +TEST_F(GeneratorCudaTest, StateRejectsCudaAndForeignInput) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + auto generator = CreateGenerator(cuda0_, 4601); + const auto valid_state = generator.get_state(); + + auto wrong_device = std::make_shared(std::vector{1}, + DataType::kUINT8, cuda0_); + EXPECT_DEATH(generator.set_state(*wrong_device), "CPU"); + + const auto cpu_state = CreateGenerator(Device(), 4601).get_state(); + EXPECT_DEATH(generator.set_state(*cpu_state), "Check failed"); + + auto cpu_generator = CreateGenerator(Device(), 4601); + EXPECT_DEATH(cpu_generator.set_state(*valid_state), "Check failed"); +} + +TEST_F(GeneratorCudaTest, CrossBackendGeneratorsAreRejected) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device cpu; + const auto cpu_generator = CreateGenerator(cpu, 8001); + const auto cuda_generator = CreateGenerator(cuda0_, 8002); + + auto cpu_input = std::make_shared(std::vector{8}, + DataType::kFLOAT32, cpu); + auto cuda_input = std::make_shared(std::vector{8}, + DataType::kFLOAT32, cuda0_); + cpu_input->Fill(1.0f); + cuda_input->Fill(1.0f); + + EXPECT_DEATH(nn::function::Rand({8}, DataType::kFLOAT32, cpu, cuda_generator), + "Generator device mismatch"); + EXPECT_DEATH( + nn::function::Rand({8}, DataType::kFLOAT32, cuda0_, cpu_generator), + "Generator device mismatch"); + EXPECT_DEATH(nn::function::Dropout(cpu_input, 0.25, true, cuda_generator), + "Generator device mismatch"); + EXPECT_DEATH(nn::function::Dropout(cuda_input, 0.25, true, cpu_generator), + "Generator device mismatch"); +} + +// Regression for the device-index check: a generator bound to cuda:0 must not +// drive a cuda:1 tensor, and the rejected calls must not advance its state. +TEST_F(GeneratorCudaTest, CudaGeneratorRejectsAnotherCudaDevice) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + REQUIRE_MIN_DEVICES(2); + const Device cuda1(Device::DeviceType::kCUDA, 1); + auto generator = CreateGenerator(cuda0_, 8401); + const auto state_before = TensorBytes(generator.get_state()); + + auto input = std::make_shared(std::vector{257}, + DataType::kFLOAT32, cuda1); + input->Fill(1.0f); + + EXPECT_DEATH(nn::function::Rand({257}, DataType::kFLOAT32, cuda1, generator), + "Generator device mismatch"); + EXPECT_DEATH(nn::function::Randn({257}, DataType::kFLOAT32, cuda1, generator), + "Generator device mismatch"); + EXPECT_DEATH(nn::function::Dropout(input, 0.25, true, generator), + "Generator device mismatch"); + + EXPECT_EQ(state_before, TensorBytes(generator.get_state())); +} + +TEST_F(GeneratorCudaTest, DefaultCudaGeneratorsAreIndependentAcrossAllDevices) { + REQUIRE_MIN_DEVICES(2); + int device_count = 0; + ASSERT_EQ(cudaGetDeviceCount(&device_count), cudaSuccess); + + ManualSeed(8501); + std::vector devices; + devices.reserve(device_count); + for (int index = 0; index < device_count; ++index) { + const Device device(Device::DeviceType::kCUDA, index); + devices.push_back(device); + EXPECT_EQ(GetDefaultGenerator(device).device(), device); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 8501U); + } + + for (int active_index = 0; active_index < device_count; ++active_index) { + std::vector> states_before; + states_before.reserve(device_count); + for (const auto &device : devices) { + states_before.push_back( + TensorBytes(GetDefaultGenerator(device).get_state())); + } + + auto tensor = std::make_shared( + std::vector{64}, DataType::kFLOAT32, devices[active_index]); + nn::init::Uniform(tensor); + + for (int observed_index = 0; observed_index < device_count; + ++observed_index) { + const auto state_after = + TensorBytes(GetDefaultGenerator(devices[observed_index]).get_state()); + if (observed_index == active_index) { + EXPECT_NE(states_before[observed_index], state_after); + } else { + EXPECT_EQ(states_before[observed_index], state_after); + } + } + } +} diff --git a/tests/generator/tensor_parallel/CMakeLists.txt b/tests/generator/tensor_parallel/CMakeLists.txt new file mode 100644 index 00000000..22a7e0ff --- /dev/null +++ b/tests/generator/tensor_parallel/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(test_rng_tensor_parallel test_rng_tensor_parallel.cc) +target_compile_options(test_rng_tensor_parallel PRIVATE -Wno-error) +target_link_libraries(test_rng_tensor_parallel PRIVATE GTest::gtest) +target_include_directories(test_rng_tensor_parallel PRIVATE ${glog_SOURCE_DIR}/src) +link_infini_train_exe(test_rng_tensor_parallel) +gtest_discover_tests(test_rng_tensor_parallel + PROPERTIES LABELS cpu TIMEOUT 40 +) diff --git a/tests/generator/tensor_parallel/test_rng_tensor_parallel.cc b/tests/generator/tensor_parallel/test_rng_tensor_parallel.cc new file mode 100644 index 00000000..0ab1197a --- /dev/null +++ b/tests/generator/tensor_parallel/test_rng_tensor_parallel.cc @@ -0,0 +1,139 @@ +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/modules/sparse.h" +#include "infini_train/include/nn/parallel/global.h" +#include "infini_train/include/nn/parallel/model_parallel_config.h" +#include "infini_train/include/nn/parallel/tensor_parallel.h" +#include "infini_train/include/rng_tracker.h" + +#include "tests/common/test_utils.h" + +namespace infini_train { +namespace { + +using nn::parallel::ColumnParallelLinear; +using nn::parallel::kTensorParallelInitRngName; +using nn::parallel::RowParallelLinear; +using test::TensorBytes; + +struct ModelParameters { + std::vector embedding; + std::vector column_weight; + std::vector column_bias; + std::vector row_weight; + std::vector row_bias; + + bool operator==(const ModelParameters &) const = default; +}; + +ModelParameters InitializeModel() { + nn::Embedding embedding(16, 8); + ColumnParallelLinear column(8, 16, true, false, false, false); + RowParallelLinear row(16, 8, true, true, true, false); + return {TensorBytes(embedding.parameter(nn::Embedding::kParamWeightName)), + TensorBytes(column.parameter(ColumnParallelLinear::kParamWeightName)), + TensorBytes(column.parameter(ColumnParallelLinear::kParamBiasName)), + TensorBytes(row.parameter(RowParallelLinear::kParamWeightName)), + TensorBytes(row.parameter(RowParallelLinear::kParamBiasName))}; +} + +std::array InitializeRanks( + const std::array &order) { + std::array parameters; + std::array, 4> start; + std::array, 4> done; + std::array threads; + for (int rank = 0; rank < 4; ++rank) { + threads[rank] = std::thread([&, rank] { + RngTracker tracker(42); + tracker.Add(kTensorParallelInitRngName, 42 + 2718 + rank % 2); + RngScope scope(tracker); + start[rank].get_future().wait(); + parameters[rank] = InitializeModel(); + done[rank].set_value(); + }); + } + for (int rank : order) { + start[rank].set_value(); + done[rank].get_future().wait(); + } + for (auto &thread : threads) { + thread.join(); + } + return parameters; +} + +TEST(RngTensorParallelTest, ShardsDifferAndReplicatedParametersMatch) { + const auto parameters = InitializeRanks({0, 1, 2, 3}); + EXPECT_EQ(parameters[0], parameters[2]); + EXPECT_EQ(parameters[1], parameters[3]); + EXPECT_EQ(parameters[0].embedding, parameters[1].embedding); + EXPECT_EQ(parameters[0].row_bias, parameters[1].row_bias); + EXPECT_NE(parameters[0].column_weight, parameters[1].column_weight); + EXPECT_NE(parameters[0].column_bias, parameters[1].column_bias); + EXPECT_NE(parameters[0].row_weight, parameters[1].row_weight); +} + +TEST(RngTensorParallelTest, ThreadOrderDoesNotChangeInitialization) { + const auto forward = InitializeRanks({0, 1, 2, 3}); + const auto reverse = InitializeRanks({3, 2, 1, 0}); + EXPECT_EQ(forward, reverse); +} + +TEST(RngTensorParallelTest, ConsecutiveLayersContinueTheShardedStream) { + RngTracker tracker(42); + tracker.Add(kTensorParallelInitRngName, 42 + 2718); + RngScope scope(tracker); + ColumnParallelLinear first(8, 16, true, false, false, false); + ColumnParallelLinear second(8, 16, true, false, false, false); + EXPECT_NE( + TensorBytes(first.parameter(ColumnParallelLinear::kParamWeightName)), + TensorBytes(second.parameter(ColumnParallelLinear::kParamWeightName))); +} + +TEST(RngTensorParallelTest, MissingNamedStreamUsesTheCurrentDefaultStream) { + ModelParameters first; + { + RngScope scope(42); + first = InitializeModel(); + } + ModelParameters replay; + { + RngScope scope(42); + replay = InitializeModel(); + } + EXPECT_EQ(first, replay); +} + +TEST(RngTensorParallelTest, UnscopedInitializationUsesTheProcessDefault) { + const auto &generator = GetDefaultGenerator(Device()); + generator.set_current_seed(42); + const auto first = InitializeModel(); + generator.set_current_seed(42); + EXPECT_EQ(first, InitializeModel()); +} + +} // namespace +} // namespace infini_train + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + infini_train::nn::parallel::ModelParallelConfig config; + config.tensor_model_parallel_size = 2; + config.pipeline_model_parallel_size = 1; + config.virtual_pipeline_model_parallel_size = 1; + config.sequence_parallel = false; + config.expert_model_parallel_size = 1; + config.expert_tensor_parallel_size = 1; + infini_train::nn::parallel::global::InitAllEnv(4, config); + return RUN_ALL_TESTS(); +} diff --git a/tests/generator/test_generator_core.cc b/tests/generator/test_generator_core.cc new file mode 100644 index 00000000..203c13b1 --- /dev/null +++ b/tests/generator/test_generator_core.cc @@ -0,0 +1,140 @@ +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/functional.h" +#include "infini_train/include/nn/init.h" +#include "infini_train/include/tensor.h" + +#include "tests/common/test_utils.h" + +using namespace infini_train; +using infini_train::test::TensorBytes; + +class GeneratorCoreTest : public infini_train::test::InfiniTrainTest {}; + +TEST_P(GeneratorCoreTest, PublicInterfaceCopyCloneAndSeed) { + const Device device = GetDevice(); + auto generator = CreateGenerator(device); + EXPECT_EQ(generator.device(), device); + EXPECT_EQ(generator.current_seed(), Generator::kDefaultSeed); + + generator.set_current_seed(1234); + EXPECT_EQ(generator.current_seed(), 1234U); + const auto first = TensorBytes( + nn::function::Rand({128}, DataType::kFLOAT32, device, generator)); + generator.set_current_seed(1234); + EXPECT_EQ(first, TensorBytes(nn::function::Rand({128}, DataType::kFLOAT32, + device, generator))); + + generator.set_current_seed(2345); + Generator alias = generator; + EXPECT_EQ(alias, generator); + const auto state_before = TensorBytes(alias.get_state()); + nn::function::Rand({32}, DataType::kFLOAT32, device, generator); + EXPECT_NE(state_before, TensorBytes(alias.get_state())); + + auto clone = generator.Clone(); + EXPECT_NE(clone, generator); + EXPECT_EQ(TensorBytes(clone.get_state()), TensorBytes(generator.get_state())); + clone.set_current_seed(3456); + EXPECT_EQ(generator.current_seed(), 2345U); + EXPECT_EQ(clone.current_seed(), 3456U); + + const uint64_t generated_seed = clone.Seed(); + EXPECT_EQ(clone.current_seed(), generated_seed); +} + +TEST_P(GeneratorCoreTest, StateRestoresUniformSequence) { + const Device device = GetDevice(); + auto generator = CreateGenerator(device, 4567); + const auto state = generator.get_state(); + EXPECT_EQ(state->Dtype(), DataType::kUINT8); + EXPECT_EQ(state->GetDevice(), Device()); + + const auto uniform = TensorBytes( + nn::function::Rand({129}, DataType::kFLOAT32, device, generator)); + generator.set_state(*state); + EXPECT_EQ(uniform, TensorBytes(nn::function::Rand({129}, DataType::kFLOAT32, + device, generator))); +} + +TEST_P(GeneratorCoreTest, StateRejectsMalformedInput) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device device = GetDevice(); + auto generator = CreateGenerator(device, 4601); + auto wrong_dtype = std::make_shared(std::vector{1}, + DataType::kFLOAT32, Device()); + EXPECT_DEATH(generator.set_state(*wrong_dtype), "UINT8"); + + const auto valid_state = generator.get_state(); + auto truncated = std::make_shared( + std::vector{ + static_cast(valid_state->SizeInBytes() - 1)}, + DataType::kUINT8, Device()); + EXPECT_DEATH(generator.set_state(*truncated), "Check failed"); +} + +TEST_P(GeneratorCoreTest, DefaultExplicitAndUndefinedPathsHaveExpectedState) { + const Device device = GetDevice(); + constexpr uint64_t seed = 7890; + + ManualSeed(seed); + const auto *first_default = &GetDefaultGenerator(device); + const auto *second_default = &GetDefaultGenerator(device); + EXPECT_EQ(first_default, second_default); + EXPECT_EQ(first_default->device(), device); + + const auto default_state = TensorBytes(first_default->get_state()); + const auto explicit_result = TensorBytes(nn::function::Rand( + {256}, DataType::kFLOAT32, device, CreateGenerator(device, seed))); + EXPECT_EQ(default_state, TensorBytes(first_default->get_state())); + EXPECT_EQ(explicit_result, + TensorBytes(nn::function::Rand({256}, DataType::kFLOAT32, device))); + + ManualSeed(seed + 1); + const auto default_result = + TensorBytes(nn::function::Rand({256}, DataType::kFLOAT32, device)); + ManualSeed(seed + 1); + std::optional undefined = std::nullopt; + EXPECT_EQ(default_result, TensorBytes(nn::function::Rand( + {256}, DataType::kFLOAT32, device, undefined))); +} + +TEST_P(GeneratorCoreTest, InitializerEntryPointsUseExplicitGenerator) { + const Device device = GetDevice(); + auto make_tensor = [&] { + return std::make_shared(std::vector{16, 16}, + DataType::kFLOAT32, device); + }; + + auto normal_a = make_tensor(); + auto normal_b = make_tensor(); + nn::init::Normal(normal_a, 1.5f, 0.5f, CreateGenerator(device, 8101)); + nn::init::Normal(normal_b, 1.5f, 0.5f, CreateGenerator(device, 8101)); + EXPECT_EQ(TensorBytes(normal_a), TensorBytes(normal_b)); + + auto uniform_a = make_tensor(); + auto uniform_b = make_tensor(); + nn::init::Uniform(uniform_a, -2.0f, 3.0f, CreateGenerator(device, 8102)); + uniform_b->Uniform(-2.0f, 3.0f, CreateGenerator(device, 8102)); + EXPECT_EQ(TensorBytes(uniform_a), TensorBytes(uniform_b)); + + auto kaiming_a = make_tensor(); + auto kaiming_b = make_tensor(); + nn::init::KaimingUniform(kaiming_a, 0.1f, nn::init::KaimingMode::kFanIn, + nn::init::NonLinearityType::kLeakyReLU, + CreateGenerator(device, 8103)); + nn::init::KaimingUniform(kaiming_b, 0.1f, nn::init::KaimingMode::kFanIn, + nn::init::NonLinearityType::kLeakyReLU, + CreateGenerator(device, 8103)); + EXPECT_EQ(TensorBytes(kaiming_a), TensorBytes(kaiming_b)); +} + +INFINI_TRAIN_REGISTER_TEST(GeneratorCoreTest); diff --git a/tests/generator/test_generator_random_ops.cc b/tests/generator/test_generator_random_ops.cc new file mode 100644 index 00000000..663463b5 --- /dev/null +++ b/tests/generator/test_generator_random_ops.cc @@ -0,0 +1,464 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/autograd/dropout.h" +#include "infini_train/include/core/runtime/device_guard.h" +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/functional.h" +#include "infini_train/include/nn/init.h" +#include "infini_train/include/tensor.h" + +#include "tests/common/test_utils.h" + +using namespace infini_train; +using infini_train::test::TensorBytes; + +namespace { + +template +void ExpectUniformRangeAndFinite(const Tensor &tensor) { + const auto *data = static_cast(tensor.DataPtr()); + for (int64_t index = 0; index < tensor.NumElements(); ++index) { + const double value = static_cast(data[index]); + EXPECT_TRUE(std::isfinite(value)); + EXPECT_GE(value, 0.0); + EXPECT_LT(value, 1.0); + } +} + +template +void ExpectFinite(const Tensor &tensor) { + const auto *data = static_cast(tensor.DataPtr()); + for (int64_t index = 0; index < tensor.NumElements(); ++index) { + EXPECT_TRUE(std::isfinite(static_cast(data[index]))); + } +} + +void CheckUniformRangeAndFinite(const std::shared_ptr &tensor) { + const Tensor host = tensor->To(Device()); + const Device source_device = tensor->GetDevice(); + core::GetDeviceGuardImpl(source_device.type()) + ->SynchronizeDevice(source_device); + switch (host.Dtype()) { + case DataType::kFLOAT16: + ExpectUniformRangeAndFinite(host); + break; + case DataType::kBFLOAT16: + ExpectUniformRangeAndFinite(host); + break; + case DataType::kFLOAT32: + ExpectUniformRangeAndFinite(host); + break; + case DataType::kFLOAT64: + ExpectUniformRangeAndFinite(host); + break; + default: + FAIL() << "Unexpected dtype"; + } +} + +void CheckFinite(const std::shared_ptr &tensor) { + const Tensor host = tensor->To(Device()); + const Device source_device = tensor->GetDevice(); + core::GetDeviceGuardImpl(source_device.type()) + ->SynchronizeDevice(source_device); + switch (host.Dtype()) { + case DataType::kFLOAT16: + ExpectFinite(host); + break; + case DataType::kBFLOAT16: + ExpectFinite(host); + break; + case DataType::kFLOAT32: + ExpectFinite(host); + break; + case DataType::kFLOAT64: + ExpectFinite(host); + break; + default: + FAIL() << "Unexpected dtype"; + } +} + +std::vector> RunRandomScript( + const Device &device, std::optional generator) { + auto input = std::make_shared(std::vector{257}, + DataType::kFLOAT32, device); + input->Fill(1.0f); + return { + TensorBytes( + nn::function::Rand({257}, DataType::kFLOAT32, device, generator)), + TensorBytes( + nn::function::Randn({257}, DataType::kFLOAT32, device, generator)), + TensorBytes(nn::function::Dropout(input, 0.25, true, generator)), + TensorBytes( + nn::function::Rand({257}, DataType::kFLOAT32, device, generator)), + }; +} + +void ExpectMaskValues(const std::shared_ptr &mask) { + EXPECT_EQ(mask->Dtype(), DataType::kUINT8); + const Tensor host = mask->To(Device()); + const Device source_device = mask->GetDevice(); + core::GetDeviceGuardImpl(source_device.type()) + ->SynchronizeDevice(source_device); + const auto *data = static_cast(host.DataPtr()); + bool saw_zero = false; + bool saw_one = false; + for (int64_t index = 0; index < host.NumElements(); ++index) { + EXPECT_TRUE(data[index] == 0 || data[index] == 1); + saw_zero |= data[index] == 0; + saw_one |= data[index] == 1; + } + EXPECT_TRUE(saw_zero); + EXPECT_TRUE(saw_one); +} + +template +void ExpectUnitInputDropoutValues(const Tensor &output, const Tensor &mask, + double p) { + const auto *output_data = static_cast(output.DataPtr()); + const auto *mask_data = static_cast(mask.DataPtr()); + const double kept_value = 1.0 / (1.0 - p); + for (int64_t index = 0; index < output.NumElements(); ++index) { + const double expected = mask_data[index] ? kept_value : 0.0; + EXPECT_NEAR(static_cast(output_data[index]), expected, 0.02); + } +} + +void ExpectUnitInputDropoutValues(const std::shared_ptr &output, + const std::shared_ptr &mask, + double p) { + const Tensor host_output = output->To(Device()); + const Tensor host_mask = mask->To(Device()); + // Each async D2H copy must complete before its host buffer is read. + const Device output_device = output->GetDevice(); + core::GetDeviceGuardImpl(output_device.type()) + ->SynchronizeDevice(output_device); + const Device mask_source_device = mask->GetDevice(); + core::GetDeviceGuardImpl(mask_source_device.type()) + ->SynchronizeDevice(mask_source_device); + switch (host_output.Dtype()) { + case DataType::kFLOAT16: + ExpectUnitInputDropoutValues(host_output, host_mask, p); + break; + case DataType::kBFLOAT16: + ExpectUnitInputDropoutValues(host_output, host_mask, p); + break; + case DataType::kFLOAT32: + ExpectUnitInputDropoutValues(host_output, host_mask, p); + break; + case DataType::kFLOAT64: + ExpectUnitInputDropoutValues(host_output, host_mask, p); + break; + default: + FAIL() << "Unexpected dtype"; + } +} + +} // namespace + +class GeneratorRandomOpsTest : public infini_train::test::InfiniTrainTest {}; + +TEST_P(GeneratorRandomOpsTest, RandAndRandnSupportAllFloatingDtypes) { + const std::vector dtypes = {DataType::kFLOAT16, DataType::kBFLOAT16, + DataType::kFLOAT32, DataType::kFLOAT64}; + const Device device = GetDevice(); + + for (const auto dtype : dtypes) { + auto uniform_first = CreateGenerator(device, 1001); + auto uniform_replay = CreateGenerator(device, 1001); + auto uniform = nn::function::Rand({4097}, dtype, device, uniform_first); + EXPECT_EQ( + TensorBytes(uniform), + TensorBytes(nn::function::Rand({4097}, dtype, device, uniform_replay))); + CheckUniformRangeAndFinite(uniform); + + auto normal_first = CreateGenerator(device, 2002); + auto normal_replay = CreateGenerator(device, 2002); + auto normal = nn::function::Randn({4097}, dtype, device, normal_first); + EXPECT_EQ(TensorBytes(normal), TensorBytes(nn::function::Randn( + {4097}, dtype, device, normal_replay))); + CheckFinite(normal); + } +} + +TEST_P(GeneratorRandomOpsTest, UniformConstantDoesNotAdvanceGenerator) { + const Device device = GetDevice(); + for (const auto dtype : {DataType::kFLOAT16, DataType::kBFLOAT16, + DataType::kFLOAT32, DataType::kFLOAT64}) { + SCOPED_TRACE(static_cast(dtype)); + auto generator = CreateGenerator(device, 2101); + nn::function::Rand({11}, dtype, device, generator); + const auto state_before = TensorBytes(generator.get_state()); + auto replay = generator.Clone(); + auto tensor = + std::make_shared(std::vector{257}, dtype, device); + auto expected = std::make_shared(tensor->Dims(), dtype, Device()); + expected->Fill(0.1f); + + nn::init::Uniform(tensor, 0.1f, 0.1f, generator); + + EXPECT_EQ(TensorBytes(tensor), TensorBytes(expected)); + EXPECT_EQ(TensorBytes(generator.get_state()), state_before); + EXPECT_EQ(TensorBytes(nn::function::Rand({129}, dtype, device, generator)), + TensorBytes(nn::function::Rand({129}, dtype, device, replay))); + } +} + +TEST_P(GeneratorRandomOpsTest, UniformEmptyTensorDoesNotAdvanceGenerator) { + const Device device = GetDevice(); + for (const auto dtype : {DataType::kFLOAT16, DataType::kBFLOAT16, + DataType::kFLOAT32, DataType::kFLOAT64}) { + SCOPED_TRACE(static_cast(dtype)); + auto generator = CreateGenerator(device, 2102); + nn::function::Rand({11}, dtype, device, generator); + const auto state_before = TensorBytes(generator.get_state()); + auto replay = generator.Clone(); + auto tensor = + std::make_shared(std::vector{2, 0, 3}, dtype, device); + + nn::init::Uniform(tensor, -2.0f, 3.0f, generator); + + EXPECT_EQ(tensor->NumElements(), 0); + EXPECT_EQ(TensorBytes(generator.get_state()), state_before); + EXPECT_EQ(TensorBytes(nn::function::Rand({129}, dtype, device, generator)), + TensorBytes(nn::function::Rand({129}, dtype, device, replay))); + } +} + +TEST_P(GeneratorRandomOpsTest, + DropoutSupportsAllFloatingDtypesWithExplicitGenerator) { + const std::vector dtypes = {DataType::kFLOAT16, DataType::kBFLOAT16, + DataType::kFLOAT32, DataType::kFLOAT64}; + const Device device = GetDevice(); + + for (const auto dtype : dtypes) { + auto input = nn::function::Rand({4097}, dtype, device, + CreateGenerator(device, 3003)); + ManualSeed(4004); + const auto default_state_before = + TensorBytes(GetDefaultGenerator(device).get_state()); + + auto dropout_first = CreateGenerator(device, 5005); + auto dropout_replay = CreateGenerator(device, 5005); + auto output = nn::function::Dropout(input, 0.25, true, dropout_first); + EXPECT_EQ(TensorBytes(output), TensorBytes(nn::function::Dropout( + input, 0.25, true, dropout_replay))); + CheckFinite(output); + EXPECT_EQ(default_state_before, + TensorBytes(GetDefaultGenerator(device).get_state())); + } +} + +TEST_P(GeneratorRandomOpsTest, SameSeedReplaysRandRandnAndDropoutScript) { + const Device device = GetDevice(); + EXPECT_EQ(RunRandomScript(device, CreateGenerator(device, 5678)), + RunRandomScript(device, CreateGenerator(device, 5678))); + EXPECT_NE(RunRandomScript(device, CreateGenerator(device, 5678)), + RunRandomScript(device, CreateGenerator(device, 5679))); + + auto generator = CreateGenerator(device, 6789); + const auto first = TensorBytes( + nn::function::Rand({256}, DataType::kFLOAT32, device, generator)); + const auto second = TensorBytes( + nn::function::Rand({256}, DataType::kFLOAT32, device, generator)); + EXPECT_NE(first, second); +} + +TEST_P(GeneratorRandomOpsTest, + DefaultGeneratorReplaysRandRandnAndDropoutScript) { + const Device device = GetDevice(); + + ManualSeed(7001); + const auto expected = RunRandomScript(device, std::nullopt); + ManualSeed(7001); + EXPECT_EQ(expected, RunRandomScript(device, std::nullopt)); + + ManualSeed(7001); + std::optional undefined = std::nullopt; + EXPECT_EQ(expected, RunRandomScript(device, undefined)); +} + +TEST_P(GeneratorRandomOpsTest, + ExplicitRandomGeneratorsDoNotAdvanceDefaultGenerator) { + const Device device = GetDevice(); + ManualSeed(7101); + const auto default_state_before = + TensorBytes(GetDefaultGenerator(device).get_state()); + + RunRandomScript(device, CreateGenerator(device, 7102)); + + EXPECT_EQ(default_state_before, + TensorBytes(GetDefaultGenerator(device).get_state())); +} + +TEST_P(GeneratorRandomOpsTest, OffsetViewsUseTheirOwnDataPointers) { + const Device device = GetDevice(); + constexpr int64_t kStorageElements = 12; + constexpr int64_t kOffsetElements = 4; + constexpr int64_t kViewElements = 4; + constexpr size_t kOffsetBytes = kOffsetElements * sizeof(float); + + auto storage = std::make_shared( + std::vector{kStorageElements}, DataType::kFLOAT32, device); + storage->Fill(-1.0f); + auto uniform_view = std::make_shared( + *storage, kOffsetBytes, std::vector{kViewElements}); + nn::init::Uniform(uniform_view, 0.0f, 1.0f, CreateGenerator(device, 7201)); + + const Tensor uniform_storage = storage->To(Device()); + const Device storage_device = storage->GetDevice(); + core::GetDeviceGuardImpl(storage_device.type()) + ->SynchronizeDevice(storage_device); + const auto *uniform_data = + static_cast(uniform_storage.DataPtr()); + for (int64_t index = 0; index < kOffsetElements; ++index) { + EXPECT_FLOAT_EQ(uniform_data[index], -1.0f); + } + for (int64_t index = kOffsetElements; index < kOffsetElements + kViewElements; + ++index) { + EXPECT_GE(uniform_data[index], 0.0f); + EXPECT_LT(uniform_data[index], 1.0f); + } + for (int64_t index = kOffsetElements + kViewElements; + index < kStorageElements; ++index) { + EXPECT_FLOAT_EQ(uniform_data[index], -1.0f); + } + + auto input_storage = std::make_shared( + std::vector{kStorageElements}, DataType::kFLOAT32, device); + input_storage->Fill(-3.0f); + auto input_view = std::make_shared( + *input_storage, kOffsetBytes, std::vector{kViewElements}); + input_view->Fill(1.0f); + const auto output = nn::function::Dropout(input_view, 0.5, true, + CreateGenerator(device, 7202)); + const Tensor host_output = output->To(Device()); + const Device output_device = output->GetDevice(); + core::GetDeviceGuardImpl(output_device.type()) + ->SynchronizeDevice(output_device); + const auto *output_data = static_cast(host_output.DataPtr()); + for (int64_t index = 0; index < kViewElements; ++index) { + EXPECT_TRUE(output_data[index] == 0.0f || + std::abs(output_data[index] - 2.0f) < 1e-6f); + } + + const Tensor input_storage_host = input_storage->To(Device()); + const Device input_storage_device = input_storage->GetDevice(); + core::GetDeviceGuardImpl(input_storage_device.type()) + ->SynchronizeDevice(input_storage_device); + const auto *input_storage_data = + static_cast(input_storage_host.DataPtr()); + for (int64_t index = 0; index < kOffsetElements; ++index) { + EXPECT_FLOAT_EQ(input_storage_data[index], -3.0f); + } + for (int64_t index = kOffsetElements + kViewElements; + index < kStorageElements; ++index) { + EXPECT_FLOAT_EQ(input_storage_data[index], -3.0f); + } +} + +TEST_P(GeneratorRandomOpsTest, DropoutBackwardSupportsAllFloatingDtypes) { + const Device device = GetDevice(); + const std::vector dtypes = {DataType::kFLOAT16, DataType::kBFLOAT16, + DataType::kFLOAT32, DataType::kFLOAT64}; + + for (const auto dtype : dtypes) { + auto input = std::make_shared(std::vector{4097}, dtype, + device, true); + input->Fill(1.0f); + auto dropout = std::make_shared( + 0.25, CreateGenerator(device, 8201)); + const auto outputs = dropout->Apply({input}); + ASSERT_EQ(outputs.size(), 2U); + ExpectMaskValues(outputs[1]); + ExpectUnitInputDropoutValues(outputs[0], outputs[1], 0.25); + + auto grad_output = std::make_shared(input->Dims(), dtype, device); + grad_output->Fill(1.0f); + const auto grad_inputs = dropout->Backward({grad_output, nullptr}); + ASSERT_EQ(grad_inputs.size(), 1U); + EXPECT_EQ(TensorBytes(outputs[0]), TensorBytes(grad_inputs[0])); + } +} + +TEST_P(GeneratorRandomOpsTest, DropoutIdentityBoundariesDoNotAdvanceGenerator) { + const Device device = GetDevice(); + auto input = std::make_shared(std::vector{64}, + DataType::kFLOAT32, device, true); + input->Fill(1.0f); + auto generator = CreateGenerator(device, 8301); + + auto expect_unchanged = [&](const auto &operation) { + const auto before = TensorBytes(generator.get_state()); + operation(); + EXPECT_EQ(before, TensorBytes(generator.get_state())); + }; + + expect_unchanged([&] { + EXPECT_EQ(nn::function::Dropout(input, 0.5, false, generator), input); + }); + expect_unchanged([&] { + EXPECT_EQ(nn::function::Dropout(input, 0.0, true, generator), input); + }); +} + +TEST_P(GeneratorRandomOpsTest, + DropoutP1ProducesZeroForwardAndBackwardWithoutAdvancing) { + const Device device = GetDevice(); + auto input = std::make_shared(std::vector{64}, + DataType::kFLOAT32, device, true); + input->Fill(1.0f); + auto generator = CreateGenerator(device, 8301); + const auto before = TensorBytes(generator.get_state()); + + auto all_dropped = nn::function::Dropout(input, 1.0, true, generator); + EXPECT_EQ(before, TensorBytes(generator.get_state())); + auto zeros = std::make_shared(input->Dims(), input->Dtype(), device); + zeros->Fill(0.0f); + EXPECT_EQ(TensorBytes(all_dropped), TensorBytes(zeros)); + + auto dropout = std::make_shared(1.0, generator); + const auto outputs = dropout->Apply({input}); + auto grad_output = + std::make_shared(input->Dims(), input->Dtype(), device); + grad_output->Fill(1.0f); + const auto grad_inputs = dropout->Backward({grad_output, nullptr}); + EXPECT_EQ(TensorBytes(grad_inputs[0]), TensorBytes(zeros)); +} + +TEST_P(GeneratorRandomOpsTest, InvalidDistributionParametersAreRejected) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device device = GetDevice(); + auto tensor = std::make_shared(std::vector{8}, + DataType::kFLOAT32, device); + auto integer_tensor = std::make_shared(std::vector{8}, + DataType::kINT32, device); + auto generator = CreateGenerator(device, 9001); + + EXPECT_DEATH(nn::init::Uniform(tensor, 2.0f, 1.0f, generator), + "uniform expects a"); + EXPECT_DEATH(nn::init::Normal(tensor, 0.0f, -1.0f, generator), "std >= 0.0"); + EXPECT_DEATH(nn::function::Dropout(tensor, -0.1, true, generator), + "between 0 and 1"); + EXPECT_DEATH(nn::function::Dropout(tensor, 1.1, true, generator), + "between 0 and 1"); + EXPECT_DEATH(nn::function::Rand({8}, DataType::kINT32, device, generator), + "floating-point"); + EXPECT_DEATH(nn::init::Uniform(integer_tensor, 0.0f, 1.0f, generator), + "floating-point"); + EXPECT_DEATH(nn::init::Normal(integer_tensor, 0.0f, 1.0f, generator), + "floating-point"); +} + +INFINI_TRAIN_REGISTER_TEST(GeneratorRandomOpsTest); diff --git a/tests/generator/test_rng_tracker.cc b/tests/generator/test_rng_tracker.cc new file mode 100644 index 00000000..70119c9e --- /dev/null +++ b/tests/generator/test_rng_tracker.cc @@ -0,0 +1,450 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "infini_train/include/datatype.h" +#include "infini_train/include/device.h" +#include "infini_train/include/generator.h" +#include "infini_train/include/nn/functional.h" +#include "infini_train/include/rng_tracker.h" +#include "infini_train/include/tensor.h" + +#include "tests/common/test_utils.h" + +using namespace infini_train; +using infini_train::test::TensorBytes; + +namespace { + +using Samples = std::vector>; + +Samples SampleScript(const Device &device, + std::optional generator = std::nullopt) { + auto input = std::make_shared(std::vector{33}, + DataType::kFLOAT32, device); + input->Fill(1.0f); + return { + TensorBytes( + nn::function::Rand({33}, DataType::kFLOAT32, device, generator)), + TensorBytes( + nn::function::Randn({33}, DataType::kFLOAT32, device, generator)), + TensorBytes( + nn::function::Randn({33}, DataType::kFLOAT64, device, generator)), + TensorBytes(nn::function::Dropout(input, 0.25, true, generator)), + }; +} + +std::array InterleavedSamples(const Device &device, bool reverse) { + std::array results; + std::array errors; + std::barrier ready(3); + std::binary_semaphore first_gate(0); + std::binary_semaphore second_gate(0); + std::array gates{&first_gate, &second_gate}; + std::binary_semaphore finished(0); + + auto worker = [&](int index) { + std::unique_ptr scope; + try { + scope = std::make_unique(9100 + index); + } catch (...) { + errors[index] = std::current_exception(); + } + ready.arrive_and_wait(); + for (int step = 0; step < 3; ++step) { + gates[index]->acquire(); + if (!errors[index]) { + try { + results[index].push_back(TensorBytes( + nn::function::Rand({17}, DataType::kFLOAT32, device))); + } catch (...) { + errors[index] = std::current_exception(); + } + } + finished.release(); + } + }; + + std::thread first(worker, 0); + std::thread second(worker, 1); + ready.arrive_and_wait(); + for (int step = 0; step < 3; ++step) { + for (int order = 0; order < 2; ++order) { + gates[reverse ? 1 - order : order]->release(); + finished.acquire(); + } + } + first.join(); + second.join(); + for (const auto &error : errors) { + if (error) { + std::rethrow_exception(error); + } + } + return results; +} + +} // namespace + +using RngTrackerTest = infini_train::test::InfiniTrainTest; + +TEST_P(RngTrackerTest, IndependentTrackersRetainTheirOwnSequences) { + const Device device = GetDevice(); + RngTracker first(9001); + RngTracker second(9001); + auto reference = CreateGenerator(device, 9001); + const auto expected_first = SampleScript(device, reference); + const auto expected_second = SampleScript(device, reference); + std::optional first_generator; + { + RngScope scope(first); + first_generator = GetDefaultGenerator(device); + EXPECT_EQ(expected_first, SampleScript(device)); + EXPECT_EQ(expected_second, SampleScript(device)); + } + { + RngScope scope(second); + EXPECT_NE(*first_generator, GetDefaultGenerator(device)); + EXPECT_EQ(expected_first, SampleScript(device)); + } + { + RngScope scope(second); + EXPECT_EQ(expected_second, SampleScript(device)); + } +} + +TEST_P(RngTrackerTest, OwningScopesStartFreshAndLeaveGlobalStateUntouched) { + const Device device = GetDevice(); + ManualSeed(9002); + Generator global = GetDefaultGenerator(device); + const auto before = TensorBytes(global.get_state()); + const auto expected = SampleScript(device, CreateGenerator(device)); + for (int attempt = 0; attempt < 2; ++attempt) { + RngScope scope; + EXPECT_NE(global, GetDefaultGenerator(device)); + EXPECT_EQ(expected, SampleScript(device)); + ManualSeed(9003); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9003)), + SampleScript(device)); + } + EXPECT_EQ(global, GetDefaultGenerator(device)); + EXPECT_EQ(before, TensorBytes(global.get_state())); + { + RngScope scope(9004); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9004)), + SampleScript(device)); + } + EXPECT_EQ(before, TensorBytes(global.get_state())); +} + +TEST_P(RngTrackerTest, NestedScopesRestoreSelectionAfterExceptions) { + const Device device = GetDevice(); + RngTracker training(9005); + RngTracker evaluation(9006); + auto reference = CreateGenerator(device, 9005); + RngScope outer(training); + Generator outer_generator = GetDefaultGenerator(device); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); + EXPECT_THROW( + { + RngScope inner(evaluation); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9006)), + SampleScript(device)); + throw std::runtime_error("leave evaluation"); + }, + std::runtime_error); + EXPECT_EQ(outer_generator, GetDefaultGenerator(device)); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); +} + +TEST_P(RngTrackerTest, ManualSeedAffectsOnlyTheSelectedStream) { + const Device device = GetDevice(); + RngTracker tracker(9007); + tracker.Add("evaluation", 9008); + RngScope outer(tracker); + auto reference = CreateGenerator(device, 9007); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); + const auto default_state = + TensorBytes(GetDefaultGenerator(device).get_state()); + { + RngScope inner(tracker, "evaluation"); + SampleScript(device); + ManualSeed(9009); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9009U); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9009)), + SampleScript(device)); + ManualSeed(9009); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9009)), + SampleScript(device)); + } + EXPECT_EQ(default_state, + TensorBytes(GetDefaultGenerator(device).get_state())); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); + + tracker.ManualSeed("evaluation", 9010); + { + RngScope inner(tracker, "evaluation"); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9010)), + SampleScript(device)); + } + tracker.ManualSeed(9011); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9011)), + SampleScript(device)); +} + +TEST_P(RngTrackerTest, SeedAndSnapshotsCoverLazilyCreatedDevices) { + const Device device = GetDevice(); + RngTracker tracker(9012); + RngScope scope(tracker); + ManualSeed(9013); + const auto snapshot = tracker.GetState(); + const auto expected = SampleScript(device, CreateGenerator(device, 9013)); + EXPECT_EQ(expected, SampleScript(device)); + ManualSeed(9014); + SampleScript(device); + tracker.SetState(*snapshot); + EXPECT_EQ(expected, SampleScript(device)); + tracker.SetState(*snapshot); + EXPECT_EQ(expected, SampleScript(device)); +} + +TEST_P(RngTrackerTest, CopiedStateRestoresAllStreamsAfterTrackerDestruction) { + const Device device = GetDevice(); + const std::string evaluation_name = "evaluation"; + std::shared_ptr exported; + Samples expected_default; + Samples expected_evaluation; + { + RngTracker original(9030); + original.Add(evaluation_name, 9031); + original.Add("unused", 9032); + RngScope outer(original); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9030U); + SampleScript(device); + { + RngScope inner(original, evaluation_name); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9031U); + SampleScript(device); + } + exported = original.GetState(); + expected_default = SampleScript(device); + RngScope inner(original, evaluation_name); + expected_evaluation = SampleScript(device); + } + ASSERT_NE(exported, nullptr); + EXPECT_TRUE(exported->GetDevice().IsCPU()); + EXPECT_EQ(exported->Dtype(), DataType::kUINT8); + ASSERT_EQ(exported->Dims().size(), 1U); + EXPECT_GT(exported->NumElements(), 0U); + + auto loaded = + std::make_shared(exported->Dims(), exported->Dtype(), Device()); + loaded->CopyFrom(*exported); + EXPECT_EQ(TensorBytes(loaded), TensorBytes(exported)); + + RngTracker restored(0); + restored.SetState(*loaded); + EXPECT_EQ(TensorBytes(restored.GetState()), TensorBytes(exported)); + loaded->Fill(0); + EXPECT_TRUE(restored.Contains(evaluation_name)); + EXPECT_TRUE(restored.Contains("unused")); + RngScope outer(restored); + EXPECT_EQ(expected_default, SampleScript(device)); + { + RngScope inner(restored, evaluation_name); + EXPECT_EQ(expected_evaluation, SampleScript(device)); + } + { + RngScope inner(restored, "unused"); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9032U); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9032)), + SampleScript(device)); + } + for (int attempt = 0; attempt < 2; ++attempt) { + restored.SetState(*exported); + EXPECT_EQ(expected_default, SampleScript(device)); + RngScope inner(restored, evaluation_name); + EXPECT_EQ(expected_evaluation, SampleScript(device)); + } +} + +TEST_P(RngTrackerTest, InvalidSerializedStatePreservesLiveGenerators) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device device = GetDevice(); + RngTracker tracker(9033); + RngScope scope(tracker); + SampleScript(device); + const Generator selected = GetDefaultGenerator(device); + const auto before = TensorBytes(selected.get_state()); + const auto valid = tracker.GetState(); + const auto bytes = TensorBytes(valid); + auto reference = selected.Clone(); + auto expect_rejected = [&](const Tensor &state, const char *error) { + EXPECT_DEATH(tracker.SetState(state), error); + EXPECT_EQ(selected, GetDefaultGenerator(device)); + EXPECT_EQ(before, TensorBytes(selected.get_state())); + EXPECT_EQ(bytes, TensorBytes(tracker.GetState())); + }; + expect_rejected( + Tensor({1}, DataType::kFLOAT32, Device()), + "RNG tracker state must be a nonempty one-dimensional CPU UINT8 tensor"); + expect_rejected( + Tensor(*valid, 0, {static_cast(valid->NumElements() - 1)}), + "Invalid RNG generator state length"); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); +} + +TEST_P(RngTrackerTest, InvalidStreamOperationsPreserveTheBinding) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device device = GetDevice(); + RngTracker tracker(9017); + EXPECT_TRUE(tracker.Contains("default")); + EXPECT_FALSE(tracker.Contains("missing")); + EXPECT_DEATH(tracker.Add("", 1), + "RNG stream name must be nonempty and unique"); + EXPECT_DEATH(tracker.Add("default", 1), + "RNG stream name must be nonempty and unique"); + EXPECT_DEATH(tracker.ManualSeed("missing", 1), "Unknown RNG stream: missing"); + const auto snapshot = tracker.GetState(); + tracker.Add("later", 9018); + EXPECT_TRUE(tracker.Contains("later")); + RngScope outer(tracker); + Generator selected = GetDefaultGenerator(device); + EXPECT_DEATH(RngScope missing(tracker, "missing"), + "Unknown RNG stream: missing"); + EXPECT_EQ(selected, GetDefaultGenerator(device)); + { + RngScope inner(tracker, "later"); + EXPECT_DEATH(tracker.SetState(*snapshot), + "Cannot restore a state that removes an active RNG stream"); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9018U); + RngScope other(9019); + EXPECT_DEATH(tracker.SetState(*snapshot), + "Cannot restore a state that removes an active RNG stream"); + } + tracker.SetState(*snapshot); + EXPECT_FALSE(tracker.Contains("later")); + EXPECT_EQ(GetDefaultGenerator(device).current_seed(), 9017U); +} + +TEST_P(RngTrackerTest, ExplicitGeneratorTakesPriorityOverTheScope) { + const Device device = GetDevice(); + RngScope scope(9019); + const auto before = TensorBytes(GetDefaultGenerator(device).get_state()); + auto explicit_generator = CreateGenerator(device, 9020); + EXPECT_NE(explicit_generator, GetDefaultGenerator(device)); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9020)), + SampleScript(device, explicit_generator)); + EXPECT_EQ(before, TensorBytes(GetDefaultGenerator(device).get_state())); + const auto explicit_state = TensorBytes(explicit_generator.get_state()); + ManualSeed(9022); + EXPECT_EQ(explicit_state, TensorBytes(explicit_generator.get_state())); + EXPECT_EQ(SampleScript(device, CreateGenerator(device, 9022)), + SampleScript(device)); +} + +TEST_P(RngTrackerTest, ChildThreadsDoNotInheritTheParentScope) { + const Device device = GetDevice(); + const Generator global = GetDefaultGenerator(device); + RngScope parent(9023); + const Generator scoped = GetDefaultGenerator(device); + EXPECT_NE(global, scoped); + auto child = std::async(std::launch::async, + [&] { return GetDefaultGenerator(device); }); + EXPECT_EQ(global, child.get()); + EXPECT_EQ(scoped, GetDefaultGenerator(device)); +} + +TEST_P(RngTrackerTest, ReversingThreadInterleavingPreservesEachSequence) { + const Device device = GetDevice(); + const auto forward = InterleavedSamples(device, false); + const auto reverse = InterleavedSamples(device, true); + EXPECT_EQ(forward, reverse); + for (int index = 0; index < 2; ++index) { + auto reference = CreateGenerator(device, 9100 + index); + for (int step = 0; step < 3; ++step) { + EXPECT_EQ(forward[index][step], + TensorBytes(nn::function::Rand({17}, DataType::kFLOAT32, device, + reference))); + } + } +} + +TEST_P(RngTrackerTest, IndependentTrackersCanSampleConcurrently) { + const Device device = GetDevice(); + std::barrier ready(2); + auto worker = [&](uint64_t seed) { + RngScope scope(seed); + ready.arrive_and_wait(); + return SampleScript(device); + }; + auto first = std::async(std::launch::async, worker, 9120); + auto second = std::async(std::launch::async, worker, 9121); + const auto first_samples = first.get(); + const auto second_samples = second.get(); + EXPECT_EQ(first_samples, SampleScript(device, CreateGenerator(device, 9120))); + EXPECT_EQ(second_samples, + SampleScript(device, CreateGenerator(device, 9121))); +} + +TEST_P(RngTrackerTest, + TrackerRejectsConcurrentBindingAndCanMoveBetweenThreads) { + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + const Device device = GetDevice(); + RngTracker tracker(9021); + auto reference = CreateGenerator(device, 9021); + { + RngScope outer(tracker); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); + { + RngScope inner(tracker); + EXPECT_EQ(SampleScript(device, reference), SampleScript(device)); + } + } + { + std::binary_semaphore held(0); + std::binary_semaphore release(0); + std::thread holder([&] { + RngScope scope(tracker); + held.release(); + release.acquire(); + }); + held.acquire(); + EXPECT_DEATH(RngScope conflicting(tracker), + "RNG tracker is in use on another thread"); + release.release(); + holder.join(); + } + { + std::binary_semaphore held(0); + std::binary_semaphore release(0); + std::thread holder([&] { + RngScope scope(tracker); + held.release(); + release.acquire(); + }); + held.acquire(); + EXPECT_DEATH(tracker.ManualSeed(0), + "RNG tracker is in use on another thread"); + release.release(); + holder.join(); + } + auto next_thread = std::async(std::launch::async, [&] { + RngScope scope(tracker); + return SampleScript(device); + }); + EXPECT_EQ(SampleScript(device, reference), next_thread.get()); +} + +INFINI_TRAIN_REGISTER_TEST(RngTrackerTest);