From 51388682bbc20bcc2983ec12dfab51e371bdc090 Mon Sep 17 00:00:00 2001 From: zhoujf1 <2047484072@qq.com> Date: Wed, 4 Nov 2026 11:50:01 +0800 Subject: [PATCH 1/4] feat: add ATen operator implementations --- src/base/accumulate_grad.h | 45 ++++ src/base/adam_accumulate_grad.h | 74 +++++++ src/base/add_backward.h | 60 ++++++ src/base/cross_entropy.h | 69 ++++++ src/base/cross_entropy_backward.h | 88 ++++++++ src/base/embedding_backward.h | 71 +++++++ src/base/masked_fill_backward.h | 71 +++++++ src/base/mean_backward.h | 70 +++++++ src/base/mul_backward.h | 77 +++++++ src/base/pow_backward.h | 56 +++++ src/base/repeat_interleave.h | 56 +++++ src/base/repeat_interleave_backward.h | 63 ++++++ src/base/rsqrt_backward.h | 50 +++++ src/base/slice.h | 60 ++++++ src/base/slice_backward.h | 63 ++++++ src/base/split.h | 76 +++++++ src/base/stack_backward.h | 69 ++++++ src/base/sub_backward.h | 60 ++++++ src/base/transpose.h | 59 ++++++ src/base/transpose_backward.h | 66 ++++++ .../vocab_parallel_cross_entropy_backward.h | 80 +++++++ .../ops/accumulate_grad/accumulate_grad.cc | 40 ++++ .../ops/accumulate_grad/accumulate_grad.h | 22 ++ .../adam_accumulate_grad.cc | 53 +++++ .../adam_accumulate_grad.h | 26 +++ src/torch/ops/add_backward/add_backward.cc | 74 +++++++ src/torch/ops/add_backward/add_backward.h | 22 ++ src/torch/ops/cross_entropy/cross_entropy.cc | 41 ++++ src/torch/ops/cross_entropy/cross_entropy.h | 22 ++ .../cross_entropy_backward.cc | 67 ++++++ .../cross_entropy_backward.h | 21 ++ .../embedding_backward/embedding_backward.cc | 54 +++++ .../embedding_backward/embedding_backward.h | 26 +++ .../masked_fill_backward.cc | 48 +++++ .../masked_fill_backward.h | 23 ++ src/torch/ops/mean_backward/mean_backward.cc | 51 +++++ src/torch/ops/mean_backward/mean_backward.h | 24 +++ src/torch/ops/mul_backward/mul_backward.cc | 83 ++++++++ src/torch/ops/mul_backward/mul_backward.h | 23 ++ src/torch/ops/pow_backward/pow_backward.cc | 49 +++++ src/torch/ops/pow_backward/pow_backward.h | 23 ++ .../repeat_interleave/repeat_interleave.cc | 38 ++++ .../ops/repeat_interleave/repeat_interleave.h | 24 +++ .../repeat_interleave_backward.cc | 45 ++++ .../repeat_interleave_backward.h | 24 +++ .../ops/rsqrt_backward/rsqrt_backward.cc | 41 ++++ src/torch/ops/rsqrt_backward/rsqrt_backward.h | 22 ++ src/torch/ops/slice/slice.cc | 46 ++++ src/torch/ops/slice/slice.h | 25 +++ .../ops/slice_backward/slice_backward.cc | 52 +++++ src/torch/ops/slice_backward/slice_backward.h | 25 +++ src/torch/ops/split/split.cc | 43 ++++ src/torch/ops/split/split.h | 23 ++ .../ops/stack_backward/stack_backward.cc | 45 ++++ src/torch/ops/stack_backward/stack_backward.h | 23 ++ src/torch/ops/sub_backward/sub_backward.cc | 75 +++++++ src/torch/ops/sub_backward/sub_backward.h | 22 ++ src/torch/ops/transpose/transpose.cc | 36 ++++ src/torch/ops/transpose/transpose.h | 23 ++ .../transpose_backward/transpose_backward.cc | 39 ++++ .../transpose_backward/transpose_backward.h | 23 ++ .../vocab_parallel_cross_entropy_backward.cc | 113 ++++++++++ .../vocab_parallel_cross_entropy_backward.h | 28 +++ tests/test_accumulate_grad.py | 87 ++++++++ tests/test_adam_accumulate_grad.py | 139 ++++++++++++ tests/test_add_backward.py | 87 ++++++++ tests/test_cross_entropy.py | 113 ++++++++++ tests/test_cross_entropy_backward.py | 130 ++++++++++++ tests/test_embedding_backward.py | 177 ++++++++++++++++ tests/test_masked_fill_backward.py | 147 +++++++++++++ tests/test_mean_backward.py | 158 ++++++++++++++ tests/test_mul_backward.py | 135 ++++++++++++ tests/test_pow_backward.py | 180 ++++++++++++++++ tests/test_repeat_interleave.py | 131 ++++++++++++ tests/test_repeat_interleave_backward.py | 143 +++++++++++++ tests/test_rsqrt_backward.py | 120 +++++++++++ tests/test_slice.py | 138 ++++++++++++ tests/test_slice_backward.py | 158 ++++++++++++++ tests/test_split.py | 119 +++++++++++ tests/test_stack_backward.py | 136 ++++++++++++ tests/test_sub_backward.py | 87 ++++++++ tests/test_transpose.py | 128 ++++++++++++ tests/test_transpose_backward.py | 141 +++++++++++++ ...t_vocab_parallel_cross_entropy_backward.py | 197 ++++++++++++++++++ 84 files changed, 5861 insertions(+) create mode 100644 src/base/accumulate_grad.h create mode 100644 src/base/adam_accumulate_grad.h create mode 100644 src/base/add_backward.h create mode 100644 src/base/cross_entropy.h create mode 100644 src/base/cross_entropy_backward.h create mode 100644 src/base/embedding_backward.h create mode 100644 src/base/masked_fill_backward.h create mode 100644 src/base/mean_backward.h create mode 100644 src/base/mul_backward.h create mode 100644 src/base/pow_backward.h create mode 100644 src/base/repeat_interleave.h create mode 100644 src/base/repeat_interleave_backward.h create mode 100644 src/base/rsqrt_backward.h create mode 100644 src/base/slice.h create mode 100644 src/base/slice_backward.h create mode 100644 src/base/split.h create mode 100644 src/base/stack_backward.h create mode 100644 src/base/sub_backward.h create mode 100644 src/base/transpose.h create mode 100644 src/base/transpose_backward.h create mode 100644 src/base/vocab_parallel_cross_entropy_backward.h create mode 100644 src/torch/ops/accumulate_grad/accumulate_grad.cc create mode 100644 src/torch/ops/accumulate_grad/accumulate_grad.h create mode 100644 src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc create mode 100644 src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h create mode 100644 src/torch/ops/add_backward/add_backward.cc create mode 100644 src/torch/ops/add_backward/add_backward.h create mode 100644 src/torch/ops/cross_entropy/cross_entropy.cc create mode 100644 src/torch/ops/cross_entropy/cross_entropy.h create mode 100644 src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc create mode 100644 src/torch/ops/cross_entropy_backward/cross_entropy_backward.h create mode 100644 src/torch/ops/embedding_backward/embedding_backward.cc create mode 100644 src/torch/ops/embedding_backward/embedding_backward.h create mode 100644 src/torch/ops/masked_fill_backward/masked_fill_backward.cc create mode 100644 src/torch/ops/masked_fill_backward/masked_fill_backward.h create mode 100644 src/torch/ops/mean_backward/mean_backward.cc create mode 100644 src/torch/ops/mean_backward/mean_backward.h create mode 100644 src/torch/ops/mul_backward/mul_backward.cc create mode 100644 src/torch/ops/mul_backward/mul_backward.h create mode 100644 src/torch/ops/pow_backward/pow_backward.cc create mode 100644 src/torch/ops/pow_backward/pow_backward.h create mode 100644 src/torch/ops/repeat_interleave/repeat_interleave.cc create mode 100644 src/torch/ops/repeat_interleave/repeat_interleave.h create mode 100644 src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc create mode 100644 src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h create mode 100644 src/torch/ops/rsqrt_backward/rsqrt_backward.cc create mode 100644 src/torch/ops/rsqrt_backward/rsqrt_backward.h create mode 100644 src/torch/ops/slice/slice.cc create mode 100644 src/torch/ops/slice/slice.h create mode 100644 src/torch/ops/slice_backward/slice_backward.cc create mode 100644 src/torch/ops/slice_backward/slice_backward.h create mode 100644 src/torch/ops/split/split.cc create mode 100644 src/torch/ops/split/split.h create mode 100644 src/torch/ops/stack_backward/stack_backward.cc create mode 100644 src/torch/ops/stack_backward/stack_backward.h create mode 100644 src/torch/ops/sub_backward/sub_backward.cc create mode 100644 src/torch/ops/sub_backward/sub_backward.h create mode 100644 src/torch/ops/transpose/transpose.cc create mode 100644 src/torch/ops/transpose/transpose.h create mode 100644 src/torch/ops/transpose_backward/transpose_backward.cc create mode 100644 src/torch/ops/transpose_backward/transpose_backward.h create mode 100644 src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc create mode 100644 src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h create mode 100644 tests/test_accumulate_grad.py create mode 100644 tests/test_adam_accumulate_grad.py create mode 100644 tests/test_add_backward.py create mode 100644 tests/test_cross_entropy.py create mode 100644 tests/test_cross_entropy_backward.py create mode 100644 tests/test_embedding_backward.py create mode 100644 tests/test_masked_fill_backward.py create mode 100644 tests/test_mean_backward.py create mode 100644 tests/test_mul_backward.py create mode 100644 tests/test_pow_backward.py create mode 100644 tests/test_repeat_interleave.py create mode 100644 tests/test_repeat_interleave_backward.py create mode 100644 tests/test_rsqrt_backward.py create mode 100644 tests/test_slice.py create mode 100644 tests/test_slice_backward.py create mode 100644 tests/test_split.py create mode 100644 tests/test_stack_backward.py create mode 100644 tests/test_sub_backward.py create mode 100644 tests/test_transpose.py create mode 100644 tests/test_transpose_backward.py create mode 100644 tests/test_vocab_parallel_cross_entropy_backward.py diff --git a/src/base/accumulate_grad.h b/src/base/accumulate_grad.h new file mode 100644 index 000000000..143da3cb7 --- /dev/null +++ b/src/base/accumulate_grad.h @@ -0,0 +1,45 @@ +#ifndef INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class AccumulateGrad : public Operator { + public: + AccumulateGrad(const Tensor gradient, const double rate, Tensor tensor) + : gradient_shape_{gradient.shape()}, + gradient_strides_{gradient.strides()}, + gradient_type_{gradient.dtype()}, + tensor_shape_{tensor.shape()}, + tensor_strides_{tensor.strides()}, + tensor_type_{tensor.dtype()}, + rate_{rate}, + device_index_{tensor.device().index()} { + assert(gradient.shape() == tensor.shape() && + "`AccumulateGrad` requires `gradient` and `tensor` to share shape"); + assert(gradient_type_ == tensor_type_ && + "`AccumulateGrad` requires `gradient` and `tensor` to share dtype"); + assert(gradient.device() == tensor.device() && + "`AccumulateGrad` requires `gradient` and `tensor` on the same device"); + } + + virtual void operator()(const Tensor gradient, const double rate, + Tensor tensor) const = 0; + + protected: + Tensor::Shape gradient_shape_; + Tensor::Strides gradient_strides_; + DataType gradient_type_; + Tensor::Shape tensor_shape_; + Tensor::Strides tensor_strides_; + DataType tensor_type_; + double rate_{1.0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ \ No newline at end of file diff --git a/src/base/adam_accumulate_grad.h b/src/base/adam_accumulate_grad.h new file mode 100644 index 000000000..ab9b79ce5 --- /dev/null +++ b/src/base/adam_accumulate_grad.h @@ -0,0 +1,74 @@ +#ifndef INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class AdamAccumulateGrad : public Operator { + public: + AdamAccumulateGrad(const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step) + : grad_shape_{grad.shape()}, + grad_strides_{grad.strides()}, + grad_type_{grad.dtype()}, + param_shape_{param.shape()}, + param_strides_{param.strides()}, + param_type_{param.dtype()}, + m_shape_{m.shape()}, + m_strides_{m.strides()}, + m_type_{m.dtype()}, + v_shape_{v.shape()}, + v_strides_{v.strides()}, + v_type_{v.dtype()}, + learning_rate_{learning_rate}, + beta1_{beta1}, + beta2_{beta2}, + eps_{eps}, + step_{step}, + device_index_{param.device().index()} { + assert(grad.shape() == param.shape() && param.shape() == m.shape() && + m.shape() == v.shape() && + "`AdamAccumulateGrad` requires grad/param/m/v to share shape"); + assert(grad_type_ == param_type_ && param_type_ == m_type_ && + m_type_ == v_type_ && "`AdamAccumulateGrad` requires matching dtypes"); + assert(grad.device() == param.device() && + "`AdamAccumulateGrad` requires grad/param on the same device"); + assert(0.0 < beta1_ && beta1_ < 1.0 && 0.0 < beta2_ && beta2_ < 1.0 && + "`AdamAccumulateGrad` requires beta1/beta2 in (0, 1)"); + assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1"); + } + + virtual void operator()(const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, + const double beta2, const double eps, + const int64_t step) const = 0; + + protected: + Tensor::Shape grad_shape_; + Tensor::Strides grad_strides_; + DataType grad_type_; + Tensor::Shape param_shape_; + Tensor::Strides param_strides_; + DataType param_type_; + Tensor::Shape m_shape_; + Tensor::Strides m_strides_; + DataType m_type_; + Tensor::Shape v_shape_; + Tensor::Strides v_strides_; + DataType v_type_; + double learning_rate_{0.0}; + double beta1_{0.9}; + double beta2_{0.999}; + double eps_{1e-8}; + int64_t step_{1}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ \ No newline at end of file diff --git a/src/base/add_backward.h b/src/base/add_backward.h new file mode 100644 index 000000000..929cb936d --- /dev/null +++ b/src/base/add_backward.h @@ -0,0 +1,60 @@ +#ifndef INFINI_OPS_BASE_ADD_BACKWARD_H_ +#define INFINI_OPS_BASE_ADD_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class AddBackward : public Operator { + public: + AddBackward(const Tensor grad_output, Tensor grad_a, Tensor grad_b) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_a_shape_{grad_a.shape()}, + grad_a_strides_{grad_a.strides()}, + grad_a_type_{grad_a.dtype()}, + grad_b_shape_{grad_b.shape()}, + grad_b_strides_{grad_b.strides()}, + grad_b_type_{grad_b.dtype()}, + device_index_{grad_a.device().index()} { + assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && + "`AddBackward` requires matching dtypes"); + assert(grad_output.device() == grad_a.device() && + grad_a.device() == grad_b.device() && + "`AddBackward` requires grad_output/grad_a/grad_b on the same device"); + ValidateBroadcast(grad_a, grad_output); + ValidateBroadcast(grad_b, grad_output); + } + + virtual void operator()(const Tensor grad_output, Tensor grad_a, + Tensor grad_b) const = 0; + + protected: + static void ValidateBroadcast(const Tensor input, const Tensor out) { + assert(input.ndim() <= out.ndim() && + "`AddBackward` input rank must not exceed grad_output rank"); + const auto offset = out.ndim() - input.ndim(); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`AddBackward` input must broadcast (right-aligned) to grad_output"); + } + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape grad_a_shape_; + Tensor::Strides grad_a_strides_; + DataType grad_a_type_; + Tensor::Shape grad_b_shape_; + Tensor::Strides grad_b_strides_; + DataType grad_b_type_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/cross_entropy.h b/src/base/cross_entropy.h new file mode 100644 index 000000000..7bec3794c --- /dev/null +++ b/src/base/cross_entropy.h @@ -0,0 +1,69 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class CrossEntropy : public Operator { + public: + // This operator follows ATen's default CrossEntropyLoss contract: + // dimension 1 stores the class logits and -100 is the ignored target. + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropy(const Tensor input, const Tensor target, Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + device_index_{out.device().index()} { + assert(input.ndim() >= 2 && "`CrossEntropy` requires input rank >= 2"); + assert(input.device() == target.device() && + target.device() == out.device() && + "`CrossEntropy` requires all tensors on the same device"); + assert((input_type_ == DataType::kFloat16 || + input_type_ == DataType::kBFloat16 || + input_type_ == DataType::kFloat32 || + input_type_ == DataType::kFloat64) && + "`CrossEntropy` requires floating-point logits"); + assert((target_type_ == DataType::kInt64 || target_type_ == DataType::kInt32 || + target_type_ == DataType::kUInt8) && + "`CrossEntropy` supports int64/int32/uint8 targets"); + assert(target.ndim() == input.ndim() - 1 && + "`CrossEntropy` target rank must be input rank - 1"); + for (Tensor::Size i = 0; i < target.ndim(); ++i) { + const auto input_dim = i == 0 ? 0 : i + 1; + assert(target.size(i) == input.size(input_dim) && + "`CrossEntropy` target shape mismatch"); + } + assert(out.ndim() == 0 && "`CrossEntropy` output must be a 0-dim scalar"); + assert(out_type_ == input_type_ && "`CrossEntropy` output dtype must match input"); + } + + virtual void operator()(const Tensor input, const Tensor target, + Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape target_shape_; + Tensor::Strides target_strides_; + DataType target_type_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType out_type_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_H_ diff --git a/src/base/cross_entropy_backward.h b/src/base/cross_entropy_backward.h new file mode 100644 index 000000000..87cc8b222 --- /dev/null +++ b/src/base/cross_entropy_backward.h @@ -0,0 +1,88 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class CrossEntropyBackward : public Operator { + public: + // Keep this synchronized with CrossEntropy's forward contract. + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropyBackward(const Tensor input, const Tensor target, + const Tensor grad_output, Tensor grad_input) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + assert(input.ndim() >= 2 && + "`CrossEntropyBackward` requires input rank >= 2"); + assert(input.device() == target.device() && + target.device() == grad_output.device() && + grad_output.device() == grad_input.device() && + "`CrossEntropyBackward` requires all tensors on the same device"); + assert((input_type_ == DataType::kFloat16 || + input_type_ == DataType::kBFloat16 || + input_type_ == DataType::kFloat32 || + input_type_ == DataType::kFloat64) && + "`CrossEntropyBackward` requires floating-point logits"); + assert((target_type_ == DataType::kInt64 || + target_type_ == DataType::kInt32 || + target_type_ == DataType::kUInt8) && + "`CrossEntropyBackward` supports int64/int32/uint8 targets"); + assert(target.ndim() == input.ndim() - 1 && + "`CrossEntropyBackward` target rank must be input rank - 1"); + for (Tensor::Size i = 0; i < target.ndim(); ++i) { + const auto input_dim = i == 0 ? 0 : i + 1; + assert(target.size(i) == input.size(input_dim) && + "`CrossEntropyBackward` target shape mismatch"); + } + assert(grad_input.shape() == input.shape() && + "`CrossEntropyBackward` grad_input shape must match input"); + assert(grad_output.ndim() == 0 && + "`CrossEntropyBackward` requires scalar grad_output"); + assert(grad_output_type_ == input_type_ && + grad_input_type_ == input_type_ && + "`CrossEntropyBackward` requires matching floating-point dtypes"); + } + + virtual void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + + Tensor::Shape target_shape_; + Tensor::Strides target_strides_; + DataType target_type_; + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/base/embedding_backward.h b/src/base/embedding_backward.h new file mode 100644 index 000000000..43fcef1d0 --- /dev/null +++ b/src/base/embedding_backward.h @@ -0,0 +1,71 @@ +#ifndef INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ +#define INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class EmbeddingBackward : public Operator { + public: + EmbeddingBackward(const Tensor grad_output, const Tensor indices, + const int64_t num_weights, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + indices_shape_{indices.shape()}, + indices_strides_{indices.strides()}, + indices_type_{indices.dtype()}, + grad_weight_shape_{grad_weight.shape()}, + grad_weight_strides_{grad_weight.strides()}, + grad_weight_type_{grad_weight.dtype()}, + num_weights_{num_weights}, + padding_idx_{padding_idx}, + scale_grad_by_freq_{scale_grad_by_freq}, + sparse_{sparse}, + device_index_{grad_weight.device().index()} { + assert((indices_type_ == DataType::kInt32 || indices_type_ == DataType::kInt64) && + "`EmbeddingBackward` requires int32 or int64 `indices`"); + assert(grad_output_type_ == grad_weight_type_ && + "`EmbeddingBackward` requires `grad_output` and `grad_weight` to have the same dtype"); + assert(grad_weight.ndim() == 2 && "`EmbeddingBackward` requires 2D `grad_weight`"); + assert(grad_weight.size(0) == num_weights_ && + "`EmbeddingBackward` requires `grad_weight` row count to equal `num_weights`"); + assert(grad_output.ndim() == indices.ndim() + 1 && + "`EmbeddingBackward` requires `grad_output` rank to be `indices` rank + 1"); + for (Tensor::Size i = 0; i < indices.ndim(); ++i) { + assert(grad_output.size(i) == indices.size(i) && + "`EmbeddingBackward` requires matching leading dims"); + } + assert(grad_output.size(-1) == grad_weight.size(1) && + "`EmbeddingBackward` requires last dim of `grad_output` to equal embedding dim"); + } + + virtual void operator()(const Tensor grad_output, const Tensor indices, + const int64_t num_weights, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape indices_shape_; + Tensor::Strides indices_strides_; + DataType indices_type_; + Tensor::Shape grad_weight_shape_; + Tensor::Strides grad_weight_strides_; + DataType grad_weight_type_; + int64_t num_weights_{0}; + int64_t padding_idx_{-1}; + bool scale_grad_by_freq_{false}; + bool sparse_{false}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/masked_fill_backward.h b/src/base/masked_fill_backward.h new file mode 100644 index 000000000..1b7e8f2f4 --- /dev/null +++ b/src/base/masked_fill_backward.h @@ -0,0 +1,71 @@ +#ifndef INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ +#define INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class MaskedFillBackward : public Operator { + public: + MaskedFillBackward(const Tensor grad_output, const Tensor mask, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + mask_shape_{mask.shape()}, + mask_strides_{mask.strides()}, + mask_type_{mask.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + assert(grad_output.shape() == grad_input.shape() && + "`MaskedFillBackward` requires grad_output/grad_input to share shape"); + assert(grad_output_type_ == grad_input_type_ && + "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); + assert(mask_type_ == DataType::kBool && + "`MaskedFillBackward` requires a bool `mask`"); + assert(mask.ndim() <= grad_output.ndim() && + "`MaskedFillBackward` mask rank must not exceed tensor rank"); + + tail_aligned_ = AlignsAt(mask, grad_output, /*tail=*/true); + if (!tail_aligned_) { + assert(AlignsAt(mask, grad_output, /*tail=*/false) && + "`MaskedFillBackward` mask must align to leading or trailing dims"); + } + } + + virtual void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const = 0; + + protected: + static bool AlignsAt(const Tensor mask, const Tensor tensor, const bool tail) { + const auto offset = tail ? tensor.ndim() - mask.ndim() : 0; + for (Tensor::Size i = 0; i < mask.ndim(); ++i) { + const auto dim = tensor.size(i + offset); + if (mask.size(i) != 1 && mask.size(i) != dim) { + return false; + } + } + return true; + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape mask_shape_; + Tensor::Strides mask_strides_; + DataType mask_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + bool tail_aligned_{true}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/mean_backward.h b/src/base/mean_backward.h new file mode 100644 index 000000000..06348c17e --- /dev/null +++ b/src/base/mean_backward.h @@ -0,0 +1,70 @@ +#ifndef INFINI_OPS_BASE_MEAN_BACKWARD_H_ +#define INFINI_OPS_BASE_MEAN_BACKWARD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class MeanBackward : public Operator { + public: + MeanBackward(const Tensor grad_output, const int64_t dim, const bool keep_dim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{NormalizeDim(dim, grad_input.ndim())}, + keep_dim_{keep_dim}, + device_index_{grad_input.device().index()} { + assert(grad_output_type_ == grad_input_type_ && + "`MeanBackward` requires matching dtypes"); + assert(grad_input.ndim() >= 1 && "`MeanBackward` requires ndim >= 1"); + if (keep_dim_) { + assert(grad_output.ndim() == grad_input.ndim() && + grad_output.size(dim_) == 1 && + "`MeanBackward` keep_dim=true requires size-1 dim in grad_output"); + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && "`MeanBackward` shape mismatch"); + } + } else { + assert(grad_output.ndim() == grad_input.ndim() - 1 && + "`MeanBackward` keep_dim=false requires rank-1 grad_output"); + for (Tensor::Size i = 0, s = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(s) == grad_input.size(i) && "`MeanBackward` shape mismatch"); + ++s; + } + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + const bool keep_dim, Tensor grad_input) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`MeanBackward` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + int64_t dim_{0}; + bool keep_dim_{false}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MEAN_BACKWARD_H_ diff --git a/src/base/mul_backward.h b/src/base/mul_backward.h new file mode 100644 index 000000000..b73f61a84 --- /dev/null +++ b/src/base/mul_backward.h @@ -0,0 +1,77 @@ +#ifndef INFINI_OPS_BASE_MUL_BACKWARD_H_ +#define INFINI_OPS_BASE_MUL_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class MulBackward : public Operator { + public: + MulBackward(const Tensor grad_output, const Tensor a, const Tensor b, + Tensor grad_a, Tensor grad_b) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + a_shape_{a.shape()}, + a_strides_{a.strides()}, + a_type_{a.dtype()}, + b_shape_{b.shape()}, + b_strides_{b.strides()}, + b_type_{b.dtype()}, + grad_a_shape_{grad_a.shape()}, + grad_a_strides_{grad_a.strides()}, + grad_a_type_{grad_a.dtype()}, + grad_b_shape_{grad_b.shape()}, + grad_b_strides_{grad_b.strides()}, + grad_b_type_{grad_b.dtype()}, + device_index_{grad_a.device().index()} { + assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && + "`MulBackward` requires matching grad dtypes"); + assert(a_type_ == grad_output_type_ && b_type_ == grad_output_type_ && + "`MulBackward` requires a/b dtype to match grad_output dtype"); + assert(grad_output.device() == a.device() && a.device() == b.device() && + "`MulBackward` requires all tensors on the same device"); + ValidateBroadcast(a, grad_output); + ValidateBroadcast(b, grad_output); + ValidateBroadcast(grad_a, grad_output); + ValidateBroadcast(grad_b, grad_output); + } + + virtual void operator()(const Tensor grad_output, const Tensor a, + const Tensor b, Tensor grad_a, + Tensor grad_b) const = 0; + + protected: + static void ValidateBroadcast(const Tensor input, const Tensor out) { + assert(input.ndim() <= out.ndim() && + "`MulBackward` input rank must not exceed grad_output rank"); + const auto offset = out.ndim() - input.ndim(); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`MulBackward` input must broadcast (right-aligned) to grad_output"); + } + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape a_shape_; + Tensor::Strides a_strides_; + DataType a_type_; + Tensor::Shape b_shape_; + Tensor::Strides b_strides_; + DataType b_type_; + Tensor::Shape grad_a_shape_; + Tensor::Strides grad_a_strides_; + DataType grad_a_type_; + Tensor::Shape grad_b_shape_; + Tensor::Strides grad_b_strides_; + DataType grad_b_type_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/pow_backward.h b/src/base/pow_backward.h new file mode 100644 index 000000000..b5f3172d6 --- /dev/null +++ b/src/base/pow_backward.h @@ -0,0 +1,56 @@ +#ifndef INFINI_OPS_BASE_POW_BACKWARD_H_ +#define INFINI_OPS_BASE_POW_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class PowBackward : public Operator { + public: + PowBackward(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + scalar_{scalar}, + scalar_is_base_{scalar_is_base}, + device_index_{grad_input.device().index()} { + assert(input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`PowBackward` requires input/grad_output/grad_input to share shape"); + assert(input_type_ == grad_output_type_ && grad_output_type_ == grad_input_type_ && + "`PowBackward` requires matching dtypes"); + assert(!scalar_is_base_ || scalar_ > 0.0 && + "`PowBackward` scalar base must be positive for log(scalar)"); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + const double scalar, const bool scalar_is_base, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + double scalar_{0.0}; + bool scalar_is_base_{false}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/repeat_interleave.h b/src/base/repeat_interleave.h new file mode 100644 index 000000000..a03624440 --- /dev/null +++ b/src/base/repeat_interleave.h @@ -0,0 +1,56 @@ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleave : public Operator { + public: + RepeatInterleave(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + repeats_{repeats}, + dim_{NormalizeDim(dim, input.ndim())}, + device_index_{out.device().index()} { + assert(repeats_ > 0 && "`RepeatInterleave` requires positive `repeats`"); + assert(input_type_ == out_type_ && "`RepeatInterleave` requires same dtype"); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + const auto expect = (i == dim_) ? input.size(i) * repeats_ : input.size(i); + assert(out.size(i) == expect && "`RepeatInterleave` output shape mismatch"); + } + } + + virtual void operator()(const Tensor input, const int64_t repeats, + const int64_t dim, Tensor out) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`RepeatInterleave` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType out_type_; + int64_t repeats_{0}; + int64_t dim_{0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ diff --git a/src/base/repeat_interleave_backward.h b/src/base/repeat_interleave_backward.h new file mode 100644 index 000000000..288284742 --- /dev/null +++ b/src/base/repeat_interleave_backward.h @@ -0,0 +1,63 @@ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleaveBackward : public Operator { + public: + RepeatInterleaveBackward(const Tensor grad_output, const int64_t dim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{NormalizeDim(dim, grad_input.ndim())}, + device_index_{grad_input.device().index()} { + assert(grad_output_type_ == grad_input_type_ && + "`RepeatInterleaveBackward` requires same dtype"); + assert(grad_output.ndim() == grad_input.ndim() && + "`RepeatInterleaveBackward` requires same rank"); + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && + "`RepeatInterleaveBackward` shape mismatch on non-repeated dims"); + } + assert(grad_input.size(dim_) > 0 && + "`RepeatInterleaveBackward` requires non-empty repeated dim"); + repeats_ = grad_output.size(dim_) / grad_input.size(dim_); + assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && + "`RepeatInterleaveBackward` grad_output dim must be divisible"); + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + Tensor grad_input) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`RepeatInterleaveBackward` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + int64_t repeats_{0}; + int64_t dim_{0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ diff --git a/src/base/rsqrt_backward.h b/src/base/rsqrt_backward.h new file mode 100644 index 000000000..d8e59c64e --- /dev/null +++ b/src/base/rsqrt_backward.h @@ -0,0 +1,50 @@ +#ifndef INFINI_OPS_BASE_RSQRT_BACKWARD_H_ +#define INFINI_OPS_BASE_RSQRT_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class RsqrtBackward : public Operator { + public: + RsqrtBackward(const Tensor grad_output, const Tensor input, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + assert(input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`RsqrtBackward` requires matching shapes"); + assert(input_type_ == grad_output_type_ && grad_output_type_ == grad_input_type_ && + "`RsqrtBackward` requires matching dtypes"); + assert(input_type_ != DataType::kInt32 && input_type_ != DataType::kInt64 && + "`RsqrtBackward` requires floating point dtypes"); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/slice.h b/src/base/slice.h new file mode 100644 index 000000000..1721fbb3e --- /dev/null +++ b/src/base/slice.h @@ -0,0 +1,60 @@ +#ifndef INFINI_OPS_BASE_SLICE_H_ +#define INFINI_OPS_BASE_SLICE_H_ + +#include +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class Slice : public Operator { + public: + Slice(const Tensor input, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + starts_{starts}, + ends_{ends}, + steps_{steps}, + device_index_{out.device().index()} { + assert(starts_.size() == input.ndim() && ends_.size() == input.ndim() && + steps_.size() == input.ndim() && + "`Slice` requires starts/ends/steps to match input rank"); + assert(out.ndim() == input.ndim() && "`Slice` requires matching rank"); + assert(input_type_ == out_type_ && "`Slice` requires same dtype"); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert(steps_[i] > 0 && "`Slice` requires positive steps (InfiniTrain semantics)"); + assert(starts_[i] <= ends_[i] && "`Slice` requires start <= end"); + const auto expected = (ends_[i] - starts_[i] + steps_[i] - 1) / steps_[i]; + assert(out.size(i) == expected && + "`Slice` output dim must equal ceil((end - start) / step)"); + } + } + + virtual void operator()(const Tensor input, const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType out_type_; + std::vector starts_; + std::vector ends_; + std::vector steps_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SLICE_H_ \ No newline at end of file diff --git a/src/base/slice_backward.h b/src/base/slice_backward.h new file mode 100644 index 000000000..540db763f --- /dev/null +++ b/src/base/slice_backward.h @@ -0,0 +1,63 @@ +#ifndef INFINI_OPS_BASE_SLICE_BACKWARD_H_ +#define INFINI_OPS_BASE_SLICE_BACKWARD_H_ + +#include +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class SliceBackward : public Operator { + public: + SliceBackward(const Tensor grad_output, const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + starts_{starts}, + ends_{ends}, + steps_{steps}, + device_index_{grad_input.device().index()} { + assert(starts_.size() == grad_input.ndim() && + "`SliceBackward` requires starts/ends/steps to match `grad_input` rank"); + assert(grad_output.ndim() == grad_input.ndim() && + "`SliceBackward` requires matching rank"); + assert(grad_output_type_ == grad_input_type_ && + "`SliceBackward` requires same dtype"); + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + assert(steps_[i] > 0 && "`SliceBackward` requires positive steps"); + assert(starts_[i] <= ends_[i] && "`SliceBackward` requires start <= end"); + const auto expected = (ends_[i] - starts_[i] + steps_[i] - 1) / steps_[i]; + assert(grad_output.size(i) == expected && + "`SliceBackward` grad_output dim must equal the slice dim"); + } + } + + virtual void operator()(const Tensor grad_output, + const std::vector starts, + const std::vector ends, + const std::vector steps, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + std::vector starts_; + std::vector ends_; + std::vector steps_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SLICE_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/split.h b/src/base/split.h new file mode 100644 index 000000000..7a2d6bdbb --- /dev/null +++ b/src/base/split.h @@ -0,0 +1,76 @@ +#ifndef INFINI_OPS_BASE_SPLIT_H_ +#define INFINI_OPS_BASE_SPLIT_H_ + +#include +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class Split : public Operator { + public: + Split(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + split_size_{split_size}, + dim_{NormalizeDim(dim, input.ndim())}, + outs_{std::move(outs)}, + device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { + assert(split_size_ > 0 && "`Split` requires positive `split_size`"); + assert(!outs_.empty() && "`Split` requires at least one output tensor"); + const auto expected = + (input.size(dim_) + split_size_ - 1) / split_size_; + assert(static_cast(outs_.size()) == expected && + "`Split` output count must equal ceil(input_size(dim) / split_size)"); + + for (std::size_t i = 0; i < outs_.size(); ++i) { + assert(outs_[i].ndim() == input.ndim() && "`Split` output rank mismatch"); + assert(outs_[i].dtype() == input_type_ && "`Split` dtype mismatch"); + for (Tensor::Size d = 0; d < input.ndim(); ++d) { + if (d != dim_) { + assert(outs_[i].size(d) == input.size(d) && "`Split` output shape mismatch"); + } + } + const auto expect_dim = std::min( + split_size_, + static_cast(input.size(dim_)) - + static_cast(i) * split_size_); + assert(outs_[i].size(dim_) == expect_dim && "`Split` output dim size mismatch"); + + out_shapes_.emplace_back(outs_[i].shape().begin(), outs_[i].shape().end()); + out_strides_.emplace_back(outs_[i].strides().begin(), outs_[i].strides().end()); + out_types_.emplace_back(outs_[i].dtype()); + } + } + + virtual void operator()(const Tensor input, const int64_t split_size, + const int64_t dim, + std::vector outs) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`Split` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + int64_t split_size_{0}; + int64_t dim_{0}; + std::vector outs_{}; + std::vector out_shapes_; + std::vector out_strides_; + std::vector out_types_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SPLIT_H_ diff --git a/src/base/stack_backward.h b/src/base/stack_backward.h new file mode 100644 index 000000000..a199d91d6 --- /dev/null +++ b/src/base/stack_backward.h @@ -0,0 +1,69 @@ +#ifndef INFINI_OPS_BASE_STACK_BACKWARD_H_ +#define INFINI_OPS_BASE_STACK_BACKWARD_H_ + +#include +#include +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class StackBackward : public Operator { + public: + StackBackward(const Tensor grad_output, const int64_t dim, + std::vector outs) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + dim_{NormalizeDim(dim, grad_output.ndim())}, + outs_{std::move(outs)}, + device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { + assert(!outs_.empty() && "`StackBackward` requires at least one output"); + assert(static_cast(outs_.size()) == grad_output.size(dim_) && + "`StackBackward` output count must equal grad_output.size(dim)"); + + for (std::size_t i = 0; i < outs_.size(); ++i) { + assert(outs_[i].ndim() == grad_output.ndim() - 1 && + "`StackBackward` output rank must be grad_output rank - 1"); + assert(outs_[i].dtype() == grad_output_type_ && "`StackBackward` dtype mismatch"); + for (Tensor::Size d = 0, s = 0; d < grad_output.ndim(); ++d) { + if (d == dim_) continue; + assert(outs_[i].size(s) == grad_output.size(d) && "`StackBackward` shape mismatch"); + ++s; + } + out_shapes_.emplace_back(outs_[i].shape().begin(), outs_[i].shape().end()); + + out_strides_.emplace_back(outs_[i].strides().begin(), outs_[i].strides().end()); + + out_types_.push_back(outs_[i].dtype()); + } + + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + std::vector outs) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`StackBackward` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + int64_t dim_{0}; + std::vector outs_{}; + std::vector out_shapes_; + std::vector out_strides_; + std::vector out_types_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_STACK_BACKWARD_H_ diff --git a/src/base/sub_backward.h b/src/base/sub_backward.h new file mode 100644 index 000000000..0ea897a56 --- /dev/null +++ b/src/base/sub_backward.h @@ -0,0 +1,60 @@ +#ifndef INFINI_OPS_BASE_SUB_BACKWARD_H_ +#define INFINI_OPS_BASE_SUB_BACKWARD_H_ + +#include + +#include "operator.h" + +namespace infini::ops { + +class SubBackward : public Operator { + public: + SubBackward(const Tensor grad_output, Tensor grad_a, Tensor grad_b) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_a_shape_{grad_a.shape()}, + grad_a_strides_{grad_a.strides()}, + grad_a_type_{grad_a.dtype()}, + grad_b_shape_{grad_b.shape()}, + grad_b_strides_{grad_b.strides()}, + grad_b_type_{grad_b.dtype()}, + device_index_{grad_a.device().index()} { + assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && + "`SubBackward` requires matching dtypes"); + assert(grad_output.device() == grad_a.device() && + grad_a.device() == grad_b.device() && + "`SubBackward` requires grad_output/grad_a/grad_b on the same device"); + ValidateBroadcast(grad_a, grad_output); + ValidateBroadcast(grad_b, grad_output); + } + + virtual void operator()(const Tensor grad_output, Tensor grad_a, + Tensor grad_b) const = 0; + + protected: + static void ValidateBroadcast(const Tensor input, const Tensor out) { + assert(input.ndim() <= out.ndim() && + "`SubBackward` input rank must not exceed grad_output rank"); + const auto offset = out.ndim() - input.ndim(); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`SubBackward` input must broadcast (right-aligned) to grad_output"); + } + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + Tensor::Shape grad_a_shape_; + Tensor::Strides grad_a_strides_; + DataType grad_a_type_; + Tensor::Shape grad_b_shape_; + Tensor::Strides grad_b_strides_; + DataType grad_b_type_; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ \ No newline at end of file diff --git a/src/base/transpose.h b/src/base/transpose.h new file mode 100644 index 000000000..cce6bdc49 --- /dev/null +++ b/src/base/transpose.h @@ -0,0 +1,59 @@ +#ifndef INFINI_OPS_BASE_TRANSPOSE_H_ +#define INFINI_OPS_BASE_TRANSPOSE_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class Transpose : public Operator { + public: + Transpose(const Tensor input, const int64_t dim0, const int64_t dim1, + Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + dim0_{NormalizeDim(dim0, input.ndim())}, + dim1_{NormalizeDim(dim1, input.ndim())}, + device_index_{out.device().index()} { + assert(input.ndim() >= 2 && "`Transpose` requires at least 2 dims"); + assert(dim0_ != dim1_ && "`Transpose` requires distinct dims"); + assert(input_type_ == out_type_ && "`Transpose` requires same dtype"); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + const auto expect = (i == dim0_) ? input.size(dim1_) + : (i == dim1_) ? input.size(dim0_) + : input.size(i); + assert(out.size(i) == expect && "`Transpose` output shape mismatch"); + } + } + + virtual void operator()(const Tensor input, const int64_t dim0, + const int64_t dim1, Tensor out) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`Transpose` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape input_shape_; + Tensor::Strides input_strides_; + DataType input_type_; + Tensor::Shape out_shape_; + Tensor::Strides out_strides_; + DataType out_type_; + int64_t dim0_{0}; + int64_t dim1_{0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_TRANSPOSE_H_ diff --git a/src/base/transpose_backward.h b/src/base/transpose_backward.h new file mode 100644 index 000000000..71b4bfc1d --- /dev/null +++ b/src/base/transpose_backward.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ +#define INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ + +#include +#include + +#include "operator.h" + +namespace infini::ops { + +class TransposeBackward : public Operator { + public: + TransposeBackward(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim0_{NormalizeDim(dim0, grad_output.ndim())}, + dim1_{NormalizeDim(dim1, grad_output.ndim())}, + device_index_{grad_input.device().index()} { + assert(grad_output.ndim() >= 2 && + "`TransposeBackward` requires at least 2 dims"); + assert(dim0_ != dim1_ && + "`TransposeBackward` requires distinct dims"); + assert(grad_output_type_ == grad_input_type_ && + "`TransposeBackward` requires same dtype"); + + for (Tensor::Size i = 0; i < grad_output.ndim(); ++i) { + const auto expect = (i == dim0_) ? grad_output.size(dim1_) + : (i == dim1_) ? grad_output.size(dim0_) + : grad_output.size(i); + assert(grad_input.size(i) == expect && + "`TransposeBackward` grad_input shape mismatch"); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) const = 0; + + protected: + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { + assert(ndim > 0 && "`TransposeBackward` requires ndim > 0"); + if (ndim == 0) return 0; + const auto d = dim % static_cast(ndim); + return d < 0 ? d + static_cast(ndim) : d; + } + + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + + int64_t dim0_{0}; + int64_t dim1_{0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ diff --git a/src/base/vocab_parallel_cross_entropy_backward.h b/src/base/vocab_parallel_cross_entropy_backward.h new file mode 100644 index 000000000..d11182df5 --- /dev/null +++ b/src/base/vocab_parallel_cross_entropy_backward.h @@ -0,0 +1,80 @@ +#ifndef INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ + +#include "operator.h" + +namespace infini::ops { + +class VocabParallelCrossEntropyBackward + : public Operator { + public: + VocabParallelCrossEntropyBackward( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + softmax_shape_{softmax_local.shape()}, + softmax_strides_{softmax_local.strides()}, + softmax_type_{softmax_local.dtype()}, + target_mask_shape_{target_mask.shape()}, + target_mask_strides_{target_mask.strides()}, + target_mask_type_{target_mask.dtype()}, + masked_target_shape_{masked_target.shape()}, + masked_target_strides_{masked_target.strides()}, + masked_target_type_{masked_target.dtype()}, + valid_mask_shape_{valid_mask_local.shape()}, + valid_mask_strides_{valid_mask_local.strides()}, + valid_mask_type_{valid_mask_local.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + vocab_size_local_{vocab_size_local}, + vocab_size_original_{vocab_size_original}, + label_smoothing_{label_smoothing}, + device_index_{grad_input.device().index()} {} + + virtual void operator()( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + Tensor::Strides grad_output_strides_; + DataType grad_output_type_; + + Tensor::Shape softmax_shape_; + Tensor::Strides softmax_strides_; + DataType softmax_type_; + + Tensor::Shape target_mask_shape_; + Tensor::Strides target_mask_strides_; + DataType target_mask_type_; + + Tensor::Shape masked_target_shape_; + Tensor::Strides masked_target_strides_; + DataType masked_target_type_; + + Tensor::Shape valid_mask_shape_; + Tensor::Strides valid_mask_strides_; + DataType valid_mask_type_; + + Tensor::Shape grad_input_shape_; + Tensor::Strides grad_input_strides_; + DataType grad_input_type_; + + int64_t vocab_size_local_{0}; + int64_t vocab_size_original_{0}; + double label_smoothing_{0.0}; + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.cc b/src/torch/ops/accumulate_grad/accumulate_grad.cc new file mode 100644 index 000000000..a3ee3f5be --- /dev/null +++ b/src/torch/ops/accumulate_grad/accumulate_grad.cc @@ -0,0 +1,40 @@ +#include "torch/ops/accumulate_grad/accumulate_grad.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor gradient, + const double rate, Tensor tensor) + : AccumulateGrad{gradient, rate, tensor}, + device_index_{tensor.device().index()} {} + +template +void Operator::operator()(const Tensor gradient, + const double rate, + Tensor tensor) const { + auto at_gradient = + ToAtenTensor(const_cast(gradient.data()), gradient_shape_, + gradient_strides_, gradient_type_, device_index_); + auto at_tensor = ToAtenTensor(tensor.data(), tensor_shape_, + tensor_strides_, tensor_type_, + device_index_); + + const auto at_rate = at_tensor.is_floating_point() + ? at::Scalar(rate) + : at::Scalar(static_cast(rate)); + at_tensor.add_(at_gradient, at_rate); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.h b/src/torch/ops/accumulate_grad/accumulate_grad.h new file mode 100644 index 000000000..3aa4e6413 --- /dev/null +++ b/src/torch/ops/accumulate_grad/accumulate_grad.h @@ -0,0 +1,22 @@ +#ifndef INFINI_OPS_TORCH_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_TORCH_ACCUMULATE_GRAD_H_ + +#include "base/accumulate_grad.h" + +namespace infini::ops { + +template +class Operator : public AccumulateGrad { + public: + Operator(const Tensor gradient, const double rate, Tensor tensor); + + + void operator()(const Tensor gradient, const double rate, Tensor tensor) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc new file mode 100644 index 000000000..29e91e64d --- /dev/null +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc @@ -0,0 +1,53 @@ +#include "torch/ops/adam_accumulate_grad/adam_accumulate_grad.h" + +#include + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, const double beta2, + const double eps, const int64_t step) + : AdamAccumulateGrad{grad, param, m, v, learning_rate, + beta1, beta2, eps, step}, + device_index_{param.device().index()} {} + +template +void Operator::operator()( + const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, const double beta2, + const double eps, const int64_t step) const { + auto at_grad = ToAtenTensor(const_cast(grad.data()), grad_shape_, + grad_strides_, grad_type_, device_index_); + auto at_param = ToAtenTensor(param.data(), param_shape_, param_strides_, + param_type_, device_index_); + auto at_m = ToAtenTensor(m.data(), m_shape_, m_strides_, m_type_, + device_index_); + auto at_v = ToAtenTensor(v.data(), v_shape_, v_strides_, v_type_, + device_index_); + + at_m.mul_(beta1_).add_(at_grad, 1.0 - beta1_); + at_v.mul_(beta2_).addcmul_(at_grad, at_grad, 1.0 - beta2_); + + const double bcm = 1.0 - std::pow(beta1_, static_cast(step_)); + const double bcv = 1.0 - std::pow(beta2_, static_cast(step_)); + const auto m_hat = at_m / bcm; + const auto v_hat = at_v / bcv; + + at_param.add_(-learning_rate_ * m_hat / (at::sqrt(v_hat) + eps_)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h new file mode 100644 index 000000000..d9f5880cd --- /dev/null +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h @@ -0,0 +1,26 @@ +#ifndef INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ + +#include "base/adam_accumulate_grad.h" + +namespace infini::ops { + +template +class Operator : public AdamAccumulateGrad { + public: + Operator(const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, const double beta2, + const double eps, const int64_t step); + + void operator()(const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, + const double beta2, const double eps, + const int64_t step) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ \ No newline at end of file diff --git a/src/torch/ops/add_backward/add_backward.cc b/src/torch/ops/add_backward/add_backward.cc new file mode 100644 index 000000000..045e924bd --- /dev/null +++ b/src/torch/ops/add_backward/add_backward.cc @@ -0,0 +1,74 @@ +#include "torch/ops/add_backward/add_backward.h" + +#include +#include +#include + +#include "torch/tensor_.h" + +namespace infini::ops { +namespace { + +at::Tensor ReduceToShape(const at::Tensor& grad, + const std::vector& target_shape) { + const int64_t ndim = static_cast(target_shape.size()); + const int64_t grad_ndim = grad.dim(); + assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); + + std::vector sum_dims; + for (int64_t i = 0; i < grad_ndim; ++i) { + const int64_t target_i = i - (grad_ndim - ndim); + if (target_i < 0) { + sum_dims.push_back(i); + continue; + } + if (target_shape[target_i] == 1 && grad.size(i) != 1) { + sum_dims.push_back(i); + } + } + + if (sum_dims.empty()) { + return grad.reshape(target_shape); + } + return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); +} + +} // namespace + +template +Operator::Operator(const Tensor grad_output, + Tensor grad_a, Tensor grad_b) + : AddBackward{grad_output, grad_a, grad_b}, + device_index_{grad_a.device().index()} {} + +template +void Operator::operator()(const Tensor grad_output, + Tensor grad_a, + Tensor grad_b) const { + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, + grad_a_strides_, grad_a_type_, + device_index_); + auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, + grad_b_strides_, grad_b_type_, + device_index_); + + at_grad_a.copy_(ReduceToShape( + at_grad, std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); + at_grad_b.copy_(ReduceToShape( + at_grad, std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/add_backward/add_backward.h b/src/torch/ops/add_backward/add_backward.h new file mode 100644 index 000000000..893222a40 --- /dev/null +++ b/src/torch/ops/add_backward/add_backward.h @@ -0,0 +1,22 @@ +#ifndef INFINI_OPS_TORCH_ADD_BACKWARD_H_ +#define INFINI_OPS_TORCH_ADD_BACKWARD_H_ + +#include "base/add_backward.h" + +namespace infini::ops { + +template +class Operator : public AddBackward { + public: + Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b); + + void operator()(const Tensor grad_output, Tensor grad_a, + Tensor grad_b) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_ADD_BACKWARD_H_ \ No newline at end of file diff --git a/src/torch/ops/cross_entropy/cross_entropy.cc b/src/torch/ops/cross_entropy/cross_entropy.cc new file mode 100644 index 000000000..38e4873bb --- /dev/null +++ b/src/torch/ops/cross_entropy/cross_entropy.cc @@ -0,0 +1,41 @@ +#include "torch/ops/cross_entropy/cross_entropy.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, + const Tensor target, Tensor out) + : CrossEntropy{input, target, out}, device_index_{out.device().index()} {} + +template +void Operator::operator()(const Tensor input, + const Tensor target, + Tensor out) const { + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_target = ToAtenTensor(const_cast(target.data()), + target_shape_, target_strides_, + target_type_, device_index_) + .to(at::kLong); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + // 默认参数:reduction=Mean, ignore_index=-100, label_smoothing=0 —— 与 InfiniTrain 一致 + auto loss = at::cross_entropy_loss(at_input, at_target); + at_out.copy_(loss); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/cross_entropy/cross_entropy.h b/src/torch/ops/cross_entropy/cross_entropy.h new file mode 100644 index 000000000..c5d8364ea --- /dev/null +++ b/src/torch/ops/cross_entropy/cross_entropy.h @@ -0,0 +1,22 @@ +#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_H_ +#define INFINI_OPS_TORCH_CROSS_ENTROPY_H_ + +#include "base/cross_entropy.h" + +namespace infini::ops { + +template +class Operator : public CrossEntropy { + public: + Operator(const Tensor input, const Tensor target, Tensor out); + + + void operator()(const Tensor input, const Tensor target, Tensor out) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc new file mode 100644 index 000000000..a25c9a490 --- /dev/null +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc @@ -0,0 +1,67 @@ +#include "torch/ops/cross_entropy_backward/cross_entropy_backward.h" + +#include + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor input, const Tensor target, const Tensor grad_output, + Tensor grad_input) + : CrossEntropyBackward{input, target, grad_output, grad_input} {} + +template +void Operator::operator()( + const Tensor input, const Tensor target, const Tensor grad_output, + Tensor grad_input) const { + auto at_input = ToAtenTensor( + const_cast(input.data()), input_shape_, input_strides_, + input_type_, device_index_); + auto at_target = ToAtenTensor( + const_cast(target.data()), target_shape_, target_strides_, + target_type_, device_index_); + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_grad_input = ToAtenTensor( + grad_input.data(), grad_input_shape_, grad_input_strides_, + grad_input_type_, device_index_); + + const auto classes = at_input.size(1); + assert(classes > 0 && "CrossEntropyBackward requires a non-empty class dimension"); + + auto gradient = at::softmax(at_input, 1).to(at_grad_input.scalar_type()); + auto target_with_class_dim = at_target.to(at::kLong).unsqueeze(1); + const auto valid_target = + target_with_class_dim.ne(CrossEntropyBackward::kIgnoreIndex); + const auto valid_range = + (target_with_class_dim >= 0) & (target_with_class_dim < classes); + assert(at::all((~valid_target) | valid_range).item() && + "CrossEntropyBackward target is outside [0, classes) or ignore_index"); + + auto safe_target = + target_with_class_dim.masked_fill(~valid_target, 0); + auto minus_one = -valid_target.to(gradient.scalar_type()); + gradient.scatter_add_(1, safe_target, minus_one); + gradient.mul_(valid_target.to(gradient.scalar_type())); + + const auto valid_count = + valid_target.sum().to(gradient.scalar_type()).clamp_min(1); + gradient.mul_(at_grad_output.reshape({1}).to(gradient.scalar_type()) / + valid_count); + at_grad_input.copy_(gradient); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h new file mode 100644 index 000000000..743d4bcbf --- /dev/null +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h @@ -0,0 +1,21 @@ +#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ + +#include "base/cross_entropy_backward.h" + +namespace infini::ops { + +template +class Operator + : public CrossEntropyBackward { + public: + Operator(const Tensor input, const Tensor target, const Tensor grad_output, + Tensor grad_input); + + void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/embedding_backward/embedding_backward.cc b/src/torch/ops/embedding_backward/embedding_backward.cc new file mode 100644 index 000000000..fd37661f5 --- /dev/null +++ b/src/torch/ops/embedding_backward/embedding_backward.cc @@ -0,0 +1,54 @@ +#include "torch/ops/embedding_backward/embedding_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad_output, const Tensor indices, const int64_t num_weights, + const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight) + : EmbeddingBackward{grad_output, indices, num_weights, padding_idx, + scale_grad_by_freq, sparse, grad_weight}, + device_index_{grad_weight.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor indices, const int64_t num_weights, + const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_indices = ToAtenTensor(const_cast(indices.data()), + indices_shape_, indices_strides_, + indices_type_, device_index_) + .to(at::kLong); + auto at_grad_weight = + ToAtenTensor(grad_weight.data(), grad_weight_shape_, + grad_weight_strides_, grad_weight_type_, device_index_); + + auto at_grad = at::embedding_dense_backward(at_grad_output, at_indices, + num_weights, padding_idx, + scale_grad_by_freq); + if (sparse) { + at_grad = at::embedding_backward(at_grad_output, at_indices, num_weights, + padding_idx, scale_grad_by_freq, true) + .to_dense(); + } + at_grad_weight.copy_(at_grad); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/embedding_backward/embedding_backward.h b/src/torch/ops/embedding_backward/embedding_backward.h new file mode 100644 index 000000000..b12838dcf --- /dev/null +++ b/src/torch/ops/embedding_backward/embedding_backward.h @@ -0,0 +1,26 @@ +#ifndef INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ +#define INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ + +#include "base/embedding_backward.h" + +namespace infini::ops { + +template +class Operator : public EmbeddingBackward { + public: + Operator(const Tensor grad_output, const Tensor indices, + const int64_t num_weights, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, Tensor grad_weight); + + void operator()(const Tensor grad_output, const Tensor indices, + const int64_t num_weights, const int64_t padding_idx, + const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ \ No newline at end of file diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc new file mode 100644 index 000000000..c42507070 --- /dev/null +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc @@ -0,0 +1,48 @@ +#include "torch/ops/masked_fill_backward/masked_fill_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor mask, + Tensor grad_input) + : MaskedFillBackward{grad_output, mask, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor mask, Tensor grad_input) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_mask = ToAtenTensor(const_cast(mask.data()), mask_shape_, + mask_strides_, mask_type_, device_index_) + .to(at::kBool); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (!tail_aligned_) { + + for (Tensor::Size i = mask_shape_.size(); i < grad_input_shape_.size(); ++i) { + at_mask = at_mask.unsqueeze(-1); + } + } + + at_grad_input.copy_(at::masked_fill(at_grad_output, at_mask, 0)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.h b/src/torch/ops/masked_fill_backward/masked_fill_backward.h new file mode 100644 index 000000000..8410d2218 --- /dev/null +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_MASKED_FILL_BACKWARD_H_ +#define INFINI_OPS_TORCH_MASKED_FILL_BACKWARD_H_ + +#include "base/masked_fill_backward.h" + +namespace infini::ops { + +template +class Operator : public MaskedFillBackward { + public: + Operator(const Tensor grad_output, const Tensor mask, + Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/mean_backward/mean_backward.cc b/src/torch/ops/mean_backward/mean_backward.cc new file mode 100644 index 000000000..6f30364ad --- /dev/null +++ b/src/torch/ops/mean_backward/mean_backward.cc @@ -0,0 +1,51 @@ +#include "torch/ops/mean_backward/mean_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, const bool keep_dim, + Tensor grad_input) + : MeanBackward{grad_output, dim, keep_dim, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()(const Tensor grad_output, + const int64_t dim, + const bool keep_dim, + Tensor grad_input) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (grad_input_shape_.empty() || grad_input_shape_[0] == 0) { + return; + } + + auto at_g = at_grad_output; + if (!keep_dim_) { + at_g = at_g.unsqueeze(dim_); + } + + const double count = static_cast(at_grad_input.numel()) / + static_cast(at_grad_output.numel()); + at_grad_input.copy_(at_g.expand(at_grad_input.sizes()) / count); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/mean_backward/mean_backward.h b/src/torch/ops/mean_backward/mean_backward.h new file mode 100644 index 000000000..a23ca23f8 --- /dev/null +++ b/src/torch/ops/mean_backward/mean_backward.h @@ -0,0 +1,24 @@ +#ifndef INFINI_OPS_TORCH_mean_backward_H_ +#define INFINI_OPS_TORCH_mean_backward_H_ + +#include "base/mean_backward.h" + +namespace infini::ops { + +template +class Operator : public MeanBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, const bool keep_dim, + Tensor grad_input); + + + void operator()(const Tensor grad_output, const int64_t dim, const bool keep_dim, + Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/mul_backward/mul_backward.cc b/src/torch/ops/mul_backward/mul_backward.cc new file mode 100644 index 000000000..726712841 --- /dev/null +++ b/src/torch/ops/mul_backward/mul_backward.cc @@ -0,0 +1,83 @@ +#include "torch/ops/mul_backward/mul_backward.h" + +#include +#include +#include + +#include "torch/tensor_.h" + +namespace infini::ops { +namespace { + +at::Tensor ReduceToShape(const at::Tensor& grad, + const std::vector& target_shape) { + const int64_t ndim = static_cast(target_shape.size()); + const int64_t grad_ndim = grad.dim(); + assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); + + std::vector sum_dims; + for (int64_t i = 0; i < grad_ndim; ++i) { + const int64_t target_i = i - (grad_ndim - ndim); + if (target_i < 0) { + sum_dims.push_back(i); + continue; + } + if (target_shape[target_i] == 1 && grad.size(i) != 1) { + sum_dims.push_back(i); + } + } + + if (sum_dims.empty()) { + return grad.reshape(target_shape); + } + return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); +} + +} // namespace + +template +Operator::Operator(const Tensor grad_output, + const Tensor a, const Tensor b, + Tensor grad_a, Tensor grad_b) + : MulBackward{grad_output, a, b, grad_a, grad_b}, + device_index_{grad_a.device().index()} {} + +template +void Operator::operator()(const Tensor grad_output, + const Tensor a, + const Tensor b, + Tensor grad_a, + Tensor grad_b) const { + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_a = ToAtenTensor(const_cast(a.data()), a_shape_, + a_strides_, a_type_, device_index_); + auto at_b = ToAtenTensor(const_cast(b.data()), b_shape_, + b_strides_, b_type_, device_index_); + auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, + grad_a_strides_, grad_a_type_, + device_index_); + auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, + grad_b_strides_, grad_b_type_, + device_index_); + + at_grad_a.copy_(ReduceToShape( + at_grad * at_b, + std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); + at_grad_b.copy_(ReduceToShape( + at_grad * at_a, + std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/mul_backward/mul_backward.h b/src/torch/ops/mul_backward/mul_backward.h new file mode 100644 index 000000000..7a02ad016 --- /dev/null +++ b/src/torch/ops/mul_backward/mul_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_MUL_BACKWARD_H_ +#define INFINI_OPS_TORCH_MUL_BACKWARD_H_ + +#include "base/mul_backward.h" + +namespace infini::ops { + +template +class Operator : public MulBackward { + public: + Operator(const Tensor grad_output, const Tensor a, const Tensor b, + Tensor grad_a, Tensor grad_b); + + void operator()(const Tensor grad_output, const Tensor a, const Tensor b, + Tensor grad_a, Tensor grad_b) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_MUL_BACKWARD_H_ \ No newline at end of file diff --git a/src/torch/ops/pow_backward/pow_backward.cc b/src/torch/ops/pow_backward/pow_backward.cc new file mode 100644 index 000000000..c80f9e8f8 --- /dev/null +++ b/src/torch/ops/pow_backward/pow_backward.cc @@ -0,0 +1,49 @@ +#include "torch/ops/pow_backward/pow_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor input, const double scalar, + const bool scalar_is_base, + Tensor grad_input) + : PowBackward{grad_output, input, scalar, scalar_is_base, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (scalar_is_base_) { + auto at_y = at::pow(scalar_, at_input); + at_grad_input.copy_(at_grad_output * at_y * std::log(scalar_)); + } else { + at_grad_input.copy_( + at_grad_output * scalar_ * at::pow(at_input, scalar_ - 1.0)); + } +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/pow_backward/pow_backward.h b/src/torch/ops/pow_backward/pow_backward.h new file mode 100644 index 000000000..c61b4ba29 --- /dev/null +++ b/src/torch/ops/pow_backward/pow_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_POW_BACKWARD_H_ +#define INFINI_OPS_TORCH_POW_BACKWARD_H_ + +#include "base/pow_backward.h" + +namespace infini::ops { + +template +class Operator : public PowBackward { + public: + Operator(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.cc b/src/torch/ops/repeat_interleave/repeat_interleave.cc new file mode 100644 index 000000000..32980c1a9 --- /dev/null +++ b/src/torch/ops/repeat_interleave/repeat_interleave.cc @@ -0,0 +1,38 @@ +#include "torch/ops/repeat_interleave/repeat_interleave.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, + const int64_t repeats, + const int64_t dim, Tensor out) + : RepeatInterleave{input, repeats, dim, out}, + device_index_{out.device().index()} {} + +template +void Operator::operator()(const Tensor input, + const int64_t repeats, + const int64_t dim, + Tensor out) const { + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + at_out.copy_(at::repeat_interleave(at_input, repeats_, dim_)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.h b/src/torch/ops/repeat_interleave/repeat_interleave.h new file mode 100644 index 000000000..d1fabc6c5 --- /dev/null +++ b/src/torch/ops/repeat_interleave/repeat_interleave.h @@ -0,0 +1,24 @@ +#ifndef INFINI_OPS_TORCH_REPEAT_INTERLEAVE_H_ +#define INFINI_OPS_TORCH_REPEAT_INTERLEAVE_H_ + +#include "base/repeat_interleave.h" + +namespace infini::ops { + +template +class Operator : public RepeatInterleave { + public: + Operator(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out); + + + void operator()(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc new file mode 100644 index 000000000..25ce36ed9 --- /dev/null +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc @@ -0,0 +1,45 @@ +#include "torch/ops/repeat_interleave_backward/repeat_interleave_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, + Tensor grad_input) + : RepeatInterleaveBackward{grad_output, dim, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const int64_t dim, Tensor grad_input) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + auto sizes = at_grad_output.sizes().vec(); + sizes[dim_] /= repeats_; + sizes.insert(sizes.begin() + dim_ + 1, repeats_); + + auto grouped = at_grad_output.reshape(sizes); + auto summed = grouped.sum(dim_ + 1); + + at_grad_input.copy_(summed); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h new file mode 100644 index 000000000..797b27755 --- /dev/null +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h @@ -0,0 +1,24 @@ +#ifndef INFINI_OPS_TORCH_REPEAT_INTERLEAVE_BACKWARD_H_ +#define INFINI_OPS_TORCH_REPEAT_INTERLEAVE_BACKWARD_H_ + +#include "base/repeat_interleave_backward.h" + +namespace infini::ops { + +template +class Operator : public RepeatInterleaveBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, + Tensor grad_input); + + + void operator()(const Tensor grad_output, const int64_t dim, + Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc new file mode 100644 index 000000000..9c5c6a9d5 --- /dev/null +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc @@ -0,0 +1,41 @@ +#include "torch/ops/rsqrt_backward/rsqrt_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor input, Tensor grad_input) + : RsqrtBackward{grad_output, input, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()(const Tensor grad_output, + const Tensor input, + Tensor grad_input) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + at_grad_input.copy_(at_grad_output * (-0.5) * at::pow(at::rsqrt(at_input), 3)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.h b/src/torch/ops/rsqrt_backward/rsqrt_backward.h new file mode 100644 index 000000000..4e94591bb --- /dev/null +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.h @@ -0,0 +1,22 @@ +#ifndef INFINI_OPS_TORCH_RSQRT_BACKWARD_H_ +#define INFINI_OPS_TORCH_RSQRT_BACKWARD_H_ + +#include "base/rsqrt_backward.h" + +namespace infini::ops { + +template +class Operator : public RsqrtBackward { + public: + Operator(const Tensor grad_output, const Tensor input, Tensor grad_input); + + + void operator()(const Tensor grad_output, const Tensor input, Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/slice/slice.cc b/src/torch/ops/slice/slice.cc new file mode 100644 index 000000000..6e05db7e2 --- /dev/null +++ b/src/torch/ops/slice/slice.cc @@ -0,0 +1,46 @@ +#include "torch/ops/slice/slice.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, + const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor out) + : Slice{input, starts, ends, steps, out}, device_index_{out.device().index()} {} + +template +void Operator::operator()( + const Tensor input, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor out) const { + using namespace torch::indexing; + + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + std::vector index; + index.reserve(starts_.size()); + for (std::size_t i = 0; i < starts_.size(); ++i) { + index.emplace_back(at::indexing::Slice(starts_[i], ends_[i], steps_[i])); + } + + at_out.copy_(at_input.index(index)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/slice/slice.h b/src/torch/ops/slice/slice.h new file mode 100644 index 000000000..da2debcf0 --- /dev/null +++ b/src/torch/ops/slice/slice.h @@ -0,0 +1,25 @@ +#ifndef INFINI_OPS_TORCH_SLICE_H_ +#define INFINI_OPS_TORCH_SLICE_H_ + +#include "base/slice.h" + +namespace infini::ops { + +template +class Operator : public Slice { + public: + Operator(const Tensor input, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor out); + + void operator()(const Tensor input, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor out) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_SLICE_H_ \ No newline at end of file diff --git a/src/torch/ops/slice_backward/slice_backward.cc b/src/torch/ops/slice_backward/slice_backward.cc new file mode 100644 index 000000000..6c30fcd84 --- /dev/null +++ b/src/torch/ops/slice_backward/slice_backward.cc @@ -0,0 +1,52 @@ +#include "torch/ops/slice_backward/slice_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad_output, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor grad_input) + : SliceBackward{grad_output, starts, ends, steps, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor grad_input) const { + using namespace torch::indexing; + + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + std::vector index; + index.reserve(starts_.size()); + for (std::size_t i = 0; i < starts_.size(); ++i) { + index.emplace_back(at::indexing::Slice(starts_[i], ends_[i], steps_[i])); + } + + + auto zeros = at::zeros(at_grad_input.sizes(), at_grad_input.options()); + zeros.index_put_(index, at_grad_output); + at_grad_input.copy_(zeros); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/slice_backward/slice_backward.h b/src/torch/ops/slice_backward/slice_backward.h new file mode 100644 index 000000000..398e11c0a --- /dev/null +++ b/src/torch/ops/slice_backward/slice_backward.h @@ -0,0 +1,25 @@ +#ifndef INFINI_OPS_TORCH_SLICE_BACKWARD_H_ +#define INFINI_OPS_TORCH_SLICE_BACKWARD_H_ + +#include "base/slice_backward.h" + +namespace infini::ops { + +template +class Operator : public SliceBackward { + public: + Operator(const Tensor grad_output, const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor grad_input); + + void operator()(const Tensor grad_output, const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/split/split.cc b/src/torch/ops/split/split.cc new file mode 100644 index 000000000..f1cba7716 --- /dev/null +++ b/src/torch/ops/split/split.cc @@ -0,0 +1,43 @@ +#include "torch/ops/split/split.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, const int64_t split_size, + const int64_t dim, std::vector outs) + : Split{input, split_size, dim, std::move(outs)}, + device_index_{Split::device_index_} {} + +template +void Operator::operator()(const Tensor input, + const int64_t split_size, + const int64_t dim, + std::vector outs) const { + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + + const auto parts = at::split(at_input, split_size_, dim_); + assert(parts.size() == outs_.size() && "`Split` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + auto at_out_i = ToAtenTensor(outs[i].data(), out_shapes_[i], + out_strides_[i], out_types_[i], + device_index_); + at_out_i.copy_(parts[i]); + } +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops diff --git a/src/torch/ops/split/split.h b/src/torch/ops/split/split.h new file mode 100644 index 000000000..29176e7c9 --- /dev/null +++ b/src/torch/ops/split/split.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_SPLIT_H_ +#define INFINI_OPS_TORCH_SPLIT_H_ + +#include "base/split.h" + +namespace infini::ops { + +template +class Operator : public Split { + public: + Operator(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs); + + void operator()(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/stack_backward/stack_backward.cc b/src/torch/ops/stack_backward/stack_backward.cc new file mode 100644 index 000000000..4f767d5a4 --- /dev/null +++ b/src/torch/ops/stack_backward/stack_backward.cc @@ -0,0 +1,45 @@ +#include "torch/ops/stack_backward/stack_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, + std::vector outs) + : StackBackward{grad_output, dim, std::move(outs)}, + device_index_{StackBackward::device_index_} {} + +template +void Operator::operator()(const Tensor grad_output, + const int64_t dim, + std::vector outs) const { + auto at_grad_output = + ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + + const auto parts = at::unbind(at_grad_output, dim_); + assert(parts.size() == outs_.size() && "`StackBackward` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + auto at_out_i = ToAtenTensor( + outs[i].data(), out_shapes_[i], + out_strides_[i], out_types_[i], + device_index_); + at_out_i.copy_(parts[i]); + } +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/stack_backward/stack_backward.h b/src/torch/ops/stack_backward/stack_backward.h new file mode 100644 index 000000000..460a12cc5 --- /dev/null +++ b/src/torch/ops/stack_backward/stack_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_STACK_BACKWARD_H_ +#define INFINI_OPS_TORCH_STACK_BACKWARD_H_ + +#include "base/stack_backward.h" + +namespace infini::ops { + +template +class Operator : public StackBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, + std::vector outs); + + void operator()(const Tensor grad_output, const int64_t dim, + std::vector outs) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/sub_backward/sub_backward.cc b/src/torch/ops/sub_backward/sub_backward.cc new file mode 100644 index 000000000..deffab7bd --- /dev/null +++ b/src/torch/ops/sub_backward/sub_backward.cc @@ -0,0 +1,75 @@ +#include "torch/ops/sub_backward/sub_backward.h" + +#include +#include +#include + +#include "torch/tensor_.h" + +namespace infini::ops { +namespace { + + +at::Tensor ReduceToShape(const at::Tensor& grad, + const std::vector& target_shape) { + const int64_t ndim = static_cast(target_shape.size()); + const int64_t grad_ndim = grad.dim(); + assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); + + std::vector sum_dims; + for (int64_t i = 0; i < grad_ndim; ++i) { + const int64_t target_i = i - (grad_ndim - ndim); + if (target_i < 0) { + sum_dims.push_back(i); + continue; + } + if (target_shape[target_i] == 1 && grad.size(i) != 1) { + sum_dims.push_back(i); + } + } + + if (sum_dims.empty()) { + return grad.reshape(target_shape); + } + return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); +} + +} // namespace + +template +Operator::Operator(const Tensor grad_output, + Tensor grad_a, Tensor grad_b) + : SubBackward{grad_output, grad_a, grad_b}, + device_index_{grad_a.device().index()} {} + +template +void Operator::operator()(const Tensor grad_output, + Tensor grad_a, + Tensor grad_b) const { + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, + grad_a_strides_, grad_a_type_, + device_index_); + auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, + grad_b_strides_, grad_b_type_, + device_index_); + + at_grad_a.copy_(ReduceToShape( + at_grad, std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); + at_grad_b.copy_(-ReduceToShape( + at_grad, std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/sub_backward/sub_backward.h b/src/torch/ops/sub_backward/sub_backward.h new file mode 100644 index 000000000..76b249c48 --- /dev/null +++ b/src/torch/ops/sub_backward/sub_backward.h @@ -0,0 +1,22 @@ +#ifndef INFINI_OPS_TORCH_SUB_BACKWARD_H_ +#define INFINI_OPS_TORCH_SUB_BACKWARD_H_ + +#include "base/sub_backward.h" + +namespace infini::ops { + +template +class Operator : public SubBackward { + public: + Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b); + + void operator()(const Tensor grad_output, Tensor grad_a, + Tensor grad_b) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_SUB_BACKWARD_H_ \ No newline at end of file diff --git a/src/torch/ops/transpose/transpose.cc b/src/torch/ops/transpose/transpose.cc new file mode 100644 index 000000000..6c95855c6 --- /dev/null +++ b/src/torch/ops/transpose/transpose.cc @@ -0,0 +1,36 @@ +#include "torch/ops/transpose/transpose.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, const int64_t dim0, + const int64_t dim1, Tensor out) + : Transpose{input, dim0, dim1, out}, device_index_{out.device().index()} {} + +template +void Operator::operator()(const Tensor input, + const int64_t dim0, + const int64_t dim1, + Tensor out) const { + auto at_input = ToAtenTensor(const_cast(input.data()), + input_shape_, input_strides_, input_type_, + device_index_); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + at_out.copy_(at_input.transpose(dim0_, dim1_)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/transpose/transpose.h b/src/torch/ops/transpose/transpose.h new file mode 100644 index 000000000..13419d981 --- /dev/null +++ b/src/torch/ops/transpose/transpose.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_TRANSPOSE_H_ +#define INFINI_OPS_TORCH_TRANSPOSE_H_ + +#include "base/transpose.h" + +namespace infini::ops { + +template +class Operator : public Transpose { + public: + Operator(const Tensor input, const int64_t dim0, const int64_t dim1, + Tensor out); + + void operator()(const Tensor input, const int64_t dim0, const int64_t dim1, + Tensor out) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/transpose_backward/transpose_backward.cc b/src/torch/ops/transpose_backward/transpose_backward.cc new file mode 100644 index 000000000..600c08022 --- /dev/null +++ b/src/torch/ops/transpose_backward/transpose_backward.cc @@ -0,0 +1,39 @@ +#include "torch/ops/transpose_backward/transpose_backward.h" + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad_output, const int64_t dim0, const int64_t dim1, + Tensor grad_input) + : TransposeBackward{grad_output, dim0, dim1, grad_input}, + device_index_{grad_input.device().index()} {} + +template +void Operator::operator()( + const Tensor grad_output, const int64_t dim0, const int64_t dim1, + Tensor grad_input) const { + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + + auto at_grad_input = ToAtenTensor( + grad_input.data(), grad_input_shape_, grad_input_strides_, + grad_input_type_, device_index_); + + at_grad_input.copy_(at_grad_output.transpose(dim0_, dim1_)); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops \ No newline at end of file diff --git a/src/torch/ops/transpose_backward/transpose_backward.h b/src/torch/ops/transpose_backward/transpose_backward.h new file mode 100644 index 000000000..08c6cf89d --- /dev/null +++ b/src/torch/ops/transpose_backward/transpose_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ +#define INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ + +#include "base/transpose_backward.h" + +namespace infini::ops { + +template +class Operator : public TransposeBackward { + public: + Operator(const Tensor grad_output, const int64_t dim0, const int64_t dim1, + Tensor grad_input); + + void operator()(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) const override; + + private: + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ \ No newline at end of file diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc new file mode 100644 index 000000000..1a5e4b24a --- /dev/null +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc @@ -0,0 +1,113 @@ +#include "torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h" + +#include + +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) + : VocabParallelCrossEntropyBackward{ + grad_output, softmax_local, target_mask, masked_target, + valid_mask_local, vocab_size_local, vocab_size_original, + label_smoothing, grad_input} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) const { + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_softmax = ToAtenTensor( + const_cast(softmax_local.data()), softmax_shape_, softmax_strides_, + softmax_type_, device_index_); + auto at_target_mask = ToAtenTensor( + const_cast(target_mask.data()), target_mask_shape_, + target_mask_strides_, target_mask_type_, device_index_); + auto at_masked_target = ToAtenTensor( + const_cast(masked_target.data()), masked_target_shape_, + masked_target_strides_, masked_target_type_, device_index_); + auto at_valid_mask = ToAtenTensor( + const_cast(valid_mask_local.data()), valid_mask_shape_, + valid_mask_strides_, valid_mask_type_, device_index_); + auto at_grad_input = ToAtenTensor( + grad_input.data(), grad_input_shape_, grad_input_strides_, + grad_input_type_, device_index_); + + assert(vocab_size_local > 0 && + "VocabParallelCrossEntropyBackward requires positive local vocab"); + assert(vocab_size_original > 0 && + "VocabParallelCrossEntropyBackward requires positive global vocab"); + assert(at_softmax.numel() % vocab_size_local == 0 && + "softmax_local size is not divisible by vocab_size_local"); + assert(at_target_mask.numel() == at_softmax.numel() / vocab_size_local && + "target_mask size mismatch"); + assert(at_masked_target.numel() == at_target_mask.numel() && + "masked_target size mismatch"); + assert(at_valid_mask.numel() == vocab_size_local && + "valid_mask_local size mismatch"); + + const auto rows = at_softmax.numel() / vocab_size_local; + assert((at_grad_output.numel() == 1 || at_grad_output.numel() == rows) && + "grad_output must be scalar or one value per row"); + + auto softmax_2d = at_softmax.reshape({rows, vocab_size_local}) + .to(at_grad_input.scalar_type()); + auto target_mask_1d = + at_target_mask.reshape({rows}).to(at_grad_input.scalar_type()); + auto target_1d = at_masked_target.reshape({rows}).to(at::kLong); + auto valid_1d = + at_valid_mask.reshape({vocab_size_local}).to(at_grad_input.scalar_type()); + + auto dloss = at_grad_output.reshape({-1}).to(at_grad_input.scalar_type()); + if (dloss.numel() == 1) { + dloss = dloss.expand({rows}); + } + + auto gradient = softmax_2d * dloss.reshape({rows, 1}); + const auto valid_rows = 1.0 - target_mask_1d; + const auto row_scale = dloss * (1.0 - label_smoothing) * valid_rows; + + auto target_one_hot = at::zeros_like(gradient); + target_one_hot.scatter_(1, target_1d.reshape({rows, 1}), 1.0); + gradient.sub_(target_one_hot * row_scale.reshape({rows, 1})); + + const auto smoothing_term = + label_smoothing / static_cast(vocab_size_original); + gradient.sub_(dloss.reshape({rows, 1}) * smoothing_term * + valid_1d.reshape({1, vocab_size_local})); + gradient.mul_(valid_1d.reshape({1, vocab_size_local})); + + at_grad_input.copy_(gradient.reshape(at_grad_input.sizes())); +} + +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; +template class Operator; + +} // namespace infini::ops diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h new file mode 100644 index 000000000..82268aa03 --- /dev/null +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h @@ -0,0 +1,28 @@ +#ifndef INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ + +#include "base/vocab_parallel_cross_entropy_backward.h" + +namespace infini::ops { + +template +class Operator + : public VocabParallelCrossEntropyBackward { + public: + Operator(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input); + + void operator()( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ diff --git a/tests/test_accumulate_grad.py b/tests/test_accumulate_grad.py new file mode 100644 index 000000000..136af0079 --- /dev/null +++ b/tests/test_accumulate_grad.py @@ -0,0 +1,87 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, rate, gradient_strides, tensor_strides) +_TEST_CASES = ( + pytest.param((17,), 0.5, None, None, id="one-dimensional"), + pytest.param((2, 3), -0.25, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + 1.5, + (30, 7, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, rate, gradient_strides, tensor_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_accumulate_grad( + shape, + rate, + gradient_strides, + tensor_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + gradient = randn_strided( + shape, + gradient_strides, + dtype=dtype, + device=device, + ) + tensor = randn_strided( + shape, + tensor_strides, + dtype=dtype, + device=device, + ) + + return Payload( + lambda *args: _accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_accumulate_grad, + (gradient, rate, tensor), + {}, + rtol=rtol, + atol=atol, + ) + + +def _accumulate_grad(gradient, rate, tensor, *, implementation_index): + infini.ops.accumulate_grad( + gradient, + rate, + tensor, + stream=get_stream(gradient.device), + implementation_index=implementation_index, + ) + + return tensor + + +def _torch_accumulate_grad(gradient, rate, tensor): + tensor.add_(gradient * rate) + + return tensor diff --git a/tests/test_adam_accumulate_grad.py b/tests/test_adam_accumulate_grad.py new file mode 100644 index 000000000..d79487458 --- /dev/null +++ b/tests/test_adam_accumulate_grad.py @@ -0,0 +1,139 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, learning_rate, beta1, beta2, eps, step, strides) +_TEST_CASES = ( + pytest.param( + (17,), + 1e-2, + 0.9, + 0.999, + 1e-8, + 1, + None, + id="one-dimensional-first-step", + ), + pytest.param( + (2, 3), + 5e-3, + 0.8, + 0.99, + 1e-6, + 3, + None, + id="two-dimensional-third-step", + ), + pytest.param( + (2, 3, 4), + 1e-2, + 0.9, + 0.999, + 1e-8, + 2, + (30, 7, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, learning_rate, beta1, beta2, eps, step, strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 2e-3, 2e-3), + (torch.bfloat16, 2e-2, 2e-2), + ), +) +def test_adam_accumulate_grad( + shape, + learning_rate, + beta1, + beta2, + eps, + step, + strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad = randn_strided(shape, strides, dtype=dtype, device=device) + param = randn_strided(shape, strides, dtype=dtype, device=device) + m = randn_strided(shape, strides, dtype=dtype, device=device) + v = randn_strided(shape, strides, dtype=dtype, device=device).abs_().add_(0.1) + + return Payload( + lambda *args: _adam_accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_adam_accumulate_grad, + (grad, param, m, v, learning_rate, beta1, beta2, eps, step), + {}, + rtol=rtol, + atol=atol, + ) + + +def _adam_accumulate_grad( + grad, + param, + m, + v, + learning_rate, + beta1, + beta2, + eps, + step, + *, + implementation_index, +): + infini.ops.adam_accumulate_grad( + grad, + param, + m, + v, + learning_rate, + beta1, + beta2, + eps, + step, + stream=get_stream(grad.device), + implementation_index=implementation_index, + ) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) + + +def _torch_adam_accumulate_grad( + grad, + param, + m, + v, + learning_rate, + beta1, + beta2, + eps, + step, +): + m.copy_(beta1 * m + (1 - beta1) * grad) + v.copy_(beta2 * v + (1 - beta2) * grad * grad) + + bias_correction_m = 1 - beta1**step + bias_correction_v = 1 - beta2**step + m_hat = m.float() / bias_correction_m + v_hat = v.float() / bias_correction_v + update = learning_rate * m_hat / (torch.sqrt(v_hat) + eps) + param.copy_(param - update.to(dtype=param.dtype)) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) diff --git a/tests/test_add_backward.py b/tests/test_add_backward.py new file mode 100644 index 000000000..e3d3ed78a --- /dev/null +++ b/tests/test_add_backward.py @@ -0,0 +1,87 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), None, None, None, id="two-dimensional"), + pytest.param((2, 3, 4), None, None, None, id="three-dimensional"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_add_backward( + shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) + grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _add_backward( + *args, + implementation_index=implementation_index, + ), + _torch_add_backward, + (grad_output, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _add_backward(grad_output, grad_a, grad_b, *, implementation_index): + infini.ops.add_backward( + grad_output, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +def _torch_add_backward(grad_output, grad_a, grad_b): + grad_a.copy_(grad_output) + grad_b.copy_(grad_output) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) diff --git a/tests/test_cross_entropy.py b/tests/test_cross_entropy.py new file mode 100644 index 000000000..899dcd360 --- /dev/null +++ b/tests/test_cross_entropy.py @@ -0,0 +1,113 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided, randint_strided + + +# Format: (input_shape, input_strides, target_strides) +# CrossEntropy follows the PyTorch convention: dimension 1 is the class dimension. +_TEST_CASES = ( + pytest.param( + (4, 7), + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 5), + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 5), + (30, 7, 1), + None, + id="non-contiguous-input", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, target_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("use_ignore_index", (False, True)) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-5, 1e-5), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_cross_entropy( + input_shape, + input_strides, + target_strides, + use_ignore_index, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + target_shape = (input_shape[0],) + input_shape[2:] + num_classes = input_shape[1] + target = randint_strided( + 0, + num_classes, + target_shape, + target_strides, + dtype=torch.int64, + device=device, + ) + if use_ignore_index: + target.reshape(-1)[0] = -100 + out = empty_strided((), None, dtype=dtype, device=device) + out.fill_(17) + + return Payload( + lambda *args: _cross_entropy( + *args, + implementation_index=implementation_index, + ), + _torch_cross_entropy, + (input, target, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _cross_entropy(input, target, out, *, implementation_index): + infini.ops.cross_entropy( + input, + target, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return out + + +def _torch_cross_entropy(input, target, out): + out.copy_( + torch.nn.functional.cross_entropy( + input, + target, + reduction="mean", + ignore_index=-100, + ) + ) + + return out diff --git a/tests/test_cross_entropy_backward.py b/tests/test_cross_entropy_backward.py new file mode 100644 index 000000000..d6a1791d8 --- /dev/null +++ b/tests/test_cross_entropy_backward.py @@ -0,0 +1,130 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randint_strided, randn_strided + + +# Format: (input_shape, input_strides, target_strides, grad_input_strides) +# CrossEntropy uses the PyTorch convention: dimension 1 is the class dimension. +_TEST_CASES = ( + pytest.param( + (4, 7), + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 5), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 5), + (30, 7, 1), + None, + (35, 8, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, target_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("use_ignore_index", (False, True)) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_cross_entropy_backward( + input_shape, + input_strides, + target_strides, + grad_input_strides, + use_ignore_index, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + target = randint_strided( + 0, + input_shape[1], + (input_shape[0],) + input_shape[2:], + target_strides, + dtype=torch.int64, + device=device, + ) + if use_ignore_index: + target.reshape(-1)[0] = -100 + grad_output = torch.randn((), dtype=dtype, device=device) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _cross_entropy_backward( + *args, + implementation_index=implementation_index, + ), + _torch_cross_entropy_backward, + (input, target, grad_output, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _cross_entropy_backward( + input, + target, + grad_output, + grad_input, + *, + implementation_index, +): + infini.ops.cross_entropy_backward( + input, + target, + grad_output, + grad_input, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_cross_entropy_backward(input, target, grad_output, grad_input): + logits = input.detach().clone().requires_grad_(True) + loss = torch.nn.functional.cross_entropy( + logits, + target, + reduction="mean", + ignore_index=-100, + ) + gradient = torch.autograd.grad(loss, logits, grad_output)[0] + grad_input.copy_(gradient) + + return grad_input diff --git a/tests/test_embedding_backward.py b/tests/test_embedding_backward.py new file mode 100644 index 000000000..e3c992bdf --- /dev/null +++ b/tests/test_embedding_backward.py @@ -0,0 +1,177 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: +# (indices, num_weights, embedding_dim, indices_strides, +# grad_output_strides, grad_weight_strides, padding_idx, +# scale_grad_by_freq) +_TEST_CASES = ( + pytest.param( + (1, 2, 1, 4, 2, 1), + 6, + 4, + None, + None, + None, + -1, + False, + id="repeated-indices", + ), + pytest.param( + ((0, 1, 0), (2, 1, 3)), + 5, + 7, + (5, 1), + (32, 8, 1), + (9, 1), + 1, + False, + id="padding-and-strides", + ), + pytest.param( + (2, 2, 2, 3, 3, 4), + 7, + 5, + None, + None, + None, + -1, + True, + id="scale-grad-by-frequency", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "indices_values", + "num_weights", + "embedding_dim", + "indices_strides", + "grad_output_strides", + "grad_weight_strides", + "padding_idx", + "scale_grad_by_freq", + ), + _TEST_CASES, +) +@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-5, 1e-5), + (torch.float16, 1e-2, 1e-2), + (torch.bfloat16, 5e-2, 5e-2), + ), +) +def test_embedding_backward( + indices_values, + num_weights, + embedding_dim, + indices_strides, + grad_output_strides, + grad_weight_strides, + padding_idx, + scale_grad_by_freq, + indices_dtype, + implementation_index, + dtype, + device, + rtol, + atol, +): + indices_values = torch.tensor(indices_values, dtype=indices_dtype) + indices = empty_strided( + indices_values.shape, + indices_strides, + dtype=indices_dtype, + device=device, + ) + indices.copy_(indices_values.to(device)) + + grad_output_shape = (*indices.shape, embedding_dim) + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_weight = empty_strided( + (num_weights, embedding_dim), + grad_weight_strides, + dtype=dtype, + device=device, + ) + grad_weight.fill_(float("nan")) + + return Payload( + lambda *args: _embedding_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + ), + lambda *args: _torch_embedding_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + ), + (grad_output, indices, grad_weight), + {}, + rtol=rtol, + atol=atol, + ) + + +def _embedding_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, + implementation_index, +): + infini.ops.embedding_backward( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + False, + grad_weight, + implementation_index=implementation_index, + stream=get_stream(grad_output.device), + ) + + return grad_weight + + +def _torch_embedding_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, +): + result = torch.ops.aten.embedding_backward.default( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + False, + ) + grad_weight.copy_(result) + + return grad_weight diff --git a/tests/test_masked_fill_backward.py b/tests/test_masked_fill_backward.py new file mode 100644 index 000000000..53f97cc35 --- /dev/null +++ b/tests/test_masked_fill_backward.py @@ -0,0 +1,147 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, mask_strides, grad_input_strides, mask_mode) +_TEST_CASES = ( + pytest.param( + (17,), + None, + None, + None, + "alternating", + id="one-dimensional", + ), + pytest.param( + (3, 5), + None, + None, + None, + "all-false", + id="all-false-mask", + ), + pytest.param( + (3, 5), + None, + None, + None, + "all-true", + id="all-true-mask", + ), + pytest.param( + (2, 3, 4), + None, + None, + None, + "alternating", + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (20, 6, 1), + (18, 5, 1), + (24, 7, 1), + "alternating", + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "grad_output_strides", + "mask_strides", + "grad_input_strides", + "mask_mode", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_masked_fill_backward( + shape, + grad_output_strides, + mask_strides, + grad_input_strides, + mask_mode, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + mask = _make_mask(shape, mask_strides, mask_mode, device) + grad_input = empty_strided( + shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _masked_fill_backward( + *args, + implementation_index=implementation_index, + ), + _torch_masked_fill_backward, + (grad_output, mask, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _make_mask(shape, strides, mode, device): + mask = empty_strided(shape, strides, dtype=torch.bool, device=device) + + if mode == "all-false": + mask.fill_(False) + elif mode == "all-true": + mask.fill_(True) + else: + values = torch.arange(mask.numel(), device=device).reshape(shape) % 3 == 0 + mask.copy_(values) + + return mask + + +def _masked_fill_backward( + grad_output, + mask, + grad_input, + *, + implementation_index, +): + infini.ops.masked_fill_backward( + grad_output, + mask, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_masked_fill_backward(grad_output, mask, grad_input): + grad_input.copy_(grad_output.masked_fill(mask, 0)) + + return grad_input diff --git a/tests/test_mean_backward.py b/tests/test_mean_backward.py new file mode 100644 index 000000000..95db25fd7 --- /dev/null +++ b/tests/test_mean_backward.py @@ -0,0 +1,158 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, dim, keep_dim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 0, + False, + None, + None, + id="one-dimensional-no-keep-dim", + ), + pytest.param( + (2, 3), + 1, + False, + None, + None, + id="two-dimensional-no-keep-dim", + ), + pytest.param( + (2, 3), + 1, + True, + None, + None, + id="two-dimensional-keep-dim", + ), + pytest.param( + (2, 3, 4), + -1, + False, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 0, + True, + None, + None, + id="three-dimensional-first-dimension-keep-dim", + ), + pytest.param( + (2, 3, 4), + 1, + False, + (10, 1), + (24, 7, 1), + id="strided-gradients", + ), + pytest.param( + (2, 3, 4), + 1, + True, + (10, 1, 1), + (24, 7, 1), + id="strided-gradients-keep-dim", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, dim, keep_dim, grad_output_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_mean_backward( + input_shape, + dim, + keep_dim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape) + grad_output_shape = list(input_shape) + if keep_dim: + grad_output_shape[normalized_dim] = 1 + else: + del grad_output_shape[normalized_dim] + + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _mean_backward( + *args, + implementation_index=implementation_index, + ), + _torch_mean_backward, + (grad_output, dim, keep_dim, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _mean_backward( + grad_output, + dim, + keep_dim, + grad_input, + *, + implementation_index, +): + infini.ops.mean_backward( + grad_output, + dim, + keep_dim, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_mean_backward(grad_output, dim, keep_dim, grad_input): + normalized_dim = dim % grad_input.ndim + reduce_size = grad_input.shape[normalized_dim] + expanded_grad = grad_output + + if not keep_dim: + expanded_grad = expanded_grad.unsqueeze(normalized_dim) + + grad_input.copy_(expanded_grad.expand_as(grad_input) / reduce_size) + + return grad_input diff --git a/tests/test_mul_backward.py b/tests/test_mul_backward.py new file mode 100644 index 000000000..888a585dd --- /dev/null +++ b/tests/test_mul_backward.py @@ -0,0 +1,135 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, a_strides, b_strides, grad_a_strides, +# grad_b_strides) +_TEST_CASES = ( + pytest.param( + (17,), + None, + None, + None, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + None, + None, + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + None, + None, + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (28, 6, 1), + (32, 5, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "grad_output_strides", + "a_strides", + "b_strides", + "grad_a_strides", + "grad_b_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_mul_backward( + shape, + grad_output_strides, + a_strides, + b_strides, + grad_a_strides, + grad_b_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + a = randn_strided(shape, a_strides, dtype=dtype, device=device) + b = randn_strided(shape, b_strides, dtype=dtype, device=device) + grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) + grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _mul_backward( + *args, + implementation_index=implementation_index, + ), + _torch_mul_backward, + (grad_output, a, b, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + *, + implementation_index, +): + infini.ops.mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +def _torch_mul_backward(grad_output, a, b, grad_a, grad_b): + grad_a.copy_(grad_output * b) + grad_b.copy_(grad_output * a) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) diff --git a/tests/test_pow_backward.py b/tests/test_pow_backward.py new file mode 100644 index 000000000..06d70d8e1 --- /dev/null +++ b/tests/test_pow_backward.py @@ -0,0 +1,180 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, scalar, scalar_is_base, grad_output_strides, input_strides, +# grad_input_strides) +_TEST_CASES = ( + pytest.param( + (17,), + 2.0, + False, + None, + None, + None, + id="one-dimensional-input-base", + ), + pytest.param( + (2, 3), + 3.0, + False, + None, + None, + None, + id="two-dimensional-input-base", + ), + pytest.param( + (2, 3, 4), + 0.5, + False, + None, + None, + None, + id="fractional-input-base", + ), + pytest.param( + (2, 3, 4), + 2.0, + True, + None, + None, + None, + id="scalar-base", + ), + pytest.param( + (2, 3, 4), + 0.5, + True, + None, + None, + None, + id="fractional-scalar-base", + ), + pytest.param( + (2, 3, 4), + 3.0, + False, + (30, 7, 1), + (28, 6, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "scalar", + "scalar_is_base", + "grad_output_strides", + "input_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_pow_backward( + shape, + scalar, + scalar_is_base, + grad_output_strides, + input_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + input = randn_strided( + shape, + input_strides, + dtype=dtype, + device=device, + ) + input.abs_().add_(0.5) + grad_input = empty_strided( + shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _pow_backward( + *args, + implementation_index=implementation_index, + ), + _torch_pow_backward, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, + *, + implementation_index, +): + infini.ops.pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, +): + scalar_tensor = torch.as_tensor( + scalar, + dtype=input.dtype, + device=input.device, + ) + + if scalar_is_base: + derivative = torch.log(scalar_tensor) * torch.pow( + scalar_tensor, + input, + ) + else: + derivative = scalar_tensor * torch.pow(input, scalar_tensor - 1) + + grad_input.copy_(grad_output * derivative) + + return grad_input diff --git a/tests/test_repeat_interleave.py b/tests/test_repeat_interleave.py new file mode 100644 index 000000000..307f216a3 --- /dev/null +++ b/tests/test_repeat_interleave.py @@ -0,0 +1,131 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, repeats, dim, input_strides, out_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 3, + 0, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + 2, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 3), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="repeat-once", + ), + pytest.param( + (2, 3, 4), + 2, + 1, + (20, 6, 1), + (32, 5, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, repeats, dim, input_strides, out_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_repeat_interleave( + input_shape, + repeats, + dim, + input_strides, + out_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + out_shape = list(input_shape) + out_shape[dim % len(input_shape)] *= repeats + out = empty_strided( + tuple(out_shape), + out_strides, + dtype=dtype, + device=device, + ) + out.fill_(17) + + return Payload( + lambda *args: _repeat_interleave( + *args, + implementation_index=implementation_index, + ), + _torch_repeat_interleave, + (input, repeats, dim, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _repeat_interleave(input, repeats, dim, out, *, implementation_index): + infini.ops.repeat_interleave( + input, + repeats, + dim, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return out + + +def _torch_repeat_interleave(input, repeats, dim, out): + out.copy_(torch.repeat_interleave(input, repeats, dim=dim)) + + return out diff --git a/tests/test_repeat_interleave_backward.py b/tests/test_repeat_interleave_backward.py new file mode 100644 index 000000000..b2160d99b --- /dev/null +++ b/tests/test_repeat_interleave_backward.py @@ -0,0 +1,143 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, repeats, dim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 3, + 0, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + 2, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 3), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="repeat-once", + ), + pytest.param( + (2, 3, 4), + 2, + 1, + (40, 8, 1), + (20, 6, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, repeats, dim, grad_output_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_repeat_interleave_backward( + input_shape, + repeats, + dim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape) + grad_output_shape = list(input_shape) + grad_output_shape[normalized_dim] *= repeats + + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _repeat_interleave_backward( + *args, + implementation_index=implementation_index, + ), + _torch_repeat_interleave_backward, + (grad_output, dim, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _repeat_interleave_backward( + grad_output, + dim, + grad_input, + *, + implementation_index, +): + infini.ops.repeat_interleave_backward( + grad_output, + dim, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_repeat_interleave_backward(grad_output, dim, grad_input): + dim = dim % grad_output.ndim + repeats = grad_output.shape[dim] // grad_input.shape[dim] + reshaped_shape = list(grad_output.shape) + reshaped_shape[dim] = grad_input.shape[dim] + reshaped_shape.insert(dim + 1, repeats) + grad_input.copy_(grad_output.reshape(reshaped_shape).sum(dim=dim + 1)) + + return grad_input diff --git a/tests/test_rsqrt_backward.py b/tests/test_rsqrt_backward.py new file mode 100644 index 000000000..9c4e8fbb3 --- /dev/null +++ b/tests/test_rsqrt_backward.py @@ -0,0 +1,120 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, input_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (17,), + None, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (28, 6, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, grad_output_strides, input_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_rsqrt_backward( + shape, + grad_output_strides, + input_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + input = randn_strided( + shape, + input_strides, + dtype=dtype, + device=device, + ) + input.abs_().add_(0.5) + grad_input = empty_strided( + shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _rsqrt_backward( + *args, + implementation_index=implementation_index, + ), + _torch_rsqrt_backward, + (grad_output, input, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _rsqrt_backward( + grad_output, + input, + grad_input, + *, + implementation_index, +): + infini.ops.rsqrt_backward( + grad_output, + input, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_rsqrt_backward(grad_output, input, grad_input): + grad_input.copy_(-0.5 * grad_output * torch.pow(input, -1.5)) + + return grad_input diff --git a/tests/test_slice.py b/tests/test_slice.py new file mode 100644 index 000000000..ed4e90acf --- /dev/null +++ b/tests/test_slice.py @@ -0,0 +1,138 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param( + (17,), + (5,), + None, + None, + (1,), + (16,), + (3,), + id="one-dimensional", + ), + pytest.param( + (6, 9), + (3, 3), + (12, 1), + (5, 1), + (1, 0), + (6, 9), + (2, 3), + id="strided-two-dimensional", + ), + pytest.param( + (4, 6, 7), + (2, 3, 3), + (60, 9, 1), + (20, 5, 1), + (-4, 1, 0), + (-1, 6, 7), + (2, 2, 3), + id="negative-indices", + ), + pytest.param( + (2, 3, 4, 5), + (2, 2, 3, 3), + None, + (24, 12, 4, 1), + (0, 0, 1, 0), + (2, 3, 4, 5), + (1, 2, 1, 2), + id="four-dimensional", + ), + pytest.param( + (5, 7), + (0, 7), + None, + None, + (3, 0), + (3, 7), + (1, 1), + id="empty-slice", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "input_shape", + "out_shape", + "input_strides", + "out_strides", + "starts", + "ends", + "steps", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_slice( + input_shape, + out_shape, + input_strides, + out_strides, + starts, + ends, + steps, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, input_strides, dtype=dtype, device=device + ) + out = empty_strided(out_shape, out_strides, dtype=dtype, device=device) + + return Payload( + lambda *args: _slice( + *args, implementation_index=implementation_index + ), + _torch_slice, + (input, starts, ends, steps, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _slice(input, starts, ends, steps, out, *, implementation_index): + infini.ops.slice( + input, + starts, + ends, + steps, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return out + + +def _torch_slice(input, starts, ends, steps, out): + out.copy_(input[_slice_index(starts, ends, steps)]) + + return out + + +def _slice_index(starts, ends, steps): + return tuple( + slice(start, end, step) + for start, end, step in zip(starts, ends, steps, strict=True) + ) diff --git a/tests/test_slice_backward.py b/tests/test_slice_backward.py new file mode 100644 index 000000000..f76839a7a --- /dev/null +++ b/tests/test_slice_backward.py @@ -0,0 +1,158 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param( + (17,), + (5,), + None, + None, + (1,), + (16,), + (3,), + id="one-dimensional", + ), + pytest.param( + (6, 9), + (3, 3), + (12, 1), + (5, 1), + (1, 0), + (6, 9), + (2, 3), + id="strided-two-dimensional", + ), + pytest.param( + (4, 6, 7), + (2, 3, 3), + (60, 9, 1), + (20, 5, 1), + (-4, 1, 0), + (-1, 6, 7), + (2, 2, 3), + id="negative-indices", + ), + pytest.param( + (2, 3, 4, 5), + (2, 2, 3, 3), + (80, 25, 6, 1), + None, + (0, 0, 1, 0), + (2, 3, 4, 5), + (1, 2, 1, 2), + id="four-dimensional", + ), + pytest.param( + (5, 7), + (0, 7), + None, + None, + (3, 0), + (3, 7), + (1, 1), + id="empty-slice", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "grad_input_shape", + "grad_output_shape", + "grad_input_strides", + "grad_output_strides", + "starts", + "ends", + "steps", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_slice_backward( + grad_input_shape, + grad_output_shape, + grad_input_strides, + grad_output_strides, + starts, + ends, + steps, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + grad_input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _slice_backward( + *args, implementation_index=implementation_index + ), + _torch_slice_backward, + (grad_output, starts, ends, steps, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _slice_backward( + grad_output, + starts, + ends, + steps, + grad_input, + *, + implementation_index, +): + infini.ops.slice_backward( + grad_output, + starts, + ends, + steps, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_slice_backward(grad_output, starts, ends, steps, grad_input): + grad_input.zero_() + grad_input[_slice_index(starts, ends, steps)] = grad_output + + return grad_input + + +def _slice_index(starts, ends, steps): + assert len(starts) == len(ends) == len(steps) + + return tuple( + slice(start, end, step) + for start, end, step in zip(starts, ends, steps) + ) diff --git a/tests/test_split.py b/tests/test_split.py new file mode 100644 index 000000000..b224262aa --- /dev/null +++ b/tests/test_split.py @@ -0,0 +1,119 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param( + (12,), + None, + 3, + 0, + id="one-dimensional-even", + ), + pytest.param( + (10,), + None, + 4, + 0, + id="one-dimensional-uneven", + ), + pytest.param( + (4, 12), + None, + 3, + 1, + id="two-dimensional", + ), + pytest.param( + (2, 5, 8), + None, + 3, + -1, + id="negative-dimension", + ), + pytest.param( + (4, 6), + (12, 2), + 2, + 1, + id="strided-input", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, split_size, dim", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_split( + input_shape, + input_strides, + split_size, + dim, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, input_strides, dtype=dtype, device=device + ) + output_shapes = tuple( + output.shape + for output in torch.split(input, split_size, dim=dim) + ) + outs = [ + empty_strided(shape, None, dtype=dtype, device=device) + for shape in output_shapes + ] + + return Payload( + lambda *args: _split( + *args, + split_size=split_size, + dim=dim, + implementation_index=implementation_index, + ), + lambda *args: _torch_split( + *args, + split_size=split_size, + dim=dim, + ), + (input, outs), + {}, + rtol=rtol, + atol=atol, + ) + + +def _split(input, outs, *, split_size, dim, implementation_index): + infini.ops.split( + input, + split_size, + dim, + outs, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return torch.cat(outs, dim=dim) + + +def _torch_split(input, outs, *, split_size, dim): + for out, expected in zip(outs, torch.split(input, split_size, dim=dim)): + out.copy_(expected) + + return torch.cat(outs, dim=dim) diff --git a/tests/test_stack_backward.py b/tests/test_stack_backward.py new file mode 100644 index 000000000..928a6c367 --- /dev/null +++ b/tests/test_stack_backward.py @@ -0,0 +1,136 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, num_inputs, dim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (2, 5), + 3, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 5), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3), + 3, + 1, + (20, 5, 1), + (5, 1), + id="strided-gradients", + ), +) + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "input_shape", + "num_inputs", + "dim", + "grad_output_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_stack_backward( + input_shape, + num_inputs, + dim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape + (0,)) + grad_output_shape = ( + input_shape[:normalized_dim] + + (num_inputs,) + + input_shape[normalized_dim:] + ) + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_inputs = [ + empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + for _ in range(num_inputs) + ] + for grad_input in grad_inputs: + grad_input.fill_(17) + + return Payload( + lambda *args: _stack_backward( + *args, + implementation_index=implementation_index, + ), + _torch_stack_backward, + (grad_output, dim, grad_inputs), + {}, + rtol=rtol, + atol=atol, + ) + + +def _stack_backward( + grad_output, + dim, + grad_inputs, + *, + implementation_index, +): + infini.ops.stack_backward( + grad_output, + dim, + grad_inputs, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return torch.stack(grad_inputs, dim=dim) + + +def _torch_stack_backward(grad_output, dim, grad_inputs): + for grad_input, expected in zip( + grad_inputs, torch.unbind(grad_output, dim=dim) + ): + grad_input.copy_(expected) + + return torch.stack(grad_inputs, dim=dim) \ No newline at end of file diff --git a/tests/test_sub_backward.py b/tests/test_sub_backward.py new file mode 100644 index 000000000..868dda678 --- /dev/null +++ b/tests/test_sub_backward.py @@ -0,0 +1,87 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), None, None, None, id="two-dimensional"), + pytest.param((2, 3, 4), None, None, None, id="three-dimensional"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_sub_backward( + shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) + grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _sub_backward( + *args, + implementation_index=implementation_index, + ), + _torch_sub_backward, + (grad_output, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _sub_backward(grad_output, grad_a, grad_b, *, implementation_index): + infini.ops.sub_backward( + grad_output, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +def _torch_sub_backward(grad_output, grad_a, grad_b): + grad_a.copy_(grad_output) + grad_b.copy_(-grad_output) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) diff --git a/tests/test_transpose.py b/tests/test_transpose.py new file mode 100644 index 000000000..aeecc5430 --- /dev/null +++ b/tests/test_transpose.py @@ -0,0 +1,128 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, input_strides, dim0, dim1, out_strides) +_TEST_CASES = ( + pytest.param( + (2, 3), + None, + 0, + 1, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + None, + 0, + 2, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param( + (2, 3, 4), + None, + -1, + -2, + None, + id="negative-dimensions", + ), + pytest.param( + (2, 3, 4), + None, + 1, + 1, + None, + id="same-dimension", + ), + pytest.param( + (2, 3, 4), + (20, 6, 1), + 0, + 2, + (10, 3, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_transpose( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + out_shape = list(input_shape) + normalized_dim0 = dim0 % len(input_shape) + normalized_dim1 = dim1 % len(input_shape) + out_shape[normalized_dim0], out_shape[normalized_dim1] = ( + out_shape[normalized_dim1], + out_shape[normalized_dim0], + ) + out = empty_strided( + tuple(out_shape), + out_strides, + dtype=dtype, + device=device, + ) + out.fill_(17) + + return Payload( + lambda *args: _transpose( + *args, + implementation_index=implementation_index, + ), + _torch_transpose, + (input, dim0, dim1, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose(input, dim0, dim1, out, *, implementation_index): + infini.ops.transpose( + input, + dim0, + dim1, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return out + + +def _torch_transpose(input, dim0, dim1, out): + out.copy_(torch.transpose(input, dim0, dim1)) + + return out diff --git a/tests/test_transpose_backward.py b/tests/test_transpose_backward.py new file mode 100644 index 000000000..78248f07b --- /dev/null +++ b/tests/test_transpose_backward.py @@ -0,0 +1,141 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (grad_input_shape, dim0, dim1, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (2, 3), + 0, + 1, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + None, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param( + (2, 3, 4), + -1, + -2, + None, + None, + id="negative-dimensions", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="same-dimension", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + (10, 3, 1), + (20, 6, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "grad_input_shape", + "dim0", + "dim1", + "grad_output_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_transpose_backward( + grad_input_shape, + dim0, + dim1, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output_shape = list(grad_input_shape) + normalized_dim0 = dim0 % len(grad_input_shape) + normalized_dim1 = dim1 % len(grad_input_shape) + grad_output_shape[normalized_dim0], grad_output_shape[normalized_dim1] = ( + grad_output_shape[normalized_dim1], + grad_output_shape[normalized_dim0], + ) + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + grad_input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _transpose_backward( + *args, + implementation_index=implementation_index, + ), + _torch_transpose_backward, + (grad_output, dim0, dim1, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + *, + implementation_index, +): + infini.ops.transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_transpose_backward(grad_output, dim0, dim1, grad_input): + grad_input.copy_(torch.transpose(grad_output, dim0, dim1)) + + return grad_input diff --git a/tests/test_vocab_parallel_cross_entropy_backward.py b/tests/test_vocab_parallel_cross_entropy_backward.py new file mode 100644 index 000000000..3747ef4f0 --- /dev/null +++ b/tests/test_vocab_parallel_cross_entropy_backward.py @@ -0,0 +1,197 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream + + +# Format: (rows, vocab_size_local, vocab_size_original, label_smoothing, +# grad_output_mode, grad_input_strides) +_TEST_CASES = ( + pytest.param( + 3, + 4, + 6, + 0.0, + "scalar", + None, + id="scalar-grad-no-smoothing", + ), + pytest.param( + 3, + 4, + 8, + 0.1, + "per-row", + None, + id="per-row-grad-with-smoothing", + ), + pytest.param( + 4, + 5, + 8, + 0.2, + "scalar", + (28, 5), + id="strided-grad-input", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "rows", + "vocab_size_local", + "vocab_size_original", + "label_smoothing", + "grad_output_mode", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 2e-3, 2e-3), + (torch.bfloat16, 2e-2, 2e-2), + ), +) +def test_vocab_parallel_cross_entropy_backward( + rows, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_output_mode, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + softmax_local = torch.rand( + (rows, vocab_size_local), + dtype=dtype, + device=device, + ).softmax(dim=-1) + target_mask = torch.tensor( + [0, 1, 0, 0][:rows], + dtype=dtype, + device=device, + ) + masked_target = torch.tensor( + [1, 2, 0, 3][:rows], + dtype=torch.int64, + device=device, + ) + valid_mask_local = torch.tensor( + [1, 1, 0, 1, 1][:vocab_size_local], + dtype=dtype, + device=device, + ) + if grad_output_mode == "scalar": + grad_output = torch.tensor(1.25, dtype=dtype, device=device) + else: + grad_output = torch.linspace( + 0.5, + 1.5, + rows, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + (rows, vocab_size_local), + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _vocab_parallel_cross_entropy_backward( + *args, + implementation_index=implementation_index, + ), + _torch_vocab_parallel_cross_entropy_backward, + ( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + ), + {}, + rtol=rtol, + atol=atol, + ) + + +def _vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + *, + implementation_index, +): + infini.ops.vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, +): + rows = softmax_local.shape[0] + dloss = grad_output.reshape(-1) + if dloss.numel() == 1: + dloss = dloss.expand(rows) + + gradient = softmax_local * dloss.reshape(rows, 1) + valid_rows = 1 - target_mask + row_scale = dloss * (1 - label_smoothing) * valid_rows + target_one_hot = torch.zeros_like(gradient) + target_one_hot.scatter_(1, masked_target.reshape(rows, 1), 1) + gradient = gradient - target_one_hot * row_scale.reshape(rows, 1) + + smoothing_term = label_smoothing / vocab_size_original + gradient = gradient - ( + dloss.reshape(rows, 1) + * smoothing_term + * valid_mask_local.reshape(1, vocab_size_local) + ) + gradient = gradient * valid_mask_local.reshape(1, vocab_size_local) + grad_input.copy_(gradient) + + return grad_input From 4d663a54f66ed556c6c67ed2a3d3b21c11d264dc Mon Sep 17 00:00:00 2001 From: zhoujf1 <2047484072@qq.com> Date: Wed, 4 Nov 2026 15:11:33 +0800 Subject: [PATCH 2/4] fix: correct ATen operator behavior and validation --- src/base/accumulate_grad.h | 18 ++++- src/base/adam_accumulate_grad.h | 55 +++++++++++--- src/base/add_backward.h | 25 +++++-- src/base/cross_entropy.h | 19 +++-- src/base/cross_entropy_backward.h | 12 +++- src/base/embedding_backward.h | 34 +++++++-- src/base/masked_fill_backward.h | 31 +++++--- src/base/mean_backward.h | 25 +++++-- src/base/mul_backward.h | 20 +++++- src/base/pow_backward.h | 34 +++++++-- src/base/repeat_interleave.h | 32 ++++++--- src/base/repeat_interleave_backward.h | 33 ++++++--- src/base/rsqrt_backward.h | 23 +++++- src/base/slice.h | 36 ++++++++-- src/base/slice_backward.h | 36 ++++++++-- src/base/split.h | 43 +++++++---- src/base/stack_backward.h | 34 ++++++--- src/base/sub_backward.h | 25 +++++-- src/base/transpose.h | 25 +++++-- src/base/transpose_backward.h | 28 +++++--- .../vocab_parallel_cross_entropy_backward.h | 46 ++++++++++-- .../ops/accumulate_grad/accumulate_grad.cc | 21 +++--- .../ops/accumulate_grad/accumulate_grad.h | 4 +- .../adam_accumulate_grad.cc | 56 ++++++++------- .../adam_accumulate_grad.h | 17 +++-- src/torch/ops/add_backward/add_backward.cc | 65 +++++------------ src/torch/ops/add_backward/add_backward.h | 2 +- src/torch/ops/cross_entropy/cross_entropy.cc | 30 ++++---- src/torch/ops/cross_entropy/cross_entropy.h | 4 +- .../cross_entropy_backward.cc | 46 ++++++------ .../cross_entropy_backward.h | 3 +- .../embedding_backward/embedding_backward.cc | 55 +++++++------- .../embedding_backward/embedding_backward.h | 5 +- .../masked_fill_backward.cc | 28 +++----- .../masked_fill_backward.h | 9 ++- src/torch/ops/mean_backward/mean_backward.cc | 36 ++++------ src/torch/ops/mean_backward/mean_backward.h | 11 ++- src/torch/ops/mul_backward/mul_backward.cc | 72 +++++-------------- src/torch/ops/mul_backward/mul_backward.h | 2 +- src/torch/ops/pow_backward/pow_backward.cc | 47 ++++++------ src/torch/ops/pow_backward/pow_backward.h | 7 +- .../repeat_interleave/repeat_interleave.cc | 21 +++--- .../ops/repeat_interleave/repeat_interleave.h | 5 +- .../repeat_interleave_backward.cc | 32 ++++----- .../repeat_interleave_backward.h | 9 ++- .../ops/rsqrt_backward/rsqrt_backward.cc | 34 ++++----- src/torch/ops/rsqrt_backward/rsqrt_backward.h | 4 +- src/torch/ops/slice/slice.cc | 35 +++++---- src/torch/ops/slice/slice.h | 8 +-- .../ops/slice_backward/slice_backward.cc | 34 ++++----- src/torch/ops/slice_backward/slice_backward.h | 13 ++-- src/torch/ops/split/split.cc | 25 +++---- src/torch/ops/split/split.h | 10 +-- .../ops/stack_backward/stack_backward.cc | 37 ++++------ src/torch/ops/stack_backward/stack_backward.h | 8 +-- src/torch/ops/sub_backward/sub_backward.cc | 66 +++++------------ src/torch/ops/sub_backward/sub_backward.h | 2 +- src/torch/ops/transpose/transpose.cc | 21 +++--- src/torch/ops/transpose/transpose.h | 10 +-- .../transpose_backward/transpose_backward.cc | 37 +++++----- .../transpose_backward/transpose_backward.h | 2 +- .../vocab_parallel_cross_entropy_backward.cc | 40 ++++------- .../vocab_parallel_cross_entropy_backward.h | 12 ++-- tests/test_accumulate_grad.py | 2 +- tests/test_adam_accumulate_grad.py | 51 +++++++------ tests/test_cross_entropy.py | 8 ++- tests/test_cross_entropy_backward.py | 8 ++- tests/test_pow_backward.py | 16 ++--- tests/test_rsqrt_backward.py | 5 +- tests/test_slice.py | 8 +-- tests/test_slice_backward.py | 7 +- tests/test_split.py | 7 +- tests/test_stack_backward.py | 11 ++- 73 files changed, 968 insertions(+), 774 deletions(-) diff --git a/src/base/accumulate_grad.h b/src/base/accumulate_grad.h index 143da3cb7..65890ad1c 100644 --- a/src/base/accumulate_grad.h +++ b/src/base/accumulate_grad.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ #include +#include #include "operator.h" @@ -18,18 +19,29 @@ class AccumulateGrad : public Operator { tensor_type_{tensor.dtype()}, rate_{rate}, device_index_{tensor.device().index()} { + AssertTrainingDevices({gradient, tensor}); assert(gradient.shape() == tensor.shape() && "`AccumulateGrad` requires `gradient` and `tensor` to share shape"); assert(gradient_type_ == tensor_type_ && "`AccumulateGrad` requires `gradient` and `tensor` to share dtype"); - assert(gradient.device() == tensor.device() && - "`AccumulateGrad` requires `gradient` and `tensor` on the same device"); + assert( + gradient.device() == tensor.device() && + "`AccumulateGrad` requires `gradient` and `tensor` on the same device"); } virtual void operator()(const Tensor gradient, const double rate, Tensor tensor) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + Tensor::Shape gradient_shape_; Tensor::Strides gradient_strides_; DataType gradient_type_; @@ -42,4 +54,4 @@ class AccumulateGrad : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ diff --git a/src/base/adam_accumulate_grad.h b/src/base/adam_accumulate_grad.h index ab9b79ce5..b753c39f1 100644 --- a/src/base/adam_accumulate_grad.h +++ b/src/base/adam_accumulate_grad.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -10,9 +11,9 @@ namespace infini::ops { class AdamAccumulateGrad : public Operator { public: - AdamAccumulateGrad(const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, - const double beta2, const double eps, const int64_t step) + AdamAccumulateGrad(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, const double eps, + const int64_t step, Tensor param, Tensor m, Tensor v) : grad_shape_{grad.shape()}, grad_strides_{grad.strides()}, grad_type_{grad.dtype()}, @@ -31,24 +32,60 @@ class AdamAccumulateGrad : public Operator { eps_{eps}, step_{step}, device_index_{param.device().index()} { + AssertTrainingDevices({grad, param, m, v}); assert(grad.shape() == param.shape() && param.shape() == m.shape() && m.shape() == v.shape() && "`AdamAccumulateGrad` requires grad/param/m/v to share shape"); - assert(grad_type_ == param_type_ && param_type_ == m_type_ && - m_type_ == v_type_ && "`AdamAccumulateGrad` requires matching dtypes"); + assert(IsTrainingFloat(param_type_) && grad_type_ == param_type_ && + "Adam parameters and gradients require matching floating dtypes."); + assert(m_type_ == v_type_ && + (m_type_ == param_type_ || (param_type_ != DataType::kFloat64 && + m_type_ == DataType::kFloat32)) && + "Adam moments must match the parameter dtype or use float32."); assert(grad.device() == param.device() && "`AdamAccumulateGrad` requires grad/param on the same device"); - assert(0.0 < beta1_ && beta1_ < 1.0 && 0.0 < beta2_ && beta2_ < 1.0 && - "`AdamAccumulateGrad` requires beta1/beta2 in (0, 1)"); + assert(0.0 <= beta1_ && beta1_ < 1.0 && 0.0 <= beta2_ && beta2_ < 1.0 && + "Adam beta values must be in [0, 1)."); + assert(learning_rate_ >= 0.0 && eps_ > 0.0 && + "Adam requires non-negative learning rate and positive epsilon."); assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1"); } + [[deprecated("Use grad, attributes, param, m, v argument order.")]] + AdamAccumulateGrad(const Tensor grad, Tensor param, Tensor m, Tensor v, + const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step) + : AdamAccumulateGrad{grad, learning_rate, beta1, beta2, eps, + step, param, m, v} {} + + [[deprecated("Use grad, attributes, param, m, v argument order.")]] virtual void operator()(const Tensor grad, Tensor param, Tensor m, Tensor v, const double learning_rate, const double beta1, const double beta2, const double eps, - const int64_t step) const = 0; + const int64_t step) const { + operator()(grad, learning_rate, beta1, beta2, eps, step, param, m, v); + } + + virtual void operator()(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, + const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static bool IsTrainingFloat(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; + } + Tensor::Shape grad_shape_; Tensor::Strides grad_strides_; DataType grad_type_; @@ -71,4 +108,4 @@ class AdamAccumulateGrad : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ diff --git a/src/base/add_backward.h b/src/base/add_backward.h index 929cb936d..491354608 100644 --- a/src/base/add_backward.h +++ b/src/base/add_backward.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_ADD_BACKWARD_H_ #include +#include #include "operator.h" @@ -20,11 +21,13 @@ class AddBackward : public Operator { grad_b_strides_{grad_b.strides()}, grad_b_type_{grad_b.dtype()}, device_index_{grad_a.device().index()} { + AssertTrainingDevices({grad_output, grad_a, grad_b}); assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && "`AddBackward` requires matching dtypes"); - assert(grad_output.device() == grad_a.device() && - grad_a.device() == grad_b.device() && - "`AddBackward` requires grad_output/grad_a/grad_b on the same device"); + assert( + grad_output.device() == grad_a.device() && + grad_a.device() == grad_b.device() && + "`AddBackward` requires grad_output/grad_a/grad_b on the same device"); ValidateBroadcast(grad_a, grad_output); ValidateBroadcast(grad_b, grad_output); } @@ -33,13 +36,23 @@ class AddBackward : public Operator { Tensor grad_b) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static void ValidateBroadcast(const Tensor input, const Tensor out) { assert(input.ndim() <= out.ndim() && "`AddBackward` input rank must not exceed grad_output rank"); const auto offset = out.ndim() - input.ndim(); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`AddBackward` input must broadcast (right-aligned) to grad_output"); + assert( + (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`AddBackward` input must broadcast (right-aligned) to grad_output"); } } @@ -57,4 +70,4 @@ class AddBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ diff --git a/src/base/cross_entropy.h b/src/base/cross_entropy.h index 7bec3794c..ac5ffadf0 100644 --- a/src/base/cross_entropy.h +++ b/src/base/cross_entropy.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -10,8 +11,6 @@ namespace infini::ops { class CrossEntropy : public Operator { public: - // This operator follows ATen's default CrossEntropyLoss contract: - // dimension 1 stores the class logits and -100 is the ignored target. static constexpr int64_t kIgnoreIndex = -100; CrossEntropy(const Tensor input, const Tensor target, Tensor out) @@ -25,6 +24,7 @@ class CrossEntropy : public Operator { out_strides_{out.strides()}, out_type_{out.dtype()}, device_index_{out.device().index()} { + AssertTrainingDevices({input, target, out}); assert(input.ndim() >= 2 && "`CrossEntropy` requires input rank >= 2"); assert(input.device() == target.device() && target.device() == out.device() && @@ -34,7 +34,8 @@ class CrossEntropy : public Operator { input_type_ == DataType::kFloat32 || input_type_ == DataType::kFloat64) && "`CrossEntropy` requires floating-point logits"); - assert((target_type_ == DataType::kInt64 || target_type_ == DataType::kInt32 || + assert((target_type_ == DataType::kInt64 || + target_type_ == DataType::kInt32 || target_type_ == DataType::kUInt8) && "`CrossEntropy` supports int64/int32/uint8 targets"); assert(target.ndim() == input.ndim() - 1 && @@ -45,13 +46,23 @@ class CrossEntropy : public Operator { "`CrossEntropy` target shape mismatch"); } assert(out.ndim() == 0 && "`CrossEntropy` output must be a 0-dim scalar"); - assert(out_type_ == input_type_ && "`CrossEntropy` output dtype must match input"); + assert(out_type_ == input_type_ && + "`CrossEntropy` output dtype must match input"); } virtual void operator()(const Tensor input, const Tensor target, Tensor out) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + Tensor::Shape input_shape_; Tensor::Strides input_strides_; DataType input_type_; diff --git a/src/base/cross_entropy_backward.h b/src/base/cross_entropy_backward.h index 87cc8b222..2ae87cfab 100644 --- a/src/base/cross_entropy_backward.h +++ b/src/base/cross_entropy_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -10,7 +11,6 @@ namespace infini::ops { class CrossEntropyBackward : public Operator { public: - // Keep this synchronized with CrossEntropy's forward contract. static constexpr int64_t kIgnoreIndex = -100; CrossEntropyBackward(const Tensor input, const Tensor target, @@ -28,6 +28,7 @@ class CrossEntropyBackward : public Operator { grad_input_strides_{grad_input.strides()}, grad_input_type_{grad_input.dtype()}, device_index_{grad_input.device().index()} { + AssertTrainingDevices({input, target, grad_output, grad_input}); assert(input.ndim() >= 2 && "`CrossEntropyBackward` requires input rank >= 2"); assert(input.device() == target.device() && @@ -64,6 +65,15 @@ class CrossEntropyBackward : public Operator { Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + Tensor::Shape input_shape_; Tensor::Strides input_strides_; DataType input_type_; diff --git a/src/base/embedding_backward.h b/src/base/embedding_backward.h index 43fcef1d0..17212202f 100644 --- a/src/base/embedding_backward.h +++ b/src/base/embedding_backward.h @@ -2,6 +2,8 @@ #define INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ #include +#include +#include #include "operator.h" @@ -27,21 +29,30 @@ class EmbeddingBackward : public Operator { scale_grad_by_freq_{scale_grad_by_freq}, sparse_{sparse}, device_index_{grad_weight.device().index()} { - assert((indices_type_ == DataType::kInt32 || indices_type_ == DataType::kInt64) && + AssertTrainingDevices({grad_output, indices, grad_weight}); + assert(!(sparse_ && scale_grad_by_freq_) && + "Sparse embedding does not support frequency scaling."); + assert((indices_type_ == DataType::kInt32 || + indices_type_ == DataType::kInt64) && "`EmbeddingBackward` requires int32 or int64 `indices`"); assert(grad_output_type_ == grad_weight_type_ && - "`EmbeddingBackward` requires `grad_output` and `grad_weight` to have the same dtype"); - assert(grad_weight.ndim() == 2 && "`EmbeddingBackward` requires 2D `grad_weight`"); + "`EmbeddingBackward` requires `grad_output` and `grad_weight` to " + "have the same dtype"); + assert(grad_weight.ndim() == 2 && + "`EmbeddingBackward` requires 2D `grad_weight`"); assert(grad_weight.size(0) == num_weights_ && - "`EmbeddingBackward` requires `grad_weight` row count to equal `num_weights`"); + "`EmbeddingBackward` requires `grad_weight` row count to equal " + "`num_weights`"); assert(grad_output.ndim() == indices.ndim() + 1 && - "`EmbeddingBackward` requires `grad_output` rank to be `indices` rank + 1"); + "`EmbeddingBackward` requires `grad_output` rank to be `indices` " + "rank + 1"); for (Tensor::Size i = 0; i < indices.ndim(); ++i) { assert(grad_output.size(i) == indices.size(i) && "`EmbeddingBackward` requires matching leading dims"); } assert(grad_output.size(-1) == grad_weight.size(1) && - "`EmbeddingBackward` requires last dim of `grad_output` to equal embedding dim"); + "`EmbeddingBackward` requires last dim of `grad_output` to equal " + "embedding dim"); } virtual void operator()(const Tensor grad_output, const Tensor indices, @@ -50,6 +61,15 @@ class EmbeddingBackward : public Operator { Tensor grad_weight) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + Tensor::Shape grad_output_shape_; Tensor::Strides grad_output_strides_; DataType grad_output_type_; @@ -68,4 +88,4 @@ class EmbeddingBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ diff --git a/src/base/masked_fill_backward.h b/src/base/masked_fill_backward.h index 1b7e8f2f4..2dadf51d3 100644 --- a/src/base/masked_fill_backward.h +++ b/src/base/masked_fill_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -22,10 +23,13 @@ class MaskedFillBackward : public Operator { grad_input_strides_{grad_input.strides()}, grad_input_type_{grad_input.dtype()}, device_index_{grad_input.device().index()} { - assert(grad_output.shape() == grad_input.shape() && - "`MaskedFillBackward` requires grad_output/grad_input to share shape"); - assert(grad_output_type_ == grad_input_type_ && - "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); + AssertTrainingDevices({grad_output, mask, grad_input}); + assert( + grad_output.shape() == grad_input.shape() && + "`MaskedFillBackward` requires grad_output/grad_input to share shape"); + assert( + grad_output_type_ == grad_input_type_ && + "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); assert(mask_type_ == DataType::kBool && "`MaskedFillBackward` requires a bool `mask`"); assert(mask.ndim() <= grad_output.ndim() && @@ -33,8 +37,9 @@ class MaskedFillBackward : public Operator { tail_aligned_ = AlignsAt(mask, grad_output, /*tail=*/true); if (!tail_aligned_) { - assert(AlignsAt(mask, grad_output, /*tail=*/false) && - "`MaskedFillBackward` mask must align to leading or trailing dims"); + assert( + AlignsAt(mask, grad_output, /*tail=*/false) && + "`MaskedFillBackward` mask must align to leading or trailing dims"); } } @@ -42,7 +47,17 @@ class MaskedFillBackward : public Operator { Tensor grad_input) const = 0; protected: - static bool AlignsAt(const Tensor mask, const Tensor tensor, const bool tail) { + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static bool AlignsAt(const Tensor mask, const Tensor tensor, + const bool tail) { const auto offset = tail ? tensor.ndim() - mask.ndim() : 0; for (Tensor::Size i = 0; i < mask.ndim(); ++i) { const auto dim = tensor.size(i + offset); @@ -68,4 +83,4 @@ class MaskedFillBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ diff --git a/src/base/mean_backward.h b/src/base/mean_backward.h index 06348c17e..137f10ad1 100644 --- a/src/base/mean_backward.h +++ b/src/base/mean_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -21,6 +22,7 @@ class MeanBackward : public Operator { dim_{NormalizeDim(dim, grad_input.ndim())}, keep_dim_{keep_dim}, device_index_{grad_input.device().index()} { + AssertTrainingDevices({grad_output, grad_input}); assert(grad_output_type_ == grad_input_type_ && "`MeanBackward` requires matching dtypes"); assert(grad_input.ndim() >= 1 && "`MeanBackward` requires ndim >= 1"); @@ -30,14 +32,16 @@ class MeanBackward : public Operator { "`MeanBackward` keep_dim=true requires size-1 dim in grad_output"); for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { if (i == dim_) continue; - assert(grad_output.size(i) == grad_input.size(i) && "`MeanBackward` shape mismatch"); + assert(grad_output.size(i) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); } } else { assert(grad_output.ndim() == grad_input.ndim() - 1 && "`MeanBackward` keep_dim=false requires rank-1 grad_output"); for (Tensor::Size i = 0, s = 0; i < grad_input.ndim(); ++i) { if (i == dim_) continue; - assert(grad_output.size(s) == grad_input.size(i) && "`MeanBackward` shape mismatch"); + assert(grad_output.size(s) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); ++s; } } @@ -47,11 +51,20 @@ class MeanBackward : public Operator { const bool keep_dim, Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`MeanBackward` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape grad_output_shape_; diff --git a/src/base/mul_backward.h b/src/base/mul_backward.h index b73f61a84..c606f8ba5 100644 --- a/src/base/mul_backward.h +++ b/src/base/mul_backward.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_MUL_BACKWARD_H_ #include +#include #include "operator.h" @@ -27,12 +28,15 @@ class MulBackward : public Operator { grad_b_strides_{grad_b.strides()}, grad_b_type_{grad_b.dtype()}, device_index_{grad_a.device().index()} { + AssertTrainingDevices({grad_output, a, b, grad_a, grad_b}); assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && "`MulBackward` requires matching grad dtypes"); assert(a_type_ == grad_output_type_ && b_type_ == grad_output_type_ && "`MulBackward` requires a/b dtype to match grad_output dtype"); assert(grad_output.device() == a.device() && a.device() == b.device() && "`MulBackward` requires all tensors on the same device"); + assert(grad_a.shape() == a.shape() && grad_b.shape() == b.shape() && + "Multiplication gradients must match their input shapes."); ValidateBroadcast(a, grad_output); ValidateBroadcast(b, grad_output); ValidateBroadcast(grad_a, grad_output); @@ -44,13 +48,23 @@ class MulBackward : public Operator { Tensor grad_b) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static void ValidateBroadcast(const Tensor input, const Tensor out) { assert(input.ndim() <= out.ndim() && "`MulBackward` input rank must not exceed grad_output rank"); const auto offset = out.ndim() - input.ndim(); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`MulBackward` input must broadcast (right-aligned) to grad_output"); + assert( + (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`MulBackward` input must broadcast (right-aligned) to grad_output"); } } @@ -74,4 +88,4 @@ class MulBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ diff --git a/src/base/pow_backward.h b/src/base/pow_backward.h index b5f3172d6..8ac2d20b7 100644 --- a/src/base/pow_backward.h +++ b/src/base/pow_backward.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_POW_BACKWARD_H_ #include +#include #include "operator.h" @@ -23,13 +24,18 @@ class PowBackward : public Operator { scalar_{scalar}, scalar_is_base_{scalar_is_base}, device_index_{grad_input.device().index()} { - assert(input.shape() == grad_output.shape() && - grad_output.shape() == grad_input.shape() && - "`PowBackward` requires input/grad_output/grad_input to share shape"); - assert(input_type_ == grad_output_type_ && grad_output_type_ == grad_input_type_ && + AssertTrainingDevices({grad_output, input, grad_input}); + assert( + input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`PowBackward` requires input/grad_output/grad_input to share shape"); + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && "`PowBackward` requires matching dtypes"); - assert(!scalar_is_base_ || scalar_ > 0.0 && - "`PowBackward` scalar base must be positive for log(scalar)"); + assert((!scalar_is_base_ || scalar_ > 0.0) && + "`PowBackward` scalar base must be positive for log(scalar)."); + assert(IsTrainingFloat(input_type_) && + "Power gradients require floating dtypes."); } virtual void operator()(const Tensor grad_output, const Tensor input, @@ -37,6 +43,20 @@ class PowBackward : public Operator { Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static bool IsTrainingFloat(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; + } + Tensor::Shape grad_output_shape_; Tensor::Strides grad_output_strides_; DataType grad_output_type_; @@ -53,4 +73,4 @@ class PowBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ diff --git a/src/base/repeat_interleave.h b/src/base/repeat_interleave.h index a03624440..e80aad14a 100644 --- a/src/base/repeat_interleave.h +++ b/src/base/repeat_interleave.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -21,11 +22,17 @@ class RepeatInterleave : public Operator { repeats_{repeats}, dim_{NormalizeDim(dim, input.ndim())}, device_index_{out.device().index()} { - assert(repeats_ > 0 && "`RepeatInterleave` requires positive `repeats`"); - assert(input_type_ == out_type_ && "`RepeatInterleave` requires same dtype"); + AssertTrainingDevices({input, out}); + assert(repeats_ >= 0 && + "`RepeatInterleave` requires non-negative repeats."); + assert(input.ndim() == out.ndim() && "RepeatInterleave ranks must match."); + assert(input_type_ == out_type_ && + "`RepeatInterleave` requires same dtype"); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - const auto expect = (i == dim_) ? input.size(i) * repeats_ : input.size(i); - assert(out.size(i) == expect && "`RepeatInterleave` output shape mismatch"); + const auto expect = + (i == dim_) ? input.size(i) * repeats_ : input.size(i); + assert(out.size(i) == expect && + "`RepeatInterleave` output shape mismatch"); } } @@ -33,11 +40,20 @@ class RepeatInterleave : public Operator { const int64_t dim, Tensor out) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`RepeatInterleave` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape input_shape_; diff --git a/src/base/repeat_interleave_backward.h b/src/base/repeat_interleave_backward.h index 288284742..bee70b8e9 100644 --- a/src/base/repeat_interleave_backward.h +++ b/src/base/repeat_interleave_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -20,6 +21,7 @@ class RepeatInterleaveBackward : public Operator { grad_input_type_{grad_input.dtype()}, dim_{NormalizeDim(dim, grad_input.ndim())}, device_index_{grad_input.device().index()} { + AssertTrainingDevices({grad_output, grad_input}); assert(grad_output_type_ == grad_input_type_ && "`RepeatInterleaveBackward` requires same dtype"); assert(grad_output.ndim() == grad_input.ndim() && @@ -29,22 +31,35 @@ class RepeatInterleaveBackward : public Operator { assert(grad_output.size(i) == grad_input.size(i) && "`RepeatInterleaveBackward` shape mismatch on non-repeated dims"); } - assert(grad_input.size(dim_) > 0 && - "`RepeatInterleaveBackward` requires non-empty repeated dim"); - repeats_ = grad_output.size(dim_) / grad_input.size(dim_); - assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && - "`RepeatInterleaveBackward` grad_output dim must be divisible"); + if (grad_input.size(dim_) == 0) { + assert(grad_output.size(dim_) == 0 && + "Empty repeated dimensions must match."); + repeats_ = 0; + } else { + repeats_ = grad_output.size(dim_) / grad_input.size(dim_); + assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && + "RepeatInterleaveBackward grad_output dim must be divisible."); + } } virtual void operator()(const Tensor grad_output, const int64_t dim, Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`RepeatInterleaveBackward` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape grad_output_shape_; diff --git a/src/base/rsqrt_backward.h b/src/base/rsqrt_backward.h index d8e59c64e..145569cbb 100644 --- a/src/base/rsqrt_backward.h +++ b/src/base/rsqrt_backward.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_RSQRT_BACKWARD_H_ #include +#include #include "operator.h" @@ -20,12 +21,14 @@ class RsqrtBackward : public Operator { grad_input_strides_{grad_input.strides()}, grad_input_type_{grad_input.dtype()}, device_index_{grad_input.device().index()} { + AssertTrainingDevices({grad_output, input, grad_input}); assert(input.shape() == grad_output.shape() && grad_output.shape() == grad_input.shape() && "`RsqrtBackward` requires matching shapes"); - assert(input_type_ == grad_output_type_ && grad_output_type_ == grad_input_type_ && + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && "`RsqrtBackward` requires matching dtypes"); - assert(input_type_ != DataType::kInt32 && input_type_ != DataType::kInt64 && + assert(IsTrainingFloat(input_type_) && "`RsqrtBackward` requires floating point dtypes"); } @@ -33,6 +36,20 @@ class RsqrtBackward : public Operator { Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static bool IsTrainingFloat(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; + } + Tensor::Shape grad_output_shape_; Tensor::Strides grad_output_strides_; DataType grad_output_type_; @@ -47,4 +64,4 @@ class RsqrtBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ diff --git a/src/base/slice.h b/src/base/slice.h index 1721fbb3e..8f456888e 100644 --- a/src/base/slice.h +++ b/src/base/slice.h @@ -1,8 +1,10 @@ #ifndef INFINI_OPS_BASE_SLICE_H_ #define INFINI_OPS_BASE_SLICE_H_ +#include #include #include +#include #include #include "operator.h" @@ -24,15 +26,16 @@ class Slice : public Operator { ends_{ends}, steps_{steps}, device_index_{out.device().index()} { + AssertTrainingDevices({input, out}); assert(starts_.size() == input.ndim() && ends_.size() == input.ndim() && steps_.size() == input.ndim() && "`Slice` requires starts/ends/steps to match input rank"); assert(out.ndim() == input.ndim() && "`Slice` requires matching rank"); assert(input_type_ == out_type_ && "`Slice` requires same dtype"); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert(steps_[i] > 0 && "`Slice` requires positive steps (InfiniTrain semantics)"); - assert(starts_[i] <= ends_[i] && "`Slice` requires start <= end"); - const auto expected = (ends_[i] - starts_[i] + steps_[i] - 1) / steps_[i]; + assert(steps_[i] > 0 && "`Slice` requires positive steps"); + const auto expected = + TrainingSliceLength(starts_[i], ends_[i], steps_[i], input.size(i)); assert(out.size(i) == expected && "`Slice` output dim must equal ceil((end - start) / step)"); } @@ -40,9 +43,32 @@ class Slice : public Operator { virtual void operator()(const Tensor input, const std::vector starts, const std::vector ends, - const std::vector steps, Tensor out) const = 0; + const std::vector steps, + Tensor out) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static int64_t TrainingSliceLength(int64_t start, int64_t end, int64_t step, + Tensor::Size size) { + assert(step > 0 && "Training slices require a positive step."); + const auto length = static_cast(size); + const auto normalize = [length](int64_t index) { + if (index < 0) index += length; + return std::clamp(index, 0, length); + }; + start = normalize(start); + end = normalize(end); + return start >= end ? 0 : 1 + (end - start - 1) / step; + } + Tensor::Shape input_shape_; Tensor::Strides input_strides_; DataType input_type_; @@ -57,4 +83,4 @@ class Slice : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_SLICE_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_SLICE_H_ diff --git a/src/base/slice_backward.h b/src/base/slice_backward.h index 540db763f..429559e14 100644 --- a/src/base/slice_backward.h +++ b/src/base/slice_backward.h @@ -1,8 +1,10 @@ #ifndef INFINI_OPS_BASE_SLICE_BACKWARD_H_ #define INFINI_OPS_BASE_SLICE_BACKWARD_H_ +#include #include #include +#include #include #include "operator.h" @@ -24,16 +26,20 @@ class SliceBackward : public Operator { ends_{ends}, steps_{steps}, device_index_{grad_input.device().index()} { + AssertTrainingDevices({grad_output, grad_input}); assert(starts_.size() == grad_input.ndim() && - "`SliceBackward` requires starts/ends/steps to match `grad_input` rank"); + ends_.size() == grad_input.ndim() && + steps_.size() == grad_input.ndim() && + "`SliceBackward` requires starts/ends/steps to match `grad_input` " + "rank"); assert(grad_output.ndim() == grad_input.ndim() && "`SliceBackward` requires matching rank"); assert(grad_output_type_ == grad_input_type_ && "`SliceBackward` requires same dtype"); for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { assert(steps_[i] > 0 && "`SliceBackward` requires positive steps"); - assert(starts_[i] <= ends_[i] && "`SliceBackward` requires start <= end"); - const auto expected = (ends_[i] - starts_[i] + steps_[i] - 1) / steps_[i]; + const auto expected = TrainingSliceLength(starts_[i], ends_[i], steps_[i], + grad_input.size(i)); assert(grad_output.size(i) == expected && "`SliceBackward` grad_output dim must equal the slice dim"); } @@ -46,6 +52,28 @@ class SliceBackward : public Operator { Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static int64_t TrainingSliceLength(int64_t start, int64_t end, int64_t step, + Tensor::Size size) { + assert(step > 0 && "Training slices require a positive step."); + const auto length = static_cast(size); + const auto normalize = [length](int64_t index) { + if (index < 0) index += length; + return std::clamp(index, 0, length); + }; + start = normalize(start); + end = normalize(end); + return start >= end ? 0 : 1 + (end - start - 1) / step; + } + Tensor::Shape grad_output_shape_; Tensor::Strides grad_output_strides_; DataType grad_output_type_; @@ -60,4 +88,4 @@ class SliceBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_SLICE_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_SLICE_BACKWARD_H_ diff --git a/src/base/split.h b/src/base/split.h index 7a2d6bdbb..935de3d99 100644 --- a/src/base/split.h +++ b/src/base/split.h @@ -3,6 +3,7 @@ #include #include +#include #include #include "operator.h" @@ -23,26 +24,31 @@ class Split : public Operator { assert(split_size_ > 0 && "`Split` requires positive `split_size`"); assert(!outs_.empty() && "`Split` requires at least one output tensor"); const auto expected = - (input.size(dim_) + split_size_ - 1) / split_size_; - assert(static_cast(outs_.size()) == expected && - "`Split` output count must equal ceil(input_size(dim) / split_size)"); + input.size(dim_) == 0 ? 1 : 1 + (input.size(dim_) - 1) / split_size_; + assert( + static_cast(outs_.size()) == expected && + "`Split` output count must equal ceil(input_size(dim) / split_size)"); for (std::size_t i = 0; i < outs_.size(); ++i) { + AssertTrainingDevices({input, outs_[i]}); assert(outs_[i].ndim() == input.ndim() && "`Split` output rank mismatch"); assert(outs_[i].dtype() == input_type_ && "`Split` dtype mismatch"); for (Tensor::Size d = 0; d < input.ndim(); ++d) { if (d != dim_) { - assert(outs_[i].size(d) == input.size(d) && "`Split` output shape mismatch"); + assert(outs_[i].size(d) == input.size(d) && + "`Split` output shape mismatch"); } } const auto expect_dim = std::min( - split_size_, - static_cast(input.size(dim_)) - - static_cast(i) * split_size_); - assert(outs_[i].size(dim_) == expect_dim && "`Split` output dim size mismatch"); + split_size_, static_cast(input.size(dim_)) - + static_cast(i) * split_size_); + assert(outs_[i].size(dim_) == expect_dim && + "`Split` output dim size mismatch"); - out_shapes_.emplace_back(outs_[i].shape().begin(), outs_[i].shape().end()); - out_strides_.emplace_back(outs_[i].strides().begin(), outs_[i].strides().end()); + out_shapes_.emplace_back(outs_[i].shape().begin(), + outs_[i].shape().end()); + out_strides_.emplace_back(outs_[i].strides().begin(), + outs_[i].strides().end()); out_types_.emplace_back(outs_[i].dtype()); } } @@ -52,11 +58,20 @@ class Split : public Operator { std::vector outs) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`Split` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape input_shape_; diff --git a/src/base/stack_backward.h b/src/base/stack_backward.h index a199d91d6..de4eb381c 100644 --- a/src/base/stack_backward.h +++ b/src/base/stack_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -25,32 +26,45 @@ class StackBackward : public Operator { "`StackBackward` output count must equal grad_output.size(dim)"); for (std::size_t i = 0; i < outs_.size(); ++i) { + AssertTrainingDevices({grad_output, outs_[i]}); assert(outs_[i].ndim() == grad_output.ndim() - 1 && "`StackBackward` output rank must be grad_output rank - 1"); - assert(outs_[i].dtype() == grad_output_type_ && "`StackBackward` dtype mismatch"); + assert(outs_[i].dtype() == grad_output_type_ && + "`StackBackward` dtype mismatch"); for (Tensor::Size d = 0, s = 0; d < grad_output.ndim(); ++d) { if (d == dim_) continue; - assert(outs_[i].size(s) == grad_output.size(d) && "`StackBackward` shape mismatch"); + assert(outs_[i].size(s) == grad_output.size(d) && + "`StackBackward` shape mismatch"); ++s; } - out_shapes_.emplace_back(outs_[i].shape().begin(), outs_[i].shape().end()); + out_shapes_.emplace_back(outs_[i].shape().begin(), + outs_[i].shape().end()); - out_strides_.emplace_back(outs_[i].strides().begin(), outs_[i].strides().end()); + out_strides_.emplace_back(outs_[i].strides().begin(), + outs_[i].strides().end()); - out_types_.push_back(outs_[i].dtype()); + out_types_.push_back(outs_[i].dtype()); } - } virtual void operator()(const Tensor grad_output, const int64_t dim, std::vector outs) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`StackBackward` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape grad_output_shape_; diff --git a/src/base/sub_backward.h b/src/base/sub_backward.h index 0ea897a56..99a996192 100644 --- a/src/base/sub_backward.h +++ b/src/base/sub_backward.h @@ -2,6 +2,7 @@ #define INFINI_OPS_BASE_SUB_BACKWARD_H_ #include +#include #include "operator.h" @@ -20,11 +21,13 @@ class SubBackward : public Operator { grad_b_strides_{grad_b.strides()}, grad_b_type_{grad_b.dtype()}, device_index_{grad_a.device().index()} { + AssertTrainingDevices({grad_output, grad_a, grad_b}); assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && "`SubBackward` requires matching dtypes"); - assert(grad_output.device() == grad_a.device() && - grad_a.device() == grad_b.device() && - "`SubBackward` requires grad_output/grad_a/grad_b on the same device"); + assert( + grad_output.device() == grad_a.device() && + grad_a.device() == grad_b.device() && + "`SubBackward` requires grad_output/grad_a/grad_b on the same device"); ValidateBroadcast(grad_a, grad_output); ValidateBroadcast(grad_b, grad_output); } @@ -33,13 +36,23 @@ class SubBackward : public Operator { Tensor grad_b) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static void ValidateBroadcast(const Tensor input, const Tensor out) { assert(input.ndim() <= out.ndim() && "`SubBackward` input rank must not exceed grad_output rank"); const auto offset = out.ndim() - input.ndim(); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert((input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`SubBackward` input must broadcast (right-aligned) to grad_output"); + assert( + (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && + "`SubBackward` input must broadcast (right-aligned) to grad_output"); } } @@ -57,4 +70,4 @@ class SubBackward : public Operator { } // namespace infini::ops -#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ diff --git a/src/base/transpose.h b/src/base/transpose.h index cce6bdc49..9297fd80a 100644 --- a/src/base/transpose.h +++ b/src/base/transpose.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -21,11 +22,12 @@ class Transpose : public Operator { dim0_{NormalizeDim(dim0, input.ndim())}, dim1_{NormalizeDim(dim1, input.ndim())}, device_index_{out.device().index()} { - assert(input.ndim() >= 2 && "`Transpose` requires at least 2 dims"); - assert(dim0_ != dim1_ && "`Transpose` requires distinct dims"); + AssertTrainingDevices({input, out}); + assert(input.ndim() >= 1 && "Transpose requires a non-scalar input."); + assert(out.ndim() == input.ndim() && "Transpose ranks must match."); assert(input_type_ == out_type_ && "`Transpose` requires same dtype"); for (Tensor::Size i = 0; i < input.ndim(); ++i) { - const auto expect = (i == dim0_) ? input.size(dim1_) + const auto expect = (i == dim0_) ? input.size(dim1_) : (i == dim1_) ? input.size(dim0_) : input.size(i); assert(out.size(i) == expect && "`Transpose` output shape mismatch"); @@ -36,11 +38,20 @@ class Transpose : public Operator { const int64_t dim1, Tensor out) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`Transpose` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape input_shape_; diff --git a/src/base/transpose_backward.h b/src/base/transpose_backward.h index 71b4bfc1d..a4ad75be2 100644 --- a/src/base/transpose_backward.h +++ b/src/base/transpose_backward.h @@ -3,6 +3,7 @@ #include #include +#include #include "operator.h" @@ -21,15 +22,15 @@ class TransposeBackward : public Operator { dim0_{NormalizeDim(dim0, grad_output.ndim())}, dim1_{NormalizeDim(dim1, grad_output.ndim())}, device_index_{grad_input.device().index()} { - assert(grad_output.ndim() >= 2 && - "`TransposeBackward` requires at least 2 dims"); - assert(dim0_ != dim1_ && - "`TransposeBackward` requires distinct dims"); + AssertTrainingDevices({grad_output, grad_input}); + assert(grad_output.ndim() >= 1 && "Transpose requires a non-scalar input."); + assert(grad_input.ndim() == grad_output.ndim() && + "Transpose ranks must match."); assert(grad_output_type_ == grad_input_type_ && "`TransposeBackward` requires same dtype"); for (Tensor::Size i = 0; i < grad_output.ndim(); ++i) { - const auto expect = (i == dim0_) ? grad_output.size(dim1_) + const auto expect = (i == dim0_) ? grad_output.size(dim1_) : (i == dim1_) ? grad_output.size(dim0_) : grad_output.size(i); assert(grad_input.size(i) == expect && @@ -41,11 +42,20 @@ class TransposeBackward : public Operator { const int64_t dim1, Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - assert(ndim > 0 && "`TransposeBackward` requires ndim > 0"); - if (ndim == 0) return 0; - const auto d = dim % static_cast(ndim); - return d < 0 ? d + static_cast(ndim) : d; + const auto rank = static_cast(ndim); + assert(rank > 0 && "Training dimension requires a non-scalar tensor."); + assert(dim >= -rank && dim < rank && "Training dimension is out of range."); + return dim < 0 ? dim + rank : dim; } Tensor::Shape grad_output_shape_; diff --git a/src/base/vocab_parallel_cross_entropy_backward.h b/src/base/vocab_parallel_cross_entropy_backward.h index d11182df5..8dfba74ec 100644 --- a/src/base/vocab_parallel_cross_entropy_backward.h +++ b/src/base/vocab_parallel_cross_entropy_backward.h @@ -1,6 +1,10 @@ #ifndef INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ #define INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#include +#include +#include + #include "operator.h" namespace infini::ops { @@ -35,16 +39,44 @@ class VocabParallelCrossEntropyBackward vocab_size_local_{vocab_size_local}, vocab_size_original_{vocab_size_original}, label_smoothing_{label_smoothing}, - device_index_{grad_input.device().index()} {} + device_index_{grad_input.device().index()} { + AssertTrainingDevices({grad_output, softmax_local, target_mask, + masked_target, valid_mask_local, grad_input}); + assert(IsTrainingFloat(grad_input_type_) && + grad_input_type_ == softmax_type_ && + grad_output_type_ == grad_input_type_ && + "Loss gradients require matching floating dtypes."); + assert(softmax_local.shape() == grad_input.shape() && + "Local softmax and gradient shapes must match."); + assert(vocab_size_local > 0 && vocab_size_original > 0 && + "Vocabulary sizes must be positive."); + assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && + "Label smoothing must be in [0, 1]."); + } - virtual void operator()( - const Tensor grad_output, const Tensor softmax_local, - const Tensor target_mask, const Tensor masked_target, - const Tensor valid_mask_local, const int64_t vocab_size_local, - const int64_t vocab_size_original, const double label_smoothing, - Tensor grad_input) const = 0; + virtual void operator()(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, + const int64_t vocab_size_local, + const int64_t vocab_size_original, + const double label_smoothing, + Tensor grad_input) const = 0; protected: + static void AssertTrainingDevices(std::initializer_list tensors) { + if (tensors.size() == 0) return; + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Training tensors must share a device type and index."); + } + } + + static bool IsTrainingFloat(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; + } + Tensor::Shape grad_output_shape_; Tensor::Strides grad_output_strides_; DataType grad_output_type_; diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.cc b/src/torch/ops/accumulate_grad/accumulate_grad.cc index a3ee3f5be..6ceefd068 100644 --- a/src/torch/ops/accumulate_grad/accumulate_grad.cc +++ b/src/torch/ops/accumulate_grad/accumulate_grad.cc @@ -1,5 +1,7 @@ #include "torch/ops/accumulate_grad/accumulate_grad.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -14,12 +16,13 @@ template void Operator::operator()(const Tensor gradient, const double rate, Tensor tensor) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_gradient = ToAtenTensor(const_cast(gradient.data()), gradient_shape_, gradient_strides_, gradient_type_, device_index_); - auto at_tensor = ToAtenTensor(tensor.data(), tensor_shape_, - tensor_strides_, tensor_type_, - device_index_); + auto at_tensor = + ToAtenTensor(tensor.data(), tensor_shape_, tensor_strides_, + tensor_type_, device_index_); const auto at_rate = at_tensor.is_floating_point() ? at::Scalar(rate) @@ -27,14 +30,6 @@ void Operator::operator()(const Tensor gradient, at_tensor.add_(at_gradient, at_rate); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AccumulateGrad, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.h b/src/torch/ops/accumulate_grad/accumulate_grad.h index 3aa4e6413..007f4d4e6 100644 --- a/src/torch/ops/accumulate_grad/accumulate_grad.h +++ b/src/torch/ops/accumulate_grad/accumulate_grad.h @@ -10,8 +10,8 @@ class Operator : public AccumulateGrad { public: Operator(const Tensor gradient, const double rate, Tensor tensor); - - void operator()(const Tensor gradient, const double rate, Tensor tensor) const override; + void operator()(const Tensor gradient, const double rate, + Tensor tensor) const override; private: int device_index_{0}; diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc index 29e91e64d..d19f30c86 100644 --- a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc @@ -2,24 +2,34 @@ #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { +template +Operator::Operator( + const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) + : AdamAccumulateGrad{grad, learning_rate, beta1, beta2, eps, + step, param, m, v}, + device_index_{param.device().index()} {} + template Operator::Operator( const Tensor grad, Tensor param, Tensor m, Tensor v, const double learning_rate, const double beta1, const double beta2, const double eps, const int64_t step) - : AdamAccumulateGrad{grad, param, m, v, learning_rate, - beta1, beta2, eps, step}, - device_index_{param.device().index()} {} + : Operator{grad, learning_rate, beta1, beta2, eps, step, param, m, v} {} template void Operator::operator()( - const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, const double beta2, - const double eps, const int64_t step) const { + const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad = ToAtenTensor(const_cast(grad.data()), grad_shape_, grad_strides_, grad_type_, device_index_); auto at_param = ToAtenTensor(param.data(), param_shape_, param_strides_, @@ -29,25 +39,23 @@ void Operator::operator()( auto at_v = ToAtenTensor(v.data(), v_shape_, v_strides_, v_type_, device_index_); - at_m.mul_(beta1_).add_(at_grad, 1.0 - beta1_); - at_v.mul_(beta2_).addcmul_(at_grad, at_grad, 1.0 - beta2_); - + const auto compute_type = + at_param.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + const auto gradient = at_grad.to(compute_type); + const auto next_m = + at_m.to(compute_type) * beta1_ + gradient * (1.0 - beta1_); + const auto next_v = + at_v.to(compute_type) * beta2_ + gradient.square() * (1.0 - beta2_); const double bcm = 1.0 - std::pow(beta1_, static_cast(step_)); const double bcv = 1.0 - std::pow(beta2_, static_cast(step_)); - const auto m_hat = at_m / bcm; - const auto v_hat = at_v / bcv; - - at_param.add_(-learning_rate_ * m_hat / (at::sqrt(v_hat) + eps_)); + const auto update = + learning_rate_ * (next_m / bcm) / (at::sqrt(next_v / bcv) + eps_); + const auto next_param = at_param.to(compute_type) - update; + at_param.copy_(next_param); + at_m.copy_(next_m); + at_v.copy_(next_v); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AdamAccumulateGrad, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h index d9f5880cd..895e70fe4 100644 --- a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h @@ -8,14 +8,21 @@ namespace infini::ops { template class Operator : public AdamAccumulateGrad { public: + using AdamAccumulateGrad::operator(); + + Operator(const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, + Tensor param, Tensor m, Tensor v); + + [[deprecated("Use grad, attributes, param, m, v argument order.")]] Operator(const Tensor grad, Tensor param, Tensor m, Tensor v, const double learning_rate, const double beta1, const double beta2, const double eps, const int64_t step); - void operator()(const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, - const double beta2, const double eps, - const int64_t step) const override; + void operator()(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, const double eps, + const int64_t step, Tensor param, Tensor m, + Tensor v) const override; private: int device_index_{0}; @@ -23,4 +30,4 @@ class Operator : public AdamAccumulateGrad { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ diff --git a/src/torch/ops/add_backward/add_backward.cc b/src/torch/ops/add_backward/add_backward.cc index 045e924bd..dfab951fe 100644 --- a/src/torch/ops/add_backward/add_backward.cc +++ b/src/torch/ops/add_backward/add_backward.cc @@ -4,37 +4,11 @@ #include #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { -namespace { - -at::Tensor ReduceToShape(const at::Tensor& grad, - const std::vector& target_shape) { - const int64_t ndim = static_cast(target_shape.size()); - const int64_t grad_ndim = grad.dim(); - assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); - - std::vector sum_dims; - for (int64_t i = 0; i < grad_ndim; ++i) { - const int64_t target_i = i - (grad_ndim - ndim); - if (target_i < 0) { - sum_dims.push_back(i); - continue; - } - if (target_shape[target_i] == 1 && grad.size(i) != 1) { - sum_dims.push_back(i); - } - } - - if (sum_dims.empty()) { - return grad.reshape(target_shape); - } - return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); -} - -} // namespace - template Operator::Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b) @@ -45,30 +19,23 @@ template void Operator::operator()(const Tensor grad_output, Tensor grad_a, Tensor grad_b) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, - grad_a_strides_, grad_a_type_, - device_index_); - auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, - grad_b_strides_, grad_b_type_, - device_index_); - - at_grad_a.copy_(ReduceToShape( - at_grad, std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); - at_grad_b.copy_(ReduceToShape( - at_grad, std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); + auto at_grad_a = + ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, + grad_a_type_, device_index_); + auto at_grad_b = + ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, + grad_b_type_, device_index_); + + at_grad_a.copy_((at_grad).sum_to_size( + std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); + at_grad_b.copy_((at_grad).sum_to_size( + std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AddBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/add_backward/add_backward.h b/src/torch/ops/add_backward/add_backward.h index 893222a40..7c2ca0a73 100644 --- a/src/torch/ops/add_backward/add_backward.h +++ b/src/torch/ops/add_backward/add_backward.h @@ -19,4 +19,4 @@ class Operator : public AddBackward { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_ADD_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_ADD_BACKWARD_H_ diff --git a/src/torch/ops/cross_entropy/cross_entropy.cc b/src/torch/ops/cross_entropy/cross_entropy.cc index 38e4873bb..2b26ef1e9 100644 --- a/src/torch/ops/cross_entropy/cross_entropy.cc +++ b/src/torch/ops/cross_entropy/cross_entropy.cc @@ -1,5 +1,7 @@ #include "torch/ops/cross_entropy/cross_entropy.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -13,29 +15,21 @@ template void Operator::operator()(const Tensor input, const Tensor target, Tensor out) const { - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); - auto at_target = ToAtenTensor(const_cast(target.data()), - target_shape_, target_strides_, - target_type_, device_index_) - .to(at::kLong); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_target = + ToAtenTensor(const_cast(target.data()), target_shape_, + target_strides_, target_type_, device_index_) + .to(at::kLong); auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, out_type_, device_index_); - // 默认参数:reduction=Mean, ignore_index=-100, label_smoothing=0 —— 与 InfiniTrain 一致 auto loss = at::cross_entropy_loss(at_input, at_target); at_out.copy_(loss); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropy, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/cross_entropy/cross_entropy.h b/src/torch/ops/cross_entropy/cross_entropy.h index c5d8364ea..f5c5a0c58 100644 --- a/src/torch/ops/cross_entropy/cross_entropy.h +++ b/src/torch/ops/cross_entropy/cross_entropy.h @@ -10,8 +10,8 @@ class Operator : public CrossEntropy { public: Operator(const Tensor input, const Tensor target, Tensor out); - - void operator()(const Tensor input, const Tensor target, Tensor out) const override; + void operator()(const Tensor input, const Tensor target, + Tensor out) const override; private: int device_index_{0}; diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc index a25c9a490..82d806bb2 100644 --- a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc @@ -2,35 +2,40 @@ #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template -Operator::Operator( - const Tensor input, const Tensor target, const Tensor grad_output, - Tensor grad_input) +Operator::Operator(const Tensor input, + const Tensor target, + const Tensor grad_output, + Tensor grad_input) : CrossEntropyBackward{input, target, grad_output, grad_input} {} template void Operator::operator()( const Tensor input, const Tensor target, const Tensor grad_output, Tensor grad_input) const { - auto at_input = ToAtenTensor( - const_cast(input.data()), input_shape_, input_strides_, - input_type_, device_index_); - auto at_target = ToAtenTensor( - const_cast(target.data()), target_shape_, target_strides_, - target_type_, device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_target = + ToAtenTensor(const_cast(target.data()), target_shape_, + target_strides_, target_type_, device_index_); auto at_grad_output = ToAtenTensor( const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_input = ToAtenTensor( - grad_input.data(), grad_input_shape_, grad_input_strides_, - grad_input_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); const auto classes = at_input.size(1); - assert(classes > 0 && "CrossEntropyBackward requires a non-empty class dimension"); + assert(classes > 0 && + "CrossEntropyBackward requires a non-empty class dimension"); auto gradient = at::softmax(at_input, 1).to(at_grad_input.scalar_type()); auto target_with_class_dim = at_target.to(at::kLong).unsqueeze(1); @@ -38,11 +43,10 @@ void Operator::operator()( target_with_class_dim.ne(CrossEntropyBackward::kIgnoreIndex); const auto valid_range = (target_with_class_dim >= 0) & (target_with_class_dim < classes); - assert(at::all((~valid_target) | valid_range).item() && + assert(at::all((~valid_target) | valid_range).template item() && "CrossEntropyBackward target is outside [0, classes) or ignore_index"); - auto safe_target = - target_with_class_dim.masked_fill(~valid_target, 0); + auto safe_target = target_with_class_dim.masked_fill(~valid_target, 0); auto minus_one = -valid_target.to(gradient.scalar_type()); gradient.scatter_add_(1, safe_target, minus_one); gradient.mul_(valid_target.to(gradient.scalar_type())); @@ -54,14 +58,6 @@ void Operator::operator()( at_grad_input.copy_(gradient); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropyBackward, 1) } // namespace infini::ops diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h index 743d4bcbf..b869d31e8 100644 --- a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h @@ -6,8 +6,7 @@ namespace infini::ops { template -class Operator - : public CrossEntropyBackward { +class Operator : public CrossEntropyBackward { public: Operator(const Tensor input, const Tensor target, const Tensor grad_output, Tensor grad_input); diff --git a/src/torch/ops/embedding_backward/embedding_backward.cc b/src/torch/ops/embedding_backward/embedding_backward.cc index fd37661f5..48486ef90 100644 --- a/src/torch/ops/embedding_backward/embedding_backward.cc +++ b/src/torch/ops/embedding_backward/embedding_backward.cc @@ -1,5 +1,7 @@ #include "torch/ops/embedding_backward/embedding_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -9,8 +11,8 @@ Operator::Operator( const Tensor grad_output, const Tensor indices, const int64_t num_weights, const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, Tensor grad_weight) - : EmbeddingBackward{grad_output, indices, num_weights, padding_idx, - scale_grad_by_freq, sparse, grad_weight}, + : EmbeddingBackward{grad_output, indices, num_weights, padding_idx, + scale_grad_by_freq, sparse, grad_weight}, device_index_{grad_weight.device().index()} {} template @@ -18,37 +20,28 @@ void Operator::operator()( const Tensor grad_output, const Tensor indices, const int64_t num_weights, const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, Tensor grad_weight) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); - auto at_indices = ToAtenTensor(const_cast(indices.data()), - indices_shape_, indices_strides_, - indices_type_, device_index_) - .to(at::kLong); - auto at_grad_weight = - ToAtenTensor(grad_weight.data(), grad_weight_shape_, - grad_weight_strides_, grad_weight_type_, device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_indices = + ToAtenTensor(const_cast(indices.data()), indices_shape_, + indices_strides_, indices_type_, device_index_) + .to(at::kLong); + auto at_grad_weight = ToAtenTensor( + grad_weight.data(), grad_weight_shape_, grad_weight_strides_, + grad_weight_type_, device_index_); - auto at_grad = at::embedding_dense_backward(at_grad_output, at_indices, - num_weights, padding_idx, - scale_grad_by_freq); - if (sparse) { - at_grad = at::embedding_backward(at_grad_output, at_indices, num_weights, - padding_idx, scale_grad_by_freq, true) - .to_dense(); - } + auto at_grad = + sparse_ ? at::embedding_backward(at_grad_output, at_indices, num_weights_, + padding_idx_, scale_grad_by_freq_, true) + .to_dense() + : at::embedding_dense_backward(at_grad_output, at_indices, + num_weights_, padding_idx_, + scale_grad_by_freq_); at_grad_weight.copy_(at_grad); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(EmbeddingBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/embedding_backward/embedding_backward.h b/src/torch/ops/embedding_backward/embedding_backward.h index b12838dcf..7224a146d 100644 --- a/src/torch/ops/embedding_backward/embedding_backward.h +++ b/src/torch/ops/embedding_backward/embedding_backward.h @@ -10,7 +10,8 @@ class Operator : public EmbeddingBackward { public: Operator(const Tensor grad_output, const Tensor indices, const int64_t num_weights, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, Tensor grad_weight); + const bool scale_grad_by_freq, const bool sparse, + Tensor grad_weight); void operator()(const Tensor grad_output, const Tensor indices, const int64_t num_weights, const int64_t padding_idx, @@ -23,4 +24,4 @@ class Operator : public EmbeddingBackward { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc index c42507070..e50190639 100644 --- a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc @@ -1,5 +1,7 @@ #include "torch/ops/masked_fill_backward/masked_fill_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -14,10 +16,10 @@ Operator::Operator(const Tensor grad_output, template void Operator::operator()( const Tensor grad_output, const Tensor mask, Tensor grad_input) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); auto at_mask = ToAtenTensor(const_cast(mask.data()), mask_shape_, mask_strides_, mask_type_, device_index_) .to(at::kBool); @@ -26,8 +28,8 @@ void Operator::operator()( grad_input_strides_, grad_input_type_, device_index_); if (!tail_aligned_) { - - for (Tensor::Size i = mask_shape_.size(); i < grad_input_shape_.size(); ++i) { + for (Tensor::Size i = mask_shape_.size(); i < grad_input_shape_.size(); + ++i) { at_mask = at_mask.unsqueeze(-1); } } @@ -35,14 +37,6 @@ void Operator::operator()( at_grad_input.copy_(at::masked_fill(at_grad_output, at_mask, 0)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MaskedFillBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.h b/src/torch/ops/masked_fill_backward/masked_fill_backward.h index 8410d2218..60e6ffcb6 100644 --- a/src/torch/ops/masked_fill_backward/masked_fill_backward.h +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.h @@ -8,11 +8,10 @@ namespace infini::ops { template class Operator : public MaskedFillBackward { public: - Operator(const Tensor grad_output, const Tensor mask, - Tensor grad_input); - - void operator()(const Tensor grad_output, const Tensor mask, - Tensor grad_input) const override; + Operator(const Tensor grad_output, const Tensor mask, Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/mean_backward/mean_backward.cc b/src/torch/ops/mean_backward/mean_backward.cc index 6f30364ad..e6c5a3bb8 100644 --- a/src/torch/ops/mean_backward/mean_backward.cc +++ b/src/torch/ops/mean_backward/mean_backward.cc @@ -1,12 +1,15 @@ #include "torch/ops/mean_backward/mean_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template Operator::Operator(const Tensor grad_output, - const int64_t dim, const bool keep_dim, + const int64_t dim, + const bool keep_dim, Tensor grad_input) : MeanBackward{grad_output, dim, keep_dim, grad_input}, device_index_{grad_input.device().index()} {} @@ -16,36 +19,27 @@ void Operator::operator()(const Tensor grad_output, const int64_t dim, const bool keep_dim, Tensor grad_input) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - if (grad_input_shape_.empty() || grad_input_shape_[0] == 0) { - return; + if (at_grad_input.numel() == 0) { + return; } auto at_g = at_grad_output; if (!keep_dim_) { - at_g = at_g.unsqueeze(dim_); + at_g = at_g.unsqueeze(dim_); } - const double count = static_cast(at_grad_input.numel()) / - static_cast(at_grad_output.numel()); + const double count = static_cast(grad_input_shape_[dim_]); at_grad_input.copy_(at_g.expand(at_grad_input.sizes()) / count); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MeanBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/mean_backward/mean_backward.h b/src/torch/ops/mean_backward/mean_backward.h index a23ca23f8..d244d6ab3 100644 --- a/src/torch/ops/mean_backward/mean_backward.h +++ b/src/torch/ops/mean_backward/mean_backward.h @@ -1,5 +1,5 @@ -#ifndef INFINI_OPS_TORCH_mean_backward_H_ -#define INFINI_OPS_TORCH_mean_backward_H_ +#ifndef INFINI_OPS_TORCH_MEAN_BACKWARD_H_ +#define INFINI_OPS_TORCH_MEAN_BACKWARD_H_ #include "base/mean_backward.h" @@ -9,11 +9,10 @@ template class Operator : public MeanBackward { public: Operator(const Tensor grad_output, const int64_t dim, const bool keep_dim, - Tensor grad_input); + Tensor grad_input); - - void operator()(const Tensor grad_output, const int64_t dim, const bool keep_dim, - Tensor grad_input) const override; + void operator()(const Tensor grad_output, const int64_t dim, + const bool keep_dim, Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/mul_backward/mul_backward.cc b/src/torch/ops/mul_backward/mul_backward.cc index 726712841..6e395fc64 100644 --- a/src/torch/ops/mul_backward/mul_backward.cc +++ b/src/torch/ops/mul_backward/mul_backward.cc @@ -4,37 +4,11 @@ #include #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { -namespace { - -at::Tensor ReduceToShape(const at::Tensor& grad, - const std::vector& target_shape) { - const int64_t ndim = static_cast(target_shape.size()); - const int64_t grad_ndim = grad.dim(); - assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); - - std::vector sum_dims; - for (int64_t i = 0; i < grad_ndim; ++i) { - const int64_t target_i = i - (grad_ndim - ndim); - if (target_i < 0) { - sum_dims.push_back(i); - continue; - } - if (target_shape[target_i] == 1 && grad.size(i) != 1) { - sum_dims.push_back(i); - } - } - - if (sum_dims.empty()) { - return grad.reshape(target_shape); - } - return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); -} - -} // namespace - template Operator::Operator(const Tensor grad_output, const Tensor a, const Tensor b, @@ -44,10 +18,10 @@ Operator::Operator(const Tensor grad_output, template void Operator::operator()(const Tensor grad_output, - const Tensor a, - const Tensor b, + const Tensor a, const Tensor b, Tensor grad_a, Tensor grad_b) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); @@ -55,29 +29,21 @@ void Operator::operator()(const Tensor grad_output, a_strides_, a_type_, device_index_); auto at_b = ToAtenTensor(const_cast(b.data()), b_shape_, b_strides_, b_type_, device_index_); - auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, - grad_a_strides_, grad_a_type_, - device_index_); - auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, - grad_b_strides_, grad_b_type_, - device_index_); - - at_grad_a.copy_(ReduceToShape( - at_grad * at_b, - std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); - at_grad_b.copy_(ReduceToShape( - at_grad * at_a, - std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); + auto at_grad_a = + ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, + grad_a_type_, device_index_); + auto at_grad_b = + ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, + grad_b_type_, device_index_); + + at_grad_a.copy_((at_grad * at_b) + .sum_to_size(std::vector(grad_a_shape_.begin(), + grad_a_shape_.end()))); + at_grad_b.copy_((at_grad * at_a) + .sum_to_size(std::vector(grad_b_shape_.begin(), + grad_b_shape_.end()))); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MulBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/mul_backward/mul_backward.h b/src/torch/ops/mul_backward/mul_backward.h index 7a02ad016..a50e741fc 100644 --- a/src/torch/ops/mul_backward/mul_backward.h +++ b/src/torch/ops/mul_backward/mul_backward.h @@ -20,4 +20,4 @@ class Operator : public MulBackward { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_MUL_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_MUL_BACKWARD_H_ diff --git a/src/torch/ops/pow_backward/pow_backward.cc b/src/torch/ops/pow_backward/pow_backward.cc index c80f9e8f8..7009421a2 100644 --- a/src/torch/ops/pow_backward/pow_backward.cc +++ b/src/torch/ops/pow_backward/pow_backward.cc @@ -1,49 +1,48 @@ #include "torch/ops/pow_backward/pow_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template Operator::Operator(const Tensor grad_output, - const Tensor input, const double scalar, + const Tensor input, + const double scalar, const bool scalar_is_base, Tensor grad_input) : PowBackward{grad_output, input, scalar, scalar_is_base, grad_input}, device_index_{grad_input.device().index()} {} template -void Operator::operator()( - const Tensor grad_output, const Tensor input, const double scalar, - const bool scalar_is_base, Tensor grad_input) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); +void Operator::operator()(const Tensor grad_output, + const Tensor input, + const double scalar, + const bool scalar_is_base, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - if (scalar_is_base_) { + if (!scalar_is_base_ && scalar_ == 0.0) { + at_grad_input.zero_(); + } else if (scalar_is_base_) { auto at_y = at::pow(scalar_, at_input); at_grad_input.copy_(at_grad_output * at_y * std::log(scalar_)); } else { - at_grad_input.copy_( - at_grad_output * scalar_ * at::pow(at_input, scalar_ - 1.0)); + at_grad_input.copy_(at_grad_output * scalar_ * + at::pow(at_input, scalar_ - 1.0)); } } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(PowBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/pow_backward/pow_backward.h b/src/torch/ops/pow_backward/pow_backward.h index c61b4ba29..493f504e5 100644 --- a/src/torch/ops/pow_backward/pow_backward.h +++ b/src/torch/ops/pow_backward/pow_backward.h @@ -9,10 +9,11 @@ template class Operator : public PowBackward { public: Operator(const Tensor grad_output, const Tensor input, const double scalar, - const bool scalar_is_base, Tensor grad_input); + const bool scalar_is_base, Tensor grad_input); - void operator()(const Tensor grad_output, const Tensor input, const double scalar, - const bool scalar_is_base, Tensor grad_input) const override; + void operator()(const Tensor grad_output, const Tensor input, + const double scalar, const bool scalar_is_base, + Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.cc b/src/torch/ops/repeat_interleave/repeat_interleave.cc index 32980c1a9..fca0e2a46 100644 --- a/src/torch/ops/repeat_interleave/repeat_interleave.cc +++ b/src/torch/ops/repeat_interleave/repeat_interleave.cc @@ -1,5 +1,7 @@ #include "torch/ops/repeat_interleave/repeat_interleave.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -16,23 +18,16 @@ void Operator::operator()(const Tensor input, const int64_t repeats, const int64_t dim, Tensor out) const { - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, out_type_, device_index_); at_out.copy_(at::repeat_interleave(at_input, repeats_, dim_)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RepeatInterleave, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.h b/src/torch/ops/repeat_interleave/repeat_interleave.h index d1fabc6c5..22e895a09 100644 --- a/src/torch/ops/repeat_interleave/repeat_interleave.h +++ b/src/torch/ops/repeat_interleave/repeat_interleave.h @@ -9,11 +9,10 @@ template class Operator : public RepeatInterleave { public: Operator(const Tensor input, const int64_t repeats, const int64_t dim, - Tensor out); - + Tensor out); void operator()(const Tensor input, const int64_t repeats, const int64_t dim, - Tensor out) const override; + Tensor out) const override; private: int device_index_{0}; diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc index 25ce36ed9..753d1fd3a 100644 --- a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc @@ -1,5 +1,7 @@ #include "torch/ops/repeat_interleave_backward/repeat_interleave_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -14,32 +16,30 @@ Operator::Operator(const Tensor grad_output, template void Operator::operator()( const Tensor grad_output, const int64_t dim, Tensor grad_input) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); + if (at_grad_input.numel() == 0) return; + if (repeats_ == 0) { + at_grad_input.zero_(); + return; + } + auto sizes = at_grad_output.sizes().vec(); sizes[dim_] /= repeats_; sizes.insert(sizes.begin() + dim_ + 1, repeats_); - auto grouped = at_grad_output.reshape(sizes); + auto grouped = at_grad_output.reshape(sizes); auto summed = grouped.sum(dim_ + 1); at_grad_input.copy_(summed); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RepeatInterleaveBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h index 797b27755..a99c33922 100644 --- a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h @@ -6,14 +6,13 @@ namespace infini::ops { template -class Operator : public RepeatInterleaveBackward { +class Operator + : public RepeatInterleaveBackward { public: - Operator(const Tensor grad_output, const int64_t dim, - Tensor grad_input); - + Operator(const Tensor grad_output, const int64_t dim, Tensor grad_input); void operator()(const Tensor grad_output, const int64_t dim, - Tensor grad_input) const override; + Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc index 9c5c6a9d5..dadd3237e 100644 --- a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc @@ -1,12 +1,15 @@ #include "torch/ops/rsqrt_backward/rsqrt_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template Operator::Operator(const Tensor grad_output, - const Tensor input, Tensor grad_input) + const Tensor input, + Tensor grad_input) : RsqrtBackward{grad_output, input, grad_input}, device_index_{grad_input.device().index()} {} @@ -14,28 +17,21 @@ template void Operator::operator()(const Tensor grad_output, const Tensor input, Tensor grad_input) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - at_grad_input.copy_(at_grad_output * (-0.5) * at::pow(at::rsqrt(at_input), 3)); + at_grad_input.copy_(at_grad_output * (-0.5) * + at::pow(at::rsqrt(at_input), 3)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RsqrtBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.h b/src/torch/ops/rsqrt_backward/rsqrt_backward.h index 4e94591bb..e2fa87fbe 100644 --- a/src/torch/ops/rsqrt_backward/rsqrt_backward.h +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.h @@ -10,8 +10,8 @@ class Operator : public RsqrtBackward { public: Operator(const Tensor grad_output, const Tensor input, Tensor grad_input); - - void operator()(const Tensor grad_output, const Tensor input, Tensor grad_input) const override; + void operator()(const Tensor grad_output, const Tensor input, + Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/slice/slice.cc b/src/torch/ops/slice/slice.cc index 6e05db7e2..a947658f5 100644 --- a/src/torch/ops/slice/slice.cc +++ b/src/torch/ops/slice/slice.cc @@ -1,5 +1,7 @@ #include "torch/ops/slice/slice.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -9,18 +11,21 @@ Operator::Operator(const Tensor input, const std::vector starts, const std::vector ends, const std::vector steps, Tensor out) - : Slice{input, starts, ends, steps, out}, device_index_{out.device().index()} {} + : Slice{input, starts, ends, steps, out}, + device_index_{out.device().index()} {} template -void Operator::operator()( - const Tensor input, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor out) const { +void Operator::operator()(const Tensor input, + const std::vector starts, + const std::vector ends, + const std::vector steps, + Tensor out) const { + const TorchStreamGuard guard{stream_, device_index_}; using namespace torch::indexing; - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, out_type_, device_index_); @@ -33,14 +38,6 @@ void Operator::operator()( at_out.copy_(at_input.index(index)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Slice, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/slice/slice.h b/src/torch/ops/slice/slice.h index da2debcf0..83358fdbd 100644 --- a/src/torch/ops/slice/slice.h +++ b/src/torch/ops/slice/slice.h @@ -12,9 +12,9 @@ class Operator : public Slice { const std::vector ends, const std::vector steps, Tensor out); - void operator()(const Tensor input, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor out) const override; + void operator()(const Tensor input, const std::vector starts, + const std::vector ends, + const std::vector steps, Tensor out) const override; private: int device_index_{0}; @@ -22,4 +22,4 @@ class Operator : public Slice { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_SLICE_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_SLICE_H_ diff --git a/src/torch/ops/slice_backward/slice_backward.cc b/src/torch/ops/slice_backward/slice_backward.cc index 6c30fcd84..aa127c1c6 100644 --- a/src/torch/ops/slice_backward/slice_backward.cc +++ b/src/torch/ops/slice_backward/slice_backward.cc @@ -1,14 +1,17 @@ #include "torch/ops/slice_backward/slice_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template -Operator::Operator( - const Tensor grad_output, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor grad_input) +Operator::Operator(const Tensor grad_output, + const std::vector starts, + const std::vector ends, + const std::vector steps, + Tensor grad_input) : SliceBackward{grad_output, starts, ends, steps, grad_input}, device_index_{grad_input.device().index()} {} @@ -17,12 +20,12 @@ void Operator::operator()( const Tensor grad_output, const std::vector starts, const std::vector ends, const std::vector steps, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; using namespace torch::indexing; - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); @@ -33,20 +36,11 @@ void Operator::operator()( index.emplace_back(at::indexing::Slice(starts_[i], ends_[i], steps_[i])); } - auto zeros = at::zeros(at_grad_input.sizes(), at_grad_input.options()); zeros.index_put_(index, at_grad_output); at_grad_input.copy_(zeros); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(SliceBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/slice_backward/slice_backward.h b/src/torch/ops/slice_backward/slice_backward.h index 398e11c0a..09b2bdf71 100644 --- a/src/torch/ops/slice_backward/slice_backward.h +++ b/src/torch/ops/slice_backward/slice_backward.h @@ -8,13 +8,14 @@ namespace infini::ops { template class Operator : public SliceBackward { public: - Operator(const Tensor grad_output, const std::vector starts, - const std::vector ends, - const std::vector steps, Tensor grad_input); + Operator(const Tensor grad_output, const std::vector starts, + const std::vector ends, const std::vector steps, + Tensor grad_input); - void operator()(const Tensor grad_output, const std::vector starts, - const std::vector ends, - const std::vector steps, Tensor grad_input) const override; + void operator()(const Tensor grad_output, const std::vector starts, + const std::vector ends, + const std::vector steps, + Tensor grad_input) const override; private: int device_index_{0}; diff --git a/src/torch/ops/split/split.cc b/src/torch/ops/split/split.cc index f1cba7716..edff2e878 100644 --- a/src/torch/ops/split/split.cc +++ b/src/torch/ops/split/split.cc @@ -1,5 +1,7 @@ #include "torch/ops/split/split.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -15,29 +17,22 @@ void Operator::operator()(const Tensor input, const int64_t split_size, const int64_t dim, std::vector outs) const { - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); const auto parts = at::split(at_input, split_size_, dim_); assert(parts.size() == outs_.size() && "`Split` part count mismatch"); for (std::size_t i = 0; i < parts.size(); ++i) { - auto at_out_i = ToAtenTensor(outs[i].data(), out_shapes_[i], - out_strides_[i], out_types_[i], - device_index_); + auto at_out_i = + ToAtenTensor(outs[i].data(), out_shapes_[i], out_strides_[i], + out_types_[i], device_index_); at_out_i.copy_(parts[i]); } } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Split, 1) } // namespace infini::ops diff --git a/src/torch/ops/split/split.h b/src/torch/ops/split/split.h index 29176e7c9..f1e3cb744 100644 --- a/src/torch/ops/split/split.h +++ b/src/torch/ops/split/split.h @@ -8,12 +8,12 @@ namespace infini::ops { template class Operator : public Split { public: - Operator(const Tensor input, const int64_t split_size, const int64_t dim, - std::vector outs); + Operator(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs); + + void operator()(const Tensor input, const int64_t split_size, + const int64_t dim, std::vector outs) const override; - void operator()(const Tensor input, const int64_t split_size, const int64_t dim, - std::vector outs) const override; - private: int device_index_{0}; }; diff --git a/src/torch/ops/stack_backward/stack_backward.cc b/src/torch/ops/stack_backward/stack_backward.cc index 4f767d5a4..06f75527e 100644 --- a/src/torch/ops/stack_backward/stack_backward.cc +++ b/src/torch/ops/stack_backward/stack_backward.cc @@ -1,5 +1,7 @@ #include "torch/ops/stack_backward/stack_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -12,34 +14,25 @@ Operator::Operator(const Tensor grad_output, device_index_{StackBackward::device_index_} {} template -void Operator::operator()(const Tensor grad_output, - const int64_t dim, - std::vector outs) const { - auto at_grad_output = - ToAtenTensor(const_cast(grad_output.data()), - grad_output_shape_, grad_output_strides_, - grad_output_type_, device_index_); +void Operator::operator()( + const Tensor grad_output, const int64_t dim, + std::vector outs) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); const auto parts = at::unbind(at_grad_output, dim_); assert(parts.size() == outs_.size() && "`StackBackward` part count mismatch"); for (std::size_t i = 0; i < parts.size(); ++i) { - auto at_out_i = ToAtenTensor( - outs[i].data(), out_shapes_[i], - out_strides_[i], out_types_[i], - device_index_); + auto at_out_i = + ToAtenTensor(outs[i].data(), out_shapes_[i], out_strides_[i], + out_types_[i], device_index_); at_out_i.copy_(parts[i]); } } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(StackBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/stack_backward/stack_backward.h b/src/torch/ops/stack_backward/stack_backward.h index 460a12cc5..b5635baeb 100644 --- a/src/torch/ops/stack_backward/stack_backward.h +++ b/src/torch/ops/stack_backward/stack_backward.h @@ -8,11 +8,11 @@ namespace infini::ops { template class Operator : public StackBackward { public: - Operator(const Tensor grad_output, const int64_t dim, - std::vector outs); + Operator(const Tensor grad_output, const int64_t dim, + std::vector outs); - void operator()(const Tensor grad_output, const int64_t dim, - std::vector outs) const override; + void operator()(const Tensor grad_output, const int64_t dim, + std::vector outs) const override; private: int device_index_{0}; diff --git a/src/torch/ops/sub_backward/sub_backward.cc b/src/torch/ops/sub_backward/sub_backward.cc index deffab7bd..8c27e50b0 100644 --- a/src/torch/ops/sub_backward/sub_backward.cc +++ b/src/torch/ops/sub_backward/sub_backward.cc @@ -4,38 +4,11 @@ #include #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { -namespace { - - -at::Tensor ReduceToShape(const at::Tensor& grad, - const std::vector& target_shape) { - const int64_t ndim = static_cast(target_shape.size()); - const int64_t grad_ndim = grad.dim(); - assert(grad_ndim >= ndim && "grad rank must not be smaller than target rank"); - - std::vector sum_dims; - for (int64_t i = 0; i < grad_ndim; ++i) { - const int64_t target_i = i - (grad_ndim - ndim); - if (target_i < 0) { - sum_dims.push_back(i); - continue; - } - if (target_shape[target_i] == 1 && grad.size(i) != 1) { - sum_dims.push_back(i); - } - } - - if (sum_dims.empty()) { - return grad.reshape(target_shape); - } - return grad.sum(sum_dims, /*keepdim=*/true).reshape(target_shape); -} - -} // namespace - template Operator::Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b) @@ -46,30 +19,23 @@ template void Operator::operator()(const Tensor grad_output, Tensor grad_a, Tensor grad_b) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_a = ToAtenTensor(grad_a.data(), grad_a_shape_, - grad_a_strides_, grad_a_type_, - device_index_); - auto at_grad_b = ToAtenTensor(grad_b.data(), grad_b_shape_, - grad_b_strides_, grad_b_type_, - device_index_); - - at_grad_a.copy_(ReduceToShape( - at_grad, std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); - at_grad_b.copy_(-ReduceToShape( - at_grad, std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); + auto at_grad_a = + ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, + grad_a_type_, device_index_); + auto at_grad_b = + ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, + grad_b_type_, device_index_); + + at_grad_a.copy_((at_grad).sum_to_size( + std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); + at_grad_b.copy_(-(at_grad).sum_to_size( + std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(SubBackward, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/sub_backward/sub_backward.h b/src/torch/ops/sub_backward/sub_backward.h index 76b249c48..c1a571e7d 100644 --- a/src/torch/ops/sub_backward/sub_backward.h +++ b/src/torch/ops/sub_backward/sub_backward.h @@ -19,4 +19,4 @@ class Operator : public SubBackward { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_SUB_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_SUB_BACKWARD_H_ diff --git a/src/torch/ops/transpose/transpose.cc b/src/torch/ops/transpose/transpose.cc index 6c95855c6..0b99e54c1 100644 --- a/src/torch/ops/transpose/transpose.cc +++ b/src/torch/ops/transpose/transpose.cc @@ -1,5 +1,7 @@ #include "torch/ops/transpose/transpose.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -14,23 +16,16 @@ void Operator::operator()(const Tensor input, const int64_t dim0, const int64_t dim1, Tensor out) const { - auto at_input = ToAtenTensor(const_cast(input.data()), - input_shape_, input_strides_, input_type_, - device_index_); + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, out_type_, device_index_); at_out.copy_(at_input.transpose(dim0_, dim1_)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Transpose, 1) -} // namespace infini::ops \ No newline at end of file +} // namespace infini::ops diff --git a/src/torch/ops/transpose/transpose.h b/src/torch/ops/transpose/transpose.h index 13419d981..50541ab12 100644 --- a/src/torch/ops/transpose/transpose.h +++ b/src/torch/ops/transpose/transpose.h @@ -8,11 +8,11 @@ namespace infini::ops { template class Operator : public Transpose { public: - Operator(const Tensor input, const int64_t dim0, const int64_t dim1, - Tensor out); - - void operator()(const Tensor input, const int64_t dim0, const int64_t dim1, - Tensor out) const override; + Operator(const Tensor input, const int64_t dim0, const int64_t dim1, + Tensor out); + + void operator()(const Tensor input, const int64_t dim0, const int64_t dim1, + Tensor out) const override; private: int device_index_{0}; diff --git a/src/torch/ops/transpose_backward/transpose_backward.cc b/src/torch/ops/transpose_backward/transpose_backward.cc index 600c08022..6084a53b0 100644 --- a/src/torch/ops/transpose_backward/transpose_backward.cc +++ b/src/torch/ops/transpose_backward/transpose_backward.cc @@ -1,39 +1,36 @@ #include "torch/ops/transpose_backward/transpose_backward.h" +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { template -Operator::Operator( - const Tensor grad_output, const int64_t dim0, const int64_t dim1, - Tensor grad_input) +Operator::Operator(const Tensor grad_output, + const int64_t dim0, + const int64_t dim1, + Tensor grad_input) : TransposeBackward{grad_output, dim0, dim1, grad_input}, device_index_{grad_input.device().index()} {} template -void Operator::operator()( - const Tensor grad_output, const int64_t dim0, const int64_t dim1, - Tensor grad_input) const { +void Operator::operator()(const Tensor grad_output, + const int64_t dim0, + const int64_t dim1, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad_output = ToAtenTensor( const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_input = ToAtenTensor( - grad_input.data(), grad_input_shape_, grad_input_strides_, - grad_input_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); at_grad_input.copy_(at_grad_output.transpose(dim0_, dim1_)); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; - -} // namespace infini::ops \ No newline at end of file +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(TransposeBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/transpose_backward/transpose_backward.h b/src/torch/ops/transpose_backward/transpose_backward.h index 08c6cf89d..b0aee44fe 100644 --- a/src/torch/ops/transpose_backward/transpose_backward.h +++ b/src/torch/ops/transpose_backward/transpose_backward.h @@ -20,4 +20,4 @@ class Operator : public TransposeBackward { } // namespace infini::ops -#endif // INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ \ No newline at end of file +#endif // INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc index 1a5e4b24a..b187c8c0b 100644 --- a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc @@ -2,6 +2,8 @@ #include +#include "torch/instantiate.h" +#include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { @@ -14,9 +16,9 @@ Operator::Operator( const int64_t vocab_size_original, const double label_smoothing, Tensor grad_input) : VocabParallelCrossEntropyBackward{ - grad_output, softmax_local, target_mask, masked_target, - valid_mask_local, vocab_size_local, vocab_size_original, - label_smoothing, grad_input} {} + grad_output, softmax_local, target_mask, + masked_target, valid_mask_local, vocab_size_local, + vocab_size_original, label_smoothing, grad_input} {} template void Operator::operator()( @@ -25,12 +27,13 @@ void Operator::operator()( const Tensor valid_mask_local, const int64_t vocab_size_local, const int64_t vocab_size_original, const double label_smoothing, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; auto at_grad_output = ToAtenTensor( const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_softmax = ToAtenTensor( - const_cast(softmax_local.data()), softmax_shape_, softmax_strides_, - softmax_type_, device_index_); + auto at_softmax = ToAtenTensor(const_cast(softmax_local.data()), + softmax_shape_, softmax_strides_, + softmax_type_, device_index_); auto at_target_mask = ToAtenTensor( const_cast(target_mask.data()), target_mask_shape_, target_mask_strides_, target_mask_type_, device_index_); @@ -40,9 +43,9 @@ void Operator::operator()( auto at_valid_mask = ToAtenTensor( const_cast(valid_mask_local.data()), valid_mask_shape_, valid_mask_strides_, valid_mask_type_, device_index_); - auto at_grad_input = ToAtenTensor( - grad_input.data(), grad_input_shape_, grad_input_strides_, - grad_input_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); assert(vocab_size_local > 0 && "VocabParallelCrossEntropyBackward requires positive local vocab"); @@ -91,23 +94,6 @@ void Operator::operator()( at_grad_input.copy_(gradient.reshape(at_grad_input.sizes())); } -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; -template class Operator; +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(VocabParallelCrossEntropyBackward, 1) } // namespace infini::ops diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h index 82268aa03..b17e4748d 100644 --- a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h @@ -15,12 +15,12 @@ class Operator const int64_t vocab_size_original, const double label_smoothing, Tensor grad_input); - void operator()( - const Tensor grad_output, const Tensor softmax_local, - const Tensor target_mask, const Tensor masked_target, - const Tensor valid_mask_local, const int64_t vocab_size_local, - const int64_t vocab_size_original, const double label_smoothing, - Tensor grad_input) const override; + void operator()(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, + const double label_smoothing, + Tensor grad_input) const override; }; } // namespace infini::ops diff --git a/tests/test_accumulate_grad.py b/tests/test_accumulate_grad.py index 136af0079..441d3a338 100644 --- a/tests/test_accumulate_grad.py +++ b/tests/test_accumulate_grad.py @@ -2,7 +2,7 @@ import pytest import torch -from tests.utils import Payload, empty_strided, get_stream, randn_strided +from tests.utils import Payload, get_stream, randn_strided # Format: (shape, rate, gradient_strides, tensor_strides) diff --git a/tests/test_adam_accumulate_grad.py b/tests/test_adam_accumulate_grad.py index d79487458..2b794bc3a 100644 --- a/tests/test_adam_accumulate_grad.py +++ b/tests/test_adam_accumulate_grad.py @@ -2,7 +2,7 @@ import pytest import torch -from tests.utils import Payload, empty_strided, get_stream, randn_strided +from tests.utils import Payload, get_stream, randn_strided # Format: (shape, learning_rate, beta1, beta2, eps, step, strides) @@ -78,7 +78,7 @@ def test_adam_accumulate_grad( implementation_index=implementation_index, ), _torch_adam_accumulate_grad, - (grad, param, m, v, learning_rate, beta1, beta2, eps, step), + (grad, learning_rate, beta1, beta2, eps, step, param, m, v), {}, rtol=rtol, atol=atol, @@ -87,27 +87,27 @@ def test_adam_accumulate_grad( def _adam_accumulate_grad( grad, - param, - m, - v, learning_rate, beta1, beta2, eps, step, + param, + m, + v, *, implementation_index, ): infini.ops.adam_accumulate_grad( grad, - param, - m, - v, learning_rate, beta1, beta2, eps, step, + param, + m, + v, stream=get_stream(grad.device), implementation_index=implementation_index, ) @@ -117,23 +117,34 @@ def _adam_accumulate_grad( def _torch_adam_accumulate_grad( grad, - param, - m, - v, learning_rate, beta1, beta2, eps, step, + param, + m, + v, ): - m.copy_(beta1 * m + (1 - beta1) * grad) - v.copy_(beta2 * v + (1 - beta2) * grad * grad) - - bias_correction_m = 1 - beta1**step - bias_correction_v = 1 - beta2**step - m_hat = m.float() / bias_correction_m - v_hat = v.float() / bias_correction_v - update = learning_rate * m_hat / (torch.sqrt(v_hat) + eps) - param.copy_(param - update.to(dtype=param.dtype)) + reference_param = torch.nn.Parameter( + param.detach().to(device="cpu", dtype=torch.float64) + ) + optimizer = torch.optim.Adam( + [reference_param], + lr=learning_rate, + betas=(beta1, beta2), + eps=eps, + foreach=False, + ) + optimizer.state[reference_param] = { + "step": torch.tensor(float(step - 1)), + "exp_avg": m.detach().to(device="cpu", dtype=torch.float64).clone(), + "exp_avg_sq": v.detach().to(device="cpu", dtype=torch.float64).clone(), + } + reference_param.grad = grad.detach().to(device="cpu", dtype=torch.float64).clone() + optimizer.step() + param.copy_(reference_param.detach()) + m.copy_(optimizer.state[reference_param]["exp_avg"]) + v.copy_(optimizer.state[reference_param]["exp_avg_sq"]) return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) diff --git a/tests/test_cross_entropy.py b/tests/test_cross_entropy.py index 899dcd360..44a6b7a96 100644 --- a/tests/test_cross_entropy.py +++ b/tests/test_cross_entropy.py @@ -2,7 +2,13 @@ import pytest import torch -from tests.utils import Payload, empty_strided, get_stream, randn_strided, randint_strided +from tests.utils import ( + Payload, + empty_strided, + get_stream, + randn_strided, + randint_strided, +) # Format: (input_shape, input_strides, target_strides) diff --git a/tests/test_cross_entropy_backward.py b/tests/test_cross_entropy_backward.py index d6a1791d8..9b52ff293 100644 --- a/tests/test_cross_entropy_backward.py +++ b/tests/test_cross_entropy_backward.py @@ -2,7 +2,13 @@ import pytest import torch -from tests.utils import Payload, empty_strided, get_stream, randint_strided, randn_strided +from tests.utils import ( + Payload, + empty_strided, + get_stream, + randint_strided, + randn_strided, +) # Format: (input_shape, input_strides, target_strides, grad_input_strides) diff --git a/tests/test_pow_backward.py b/tests/test_pow_backward.py index 06d70d8e1..74f88a31e 100644 --- a/tests/test_pow_backward.py +++ b/tests/test_pow_backward.py @@ -161,20 +161,14 @@ def _torch_pow_backward( scalar_is_base, grad_input, ): - scalar_tensor = torch.as_tensor( - scalar, - dtype=input.dtype, - device=input.device, - ) + differentiable_input = input.detach().clone().requires_grad_(True) if scalar_is_base: - derivative = torch.log(scalar_tensor) * torch.pow( - scalar_tensor, - input, - ) + result = torch.pow(scalar, differentiable_input) else: - derivative = scalar_tensor * torch.pow(input, scalar_tensor - 1) + result = torch.pow(differentiable_input, scalar) - grad_input.copy_(grad_output * derivative) + gradient = torch.autograd.grad(result, differentiable_input, grad_output)[0] + grad_input.copy_(gradient) return grad_input diff --git a/tests/test_rsqrt_backward.py b/tests/test_rsqrt_backward.py index 9c4e8fbb3..f6ec99142 100644 --- a/tests/test_rsqrt_backward.py +++ b/tests/test_rsqrt_backward.py @@ -115,6 +115,9 @@ def _rsqrt_backward( def _torch_rsqrt_backward(grad_output, input, grad_input): - grad_input.copy_(-0.5 * grad_output * torch.pow(input, -1.5)) + differentiable_input = input.detach().clone().requires_grad_(True) + result = torch.rsqrt(differentiable_input) + gradient = torch.autograd.grad(result, differentiable_input, grad_output)[0] + grad_input.copy_(gradient) return grad_input diff --git a/tests/test_slice.py b/tests/test_slice.py index ed4e90acf..bdee862ba 100644 --- a/tests/test_slice.py +++ b/tests/test_slice.py @@ -94,15 +94,11 @@ def test_slice( rtol, atol, ): - input = randn_strided( - input_shape, input_strides, dtype=dtype, device=device - ) + input = randn_strided(input_shape, input_strides, dtype=dtype, device=device) out = empty_strided(out_shape, out_strides, dtype=dtype, device=device) return Payload( - lambda *args: _slice( - *args, implementation_index=implementation_index - ), + lambda *args: _slice(*args, implementation_index=implementation_index), _torch_slice, (input, starts, ends, steps, out), {}, diff --git a/tests/test_slice_backward.py b/tests/test_slice_backward.py index f76839a7a..8f90bd349 100644 --- a/tests/test_slice_backward.py +++ b/tests/test_slice_backward.py @@ -109,9 +109,7 @@ def test_slice_backward( grad_input.fill_(17) return Payload( - lambda *args: _slice_backward( - *args, implementation_index=implementation_index - ), + lambda *args: _slice_backward(*args, implementation_index=implementation_index), _torch_slice_backward, (grad_output, starts, ends, steps, grad_input), {}, @@ -153,6 +151,5 @@ def _slice_index(starts, ends, steps): assert len(starts) == len(ends) == len(steps) return tuple( - slice(start, end, step) - for start, end, step in zip(starts, ends, steps) + slice(start, end, step) for start, end, step in zip(starts, ends, steps) ) diff --git a/tests/test_split.py b/tests/test_split.py index b224262aa..01b7b18fd 100644 --- a/tests/test_split.py +++ b/tests/test_split.py @@ -68,12 +68,9 @@ def test_split( rtol, atol, ): - input = randn_strided( - input_shape, input_strides, dtype=dtype, device=device - ) + input = randn_strided(input_shape, input_strides, dtype=dtype, device=device) output_shapes = tuple( - output.shape - for output in torch.split(input, split_size, dim=dim) + output.shape for output in torch.split(input, split_size, dim=dim) ) outs = [ empty_strided(shape, None, dtype=dtype, device=device) diff --git a/tests/test_stack_backward.py b/tests/test_stack_backward.py index 928a6c367..c50931b85 100644 --- a/tests/test_stack_backward.py +++ b/tests/test_stack_backward.py @@ -41,6 +41,7 @@ ), ) + @pytest.mark.auto_act_and_assert @pytest.mark.parametrize( ( @@ -74,9 +75,7 @@ def test_stack_backward( ): normalized_dim = dim % len(input_shape + (0,)) grad_output_shape = ( - input_shape[:normalized_dim] - + (num_inputs,) - + input_shape[normalized_dim:] + input_shape[:normalized_dim] + (num_inputs,) + input_shape[normalized_dim:] ) grad_output = randn_strided( grad_output_shape, @@ -128,9 +127,7 @@ def _stack_backward( def _torch_stack_backward(grad_output, dim, grad_inputs): - for grad_input, expected in zip( - grad_inputs, torch.unbind(grad_output, dim=dim) - ): + for grad_input, expected in zip(grad_inputs, torch.unbind(grad_output, dim=dim)): grad_input.copy_(expected) - return torch.stack(grad_inputs, dim=dim) \ No newline at end of file + return torch.stack(grad_inputs, dim=dim) From 9e92a80fbf008957085c5228c45d1fa87d914595 Mon Sep 17 00:00:00 2001 From: zhoujf1 <2047484072@qq.com> Date: Wed, 18 Nov 2026 01:12:40 +0800 Subject: [PATCH 3/4] fix: address review feedback for ATen operators --- scripts/generate_torch_ops.py | 3594 ++++++++--------- scripts/generate_wrappers.py | 2 +- scripts/torch_ops.yaml | 4 + src/base/accumulate_grad.h | 112 +- src/base/adam_accumulate_grad.h | 212 +- src/base/add_backward.h | 136 +- src/base/cross_entropy.h | 144 +- src/base/cross_entropy_backward.h | 175 +- src/base/cross_entropy_common.h | 78 + src/base/embedding_backward.h | 91 - src/base/masked_fill_backward.h | 183 +- src/base/mean_backward.h | 162 +- src/base/mul_backward.h | 183 +- src/base/pow_backward.h | 149 +- src/base/repeat_interleave.h | 138 +- src/base/repeat_interleave_backward.h | 150 +- src/base/rsqrt_backward.h | 129 +- src/base/slice.h | 86 - src/base/slice_backward.h | 91 - src/base/split.h | 179 +- src/base/stack_backward.h | 163 +- src/base/sub_backward.h | 136 +- src/base/transpose.h | 70 - src/base/transpose_backward.h | 142 +- src/base/validation.h | 50 + .../vocab_parallel_cross_entropy_backward.h | 225 +- .../ops/accumulate_grad/accumulate_grad.cc | 12 +- .../ops/accumulate_grad/accumulate_grad.h | 3 - .../adam_accumulate_grad.cc | 29 +- .../adam_accumulate_grad.h | 10 - src/torch/ops/add_backward/add_backward.cc | 45 +- src/torch/ops/add_backward/add_backward.h | 10 +- src/torch/ops/cross_entropy/cross_entropy.cc | 52 +- src/torch/ops/cross_entropy/cross_entropy.h | 11 +- .../cross_entropy_backward.cc | 157 +- .../cross_entropy_backward.h | 43 +- .../embedding_backward/embedding_backward.cc | 47 - .../embedding_backward/embedding_backward.h | 27 - .../masked_fill_backward.cc | 22 +- .../masked_fill_backward.h | 3 - src/torch/ops/mean_backward/mean_backward.cc | 10 +- src/torch/ops/mean_backward/mean_backward.h | 7 +- src/torch/ops/mul_backward/mul_backward.cc | 63 +- src/torch/ops/mul_backward/mul_backward.h | 12 +- src/torch/ops/pow_backward/pow_backward.cc | 26 +- src/torch/ops/pow_backward/pow_backward.h | 3 - .../repeat_interleave/repeat_interleave.cc | 3 +- .../ops/repeat_interleave/repeat_interleave.h | 3 - .../repeat_interleave_backward.cc | 3 +- .../repeat_interleave_backward.h | 3 - .../ops/rsqrt_backward/rsqrt_backward.cc | 12 +- src/torch/ops/rsqrt_backward/rsqrt_backward.h | 3 - src/torch/ops/slice/slice.cc | 43 - src/torch/ops/slice/slice.h | 25 - .../ops/slice_backward/slice_backward.cc | 46 - src/torch/ops/slice_backward/slice_backward.h | 26 - src/torch/ops/split/split.cc | 84 +- src/torch/ops/split/split.h | 3 - .../ops/stack_backward/stack_backward.cc | 85 +- src/torch/ops/stack_backward/stack_backward.h | 7 +- src/torch/ops/sub_backward/sub_backward.cc | 45 +- src/torch/ops/sub_backward/sub_backward.h | 10 +- src/torch/ops/transpose/transpose.cc | 31 - src/torch/ops/transpose/transpose.h | 23 - .../transpose_backward/transpose_backward.cc | 5 +- .../transpose_backward/transpose_backward.h | 3 - .../vocab_parallel_cross_entropy_backward.cc | 28 +- src/torch/storage_overlap.h | 23 + tests/test_accumulate_grad.py | 250 +- tests/test_adam_accumulate_grad.py | 460 ++- tests/test_add_backward.py | 493 ++- tests/test_cross_entropy.py | 672 ++- tests/test_cross_entropy_backward.py | 459 ++- tests/test_embedding_backward.py | 177 - tests/test_embedding_dense_backward.py | 590 +++ tests/test_generate_torch_ops.py | 1074 ++--- tests/test_generate_wrappers.py | 24 + tests/test_masked_fill_backward.py | 611 ++- tests/test_mean_backward.py | 18 +- tests/test_mul_backward.py | 600 ++- tests/test_pow_backward.py | 241 +- tests/test_rsqrt_backward.py | 42 +- tests/test_slice.py | 134 - tests/test_slice_backward.py | 247 +- tests/test_slice_copy.py | 84 + tests/test_split.py | 62 + tests/test_stack_backward.py | 61 + tests/test_sub_backward.py | 501 ++- tests/test_transpose.py | 128 - tests/test_transpose_backward.py | 297 +- tests/test_transpose_copy.py | 416 ++ ...t_vocab_parallel_cross_entropy_backward.py | 72 +- 92 files changed, 9174 insertions(+), 6429 deletions(-) create mode 100644 src/base/cross_entropy_common.h delete mode 100644 src/base/embedding_backward.h delete mode 100644 src/base/slice.h delete mode 100644 src/base/slice_backward.h delete mode 100644 src/base/transpose.h create mode 100644 src/base/validation.h delete mode 100644 src/torch/ops/embedding_backward/embedding_backward.cc delete mode 100644 src/torch/ops/embedding_backward/embedding_backward.h delete mode 100644 src/torch/ops/slice/slice.cc delete mode 100644 src/torch/ops/slice/slice.h delete mode 100644 src/torch/ops/slice_backward/slice_backward.cc delete mode 100644 src/torch/ops/slice_backward/slice_backward.h delete mode 100644 src/torch/ops/transpose/transpose.cc delete mode 100644 src/torch/ops/transpose/transpose.h create mode 100644 src/torch/storage_overlap.h delete mode 100644 tests/test_embedding_backward.py create mode 100644 tests/test_embedding_dense_backward.py delete mode 100644 tests/test_slice.py create mode 100644 tests/test_slice_copy.py delete mode 100644 tests/test_transpose.py create mode 100644 tests/test_transpose_copy.py diff --git a/scripts/generate_torch_ops.py b/scripts/generate_torch_ops.py index 4fd92024b..a67818d16 100644 --- a/scripts/generate_torch_ops.py +++ b/scripts/generate_torch_ops.py @@ -1,1797 +1,1797 @@ -"""Generate InfiniOps PyTorch wrappers from ATen `native_functions.yaml`. - -For each op listed in `scripts/torch_ops.yaml`, this script finds the `.out` -variant in PyTorch's locally installed ATen schemas, including variants declared -through `autogen`, and emits: - - `generated/base/.h` — the InfiniOps base class - `class : public Operator<>`, with constructors and pure-virtual - `operator()` overloads mirroring the selected ATen schemas. - - `generated/torch//.h` and `.cc` — the PyTorch backend - `Operator<, kDev, 8>` that calls `at::_out(out, ...)`. - - `generated/torch_ops_metadata.json` — the kind (`unary` / `binary` / - `binary_alpha`) of every successfully-generated op, consumed by the - parametrized test suite. - -Slot 8 is the reserved convention for PyTorch backends; slots 0-7 are -left for native or vendor implementations. (The slot must also be > 0 -to side-step a partial-specialization-after-instantiation conflict with -the primary template `Operator<>` instantiated at index 0.) - -The generated files are not committed; CMake regenerates them at configure -time when `WITH_TORCH=ON`. -""" - -import argparse -import collections -import dataclasses -import importlib.util -import json -import os -import pathlib -import re -import shutil -import subprocess -import sys - -import yaml - -_SCRIPTS_DIR = pathlib.Path(__file__).resolve().parent -sys.path.insert(0, str(_SCRIPTS_DIR)) -import ops_config # noqa: E402 - -_REPO_ROOT = _SCRIPTS_DIR.parent -_OPS_YAML_PATH = _SCRIPTS_DIR / "torch_ops.yaml" -_BASE_DIR = _REPO_ROOT / "src" / "base" -_GENERATED_DIR = _REPO_ROOT / "generated" -_GENERATED_BASE_DIR = _GENERATED_DIR / "base" -_GENERATED_TORCH_DIR = _GENERATED_DIR / "torch" -_METADATA_PATH = _GENERATED_DIR / "torch_ops_metadata.json" - -# Reserved slot for PyTorch backends. Native and vendor implementations -# claim slots 0-7; PyTorch wrappers always live at 8. -_PYTORCH_SLOT = 8 - -# ATen uses symbolic names for some `int`/`float` defaults (e.g. -# `reduction=Mean`). Map them to C++ identifiers usable in a call. -_ENUM_DEFAULTS = { - "Mean": "at::Reduction::Mean", - "Sum": "at::Reduction::Sum", - "Contiguous": "at::MemoryFormat::Contiguous", -} - -# Default PyTorch schema label used only in diagnostics when CMake does not -# provide the locally installed torch version. Codegen reads the actual schema -# from installed `torchgen/packaged/ATen/native/native_functions.yaml`. -_DEFAULT_PYTORCH_VERSION = "v2.4.0" - -# YAML scalar-type tokens → C++ types. Reference types (e.g. `const Scalar&`) -# are not used so the generated signatures match the existing hand-written -# ones, which pass by value to keep pybind11 binding generation simple. -_SCALAR_TYPE_MAP = { - # `at::Scalar` is implicitly constructible from `double`, so we expose - # scalars as `double` in the base class to keep it torch-independent. - "Scalar": "double", - "int": "int64_t", - "bool": "bool", - "float": "double", - # `SymInt` / `SymInt[]` exist for `torch.compile` internals; at runtime - # they're just `int64`/IntArrayRef. - "SymInt": "int64_t", - # `str` for required string params (e.g. `index_reduce.reduce`). - # `std::string` marshals through pybind11 cleanly and converts - # implicitly to ATen's `c10::string_view`. - "str": "std::string", - "ScalarType": "DataType", -} - -# `Dimname` overloads (named-tensor dim) are skipped — passing them -# from Python to ATen requires a wrapper conversion through -# `at::Dimname::fromSymbol(...)` that doesn't fit the cleanly-rendered -# 1:1 arg model, and named tensors remain experimental in PyTorch. -# The int-dim overload is always emitted alongside, so we lose nothing -# user-visible. - -# Optional ATen types and their typed empty optional values at the call site. -# We use typed `c10::optional{}` values rather than bare `at::nullopt` so -# the compiler can disambiguate ops with multiple `_out` overloads (e.g. -# `clamp_out` accepts both `optional` and `optional` for -# `min`/`max`). -_NULLOPT_BY_TYPE = { - "Scalar?": "c10::optional{}", - "int?": "c10::optional{}", - "bool?": "c10::optional{}", - "float?": "c10::optional{}", - "str?": "c10::optional{}", - "ScalarType?": "c10::optional{}", - "MemoryFormat?": "c10::optional{}", - "Layout?": "c10::optional{}", - "Device?": "c10::optional{}", - "Generator?": "c10::optional{}", - "Tensor?": "c10::optional{}", - "Tensor?[]": "c10::List>{}", - "int[]?": "c10::optional{}", - "int[1]?": "c10::optional{}", - "int[2]?": "c10::optional{}", - "int[3]?": "c10::optional{}", - "SymInt?": "c10::optional{}", - "SymInt[]?": "c10::optional{}", - "SymInt[1]?": "c10::optional{}", - "SymInt[2]?": "c10::optional{}", - "SymInt[3]?": "c10::optional{}", - "float[]?": "c10::optional>{}", -} - -# Optional ATen types that have a stable InfiniOps representation. These are -# exposed in generated base signatures and converted back to ATen optionals in -# the PyTorch backend. PyTorch-internal concepts without a public InfiniOps -# analogue stay hidden and are still passed as typed empty optionals. -_EXPOSED_OPTIONAL_CPP_TYPES = { - "Scalar?": "std::optional", - "int?": "std::optional", - "bool?": "std::optional", - "float?": "std::optional", - "str?": "std::optional", - "ScalarType?": "std::optional", - "Tensor?": "std::optional", - "Tensor?[]": "std::vector>", - "int[]?": "std::optional>", - "int[1]?": "std::optional>", - "int[2]?": "std::optional>", - "int[3]?": "std::optional>", - "SymInt?": "std::optional", - "SymInt[]?": "std::optional>", - "SymInt[1]?": "std::optional>", - "SymInt[2]?": "std::optional>", - "SymInt[3]?": "std::optional>", - "float[]?": "std::optional>", -} -_HARDCODE_NULLOPT_TYPES = frozenset( - set(_NULLOPT_BY_TYPE) - set(_EXPOSED_OPTIONAL_CPP_TYPES) -) - - -def _normalize_cpp_type(cpp_type: str) -> str: - text = cpp_type.strip() - - if text.startswith("const "): - text = text[len("const ") :] - - return text.rstrip("&").strip() - - -def _write_text_if_changed(path: pathlib.Path, content: str) -> bool: - """Write `content` only when the file's bytes would change.""" - path.parent.mkdir(parents=True, exist_ok=True) - - if path.exists() and path.read_text() == content: - return False - - path.write_text(content) - return True - - -def _prune_empty_dirs(root: pathlib.Path) -> None: - if not root.exists(): - return - - for child in sorted(root.iterdir(), reverse=True): - if child.is_dir(): - _prune_empty_dirs(child) - - if root.is_dir() and not any(root.iterdir()): - root.rmdir() - - -def _remove_stale_files(root: pathlib.Path, expected_files: set[pathlib.Path]) -> None: - if not root.exists(): - return - - for path in sorted(root.rglob("*"), reverse=True): - if path.is_file() and path not in expected_files: - path.unlink() - - _prune_empty_dirs(root) - - -@dataclasses.dataclass -class Param: - name: str - aten_type: str - default: str | None - keyword_only: bool - cpp_type_override: str | None = None - - @property - def api_name(self) -> str: - if self.name == "self": - return "input" - - return self.name - - @property - def is_tensor(self) -> bool: - if self.cpp_type_override is not None: - return _normalize_cpp_type(self.cpp_type_override) == "Tensor" - - # Real tensors only. `Tensor?` and `Tensor[]` have separate handling. - if self.aten_type.endswith("[]"): - return False - - return self.aten_type == "Tensor" or self.aten_type.startswith("Tensor(") - - @property - def is_optional_tensor(self) -> bool: - if self.cpp_type_override is not None: - return ( - _normalize_cpp_type(self.cpp_type_override) == "std::optional" - ) - - return self.aten_type == "Tensor?" - - @property - def is_tensor_list(self) -> bool: - if self.cpp_type_override is not None: - return _normalize_cpp_type(self.cpp_type_override) == "std::vector" - - return self.aten_type == "Tensor[]" - - @property - def is_optional_tensor_list(self) -> bool: - if self.cpp_type_override is not None: - return ( - _normalize_cpp_type(self.cpp_type_override) - == "std::vector>" - ) - - return self.aten_type == "Tensor?[]" - - @property - def is_mutable_tensor(self) -> bool: - # Mutable tensors carry `!` in their alias annotation, e.g. - # `Tensor(a!)`. - - return self.is_tensor and "!" in self.aten_type - - @property - def is_out(self) -> bool: - # In ATen `_out` schemas, output tensors are keyword-only mutable tensor - # params. Some mutable tensors are real inputs (`running_mean` / - # `running_var` in `_batch_norm_with_update`) and must stay in schema - # order, so mutability alone is not enough. - - return self.is_mutable_tensor and self.keyword_only - - @property - def is_hardcoded_nullopt(self) -> bool: - """If `True`, the param is omitted from the user-facing API and - passed as `at::nullopt` to ATen.""" - - return self.aten_type in _HARDCODE_NULLOPT_TYPES - - @property - def is_hidden(self) -> bool: - """True if the param is omitted from the user-facing API. - - Default-valued non-optional params (\\`bool\\`, \\`int\\`, \\`float\\`, - \\`str\\`, \\`int[N]\\`, …) used to be hidden as a convenience, but - reviewers consistently flagged the resulting omissions — - \\`bool upper/transpose/unitriangular\\` on \\`triangular_solve\\`, - \\`int diagonal\\` on \\`triu\\`, \\`str ord\\` on \\`linalg_matrix_norm\\`, - \\`int n\\` on the special chebyshev family, etc. — as missing - semantic controls. They are now exposed and forwarded to ATen. - - Optional ATen types with a stable InfiniOps representation - (\\`Tensor?\\`, \\`Scalar?\\`, \\`int?\\`, …) are exposed as - \\`std::optional\\`. ATen-internal types like - \\`Generator?\\`/\\`Layout?\\` still have no InfiniOps analogue, so - they remain hidden. - """ - - return self.is_hardcoded_nullopt - - def hidden_value(self) -> str: - """C++ literal substituted for a hidden param in the ATen call.""" - - if self.is_hardcoded_nullopt: - return _NULLOPT_BY_TYPE[self.aten_type] - - if self.default == "True": - return "true" - - if self.default == "False": - return "false" - - if self.aten_type.startswith(("int[", "SymInt[")) and self.default is not None: - # `int[N]=[a, b, c]` → `{a, b, c}`; `int[N]=0` (scalar default - # for list type) → `{0, 0, ...}` replicated to size N. - if self.default.startswith("["): - return "{" + self.default[1:-1] + "}" - - size_match = re.search(r"\[(\d+)\]", self.aten_type) - n = int(size_match.group(1)) if size_match else 1 - - return "{" + ", ".join([self.default] * n) + "}" - - if self.aten_type == "str" and self.default is not None: - # YAML uses single-quoted strings (e.g. `'none'`); C++ char - # literals also use single quotes, so swap to doubles. - - return '"' + self.default.strip("'\"") + '"' - - if self.aten_type in {"int", "float", "SymInt"} and self.default is not None: - # Translate known ATen enum defaults to their C++ identifiers. - - return _ENUM_DEFAULTS.get(self.default, self.default) - - raise AssertionError( - f"param {self.name!r} of type {self.aten_type!r} with default " - f"{self.default!r} is not hidden" - ) - - @property - def cpp_type(self) -> str: - if self.cpp_type_override is not None: - return self.cpp_type_override - - if self.is_tensor: - return "Tensor" - - if self.is_tensor_list: - return "std::vector" - - if self.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: - return _EXPOSED_OPTIONAL_CPP_TYPES[self.aten_type] - - if self.is_hidden: - # Not exposed — the ATen call substitutes a hardcoded value - # so the `cpp_type` is irrelevant. - - return "void" - - bare = self.aten_type.rstrip("?") - # Required `int[N]` / `SymInt[N]` (no default) — pybind11 accepts - # a Python list of ints into `std::vector`, which ATen - # promotes to `IntArrayRef` implicitly. - if bare.startswith(("int[", "SymInt[")) or bare in {"int[]", "SymInt[]"}: - return "std::vector" - - try: - return _SCALAR_TYPE_MAP[bare] - except KeyError as exc: - raise NotImplementedError( - f"unsupported ATen type {self.aten_type!r} for param {self.name!r}" - ) from exc - - -@dataclasses.dataclass -class Op: - aten_name: str - overload: str - params: list[Param] - signature_params: list[Param] | None = None - param_bindings: list[Param | None] | None = None - explicit_inplace_out: Param | None = None - - @property - def pascal_name(self) -> str: - return _snake_to_pascal(self.infini_name) - - @property - def infini_name(self) -> str: - """InfiniOps public op name. - - ATen disambiguates `_out` overloads with suffixes like `Tensor_Tensor_out`, - `out_x`, `forward_output`, `grad_input`, but reviewers consistently - flag those suffixes as bad public-API naming when they leak into - InfiniOps class names. Different ATen overloads of the same base op - become overloaded `operator()` methods on a single class instead. When - two overloads collapse to the same visible C++ signature after hidden - defaults, `_dedupe_visible_overloads` keeps only one. - - ATen-internal leading underscores and in-place trailing underscores are - also normalized so generated class names do not collide with existing - public ops (`_softmax` → `AtenSoftmax`, `add_` → `AddInplace`). - """ - - return _public_op_name(self.aten_name) - - @property - def is_inplace(self) -> bool: - """True for ATen's single-underscore in-place variants.""" - - return _is_inplace_aten_name(self.aten_name) - - @property - def tensor_params(self) -> list[Param]: - return [p for p in self.params if p.is_tensor] - - @property - def out_params(self) -> list[Param]: - """Mutable tensor outputs. Most ops have one (`Tensor(a!) out`); - multi-output ops like `frexp` or `sort` have several - (`Tensor(a!) values`, `Tensor(b!) indices`).""" - - if self.explicit_inplace_out is not None: - return [self.explicit_inplace_out] - - if self.is_inplace: - return [self.params[0]] - - return [p for p in self.params if p.is_out] - - @property - def out_param(self) -> Param: - """Single-output convenience. Asserts there's exactly one.""" - outs = self.out_params - assert len(outs) == 1, f"op {self.aten_name!r} has {len(outs)} out tensors" - - return outs[0] - - @property - def visible_params(self) -> list[Param]: - """Params the wrapper exposes to the user; hidden ones (hardcoded - optional nullopt, default-`False`/`True` bools) are filtered.""" - - if self.signature_params is not None: - return self.signature_params - - return [p for p in self.params if not p.is_hidden] - - def api_param_for(self, schema_index: int) -> Param | None: - """Return the public signature param bound to a schema param.""" - - if self.param_bindings is not None: - return self.param_bindings[schema_index] - - param = self.params[schema_index] - - if param.is_hidden: - return None - - return param - - @property - def is_testable(self) -> bool: - """Cheap structural check: at least one out tensor, and the first - constructor parameter is a tensor. The latter is needed because - `Operator::Make(Tensor tensor, Args... args)` dispatches on - `tensor.device()`, so an op like `pow.Scalar_out(Scalar self, - Tensor exponent, *, Tensor(a!) out)` cannot be wired up without - a separate dispatch path. Generators like `arange` / `linspace` - also fall under this rule (no input tensors at all).""" - - if not self.out_params: - return False - - if self.is_inplace: - return self.params[0].is_mutable_tensor - - # `params` includes out tensors at the end; check the first - # non-out param. If there are no non-out params (`empty.out`, - # `arange.out`), this op also fails the dispatch precondition. - non_out = [p for p in self.params if not p.is_out] - - if not non_out: - return False - - return non_out[0].is_tensor or non_out[0].is_tensor_list - - -_FUNC_RE = re.compile( - r"^(?P[a-zA-Z_][a-zA-Z0-9_]*)" - r"(?:\.(?P\w+))?" - r"\((?P.*)\)\s*->\s*.+$" -) - -_ARG_RE = re.compile( - r"^(?P\S+(?:\([^)]*\))?\??)" # type with optional alias and `?` - r"\s+(?P\w+)" - r"(?:\s*=\s*(?P.+))?$" -) - - -def _parse_func(func_str: str) -> Op: - m = _FUNC_RE.match(func_str) - - if not m: - raise ValueError(f"could not parse func: {func_str!r}") - - return Op( - aten_name=m.group("name"), - overload=m.group("overload") or "", - params=_parse_args(m.group("args")), - ) - - -def _parse_args(args_str: str) -> list[Param]: - params: list[Param] = [] - keyword_only = False - - for token in _split_args(args_str): - if token == "*": - keyword_only = True - continue - - params.append(_parse_one_arg(token, keyword_only)) - - return params - - -def _split_args(args_str: str) -> list[str]: - """Split on top-level commas, respecting `(...)` and `[...]`.""" - parts: list[str] = [] - depth = 0 - current: list[str] = [] - - for ch in args_str: - if ch in "([": - depth += 1 - current.append(ch) - elif ch in ")]": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - piece = "".join(current).strip() - - if piece: - parts.append(piece) - - current = [] - else: - current.append(ch) - - tail = "".join(current).strip() - - if tail: - parts.append(tail) - - return parts - - -def _parse_one_arg(token: str, keyword_only: bool) -> Param: - m = _ARG_RE.match(token) - - if not m: - raise ValueError(f"could not parse arg: {token!r}") - - name = m.group("name") - return Param( - name=name, - aten_type=m.group("type"), - default=m.group("default"), - keyword_only=keyword_only, - ) - - -_BASE_OPERATOR_RE = re.compile( - r"\bvirtual\s+void\s+operator\(\)\s*\((?P.*?)\)\s*const" - r"(?:\s*=\s*0|\s*\{)", - re.S, -) - - -def _split_cpp_params(args_str: str) -> list[str]: - parts: list[str] = [] - depth = 0 - current: list[str] = [] - - for ch in args_str: - if ch in "([<": - depth += 1 - current.append(ch) - elif ch in ")]>": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - piece = "".join(current).strip() - - if piece: - parts.append(piece) - - current = [] - else: - current.append(ch) - - tail = "".join(current).strip() - - if tail: - parts.append(tail) - - return parts - - -def _parse_base_signature_param(token: str) -> tuple[str, str]: - token = token.split("=", 1)[0].strip() - token = re.sub(r"\s+", " ", token) - match = re.match(r"(?P.+?)\s+(?P\w+)$", token) - - if not match: - raise ValueError(f"could not parse base operator param: {token!r}") - - return _normalize_cpp_type(match.group("type")), match.group("name") - - -def _parse_base_operator_signatures( - base_path: pathlib.Path, -) -> list[list[tuple[str, str]]]: - source = base_path.read_text() - signatures = [] - - for match in _BASE_OPERATOR_RE.finditer(source): - args = match.group("args").strip() - - if not args: - signatures.append([]) - continue - - signatures.append( - [_parse_base_signature_param(token) for token in _split_cpp_params(args)] - ) - - return signatures - - -def _cpp_types_compatible(schema_cpp_type: str, base_cpp_type: str) -> bool: - schema = _normalize_cpp_type(schema_cpp_type) - base = _normalize_cpp_type(base_cpp_type) - - if schema == base: - return True - - compatible_pairs = { - ("double", "float"), - ("int64_t", "int"), - ("std::optional", "std::optional"), - ("std::optional", "std::optional"), - } - - return (schema, base) in compatible_pairs - - -def _cpp_param_compatible( - schema_param: Param, base_cpp_type: str, base_name: str -) -> bool: - if schema_param.api_name != base_name: - return False - - return _cpp_types_compatible(schema_param.cpp_type, base_cpp_type) - - -def _bind_base_signature(op: Op, signature: list[tuple[str, str]]) -> Op | None: - bindings: list[Param | None] = [None] * len(op.params) - signature_params = [] - explicit_inplace_out = None - - if ( - op.is_inplace - and signature - and signature[-1][1] == "out" - and _cpp_types_compatible("Tensor", signature[-1][0]) - ): - out_cpp_type, out_name = signature[-1] - explicit_inplace_out = Param( - name=out_name, - aten_type="Tensor(a!)", - default=None, - keyword_only=True, - cpp_type_override=out_cpp_type, - ) - signature = signature[:-1] - - for base_cpp_type, base_name in signature: - matched_index = None - - for index, schema_param in enumerate(op.params): - if bindings[index] is not None: - continue - - if _cpp_param_compatible(schema_param, base_cpp_type, base_name): - matched_index = index - break - - if matched_index is None: - return None - - schema_param = op.params[matched_index] - api_param = dataclasses.replace( - schema_param, name=base_name, cpp_type_override=base_cpp_type - ) - bindings[matched_index] = api_param - signature_params.append(api_param) - - if any( - binding is None and not _is_omittable_param(param) - for param, binding in zip(op.params, bindings) - ): - return None - - if explicit_inplace_out is not None: - signature_params.append(explicit_inplace_out) - - return dataclasses.replace( - op, - signature_params=signature_params, - param_bindings=bindings, - explicit_inplace_out=explicit_inplace_out, - ) - - -def _bind_existing_base_overloads( - name: str, ops: list[Op] -) -> tuple[list[Op], list[str]]: - base_path = _base_path(name) - signatures = _parse_base_operator_signatures(base_path) - bound_ops = [] - warnings = [] - - for signature in signatures: - matches = [ - bound - for op in ops - if (bound := _bind_base_signature(op, signature)) is not None - ] - - if not matches: - rendered = ", ".join(f"{cpp_type} {param}" for cpp_type, param in signature) - warnings.append( - f"base overload `operator()({rendered})` does not match any " - "usable ATen schema" - ) - continue - - matches.sort( - key=lambda match: sum( - binding is None for binding in match.param_bindings or () - ) - ) - bound_ops.append(matches[0]) - - return bound_ops, warnings - - -def _snake_to_pascal(s: str) -> str: - return "".join(p.capitalize() for p in s.split("_")) - - -_OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") - - -def _is_inplace_aten_name(name: str) -> bool: - """Return whether `name` is an ATen in-place operator name.""" - - return name.endswith("_") and not name.endswith("__") - - -def _public_op_name(aten_name: str) -> str: - """Map ATen-only spelling to stable InfiniOps public names.""" - - public_name = aten_name - file_prefixes = [] - - if public_name.startswith("_"): - file_prefixes.append("internal") - public_name = public_name.lstrip("_") - - if _is_inplace_aten_name(public_name): - public_name = public_name[:-1] - - return "_".join(file_prefixes + [public_name]) - - -def _op_namespace_parts(name: str) -> tuple[str, ...]: - parts = name.split("_") - namespaces = [] - - if parts and parts[0] == "internal": - namespaces.append(parts.pop(0)) - - if parts and parts[0] in _OP_NAMESPACE_PREFIXES: - namespaces.append(parts.pop(0)) - - return tuple(namespaces) - - -def _op_class_stem(name: str) -> str: - parts = name.split("_") - - if parts and parts[0] == "internal": - parts.pop(0) - - if parts and parts[0] in _OP_NAMESPACE_PREFIXES: - parts.pop(0) - - return "_".join(parts) - - -def _op_class_name(name: str) -> str: - return _snake_to_pascal(_op_class_stem(name)) - - -def _op_cpp_type(name: str) -> str: - parts = (*_op_namespace_parts(name), _op_class_name(name)) - - return "::".join(parts) - - -def _op_namespace_open(name: str) -> str: - parts = _op_namespace_parts(name) - - if not parts: - return "namespace infini::ops {" - - return f"namespace infini::ops::{'::'.join(parts)} {{" - - -def _op_namespace_close(name: str) -> str: - parts = _op_namespace_parts(name) - - if not parts: - return "} // namespace infini::ops" - - return f"}} // namespace infini::ops::{'::'.join(parts)}" - - -def _base_path(op_name: str) -> pathlib.Path: - return _BASE_DIR / f"{op_name}.h" - - -def _is_omittable_param(param: Param) -> bool: - return param.aten_type in _NULLOPT_BY_TYPE or param.default is not None - - -def _default_call_value(param: Param) -> str: - if param.aten_type in _NULLOPT_BY_TYPE: - return _NULLOPT_BY_TYPE[param.aten_type] - - if param.default is not None: - return _translate_default(param) - - return param.hidden_value() - - -def _load_aten_entries(version: str) -> list[dict]: - """Return parsed ATen schemas bundled with installed `torchgen`. - - `WITH_TORCH=ON` already requires a local PyTorch installation. PyTorch - wheels ship the matching ATen schema under `torchgen/packaged`, including - vendor forks, so codegen should not depend on fetching PyTorch sources from - GitHub during CI builds. The official parser expands schemas declared via - `autogen`, such as `relu.out`, before InfiniOps filters for out variants. - """ - - try: - packaged, expanded_autogen = _load_packaged_aten_entries() - except RuntimeError as error: - raise RuntimeError(f"{error} for PyTorch schema {version!r}") from error - - print( - "using installed `torchgen` ATen schemas " - f"for PyTorch schema {version} " - f"({'with' if expanded_autogen else 'without'} `autogen` expansion).", - file=sys.stderr, - ) - - return packaged - - -def _load_packaged_aten_entries() -> tuple[list[dict], bool]: - """Parse the ATen schemas bundled with installed `torchgen`. - - PyTorch wheels install `torchgen/packaged/ATen/native/native_functions.yaml`; - using it lets offline CI images generate wrappers against the exact schema - shipped with their installed torch fork. - """ - - spec = importlib.util.find_spec("torchgen") - - if spec is None: - raise RuntimeError("could not find the installed `torchgen` package") - - if spec.submodule_search_locations is None: - raise RuntimeError("installed `torchgen` is not a package") - - for root in spec.submodule_search_locations: - native_dir = pathlib.Path(root) / "packaged" / "ATen" / "native" - native_path = native_dir / "native_functions.yaml" - tags_path = native_dir / "tags.yaml" - - if not native_path.is_file(): - continue - - if not tags_path.is_file(): - print( - "warning: could not expand ATen `autogen` schemas because " - f"`{tags_path}` is missing; using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - try: - from torchgen.gen import parse_native_yaml - except (ImportError, AttributeError) as error: - print( - "warning: could not expand ATen `autogen` schemas because " - f"`torchgen.gen.parse_native_yaml` is unavailable ({error}); " - "using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - try: - parsed = parse_native_yaml(str(native_path), str(tags_path)) - except Exception as error: - print( - "warning: could not expand ATen `autogen` schemas from " - f"`{native_path}` ({type(error).__name__}: {error}); " - "using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - return ( - [{"func": str(function.func)} for function in parsed.native_functions], - True, - ) - - raise RuntimeError( - "could not find installed `torchgen` packaged `native_functions.yaml`" - ) - - -def _load_explicit_aten_entries(native_path: pathlib.Path) -> list[dict]: - entries = yaml.safe_load(native_path.read_text()) - - if not isinstance(entries, list): - raise RuntimeError( - f"expected a list of ATen schemas in `{native_path}`, " - f"got {type(entries).__name__}" - ) - - return entries - - -def _find_out_entries(entries: list[dict], op_name: str) -> list[dict]: - """Return all out-variant entries for `op_name`, with the bare - `.out(` form first and overload-suffixed variants - (e.g. `pow.Tensor_Tensor_out(`, `kthvalue.values(`) after. An - entry counts as an out-variant when it (a) is named - `.out`, (b) ends in `_out`, or (c) carries a - `Tensor(!)` mutability annotation — that last case covers - multi-output ops named after their output tensors - (`kthvalue.values`, `mode.values`, …).""" - bare_prefix = f"{op_name}.out(" - op_overload = re.compile(rf"^{re.escape(op_name)}\.\w+\(") - mut_tensor = re.compile(r"Tensor\([a-z]!\)") - bare: list[dict] = [] - others: list[dict] = [] - - for entry in entries: - func = entry.get("func", "") - - if func.startswith(bare_prefix): - bare.append(entry) - elif op_overload.match(func) and ( - func.split("(", 1)[0].endswith("_out") or mut_tensor.search(func) - ): - others.append(entry) - - return bare + others - - -def _format_signature(op: Op, *, include_defaults: bool = False) -> str: - parts = [] - - for param in op.visible_params: - prefix = "" if param.is_mutable_tensor else "const " - text = f"{prefix}{param.cpp_type} {param.api_name}" - - if include_defaults and param.default is not None: - text += f" = {_translate_default(param)}" - - parts.append(text) - - return ", ".join(parts) - - -def _visible_signature_key(op: Op) -> tuple[str, ...]: - """C++ overload identity for the user-facing API. - - Parameter names and top-level `const` do not distinguish C++ overloads, so - only the exposed C++ type sequence participates in duplicate detection. - """ - - return tuple(param.cpp_type for param in op.visible_params) - - -def _canonical_overload_score(index: int, op: Op) -> tuple[bool, int, int, str, int]: - """Sort key for duplicate visible signatures. - - Prefer the canonical unsuffixed InfiniOps name, then the schema that hides - fewer ATen-only defaults, then the shorter deterministic name. - """ - - return ( - op.infini_name != op.aten_name, - sum(param.is_hidden for param in op.params), - len(op.infini_name), - op.infini_name, - index, - ) - - -def _dedupe_visible_overloads(ops: list[Op]) -> tuple[list[Op], list[tuple[Op, Op]]]: - """Drop overloads that collapse to the same visible C++ signature. - - Returns the selected overloads in the original schema order plus a list of - `(skipped, kept)` duplicate pairs for diagnostics. - """ - winners: dict[tuple[str, ...], tuple[int, Op]] = {} - duplicates: list[tuple[Op, tuple[str, ...]]] = [] - - for index, op in enumerate(ops): - key = _visible_signature_key(op) - current = winners.get(key) - - if current is None: - winners[key] = (index, op) - continue - - current_index, current_op = current - - if _canonical_overload_score(index, op) < _canonical_overload_score( - current_index, current_op - ): - duplicates.append((current_op, key)) - winners[key] = (index, op) - else: - duplicates.append((op, key)) - - selected_indices = {index for index, _ in winners.values()} - selected = [op for index, op in enumerate(ops) if index in selected_indices] - duplicate_pairs = [ - (skipped, winners[key][1]) - for skipped, key in duplicates - if winners[key][1] is not skipped - ] - - return selected, duplicate_pairs - - -def _translate_default(param: Param) -> str: - """Translate a YAML default literal to a C++ literal.""" - raw = param.default - - if raw == "True": - return "true" - - if raw == "False": - return "false" - - if raw == "None": - return "{}" - - return raw # numeric literals (`0`, `1`, `1.0`) pass through - - -def _generate_base_includes(ops: list[Op]) -> str: - cpp_types = [param.cpp_type for op in ops for param in op.visible_params] - includes = [] - - if any("std::optional" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - if any("std::string" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - if any("std::vector" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - includes.append('#include "operator.h"') - - return "\n".join(includes) - - -def _generate_base_header(name: str, ops: list[Op]) -> str: - pascal = _op_class_name(name) - - member_decls = [] - tensor_member_order = [] - seen_tensor_members = set() - optional_tensor_member_order = [] - seen_optional_tensor_members = set() - scalar_member_order = [] - scalar_member_types = {} - - for op in ops: - for param in op.tensor_params: - api_name = param.api_name - - if api_name in seen_tensor_members: - continue - - seen_tensor_members.add(api_name) - tensor_member_order.append(api_name) - member_decls.append(f" Tensor::Shape {api_name}_shape_;") - member_decls.append(f" Tensor::Strides {api_name}_strides_;") - member_decls.append(f" DataType {api_name}_type_;") - - for param in op.visible_params: - api_name = param.api_name - - if not param.is_optional_tensor or api_name in seen_optional_tensor_members: - continue - - seen_optional_tensor_members.add(api_name) - optional_tensor_member_order.append(api_name) - member_decls.append(f" bool has_{api_name}_{{false}};") - member_decls.append(f" Tensor::Shape {api_name}_shape_;") - member_decls.append(f" Tensor::Strides {api_name}_strides_;") - member_decls.append(f" DataType {api_name}_type_{{DataType::kFloat32}};") - - # Visible non-tensor params (scalars, strings, vectors) are also - # stored on the base so backends can dispatch on them later — not - # only at the moment `operator()` is invoked. Reviewers flagged - # this on multiple PRs (e.g. `n` on - # `special_chebyshev_polynomial_v_n_scalar`). Same-named params - # across overloads must share a type; if they conflict, the second - # overload's member is dropped (later constructors leave it - # default-initialised). - for param in op.visible_params: - api_name = param.api_name - - if ( - param.is_tensor - or param.is_optional_tensor - or api_name in scalar_member_types - ): - continue - - scalar_member_order.append(api_name) - scalar_member_types[api_name] = param.cpp_type - member_decls.append(f" {param.cpp_type} {api_name}_{{}};") - - member_decls.append(" int device_index_{0};") - - constructors = [] - calls = [] - - for op in ops: - init_pieces = [] - tensor_params = {param.api_name: param for param in op.tensor_params} - optional_tensor_params = { - param.api_name: param - for param in op.visible_params - if param.is_optional_tensor - } - scalar_params = { - param.api_name: param - for param in op.visible_params - if not param.is_tensor - and not param.is_optional_tensor - and scalar_member_types.get(param.api_name) == param.cpp_type - } - - for param_name in tensor_member_order: - param = tensor_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" {api_name}_shape_{{{api_name}.shape()}}") - init_pieces.append(f" {api_name}_strides_{{{api_name}.strides()}}") - init_pieces.append(f" {api_name}_type_{{{api_name}.dtype()}}") - - for param_name in optional_tensor_member_order: - param = optional_tensor_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" has_{api_name}_{{{api_name}.has_value()}}") - init_pieces.append( - f" {api_name}_shape_{{{api_name} ? " - f"Tensor::Shape{{{api_name}->shape()}} : Tensor::Shape{{}}}}" - ) - init_pieces.append( - f" {api_name}_strides_{{{api_name} ? " - f"Tensor::Strides{{{api_name}->strides()}} : Tensor::Strides{{}}}}" - ) - init_pieces.append( - f" {api_name}_type_{{{api_name} ? " - f"{api_name}->dtype() : DataType::kFloat32}}" - ) - - for param_name in scalar_member_order: - param = scalar_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" {api_name}_{{{api_name}}}") - - # All out tensors share a device; use the first one. Keep this last - # so initializer order follows the member declaration order. - init_pieces.append( - f" device_index_{{{op.out_params[0].api_name}.device().index()}}" - ) - - init_list = ",\n".join(init_pieces).lstrip() - constructors.append( - f" {pascal}({_format_signature(op)})\n : {init_list} {{}}" - ) - calls.append(f" virtual void operator()({_format_signature(op)}) const = 0;") - - return _BASE_TEMPLATE.format( - name_uc=name.upper(), - pascal=pascal, - namespace_open=_op_namespace_open(name), - namespace_close=_op_namespace_close(name), - includes=_generate_base_includes(ops), - constructors="\n\n".join(constructors), - op_calls="\n\n".join(calls), - member_decls="\n\n".join(member_decls), - ) - - -def _generate_torch_header(name: str, ops: list[Op]) -> str: - pascal = _op_class_name(name) - op_type = _op_cpp_type(name) - op_calls = "\n\n".join( - f" void operator()({_format_signature(op)}) const override;" for op in ops - ) - - return _TORCH_HEADER_TEMPLATE.format( - name_uc=name.upper(), - name=name, - pascal=pascal, - op_type=op_type, - op_calls=op_calls, - slot=_PYTORCH_SLOT, - ) - - -def _generate_torch_method_source(name: str, op: Op) -> str: - op_type = _op_cpp_type(name) - conversion_lines = [] - out_device_index = f"{op.out_params[0].api_name}.device().index()" - - def _optional_aten_type(param: Param) -> str: - return _NULLOPT_BY_TYPE[param.aten_type].removesuffix("{}") - - def _optional_aten_value(schema_param: Param, api_param: Param) -> str: - api_name = api_param.api_name - - if schema_param.aten_type == "Tensor?": - data_expr = f"const_cast({api_name}->data())" - - return ( - f"ToAtenTensor({data_expr}, {api_name}->shape(), " - f"{api_name}->strides(), {api_name}->dtype(), " - f"device_index)" - ) - - if schema_param.aten_type == "Scalar?": - return f"at::Scalar(*{api_name})" - - if schema_param.aten_type == "ScalarType?": - return f"ToAtenDataType(*{api_name})" - - if schema_param.aten_type == "str?": - return f"c10::string_view(*{api_name})" - - if schema_param.aten_type.startswith( - ("int[", "SymInt[") - ) or schema_param.aten_type in { - "int[]?", - "SymInt[]?", - }: - return f"at::IntArrayRef(*{api_name})" - - if schema_param.aten_type == "float[]?": - return f"at::ArrayRef(*{api_name})" - - return f"*{api_name}" - - def _append_optional_conversion(schema_param: Param, api_param: Param) -> None: - api_name = api_param.api_name - optional_type = _optional_aten_type(schema_param) - conversion_lines.append(f" {optional_type} at_{schema_param.name};") - conversion_lines.append(f" if ({api_name}.has_value()) {{") - conversion_lines.append( - f" at_{schema_param.name} = " - f"{optional_type}{{{_optional_aten_value(schema_param, api_param)}}};" - ) - conversion_lines.append(" }") - - for schema_index, param in enumerate(op.params): - if ( - not param.is_tensor - and not param.is_tensor_list - and not param.is_optional_tensor_list - ): - continue - - api_param = op.api_param_for(schema_index) - - if api_param is None: - continue - - api_name = api_param.api_name - - if param.is_optional_tensor_list: - conversion_lines.append( - f" c10::List> at_{param.name};" - ) - conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") - conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") - conversion_lines.append(" if (tensor.has_value()) {") - conversion_lines.append( - " at_" - f"{param.name}.push_back(ToAtenTensor(" - "const_cast(tensor->data()), tensor->shape(), " - "tensor->strides(), tensor->dtype(), device_index));" - ) - conversion_lines.append(" } else {") - conversion_lines.append(f" at_{param.name}.push_back(c10::nullopt);") - conversion_lines.append(" }") - conversion_lines.append(" }") - continue - - if param.is_tensor_list: - conversion_lines.append(f" std::vector at_{param.name};") - conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") - conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") - conversion_lines.append( - " at_" - f"{param.name}.push_back(ToAtenTensor(" - "const_cast(tensor.data()), tensor.shape(), tensor.strides(), " - "tensor.dtype(), device_index));" - ) - conversion_lines.append(" }") - continue - - use_mutable_data = param.is_mutable_tensor and not ( - op.explicit_inplace_out is not None and schema_index == 0 - ) - data_expr = ( - f"{api_name}.data()" - if use_mutable_data - else f"const_cast({api_name}.data())" - ) - conversion_lines.append( - f" auto at_{param.name} = ToAtenTensor(\n" - f" {data_expr}, {api_name}_shape_, {api_name}_strides_,\n" - f" {api_name}_type_, device_index);" - ) - - if op.explicit_inplace_out is not None: - out_name = op.explicit_inplace_out.api_name - conversion_lines.append( - f" auto at_{out_name} = ToAtenTensor(\n" - f" {out_name}.data(), {out_name}_shape_, {out_name}_strides_,\n" - f" {out_name}_type_, device_index);" - ) - - for schema_index, param in enumerate(op.params): - api_param = op.api_param_for(schema_index) - - if ( - api_param is not None - and param.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES - and not param.is_optional_tensor_list - ): - _append_optional_conversion(param, api_param) - - def _render_arg(schema_index, p): - api_param = op.api_param_for(schema_index) - - if api_param is None: - return _default_call_value(p) - - if p.is_hidden: - return _default_call_value(p) - - if p.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: - return f"at_{p.name}" - - if p.is_tensor or p.is_tensor_list: - return f"at_{p.name}" - - if p.aten_type == "ScalarType": - return f"ToAtenDataType({api_param.api_name})" - - return api_param.api_name - - if op.is_inplace: - # In-place ATen calls keep the mutable input in positional order, - # unlike `_out` calls which place output tensors first. - input_param = op.params[0] - arg_order = op.params[1:] - rendered_args = ", ".join( - _render_arg(index + 1, p) for index, p in enumerate(arg_order) - ) - if op.explicit_inplace_out is not None: - out_name = op.explicit_inplace_out.api_name - aten_call = f"at_{out_name}.copy_(at_{input_param.name});\n" - aten_call += f" at_{out_name}.{op.aten_name}({rendered_args})" - else: - aten_call = f"at_{input_param.name}.{op.aten_name}({rendered_args})" - else: - # ATen `_out` form puts all out tensors first, then non-out params - # in YAML order. Hardcoded-nullopt params become `at::nullopt`. - arg_order = [(index, p) for index, p in enumerate(op.params) if p.is_out] + [ - (index, p) for index, p in enumerate(op.params) if not p.is_out - ] - rendered_args = ", ".join(_render_arg(index, p) for index, p in arg_order) - aten_call = f"at::{op.aten_name}_out({rendered_args})" - - return _TORCH_METHOD_TEMPLATE.format( - op_type=op_type, - op_call_signature=_format_signature(op), - tensor_conversions="\n".join(conversion_lines), - out_device_index=out_device_index, - # The generated call expression resolves the right kernel via C++ - # overload resolution from the argument types we pass. - aten_call=aten_call, - slot=_PYTORCH_SLOT, - ) - - -def _generate_torch_source(name: str, ops: list[Op]) -> str: - op_type = _op_cpp_type(name) - methods = "\n\n".join(_generate_torch_method_source(name, op) for op in ops) - return _TORCH_SOURCE_TEMPLATE.format( - name=name, - methods=methods, - op_type=op_type, - slot=_PYTORCH_SLOT, - ) - - -_BASE_TEMPLATE = """\ -#ifndef INFINI_OPS_BASE_{name_uc}_H_ -#define INFINI_OPS_BASE_{name_uc}_H_ - -{includes} - -{namespace_open} - -class {pascal} : public Operator<{pascal}> {{ - public: -{constructors} - -{op_calls} - - protected: -{member_decls} -}}; - -{namespace_close} - -#endif -""" - - -_TORCH_HEADER_TEMPLATE = """\ -#ifndef INFINI_OPS_TORCH_{name_uc}_H_ -#define INFINI_OPS_TORCH_{name_uc}_H_ - -#include "base/{name}.h" - -namespace infini::ops {{ - -template -class Operator<{op_type}, kDev, {slot}> : public {op_type} {{ - public: - using {op_type}::{pascal}; - -{op_calls} -}}; - -}} // namespace infini::ops - -#endif -""" - - -_TORCH_METHOD_TEMPLATE = """\ -template -void Operator<{op_type}, kDev, {slot}>::operator()({op_call_signature}) const {{ - const auto device_index = {out_device_index}; - const TorchStreamGuard stream_guard{{stream_, device_index}}; -{tensor_conversions} - - {aten_call}; -}} -""" - - -_TORCH_SOURCE_TEMPLATE = """\ -#include "torch/{name}/{name}.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops {{ - -{methods} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR({op_type}, {slot}) - -}} // namespace infini::ops -""" - - -def _find_clang_format() -> str | None: - """Return the path to a `clang-format` binary, or `None` if unavailable. - - Generated files live under `generated/`, which is gitignored, so offline CI - containers should not block or try network access just to format them. - """ - - found = shutil.which("clang-format") - - if found: - return found - - print( - "`clang-format` not found on PATH; generated files will be emitted " - "without formatting.", - file=sys.stderr, - ) - - return None - - -def _clang_format(text: str, path: pathlib.Path) -> str: - """Pipe `text` through `clang-format` so generated headers / sources - satisfy the same style check (`clang-format` v21) that CI runs. - `path` informs include sorting (the file's own header should come - first in a `.cc`). If no `clang-format` binary is available, return - the input unchanged.""" - - if _CLANG_FORMAT is None: - return text - - return subprocess.run( - [_CLANG_FORMAT, f"--assume-filename={path}"], - input=text, - capture_output=True, - text=True, - check=True, - ).stdout - - -def _emit(name: str, ops: list[Op], *, emit_base: bool) -> set[pathlib.Path]: - base_path = _GENERATED_BASE_DIR / f"{name}.h" - torch_dir = _GENERATED_TORCH_DIR / name - torch_header_path = torch_dir / f"{name}.h" - torch_source_path = torch_dir / f"{name}.cc" - emitted_paths = set() - - if emit_base: - _write_text_if_changed( - base_path, _clang_format(_generate_base_header(name, ops), base_path) - ) - emitted_paths.add(base_path) - - _write_text_if_changed( - torch_header_path, - _clang_format(_generate_torch_header(name, ops), torch_header_path), - ) - _write_text_if_changed( - torch_source_path, - _clang_format(_generate_torch_source(name, ops), torch_source_path), - ) - emitted_paths.add(torch_header_path) - emitted_paths.add(torch_source_path) - - return emitted_paths - - -def _select_op_names(cli_ops, default_ops, config): - if config is None: - return cli_ops or default_ops - - aten_names_by_public_name = collections.defaultdict(list) - for op_name in default_ops: - aten_names_by_public_name[_public_op_name(op_name)].append(op_name) - - selected_public_names = ops_config.torch_op_names( - config, aten_names_by_public_name, _PYTORCH_SLOT - ) - selected = [ - aten_name - for public_name in selected_public_names - for aten_name in aten_names_by_public_name.get(public_name, (public_name,)) - ] - - if cli_ops: - allowed = set(cli_ops) - selected = [op_name for op_name in selected if op_name in allowed] - - return selected - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument( - "--ops", - nargs="*", - help="Override the op allowlist. If omitted, reads `scripts/torch_ops.yaml`.", - ) - parser.add_argument( - "--ops-config", - type=pathlib.Path, - help="Path to an `ops.json` operator and implementation selection.", - ) - parser.add_argument( - "--pytorch-version", - default=os.environ.get("INFINI_OPS_PYTORCH_VERSION", _DEFAULT_PYTORCH_VERSION), - help=( - "PyTorch version label used in diagnostics. Schemas are read from " - "the locally installed `torchgen` package. Default: `%(default)s`. " - "Can also be set via the `INFINI_OPS_PYTORCH_VERSION` environment " - "variable." - ), - ) - args = parser.parse_args() - - global _CLANG_FORMAT - _CLANG_FORMAT = _find_clang_format() - - default_ops = yaml.safe_load(_OPS_YAML_PATH.read_text()) - config = ops_config.load_ops_config(args.ops_config) if args.ops_config else None - op_names = _select_op_names(args.ops, default_ops, config) - aten_entries = _load_aten_entries(args.pytorch_version) - - skipped: list[tuple[str, str]] = [] - metadata: list[dict] = [] - expected_base_files: set[pathlib.Path] = set() - expected_torch_files: set[pathlib.Path] = set() - ops_by_public_name: dict[str, list[Op]] = collections.defaultdict(list) - - for name in op_names: - candidates = _find_out_entries(aten_entries, name) - - if not candidates: - skipped.append((name, f"no `.out` variant for `{name}` in YAML")) - continue - - usable: list[Op] = [] - last_reason = "" - - for entry in candidates: - try: - op = _parse_func(entry["func"]) - - for param in op.params: - _ = param.cpp_type # eagerly raise on unsupported types - except (NotImplementedError, ValueError) as exc: - last_reason = str(exc) - continue - - if not op.is_testable: - last_reason = "no testable tensor input/output pair" - continue - - usable.append(op) - - if not usable: - skipped.append((name, last_reason or "no usable overload")) - continue - - usable, duplicate_overloads = _dedupe_visible_overloads(usable) - - for skipped_op, kept_op in duplicate_overloads: - skipped.append( - ( - skipped_op.infini_name, - "duplicate visible C++ signature for " - f"`{name}`; using `{kept_op.infini_name}`", - ) - ) - - ops_by_public_name[usable[0].infini_name].extend(usable) - - for public_name, usable in ops_by_public_name.items(): - usable, duplicate_overloads = _dedupe_visible_overloads(usable) - - for skipped_op, kept_op in duplicate_overloads: - skipped.append( - ( - skipped_op.infini_name, - "duplicate visible C++ signature for " - f"`{public_name}`; using `{kept_op.infini_name}`", - ) - ) - - base_exists = _base_path(public_name).exists() - - if base_exists: - usable, base_warnings = _bind_existing_base_overloads(public_name, usable) - - for warning in base_warnings: - skipped.append((public_name, warning)) - - if not usable: - skipped.append( - ( - public_name, - "existing base has no overload compatible with ATen schema", - ) - ) - continue - - # Emit one InfiniOps wrapper per ATen op. Distinct visible overloads - # become overloaded constructors / `operator()` methods on the same - # class (`Pow` exposes both tensor and scalar exponents). Overloads - # that collapse to the same C++ signature after hidden defaults are - # skipped above. When a hand-written `src/base/.h` exists, - # skip emitting `generated/base/.h` so the hand-written one - # wins (the generated torch source's `#include "base/.h"` - # resolves through `src/` first). Signature mismatches surface as - # compile errors with a clear message — drop the op from the YAML - # to suppress. - emitted_paths = _emit(public_name, usable, emit_base=not base_exists) - expected_base_files.update( - path for path in emitted_paths if path.is_relative_to(_GENERATED_BASE_DIR) - ) - expected_torch_files.update( - path for path in emitted_paths if path.is_relative_to(_GENERATED_TORCH_DIR) - ) - - for op in usable: - metadata.append( - { - "name": public_name, - "aten_name": op.aten_name, - "overload_name": op.infini_name, - "params": [ - { - "name": p.api_name, - "type": p.aten_type, - "is_tensor": p.is_tensor, - "is_out": p.is_out, - "schema_index": next( - ( - schema_index - for schema_index in range(len(op.params)) - if op.api_param_for(schema_index) is p - ), - None, - ), - } - for p in op.visible_params - ], - } - ) - - _GENERATED_DIR.mkdir(parents=True, exist_ok=True) - _write_text_if_changed( - _METADATA_PATH, json.dumps({"ops": metadata}, indent=2) + "\n" - ) - _remove_stale_files(_GENERATED_BASE_DIR, expected_base_files) - _remove_stale_files(_GENERATED_TORCH_DIR, expected_torch_files) - - generated_names = sorted({m["name"] for m in metadata}) - print( - f"generated {len(metadata)} overloads across {len(generated_names)} ops: " - f"{generated_names}" - ) - - for name, reason in skipped: - print(f" skipped {name!r}: {reason}", file=sys.stderr) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""Generate InfiniOps PyTorch wrappers from ATen `native_functions.yaml`. + +For each op listed in `scripts/torch_ops.yaml`, this script finds the `.out` +variant in PyTorch's locally installed ATen schemas, including variants declared +through `autogen`, and emits: + - `generated/base/.h` — the InfiniOps base class + `class : public Operator<>`, with constructors and pure-virtual + `operator()` overloads mirroring the selected ATen schemas. + - `generated/torch//.h` and `.cc` — the PyTorch backend + `Operator<, kDev, 8>` that calls `at::_out(out, ...)`. + - `generated/torch_ops_metadata.json` — the kind (`unary` / `binary` / + `binary_alpha`) of every successfully-generated op, consumed by the + parametrized test suite. + +Slot 8 is the reserved convention for PyTorch backends; slots 0-7 are +left for native or vendor implementations. (The slot must also be > 0 +to side-step a partial-specialization-after-instantiation conflict with +the primary template `Operator<>` instantiated at index 0.) + +The generated files are not committed; CMake regenerates them at configure +time when `WITH_TORCH=ON`. +""" + +import argparse +import collections +import dataclasses +import importlib.util +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys + +import yaml + +_SCRIPTS_DIR = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPTS_DIR)) +import ops_config # noqa: E402 + +_REPO_ROOT = _SCRIPTS_DIR.parent +_OPS_YAML_PATH = _SCRIPTS_DIR / "torch_ops.yaml" +_BASE_DIR = _REPO_ROOT / "src" / "base" +_GENERATED_DIR = _REPO_ROOT / "generated" +_GENERATED_BASE_DIR = _GENERATED_DIR / "base" +_GENERATED_TORCH_DIR = _GENERATED_DIR / "torch" +_METADATA_PATH = _GENERATED_DIR / "torch_ops_metadata.json" + +# Reserved slot for PyTorch backends. Native and vendor implementations +# claim slots 0-7; PyTorch wrappers always live at 8. +_PYTORCH_SLOT = 8 + +# ATen uses symbolic names for some `int`/`float` defaults (e.g. +# `reduction=Mean`). Map them to C++ identifiers usable in a call. +_ENUM_DEFAULTS = { + "Mean": "at::Reduction::Mean", + "Sum": "at::Reduction::Sum", + "Contiguous": "at::MemoryFormat::Contiguous", +} + +# Default PyTorch schema label used only in diagnostics when CMake does not +# provide the locally installed torch version. Codegen reads the actual schema +# from installed `torchgen/packaged/ATen/native/native_functions.yaml`. +_DEFAULT_PYTORCH_VERSION = "v2.4.0" + +# YAML scalar-type tokens → C++ types. Reference types (e.g. `const Scalar&`) +# are not used so the generated signatures match the existing hand-written +# ones, which pass by value to keep pybind11 binding generation simple. +_SCALAR_TYPE_MAP = { + # `at::Scalar` is implicitly constructible from `double`, so we expose + # scalars as `double` in the base class to keep it torch-independent. + "Scalar": "double", + "int": "int64_t", + "bool": "bool", + "float": "double", + # `SymInt` / `SymInt[]` exist for `torch.compile` internals; at runtime + # they're just `int64`/IntArrayRef. + "SymInt": "int64_t", + # `str` for required string params (e.g. `index_reduce.reduce`). + # `std::string` marshals through pybind11 cleanly and converts + # implicitly to ATen's `c10::string_view`. + "str": "std::string", + "ScalarType": "DataType", +} + +# `Dimname` overloads (named-tensor dim) are skipped — passing them +# from Python to ATen requires a wrapper conversion through +# `at::Dimname::fromSymbol(...)` that doesn't fit the cleanly-rendered +# 1:1 arg model, and named tensors remain experimental in PyTorch. +# The int-dim overload is always emitted alongside, so we lose nothing +# user-visible. + +# Optional ATen types and their typed empty optional values at the call site. +# We use typed `c10::optional{}` values rather than bare `at::nullopt` so +# the compiler can disambiguate ops with multiple `_out` overloads (e.g. +# `clamp_out` accepts both `optional` and `optional` for +# `min`/`max`). +_NULLOPT_BY_TYPE = { + "Scalar?": "c10::optional{}", + "int?": "c10::optional{}", + "bool?": "c10::optional{}", + "float?": "c10::optional{}", + "str?": "c10::optional{}", + "ScalarType?": "c10::optional{}", + "MemoryFormat?": "c10::optional{}", + "Layout?": "c10::optional{}", + "Device?": "c10::optional{}", + "Generator?": "c10::optional{}", + "Tensor?": "c10::optional{}", + "Tensor?[]": "c10::List>{}", + "int[]?": "c10::optional{}", + "int[1]?": "c10::optional{}", + "int[2]?": "c10::optional{}", + "int[3]?": "c10::optional{}", + "SymInt?": "c10::optional{}", + "SymInt[]?": "c10::optional{}", + "SymInt[1]?": "c10::optional{}", + "SymInt[2]?": "c10::optional{}", + "SymInt[3]?": "c10::optional{}", + "float[]?": "c10::optional>{}", +} + +# Optional ATen types that have a stable InfiniOps representation. These are +# exposed in generated base signatures and converted back to ATen optionals in +# the PyTorch backend. PyTorch-internal concepts without a public InfiniOps +# analogue stay hidden and are still passed as typed empty optionals. +_EXPOSED_OPTIONAL_CPP_TYPES = { + "Scalar?": "std::optional", + "int?": "std::optional", + "bool?": "std::optional", + "float?": "std::optional", + "str?": "std::optional", + "ScalarType?": "std::optional", + "Tensor?": "std::optional", + "Tensor?[]": "std::vector>", + "int[]?": "std::optional>", + "int[1]?": "std::optional>", + "int[2]?": "std::optional>", + "int[3]?": "std::optional>", + "SymInt?": "std::optional", + "SymInt[]?": "std::optional>", + "SymInt[1]?": "std::optional>", + "SymInt[2]?": "std::optional>", + "SymInt[3]?": "std::optional>", + "float[]?": "std::optional>", +} +_HARDCODE_NULLOPT_TYPES = frozenset( + set(_NULLOPT_BY_TYPE) - set(_EXPOSED_OPTIONAL_CPP_TYPES) +) + + +def _normalize_cpp_type(cpp_type: str) -> str: + text = cpp_type.strip() + + if text.startswith("const "): + text = text[len("const ") :] + + return text.rstrip("&").strip() + + +def _write_text_if_changed(path: pathlib.Path, content: str) -> bool: + """Write `content` only when the file's bytes would change.""" + path.parent.mkdir(parents=True, exist_ok=True) + + if path.exists() and path.read_text() == content: + return False + + path.write_text(content) + return True + + +def _prune_empty_dirs(root: pathlib.Path) -> None: + if not root.exists(): + return + + for child in sorted(root.iterdir(), reverse=True): + if child.is_dir(): + _prune_empty_dirs(child) + + if root.is_dir() and not any(root.iterdir()): + root.rmdir() + + +def _remove_stale_files(root: pathlib.Path, expected_files: set[pathlib.Path]) -> None: + if not root.exists(): + return + + for path in sorted(root.rglob("*"), reverse=True): + if path.is_file() and path not in expected_files: + path.unlink() + + _prune_empty_dirs(root) + + +@dataclasses.dataclass +class Param: + name: str + aten_type: str + default: str | None + keyword_only: bool + cpp_type_override: str | None = None + + @property + def api_name(self) -> str: + if self.name == "self": + return "input" + + return self.name + + @property + def is_tensor(self) -> bool: + if self.cpp_type_override is not None: + return _normalize_cpp_type(self.cpp_type_override) == "Tensor" + + # Real tensors only. `Tensor?` and `Tensor[]` have separate handling. + if self.aten_type.endswith("[]"): + return False + + return self.aten_type == "Tensor" or self.aten_type.startswith("Tensor(") + + @property + def is_optional_tensor(self) -> bool: + if self.cpp_type_override is not None: + return ( + _normalize_cpp_type(self.cpp_type_override) == "std::optional" + ) + + return self.aten_type == "Tensor?" + + @property + def is_tensor_list(self) -> bool: + if self.cpp_type_override is not None: + return _normalize_cpp_type(self.cpp_type_override) == "std::vector" + + return self.aten_type == "Tensor[]" + + @property + def is_optional_tensor_list(self) -> bool: + if self.cpp_type_override is not None: + return ( + _normalize_cpp_type(self.cpp_type_override) + == "std::vector>" + ) + + return self.aten_type == "Tensor?[]" + + @property + def is_mutable_tensor(self) -> bool: + # Mutable tensors carry `!` in their alias annotation, e.g. + # `Tensor(a!)`. + + return self.is_tensor and "!" in self.aten_type + + @property + def is_out(self) -> bool: + # In ATen `_out` schemas, output tensors are keyword-only mutable tensor + # params. Some mutable tensors are real inputs (`running_mean` / + # `running_var` in `_batch_norm_with_update`) and must stay in schema + # order, so mutability alone is not enough. + + return self.is_mutable_tensor and self.keyword_only + + @property + def is_hardcoded_nullopt(self) -> bool: + """If `True`, the param is omitted from the user-facing API and + passed as `at::nullopt` to ATen.""" + + return self.aten_type in _HARDCODE_NULLOPT_TYPES + + @property + def is_hidden(self) -> bool: + """True if the param is omitted from the user-facing API. + + Default-valued non-optional params (\\`bool\\`, \\`int\\`, \\`float\\`, + \\`str\\`, \\`int[N]\\`, …) used to be hidden as a convenience, but + reviewers consistently flagged the resulting omissions — + \\`bool upper/transpose/unitriangular\\` on \\`triangular_solve\\`, + \\`int diagonal\\` on \\`triu\\`, \\`str ord\\` on \\`linalg_matrix_norm\\`, + \\`int n\\` on the special chebyshev family, etc. — as missing + semantic controls. They are now exposed and forwarded to ATen. + + Optional ATen types with a stable InfiniOps representation + (\\`Tensor?\\`, \\`Scalar?\\`, \\`int?\\`, …) are exposed as + \\`std::optional\\`. ATen-internal types like + \\`Generator?\\`/\\`Layout?\\` still have no InfiniOps analogue, so + they remain hidden. + """ + + return self.is_hardcoded_nullopt + + def hidden_value(self) -> str: + """C++ literal substituted for a hidden param in the ATen call.""" + + if self.is_hardcoded_nullopt: + return _NULLOPT_BY_TYPE[self.aten_type] + + if self.default == "True": + return "true" + + if self.default == "False": + return "false" + + if self.aten_type.startswith(("int[", "SymInt[")) and self.default is not None: + # `int[N]=[a, b, c]` → `{a, b, c}`; `int[N]=0` (scalar default + # for list type) → `{0, 0, ...}` replicated to size N. + if self.default.startswith("["): + return "{" + self.default[1:-1] + "}" + + size_match = re.search(r"\[(\d+)\]", self.aten_type) + n = int(size_match.group(1)) if size_match else 1 + + return "{" + ", ".join([self.default] * n) + "}" + + if self.aten_type == "str" and self.default is not None: + # YAML uses single-quoted strings (e.g. `'none'`); C++ char + # literals also use single quotes, so swap to doubles. + + return '"' + self.default.strip("'\"") + '"' + + if self.aten_type in {"int", "float", "SymInt"} and self.default is not None: + # Translate known ATen enum defaults to their C++ identifiers. + + return _ENUM_DEFAULTS.get(self.default, self.default) + + raise AssertionError( + f"param {self.name!r} of type {self.aten_type!r} with default " + f"{self.default!r} is not hidden" + ) + + @property + def cpp_type(self) -> str: + if self.cpp_type_override is not None: + return self.cpp_type_override + + if self.is_tensor: + return "Tensor" + + if self.is_tensor_list: + return "std::vector" + + if self.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: + return _EXPOSED_OPTIONAL_CPP_TYPES[self.aten_type] + + if self.is_hidden: + # Not exposed — the ATen call substitutes a hardcoded value + # so the `cpp_type` is irrelevant. + + return "void" + + bare = self.aten_type.rstrip("?") + # Required `int[N]` / `SymInt[N]` (no default) — pybind11 accepts + # a Python list of ints into `std::vector`, which ATen + # promotes to `IntArrayRef` implicitly. + if bare.startswith(("int[", "SymInt[")) or bare in {"int[]", "SymInt[]"}: + return "std::vector" + + try: + return _SCALAR_TYPE_MAP[bare] + except KeyError as exc: + raise NotImplementedError( + f"unsupported ATen type {self.aten_type!r} for param {self.name!r}" + ) from exc + + +@dataclasses.dataclass +class Op: + aten_name: str + overload: str + params: list[Param] + signature_params: list[Param] | None = None + param_bindings: list[Param | None] | None = None + explicit_inplace_out: Param | None = None + + @property + def pascal_name(self) -> str: + return _snake_to_pascal(self.infini_name) + + @property + def infini_name(self) -> str: + """InfiniOps public op name. + + ATen disambiguates `_out` overloads with suffixes like `Tensor_Tensor_out`, + `out_x`, `forward_output`, `grad_input`, but reviewers consistently + flag those suffixes as bad public-API naming when they leak into + InfiniOps class names. Different ATen overloads of the same base op + become overloaded `operator()` methods on a single class instead. When + two overloads collapse to the same visible C++ signature after hidden + defaults, `_dedupe_visible_overloads` keeps only one. + + ATen-internal leading underscores and in-place trailing underscores are + also normalized so generated class names do not collide with existing + public ops (`_softmax` → `AtenSoftmax`, `add_` → `AddInplace`). + """ + + return _public_op_name(self.aten_name) + + @property + def is_inplace(self) -> bool: + """True for ATen's single-underscore in-place variants.""" + + return _is_inplace_aten_name(self.aten_name) + + @property + def tensor_params(self) -> list[Param]: + return [p for p in self.params if p.is_tensor] + + @property + def out_params(self) -> list[Param]: + """Mutable tensor outputs. Most ops have one (`Tensor(a!) out`); + multi-output ops like `frexp` or `sort` have several + (`Tensor(a!) values`, `Tensor(b!) indices`).""" + + if self.explicit_inplace_out is not None: + return [self.explicit_inplace_out] + + if self.is_inplace: + return [self.params[0]] + + return [p for p in self.params if p.is_out] + + @property + def out_param(self) -> Param: + """Single-output convenience. Asserts there's exactly one.""" + outs = self.out_params + assert len(outs) == 1, f"op {self.aten_name!r} has {len(outs)} out tensors" + + return outs[0] + + @property + def visible_params(self) -> list[Param]: + """Params the wrapper exposes to the user; hidden ones (hardcoded + optional nullopt, default-`False`/`True` bools) are filtered.""" + + if self.signature_params is not None: + return self.signature_params + + return [p for p in self.params if not p.is_hidden] + + def api_param_for(self, schema_index: int) -> Param | None: + """Return the public signature param bound to a schema param.""" + + if self.param_bindings is not None: + return self.param_bindings[schema_index] + + param = self.params[schema_index] + + if param.is_hidden: + return None + + return param + + @property + def is_testable(self) -> bool: + """Cheap structural check: at least one out tensor, and the first + constructor parameter is a tensor. The latter is needed because + `Operator::Make(Tensor tensor, Args... args)` dispatches on + `tensor.device()`, so an op like `pow.Scalar_out(Scalar self, + Tensor exponent, *, Tensor(a!) out)` cannot be wired up without + a separate dispatch path. Generators like `arange` / `linspace` + also fall under this rule (no input tensors at all).""" + + if not self.out_params: + return False + + if self.is_inplace: + return self.params[0].is_mutable_tensor + + # `params` includes out tensors at the end; check the first + # non-out param. If there are no non-out params (`empty.out`, + # `arange.out`), this op also fails the dispatch precondition. + non_out = [p for p in self.params if not p.is_out] + + if not non_out: + return False + + return non_out[0].is_tensor or non_out[0].is_tensor_list + + +_FUNC_RE = re.compile( + r"^(?P[a-zA-Z_][a-zA-Z0-9_]*)" + r"(?:\.(?P\w+))?" + r"\((?P.*)\)\s*->\s*.+$" +) + +_ARG_RE = re.compile( + r"^(?P\S+(?:\([^)]*\))?\??)" # type with optional alias and `?` + r"\s+(?P\w+)" + r"(?:\s*=\s*(?P.+))?$" +) + + +def _parse_func(func_str: str) -> Op: + m = _FUNC_RE.match(func_str) + + if not m: + raise ValueError(f"could not parse func: {func_str!r}") + + return Op( + aten_name=m.group("name"), + overload=m.group("overload") or "", + params=_parse_args(m.group("args")), + ) + + +def _parse_args(args_str: str) -> list[Param]: + params: list[Param] = [] + keyword_only = False + + for token in _split_args(args_str): + if token == "*": + keyword_only = True + continue + + params.append(_parse_one_arg(token, keyword_only)) + + return params + + +def _split_args(args_str: str) -> list[str]: + """Split on top-level commas, respecting `(...)` and `[...]`.""" + parts: list[str] = [] + depth = 0 + current: list[str] = [] + + for ch in args_str: + if ch in "([": + depth += 1 + current.append(ch) + elif ch in ")]": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + piece = "".join(current).strip() + + if piece: + parts.append(piece) + + current = [] + else: + current.append(ch) + + tail = "".join(current).strip() + + if tail: + parts.append(tail) + + return parts + + +def _parse_one_arg(token: str, keyword_only: bool) -> Param: + m = _ARG_RE.match(token) + + if not m: + raise ValueError(f"could not parse arg: {token!r}") + + name = m.group("name") + return Param( + name=name, + aten_type=m.group("type"), + default=m.group("default"), + keyword_only=keyword_only, + ) + + +_BASE_OPERATOR_RE = re.compile( + r"\bvirtual\s+void\s+operator\(\)\s*\((?P.*?)\)\s*const" + r"(?:\s*=\s*0|\s*\{)", + re.S, +) + + +def _split_cpp_params(args_str: str) -> list[str]: + parts: list[str] = [] + depth = 0 + current: list[str] = [] + + for ch in args_str: + if ch in "([<": + depth += 1 + current.append(ch) + elif ch in ")]>": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + piece = "".join(current).strip() + + if piece: + parts.append(piece) + + current = [] + else: + current.append(ch) + + tail = "".join(current).strip() + + if tail: + parts.append(tail) + + return parts + + +def _parse_base_signature_param(token: str) -> tuple[str, str]: + token = token.split("=", 1)[0].strip() + token = re.sub(r"\s+", " ", token) + match = re.match(r"(?P.+?)\s+(?P\w+)$", token) + + if not match: + raise ValueError(f"could not parse base operator param: {token!r}") + + return _normalize_cpp_type(match.group("type")), match.group("name") + + +def _parse_base_operator_signatures( + base_path: pathlib.Path, +) -> list[list[tuple[str, str]]]: + source = base_path.read_text() + signatures = [] + + for match in _BASE_OPERATOR_RE.finditer(source): + args = match.group("args").strip() + + if not args: + signatures.append([]) + continue + + signatures.append( + [_parse_base_signature_param(token) for token in _split_cpp_params(args)] + ) + + return signatures + + +def _cpp_types_compatible(schema_cpp_type: str, base_cpp_type: str) -> bool: + schema = _normalize_cpp_type(schema_cpp_type) + base = _normalize_cpp_type(base_cpp_type) + + if schema == base: + return True + + compatible_pairs = { + ("double", "float"), + ("int64_t", "int"), + ("std::optional", "std::optional"), + ("std::optional", "std::optional"), + } + + return (schema, base) in compatible_pairs + + +def _cpp_param_compatible( + schema_param: Param, base_cpp_type: str, base_name: str +) -> bool: + if schema_param.api_name != base_name: + return False + + return _cpp_types_compatible(schema_param.cpp_type, base_cpp_type) + + +def _bind_base_signature(op: Op, signature: list[tuple[str, str]]) -> Op | None: + bindings: list[Param | None] = [None] * len(op.params) + signature_params = [] + explicit_inplace_out = None + + if ( + op.is_inplace + and signature + and signature[-1][1] == "out" + and _cpp_types_compatible("Tensor", signature[-1][0]) + ): + out_cpp_type, out_name = signature[-1] + explicit_inplace_out = Param( + name=out_name, + aten_type="Tensor(a!)", + default=None, + keyword_only=True, + cpp_type_override=out_cpp_type, + ) + signature = signature[:-1] + + for base_cpp_type, base_name in signature: + matched_index = None + + for index, schema_param in enumerate(op.params): + if bindings[index] is not None: + continue + + if _cpp_param_compatible(schema_param, base_cpp_type, base_name): + matched_index = index + break + + if matched_index is None: + return None + + schema_param = op.params[matched_index] + api_param = dataclasses.replace( + schema_param, name=base_name, cpp_type_override=base_cpp_type + ) + bindings[matched_index] = api_param + signature_params.append(api_param) + + if any( + binding is None and not _is_omittable_param(param) + for param, binding in zip(op.params, bindings) + ): + return None + + if explicit_inplace_out is not None: + signature_params.append(explicit_inplace_out) + + return dataclasses.replace( + op, + signature_params=signature_params, + param_bindings=bindings, + explicit_inplace_out=explicit_inplace_out, + ) + + +def _bind_existing_base_overloads( + name: str, ops: list[Op] +) -> tuple[list[Op], list[str]]: + base_path = _base_path(name) + signatures = _parse_base_operator_signatures(base_path) + bound_ops = [] + warnings = [] + + for signature in signatures: + matches = [ + bound + for op in ops + if (bound := _bind_base_signature(op, signature)) is not None + ] + + if not matches: + rendered = ", ".join(f"{cpp_type} {param}" for cpp_type, param in signature) + warnings.append( + f"base overload `operator()({rendered})` does not match any " + "usable ATen schema" + ) + continue + + matches.sort( + key=lambda match: sum( + binding is None for binding in match.param_bindings or () + ) + ) + bound_ops.append(matches[0]) + + return bound_ops, warnings + + +def _snake_to_pascal(s: str) -> str: + return "".join(p.capitalize() for p in s.split("_")) + + +_OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") + + +def _is_inplace_aten_name(name: str) -> bool: + """Return whether `name` is an ATen in-place operator name.""" + + return name.endswith("_") and not name.endswith("__") + + +def _public_op_name(aten_name: str) -> str: + """Map ATen-only spelling to stable InfiniOps public names.""" + + public_name = aten_name + file_prefixes = [] + + if public_name.startswith("_"): + file_prefixes.append("internal") + public_name = public_name.lstrip("_") + + if _is_inplace_aten_name(public_name): + public_name = public_name[:-1] + + return "_".join(file_prefixes + [public_name]) + + +def _op_namespace_parts(name: str) -> tuple[str, ...]: + parts = name.split("_") + namespaces = [] + + if parts and parts[0] == "internal": + namespaces.append(parts.pop(0)) + + if parts and parts[0] in _OP_NAMESPACE_PREFIXES: + namespaces.append(parts.pop(0)) + + return tuple(namespaces) + + +def _op_class_stem(name: str) -> str: + parts = name.split("_") + + if parts and parts[0] == "internal": + parts.pop(0) + + if parts and parts[0] in _OP_NAMESPACE_PREFIXES: + parts.pop(0) + + return "_".join(parts) + + +def _op_class_name(name: str) -> str: + return _snake_to_pascal(_op_class_stem(name)) + + +def _op_cpp_type(name: str) -> str: + parts = (*_op_namespace_parts(name), _op_class_name(name)) + + return "::".join(parts) + + +def _op_namespace_open(name: str) -> str: + parts = _op_namespace_parts(name) + + if not parts: + return "namespace infini::ops {" + + return f"namespace infini::ops::{'::'.join(parts)} {{" + + +def _op_namespace_close(name: str) -> str: + parts = _op_namespace_parts(name) + + if not parts: + return "} // namespace infini::ops" + + return f"}} // namespace infini::ops::{'::'.join(parts)}" + + +def _base_path(op_name: str) -> pathlib.Path: + return _BASE_DIR / f"{op_name}.h" + + +def _is_omittable_param(param: Param) -> bool: + return param.aten_type in _NULLOPT_BY_TYPE or param.default is not None + + +def _default_call_value(param: Param) -> str: + if param.aten_type in _NULLOPT_BY_TYPE: + return _NULLOPT_BY_TYPE[param.aten_type] + + if param.default is not None: + return _translate_default(param) + + return param.hidden_value() + + +def _load_aten_entries(version: str) -> list[dict]: + """Return parsed ATen schemas bundled with installed `torchgen`. + + `WITH_TORCH=ON` already requires a local PyTorch installation. PyTorch + wheels ship the matching ATen schema under `torchgen/packaged`, including + vendor forks, so codegen should not depend on fetching PyTorch sources from + GitHub during CI builds. The official parser expands schemas declared via + `autogen`, such as `relu.out`, before InfiniOps filters for out variants. + """ + + try: + packaged, expanded_autogen = _load_packaged_aten_entries() + except RuntimeError as error: + raise RuntimeError(f"{error} for PyTorch schema {version!r}") from error + + print( + "using installed `torchgen` ATen schemas " + f"for PyTorch schema {version} " + f"({'with' if expanded_autogen else 'without'} `autogen` expansion).", + file=sys.stderr, + ) + + return packaged + + +def _load_packaged_aten_entries() -> tuple[list[dict], bool]: + """Parse the ATen schemas bundled with installed `torchgen`. + + PyTorch wheels install `torchgen/packaged/ATen/native/native_functions.yaml`; + using it lets offline CI images generate wrappers against the exact schema + shipped with their installed torch fork. + """ + + spec = importlib.util.find_spec("torchgen") + + if spec is None: + raise RuntimeError("could not find the installed `torchgen` package") + + if spec.submodule_search_locations is None: + raise RuntimeError("installed `torchgen` is not a package") + + for root in spec.submodule_search_locations: + native_dir = pathlib.Path(root) / "packaged" / "ATen" / "native" + native_path = native_dir / "native_functions.yaml" + tags_path = native_dir / "tags.yaml" + + if not native_path.is_file(): + continue + + if not tags_path.is_file(): + print( + "warning: could not expand ATen `autogen` schemas because " + f"`{tags_path}` is missing; using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + try: + from torchgen.gen import parse_native_yaml + except (ImportError, AttributeError) as error: + print( + "warning: could not expand ATen `autogen` schemas because " + f"`torchgen.gen.parse_native_yaml` is unavailable ({error}); " + "using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + try: + parsed = parse_native_yaml(str(native_path), str(tags_path)) + except Exception as error: + print( + "warning: could not expand ATen `autogen` schemas from " + f"`{native_path}` ({type(error).__name__}: {error}); " + "using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + return ( + [{"func": str(function.func)} for function in parsed.native_functions], + True, + ) + + raise RuntimeError( + "could not find installed `torchgen` packaged `native_functions.yaml`" + ) + + +def _load_explicit_aten_entries(native_path: pathlib.Path) -> list[dict]: + entries = yaml.safe_load(native_path.read_text()) + + if not isinstance(entries, list): + raise RuntimeError( + f"expected a list of ATen schemas in `{native_path}`, " + f"got {type(entries).__name__}" + ) + + return entries + + +def _find_out_entries(entries: list[dict], op_name: str) -> list[dict]: + """Return all out-variant entries for `op_name`, with the bare + `.out(` form first and overload-suffixed variants + (e.g. `pow.Tensor_Tensor_out(`, `kthvalue.values(`) after. An + entry counts as an out-variant when it (a) is named + `.out`, (b) ends in `_out`, or (c) carries a + `Tensor(!)` mutability annotation — that last case covers + multi-output ops named after their output tensors + (`kthvalue.values`, `mode.values`, …).""" + bare_prefix = f"{op_name}.out(" + op_overload = re.compile(rf"^{re.escape(op_name)}\.\w+\(") + mut_tensor = re.compile(r"Tensor\([a-z]!\)") + bare: list[dict] = [] + others: list[dict] = [] + + for entry in entries: + func = entry.get("func", "") + + if func.startswith(bare_prefix): + bare.append(entry) + elif op_overload.match(func) and ( + func.split("(", 1)[0].endswith("_out") or mut_tensor.search(func) + ): + others.append(entry) + + return bare + others + + +def _format_signature(op: Op, *, include_defaults: bool = False) -> str: + parts = [] + + for param in op.visible_params: + prefix = "" if param.is_mutable_tensor else "const " + text = f"{prefix}{param.cpp_type} {param.api_name}" + + if include_defaults and param.default is not None: + text += f" = {_translate_default(param)}" + + parts.append(text) + + return ", ".join(parts) + + +def _visible_signature_key(op: Op) -> tuple[str, ...]: + """C++ overload identity for the user-facing API. + + Parameter names and top-level `const` do not distinguish C++ overloads, so + only the exposed C++ type sequence participates in duplicate detection. + """ + + return tuple(param.cpp_type for param in op.visible_params) + + +def _canonical_overload_score(index: int, op: Op) -> tuple[bool, int, int, str, int]: + """Sort key for duplicate visible signatures. + + Prefer the canonical unsuffixed InfiniOps name, then the schema that hides + fewer ATen-only defaults, then the shorter deterministic name. + """ + + return ( + op.infini_name != op.aten_name, + sum(param.is_hidden for param in op.params), + len(op.infini_name), + op.infini_name, + index, + ) + + +def _dedupe_visible_overloads(ops: list[Op]) -> tuple[list[Op], list[tuple[Op, Op]]]: + """Drop overloads that collapse to the same visible C++ signature. + + Returns the selected overloads in the original schema order plus a list of + `(skipped, kept)` duplicate pairs for diagnostics. + """ + winners: dict[tuple[str, ...], tuple[int, Op]] = {} + duplicates: list[tuple[Op, tuple[str, ...]]] = [] + + for index, op in enumerate(ops): + key = _visible_signature_key(op) + current = winners.get(key) + + if current is None: + winners[key] = (index, op) + continue + + current_index, current_op = current + + if _canonical_overload_score(index, op) < _canonical_overload_score( + current_index, current_op + ): + duplicates.append((current_op, key)) + winners[key] = (index, op) + else: + duplicates.append((op, key)) + + selected_indices = {index for index, _ in winners.values()} + selected = [op for index, op in enumerate(ops) if index in selected_indices] + duplicate_pairs = [ + (skipped, winners[key][1]) + for skipped, key in duplicates + if winners[key][1] is not skipped + ] + + return selected, duplicate_pairs + + +def _translate_default(param: Param) -> str: + """Translate a YAML default literal to a C++ literal.""" + raw = param.default + + if raw == "True": + return "true" + + if raw == "False": + return "false" + + if raw == "None": + return "{}" + + return raw # numeric literals (`0`, `1`, `1.0`) pass through + + +def _generate_base_includes(ops: list[Op]) -> str: + cpp_types = [param.cpp_type for op in ops for param in op.visible_params] + includes = [] + + if any("std::optional" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + if any("std::string" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + if any("std::vector" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + includes.append('#include "operator.h"') + + return "\n".join(includes) + + +def _generate_base_header(name: str, ops: list[Op]) -> str: + pascal = _op_class_name(name) + + member_decls = [] + tensor_member_order = [] + seen_tensor_members = set() + optional_tensor_member_order = [] + seen_optional_tensor_members = set() + scalar_member_order = [] + scalar_member_types = {} + + for op in ops: + for param in op.tensor_params: + api_name = param.api_name + + if api_name in seen_tensor_members: + continue + + seen_tensor_members.add(api_name) + tensor_member_order.append(api_name) + member_decls.append(f" Tensor::Shape {api_name}_shape_;") + member_decls.append(f" Tensor::Strides {api_name}_strides_;") + member_decls.append(f" DataType {api_name}_type_;") + + for param in op.visible_params: + api_name = param.api_name + + if not param.is_optional_tensor or api_name in seen_optional_tensor_members: + continue + + seen_optional_tensor_members.add(api_name) + optional_tensor_member_order.append(api_name) + member_decls.append(f" bool has_{api_name}_{{false}};") + member_decls.append(f" Tensor::Shape {api_name}_shape_;") + member_decls.append(f" Tensor::Strides {api_name}_strides_;") + member_decls.append(f" DataType {api_name}_type_{{DataType::kFloat32}};") + + # Visible non-tensor params (scalars, strings, vectors) are also + # stored on the base so backends can dispatch on them later — not + # only at the moment `operator()` is invoked. Reviewers flagged + # this on multiple PRs (e.g. `n` on + # `special_chebyshev_polynomial_v_n_scalar`). Same-named params + # across overloads must share a type; if they conflict, the second + # overload's member is dropped (later constructors leave it + # default-initialised). + for param in op.visible_params: + api_name = param.api_name + + if ( + param.is_tensor + or param.is_optional_tensor + or api_name in scalar_member_types + ): + continue + + scalar_member_order.append(api_name) + scalar_member_types[api_name] = param.cpp_type + member_decls.append(f" {param.cpp_type} {api_name}_{{}};") + + member_decls.append(" int device_index_{0};") + + constructors = [] + calls = [] + + for op in ops: + init_pieces = [] + tensor_params = {param.api_name: param for param in op.tensor_params} + optional_tensor_params = { + param.api_name: param + for param in op.visible_params + if param.is_optional_tensor + } + scalar_params = { + param.api_name: param + for param in op.visible_params + if not param.is_tensor + and not param.is_optional_tensor + and scalar_member_types.get(param.api_name) == param.cpp_type + } + + for param_name in tensor_member_order: + param = tensor_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" {api_name}_shape_{{{api_name}.shape()}}") + init_pieces.append(f" {api_name}_strides_{{{api_name}.strides()}}") + init_pieces.append(f" {api_name}_type_{{{api_name}.dtype()}}") + + for param_name in optional_tensor_member_order: + param = optional_tensor_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" has_{api_name}_{{{api_name}.has_value()}}") + init_pieces.append( + f" {api_name}_shape_{{{api_name} ? " + f"Tensor::Shape{{{api_name}->shape()}} : Tensor::Shape{{}}}}" + ) + init_pieces.append( + f" {api_name}_strides_{{{api_name} ? " + f"Tensor::Strides{{{api_name}->strides()}} : Tensor::Strides{{}}}}" + ) + init_pieces.append( + f" {api_name}_type_{{{api_name} ? " + f"{api_name}->dtype() : DataType::kFloat32}}" + ) + + for param_name in scalar_member_order: + param = scalar_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" {api_name}_{{{api_name}}}") + + # All out tensors share a device; use the first one. Keep this last + # so initializer order follows the member declaration order. + init_pieces.append( + f" device_index_{{{op.out_params[0].api_name}.device().index()}}" + ) + + init_list = ",\n".join(init_pieces).lstrip() + constructors.append( + f" {pascal}({_format_signature(op)})\n : {init_list} {{}}" + ) + calls.append(f" virtual void operator()({_format_signature(op)}) const = 0;") + + return _BASE_TEMPLATE.format( + name_uc=name.upper(), + pascal=pascal, + namespace_open=_op_namespace_open(name), + namespace_close=_op_namespace_close(name), + includes=_generate_base_includes(ops), + constructors="\n\n".join(constructors), + op_calls="\n\n".join(calls), + member_decls="\n\n".join(member_decls), + ) + + +def _generate_torch_header(name: str, ops: list[Op]) -> str: + pascal = _op_class_name(name) + op_type = _op_cpp_type(name) + op_calls = "\n\n".join( + f" void operator()({_format_signature(op)}) const override;" for op in ops + ) + + return _TORCH_HEADER_TEMPLATE.format( + name_uc=name.upper(), + name=name, + pascal=pascal, + op_type=op_type, + op_calls=op_calls, + slot=_PYTORCH_SLOT, + ) + + +def _generate_torch_method_source(name: str, op: Op) -> str: + op_type = _op_cpp_type(name) + conversion_lines = [] + out_device_index = f"{op.out_params[0].api_name}.device().index()" + + def _optional_aten_type(param: Param) -> str: + return _NULLOPT_BY_TYPE[param.aten_type].removesuffix("{}") + + def _optional_aten_value(schema_param: Param, api_param: Param) -> str: + api_name = api_param.api_name + + if schema_param.aten_type == "Tensor?": + data_expr = f"const_cast({api_name}->data())" + + return ( + f"ToAtenTensor({data_expr}, {api_name}->shape(), " + f"{api_name}->strides(), {api_name}->dtype(), " + f"device_index)" + ) + + if schema_param.aten_type == "Scalar?": + return f"at::Scalar(*{api_name})" + + if schema_param.aten_type == "ScalarType?": + return f"ToAtenDataType(*{api_name})" + + if schema_param.aten_type == "str?": + return f"c10::string_view(*{api_name})" + + if schema_param.aten_type.startswith( + ("int[", "SymInt[") + ) or schema_param.aten_type in { + "int[]?", + "SymInt[]?", + }: + return f"at::IntArrayRef(*{api_name})" + + if schema_param.aten_type == "float[]?": + return f"at::ArrayRef(*{api_name})" + + return f"*{api_name}" + + def _append_optional_conversion(schema_param: Param, api_param: Param) -> None: + api_name = api_param.api_name + optional_type = _optional_aten_type(schema_param) + conversion_lines.append(f" {optional_type} at_{schema_param.name};") + conversion_lines.append(f" if ({api_name}.has_value()) {{") + conversion_lines.append( + f" at_{schema_param.name} = " + f"{optional_type}{{{_optional_aten_value(schema_param, api_param)}}};" + ) + conversion_lines.append(" }") + + for schema_index, param in enumerate(op.params): + if ( + not param.is_tensor + and not param.is_tensor_list + and not param.is_optional_tensor_list + ): + continue + + api_param = op.api_param_for(schema_index) + + if api_param is None: + continue + + api_name = api_param.api_name + + if param.is_optional_tensor_list: + conversion_lines.append( + f" c10::List> at_{param.name};" + ) + conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") + conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") + conversion_lines.append(" if (tensor.has_value()) {") + conversion_lines.append( + " at_" + f"{param.name}.push_back(ToAtenTensor(" + "const_cast(tensor->data()), tensor->shape(), " + "tensor->strides(), tensor->dtype(), device_index));" + ) + conversion_lines.append(" } else {") + conversion_lines.append(f" at_{param.name}.push_back(c10::nullopt);") + conversion_lines.append(" }") + conversion_lines.append(" }") + continue + + if param.is_tensor_list: + conversion_lines.append(f" std::vector at_{param.name};") + conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") + conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") + conversion_lines.append( + " at_" + f"{param.name}.push_back(ToAtenTensor(" + "const_cast(tensor.data()), tensor.shape(), tensor.strides(), " + "tensor.dtype(), device_index));" + ) + conversion_lines.append(" }") + continue + + use_mutable_data = param.is_mutable_tensor and not ( + op.explicit_inplace_out is not None and schema_index == 0 + ) + data_expr = ( + f"{api_name}.data()" + if use_mutable_data + else f"const_cast({api_name}.data())" + ) + conversion_lines.append( + f" auto at_{param.name} = ToAtenTensor(\n" + f" {data_expr}, {api_name}_shape_, {api_name}_strides_,\n" + f" {api_name}_type_, device_index);" + ) + + if op.explicit_inplace_out is not None: + out_name = op.explicit_inplace_out.api_name + conversion_lines.append( + f" auto at_{out_name} = ToAtenTensor(\n" + f" {out_name}.data(), {out_name}_shape_, {out_name}_strides_,\n" + f" {out_name}_type_, device_index);" + ) + + for schema_index, param in enumerate(op.params): + api_param = op.api_param_for(schema_index) + + if ( + api_param is not None + and param.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES + and not param.is_optional_tensor_list + ): + _append_optional_conversion(param, api_param) + + def _render_arg(schema_index, p): + api_param = op.api_param_for(schema_index) + + if api_param is None: + return _default_call_value(p) + + if p.is_hidden: + return _default_call_value(p) + + if p.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: + return f"at_{p.name}" + + if p.is_tensor or p.is_tensor_list: + return f"at_{p.name}" + + if p.aten_type == "ScalarType": + return f"ToAtenDataType({api_param.api_name})" + + return api_param.api_name + + if op.is_inplace: + # In-place ATen calls keep the mutable input in positional order, + # unlike `_out` calls which place output tensors first. + input_param = op.params[0] + arg_order = op.params[1:] + rendered_args = ", ".join( + _render_arg(index + 1, p) for index, p in enumerate(arg_order) + ) + if op.explicit_inplace_out is not None: + out_name = op.explicit_inplace_out.api_name + aten_call = f"at_{out_name}.copy_(at_{input_param.name});\n" + aten_call += f" at_{out_name}.{op.aten_name}({rendered_args})" + else: + aten_call = f"at_{input_param.name}.{op.aten_name}({rendered_args})" + else: + # ATen `_out` form puts all out tensors first, then non-out params + # in YAML order. Hardcoded-nullopt params become `at::nullopt`. + arg_order = [(index, p) for index, p in enumerate(op.params) if p.is_out] + [ + (index, p) for index, p in enumerate(op.params) if not p.is_out + ] + rendered_args = ", ".join(_render_arg(index, p) for index, p in arg_order) + aten_call = f"at::{op.aten_name}_out({rendered_args})" + + return _TORCH_METHOD_TEMPLATE.format( + op_type=op_type, + op_call_signature=_format_signature(op), + tensor_conversions="\n".join(conversion_lines), + out_device_index=out_device_index, + # The generated call expression resolves the right kernel via C++ + # overload resolution from the argument types we pass. + aten_call=aten_call, + slot=_PYTORCH_SLOT, + ) + + +def _generate_torch_source(name: str, ops: list[Op]) -> str: + op_type = _op_cpp_type(name) + methods = "\n\n".join(_generate_torch_method_source(name, op) for op in ops) + return _TORCH_SOURCE_TEMPLATE.format( + name=name, + methods=methods, + op_type=op_type, + slot=_PYTORCH_SLOT, + ) + + +_BASE_TEMPLATE = """\ +#ifndef INFINI_OPS_BASE_{name_uc}_H_ +#define INFINI_OPS_BASE_{name_uc}_H_ + +{includes} + +{namespace_open} + +class {pascal} : public Operator<{pascal}> {{ + public: +{constructors} + +{op_calls} + + protected: +{member_decls} +}}; + +{namespace_close} + +#endif +""" + + +_TORCH_HEADER_TEMPLATE = """\ +#ifndef INFINI_OPS_TORCH_{name_uc}_H_ +#define INFINI_OPS_TORCH_{name_uc}_H_ + +#include "base/{name}.h" + +namespace infini::ops {{ + +template +class Operator<{op_type}, kDev, {slot}> : public {op_type} {{ + public: + using {op_type}::{pascal}; + +{op_calls} +}}; + +}} // namespace infini::ops + +#endif +""" + + +_TORCH_METHOD_TEMPLATE = """\ +template +void Operator<{op_type}, kDev, {slot}>::operator()({op_call_signature}) const {{ + const auto device_index = {out_device_index}; + const TorchStreamGuard stream_guard{{stream_, device_index}}; +{tensor_conversions} + + {aten_call}; +}} +""" + + +_TORCH_SOURCE_TEMPLATE = """\ +#include "torch/{name}/{name}.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops {{ + +{methods} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR({op_type}, {slot}) + +}} // namespace infini::ops +""" + + +def _find_clang_format() -> str | None: + """Return the path to a `clang-format` binary, or `None` if unavailable. + + Generated files live under `generated/`, which is gitignored, so offline CI + containers should not block or try network access just to format them. + """ + + found = shutil.which("clang-format") + + if found: + return found + + print( + "`clang-format` not found on PATH; generated files will be emitted " + "without formatting.", + file=sys.stderr, + ) + + return None + + +def _clang_format(text: str, path: pathlib.Path) -> str: + """Pipe `text` through `clang-format` so generated headers / sources + satisfy the same style check (`clang-format` v21) that CI runs. + `path` informs include sorting (the file's own header should come + first in a `.cc`). If no `clang-format` binary is available, return + the input unchanged.""" + + if _CLANG_FORMAT is None: + return text + + return subprocess.run( + [_CLANG_FORMAT, f"--assume-filename={path}"], + input=text, + capture_output=True, + text=True, + check=True, + ).stdout + + +def _emit(name: str, ops: list[Op], *, emit_base: bool) -> set[pathlib.Path]: + base_path = _GENERATED_BASE_DIR / f"{name}.h" + torch_dir = _GENERATED_TORCH_DIR / name + torch_header_path = torch_dir / f"{name}.h" + torch_source_path = torch_dir / f"{name}.cc" + emitted_paths = set() + + if emit_base: + _write_text_if_changed( + base_path, _clang_format(_generate_base_header(name, ops), base_path) + ) + emitted_paths.add(base_path) + + _write_text_if_changed( + torch_header_path, + _clang_format(_generate_torch_header(name, ops), torch_header_path), + ) + _write_text_if_changed( + torch_source_path, + _clang_format(_generate_torch_source(name, ops), torch_source_path), + ) + emitted_paths.add(torch_header_path) + emitted_paths.add(torch_source_path) + + return emitted_paths + + +def _select_op_names(cli_ops, default_ops, config): + if config is None: + return cli_ops or default_ops + + aten_names_by_public_name = collections.defaultdict(list) + for op_name in default_ops: + aten_names_by_public_name[_public_op_name(op_name)].append(op_name) + + selected_public_names = ops_config.torch_op_names( + config, aten_names_by_public_name, _PYTORCH_SLOT + ) + selected = [ + aten_name + for public_name in selected_public_names + for aten_name in aten_names_by_public_name.get(public_name, (public_name,)) + ] + + if cli_ops: + allowed = set(cli_ops) + selected = [op_name for op_name in selected if op_name in allowed] + + return selected + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--ops", + nargs="*", + help="Override the op allowlist. If omitted, reads `scripts/torch_ops.yaml`.", + ) + parser.add_argument( + "--ops-config", + type=pathlib.Path, + help="Path to an `ops.json` operator and implementation selection.", + ) + parser.add_argument( + "--pytorch-version", + default=os.environ.get("INFINI_OPS_PYTORCH_VERSION", _DEFAULT_PYTORCH_VERSION), + help=( + "PyTorch version label used in diagnostics. Schemas are read from " + "the locally installed `torchgen` package. Default: `%(default)s`. " + "Can also be set via the `INFINI_OPS_PYTORCH_VERSION` environment " + "variable." + ), + ) + args = parser.parse_args() + + global _CLANG_FORMAT + _CLANG_FORMAT = _find_clang_format() + + default_ops = yaml.safe_load(_OPS_YAML_PATH.read_text()) + config = ops_config.load_ops_config(args.ops_config) if args.ops_config else None + op_names = _select_op_names(args.ops, default_ops, config) + aten_entries = _load_aten_entries(args.pytorch_version) + + skipped: list[tuple[str, str]] = [] + metadata: list[dict] = [] + expected_base_files: set[pathlib.Path] = set() + expected_torch_files: set[pathlib.Path] = set() + ops_by_public_name: dict[str, list[Op]] = collections.defaultdict(list) + + for name in op_names: + candidates = _find_out_entries(aten_entries, name) + + if not candidates: + skipped.append((name, f"no `.out` variant for `{name}` in YAML")) + continue + + usable: list[Op] = [] + last_reason = "" + + for entry in candidates: + try: + op = _parse_func(entry["func"]) + + for param in op.params: + _ = param.cpp_type # eagerly raise on unsupported types + except (NotImplementedError, ValueError) as exc: + last_reason = str(exc) + continue + + if not op.is_testable: + last_reason = "no testable tensor input/output pair" + continue + + usable.append(op) + + if not usable: + skipped.append((name, last_reason or "no usable overload")) + continue + + usable, duplicate_overloads = _dedupe_visible_overloads(usable) + + for skipped_op, kept_op in duplicate_overloads: + skipped.append( + ( + skipped_op.infini_name, + "duplicate visible C++ signature for " + f"`{name}`; using `{kept_op.infini_name}`", + ) + ) + + ops_by_public_name[usable[0].infini_name].extend(usable) + + for public_name, usable in ops_by_public_name.items(): + usable, duplicate_overloads = _dedupe_visible_overloads(usable) + + for skipped_op, kept_op in duplicate_overloads: + skipped.append( + ( + skipped_op.infini_name, + "duplicate visible C++ signature for " + f"`{public_name}`; using `{kept_op.infini_name}`", + ) + ) + + base_exists = _base_path(public_name).exists() + + if base_exists: + usable, base_warnings = _bind_existing_base_overloads(public_name, usable) + + for warning in base_warnings: + skipped.append((public_name, warning)) + + if not usable: + skipped.append( + ( + public_name, + "existing base has no overload compatible with ATen schema", + ) + ) + continue + + # Emit one InfiniOps wrapper per ATen op. Distinct visible overloads + # become overloaded constructors / `operator()` methods on the same + # class (`Pow` exposes both tensor and scalar exponents). Overloads + # that collapse to the same C++ signature after hidden defaults are + # skipped above. When a hand-written `src/base/.h` exists, + # skip emitting `generated/base/.h` so the hand-written one + # wins (the generated torch source's `#include "base/.h"` + # resolves through `src/` first). Signature mismatches surface as + # compile errors with a clear message — drop the op from the YAML + # to suppress. + emitted_paths = _emit(public_name, usable, emit_base=not base_exists) + expected_base_files.update( + path for path in emitted_paths if path.is_relative_to(_GENERATED_BASE_DIR) + ) + expected_torch_files.update( + path for path in emitted_paths if path.is_relative_to(_GENERATED_TORCH_DIR) + ) + + for op in usable: + metadata.append( + { + "name": public_name, + "aten_name": op.aten_name, + "overload_name": op.infini_name, + "params": [ + { + "name": p.api_name, + "type": p.aten_type, + "is_tensor": p.is_tensor, + "is_out": p.is_out, + "schema_index": next( + ( + schema_index + for schema_index in range(len(op.params)) + if op.api_param_for(schema_index) is p + ), + None, + ), + } + for p in op.visible_params + ], + } + ) + + _GENERATED_DIR.mkdir(parents=True, exist_ok=True) + _write_text_if_changed( + _METADATA_PATH, json.dumps({"ops": metadata}, indent=2) + "\n" + ) + _remove_stale_files(_GENERATED_BASE_DIR, expected_base_files) + _remove_stale_files(_GENERATED_TORCH_DIR, expected_torch_files) + + generated_names = sorted({m["name"] for m in metadata}) + print( + f"generated {len(metadata)} overloads across {len(generated_names)} ops: " + f"{generated_names}" + ) + + for name, reason in skipped: + print(f" skipped {name!r}: {reason}", file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py index a4cb10c62..093c27022 100644 --- a/scripts/generate_wrappers.py +++ b/scripts/generate_wrappers.py @@ -272,7 +272,7 @@ def _parse_operator_header(op_name): constructors = [ _ParsedFunction(_parse_parameter_list(params)) for params in _find_signature_parameters( - class_body, rf"(?:explicit\s+)?{class_name}\s*\(" + class_body, rf"\b(?:explicit\s+)?{class_name}\s*\(" ) ] calls = [ diff --git a/scripts/torch_ops.yaml b/scripts/torch_ops.yaml index 8ab9c4398..3b4608d10 100644 --- a/scripts/torch_ops.yaml +++ b/scripts/torch_ops.yaml @@ -100,6 +100,7 @@ - dstack - elu - elu_backward +- embedding_dense_backward - empty - eq - erf @@ -363,6 +364,8 @@ - sin - sinc - sinh +- slice_backward +- slice_copy - slogdet - slow_conv3d - slow_conv3d_forward @@ -454,6 +457,7 @@ - threshold - threshold_backward - topk +- transpose_copy - triangular_solve - tril - triu diff --git a/src/base/accumulate_grad.h b/src/base/accumulate_grad.h index 65890ad1c..eeeafff8e 100644 --- a/src/base/accumulate_grad.h +++ b/src/base/accumulate_grad.h @@ -1,57 +1,55 @@ -#ifndef INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ -#define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class AccumulateGrad : public Operator { - public: - AccumulateGrad(const Tensor gradient, const double rate, Tensor tensor) - : gradient_shape_{gradient.shape()}, - gradient_strides_{gradient.strides()}, - gradient_type_{gradient.dtype()}, - tensor_shape_{tensor.shape()}, - tensor_strides_{tensor.strides()}, - tensor_type_{tensor.dtype()}, - rate_{rate}, - device_index_{tensor.device().index()} { - AssertTrainingDevices({gradient, tensor}); - assert(gradient.shape() == tensor.shape() && - "`AccumulateGrad` requires `gradient` and `tensor` to share shape"); - assert(gradient_type_ == tensor_type_ && - "`AccumulateGrad` requires `gradient` and `tensor` to share dtype"); - assert( - gradient.device() == tensor.device() && - "`AccumulateGrad` requires `gradient` and `tensor` on the same device"); - } - - virtual void operator()(const Tensor gradient, const double rate, - Tensor tensor) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - Tensor::Shape gradient_shape_; - Tensor::Strides gradient_strides_; - DataType gradient_type_; - Tensor::Shape tensor_shape_; - Tensor::Strides tensor_strides_; - DataType tensor_type_; - double rate_{1.0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ +#ifndef INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AccumulateGrad : public Operator { + public: + AccumulateGrad(const Tensor gradient, const double rate, Tensor tensor) + : gradient_shape_{gradient.shape()}, + gradient_strides_{gradient.strides()}, + gradient_type_{gradient.dtype()}, + tensor_shape_{tensor.shape()}, + tensor_strides_{tensor.strides()}, + tensor_type_{tensor.dtype()}, + rate_{rate}, + device_index_{tensor.device().index()} { + validation::AssertSameDevice({gradient, tensor}); + assert(validation::IsFloatingPoint(tensor_type_) && + "`AccumulateGrad` requires floating-point tensors."); + assert(gradient.shape() == tensor.shape() && + "`AccumulateGrad` requires `gradient` and `tensor` to share shape"); + assert(gradient_type_ == tensor_type_ && + "`AccumulateGrad` requires `gradient` and `tensor` to share dtype"); + } + + // Gradient and tensor may overlap, including partial external storage spans. + virtual void operator()(const Tensor gradient, const double rate, + Tensor tensor) const = 0; + + protected: + Tensor::Shape gradient_shape_; + + Tensor::Strides gradient_strides_; + + DataType gradient_type_; + + Tensor::Shape tensor_shape_; + + Tensor::Strides tensor_strides_; + + DataType tensor_type_; + + double rate_{1.0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ diff --git a/src/base/adam_accumulate_grad.h b/src/base/adam_accumulate_grad.h index b753c39f1..638d7fa3a 100644 --- a/src/base/adam_accumulate_grad.h +++ b/src/base/adam_accumulate_grad.h @@ -1,111 +1,101 @@ -#ifndef INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ -#define INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class AdamAccumulateGrad : public Operator { - public: - AdamAccumulateGrad(const Tensor grad, const double learning_rate, - const double beta1, const double beta2, const double eps, - const int64_t step, Tensor param, Tensor m, Tensor v) - : grad_shape_{grad.shape()}, - grad_strides_{grad.strides()}, - grad_type_{grad.dtype()}, - param_shape_{param.shape()}, - param_strides_{param.strides()}, - param_type_{param.dtype()}, - m_shape_{m.shape()}, - m_strides_{m.strides()}, - m_type_{m.dtype()}, - v_shape_{v.shape()}, - v_strides_{v.strides()}, - v_type_{v.dtype()}, - learning_rate_{learning_rate}, - beta1_{beta1}, - beta2_{beta2}, - eps_{eps}, - step_{step}, - device_index_{param.device().index()} { - AssertTrainingDevices({grad, param, m, v}); - assert(grad.shape() == param.shape() && param.shape() == m.shape() && - m.shape() == v.shape() && - "`AdamAccumulateGrad` requires grad/param/m/v to share shape"); - assert(IsTrainingFloat(param_type_) && grad_type_ == param_type_ && - "Adam parameters and gradients require matching floating dtypes."); - assert(m_type_ == v_type_ && - (m_type_ == param_type_ || (param_type_ != DataType::kFloat64 && - m_type_ == DataType::kFloat32)) && - "Adam moments must match the parameter dtype or use float32."); - assert(grad.device() == param.device() && - "`AdamAccumulateGrad` requires grad/param on the same device"); - assert(0.0 <= beta1_ && beta1_ < 1.0 && 0.0 <= beta2_ && beta2_ < 1.0 && - "Adam beta values must be in [0, 1)."); - assert(learning_rate_ >= 0.0 && eps_ > 0.0 && - "Adam requires non-negative learning rate and positive epsilon."); - assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1"); - } - - [[deprecated("Use grad, attributes, param, m, v argument order.")]] - AdamAccumulateGrad(const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, - const double beta2, const double eps, const int64_t step) - : AdamAccumulateGrad{grad, learning_rate, beta1, beta2, eps, - step, param, m, v} {} - - [[deprecated("Use grad, attributes, param, m, v argument order.")]] - virtual void operator()(const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, - const double beta2, const double eps, - const int64_t step) const { - operator()(grad, learning_rate, beta1, beta2, eps, step, param, m, v); - } - - virtual void operator()(const Tensor grad, const double learning_rate, - const double beta1, const double beta2, - const double eps, const int64_t step, Tensor param, - Tensor m, Tensor v) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static bool IsTrainingFloat(DataType dtype) { - return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || - dtype == DataType::kFloat32 || dtype == DataType::kFloat64; - } - - Tensor::Shape grad_shape_; - Tensor::Strides grad_strides_; - DataType grad_type_; - Tensor::Shape param_shape_; - Tensor::Strides param_strides_; - DataType param_type_; - Tensor::Shape m_shape_; - Tensor::Strides m_strides_; - DataType m_type_; - Tensor::Shape v_shape_; - Tensor::Strides v_strides_; - DataType v_type_; - double learning_rate_{0.0}; - double beta1_{0.9}; - double beta2_{0.999}; - double eps_{1e-8}; - int64_t step_{1}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ +#ifndef INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AdamAccumulateGrad : public Operator { + public: + AdamAccumulateGrad(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, const double eps, + const int64_t step, Tensor param, Tensor m, Tensor v) + : grad_shape_{grad.shape()}, + grad_strides_{grad.strides()}, + grad_type_{grad.dtype()}, + param_shape_{param.shape()}, + param_strides_{param.strides()}, + param_type_{param.dtype()}, + m_shape_{m.shape()}, + m_strides_{m.strides()}, + m_type_{m.dtype()}, + v_shape_{v.shape()}, + v_strides_{v.strides()}, + v_type_{v.dtype()}, + learning_rate_{learning_rate}, + beta1_{beta1}, + beta2_{beta2}, + eps_{eps}, + step_{step}, + device_index_{param.device().index()} { + validation::AssertSameDevice({grad, param, m, v}); + assert(grad.shape() == param.shape() && param.shape() == m.shape() && + m.shape() == v.shape() && + "`AdamAccumulateGrad` requires grad/param/m/v to share shape"); + assert(validation::IsFloatingPoint(param_type_) && + grad_type_ == param_type_ && + "Adam parameters and gradients require matching floating dtypes."); + assert(m_type_ == v_type_ && + (m_type_ == param_type_ || (param_type_ != DataType::kFloat64 && + m_type_ == DataType::kFloat32)) && + "Adam moments must match the parameter dtype or use float32."); + + assert(0.0 <= beta1_ && beta1_ < 1.0 && 0.0 <= beta2_ && beta2_ < 1.0 && + "Adam beta values must be in [0, 1)."); + assert(learning_rate_ >= 0.0 && eps_ > 0.0 && + "Adam requires non-negative learning rate and positive epsilon."); + + assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1"); + } + + // Grad may alias outputs; param, m and v byte spans must not overlap. + virtual void operator()(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, + const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) const = 0; + + protected: + Tensor::Shape grad_shape_; + + Tensor::Strides grad_strides_; + + DataType grad_type_; + + Tensor::Shape param_shape_; + + Tensor::Strides param_strides_; + + DataType param_type_; + + Tensor::Shape m_shape_; + + Tensor::Strides m_strides_; + + DataType m_type_; + + Tensor::Shape v_shape_; + + Tensor::Strides v_strides_; + + DataType v_type_; + + double learning_rate_{0.0}; + + double beta1_{0.9}; + + double beta2_{0.999}; + + double eps_{1e-8}; + + int64_t step_{1}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ diff --git a/src/base/add_backward.h b/src/base/add_backward.h index 491354608..230c4b152 100644 --- a/src/base/add_backward.h +++ b/src/base/add_backward.h @@ -1,73 +1,63 @@ -#ifndef INFINI_OPS_BASE_ADD_BACKWARD_H_ -#define INFINI_OPS_BASE_ADD_BACKWARD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class AddBackward : public Operator { - public: - AddBackward(const Tensor grad_output, Tensor grad_a, Tensor grad_b) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_a_shape_{grad_a.shape()}, - grad_a_strides_{grad_a.strides()}, - grad_a_type_{grad_a.dtype()}, - grad_b_shape_{grad_b.shape()}, - grad_b_strides_{grad_b.strides()}, - grad_b_type_{grad_b.dtype()}, - device_index_{grad_a.device().index()} { - AssertTrainingDevices({grad_output, grad_a, grad_b}); - assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && - "`AddBackward` requires matching dtypes"); - assert( - grad_output.device() == grad_a.device() && - grad_a.device() == grad_b.device() && - "`AddBackward` requires grad_output/grad_a/grad_b on the same device"); - ValidateBroadcast(grad_a, grad_output); - ValidateBroadcast(grad_b, grad_output); - } - - virtual void operator()(const Tensor grad_output, Tensor grad_a, - Tensor grad_b) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static void ValidateBroadcast(const Tensor input, const Tensor out) { - assert(input.ndim() <= out.ndim() && - "`AddBackward` input rank must not exceed grad_output rank"); - const auto offset = out.ndim() - input.ndim(); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert( - (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`AddBackward` input must broadcast (right-aligned) to grad_output"); - } - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape grad_a_shape_; - Tensor::Strides grad_a_strides_; - DataType grad_a_type_; - Tensor::Shape grad_b_shape_; - Tensor::Strides grad_b_strides_; - DataType grad_b_type_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_ADD_BACKWARD_H_ +#define INFINI_OPS_BASE_ADD_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AddBackward : public Operator { + public: + AddBackward(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`AddBackward` requires matching dtypes"); + + validation::AssertBroadcastsTo(grad_input, grad_output); + + validation::AssertBroadcastsTo(grad_other, grad_output); + } + + // Inputs may alias outputs; output byte spans must not overlap. + virtual void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ diff --git a/src/base/cross_entropy.h b/src/base/cross_entropy.h index ac5ffadf0..e59123d6e 100644 --- a/src/base/cross_entropy.h +++ b/src/base/cross_entropy.h @@ -1,80 +1,64 @@ -#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_H_ -#define INFINI_OPS_BASE_CROSS_ENTROPY_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class CrossEntropy : public Operator { - public: - static constexpr int64_t kIgnoreIndex = -100; - - CrossEntropy(const Tensor input, const Tensor target, Tensor out) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - target_shape_{target.shape()}, - target_strides_{target.strides()}, - target_type_{target.dtype()}, - out_shape_{out.shape()}, - out_strides_{out.strides()}, - out_type_{out.dtype()}, - device_index_{out.device().index()} { - AssertTrainingDevices({input, target, out}); - assert(input.ndim() >= 2 && "`CrossEntropy` requires input rank >= 2"); - assert(input.device() == target.device() && - target.device() == out.device() && - "`CrossEntropy` requires all tensors on the same device"); - assert((input_type_ == DataType::kFloat16 || - input_type_ == DataType::kBFloat16 || - input_type_ == DataType::kFloat32 || - input_type_ == DataType::kFloat64) && - "`CrossEntropy` requires floating-point logits"); - assert((target_type_ == DataType::kInt64 || - target_type_ == DataType::kInt32 || - target_type_ == DataType::kUInt8) && - "`CrossEntropy` supports int64/int32/uint8 targets"); - assert(target.ndim() == input.ndim() - 1 && - "`CrossEntropy` target rank must be input rank - 1"); - for (Tensor::Size i = 0; i < target.ndim(); ++i) { - const auto input_dim = i == 0 ? 0 : i + 1; - assert(target.size(i) == input.size(input_dim) && - "`CrossEntropy` target shape mismatch"); - } - assert(out.ndim() == 0 && "`CrossEntropy` output must be a 0-dim scalar"); - assert(out_type_ == input_type_ && - "`CrossEntropy` output dtype must match input"); - } - - virtual void operator()(const Tensor input, const Tensor target, - Tensor out) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape target_shape_; - Tensor::Strides target_strides_; - DataType target_type_; - Tensor::Shape out_shape_; - Tensor::Strides out_strides_; - DataType out_type_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_CROSS_ENTROPY_H_ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_H_ + +#include +#include +#include +#include + +#include "base/cross_entropy_common.h" +#include "operator.h" + +namespace infini::ops { + +class CrossEntropy : public Operator { + public: + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropy(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + device_index_{out.device().index()} { + ValidateCrossEntropy(input, target, weight, ignore_index, reduction, + label_smoothing, out); + } + + virtual void operator()(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, + Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape target_shape_; + + Tensor::Strides target_strides_; + + DataType target_type_; + + Tensor::Shape out_shape_; + + Tensor::Strides out_strides_; + + DataType out_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_H_ diff --git a/src/base/cross_entropy_backward.h b/src/base/cross_entropy_backward.h index 2ae87cfab..50085b78d 100644 --- a/src/base/cross_entropy_backward.h +++ b/src/base/cross_entropy_backward.h @@ -1,98 +1,77 @@ -#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ -#define INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class CrossEntropyBackward : public Operator { - public: - static constexpr int64_t kIgnoreIndex = -100; - - CrossEntropyBackward(const Tensor input, const Tensor target, - const Tensor grad_output, Tensor grad_input) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - target_shape_{target.shape()}, - target_strides_{target.strides()}, - target_type_{target.dtype()}, - grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({input, target, grad_output, grad_input}); - assert(input.ndim() >= 2 && - "`CrossEntropyBackward` requires input rank >= 2"); - assert(input.device() == target.device() && - target.device() == grad_output.device() && - grad_output.device() == grad_input.device() && - "`CrossEntropyBackward` requires all tensors on the same device"); - assert((input_type_ == DataType::kFloat16 || - input_type_ == DataType::kBFloat16 || - input_type_ == DataType::kFloat32 || - input_type_ == DataType::kFloat64) && - "`CrossEntropyBackward` requires floating-point logits"); - assert((target_type_ == DataType::kInt64 || - target_type_ == DataType::kInt32 || - target_type_ == DataType::kUInt8) && - "`CrossEntropyBackward` supports int64/int32/uint8 targets"); - assert(target.ndim() == input.ndim() - 1 && - "`CrossEntropyBackward` target rank must be input rank - 1"); - for (Tensor::Size i = 0; i < target.ndim(); ++i) { - const auto input_dim = i == 0 ? 0 : i + 1; - assert(target.size(i) == input.size(input_dim) && - "`CrossEntropyBackward` target shape mismatch"); - } - assert(grad_input.shape() == input.shape() && - "`CrossEntropyBackward` grad_input shape must match input"); - assert(grad_output.ndim() == 0 && - "`CrossEntropyBackward` requires scalar grad_output"); - assert(grad_output_type_ == input_type_ && - grad_input_type_ == input_type_ && - "`CrossEntropyBackward` requires matching floating-point dtypes"); - } - - virtual void operator()(const Tensor input, const Tensor target, - const Tensor grad_output, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - - Tensor::Shape target_shape_; - Tensor::Strides target_strides_; - DataType target_type_; - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ + +#include +#include + +#include "base/cross_entropy_common.h" +#include "operator.h" + +namespace infini::ops { + +class CrossEntropyBackward : public Operator { + public: + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropyBackward(const Tensor input, const Tensor target, + const Tensor grad_output, std::optional weight, + int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + ValidateCrossEntropy(input, target, weight, ignore_index, reduction, + label_smoothing, grad_output); + validation::AssertSameDevice({input, grad_input}); + assert(grad_input.shape() == input.shape() && + grad_input.dtype() == input.dtype() && + "`CrossEntropyBackward` grad_input must match logits metadata."); + } + + virtual void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape target_shape_; + + Tensor::Strides target_strides_; + + DataType target_type_; + + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/base/cross_entropy_common.h b/src/base/cross_entropy_common.h new file mode 100644 index 000000000..c9c4496b0 --- /dev/null +++ b/src/base/cross_entropy_common.h @@ -0,0 +1,78 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ + +#include +#include + +#include "base/validation.h" + +namespace infini::ops { + +inline DataType PromoteCrossEntropyType(DataType a, DataType b) { + if (a == b) return a; + if (a == DataType::kFloat64 || b == DataType::kFloat64) { + return DataType::kFloat64; + } + return DataType::kFloat32; +} + +inline void ValidateCrossEntropy(const Tensor input, const Tensor target, + std::optional weight, + int64_t ignore_index, + const std::string& reduction, + double label_smoothing, const Tensor loss) { + validation::AssertSameDevice({input, target, loss}); + + assert(input.ndim() >= 1 && "`CrossEntropy` requires input rank >= 1."); + assert(validation::IsFloatingPoint(input.dtype()) && + "`CrossEntropy` requires floating-point logits."); + assert((reduction == "none" || reduction == "mean" || reduction == "sum") && + "`CrossEntropy` reduction must be none, mean or sum."); + assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && + "`CrossEntropy` label_smoothing must be in [0, 1]."); + const auto class_dim = input.ndim() == 1 ? 0 : 1; + assert(input.size(class_dim) > 0 && + "`CrossEntropy` requires a non-empty class dimension."); + Tensor::Shape loss_shape; + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + if (i != class_dim) loss_shape.push_back(input.size(i)); + } + const bool probability_target = target.shape() == input.shape(); + + auto result_type = input.dtype(); + if (probability_target) { + assert(validation::IsFloatingPoint(target.dtype()) && + "`CrossEntropy` probability target must be floating-point."); + assert(ignore_index < 0 && + "`CrossEntropy` probability target does not support ignore_index."); + result_type = PromoteCrossEntropyType(result_type, target.dtype()); + } else { + assert((target.dtype() == DataType::kInt64 || + target.dtype() == DataType::kInt32 || + target.dtype() == DataType::kUInt8) && + "`CrossEntropy` class target must be int64, int32 or uint8."); + assert( + target.shape() == loss_shape && + "`CrossEntropy` class target shape must remove the class dimension."); + } + if (weight) { + validation::AssertSameDevice({input, *weight}); + assert(weight->ndim() == 1 && weight->size(0) == input.size(class_dim) && + "`CrossEntropy` weight must have shape (C,)."); + if (probability_target) { + if (validation::IsFloatingPoint(weight->dtype())) { + result_type = PromoteCrossEntropyType(result_type, weight->dtype()); + } + } else { + assert(weight->dtype() == input.dtype() && + "`CrossEntropy` class-target weight dtype must match logits."); + } + } + assert(loss.shape() == (reduction == "none" ? loss_shape : Tensor::Shape{}) && + "`CrossEntropy` loss shape does not match reduction."); + assert(loss.dtype() == result_type && "`CrossEntropy` loss dtype mismatch."); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ diff --git a/src/base/embedding_backward.h b/src/base/embedding_backward.h deleted file mode 100644 index 17212202f..000000000 --- a/src/base/embedding_backward.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ -#define INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class EmbeddingBackward : public Operator { - public: - EmbeddingBackward(const Tensor grad_output, const Tensor indices, - const int64_t num_weights, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - indices_shape_{indices.shape()}, - indices_strides_{indices.strides()}, - indices_type_{indices.dtype()}, - grad_weight_shape_{grad_weight.shape()}, - grad_weight_strides_{grad_weight.strides()}, - grad_weight_type_{grad_weight.dtype()}, - num_weights_{num_weights}, - padding_idx_{padding_idx}, - scale_grad_by_freq_{scale_grad_by_freq}, - sparse_{sparse}, - device_index_{grad_weight.device().index()} { - AssertTrainingDevices({grad_output, indices, grad_weight}); - assert(!(sparse_ && scale_grad_by_freq_) && - "Sparse embedding does not support frequency scaling."); - assert((indices_type_ == DataType::kInt32 || - indices_type_ == DataType::kInt64) && - "`EmbeddingBackward` requires int32 or int64 `indices`"); - assert(grad_output_type_ == grad_weight_type_ && - "`EmbeddingBackward` requires `grad_output` and `grad_weight` to " - "have the same dtype"); - assert(grad_weight.ndim() == 2 && - "`EmbeddingBackward` requires 2D `grad_weight`"); - assert(grad_weight.size(0) == num_weights_ && - "`EmbeddingBackward` requires `grad_weight` row count to equal " - "`num_weights`"); - assert(grad_output.ndim() == indices.ndim() + 1 && - "`EmbeddingBackward` requires `grad_output` rank to be `indices` " - "rank + 1"); - for (Tensor::Size i = 0; i < indices.ndim(); ++i) { - assert(grad_output.size(i) == indices.size(i) && - "`EmbeddingBackward` requires matching leading dims"); - } - assert(grad_output.size(-1) == grad_weight.size(1) && - "`EmbeddingBackward` requires last dim of `grad_output` to equal " - "embedding dim"); - } - - virtual void operator()(const Tensor grad_output, const Tensor indices, - const int64_t num_weights, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape indices_shape_; - Tensor::Strides indices_strides_; - DataType indices_type_; - Tensor::Shape grad_weight_shape_; - Tensor::Strides grad_weight_strides_; - DataType grad_weight_type_; - int64_t num_weights_{0}; - int64_t padding_idx_{-1}; - bool scale_grad_by_freq_{false}; - bool sparse_{false}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_EMBEDDING_BACKWARD_H_ diff --git a/src/base/masked_fill_backward.h b/src/base/masked_fill_backward.h index 2dadf51d3..987ba33b9 100644 --- a/src/base/masked_fill_backward.h +++ b/src/base/masked_fill_backward.h @@ -1,86 +1,97 @@ -#ifndef INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ -#define INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class MaskedFillBackward : public Operator { - public: - MaskedFillBackward(const Tensor grad_output, const Tensor mask, - Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - mask_shape_{mask.shape()}, - mask_strides_{mask.strides()}, - mask_type_{mask.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, mask, grad_input}); - assert( - grad_output.shape() == grad_input.shape() && - "`MaskedFillBackward` requires grad_output/grad_input to share shape"); - assert( - grad_output_type_ == grad_input_type_ && - "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); - assert(mask_type_ == DataType::kBool && - "`MaskedFillBackward` requires a bool `mask`"); - assert(mask.ndim() <= grad_output.ndim() && - "`MaskedFillBackward` mask rank must not exceed tensor rank"); - - tail_aligned_ = AlignsAt(mask, grad_output, /*tail=*/true); - if (!tail_aligned_) { - assert( - AlignsAt(mask, grad_output, /*tail=*/false) && - "`MaskedFillBackward` mask must align to leading or trailing dims"); - } - } - - virtual void operator()(const Tensor grad_output, const Tensor mask, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static bool AlignsAt(const Tensor mask, const Tensor tensor, - const bool tail) { - const auto offset = tail ? tensor.ndim() - mask.ndim() : 0; - for (Tensor::Size i = 0; i < mask.ndim(); ++i) { - const auto dim = tensor.size(i + offset); - if (mask.size(i) != 1 && mask.size(i) != dim) { - return false; - } - } - return true; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape mask_shape_; - Tensor::Strides mask_strides_; - DataType mask_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - bool tail_aligned_{true}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ +#define INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ + +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MaskedFillBackward : public Operator { + public: + MaskedFillBackward(const Tensor grad_output, const Tensor mask, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + mask_shape_{mask.shape()}, + mask_strides_{mask.strides()}, + mask_type_{mask.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, mask, grad_input}); + assert( + grad_output_type_ == grad_input_type_ && + "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); + assert(mask_type_ == DataType::kBool && + "`MaskedFillBackward` requires a bool `mask`"); + assert(IsValidBroadcastShape(grad_input_shape_, mask_shape_, + grad_output_shape_) && + "`MaskedFillBackward` requires grad_output shape to equal the " + "broadcast shape of grad_input and mask"); + } + + virtual void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const = 0; + + protected: + static bool IsValidBroadcastShape(const Tensor::Shape& input_shape, + const Tensor::Shape& mask_shape, + const Tensor::Shape& output_shape) { + const auto ndim = input_shape.size() > mask_shape.size() + ? input_shape.size() + : mask_shape.size(); + if (output_shape.size() != ndim) { + return false; + } + + const auto input_offset = ndim - input_shape.size(); + + const auto mask_offset = ndim - mask_shape.size(); + + for (std::size_t i = 0; i < ndim; ++i) { + const auto input_dim = + i < input_offset ? 1 : input_shape[i - input_offset]; + const auto mask_dim = i < mask_offset ? 1 : mask_shape[i - mask_offset]; + + if (input_dim != mask_dim && input_dim != 1 && mask_dim != 1) { + return false; + } + + const auto output_dim = input_dim == 1 ? mask_dim : input_dim; + if (output_shape[i] != output_dim) { + return false; + } + } + return true; + } + + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape mask_shape_; + + Tensor::Strides mask_strides_; + + DataType mask_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ diff --git a/src/base/mean_backward.h b/src/base/mean_backward.h index 137f10ad1..3e448e18f 100644 --- a/src/base/mean_backward.h +++ b/src/base/mean_backward.h @@ -1,83 +1,79 @@ -#ifndef INFINI_OPS_BASE_MEAN_BACKWARD_H_ -#define INFINI_OPS_BASE_MEAN_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class MeanBackward : public Operator { - public: - MeanBackward(const Tensor grad_output, const int64_t dim, const bool keep_dim, - Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - dim_{NormalizeDim(dim, grad_input.ndim())}, - keep_dim_{keep_dim}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, grad_input}); - assert(grad_output_type_ == grad_input_type_ && - "`MeanBackward` requires matching dtypes"); - assert(grad_input.ndim() >= 1 && "`MeanBackward` requires ndim >= 1"); - if (keep_dim_) { - assert(grad_output.ndim() == grad_input.ndim() && - grad_output.size(dim_) == 1 && - "`MeanBackward` keep_dim=true requires size-1 dim in grad_output"); - for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { - if (i == dim_) continue; - assert(grad_output.size(i) == grad_input.size(i) && - "`MeanBackward` shape mismatch"); - } - } else { - assert(grad_output.ndim() == grad_input.ndim() - 1 && - "`MeanBackward` keep_dim=false requires rank-1 grad_output"); - for (Tensor::Size i = 0, s = 0; i < grad_input.ndim(); ++i) { - if (i == dim_) continue; - assert(grad_output.size(s) == grad_input.size(i) && - "`MeanBackward` shape mismatch"); - ++s; - } - } - } - - virtual void operator()(const Tensor grad_output, const int64_t dim, - const bool keep_dim, Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - int64_t dim_{0}; - bool keep_dim_{false}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_MEAN_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_MEAN_BACKWARD_H_ +#define INFINI_OPS_BASE_MEAN_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MeanBackward : public Operator { + public: + MeanBackward(const Tensor grad_output, const int64_t dim, const bool keepdim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{validation::NormalizeDim(dim, grad_input.ndim())}, + keepdim_{keepdim}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output_type_ == grad_input_type_ && + "`MeanBackward` requires matching dtypes"); + + assert(grad_input.ndim() >= 1 && "`MeanBackward` requires ndim >= 1"); + if (keepdim_) { + assert(grad_output.ndim() == grad_input.ndim() && + grad_output.size(dim_) == 1 && + "`MeanBackward` keepdim=true requires size-1 dim in grad_output"); + + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); + } + } else { + assert(grad_output.ndim() == grad_input.ndim() - 1 && + "`MeanBackward` keepdim=false requires rank-1 grad_output"); + + for (Tensor::Size i = 0, s = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(s) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); + + ++s; + } + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + const bool keepdim, Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t dim_{0}; + + bool keepdim_{false}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MEAN_BACKWARD_H_ diff --git a/src/base/mul_backward.h b/src/base/mul_backward.h index c606f8ba5..cd86d93cf 100644 --- a/src/base/mul_backward.h +++ b/src/base/mul_backward.h @@ -1,91 +1,92 @@ -#ifndef INFINI_OPS_BASE_MUL_BACKWARD_H_ -#define INFINI_OPS_BASE_MUL_BACKWARD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class MulBackward : public Operator { - public: - MulBackward(const Tensor grad_output, const Tensor a, const Tensor b, - Tensor grad_a, Tensor grad_b) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - a_shape_{a.shape()}, - a_strides_{a.strides()}, - a_type_{a.dtype()}, - b_shape_{b.shape()}, - b_strides_{b.strides()}, - b_type_{b.dtype()}, - grad_a_shape_{grad_a.shape()}, - grad_a_strides_{grad_a.strides()}, - grad_a_type_{grad_a.dtype()}, - grad_b_shape_{grad_b.shape()}, - grad_b_strides_{grad_b.strides()}, - grad_b_type_{grad_b.dtype()}, - device_index_{grad_a.device().index()} { - AssertTrainingDevices({grad_output, a, b, grad_a, grad_b}); - assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && - "`MulBackward` requires matching grad dtypes"); - assert(a_type_ == grad_output_type_ && b_type_ == grad_output_type_ && - "`MulBackward` requires a/b dtype to match grad_output dtype"); - assert(grad_output.device() == a.device() && a.device() == b.device() && - "`MulBackward` requires all tensors on the same device"); - assert(grad_a.shape() == a.shape() && grad_b.shape() == b.shape() && - "Multiplication gradients must match their input shapes."); - ValidateBroadcast(a, grad_output); - ValidateBroadcast(b, grad_output); - ValidateBroadcast(grad_a, grad_output); - ValidateBroadcast(grad_b, grad_output); - } - - virtual void operator()(const Tensor grad_output, const Tensor a, - const Tensor b, Tensor grad_a, - Tensor grad_b) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static void ValidateBroadcast(const Tensor input, const Tensor out) { - assert(input.ndim() <= out.ndim() && - "`MulBackward` input rank must not exceed grad_output rank"); - const auto offset = out.ndim() - input.ndim(); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert( - (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`MulBackward` input must broadcast (right-aligned) to grad_output"); - } - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape a_shape_; - Tensor::Strides a_strides_; - DataType a_type_; - Tensor::Shape b_shape_; - Tensor::Strides b_strides_; - DataType b_type_; - Tensor::Shape grad_a_shape_; - Tensor::Strides grad_a_strides_; - DataType grad_a_type_; - Tensor::Shape grad_b_shape_; - Tensor::Strides grad_b_strides_; - DataType grad_b_type_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_MUL_BACKWARD_H_ +#define INFINI_OPS_BASE_MUL_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MulBackward : public Operator { + public: + MulBackward(const Tensor grad_output, const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + other_shape_{other.shape()}, + other_strides_{other.strides()}, + other_type_{other.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice( + {grad_output, input, other, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`MulBackward` requires matching grad dtypes"); + assert( + input_type_ == grad_output_type_ && other_type_ == grad_output_type_ && + "`MulBackward` requires input/other dtype to match grad_output dtype"); + + assert(grad_input.shape() == input.shape() && + grad_other.shape() == other.shape() && + "Multiplication gradients must match their input shapes."); + + validation::AssertBroadcastsTo(input, grad_output); + + validation::AssertBroadcastsTo(other, grad_output); + } + + // Aliasing: inputs may overlap each other and either output. + // The storage byte spans of grad_input and grad_other must not overlap, + // including gaps in non-contiguous layouts. + virtual void operator()(const Tensor grad_output, const Tensor input, + const Tensor other, Tensor grad_input, + Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape other_shape_; + + Tensor::Strides other_strides_; + + DataType other_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ diff --git a/src/base/pow_backward.h b/src/base/pow_backward.h index 8ac2d20b7..c184c7393 100644 --- a/src/base/pow_backward.h +++ b/src/base/pow_backward.h @@ -1,76 +1,73 @@ -#ifndef INFINI_OPS_BASE_POW_BACKWARD_H_ -#define INFINI_OPS_BASE_POW_BACKWARD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class PowBackward : public Operator { - public: - PowBackward(const Tensor grad_output, const Tensor input, const double scalar, - const bool scalar_is_base, Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - scalar_{scalar}, - scalar_is_base_{scalar_is_base}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, input, grad_input}); - assert( - input.shape() == grad_output.shape() && - grad_output.shape() == grad_input.shape() && - "`PowBackward` requires input/grad_output/grad_input to share shape"); - assert(input_type_ == grad_output_type_ && - grad_output_type_ == grad_input_type_ && - "`PowBackward` requires matching dtypes"); - assert((!scalar_is_base_ || scalar_ > 0.0) && - "`PowBackward` scalar base must be positive for log(scalar)."); - assert(IsTrainingFloat(input_type_) && - "Power gradients require floating dtypes."); - } - - virtual void operator()(const Tensor grad_output, const Tensor input, - const double scalar, const bool scalar_is_base, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static bool IsTrainingFloat(DataType dtype) { - return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || - dtype == DataType::kFloat32 || dtype == DataType::kFloat64; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - double scalar_{0.0}; - bool scalar_is_base_{false}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_POW_BACKWARD_H_ +#define INFINI_OPS_BASE_POW_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class PowBackward : public Operator { + public: + PowBackward(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + scalar_{scalar}, + scalar_is_base_{scalar_is_base}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, input, grad_input}); + assert( + input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`PowBackward` requires input/grad_output/grad_input to share shape"); + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && + "`PowBackward` requires matching dtypes"); + assert((!scalar_is_base_ || scalar_ > 0.0) && + "`PowBackward` scalar base must be positive for log(scalar)."); + assert(validation::IsFloatingPoint(input_type_) && + "Power gradients require floating dtypes."); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + const double scalar, const bool scalar_is_base, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + double scalar_{0.0}; + + bool scalar_is_base_{false}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ diff --git a/src/base/repeat_interleave.h b/src/base/repeat_interleave.h index e80aad14a..5afae25d8 100644 --- a/src/base/repeat_interleave.h +++ b/src/base/repeat_interleave.h @@ -1,72 +1,66 @@ -#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ -#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class RepeatInterleave : public Operator { - public: - RepeatInterleave(const Tensor input, const int64_t repeats, const int64_t dim, - Tensor out) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - out_shape_{out.shape()}, - out_strides_{out.strides()}, - out_type_{out.dtype()}, - repeats_{repeats}, - dim_{NormalizeDim(dim, input.ndim())}, - device_index_{out.device().index()} { - AssertTrainingDevices({input, out}); - assert(repeats_ >= 0 && - "`RepeatInterleave` requires non-negative repeats."); - assert(input.ndim() == out.ndim() && "RepeatInterleave ranks must match."); - assert(input_type_ == out_type_ && - "`RepeatInterleave` requires same dtype"); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - const auto expect = - (i == dim_) ? input.size(i) * repeats_ : input.size(i); - assert(out.size(i) == expect && - "`RepeatInterleave` output shape mismatch"); - } - } - - virtual void operator()(const Tensor input, const int64_t repeats, - const int64_t dim, Tensor out) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape out_shape_; - Tensor::Strides out_strides_; - DataType out_type_; - int64_t repeats_{0}; - int64_t dim_{0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleave : public Operator { + public: + RepeatInterleave(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + repeats_{repeats}, + dim_{validation::NormalizeDim(dim, input.ndim())}, + device_index_{out.device().index()} { + validation::AssertSameDevice({input, out}); + assert(repeats_ >= 0 && + "`RepeatInterleave` requires non-negative repeats."); + + assert(input.ndim() == out.ndim() && "RepeatInterleave ranks must match."); + assert(input_type_ == out_type_ && + "`RepeatInterleave` requires same dtype"); + + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + const auto expect = + (i == dim_) ? input.size(i) * repeats_ : input.size(i); + assert(out.size(i) == expect && + "`RepeatInterleave` output shape mismatch"); + } + } + + virtual void operator()(const Tensor input, const int64_t repeats, + const int64_t dim, Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape out_shape_; + + Tensor::Strides out_strides_; + + DataType out_type_; + + int64_t repeats_{0}; + + int64_t dim_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ diff --git a/src/base/repeat_interleave_backward.h b/src/base/repeat_interleave_backward.h index bee70b8e9..b8d586fb0 100644 --- a/src/base/repeat_interleave_backward.h +++ b/src/base/repeat_interleave_backward.h @@ -1,78 +1,72 @@ -#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ -#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class RepeatInterleaveBackward : public Operator { - public: - RepeatInterleaveBackward(const Tensor grad_output, const int64_t dim, - Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - dim_{NormalizeDim(dim, grad_input.ndim())}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, grad_input}); - assert(grad_output_type_ == grad_input_type_ && - "`RepeatInterleaveBackward` requires same dtype"); - assert(grad_output.ndim() == grad_input.ndim() && - "`RepeatInterleaveBackward` requires same rank"); - for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { - if (i == dim_) continue; - assert(grad_output.size(i) == grad_input.size(i) && - "`RepeatInterleaveBackward` shape mismatch on non-repeated dims"); - } - if (grad_input.size(dim_) == 0) { - assert(grad_output.size(dim_) == 0 && - "Empty repeated dimensions must match."); - repeats_ = 0; - } else { - repeats_ = grad_output.size(dim_) / grad_input.size(dim_); - assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && - "RepeatInterleaveBackward grad_output dim must be divisible."); - } - } - - virtual void operator()(const Tensor grad_output, const int64_t dim, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - int64_t repeats_{0}; - int64_t dim_{0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleaveBackward : public Operator { + public: + RepeatInterleaveBackward(const Tensor grad_output, const int64_t dim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{validation::NormalizeDim(dim, grad_input.ndim())}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output_type_ == grad_input_type_ && + "`RepeatInterleaveBackward` requires same dtype"); + assert(grad_output.ndim() == grad_input.ndim() && + "`RepeatInterleaveBackward` requires same rank"); + + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && + "`RepeatInterleaveBackward` shape mismatch on non-repeated dims"); + } + if (grad_input.size(dim_) == 0) { + assert(grad_output.size(dim_) == 0 && + "Empty repeated dimensions must match."); + + repeats_ = 0; + } else { + repeats_ = grad_output.size(dim_) / grad_input.size(dim_); + assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && + "RepeatInterleaveBackward grad_output dim must be divisible."); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t repeats_{0}; + + int64_t dim_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ diff --git a/src/base/rsqrt_backward.h b/src/base/rsqrt_backward.h index 145569cbb..28be8d647 100644 --- a/src/base/rsqrt_backward.h +++ b/src/base/rsqrt_backward.h @@ -1,67 +1,62 @@ -#ifndef INFINI_OPS_BASE_RSQRT_BACKWARD_H_ -#define INFINI_OPS_BASE_RSQRT_BACKWARD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class RsqrtBackward : public Operator { - public: - RsqrtBackward(const Tensor grad_output, const Tensor input, Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, input, grad_input}); - assert(input.shape() == grad_output.shape() && - grad_output.shape() == grad_input.shape() && - "`RsqrtBackward` requires matching shapes"); - assert(input_type_ == grad_output_type_ && - grad_output_type_ == grad_input_type_ && - "`RsqrtBackward` requires matching dtypes"); - assert(IsTrainingFloat(input_type_) && - "`RsqrtBackward` requires floating point dtypes"); - } - - virtual void operator()(const Tensor grad_output, const Tensor input, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static bool IsTrainingFloat(DataType dtype) { - return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || - dtype == DataType::kFloat32 || dtype == DataType::kFloat64; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_RSQRT_BACKWARD_H_ +#define INFINI_OPS_BASE_RSQRT_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RsqrtBackward : public Operator { + public: + RsqrtBackward(const Tensor grad_output, const Tensor input, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, input, grad_input}); + assert(input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`RsqrtBackward` requires matching shapes"); + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && + "`RsqrtBackward` requires matching dtypes"); + assert(validation::IsFloatingPoint(input_type_) && + "`RsqrtBackward` requires floating point dtypes"); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ diff --git a/src/base/slice.h b/src/base/slice.h deleted file mode 100644 index 8f456888e..000000000 --- a/src/base/slice.h +++ /dev/null @@ -1,86 +0,0 @@ -#ifndef INFINI_OPS_BASE_SLICE_H_ -#define INFINI_OPS_BASE_SLICE_H_ - -#include -#include -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class Slice : public Operator { - public: - Slice(const Tensor input, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor out) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - out_shape_{out.shape()}, - out_strides_{out.strides()}, - out_type_{out.dtype()}, - starts_{starts}, - ends_{ends}, - steps_{steps}, - device_index_{out.device().index()} { - AssertTrainingDevices({input, out}); - assert(starts_.size() == input.ndim() && ends_.size() == input.ndim() && - steps_.size() == input.ndim() && - "`Slice` requires starts/ends/steps to match input rank"); - assert(out.ndim() == input.ndim() && "`Slice` requires matching rank"); - assert(input_type_ == out_type_ && "`Slice` requires same dtype"); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert(steps_[i] > 0 && "`Slice` requires positive steps"); - const auto expected = - TrainingSliceLength(starts_[i], ends_[i], steps_[i], input.size(i)); - assert(out.size(i) == expected && - "`Slice` output dim must equal ceil((end - start) / step)"); - } - } - - virtual void operator()(const Tensor input, const std::vector starts, - const std::vector ends, - const std::vector steps, - Tensor out) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t TrainingSliceLength(int64_t start, int64_t end, int64_t step, - Tensor::Size size) { - assert(step > 0 && "Training slices require a positive step."); - const auto length = static_cast(size); - const auto normalize = [length](int64_t index) { - if (index < 0) index += length; - return std::clamp(index, 0, length); - }; - start = normalize(start); - end = normalize(end); - return start >= end ? 0 : 1 + (end - start - 1) / step; - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape out_shape_; - Tensor::Strides out_strides_; - DataType out_type_; - std::vector starts_; - std::vector ends_; - std::vector steps_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_SLICE_H_ diff --git a/src/base/slice_backward.h b/src/base/slice_backward.h deleted file mode 100644 index 429559e14..000000000 --- a/src/base/slice_backward.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef INFINI_OPS_BASE_SLICE_BACKWARD_H_ -#define INFINI_OPS_BASE_SLICE_BACKWARD_H_ - -#include -#include -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class SliceBackward : public Operator { - public: - SliceBackward(const Tensor grad_output, const std::vector starts, - const std::vector ends, - const std::vector steps, Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - starts_{starts}, - ends_{ends}, - steps_{steps}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, grad_input}); - assert(starts_.size() == grad_input.ndim() && - ends_.size() == grad_input.ndim() && - steps_.size() == grad_input.ndim() && - "`SliceBackward` requires starts/ends/steps to match `grad_input` " - "rank"); - assert(grad_output.ndim() == grad_input.ndim() && - "`SliceBackward` requires matching rank"); - assert(grad_output_type_ == grad_input_type_ && - "`SliceBackward` requires same dtype"); - for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { - assert(steps_[i] > 0 && "`SliceBackward` requires positive steps"); - const auto expected = TrainingSliceLength(starts_[i], ends_[i], steps_[i], - grad_input.size(i)); - assert(grad_output.size(i) == expected && - "`SliceBackward` grad_output dim must equal the slice dim"); - } - } - - virtual void operator()(const Tensor grad_output, - const std::vector starts, - const std::vector ends, - const std::vector steps, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t TrainingSliceLength(int64_t start, int64_t end, int64_t step, - Tensor::Size size) { - assert(step > 0 && "Training slices require a positive step."); - const auto length = static_cast(size); - const auto normalize = [length](int64_t index) { - if (index < 0) index += length; - return std::clamp(index, 0, length); - }; - start = normalize(start); - end = normalize(end); - return start >= end ? 0 : 1 + (end - start - 1) / step; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - std::vector starts_; - std::vector ends_; - std::vector steps_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_SLICE_BACKWARD_H_ diff --git a/src/base/split.h b/src/base/split.h index 935de3d99..475e3bbe9 100644 --- a/src/base/split.h +++ b/src/base/split.h @@ -1,91 +1,88 @@ -#ifndef INFINI_OPS_BASE_SPLIT_H_ -#define INFINI_OPS_BASE_SPLIT_H_ - -#include -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class Split : public Operator { - public: - Split(const Tensor input, const int64_t split_size, const int64_t dim, - std::vector outs) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - split_size_{split_size}, - dim_{NormalizeDim(dim, input.ndim())}, - outs_{std::move(outs)}, - device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { - assert(split_size_ > 0 && "`Split` requires positive `split_size`"); - assert(!outs_.empty() && "`Split` requires at least one output tensor"); - const auto expected = - input.size(dim_) == 0 ? 1 : 1 + (input.size(dim_) - 1) / split_size_; - assert( - static_cast(outs_.size()) == expected && - "`Split` output count must equal ceil(input_size(dim) / split_size)"); - - for (std::size_t i = 0; i < outs_.size(); ++i) { - AssertTrainingDevices({input, outs_[i]}); - assert(outs_[i].ndim() == input.ndim() && "`Split` output rank mismatch"); - assert(outs_[i].dtype() == input_type_ && "`Split` dtype mismatch"); - for (Tensor::Size d = 0; d < input.ndim(); ++d) { - if (d != dim_) { - assert(outs_[i].size(d) == input.size(d) && - "`Split` output shape mismatch"); - } - } - const auto expect_dim = std::min( - split_size_, static_cast(input.size(dim_)) - - static_cast(i) * split_size_); - assert(outs_[i].size(dim_) == expect_dim && - "`Split` output dim size mismatch"); - - out_shapes_.emplace_back(outs_[i].shape().begin(), - outs_[i].shape().end()); - out_strides_.emplace_back(outs_[i].strides().begin(), - outs_[i].strides().end()); - out_types_.emplace_back(outs_[i].dtype()); - } - } - - virtual void operator()(const Tensor input, const int64_t split_size, - const int64_t dim, - std::vector outs) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - int64_t split_size_{0}; - int64_t dim_{0}; - std::vector outs_{}; - std::vector out_shapes_; - std::vector out_strides_; - std::vector out_types_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_SPLIT_H_ +#ifndef INFINI_OPS_BASE_SPLIT_H_ +#define INFINI_OPS_BASE_SPLIT_H_ + +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class Split : public Operator { + public: + Split(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + split_size_{split_size}, + dim_{validation::NormalizeDim(dim, input.ndim())}, + outs_{std::move(outs)}, + device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { + assert(split_size_ > 0 && "`Split` requires positive `split_size`"); + + assert(!outs_.empty() && "`Split` requires at least one output tensor"); + const auto expected = + input.size(dim_) == 0 ? 1 : 1 + (input.size(dim_) - 1) / split_size_; + assert( + static_cast(outs_.size()) == expected && + "`Split` output count must equal ceil(input_size(dim) / split_size)"); + + for (std::size_t i = 0; i < outs_.size(); ++i) { + validation::AssertSameDevice({input, outs_[i]}); + assert(outs_[i].ndim() == input.ndim() && "`Split` output rank mismatch"); + + assert(outs_[i].dtype() == input_type_ && "`Split` dtype mismatch"); + + for (Tensor::Size d = 0; d < input.ndim(); ++d) { + if (d != dim_) { + assert(outs_[i].size(d) == input.size(d) && + "`Split` output shape mismatch"); + } + } + const auto expect_dim = std::min( + split_size_, static_cast(input.size(dim_)) - + static_cast(i) * split_size_); + assert(outs_[i].size(dim_) == expect_dim && + "`Split` output dim size mismatch"); + + out_shapes_.emplace_back(outs_[i].shape().begin(), + outs_[i].shape().end()); + out_strides_.emplace_back(outs_[i].strides().begin(), + outs_[i].strides().end()); + + out_types_.emplace_back(outs_[i].dtype()); + } + } + + virtual void operator()(const Tensor input, const int64_t split_size, + const int64_t dim, + std::vector outs) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + int64_t split_size_{0}; + + int64_t dim_{0}; + + std::vector outs_{}; + + std::vector out_shapes_; + + std::vector out_strides_; + + std::vector out_types_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SPLIT_H_ diff --git a/src/base/stack_backward.h b/src/base/stack_backward.h index de4eb381c..9fcf6b7ec 100644 --- a/src/base/stack_backward.h +++ b/src/base/stack_backward.h @@ -1,83 +1,80 @@ -#ifndef INFINI_OPS_BASE_STACK_BACKWARD_H_ -#define INFINI_OPS_BASE_STACK_BACKWARD_H_ - -#include -#include -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class StackBackward : public Operator { - public: - StackBackward(const Tensor grad_output, const int64_t dim, - std::vector outs) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - dim_{NormalizeDim(dim, grad_output.ndim())}, - outs_{std::move(outs)}, - device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { - assert(!outs_.empty() && "`StackBackward` requires at least one output"); - assert(static_cast(outs_.size()) == grad_output.size(dim_) && - "`StackBackward` output count must equal grad_output.size(dim)"); - - for (std::size_t i = 0; i < outs_.size(); ++i) { - AssertTrainingDevices({grad_output, outs_[i]}); - assert(outs_[i].ndim() == grad_output.ndim() - 1 && - "`StackBackward` output rank must be grad_output rank - 1"); - assert(outs_[i].dtype() == grad_output_type_ && - "`StackBackward` dtype mismatch"); - for (Tensor::Size d = 0, s = 0; d < grad_output.ndim(); ++d) { - if (d == dim_) continue; - assert(outs_[i].size(s) == grad_output.size(d) && - "`StackBackward` shape mismatch"); - ++s; - } - out_shapes_.emplace_back(outs_[i].shape().begin(), - outs_[i].shape().end()); - - out_strides_.emplace_back(outs_[i].strides().begin(), - outs_[i].strides().end()); - - out_types_.push_back(outs_[i].dtype()); - } - } - - virtual void operator()(const Tensor grad_output, const int64_t dim, - std::vector outs) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - int64_t dim_{0}; - std::vector outs_{}; - std::vector out_shapes_; - std::vector out_strides_; - std::vector out_types_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_STACK_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_STACK_BACKWARD_H_ +#define INFINI_OPS_BASE_STACK_BACKWARD_H_ + +#include +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class StackBackward : public Operator { + public: + StackBackward(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + dim_{validation::NormalizeDim(dim, grad_output.ndim())}, + grad_inputs_{std::move(grad_inputs)}, + device_index_{ + grad_inputs_.empty() ? 0 : grad_inputs_.front().device().index()} { + assert(!grad_inputs_.empty() && + "`StackBackward` requires at least one output"); + assert(static_cast(grad_inputs_.size()) == + grad_output.size(dim_) && + "`StackBackward` output count must equal grad_output.size(dim)"); + + for (std::size_t i = 0; i < grad_inputs_.size(); ++i) { + validation::AssertSameDevice({grad_output, grad_inputs_[i]}); + assert(grad_inputs_[i].ndim() == grad_output.ndim() - 1 && + "`StackBackward` output rank must be grad_output rank - 1"); + assert(grad_inputs_[i].dtype() == grad_output_type_ && + "`StackBackward` dtype mismatch"); + + for (Tensor::Size d = 0, s = 0; d < grad_output.ndim(); ++d) { + if (d == dim_) continue; + assert(grad_inputs_[i].size(s) == grad_output.size(d) && + "`StackBackward` shape mismatch"); + + ++s; + } + grad_input_shapes_.emplace_back(grad_inputs_[i].shape().begin(), + grad_inputs_[i].shape().end()); + + grad_input_strides_.emplace_back(grad_inputs_[i].strides().begin(), + grad_inputs_[i].strides().end()); + + grad_input_types_.push_back(grad_inputs_[i].dtype()); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + int64_t dim_{0}; + + std::vector grad_inputs_{}; + + std::vector grad_input_shapes_; + + std::vector grad_input_strides_; + + std::vector grad_input_types_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_STACK_BACKWARD_H_ diff --git a/src/base/sub_backward.h b/src/base/sub_backward.h index 99a996192..71dcfd37a 100644 --- a/src/base/sub_backward.h +++ b/src/base/sub_backward.h @@ -1,73 +1,63 @@ -#ifndef INFINI_OPS_BASE_SUB_BACKWARD_H_ -#define INFINI_OPS_BASE_SUB_BACKWARD_H_ - -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class SubBackward : public Operator { - public: - SubBackward(const Tensor grad_output, Tensor grad_a, Tensor grad_b) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_a_shape_{grad_a.shape()}, - grad_a_strides_{grad_a.strides()}, - grad_a_type_{grad_a.dtype()}, - grad_b_shape_{grad_b.shape()}, - grad_b_strides_{grad_b.strides()}, - grad_b_type_{grad_b.dtype()}, - device_index_{grad_a.device().index()} { - AssertTrainingDevices({grad_output, grad_a, grad_b}); - assert(grad_output_type_ == grad_a_type_ && grad_a_type_ == grad_b_type_ && - "`SubBackward` requires matching dtypes"); - assert( - grad_output.device() == grad_a.device() && - grad_a.device() == grad_b.device() && - "`SubBackward` requires grad_output/grad_a/grad_b on the same device"); - ValidateBroadcast(grad_a, grad_output); - ValidateBroadcast(grad_b, grad_output); - } - - virtual void operator()(const Tensor grad_output, Tensor grad_a, - Tensor grad_b) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static void ValidateBroadcast(const Tensor input, const Tensor out) { - assert(input.ndim() <= out.ndim() && - "`SubBackward` input rank must not exceed grad_output rank"); - const auto offset = out.ndim() - input.ndim(); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - assert( - (input.size(i) == 1 || input.size(i) == out.size(i + offset)) && - "`SubBackward` input must broadcast (right-aligned) to grad_output"); - } - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - Tensor::Shape grad_a_shape_; - Tensor::Strides grad_a_strides_; - DataType grad_a_type_; - Tensor::Shape grad_b_shape_; - Tensor::Strides grad_b_strides_; - DataType grad_b_type_; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_SUB_BACKWARD_H_ +#define INFINI_OPS_BASE_SUB_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class SubBackward : public Operator { + public: + SubBackward(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`SubBackward` requires matching dtypes"); + + validation::AssertBroadcastsTo(grad_input, grad_output); + + validation::AssertBroadcastsTo(grad_other, grad_output); + } + + // Inputs may alias outputs; output byte spans must not overlap. + virtual void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ diff --git a/src/base/transpose.h b/src/base/transpose.h deleted file mode 100644 index 9297fd80a..000000000 --- a/src/base/transpose.h +++ /dev/null @@ -1,70 +0,0 @@ -#ifndef INFINI_OPS_BASE_TRANSPOSE_H_ -#define INFINI_OPS_BASE_TRANSPOSE_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class Transpose : public Operator { - public: - Transpose(const Tensor input, const int64_t dim0, const int64_t dim1, - Tensor out) - : input_shape_{input.shape()}, - input_strides_{input.strides()}, - input_type_{input.dtype()}, - out_shape_{out.shape()}, - out_strides_{out.strides()}, - out_type_{out.dtype()}, - dim0_{NormalizeDim(dim0, input.ndim())}, - dim1_{NormalizeDim(dim1, input.ndim())}, - device_index_{out.device().index()} { - AssertTrainingDevices({input, out}); - assert(input.ndim() >= 1 && "Transpose requires a non-scalar input."); - assert(out.ndim() == input.ndim() && "Transpose ranks must match."); - assert(input_type_ == out_type_ && "`Transpose` requires same dtype"); - for (Tensor::Size i = 0; i < input.ndim(); ++i) { - const auto expect = (i == dim0_) ? input.size(dim1_) - : (i == dim1_) ? input.size(dim0_) - : input.size(i); - assert(out.size(i) == expect && "`Transpose` output shape mismatch"); - } - } - - virtual void operator()(const Tensor input, const int64_t dim0, - const int64_t dim1, Tensor out) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape input_shape_; - Tensor::Strides input_strides_; - DataType input_type_; - Tensor::Shape out_shape_; - Tensor::Strides out_strides_; - DataType out_type_; - int64_t dim0_{0}; - int64_t dim1_{0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_TRANSPOSE_H_ diff --git a/src/base/transpose_backward.h b/src/base/transpose_backward.h index a4ad75be2..8f80bac68 100644 --- a/src/base/transpose_backward.h +++ b/src/base/transpose_backward.h @@ -1,76 +1,66 @@ -#ifndef INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ -#define INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class TransposeBackward : public Operator { - public: - TransposeBackward(const Tensor grad_output, const int64_t dim0, - const int64_t dim1, Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - dim0_{NormalizeDim(dim0, grad_output.ndim())}, - dim1_{NormalizeDim(dim1, grad_output.ndim())}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, grad_input}); - assert(grad_output.ndim() >= 1 && "Transpose requires a non-scalar input."); - assert(grad_input.ndim() == grad_output.ndim() && - "Transpose ranks must match."); - assert(grad_output_type_ == grad_input_type_ && - "`TransposeBackward` requires same dtype"); - - for (Tensor::Size i = 0; i < grad_output.ndim(); ++i) { - const auto expect = (i == dim0_) ? grad_output.size(dim1_) - : (i == dim1_) ? grad_output.size(dim0_) - : grad_output.size(i); - assert(grad_input.size(i) == expect && - "`TransposeBackward` grad_input shape mismatch"); - } - } - - virtual void operator()(const Tensor grad_output, const int64_t dim0, - const int64_t dim1, Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static int64_t NormalizeDim(const int64_t dim, const Tensor::Size ndim) { - const auto rank = static_cast(ndim); - assert(rank > 0 && "Training dimension requires a non-scalar tensor."); - assert(dim >= -rank && dim < rank && "Training dimension is out of range."); - return dim < 0 ? dim + rank : dim; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - - int64_t dim0_{0}; - int64_t dim1_{0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ +#define INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class TransposeBackward : public Operator { + public: + TransposeBackward(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim0_{validation::NormalizeDim(dim0, grad_output.ndim())}, + dim1_{validation::NormalizeDim(dim1, grad_output.ndim())}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output.ndim() >= 1 && "Transpose requires a non-scalar input."); + assert(grad_input.ndim() == grad_output.ndim() && + "Transpose ranks must match."); + assert(grad_output_type_ == grad_input_type_ && + "`TransposeBackward` requires same dtype"); + + for (Tensor::Size i = 0; i < grad_output.ndim(); ++i) { + const auto expect = (i == dim0_) ? grad_output.size(dim1_) + : (i == dim1_) ? grad_output.size(dim0_) + : grad_output.size(i); + assert(grad_input.size(i) == expect && + "`TransposeBackward` grad_input shape mismatch"); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t dim0_{0}; + + int64_t dim1_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ diff --git a/src/base/validation.h b/src/base/validation.h new file mode 100644 index 000000000..23900b0db --- /dev/null +++ b/src/base/validation.h @@ -0,0 +1,50 @@ +#ifndef INFINI_OPS_BASE_VALIDATION_H_ +#define INFINI_OPS_BASE_VALIDATION_H_ + +#include +#include +#include + +#include "data_type.h" +#include "tensor.h" + +namespace infini::ops::validation { + +inline void AssertSameDevice(std::initializer_list tensors) { + if (tensors.size() == 0) return; + + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Tensors must share a device type and index."); + } +} + +inline bool IsFloatingPoint(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; +} + +inline int64_t NormalizeDim(int64_t dim, Tensor::Size ndim) { + const auto rank = static_cast(ndim); + + assert(rank > 0 && "Dimension requires a non-scalar tensor."); + + assert(dim >= -rank && dim < rank && "Dimension is out of range."); + + return dim < 0 ? dim + rank : dim; +} + +inline void AssertBroadcastsTo(const Tensor input, const Tensor output) { + assert(input.ndim() <= output.ndim() && + "Input rank must not exceed output rank."); + const auto offset = output.ndim() - input.ndim(); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert((input.size(i) == 1 || input.size(i) == output.size(i + offset)) && + "Input must broadcast to output with right-aligned dimensions."); + } +} + +} // namespace infini::ops::validation + +#endif // INFINI_OPS_BASE_VALIDATION_H_ diff --git a/src/base/vocab_parallel_cross_entropy_backward.h b/src/base/vocab_parallel_cross_entropy_backward.h index 8dfba74ec..0279d27a3 100644 --- a/src/base/vocab_parallel_cross_entropy_backward.h +++ b/src/base/vocab_parallel_cross_entropy_backward.h @@ -1,112 +1,113 @@ -#ifndef INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ -#define INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ - -#include -#include -#include - -#include "operator.h" - -namespace infini::ops { - -class VocabParallelCrossEntropyBackward - : public Operator { - public: - VocabParallelCrossEntropyBackward( - const Tensor grad_output, const Tensor softmax_local, - const Tensor target_mask, const Tensor masked_target, - const Tensor valid_mask_local, const int64_t vocab_size_local, - const int64_t vocab_size_original, const double label_smoothing, - Tensor grad_input) - : grad_output_shape_{grad_output.shape()}, - grad_output_strides_{grad_output.strides()}, - grad_output_type_{grad_output.dtype()}, - softmax_shape_{softmax_local.shape()}, - softmax_strides_{softmax_local.strides()}, - softmax_type_{softmax_local.dtype()}, - target_mask_shape_{target_mask.shape()}, - target_mask_strides_{target_mask.strides()}, - target_mask_type_{target_mask.dtype()}, - masked_target_shape_{masked_target.shape()}, - masked_target_strides_{masked_target.strides()}, - masked_target_type_{masked_target.dtype()}, - valid_mask_shape_{valid_mask_local.shape()}, - valid_mask_strides_{valid_mask_local.strides()}, - valid_mask_type_{valid_mask_local.dtype()}, - grad_input_shape_{grad_input.shape()}, - grad_input_strides_{grad_input.strides()}, - grad_input_type_{grad_input.dtype()}, - vocab_size_local_{vocab_size_local}, - vocab_size_original_{vocab_size_original}, - label_smoothing_{label_smoothing}, - device_index_{grad_input.device().index()} { - AssertTrainingDevices({grad_output, softmax_local, target_mask, - masked_target, valid_mask_local, grad_input}); - assert(IsTrainingFloat(grad_input_type_) && - grad_input_type_ == softmax_type_ && - grad_output_type_ == grad_input_type_ && - "Loss gradients require matching floating dtypes."); - assert(softmax_local.shape() == grad_input.shape() && - "Local softmax and gradient shapes must match."); - assert(vocab_size_local > 0 && vocab_size_original > 0 && - "Vocabulary sizes must be positive."); - assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && - "Label smoothing must be in [0, 1]."); - } - - virtual void operator()(const Tensor grad_output, const Tensor softmax_local, - const Tensor target_mask, const Tensor masked_target, - const Tensor valid_mask_local, - const int64_t vocab_size_local, - const int64_t vocab_size_original, - const double label_smoothing, - Tensor grad_input) const = 0; - - protected: - static void AssertTrainingDevices(std::initializer_list tensors) { - if (tensors.size() == 0) return; - const auto device = tensors.begin()->device(); - for (const auto& tensor : tensors) { - assert(tensor.device() == device && - "Training tensors must share a device type and index."); - } - } - - static bool IsTrainingFloat(DataType dtype) { - return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || - dtype == DataType::kFloat32 || dtype == DataType::kFloat64; - } - - Tensor::Shape grad_output_shape_; - Tensor::Strides grad_output_strides_; - DataType grad_output_type_; - - Tensor::Shape softmax_shape_; - Tensor::Strides softmax_strides_; - DataType softmax_type_; - - Tensor::Shape target_mask_shape_; - Tensor::Strides target_mask_strides_; - DataType target_mask_type_; - - Tensor::Shape masked_target_shape_; - Tensor::Strides masked_target_strides_; - DataType masked_target_type_; - - Tensor::Shape valid_mask_shape_; - Tensor::Strides valid_mask_strides_; - DataType valid_mask_type_; - - Tensor::Shape grad_input_shape_; - Tensor::Strides grad_input_strides_; - DataType grad_input_type_; - - int64_t vocab_size_local_{0}; - int64_t vocab_size_original_{0}; - double label_smoothing_{0.0}; - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#ifndef INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class VocabParallelCrossEntropyBackward + : public Operator { + public: + VocabParallelCrossEntropyBackward( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + softmax_shape_{softmax_local.shape()}, + softmax_strides_{softmax_local.strides()}, + softmax_type_{softmax_local.dtype()}, + target_mask_shape_{target_mask.shape()}, + target_mask_strides_{target_mask.strides()}, + target_mask_type_{target_mask.dtype()}, + masked_target_shape_{masked_target.shape()}, + masked_target_strides_{masked_target.strides()}, + masked_target_type_{masked_target.dtype()}, + valid_mask_shape_{valid_mask_local.shape()}, + valid_mask_strides_{valid_mask_local.strides()}, + valid_mask_type_{valid_mask_local.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + vocab_size_local_{vocab_size_local}, + vocab_size_original_{vocab_size_original}, + label_smoothing_{label_smoothing}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, softmax_local, target_mask, + masked_target, valid_mask_local, grad_input}); + assert(validation::IsFloatingPoint(grad_input_type_) && + grad_input_type_ == softmax_type_ && + grad_output_type_ == grad_input_type_ && + "Loss gradients require matching floating dtypes."); + assert(softmax_local.shape() == grad_input.shape() && + "Local softmax and gradient shapes must match."); + assert(vocab_size_local > 0 && vocab_size_original > 0 && + "Vocabulary sizes must be positive."); + assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && + "Label smoothing must be in [0, 1]."); + } + + virtual void operator()(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, + const int64_t vocab_size_local, + const int64_t vocab_size_original, + const double label_smoothing, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape softmax_shape_; + + Tensor::Strides softmax_strides_; + + DataType softmax_type_; + + Tensor::Shape target_mask_shape_; + + Tensor::Strides target_mask_strides_; + + DataType target_mask_type_; + + Tensor::Shape masked_target_shape_; + + Tensor::Strides masked_target_strides_; + + DataType masked_target_type_; + + Tensor::Shape valid_mask_shape_; + + Tensor::Strides valid_mask_strides_; + + DataType valid_mask_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t vocab_size_local_{0}; + + int64_t vocab_size_original_{0}; + + double label_smoothing_{0.0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.cc b/src/torch/ops/accumulate_grad/accumulate_grad.cc index 6ceefd068..37a46aac6 100644 --- a/src/torch/ops/accumulate_grad/accumulate_grad.cc +++ b/src/torch/ops/accumulate_grad/accumulate_grad.cc @@ -1,6 +1,7 @@ #include "torch/ops/accumulate_grad/accumulate_grad.h" #include "torch/instantiate.h" +#include "torch/storage_overlap.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" @@ -9,8 +10,7 @@ namespace infini::ops { template Operator::Operator(const Tensor gradient, const double rate, Tensor tensor) - : AccumulateGrad{gradient, rate, tensor}, - device_index_{tensor.device().index()} {} + : AccumulateGrad{gradient, rate, tensor} {} template void Operator::operator()(const Tensor gradient, @@ -24,10 +24,10 @@ void Operator::operator()(const Tensor gradient, ToAtenTensor(tensor.data(), tensor_shape_, tensor_strides_, tensor_type_, device_index_); - const auto at_rate = at_tensor.is_floating_point() - ? at::Scalar(rate) - : at::Scalar(static_cast(rate)); - at_tensor.add_(at_gradient, at_rate); + if (StorageSpansOverlap(at_tensor, at_gradient)) { + at_gradient = at_gradient.clone(); + } + at_tensor.add_(at_gradient, rate); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AccumulateGrad, 1) diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.h b/src/torch/ops/accumulate_grad/accumulate_grad.h index 007f4d4e6..863186cdb 100644 --- a/src/torch/ops/accumulate_grad/accumulate_grad.h +++ b/src/torch/ops/accumulate_grad/accumulate_grad.h @@ -12,9 +12,6 @@ class Operator : public AccumulateGrad { void operator()(const Tensor gradient, const double rate, Tensor tensor) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc index d19f30c86..6ca6cf04b 100644 --- a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc @@ -3,6 +3,7 @@ #include #include "torch/instantiate.h" +#include "torch/storage_overlap.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" @@ -14,15 +15,7 @@ Operator::Operator( const double beta2, const double eps, const int64_t step, Tensor param, Tensor m, Tensor v) : AdamAccumulateGrad{grad, learning_rate, beta1, beta2, eps, - step, param, m, v}, - device_index_{param.device().index()} {} - -template -Operator::Operator( - const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, const double beta2, - const double eps, const int64_t step) - : Operator{grad, learning_rate, beta1, beta2, eps, step, param, m, v} {} + step, param, m, v} {} template void Operator::operator()( @@ -39,18 +32,22 @@ void Operator::operator()( auto at_v = ToAtenTensor(v.data(), v_shape_, v_strides_, v_type_, device_index_); + assert(!StorageSpansOverlap(at_param, at_m) && + !StorageSpansOverlap(at_param, at_v) && + !StorageSpansOverlap(at_m, at_v) && + "`AdamAccumulateGrad` output storage must not overlap."); + const auto compute_type = at_param.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; const auto gradient = at_grad.to(compute_type); - const auto next_m = - at_m.to(compute_type) * beta1_ + gradient * (1.0 - beta1_); - const auto next_v = - at_v.to(compute_type) * beta2_ + gradient.square() * (1.0 - beta2_); + const auto next_m = at::lerp(at_m.to(compute_type), gradient, 1.0 - beta1_); + auto next_v = at_v.to(compute_type) * beta2_; + next_v.addcmul_(gradient, gradient, 1.0 - beta2_); const double bcm = 1.0 - std::pow(beta1_, static_cast(step_)); const double bcv = 1.0 - std::pow(beta2_, static_cast(step_)); - const auto update = - learning_rate_ * (next_m / bcm) / (at::sqrt(next_v / bcv) + eps_); - const auto next_param = at_param.to(compute_type) - update; + const auto denominator = at::sqrt(next_v) / std::sqrt(bcv) + eps_; + const auto next_param = at::addcdiv(at_param.to(compute_type), next_m, + denominator, -learning_rate_ / bcm); at_param.copy_(next_param); at_m.copy_(next_m); at_v.copy_(next_v); diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h index 895e70fe4..8579c0206 100644 --- a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h @@ -8,24 +8,14 @@ namespace infini::ops { template class Operator : public AdamAccumulateGrad { public: - using AdamAccumulateGrad::operator(); - Operator(const Tensor grad, const double learning_rate, const double beta1, const double beta2, const double eps, const int64_t step, Tensor param, Tensor m, Tensor v); - [[deprecated("Use grad, attributes, param, m, v argument order.")]] - Operator(const Tensor grad, Tensor param, Tensor m, Tensor v, - const double learning_rate, const double beta1, const double beta2, - const double eps, const int64_t step); - void operator()(const Tensor grad, const double learning_rate, const double beta1, const double beta2, const double eps, const int64_t step, Tensor param, Tensor m, Tensor v) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/add_backward/add_backward.cc b/src/torch/ops/add_backward/add_backward.cc index dfab951fe..a59745add 100644 --- a/src/torch/ops/add_backward/add_backward.cc +++ b/src/torch/ops/add_backward/add_backward.cc @@ -5,35 +5,48 @@ #include #include "torch/instantiate.h" +#include "torch/storage_overlap.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { + template Operator::Operator(const Tensor grad_output, - Tensor grad_a, Tensor grad_b) - : AddBackward{grad_output, grad_a, grad_b}, - device_index_{grad_a.device().index()} {} + const double alpha, Tensor grad_input, + Tensor grad_other) + : AddBackward{grad_output, alpha, grad_input, grad_other} {} template void Operator::operator()(const Tensor grad_output, - Tensor grad_a, - Tensor grad_b) const { + const double alpha, + Tensor grad_input, + Tensor grad_other) const { const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_a = - ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, - grad_a_type_, device_index_); - auto at_grad_b = - ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, - grad_b_type_, device_index_); - - at_grad_a.copy_((at_grad).sum_to_size( - std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); - at_grad_b.copy_((at_grad).sum_to_size( - std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`AddBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = at_grad.sum_to_size(grad_input_sizes).clone(); + auto grad_other_result = (at_grad * alpha).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AddBackward, 1) diff --git a/src/torch/ops/add_backward/add_backward.h b/src/torch/ops/add_backward/add_backward.h index 7c2ca0a73..b534455db 100644 --- a/src/torch/ops/add_backward/add_backward.h +++ b/src/torch/ops/add_backward/add_backward.h @@ -8,13 +8,11 @@ namespace infini::ops { template class Operator : public AddBackward { public: - Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b); + Operator(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other); - void operator()(const Tensor grad_output, Tensor grad_a, - Tensor grad_b) const override; - - private: - int device_index_{0}; + void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const override; }; } // namespace infini::ops diff --git a/src/torch/ops/cross_entropy/cross_entropy.cc b/src/torch/ops/cross_entropy/cross_entropy.cc index 2b26ef1e9..5eabdecc1 100644 --- a/src/torch/ops/cross_entropy/cross_entropy.cc +++ b/src/torch/ops/cross_entropy/cross_entropy.cc @@ -1,5 +1,7 @@ #include "torch/ops/cross_entropy/cross_entropy.h" +#include + #include "torch/instantiate.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" @@ -8,25 +10,59 @@ namespace infini::ops { template Operator::Operator(const Tensor input, - const Tensor target, Tensor out) - : CrossEntropy{input, target, out}, device_index_{out.device().index()} {} + const Tensor target, + std::optional weight, + int64_t ignore_index, + std::string reduction, + double label_smoothing, Tensor out) + : CrossEntropy{input, target, weight, ignore_index, + reduction, label_smoothing, out} {} template -void Operator::operator()(const Tensor input, - const Tensor target, - Tensor out) const { +void Operator::operator()( + const Tensor input, const Tensor target, std::optional weight, + int64_t ignore_index, std::string reduction, double label_smoothing, + Tensor out) const { const TorchStreamGuard guard{stream_, device_index_}; auto at_input = ToAtenTensor(const_cast(input.data()), input_shape_, input_strides_, input_type_, device_index_); auto at_target = ToAtenTensor(const_cast(target.data()), target_shape_, - target_strides_, target_type_, device_index_) - .to(at::kLong); + target_strides_, target_type_, device_index_); auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, out_type_, device_index_); - auto loss = at::cross_entropy_loss(at_input, at_target); + std::optional at_weight; + if (weight) { + at_weight = + ToAtenTensor(const_cast(weight->data()), weight->shape(), + weight->strides(), weight->dtype(), device_index_); + } + + const bool probability_target = target_shape_ == input_shape_; + if (!probability_target) at_target = at_target.to(at::kLong); + + const auto promote_low_precision = [](at::Tensor value) { + if (value.scalar_type() == at::kHalf || + value.scalar_type() == at::kBFloat16) { + return value.to(at::kFloat); + } + return value; + }; + at_input = promote_low_precision(at_input); + if (probability_target) at_target = promote_low_precision(at_target); + if (at_weight) *at_weight = promote_low_precision(*at_weight); + + assert((reduction == "none" || reduction == "mean" || reduction == "sum") && + "Invalid CrossEntropy reduction"); + const int64_t at_reduction = reduction == "none" ? at::Reduction::None + : reduction == "sum" ? at::Reduction::Sum + : at::Reduction::Mean; + + auto loss = + at::cross_entropy_loss(at_input, at_target, at_weight, at_reduction, + ignore_index, label_smoothing); at_out.copy_(loss); } diff --git a/src/torch/ops/cross_entropy/cross_entropy.h b/src/torch/ops/cross_entropy/cross_entropy.h index f5c5a0c58..87adfb06a 100644 --- a/src/torch/ops/cross_entropy/cross_entropy.h +++ b/src/torch/ops/cross_entropy/cross_entropy.h @@ -8,15 +8,16 @@ namespace infini::ops { template class Operator : public CrossEntropy { public: - Operator(const Tensor input, const Tensor target, Tensor out); + Operator(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor out); void operator()(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor out) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops -#endif +#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_H_ diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc index 82d806bb2..3cb92b777 100644 --- a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc @@ -1,63 +1,94 @@ -#include "torch/ops/cross_entropy_backward/cross_entropy_backward.h" - -#include - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor input, - const Tensor target, - const Tensor grad_output, - Tensor grad_input) - : CrossEntropyBackward{input, target, grad_output, grad_input} {} - -template -void Operator::operator()( - const Tensor input, const Tensor target, const Tensor grad_output, - Tensor grad_input) const { - const TorchStreamGuard guard{stream_, device_index_}; - auto at_input = - ToAtenTensor(const_cast(input.data()), input_shape_, - input_strides_, input_type_, device_index_); - auto at_target = - ToAtenTensor(const_cast(target.data()), target_shape_, - target_strides_, target_type_, device_index_); - auto at_grad_output = ToAtenTensor( - const_cast(grad_output.data()), grad_output_shape_, - grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_input = - ToAtenTensor(grad_input.data(), grad_input_shape_, - grad_input_strides_, grad_input_type_, device_index_); - - const auto classes = at_input.size(1); - assert(classes > 0 && - "CrossEntropyBackward requires a non-empty class dimension"); - - auto gradient = at::softmax(at_input, 1).to(at_grad_input.scalar_type()); - auto target_with_class_dim = at_target.to(at::kLong).unsqueeze(1); - const auto valid_target = - target_with_class_dim.ne(CrossEntropyBackward::kIgnoreIndex); - const auto valid_range = - (target_with_class_dim >= 0) & (target_with_class_dim < classes); - assert(at::all((~valid_target) | valid_range).template item() && - "CrossEntropyBackward target is outside [0, classes) or ignore_index"); - - auto safe_target = target_with_class_dim.masked_fill(~valid_target, 0); - auto minus_one = -valid_target.to(gradient.scalar_type()); - gradient.scatter_add_(1, safe_target, minus_one); - gradient.mul_(valid_target.to(gradient.scalar_type())); - - const auto valid_count = - valid_target.sum().to(gradient.scalar_type()).clamp_min(1); - gradient.mul_(at_grad_output.reshape({1}).to(gradient.scalar_type()) / - valid_count); - at_grad_input.copy_(gradient); -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropyBackward, 1) - -} // namespace infini::ops +#include "torch/ops/cross_entropy_backward/cross_entropy_backward.h" + +#include + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) + : CrossEntropyBackward{input, target, grad_output, + weight, ignore_index, reduction, + label_smoothing, grad_input} {} + +template +void Operator::operator()( + const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto x = ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto target_tensor = + ToAtenTensor(const_cast(target.data()), target_shape_, + target_strides_, target_type_, device_index_); + auto upstream = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto out = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + // Accumulate low-precision derivatives and reduction denominators in FP32. + const auto compute_type = + x.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + x = x.to(compute_type); + if (upstream.scalar_type() == at::kHalf || + upstream.scalar_type() == at::kBFloat16) { + upstream = upstream.to(at::kFloat); + } + const int64_t dim = x.dim() == 1 ? 0 : 1; + const auto classes = x.size(dim); + const bool probability_target = target_shape_ == input_shape_; + std::vector weight_shape(x.dim(), 1); + weight_shape[dim] = classes; + auto class_weight = at::ones({classes}, x.options()); + if (weight) { + class_weight = + ToAtenTensor(const_cast(weight->data()), weight->shape(), + weight->strides(), weight->dtype(), device_index_); + if (class_weight.scalar_type() != at::kDouble) { + class_weight = class_weight.to(compute_type); + } + } + at::Tensor valid; + at::Tensor denominator; + at::Tensor distribution; + if (probability_target) { + if (target_tensor.scalar_type() != at::kDouble) { + target_tensor = target_tensor.to(compute_type); + } + distribution = + target_tensor * (1.0 - label_smoothing) + label_smoothing / classes; + denominator = at::scalar_tensor(x.numel() / classes, x.options()); + } else { + auto indices = target_tensor.to(at::kLong); + valid = indices.ne(ignore_index); + auto safe_indices = indices.masked_fill(~valid, 0); + distribution = at::full_like(x, label_smoothing / classes); + auto selected = distribution.gather(dim, safe_indices.unsqueeze(dim)); + distribution.scatter_(dim, safe_indices.unsqueeze(dim), + selected + (1.0 - label_smoothing)); + const auto flat_indices = safe_indices.reshape({-1}); + const auto flat_weights = class_weight.index_select(0, flat_indices); + auto selected_weight = flat_weights.reshape(indices.sizes()); + denominator = selected_weight.masked_fill(~valid, 0).sum(); + } + auto weighted = distribution * class_weight.reshape(weight_shape); + auto gradient = at::softmax(x, dim) * weighted.sum(dim, true) - weighted; + if (reduction == "mean") gradient = gradient / denominator; + if (reduction == "none") upstream = upstream.unsqueeze(dim); + gradient = gradient * upstream; + // Ignored positions stay zero even when a mean has zero total weight. + if (!probability_target) gradient.masked_fill_(~valid.unsqueeze(dim), 0); + out.copy_(gradient); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropyBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h index b869d31e8..435b08062 100644 --- a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h @@ -1,20 +1,23 @@ -#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ -#define INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ - -#include "base/cross_entropy_backward.h" - -namespace infini::ops { - -template -class Operator : public CrossEntropyBackward { - public: - Operator(const Tensor input, const Tensor target, const Tensor grad_output, - Tensor grad_input); - - void operator()(const Tensor input, const Tensor target, - const Tensor grad_output, Tensor grad_input) const override; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ +#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ + +#include "base/cross_entropy_backward.h" + +namespace infini::ops { + +template +class Operator : public CrossEntropyBackward { + public: + Operator(const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor grad_input); + + void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, std::optional weight, + int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/embedding_backward/embedding_backward.cc b/src/torch/ops/embedding_backward/embedding_backward.cc deleted file mode 100644 index 48486ef90..000000000 --- a/src/torch/ops/embedding_backward/embedding_backward.cc +++ /dev/null @@ -1,47 +0,0 @@ -#include "torch/ops/embedding_backward/embedding_backward.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator( - const Tensor grad_output, const Tensor indices, const int64_t num_weights, - const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight) - : EmbeddingBackward{grad_output, indices, num_weights, padding_idx, - scale_grad_by_freq, sparse, grad_weight}, - device_index_{grad_weight.device().index()} {} - -template -void Operator::operator()( - const Tensor grad_output, const Tensor indices, const int64_t num_weights, - const int64_t padding_idx, const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight) const { - const TorchStreamGuard guard{stream_, device_index_}; - auto at_grad_output = ToAtenTensor( - const_cast(grad_output.data()), grad_output_shape_, - grad_output_strides_, grad_output_type_, device_index_); - auto at_indices = - ToAtenTensor(const_cast(indices.data()), indices_shape_, - indices_strides_, indices_type_, device_index_) - .to(at::kLong); - auto at_grad_weight = ToAtenTensor( - grad_weight.data(), grad_weight_shape_, grad_weight_strides_, - grad_weight_type_, device_index_); - - auto at_grad = - sparse_ ? at::embedding_backward(at_grad_output, at_indices, num_weights_, - padding_idx_, scale_grad_by_freq_, true) - .to_dense() - : at::embedding_dense_backward(at_grad_output, at_indices, - num_weights_, padding_idx_, - scale_grad_by_freq_); - at_grad_weight.copy_(at_grad); -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(EmbeddingBackward, 1) - -} // namespace infini::ops diff --git a/src/torch/ops/embedding_backward/embedding_backward.h b/src/torch/ops/embedding_backward/embedding_backward.h deleted file mode 100644 index 7224a146d..000000000 --- a/src/torch/ops/embedding_backward/embedding_backward.h +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ -#define INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ - -#include "base/embedding_backward.h" - -namespace infini::ops { - -template -class Operator : public EmbeddingBackward { - public: - Operator(const Tensor grad_output, const Tensor indices, - const int64_t num_weights, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight); - - void operator()(const Tensor grad_output, const Tensor indices, - const int64_t num_weights, const int64_t padding_idx, - const bool scale_grad_by_freq, const bool sparse, - Tensor grad_weight) const override; - - private: - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_TORCH_EMBEDDING_BACKWARD_H_ diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc index e50190639..cf549673a 100644 --- a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc @@ -1,5 +1,7 @@ #include "torch/ops/masked_fill_backward/masked_fill_backward.h" +#include + #include "torch/instantiate.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" @@ -10,31 +12,29 @@ template Operator::Operator(const Tensor grad_output, const Tensor mask, Tensor grad_input) - : MaskedFillBackward{grad_output, mask, grad_input}, - device_index_{grad_input.device().index()} {} + : MaskedFillBackward{grad_output, mask, grad_input} {} template void Operator::operator()( const Tensor grad_output, const Tensor mask, Tensor grad_input) const { const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); auto at_mask = ToAtenTensor(const_cast(mask.data()), mask_shape_, - mask_strides_, mask_type_, device_index_) - .to(at::kBool); + mask_strides_, mask_type_, device_index_); + at_mask = at_mask.to(at::kBool); auto at_grad_input = ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - if (!tail_aligned_) { - for (Tensor::Size i = mask_shape_.size(); i < grad_input_shape_.size(); - ++i) { - at_mask = at_mask.unsqueeze(-1); - } - } + auto masked_grad = at::masked_fill(at_grad_output, at_mask, 0); + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + auto result = masked_grad.sum_to_size(grad_input_sizes); - at_grad_input.copy_(at::masked_fill(at_grad_output, at_mask, 0)); + at_grad_input.copy_(result); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MaskedFillBackward, 1) diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.h b/src/torch/ops/masked_fill_backward/masked_fill_backward.h index 60e6ffcb6..d64f03f3c 100644 --- a/src/torch/ops/masked_fill_backward/masked_fill_backward.h +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.h @@ -12,9 +12,6 @@ class Operator : public MaskedFillBackward { void operator()(const Tensor grad_output, const Tensor mask, Tensor grad_input) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/mean_backward/mean_backward.cc b/src/torch/ops/mean_backward/mean_backward.cc index e6c5a3bb8..8fef69645 100644 --- a/src/torch/ops/mean_backward/mean_backward.cc +++ b/src/torch/ops/mean_backward/mean_backward.cc @@ -8,16 +8,14 @@ namespace infini::ops { template Operator::Operator(const Tensor grad_output, - const int64_t dim, - const bool keep_dim, + const int64_t dim, const bool keepdim, Tensor grad_input) - : MeanBackward{grad_output, dim, keep_dim, grad_input}, - device_index_{grad_input.device().index()} {} + : MeanBackward{grad_output, dim, keepdim, grad_input} {} template void Operator::operator()(const Tensor grad_output, const int64_t dim, - const bool keep_dim, + const bool keepdim, Tensor grad_input) const { const TorchStreamGuard guard{stream_, device_index_}; auto at_grad_output = ToAtenTensor( @@ -32,7 +30,7 @@ void Operator::operator()(const Tensor grad_output, } auto at_g = at_grad_output; - if (!keep_dim_) { + if (!keepdim_) { at_g = at_g.unsqueeze(dim_); } diff --git a/src/torch/ops/mean_backward/mean_backward.h b/src/torch/ops/mean_backward/mean_backward.h index d244d6ab3..11b80949b 100644 --- a/src/torch/ops/mean_backward/mean_backward.h +++ b/src/torch/ops/mean_backward/mean_backward.h @@ -8,14 +8,11 @@ namespace infini::ops { template class Operator : public MeanBackward { public: - Operator(const Tensor grad_output, const int64_t dim, const bool keep_dim, + Operator(const Tensor grad_output, const int64_t dim, const bool keepdim, Tensor grad_input); void operator()(const Tensor grad_output, const int64_t dim, - const bool keep_dim, Tensor grad_input) const override; - - private: - int device_index_{0}; + const bool keepdim, Tensor grad_input) const override; }; } // namespace infini::ops diff --git a/src/torch/ops/mul_backward/mul_backward.cc b/src/torch/ops/mul_backward/mul_backward.cc index 6e395fc64..a958610e0 100644 --- a/src/torch/ops/mul_backward/mul_backward.cc +++ b/src/torch/ops/mul_backward/mul_backward.cc @@ -1,47 +1,60 @@ #include "torch/ops/mul_backward/mul_backward.h" -#include +#include #include +#include #include #include "torch/instantiate.h" +#include "torch/storage_overlap.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { + template Operator::Operator(const Tensor grad_output, - const Tensor a, const Tensor b, - Tensor grad_a, Tensor grad_b) - : MulBackward{grad_output, a, b, grad_a, grad_b}, - device_index_{grad_a.device().index()} {} + const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other) + : MulBackward{grad_output, input, other, grad_input, grad_other} {} template void Operator::operator()(const Tensor grad_output, - const Tensor a, const Tensor b, - Tensor grad_a, - Tensor grad_b) const { + const Tensor input, + const Tensor other, + Tensor grad_input, + Tensor grad_other) const { const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_a = ToAtenTensor(const_cast(a.data()), a_shape_, - a_strides_, a_type_, device_index_); - auto at_b = ToAtenTensor(const_cast(b.data()), b_shape_, - b_strides_, b_type_, device_index_); - auto at_grad_a = - ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, - grad_a_type_, device_index_); - auto at_grad_b = - ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, - grad_b_type_, device_index_); - - at_grad_a.copy_((at_grad * at_b) - .sum_to_size(std::vector(grad_a_shape_.begin(), - grad_a_shape_.end()))); - at_grad_b.copy_((at_grad * at_a) - .sum_to_size(std::vector(grad_b_shape_.begin(), - grad_b_shape_.end()))); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_other = + ToAtenTensor(const_cast(other.data()), other_shape_, + other_strides_, other_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`MulBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = (at_grad * at_other).sum_to_size(grad_input_sizes); + auto grad_other_result = (at_grad * at_input).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MulBackward, 1) diff --git a/src/torch/ops/mul_backward/mul_backward.h b/src/torch/ops/mul_backward/mul_backward.h index a50e741fc..1334cd99f 100644 --- a/src/torch/ops/mul_backward/mul_backward.h +++ b/src/torch/ops/mul_backward/mul_backward.h @@ -8,14 +8,12 @@ namespace infini::ops { template class Operator : public MulBackward { public: - Operator(const Tensor grad_output, const Tensor a, const Tensor b, - Tensor grad_a, Tensor grad_b); + Operator(const Tensor grad_output, const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other); - void operator()(const Tensor grad_output, const Tensor a, const Tensor b, - Tensor grad_a, Tensor grad_b) const override; - - private: - int device_index_{0}; + void operator()(const Tensor grad_output, const Tensor input, + const Tensor other, Tensor grad_input, + Tensor grad_other) const override; }; } // namespace infini::ops diff --git a/src/torch/ops/pow_backward/pow_backward.cc b/src/torch/ops/pow_backward/pow_backward.cc index 7009421a2..a06f303fb 100644 --- a/src/torch/ops/pow_backward/pow_backward.cc +++ b/src/torch/ops/pow_backward/pow_backward.cc @@ -12,8 +12,7 @@ Operator::Operator(const Tensor grad_output, const double scalar, const bool scalar_is_base, Tensor grad_input) - : PowBackward{grad_output, input, scalar, scalar_is_base, grad_input}, - device_index_{grad_input.device().index()} {} + : PowBackward{grad_output, input, scalar, scalar_is_base, grad_input} {} template void Operator::operator()(const Tensor grad_output, @@ -34,12 +33,25 @@ void Operator::operator()(const Tensor grad_output, if (!scalar_is_base_ && scalar_ == 0.0) { at_grad_input.zero_(); - } else if (scalar_is_base_) { - auto at_y = at::pow(scalar_, at_input); - at_grad_input.copy_(at_grad_output * at_y * std::log(scalar_)); + return; + } + auto at_compute_input = at_input; + auto at_compute_grad_output = at_grad_output; + if (at_input.scalar_type() == at::kHalf || + at_input.scalar_type() == at::kBFloat16) { + at_compute_input = at_input.to(at::kFloat); + } + if (at_grad_output.scalar_type() == at::kHalf || + at_grad_output.scalar_type() == at::kBFloat16) { + at_compute_grad_output = at_grad_output.to(at::kFloat); + } + + if (scalar_is_base_) { + auto at_local_grad = at::pow(scalar_, at_compute_input) * std::log(scalar_); + at_grad_input.copy_(at_compute_grad_output * at_local_grad); } else { - at_grad_input.copy_(at_grad_output * scalar_ * - at::pow(at_input, scalar_ - 1.0)); + auto at_local_grad = scalar_ * at::pow(at_compute_input, scalar_ - 1.0); + at_grad_input.copy_(at_compute_grad_output * at_local_grad); } } diff --git a/src/torch/ops/pow_backward/pow_backward.h b/src/torch/ops/pow_backward/pow_backward.h index 493f504e5..977a3c847 100644 --- a/src/torch/ops/pow_backward/pow_backward.h +++ b/src/torch/ops/pow_backward/pow_backward.h @@ -14,9 +14,6 @@ class Operator : public PowBackward { void operator()(const Tensor grad_output, const Tensor input, const double scalar, const bool scalar_is_base, Tensor grad_input) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.cc b/src/torch/ops/repeat_interleave/repeat_interleave.cc index fca0e2a46..14209eeec 100644 --- a/src/torch/ops/repeat_interleave/repeat_interleave.cc +++ b/src/torch/ops/repeat_interleave/repeat_interleave.cc @@ -10,8 +10,7 @@ template Operator::Operator(const Tensor input, const int64_t repeats, const int64_t dim, Tensor out) - : RepeatInterleave{input, repeats, dim, out}, - device_index_{out.device().index()} {} + : RepeatInterleave{input, repeats, dim, out} {} template void Operator::operator()(const Tensor input, diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.h b/src/torch/ops/repeat_interleave/repeat_interleave.h index 22e895a09..b11812c6f 100644 --- a/src/torch/ops/repeat_interleave/repeat_interleave.h +++ b/src/torch/ops/repeat_interleave/repeat_interleave.h @@ -13,9 +13,6 @@ class Operator : public RepeatInterleave { void operator()(const Tensor input, const int64_t repeats, const int64_t dim, Tensor out) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc index 753d1fd3a..0752fa0e8 100644 --- a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc @@ -10,8 +10,7 @@ template Operator::Operator(const Tensor grad_output, const int64_t dim, Tensor grad_input) - : RepeatInterleaveBackward{grad_output, dim, grad_input}, - device_index_{grad_input.device().index()} {} + : RepeatInterleaveBackward{grad_output, dim, grad_input} {} template void Operator::operator()( diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h index a99c33922..267ec765e 100644 --- a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h @@ -13,9 +13,6 @@ class Operator void operator()(const Tensor grad_output, const int64_t dim, Tensor grad_input) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc index dadd3237e..0a4e07ca1 100644 --- a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc @@ -10,8 +10,7 @@ template Operator::Operator(const Tensor grad_output, const Tensor input, Tensor grad_input) - : RsqrtBackward{grad_output, input, grad_input}, - device_index_{grad_input.device().index()} {} + : RsqrtBackward{grad_output, input, grad_input} {} template void Operator::operator()(const Tensor grad_output, @@ -28,8 +27,13 @@ void Operator::operator()(const Tensor grad_output, ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - at_grad_input.copy_(at_grad_output * (-0.5) * - at::pow(at::rsqrt(at_input), 3)); + const auto compute_type = + at_input.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + const auto input_value = at_input.to(compute_type); + const auto upstream = at_grad_output.to(compute_type); + // Avoid cubing rsqrt before a small upstream gradient can scale it down. + const auto result = upstream * (-0.5) * at::rsqrt(input_value) / input_value; + at_grad_input.copy_(result); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RsqrtBackward, 1) diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.h b/src/torch/ops/rsqrt_backward/rsqrt_backward.h index e2fa87fbe..cae79d160 100644 --- a/src/torch/ops/rsqrt_backward/rsqrt_backward.h +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.h @@ -12,9 +12,6 @@ class Operator : public RsqrtBackward { void operator()(const Tensor grad_output, const Tensor input, Tensor grad_input) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/slice/slice.cc b/src/torch/ops/slice/slice.cc deleted file mode 100644 index a947658f5..000000000 --- a/src/torch/ops/slice/slice.cc +++ /dev/null @@ -1,43 +0,0 @@ -#include "torch/ops/slice/slice.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor input, - const std::vector starts, - const std::vector ends, - const std::vector steps, Tensor out) - : Slice{input, starts, ends, steps, out}, - device_index_{out.device().index()} {} - -template -void Operator::operator()(const Tensor input, - const std::vector starts, - const std::vector ends, - const std::vector steps, - Tensor out) const { - const TorchStreamGuard guard{stream_, device_index_}; - using namespace torch::indexing; - - auto at_input = - ToAtenTensor(const_cast(input.data()), input_shape_, - input_strides_, input_type_, device_index_); - auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, - out_type_, device_index_); - - std::vector index; - index.reserve(starts_.size()); - for (std::size_t i = 0; i < starts_.size(); ++i) { - index.emplace_back(at::indexing::Slice(starts_[i], ends_[i], steps_[i])); - } - - at_out.copy_(at_input.index(index)); -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Slice, 1) - -} // namespace infini::ops diff --git a/src/torch/ops/slice/slice.h b/src/torch/ops/slice/slice.h deleted file mode 100644 index 83358fdbd..000000000 --- a/src/torch/ops/slice/slice.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef INFINI_OPS_TORCH_SLICE_H_ -#define INFINI_OPS_TORCH_SLICE_H_ - -#include "base/slice.h" - -namespace infini::ops { - -template -class Operator : public Slice { - public: - Operator(const Tensor input, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor out); - - void operator()(const Tensor input, const std::vector starts, - const std::vector ends, - const std::vector steps, Tensor out) const override; - - private: - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif // INFINI_OPS_TORCH_SLICE_H_ diff --git a/src/torch/ops/slice_backward/slice_backward.cc b/src/torch/ops/slice_backward/slice_backward.cc deleted file mode 100644 index aa127c1c6..000000000 --- a/src/torch/ops/slice_backward/slice_backward.cc +++ /dev/null @@ -1,46 +0,0 @@ -#include "torch/ops/slice_backward/slice_backward.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor grad_output, - const std::vector starts, - const std::vector ends, - const std::vector steps, - Tensor grad_input) - : SliceBackward{grad_output, starts, ends, steps, grad_input}, - device_index_{grad_input.device().index()} {} - -template -void Operator::operator()( - const Tensor grad_output, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor grad_input) const { - const TorchStreamGuard guard{stream_, device_index_}; - using namespace torch::indexing; - - auto at_grad_output = ToAtenTensor( - const_cast(grad_output.data()), grad_output_shape_, - grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_input = - ToAtenTensor(grad_input.data(), grad_input_shape_, - grad_input_strides_, grad_input_type_, device_index_); - - std::vector index; - index.reserve(starts_.size()); - for (std::size_t i = 0; i < starts_.size(); ++i) { - index.emplace_back(at::indexing::Slice(starts_[i], ends_[i], steps_[i])); - } - - auto zeros = at::zeros(at_grad_input.sizes(), at_grad_input.options()); - zeros.index_put_(index, at_grad_output); - at_grad_input.copy_(zeros); -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(SliceBackward, 1) - -} // namespace infini::ops diff --git a/src/torch/ops/slice_backward/slice_backward.h b/src/torch/ops/slice_backward/slice_backward.h deleted file mode 100644 index 09b2bdf71..000000000 --- a/src/torch/ops/slice_backward/slice_backward.h +++ /dev/null @@ -1,26 +0,0 @@ -#ifndef INFINI_OPS_TORCH_SLICE_BACKWARD_H_ -#define INFINI_OPS_TORCH_SLICE_BACKWARD_H_ - -#include "base/slice_backward.h" - -namespace infini::ops { - -template -class Operator : public SliceBackward { - public: - Operator(const Tensor grad_output, const std::vector starts, - const std::vector ends, const std::vector steps, - Tensor grad_input); - - void operator()(const Tensor grad_output, const std::vector starts, - const std::vector ends, - const std::vector steps, - Tensor grad_input) const override; - - private: - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif diff --git a/src/torch/ops/split/split.cc b/src/torch/ops/split/split.cc index edff2e878..a8d350cc6 100644 --- a/src/torch/ops/split/split.cc +++ b/src/torch/ops/split/split.cc @@ -1,38 +1,46 @@ -#include "torch/ops/split/split.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor input, const int64_t split_size, - const int64_t dim, std::vector outs) - : Split{input, split_size, dim, std::move(outs)}, - device_index_{Split::device_index_} {} - -template -void Operator::operator()(const Tensor input, - const int64_t split_size, - const int64_t dim, - std::vector outs) const { - const TorchStreamGuard guard{stream_, device_index_}; - auto at_input = - ToAtenTensor(const_cast(input.data()), input_shape_, - input_strides_, input_type_, device_index_); - - const auto parts = at::split(at_input, split_size_, dim_); - assert(parts.size() == outs_.size() && "`Split` part count mismatch"); - - for (std::size_t i = 0; i < parts.size(); ++i) { - auto at_out_i = - ToAtenTensor(outs[i].data(), out_shapes_[i], out_strides_[i], - out_types_[i], device_index_); - at_out_i.copy_(parts[i]); - } -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Split, 1) - -} // namespace infini::ops +#include "torch/ops/split/split.h" + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, const int64_t split_size, + const int64_t dim, std::vector outs) + : Split{input, split_size, dim, std::move(outs)} {} + +template +void Operator::operator()(const Tensor input, + const int64_t split_size, + const int64_t dim, + std::vector outs) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + + // Preserve every source slice before any output can overwrite it. + const auto parts = at::split(at_input.clone(), split_size_, dim_); + std::vector outputs; + for (std::size_t i = 0; i < outs.size(); ++i) { + outputs.push_back(ToAtenTensor(outs[i].data(), out_shapes_[i], + out_strides_[i], out_types_[i], + device_index_)); + for (std::size_t j = 0; j < i; ++j) { + assert(!StorageSpansOverlap(outputs[i], outputs[j]) && + "Output storage must not overlap."); + } + } + assert(parts.size() == outs_.size() && "`Split` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + outputs[i].copy_(parts[i]); + } +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Split, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/split/split.h b/src/torch/ops/split/split.h index f1e3cb744..442664781 100644 --- a/src/torch/ops/split/split.h +++ b/src/torch/ops/split/split.h @@ -13,9 +13,6 @@ class Operator : public Split { void operator()(const Tensor input, const int64_t split_size, const int64_t dim, std::vector outs) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/stack_backward/stack_backward.cc b/src/torch/ops/stack_backward/stack_backward.cc index 06f75527e..8c66a5ea6 100644 --- a/src/torch/ops/stack_backward/stack_backward.cc +++ b/src/torch/ops/stack_backward/stack_backward.cc @@ -1,38 +1,47 @@ -#include "torch/ops/stack_backward/stack_backward.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor grad_output, - const int64_t dim, - std::vector outs) - : StackBackward{grad_output, dim, std::move(outs)}, - device_index_{StackBackward::device_index_} {} - -template -void Operator::operator()( - const Tensor grad_output, const int64_t dim, - std::vector outs) const { - const TorchStreamGuard guard{stream_, device_index_}; - auto at_grad_output = ToAtenTensor( - const_cast(grad_output.data()), grad_output_shape_, - grad_output_strides_, grad_output_type_, device_index_); - - const auto parts = at::unbind(at_grad_output, dim_); - assert(parts.size() == outs_.size() && "`StackBackward` part count mismatch"); - - for (std::size_t i = 0; i < parts.size(); ++i) { - auto at_out_i = - ToAtenTensor(outs[i].data(), out_shapes_[i], out_strides_[i], - out_types_[i], device_index_); - at_out_i.copy_(parts[i]); - } -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(StackBackward, 1) - -} // namespace infini::ops +#include "torch/ops/stack_backward/stack_backward.h" + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, + std::vector grad_inputs) + : StackBackward{grad_output, dim, std::move(grad_inputs)} {} + +template +void Operator::operator()( + const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + + // Preserve every source slice before any output can overwrite it. + const auto parts = at::unbind(at_grad_output.clone(), dim_); + std::vector outputs; + for (std::size_t i = 0; i < grad_inputs.size(); ++i) { + outputs.push_back(ToAtenTensor( + grad_inputs[i].data(), grad_input_shapes_[i], grad_input_strides_[i], + grad_input_types_[i], device_index_)); + for (std::size_t j = 0; j < i; ++j) { + assert(!StorageSpansOverlap(outputs[i], outputs[j]) && + "Output storage must not overlap."); + } + } + assert(parts.size() == grad_inputs_.size() && + "`StackBackward` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + outputs[i].copy_(parts[i]); + } +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(StackBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/stack_backward/stack_backward.h b/src/torch/ops/stack_backward/stack_backward.h index b5635baeb..e51b2a393 100644 --- a/src/torch/ops/stack_backward/stack_backward.h +++ b/src/torch/ops/stack_backward/stack_backward.h @@ -9,13 +9,10 @@ template class Operator : public StackBackward { public: Operator(const Tensor grad_output, const int64_t dim, - std::vector outs); + std::vector grad_inputs); void operator()(const Tensor grad_output, const int64_t dim, - std::vector outs) const override; - - private: - int device_index_{0}; + std::vector grad_inputs) const override; }; } // namespace infini::ops diff --git a/src/torch/ops/sub_backward/sub_backward.cc b/src/torch/ops/sub_backward/sub_backward.cc index 8c27e50b0..a539363de 100644 --- a/src/torch/ops/sub_backward/sub_backward.cc +++ b/src/torch/ops/sub_backward/sub_backward.cc @@ -5,35 +5,48 @@ #include #include "torch/instantiate.h" +#include "torch/storage_overlap.h" #include "torch/stream_guard.h" #include "torch/tensor_.h" namespace infini::ops { + template Operator::Operator(const Tensor grad_output, - Tensor grad_a, Tensor grad_b) - : SubBackward{grad_output, grad_a, grad_b}, - device_index_{grad_a.device().index()} {} + const double alpha, Tensor grad_input, + Tensor grad_other) + : SubBackward{grad_output, alpha, grad_input, grad_other} {} template void Operator::operator()(const Tensor grad_output, - Tensor grad_a, - Tensor grad_b) const { + const double alpha, + Tensor grad_input, + Tensor grad_other) const { const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), grad_output_shape_, grad_output_strides_, grad_output_type_, device_index_); - auto at_grad_a = - ToAtenTensor(grad_a.data(), grad_a_shape_, grad_a_strides_, - grad_a_type_, device_index_); - auto at_grad_b = - ToAtenTensor(grad_b.data(), grad_b_shape_, grad_b_strides_, - grad_b_type_, device_index_); - - at_grad_a.copy_((at_grad).sum_to_size( - std::vector(grad_a_shape_.begin(), grad_a_shape_.end()))); - at_grad_b.copy_(-(at_grad).sum_to_size( - std::vector(grad_b_shape_.begin(), grad_b_shape_.end()))); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`SubBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = at_grad.sum_to_size(grad_input_sizes).clone(); + auto grad_other_result = (at_grad * (-alpha)).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(SubBackward, 1) diff --git a/src/torch/ops/sub_backward/sub_backward.h b/src/torch/ops/sub_backward/sub_backward.h index c1a571e7d..46fbe31d8 100644 --- a/src/torch/ops/sub_backward/sub_backward.h +++ b/src/torch/ops/sub_backward/sub_backward.h @@ -8,13 +8,11 @@ namespace infini::ops { template class Operator : public SubBackward { public: - Operator(const Tensor grad_output, Tensor grad_a, Tensor grad_b); + Operator(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other); - void operator()(const Tensor grad_output, Tensor grad_a, - Tensor grad_b) const override; - - private: - int device_index_{0}; + void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const override; }; } // namespace infini::ops diff --git a/src/torch/ops/transpose/transpose.cc b/src/torch/ops/transpose/transpose.cc deleted file mode 100644 index 0b99e54c1..000000000 --- a/src/torch/ops/transpose/transpose.cc +++ /dev/null @@ -1,31 +0,0 @@ -#include "torch/ops/transpose/transpose.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops { - -template -Operator::Operator(const Tensor input, const int64_t dim0, - const int64_t dim1, Tensor out) - : Transpose{input, dim0, dim1, out}, device_index_{out.device().index()} {} - -template -void Operator::operator()(const Tensor input, - const int64_t dim0, - const int64_t dim1, - Tensor out) const { - const TorchStreamGuard guard{stream_, device_index_}; - auto at_input = - ToAtenTensor(const_cast(input.data()), input_shape_, - input_strides_, input_type_, device_index_); - auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, - out_type_, device_index_); - - at_out.copy_(at_input.transpose(dim0_, dim1_)); -} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Transpose, 1) - -} // namespace infini::ops diff --git a/src/torch/ops/transpose/transpose.h b/src/torch/ops/transpose/transpose.h deleted file mode 100644 index 50541ab12..000000000 --- a/src/torch/ops/transpose/transpose.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef INFINI_OPS_TORCH_TRANSPOSE_H_ -#define INFINI_OPS_TORCH_TRANSPOSE_H_ - -#include "base/transpose.h" - -namespace infini::ops { - -template -class Operator : public Transpose { - public: - Operator(const Tensor input, const int64_t dim0, const int64_t dim1, - Tensor out); - - void operator()(const Tensor input, const int64_t dim0, const int64_t dim1, - Tensor out) const override; - - private: - int device_index_{0}; -}; - -} // namespace infini::ops - -#endif diff --git a/src/torch/ops/transpose_backward/transpose_backward.cc b/src/torch/ops/transpose_backward/transpose_backward.cc index 6084a53b0..03bc408e4 100644 --- a/src/torch/ops/transpose_backward/transpose_backward.cc +++ b/src/torch/ops/transpose_backward/transpose_backward.cc @@ -11,8 +11,7 @@ Operator::Operator(const Tensor grad_output, const int64_t dim0, const int64_t dim1, Tensor grad_input) - : TransposeBackward{grad_output, dim0, dim1, grad_input}, - device_index_{grad_input.device().index()} {} + : TransposeBackward{grad_output, dim0, dim1, grad_input} {} template void Operator::operator()(const Tensor grad_output, @@ -28,7 +27,7 @@ void Operator::operator()(const Tensor grad_output, ToAtenTensor(grad_input.data(), grad_input_shape_, grad_input_strides_, grad_input_type_, device_index_); - at_grad_input.copy_(at_grad_output.transpose(dim0_, dim1_)); + at_grad_input.copy_(at_grad_output.transpose(dim0_, dim1_).clone()); } INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(TransposeBackward, 1) diff --git a/src/torch/ops/transpose_backward/transpose_backward.h b/src/torch/ops/transpose_backward/transpose_backward.h index b0aee44fe..b17850c9f 100644 --- a/src/torch/ops/transpose_backward/transpose_backward.h +++ b/src/torch/ops/transpose_backward/transpose_backward.h @@ -13,9 +13,6 @@ class Operator : public TransposeBackward { void operator()(const Tensor grad_output, const int64_t dim0, const int64_t dim1, Tensor grad_input) const override; - - private: - int device_index_{0}; }; } // namespace infini::ops diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc index b187c8c0b..252867095 100644 --- a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc @@ -64,31 +64,33 @@ void Operator::operator()( assert((at_grad_output.numel() == 1 || at_grad_output.numel() == rows) && "grad_output must be scalar or one value per row"); - auto softmax_2d = at_softmax.reshape({rows, vocab_size_local}) - .to(at_grad_input.scalar_type()); - auto target_mask_1d = - at_target_mask.reshape({rows}).to(at_grad_input.scalar_type()); + const auto compute_type = + at_grad_input.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + auto softmax_2d = + at_softmax.reshape({rows, vocab_size_local}).to(compute_type); + auto target_mask_1d = at_target_mask.reshape({rows}).to(compute_type); auto target_1d = at_masked_target.reshape({rows}).to(at::kLong); - auto valid_1d = - at_valid_mask.reshape({vocab_size_local}).to(at_grad_input.scalar_type()); + auto valid_1d = at_valid_mask.reshape({vocab_size_local}).to(compute_type); - auto dloss = at_grad_output.reshape({-1}).to(at_grad_input.scalar_type()); + auto dloss = at_grad_output.reshape({-1}).to(compute_type); if (dloss.numel() == 1) { dloss = dloss.expand({rows}); } - auto gradient = softmax_2d * dloss.reshape({rows, 1}); const auto valid_rows = 1.0 - target_mask_1d; - const auto row_scale = dloss * (1.0 - label_smoothing) * valid_rows; - auto target_one_hot = at::zeros_like(gradient); + auto target_one_hot = at::zeros_like(softmax_2d); target_one_hot.scatter_(1, target_1d.reshape({rows, 1}), 1.0); - gradient.sub_(target_one_hot * row_scale.reshape({rows, 1})); + const auto local_target = target_one_hot * valid_rows.reshape({rows, 1}); + // Preserve small smoothing corrections before scaling by the upstream + // gradient. + auto gradient = softmax_2d - local_target; + gradient.add_(local_target * label_smoothing); const auto smoothing_term = label_smoothing / static_cast(vocab_size_original); - gradient.sub_(dloss.reshape({rows, 1}) * smoothing_term * - valid_1d.reshape({1, vocab_size_local})); + gradient.sub_(smoothing_term * valid_1d.reshape({1, vocab_size_local})); + gradient.mul_(dloss.reshape({rows, 1})); gradient.mul_(valid_1d.reshape({1, vocab_size_local})); at_grad_input.copy_(gradient.reshape(at_grad_input.sizes())); diff --git a/src/torch/storage_overlap.h b/src/torch/storage_overlap.h new file mode 100644 index 000000000..064ba5b1d --- /dev/null +++ b/src/torch/storage_overlap.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ +#define INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ + +#include + +#include + +namespace infini::ops { + +// External pointers have independent ATen Storage identities. Include stride +// gaps in the byte span, conservatively treating interleaved views as overlap. +inline bool StorageSpansOverlap(const at::Tensor& a, const at::Tensor& b) { + if (a.numel() == 0 || b.numel() == 0) return false; + const auto ap = reinterpret_cast(a.data_ptr()); + const auto bp = reinterpret_cast(b.data_ptr()); + const auto an = a.storage().nbytes(); + const auto bn = b.storage().nbytes(); + return an && bn && (ap <= bp ? bp - ap < an : ap - bp < bn); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ diff --git a/tests/test_accumulate_grad.py b/tests/test_accumulate_grad.py index 441d3a338..72138b1c9 100644 --- a/tests/test_accumulate_grad.py +++ b/tests/test_accumulate_grad.py @@ -1,87 +1,163 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, get_stream, randn_strided - - -# Format: (shape, rate, gradient_strides, tensor_strides) -_TEST_CASES = ( - pytest.param((17,), 0.5, None, None, id="one-dimensional"), - pytest.param((2, 3), -0.25, None, None, id="two-dimensional"), - pytest.param( - (2, 3, 4), - 1.5, - (30, 7, 1), - (35, 8, 1), - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "shape, rate, gradient_strides, tensor_strides", - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-6, 1e-6), - (torch.float16, 1e-3, 1e-3), - (torch.bfloat16, 1e-2, 1e-2), - ), -) -def test_accumulate_grad( - shape, - rate, - gradient_strides, - tensor_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - gradient = randn_strided( - shape, - gradient_strides, - dtype=dtype, - device=device, - ) - tensor = randn_strided( - shape, - tensor_strides, - dtype=dtype, - device=device, - ) - - return Payload( - lambda *args: _accumulate_grad( - *args, - implementation_index=implementation_index, - ), - _torch_accumulate_grad, - (gradient, rate, tensor), - {}, - rtol=rtol, - atol=atol, - ) - - -def _accumulate_grad(gradient, rate, tensor, *, implementation_index): - infini.ops.accumulate_grad( - gradient, - rate, - tensor, - stream=get_stream(gradient.device), - implementation_index=implementation_index, - ) - - return tensor - - -def _torch_accumulate_grad(gradient, rate, tensor): - tensor.add_(gradient * rate) - - return tensor +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +# Format: (shape, rate, gradient_strides, tensor_strides) +_TEST_CASES = ( + pytest.param((17,), 0.5, None, None, id="one-dimensional"), + pytest.param((2, 3), -0.25, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + 1.5, + (30, 7, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, rate, gradient_strides, tensor_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_accumulate_grad( + shape, + rate, + gradient_strides, + tensor_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + gradient = randn_strided( + shape, + gradient_strides, + dtype=dtype, + device=device, + ) + tensor = randn_strided( + shape, + tensor_strides, + dtype=dtype, + device=device, + ) + + return Payload( + lambda *args: _accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_accumulate_grad, + (gradient, rate, tensor), + {}, + rtol=rtol, + atol=atol, + ) + + +def _accumulate_grad(gradient, rate, tensor, *, implementation_index): + infini.ops.accumulate_grad( + gradient, + rate, + tensor, + stream=get_stream(gradient.device), + implementation_index=implementation_index, + ) + + return tensor + + +def _torch_accumulate_grad(gradient, rate, tensor): + tensor.add_(gradient * rate) + + return tensor + + +@pytest.mark.parametrize("rate", (0.0, 0.5, -1.0)) +def test_accumulate_grad_repeated(rate, device, implementation_index): + tensor = torch.ones(4, device=device) + gradient = torch.full_like(tensor, 2) + expected = tensor.clone() + + for _ in range(5): + expected.add_(gradient, alpha=rate) + _accumulate_grad( + gradient, rate, tensor, implementation_index=implementation_index + ) + torch.testing.assert_close(tensor, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "dtype", (torch.float16, torch.bfloat16, torch.float32, torch.float64) +) +@pytest.mark.parametrize("rate", (0.0, 0.5)) +@pytest.mark.parametrize("stride", (1, 2)) +@pytest.mark.parametrize("offset", (-1, 0, 1, 80)) +def test_accumulate_grad_external_overlap( + dtype, rate, stride, offset, device, implementation_index +): + backing = torch.arange(256, dtype=dtype, device=device) + tensor = backing.as_strided((32,), (stride,), 8) + gradient = backing.as_strided((32,), (stride,), 8 + offset) + expected = tensor.clone().add_(gradient.clone(), alpha=rate) + expected_backing = backing.clone() + expected_backing.as_strided((32,), (stride,), 8).copy_(expected) + _accumulate_grad(gradient, rate, tensor, implementation_index=implementation_index) + torch.testing.assert_close(backing, expected_backing, rtol=0, atol=0) + + +def test_accumulate_grad_rejects_integer(device, implementation_index): + setup = f"x = torch.ones(4, dtype=torch.int32, device={str(device)!r})\n" + probe = setup + "infini.ops.AccumulateGrad(x, 0.5, x)\n" + source = ( + setup + + f"infini.ops.accumulate_grad(x, 0.5, x, implementation_index={implementation_index}, stream=get_stream(x.device))\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "requires floating-point tensors"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_adam_accumulate_grad.py b/tests/test_adam_accumulate_grad.py index 2b794bc3a..c8d33d194 100644 --- a/tests/test_adam_accumulate_grad.py +++ b/tests/test_adam_accumulate_grad.py @@ -1,150 +1,310 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, get_stream, randn_strided - - -# Format: (shape, learning_rate, beta1, beta2, eps, step, strides) -_TEST_CASES = ( - pytest.param( - (17,), - 1e-2, - 0.9, - 0.999, - 1e-8, - 1, - None, - id="one-dimensional-first-step", - ), - pytest.param( - (2, 3), - 5e-3, - 0.8, - 0.99, - 1e-6, - 3, - None, - id="two-dimensional-third-step", - ), - pytest.param( - (2, 3, 4), - 1e-2, - 0.9, - 0.999, - 1e-8, - 2, - (30, 7, 1), - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "shape, learning_rate, beta1, beta2, eps, step, strides", - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-5, 1e-6), - (torch.float16, 2e-3, 2e-3), - (torch.bfloat16, 2e-2, 2e-2), - ), -) -def test_adam_accumulate_grad( - shape, - learning_rate, - beta1, - beta2, - eps, - step, - strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad = randn_strided(shape, strides, dtype=dtype, device=device) - param = randn_strided(shape, strides, dtype=dtype, device=device) - m = randn_strided(shape, strides, dtype=dtype, device=device) - v = randn_strided(shape, strides, dtype=dtype, device=device).abs_().add_(0.1) - - return Payload( - lambda *args: _adam_accumulate_grad( - *args, - implementation_index=implementation_index, - ), - _torch_adam_accumulate_grad, - (grad, learning_rate, beta1, beta2, eps, step, param, m, v), - {}, - rtol=rtol, - atol=atol, - ) - - -def _adam_accumulate_grad( - grad, - learning_rate, - beta1, - beta2, - eps, - step, - param, - m, - v, - *, - implementation_index, -): - infini.ops.adam_accumulate_grad( - grad, - learning_rate, - beta1, - beta2, - eps, - step, - param, - m, - v, - stream=get_stream(grad.device), - implementation_index=implementation_index, - ) - - return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) - - -def _torch_adam_accumulate_grad( - grad, - learning_rate, - beta1, - beta2, - eps, - step, - param, - m, - v, -): - reference_param = torch.nn.Parameter( - param.detach().to(device="cpu", dtype=torch.float64) - ) - optimizer = torch.optim.Adam( - [reference_param], - lr=learning_rate, - betas=(beta1, beta2), - eps=eps, - foreach=False, - ) - optimizer.state[reference_param] = { - "step": torch.tensor(float(step - 1)), - "exp_avg": m.detach().to(device="cpu", dtype=torch.float64).clone(), - "exp_avg_sq": v.detach().to(device="cpu", dtype=torch.float64).clone(), - } - reference_param.grad = grad.detach().to(device="cpu", dtype=torch.float64).clone() - optimizer.step() - param.copy_(reference_param.detach()) - m.copy_(optimizer.state[reference_param]["exp_avg"]) - v.copy_(optimizer.state[reference_param]["exp_avg_sq"]) - - return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) +import json +import os +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +# Format: (shape, learning_rate, beta1, beta2, eps, step, strides) +_TEST_CASES = ( + pytest.param( + (17,), + 1e-2, + 0.9, + 0.999, + 1e-8, + 1, + None, + id="one-dimensional-first-step", + ), + pytest.param( + (2, 3), + 5e-3, + 0.8, + 0.99, + 1e-6, + 3, + None, + id="two-dimensional-third-step", + ), + pytest.param( + (2, 3, 4), + 1e-2, + 0.9, + 0.999, + 1e-8, + 2, + (30, 7, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, learning_rate, beta1, beta2, eps, step, strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 2e-3, 2e-3), + (torch.bfloat16, 2e-2, 2e-2), + ), +) +def test_adam_accumulate_grad( + shape, + learning_rate, + beta1, + beta2, + eps, + step, + strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad = randn_strided(shape, strides, dtype=dtype, device=device) + param = randn_strided(shape, strides, dtype=dtype, device=device) + m = randn_strided(shape, strides, dtype=dtype, device=device) + v = randn_strided(shape, strides, dtype=dtype, device=device).abs_().add_(0.1) + + return Payload( + lambda *args: _adam_accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_adam_accumulate_grad, + (grad, learning_rate, beta1, beta2, eps, step, param, m, v), + {}, + rtol=rtol, + atol=atol, + ) + + +def _adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, + *, + implementation_index, +): + infini.ops.adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, + stream=get_stream(grad.device), + implementation_index=implementation_index, + ) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) + + +def _torch_adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, +): + reference_param = torch.nn.Parameter( + param.detach().to(device="cpu", dtype=torch.float64) + ) + optimizer = torch.optim.Adam( + [reference_param], + lr=learning_rate, + betas=(beta1, beta2), + eps=eps, + foreach=False, + ) + optimizer.state[reference_param] = { + "step": torch.tensor(float(step - 1)), + "exp_avg": m.detach().to(device="cpu", dtype=torch.float64).clone(), + "exp_avg_sq": v.detach().to(device="cpu", dtype=torch.float64).clone(), + } + reference_param.grad = grad.detach().to(device="cpu", dtype=torch.float64).clone() + optimizer.step() + param.copy_(reference_param.detach()) + m.copy_(optimizer.state[reference_param]["exp_avg"]) + v.copy_(optimizer.state[reference_param]["exp_avg_sq"]) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) + + +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +@pytest.mark.parametrize("gradient_scale", (0.0, 1e-7, 1.0)) +def test_adam_accumulate_grad_mixed_precision_multistep( + dtype, + gradient_scale, + device, + implementation_index, +): + param = torch.linspace(-1, 1, 12, device=device).reshape(3, 4).to(dtype) + m = torch.zeros((3, 4), dtype=torch.float32, device=device) + v = torch.zeros_like(m) + expected_param, expected_m, expected_v = param.clone(), m.clone(), v.clone() + + for step in range(1, 9): + grad = torch.full_like(param, gradient_scale * (-1 if step % 2 else 1)) + _torch_adam_accumulate_grad( + grad, 0.01, 0.9, 0.999, 1e-8, step, expected_param, expected_m, expected_v + ) + _adam_accumulate_grad( + grad, + 0.01, + 0.9, + 0.999, + 1e-8, + step, + param, + m, + v, + implementation_index=implementation_index, + ) + torch.testing.assert_close(param, expected_param, rtol=0, atol=0) + torch.testing.assert_close(m, expected_m, rtol=1e-5, atol=1e-12) + torch.testing.assert_close(v, expected_v, rtol=1e-5, atol=1e-20) + + +@pytest.mark.parametrize( + "dtype,gradient,moment,variance", + ( + (torch.float32, 1e20, 0.0, 0.0), + (torch.float32, 0.0, 1e19, 1e37), + (torch.float64, 1e155, 0.0, 0.0), + ), +) +def test_adam_accumulate_grad_finite_intermediates( + dtype, gradient, moment, variance, device, implementation_index +): + grad = torch.full((3,), gradient, dtype=dtype, device=device) + param = torch.zeros_like(grad) + m, v = torch.full_like(grad, moment), torch.full_like(grad, variance) + expected = [value.clone() for value in (param, m, v)] + _torch_adam_accumulate_grad(grad, 0.001, 0.9, 0.999, 1e-8, 1, *expected) + _adam_accumulate_grad( + grad, + 0.001, + 0.9, + 0.999, + 1e-8, + 1, + param, + m, + v, + implementation_index=implementation_index, + ) + + for actual, reference in zip((param, m, v), expected): + assert torch.isfinite(actual).all().item() + rtol = 1e-12 if dtype == torch.float64 else 1e-5 + torch.testing.assert_close(actual, reference, rtol=rtol, atol=0) + + +@pytest.mark.parametrize("dtype", (torch.float32, torch.float64)) +@pytest.mark.parametrize("stride", (1, 2)) +@pytest.mark.parametrize( + "alias,offset", + [("none", 0)] + + [(name, shift) for name in ("param", "m", "v") for shift in (-1, 0, 1)], +) +def test_adam_accumulate_grad_input_output_overlap( + dtype, stride, alias, offset, device, implementation_index +): + backing = torch.linspace(0.25, 2.0, 128, dtype=dtype, device=device) + positions = {"param": 8, "m": 40, "v": 72} + param, m, v = [ + backing.as_strided((6,), (stride,), pos) for pos in positions.values() + ] + grad = torch.full_like(param, 0.5) + + if alias != "none": + grad = backing.as_strided((6,), (stride,), positions[alias] + offset) + + expected = [value.clone() for value in (param, m, v)] + _torch_adam_accumulate_grad(grad.clone(), 0.001, 0.9, 0.999, 1e-8, 1, *expected) + expected_backing = backing.clone() + + for pos, result in zip(positions.values(), expected): + expected_backing.as_strided((6,), (stride,), pos).copy_(result) + + _adam_accumulate_grad( + grad, + 0.001, + 0.9, + 0.999, + 1e-8, + 1, + param, + m, + v, + implementation_index=implementation_index, + ) + torch.testing.assert_close(backing, expected_backing, rtol=1e-5, atol=1e-8) + + +@pytest.mark.skipif( + os.environ.get("INFINI_OPS_TEST_ASSERTIONS") != "1", + reason="set INFINI_OPS_TEST_ASSERTIONS=1 with an assertions-enabled build", +) +@pytest.mark.parametrize("pair", (("param", "m"), ("param", "v"), ("m", "v"))) +@pytest.mark.parametrize("layout", ("full", "partial", "strided", "interleaved")) +@pytest.mark.parametrize("reverse", (False, True)) +def test_adam_accumulate_grad_rejects_output_overlap( + pair, layout, reverse, device, implementation_index +): + stride = 2 if layout in ("strided", "interleaved") else 1 + offsets = (8, {"full": 8, "partial": 10, "strided": 10, "interleaved": 9}[layout]) + + if reverse: + offsets = offsets[::-1] + + source = f""" +import sys +sys.path[:] = {json.dumps(sys.path)} +import infini.ops +import torch +from tests.utils import get_stream +device = torch.device({str(device)!r}) +backing = torch.ones(128, device=device) +outputs = {{name: torch.ones(6, device=device) for name in ('param', 'm', 'v')}} +outputs[{pair[0]!r}] = backing.as_strided((6,), ({stride},), {offsets[0]}) +outputs[{pair[1]!r}] = backing.as_strided((6,), ({stride},), {offsets[1]}) +grad = torch.full((6,), 0.5, device=device) +infini.ops.adam_accumulate_grad(grad, 0.001, 0.9, 0.999, 1e-8, 1, + outputs['param'], outputs['m'], outputs['v'], + implementation_index={implementation_index}, stream=get_stream(device)) +""" + result = subprocess.run( + [sys.executable, "-c", source], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + assert result.returncode != 0, "Overlapping Adam outputs were accepted" + assert "output storage must not overlap" in result.stdout + result.stderr diff --git a/tests/test_add_backward.py b/tests/test_add_backward.py index e3d3ed78a..c2789a373 100644 --- a/tests/test_add_backward.py +++ b/tests/test_add_backward.py @@ -1,87 +1,406 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (shape, grad_output_strides, grad_a_strides, grad_b_strides) -_TEST_CASES = ( - pytest.param((17,), None, None, None, id="one-dimensional"), - pytest.param((2, 3), None, None, None, id="two-dimensional"), - pytest.param((2, 3, 4), None, None, None, id="three-dimensional"), - pytest.param( - (2, 3, 4), - (30, 7, 1), - (35, 8, 1), - (36, 9, 1), - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "shape, grad_output_strides, grad_a_strides, grad_b_strides", - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_add_backward( - shape, - grad_output_strides, - grad_a_strides, - grad_b_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output = randn_strided( - shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) - grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) - grad_a.fill_(17) - grad_b.fill_(17) - - return Payload( - lambda *args: _add_backward( - *args, - implementation_index=implementation_index, - ), - _torch_add_backward, - (grad_output, grad_a, grad_b), - {}, - rtol=rtol, - atol=atol, - ) - - -def _add_backward(grad_output, grad_a, grad_b, *, implementation_index): - infini.ops.add_backward( - grad_output, - grad_a, - grad_b, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) - - -def _torch_add_backward(grad_output, grad_a, grad_b): - grad_a.copy_(grad_output) - grad_b.copy_(grad_output) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param((17,), (17,), (17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), (2, 3), (2, 3), None, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), + pytest.param( + (2, 3), + (2, 3), + (2, 3), + (1, 2), + (1, 2), + (1, 2), + id="transposed-layout", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (4,), + None, + None, + None, + id="broadcast-b-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (4,), + (2, 3, 4), + None, + None, + None, + id="broadcast-a-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + id="broadcast-both-inputs", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + (30, 7, 1), + (11, 7, 2), + (15, 3, 1), + id="broadcast-strided-tensors", + ), + pytest.param( + (2, 3), + (), + (2, 3), + None, + None, + None, + id="broadcast-scalar-a", + ), + pytest.param( + (2, 3), + (2, 3), + (), + None, + None, + None, + id="broadcast-scalar-b", + ), + pytest.param((), (), (), None, None, None, id="scalar-output"), + pytest.param( + (1, 3, 1), + (3, 1), + (1, 3, 1), + None, + None, + None, + id="singleton-dimensions", + ), + pytest.param( + (0, 3), + (0, 3), + (0, 3), + None, + None, + None, + id="empty-output", + ), + pytest.param( + (0, 3), + (1, 3), + (0, 3), + None, + None, + None, + id="empty-output-nonempty-grad-a", + ), + pytest.param( + (0, 3), + (0, 3), + (1, 3), + None, + None, + None, + id="empty-output-nonempty-grad-b", + ), +) + +_ALPHAS = ( + pytest.param(1.0, id="alpha-one"), + pytest.param(0.0, id="alpha-zero"), + pytest.param(2.0, id="alpha-two"), + pytest.param(-1.0, id="alpha-negative"), + pytest.param(0.5, id="alpha-fractional"), +) + +_DTYPES = ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "grad_output_shape, grad_a_shape, grad_b_shape, " + "grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_add_backward( + grad_output_shape, + grad_a_shape, + grad_b_shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + alpha, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided( + grad_a_shape, + grad_a_strides, + dtype=dtype, + device=device, + ) + grad_b = empty_strided( + grad_b_shape, + grad_b_strides, + dtype=dtype, + device=device, + ) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _add_backward( + *args, + implementation_index=implementation_index, + ), + _torch_add_backward, + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pack_result(grad_output, grad_a, grad_b): + return torch.cat( + (grad_a.reshape(-1), grad_b.reshape(-1), grad_output.reshape(-1)), + ) + + +def _add_backward(grad_output, alpha, grad_a, grad_b, *, implementation_index): + infini.ops.add_backward( + grad_output, + alpha, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return _pack_result(grad_output, grad_a, grad_b) + + +def _torch_add_backward(grad_output, alpha, grad_a, grad_b): + with torch.enable_grad(): + a = torch.zeros_like(grad_a, requires_grad=True) + b = torch.zeros_like(grad_b, requires_grad=True) + out = torch.add(a, b, alpha=alpha) + expected_a, expected_b = torch.autograd.grad( + out, + (a, b), + grad_outputs=grad_output, + ) + + expected_a = expected_a.clone() + expected_b = expected_b.clone() + with torch.no_grad(): + grad_a.copy_(expected_a) + grad_b.copy_(expected_b) + return _pack_result(grad_output, grad_a, grad_b) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("target", ("grad_a", "grad_b")) +@pytest.mark.parametrize("layout", ("exact", "partial", "strided", "broadcast")) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_add_backward_alias( + target, + layout, + alpha, + implementation_index, + dtype, + device, +): + source = torch.arange(-16, 16, dtype=dtype, device=device) + + return Payload( + lambda values: _run_alias( + values, + alpha, + target, + layout, + implementation_index=implementation_index, + ), + lambda values: _run_alias(values, alpha, target, layout), + (source,), + {}, + rtol=0, + atol=0, + ) + + +def _run_alias(source, alpha, target, layout, *, implementation_index=None): + storage = source.clone() + grad_output = storage[:6].view(2, 3) + if layout == "exact": + shared_output = grad_output + elif layout == "partial": + shared_output = storage[1:7].view(2, 3) + elif layout == "strided": + shared_output = storage[:6].view(3, 2).transpose(0, 1) + elif layout == "broadcast": + shared_output = storage[:3].view(1, 3) + else: + raise ValueError(f"Unknown alias layout: {layout}") + + independent_output = torch.full_like(grad_output, 17) + if target == "grad_a": + grad_a, grad_b = shared_output, independent_output + else: + grad_a, grad_b = independent_output, shared_output + + if implementation_index is None: + _torch_add_backward(grad_output, alpha, grad_a, grad_b) + else: + _add_backward( + grad_output, + alpha, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1), storage)) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +@pytest.mark.parametrize( + "alpha, grad_value, grad_a_shape, grad_b_shape, expected_a, expected_b", + ( + pytest.param(2.0, 1.0, (2, 3), (2, 3), 1.0, 2.0, id="review-example"), + pytest.param(2.0, 1.0, (2, 3), (3,), 1.0, 4.0, id="scaled-reduction"), + pytest.param(0.0, 1.0, (2, 3), (3,), 1.0, 0.0, id="zero-alpha"), + pytest.param(-1.0, 1.0, (2, 3), (3,), 1.0, -2.0, id="negative-alpha"), + pytest.param(0.5, 1.0, (2, 3), (3,), 1.0, 1.0, id="fractional-alpha"), + pytest.param(2.0, 1.0, (1, 3), (2, 1), 2.0, 6.0, id="both-reduced"), + pytest.param( + 0.5, 40000.0, (2, 3), (2, 3), 40000.0, 20000.0, id="large-finite-gradient" + ), + pytest.param( + 0.0, 40000.0, (2, 3), (3,), 40000.0, 0.0, id="zero-alpha-before-reduction" + ), + ), +) +def test_add_backward_known_values( + alpha, + grad_value, + grad_a_shape, + grad_b_shape, + expected_a, + expected_b, + implementation_index, + dtype, + device, +): + grad_output = torch.full((2, 3), grad_value, dtype=dtype, device=device) + grad_a = torch.full(grad_a_shape, 17.0, dtype=dtype, device=device) + grad_b = torch.full(grad_b_shape, 17.0, dtype=dtype, device=device) + expected = _pack_result( + grad_output, + torch.full_like(grad_a, expected_a), + torch.full_like(grad_b, expected_b), + ) + + return Payload( + lambda *args: _add_backward( + *args, + implementation_index=implementation_index, + ), + lambda *args: expected.clone(), + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("offset", (0, 2)) +@pytest.mark.parametrize("strides", ((3, 1), (6, 2))) +def test_add_backward_rejects_output_overlap( + offset, strides, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = setup + "infini.ops.AddBackward(x, 2.0, torch.empty(4, device=device), x)\n" + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {strides!r}, 0) +b = backing.as_strided((2, 3), {strides!r}, {offset}) +infini.ops.add_backward(x, 2.0, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_cross_entropy.py b/tests/test_cross_entropy.py index 44a6b7a96..1fd5c6033 100644 --- a/tests/test_cross_entropy.py +++ b/tests/test_cross_entropy.py @@ -1,119 +1,631 @@ +import math + import infini.ops import pytest import torch +import torch.nn.functional as F -from tests.utils import ( - Payload, - empty_strided, - get_stream, - randn_strided, - randint_strided, -) +from tests.utils import Payload, empty_strided, get_stream, randn_strided -# Format: (input_shape, input_strides, target_strides) -# CrossEntropy follows the PyTorch convention: dimension 1 is the class dimension. -_TEST_CASES = ( - pytest.param( - (4, 7), - None, - None, - id="two-dimensional", - ), - pytest.param( - (2, 3, 5), - None, - None, - id="three-dimensional", - ), - pytest.param( - (2, 3, 5), - (30, 7, 1), - None, - id="non-contiguous-input", - ), +_DTYPES = (torch.float32, torch.float64, torch.float16, torch.bfloat16) +_REDUCTIONS = ("none", "mean", "sum") +# Dimension 0 is the class dimension for (C,); otherwise it is dimension 1. +_SHAPES = ( + pytest.param((7,), False, id="unbatched"), + pytest.param((1, 7), False, id="batch-one"), + pytest.param((4, 7), False, id="two-dimensional"), + pytest.param((2, 3, 5), False, id="three-dimensional"), + pytest.param((2, 3, 2, 4), False, id="four-dimensional"), + pytest.param((3, 1), False, id="one-class"), + pytest.param((7,), True, id="unbatched-strided"), + pytest.param((2, 3, 5), True, id="all-tensors-strided"), ) -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "input_shape, input_strides, target_strides", - _TEST_CASES, -) -@pytest.mark.parametrize("use_ignore_index", (False, True)) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-5, 1e-5), - (torch.float16, 1e-3, 1e-3), - (torch.bfloat16, 1e-2, 1e-2), - ), -) -def test_cross_entropy( - input_shape, - input_strides, - target_strides, - use_ignore_index, +def _tolerance(dtype): + return { + torch.float64: (1e-10, 1e-10), + torch.float32: (1e-5, 1e-5), + torch.float16: (1e-3, 1e-3), + torch.bfloat16: (1e-2, 1e-2), + }[dtype] + + +def _strides(shape, strided): + if not strided or not shape: + return None + result = [] + step = 2 + for size in reversed(shape): + result.append(step) + step *= max(size, 1) + return tuple(reversed(result)) + + +def _class_dim(shape): + return 0 if len(shape) == 1 else 1 + + +def _loss_shape(shape): + dim = _class_dim(shape) + return shape[:dim] + shape[dim + 1 :] + + +def _promote_low_precision(tensor): + if tensor is not None and tensor.dtype in (torch.float16, torch.bfloat16): + return tensor.float() + return tensor + + +def _result_dtype(input, target, weight): + dtype = input.dtype + if target.is_floating_point(): + dtype = torch.promote_types(dtype, target.dtype) + if weight is not None: + dtype = torch.promote_types(dtype, weight.dtype) + return dtype + + +def _expected(input, target, weight, ignore_index, reduction, label_smoothing): + # FP16/BF16 reference uses FP32 intermediates, not same-dtype Torch CE. + target = ( + _promote_low_precision(target) if target.is_floating_point() else target.long() + ) + return F.cross_entropy( + _promote_low_precision(input), + target, + weight=_promote_low_precision(weight), + ignore_index=ignore_index, + reduction=reduction, + label_smoothing=label_smoothing, + ) + + +def _cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + *, implementation_index, - dtype, - device, - rtol, - atol, ): + # Requires the binding for the new canonical C++ argument order. + infini.ops.cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + return out + + +def _torch_cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, +): + out.copy_( + _expected( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + ) + ) + return out + + +def _data(shape, probability_target, weighted, strided, dtype, device): input = randn_strided( - input_shape, - input_strides, + shape, + _strides(shape, strided), dtype=dtype, device=device, ) - target_shape = (input_shape[0],) + input_shape[2:] - num_classes = input_shape[1] - target = randint_strided( - 0, - num_classes, - target_shape, - target_strides, - dtype=torch.int64, - device=device, + dim = _class_dim(shape) + classes = shape[dim] + if probability_target: + target = empty_strided( + shape, + _strides(shape, strided), + dtype=dtype, + device=device, + ) + target.copy_(torch.randn(shape, device=device).softmax(dim=dim)) + else: + target_shape = _loss_shape(shape) + target = empty_strided( + target_shape, + _strides(target_shape, strided), + dtype=torch.int64, + device=device, + ) + target.copy_(torch.randint(classes, target_shape, device=device)) + weight = None + if weighted: + weight = empty_strided( + (classes,), + _strides((classes,), strided), + dtype=dtype, + device=device, + ) + weight.copy_(torch.linspace(0.25, 2.0, classes, device=device)) + return input, target, weight + + +def _out(input, target, weight, reduction, strided=False): + shape = _loss_shape(tuple(input.shape)) if reduction == "none" else () + out = empty_strided( + shape, + _strides(shape, strided), + dtype=_result_dtype(input, target, weight), + device=input.device, ) - if use_ignore_index: - target.reshape(-1)[0] = -100 - out = empty_strided((), None, dtype=dtype, device=device) out.fill_(17) + return out + + +def _check( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + implementation_index, + out=None, +): + if out is None: + out = _out(input, target, weight, reduction) + # Compute and detach the reference storage BEFORE the backend can write + # an aliased output. Merely assigning another Tensor variable is not a copy. + expected = ( + _expected( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + ) + .to(out.dtype) + .clone() + ) + _cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + implementation_index=implementation_index, + ) + rtol, atol = _tolerance(out.dtype) + torch.testing.assert_close(out, expected, rtol=rtol, atol=atol, equal_nan=True) + return out + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strided", _SHAPES) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2, 1.0)) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy( + shape, + strided, + probability_target, + weighted, + reduction, + label_smoothing, + implementation_index, + dtype, + device, +): + input, target, weight = _data( + shape, + probability_target, + weighted, + strided, + dtype, + device, + ) + out = _out(input, target, weight, reduction, strided) + rtol, atol = _tolerance(out.dtype) return Payload( lambda *args: _cross_entropy( *args, implementation_index=implementation_index, ), _torch_cross_entropy, - (input, target, out), + (input, target, weight, -100, reduction, label_smoothing, out), {}, rtol=rtol, atol=atol, ) -def _cross_entropy(input, target, out, *, implementation_index): - infini.ops.cross_entropy( +@pytest.mark.parametrize("pattern", ("partial", "one-valid", "all")) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2)) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy_ignored_targets( + pattern, + weighted, + reduction, + label_smoothing, + implementation_index, + dtype, + device, +): + input, target, weight = _data((2, 3, 5), False, weighted, True, dtype, device) + if pattern == "partial": + target[0, ::2] = -100 + else: + target.fill_(-100) + if pattern == "one-valid": + target[1, 3] = 1 + out = _check( input, target, + weight, + -100, + reduction, + label_smoothing, + implementation_index, + ) + if pattern == "all": + if reduction == "mean": + assert torch.isnan(out).item() + else: + assert torch.count_nonzero(out).item() == 0 + elif reduction == "none": + assert torch.count_nonzero(out[target == -100]).item() == 0 + + +@pytest.mark.parametrize("ignore_index", (-7, 1, 255)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_custom_ignore_index( + ignore_index, + reduction, + implementation_index, + device, +): + input = torch.zeros((3, 3), device=device) + target = torch.tensor([ignore_index, 0, 2], device=device) + out = _check( + input, target, None, ignore_index, reduction, 0.0, implementation_index + ) + expected = { + "none": [0.0, math.log(3), math.log(3)], + "mean": math.log(3), + "sum": 2 * math.log(3), + }[reduction] + torch.testing.assert_close(out, torch.tensor(expected, device=device)) + + +@pytest.mark.parametrize("target_dtype", (torch.int32, torch.uint8)) +def test_cross_entropy_integer_target_extensions( + target_dtype, + implementation_index, + device, +): + input = torch.zeros((3, 3), device=device) + target = torch.tensor([0, 1, 255], dtype=target_dtype, device=device) + _check(input, target, None, 255, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("all_zero", (False, True)) +def test_cross_entropy_zero_weights( + probability_target, + reduction, + all_zero, + implementation_index, + device, +): + input, target, _ = _data( + (3, 3), probability_target, False, False, torch.float32, device + ) + if not probability_target: + target.copy_(torch.tensor([0, 1, 2], device=device)) + weight = torch.tensor([0.0, 0.0 if all_zero else 2.0, 0.0], device=device) + out = _check(input, target, weight, -100, reduction, 0.2, implementation_index) + if all_zero: + if not probability_target and reduction == "mean": + assert torch.isnan(out).item() + else: + assert torch.count_nonzero(out).item() == 0 + + +def test_cross_entropy_probability_zero_row(implementation_index, device): + input = torch.zeros((2, 3), device=device) + target = torch.tensor([[1.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=device) + # A zero probability row does not remove that row from the mean divisor. + out = _check(input, target, None, -100, "mean", 0.0, implementation_index) + torch.testing.assert_close(out, torch.tensor(math.log(3) / 2, device=device)) + + +@pytest.mark.parametrize("probability_target", (False, True)) +def test_cross_entropy_weighted_mean_divisor( + probability_target, + implementation_index, + device, +): + input = torch.zeros((2, 2), device=device) + target = torch.tensor([0, 1], device=device) + weight = torch.tensor([1.0, 3.0], device=device) + if probability_target: + target = F.one_hot(target, num_classes=2).float() + out = _check(input, target, weight, -100, "mean", 0.0, implementation_index) + # Same one-hot labels, different upstream mean denominators: 4 vs 2. + expected = math.log(2) * (2 if probability_target else 1) + torch.testing.assert_close(out, torch.tensor(expected, device=device)) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2, 1.0)) +def test_cross_entropy_smoothing_analytic( + probability_target, + reduction, + label_smoothing, + implementation_index, + device, +): + # Each row assigns probability 3/4 to its correct class and 1/4 to the + # other class. Calculate the answer without calling Torch CE. + input = torch.tensor( + [[math.log(3), 0.0], [0.0, math.log(3)]], dtype=torch.float64, device=device + ) + target = torch.tensor([0, 1], device=device) + weight = torch.tensor([1.0, 3.0], dtype=torch.float64, device=device) + if probability_target: + target = F.one_hot(target, num_classes=2).double() + a = 1 - label_smoothing / 2 + b = label_smoothing / 2 + losses = [ + -a * math.log(0.75) - 3 * b * math.log(0.25), + -3 * a * math.log(0.75) - b * math.log(0.25), + ] + expected = torch.tensor(losses, dtype=torch.float64, device=device) + if reduction == "sum": + expected = expected.sum() + elif reduction == "mean": + expected = expected.sum() / (2 if probability_target else 4) + out = _check( + input, target, weight, -100, reduction, label_smoothing, implementation_index + ) + torch.testing.assert_close(out, expected, rtol=1e-10, atol=1e-10) + + +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_unbatched_ignored(reduction, implementation_index, device): + input = torch.zeros((3,), device=device) + target = torch.tensor(-100, device=device) + out = _check(input, target, None, -100, reduction, 0.2, implementation_index) + if reduction == "mean": + assert torch.isnan(out).item() + else: + assert out.item() == 0 + + +@pytest.mark.parametrize("shape", ((0, 3), (2, 3, 0))) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_empty( + shape, + probability_target, + reduction, + implementation_index, + device, +): + input, target, weight = _data( + shape, probability_target, True, False, torch.float32, device + ) + out = _check(input, target, weight, -100, reduction, 0.2, implementation_index) + if reduction == "mean": + assert torch.isnan(out).item() + elif reduction == "sum": + assert out.item() == 0 + else: + assert out.numel() == 0 + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy_extreme_logits( + probability_target, + reduction, + dtype, + implementation_index, + device, +): + input = torch.tensor( + [ + [1000.0, -1000.0, 0.0], + [-1000.0, 1000.0, 0.0], + [1000.0, 1000.0, 1000.0], + [-1000.0, -1000.0, -1000.0], + ], + dtype=dtype, + device=device, + ) + target = torch.tensor([1, 1, 0, 2], device=device) + if probability_target: + target = F.one_hot(target, num_classes=3).to(dtype) + out = _check(input, target, None, -100, reduction, 0.0, implementation_index) + losses = torch.tensor( + [2000.0, 0.0, math.log(3), math.log(3)], dtype=torch.float64, device=device + ) + expected = ( + losses + if reduction == "none" + else (losses.mean() if reduction == "mean" else losses.sum()) + ) + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).all().item() + torch.testing.assert_close(out, expected.to(dtype), rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("count", (32768, 65504, 65505, 65520, 65536, 131072)) +@pytest.mark.parametrize("classes", (2, 4)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +def test_cross_entropy_long_sequence( + count, + classes, + weighted, + dtype, + implementation_index, + device, +): + # Shape (1,C,L) deliberately exercises the sequence path. With a constant + # class weight of 2, even count=32768 has total_weight > FP16's maximum. + input = torch.zeros((1, classes, count + 17), dtype=dtype, device=device) + target = torch.full((1, count + 17), -100, dtype=torch.int64, device=device) + target[:, :count] = 0 + weight = ( + torch.full((classes,), 2.0, dtype=dtype, device=device) if weighted else None + ) + out = _check(input, target, weight, -100, "mean", 0.0, implementation_index) + # Independent mathematical oracle: each prediction is uniform. + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).item() + torch.testing.assert_close( out, - stream=get_stream(input.device), - implementation_index=implementation_index, + torch.tensor(math.log(classes), dtype=dtype, device=device), + rtol=rtol, + atol=atol, ) - return out +def test_cross_entropy_sum_output_overflow(implementation_index, device): + # FP32 intermediates cannot make a genuinely unrepresentable FP16 output + # finite. This must remain +inf, unlike the finite long-sequence mean. + input = torch.zeros((1, 4, 65536), dtype=torch.float16, device=device) + target = torch.zeros((1, 65536), dtype=torch.int64, device=device) + out = _check(input, target, None, -100, "sum", 0.0, implementation_index) + assert torch.isposinf(out).item() -def _torch_cross_entropy(input, target, out): - out.copy_( - torch.nn.functional.cross_entropy( - input, - target, - reduction="mean", - ignore_index=-100, - ) + +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +def test_cross_entropy_long_probability_target(dtype, implementation_index, device): + input = torch.zeros((1, 4, 65536), dtype=dtype, device=device) + target = torch.full_like(input, 0.25) + out = _check(input, target, None, -100, "mean", 0.2, implementation_index) + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).item() + torch.testing.assert_close( + out, + torch.tensor(math.log(4), dtype=dtype, device=device), + rtol=rtol, + atol=atol, ) - return out + +@pytest.mark.parametrize( + "input_dtype,target_dtype", + ( + (torch.float32, torch.float64), + (torch.float16, torch.float32), + (torch.float16, torch.bfloat16), + ), +) +def test_cross_entropy_probability_dtype_promotion( + input_dtype, + target_dtype, + implementation_index, + device, +): + input, target, weight = _data((2, 3), True, True, True, input_dtype, device) + target = target.to(target_dtype) + _check(input, target, weight, -100, "none", 0.2, implementation_index) + + +def test_cross_entropy_probability_weight_dtype_promotion(implementation_index, device): + input, target, weight = _data((2, 3), True, True, True, torch.float32, device) + _check(input, target, weight.double(), -100, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_transposed_layout( + probability_target, + reduction, + implementation_index, + device, +): + input = torch.randn((3, 2, 5), device=device).transpose(0, 1) + if probability_target: + target = torch.randn((3, 2, 5), device=device).softmax(0).transpose(0, 1) + else: + target = (torch.arange(10, device=device).reshape(5, 2) % 3).t() + target[0, 0] = -100 + weight = torch.tensor([0.5, 17.0, 2.0, 17.0, 3.0], device=device)[::2] + out = ( + torch.full((5, 2), 17.0, device=device).t() + if reduction == "none" + else torch.full((), 17.0, device=device) + ) + _check(input, target, weight, -100, reduction, 0.2, implementation_index, out=out) + + +@pytest.mark.parametrize( + "alias", + ( + "input-full", + "input-partial", + "target-full", + "target-partial", + "weight", + ), +) +@pytest.mark.parametrize("reduction", ("none", "mean")) +def test_cross_entropy_aliasing(alias, reduction, implementation_index, device): + full = alias in ("input-full", "target-full") + shape = (1,) if full and reduction == "mean" else ((2, 1) if full else (2, 3)) + probability_target = alias.startswith("target") + input, target, weight = _data( + shape, probability_target, True, True, torch.float32, device + ) + if alias == "weight": + out = weight[:2] if reduction == "none" else weight[1] + else: + source = target if probability_target else input + if full: + out = source.squeeze(1) if reduction == "none" else source[0] + else: + out = source[:, 0] if reduction == "none" else source[0, 1] + _check(input, target, weight, -100, reduction, 0.2, implementation_index, out=out) + + +def test_cross_entropy_strided_output_guards(implementation_index, device): + input, target, weight = _data((2, 3), False, True, True, torch.float32, device) + backing = torch.full((7,), 17.0, device=device) + out = backing[1:5:2] + _check(input, target, weight, -100, "none", 0.2, implementation_index, out=out) + untouched = torch.tensor([0, 2, 4, 5, 6], device=device) + torch.testing.assert_close( + backing[untouched], torch.full((5,), 17.0, device=device) + ) diff --git a/tests/test_cross_entropy_backward.py b/tests/test_cross_entropy_backward.py index 9b52ff293..5986f984f 100644 --- a/tests/test_cross_entropy_backward.py +++ b/tests/test_cross_entropy_backward.py @@ -1,136 +1,323 @@ -import infini.ops -import pytest -import torch - -from tests.utils import ( - Payload, - empty_strided, - get_stream, - randint_strided, - randn_strided, -) - - -# Format: (input_shape, input_strides, target_strides, grad_input_strides) -# CrossEntropy uses the PyTorch convention: dimension 1 is the class dimension. -_TEST_CASES = ( - pytest.param( - (4, 7), - None, - None, - None, - id="two-dimensional", - ), - pytest.param( - (2, 3, 5), - None, - None, - None, - id="three-dimensional", - ), - pytest.param( - (2, 3, 5), - (30, 7, 1), - None, - (35, 8, 1), - id="strided-input-and-output", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "input_shape, input_strides, target_strides, grad_input_strides", - _TEST_CASES, -) -@pytest.mark.parametrize("use_ignore_index", (False, True)) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-6, 1e-6), - (torch.float16, 1e-3, 1e-3), - (torch.bfloat16, 1e-2, 1e-2), - ), -) -def test_cross_entropy_backward( - input_shape, - input_strides, - target_strides, - grad_input_strides, - use_ignore_index, - implementation_index, - dtype, - device, - rtol, - atol, -): - input = randn_strided( - input_shape, - input_strides, - dtype=dtype, - device=device, - ) - target = randint_strided( - 0, - input_shape[1], - (input_shape[0],) + input_shape[2:], - target_strides, - dtype=torch.int64, - device=device, - ) - if use_ignore_index: - target.reshape(-1)[0] = -100 - grad_output = torch.randn((), dtype=dtype, device=device) - grad_input = empty_strided( - input_shape, - grad_input_strides, - dtype=dtype, - device=device, - ) - grad_input.fill_(17) - - return Payload( - lambda *args: _cross_entropy_backward( - *args, - implementation_index=implementation_index, - ), - _torch_cross_entropy_backward, - (input, target, grad_output, grad_input), - {}, - rtol=rtol, - atol=atol, - ) - - -def _cross_entropy_backward( - input, - target, - grad_output, - grad_input, - *, - implementation_index, -): - infini.ops.cross_entropy_backward( - input, - target, - grad_output, - grad_input, - stream=get_stream(input.device), - implementation_index=implementation_index, - ) - - return grad_input - - -def _torch_cross_entropy_backward(input, target, grad_output, grad_input): - logits = input.detach().clone().requires_grad_(True) - loss = torch.nn.functional.cross_entropy( - logits, - target, - reduction="mean", - ignore_index=-100, - ) - gradient = torch.autograd.grad(loss, logits, grad_output)[0] - grad_input.copy_(gradient) - - return grad_input +import infini.ops +import pytest +import torch + +from tests.utils import ( + Payload, + empty_strided, + get_stream, + randint_strided, + randn_strided, +) + + +# Format: (input_shape, input_strides, target_strides, grad_input_strides) +# CrossEntropy uses the PyTorch convention: dimension 1 is the class dimension. +_TEST_CASES = ( + pytest.param( + (4, 7), + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 5), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 5), + (30, 7, 1), + None, + (35, 8, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, target_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("use_ignore_index", (False, True)) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_cross_entropy_backward( + input_shape, + input_strides, + target_strides, + grad_input_strides, + use_ignore_index, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + target = randint_strided( + 0, + input_shape[1], + (input_shape[0],) + input_shape[2:], + target_strides, + dtype=torch.int64, + device=device, + ) + if use_ignore_index: + target.reshape(-1)[0] = -100 + grad_output = torch.randn((), dtype=dtype, device=device) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _cross_entropy_backward( + *args, + implementation_index=implementation_index, + ), + _torch_cross_entropy_backward, + (input, target, grad_output, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _cross_entropy_backward( + input, + target, + grad_output, + grad_input, + *, + implementation_index, +): + infini.ops.cross_entropy_backward( + input, + target, + grad_output, + None, + -100, + "mean", + 0.0, + grad_input, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_cross_entropy_backward(input, target, grad_output, grad_input): + logits = input.detach().clone().requires_grad_(True) + loss = torch.nn.functional.cross_entropy( + logits, + target, + reduction="mean", + ignore_index=-100, + ) + gradient = torch.autograd.grad(loss, logits, grad_output)[0] + grad_input.copy_(gradient) + + return grad_input + + +# Forward API coverage is shared so backward stays aligned with every attribute. +@pytest.mark.parametrize( + "shape,strided", (((7,), False), ((4, 7), False), ((2, 3, 5), True)) +) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", ("none", "sum", "mean")) +@pytest.mark.parametrize("smoothing", (0.0, 0.2, 1.0)) +@pytest.mark.parametrize( + "dtype", (torch.float64, torch.float32, torch.float16, torch.bfloat16) +) +def test_cross_entropy_backward_full_contract( + shape, + strided, + probability_target, + weighted, + reduction, + smoothing, + dtype, + device, + implementation_index, +): + from tests.test_cross_entropy import _data + + input, target, weight = _data( + shape, probability_target, weighted, strided, dtype, device + ) + _check_full_backward( + input, target, weight, -100, reduction, smoothing, implementation_index + ) + + +def _check_full_backward( + input, + target, + weight, + ignore_index, + reduction, + smoothing, + implementation_index, + out=None, +): + from tests.test_cross_entropy import _expected, _result_dtype, _tolerance + + with torch.enable_grad(): + logits = input.detach().clone().requires_grad_(True) + loss = _expected( + logits, target.detach().clone(), weight, ignore_index, reduction, smoothing + ) + upstream = torch.full_like( + loss, 0.75, dtype=_result_dtype(input, target, weight) + ) + expected = torch.autograd.grad(loss, logits, upstream)[0].detach().clone() + + if out is None: + out = torch.empty_like(input) + + infini.ops.cross_entropy_backward( + input, + target, + upstream, + weight, + ignore_index, + reduction, + smoothing, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + rtol, atol = _tolerance(input.dtype) + torch.testing.assert_close(out, expected, rtol=rtol, atol=atol, equal_nan=True) + return out + + +@pytest.mark.parametrize("count", (65504, 65536, 131072)) +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +@pytest.mark.parametrize("mode", ("ordinary", "all-ignore", "one-valid", "extreme")) +def test_cross_entropy_backward_long_sequence( + count, dtype, mode, device, implementation_index +): + input = torch.zeros((1, 4, count), dtype=dtype, device=device) + target = torch.zeros((1, count), dtype=torch.int64, device=device) + + if mode == "all-ignore": + target.fill_(-100) + elif mode == "one-valid": + target.fill_(-100) + target[0, 0] = 0 + elif mode == "extreme": + input[:, 1, :] = 1000 + + out = _check_full_backward( + input, target, None, -100, "mean", 0.0, implementation_index + ) + + if mode == "ordinary": + expected = torch.full_like(input, 0.75 * 0.25 / count) + expected[:, 0, :] = -0.75 * 0.75 / count + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + if mode == "all-ignore": + assert torch.count_nonzero(out) == 0 + + +@pytest.mark.parametrize("smoothing", (0.0, 0.2)) +@pytest.mark.parametrize("reduction", ("none", "sum", "mean")) +def test_cross_entropy_backward_zero_weights( + smoothing, reduction, device, implementation_index +): + input = torch.randn((3, 4), device=device) + target = torch.tensor([-7, 1, 2], device=device) + weight = torch.zeros(4, device=device) + _check_full_backward( + input, target, weight, -7, reduction, smoothing, implementation_index + ) + + +@pytest.mark.parametrize("target_dtype", (torch.float16, torch.float32, torch.float64)) +@pytest.mark.parametrize("weight_dtype", (torch.float32, torch.float64)) +def test_cross_entropy_backward_probability_promotion( + target_dtype, + weight_dtype, + device, + implementation_index, +): + input = torch.randn((2, 3), device=device) + target = torch.randn((2, 3), device=device).softmax(1).to(target_dtype) + weight = torch.tensor([0.5, 1.0, 2.0], dtype=weight_dtype, device=device) + _check_full_backward(input, target, weight, -100, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("ignore_index", (-7, 1, 255)) +@pytest.mark.parametrize("reduction", ("none", "mean", "sum")) +@pytest.mark.parametrize("smoothing", (0.0, 0.2)) +def test_cross_entropy_backward_custom_ignore_index( + ignore_index, + reduction, + smoothing, + device, + implementation_index, +): + input = torch.randn((2, 3), device=device) + target = torch.tensor([ignore_index, 2], device=device) + weight = torch.tensor([0.5, 1.0, 2.0], device=device) + _check_full_backward( + input, target, weight, ignore_index, reduction, smoothing, implementation_index + ) + + +@pytest.mark.parametrize("alias", ("input", "upstream", "weight")) +def test_cross_entropy_backward_alias(alias, device, implementation_index): + input = torch.randn((1, 3), device=device) + target = torch.tensor([1], device=device) + weight = torch.tensor([0.5, 1.0, 2.0], device=device) + # Input/output and weight/output may share memory; snapshot the oracle first. + out = input if alias == "input" else weight.view(1, 3) + + if alias == "upstream": + upstream = out.reshape(-1)[:1].view(()) + upstream.fill_(0.75) + with torch.enable_grad(): + logits = input.detach().clone().requires_grad_(True) + loss = torch.nn.functional.cross_entropy(logits, target, weight=weight) + expected = torch.autograd.grad(loss, logits, upstream.clone())[0] + infini.ops.cross_entropy_backward( + input, + target, + upstream, + weight, + -100, + "mean", + 0.0, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected) + else: + _check_full_backward( + input, target, weight, -100, "mean", 0.2, implementation_index, out + ) diff --git a/tests/test_embedding_backward.py b/tests/test_embedding_backward.py deleted file mode 100644 index e3c992bdf..000000000 --- a/tests/test_embedding_backward.py +++ /dev/null @@ -1,177 +0,0 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: -# (indices, num_weights, embedding_dim, indices_strides, -# grad_output_strides, grad_weight_strides, padding_idx, -# scale_grad_by_freq) -_TEST_CASES = ( - pytest.param( - (1, 2, 1, 4, 2, 1), - 6, - 4, - None, - None, - None, - -1, - False, - id="repeated-indices", - ), - pytest.param( - ((0, 1, 0), (2, 1, 3)), - 5, - 7, - (5, 1), - (32, 8, 1), - (9, 1), - 1, - False, - id="padding-and-strides", - ), - pytest.param( - (2, 2, 2, 3, 3, 4), - 7, - 5, - None, - None, - None, - -1, - True, - id="scale-grad-by-frequency", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "indices_values", - "num_weights", - "embedding_dim", - "indices_strides", - "grad_output_strides", - "grad_weight_strides", - "padding_idx", - "scale_grad_by_freq", - ), - _TEST_CASES, -) -@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-5, 1e-5), - (torch.float16, 1e-2, 1e-2), - (torch.bfloat16, 5e-2, 5e-2), - ), -) -def test_embedding_backward( - indices_values, - num_weights, - embedding_dim, - indices_strides, - grad_output_strides, - grad_weight_strides, - padding_idx, - scale_grad_by_freq, - indices_dtype, - implementation_index, - dtype, - device, - rtol, - atol, -): - indices_values = torch.tensor(indices_values, dtype=indices_dtype) - indices = empty_strided( - indices_values.shape, - indices_strides, - dtype=indices_dtype, - device=device, - ) - indices.copy_(indices_values.to(device)) - - grad_output_shape = (*indices.shape, embedding_dim) - grad_output = randn_strided( - grad_output_shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - grad_weight = empty_strided( - (num_weights, embedding_dim), - grad_weight_strides, - dtype=dtype, - device=device, - ) - grad_weight.fill_(float("nan")) - - return Payload( - lambda *args: _embedding_backward( - *args, - num_weights=num_weights, - padding_idx=padding_idx, - scale_grad_by_freq=scale_grad_by_freq, - implementation_index=implementation_index, - ), - lambda *args: _torch_embedding_backward( - *args, - num_weights=num_weights, - padding_idx=padding_idx, - scale_grad_by_freq=scale_grad_by_freq, - ), - (grad_output, indices, grad_weight), - {}, - rtol=rtol, - atol=atol, - ) - - -def _embedding_backward( - grad_output, - indices, - grad_weight, - *, - num_weights, - padding_idx, - scale_grad_by_freq, - implementation_index, -): - infini.ops.embedding_backward( - grad_output, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - False, - grad_weight, - implementation_index=implementation_index, - stream=get_stream(grad_output.device), - ) - - return grad_weight - - -def _torch_embedding_backward( - grad_output, - indices, - grad_weight, - *, - num_weights, - padding_idx, - scale_grad_by_freq, -): - result = torch.ops.aten.embedding_backward.default( - grad_output, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - False, - ) - grad_weight.copy_(result) - - return grad_weight diff --git a/tests/test_embedding_dense_backward.py b/tests/test_embedding_dense_backward.py new file mode 100644 index 000000000..9a6b80c09 --- /dev/null +++ b/tests/test_embedding_dense_backward.py @@ -0,0 +1,590 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Dense-only interface: there is deliberately no sparse argument. +# (indices, num_weights, embedding_dim, indices_strides, +# grad_output_strides, grad_weight_strides, padding_idx, scale_grad_by_freq) +_TEST_CASES = ( + pytest.param( + (1, 2, 1, 4, 2, 1), 6, 4, None, None, None, -1, False, id="repeated-indices" + ), + pytest.param( + ((0, 1, 0), (2, 1, 3)), + 5, + 7, + (5, 1), + (32, 8, 1), + (9, 1), + 1, + False, + id="padding-and-strides", + ), + pytest.param( + (2, 2, 2, 3, 3, 4), + 7, + 5, + None, + None, + None, + -1, + True, + id="scale-grad-by-frequency", + ), + pytest.param( + (1, 2, 1, 1, 2, 4), + 6, + 5, + (2,), + (13, 2), + (12, 2), + 1, + True, + id="padding-with-frequency-and-strides", + ), + pytest.param((), 4, 3, None, None, None, -1, False, id="empty"), + pytest.param( + (), + 4, + 3, + (2,), + (7, 2), + (8, 2), + 0, + True, + id="empty-with-padding-frequency-and-strides", + ), + pytest.param( + ((), ()), 4, 3, None, None, None, -1, True, id="multidimensional-empty" + ), + pytest.param((1, 1, 1, 1), 4, 3, None, None, None, 1, False, id="all-padding"), + pytest.param( + ((2, 2), (2, 2)), + 4, + 3, + (5, 2), + (15, 6, 1), + (8, 2), + 2, + True, + id="all-padding-with-frequency", + ), + pytest.param( + (0, 0, 3, 0), 5, 3, None, None, None, 0, True, id="one-non-padding-occurrence" + ), + pytest.param(2, 4, 5, None, (2,), (12, 2), -1, True, id="scalar-index"), + pytest.param(2, 4, 5, None, None, None, 2, True, id="scalar-padding"), + pytest.param( + (((0, 2), (1, 0)), ((2, 1), (0, 2))), + 4, + 3, + None, + None, + None, + 1, + True, + id="three-dimensional-indices", + ), + pytest.param( + (0, 0, 0), 1, 1, None, None, None, -1, True, id="one-weight-one-feature" + ), + pytest.param( + (0, 5, 0, 5), + 6, + 33, + None, + None, + None, + -1, + False, + id="first-last-rows-and-wide-embedding", + ), + pytest.param((0, 3, 3), 4, 1, None, None, None, 3, True, id="last-row-padding"), + pytest.param( + (0, 3, 3), + 4, + 1, + None, + None, + None, + -1, + True, + id="minus-one-does-not-mean-last-row-padding", + ), + pytest.param((), 0, 3, None, None, None, -1, False, id="empty-vocabulary"), + pytest.param( + (), 0, 3, None, None, None, -1, True, id="empty-vocabulary-with-frequency" + ), + pytest.param( + (0, 1, 1), 4, 0, None, None, None, -1, False, id="zero-embedding-dimension" + ), + pytest.param( + (0, 1, 1), + 4, + 0, + None, + None, + None, + -1, + True, + id="zero-embedding-dimension-with-frequency", + ), +) + +_DTYPES = ( + pytest.param(torch.float32, 1e-5, 1e-5, id="float32"), + pytest.param(torch.float64, 1e-10, 1e-10, id="float64"), + pytest.param(torch.float16, 1e-2, 1e-2, id="float16"), + pytest.param(torch.bfloat16, 5e-2, 5e-2, id="bfloat16"), +) + + +def _embedding_dense_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, + implementation_index, +): + infini.ops.embedding_dense_backward( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + grad_weight, + implementation_index=implementation_index, + stream=get_stream(grad_output.device), + ) + return grad_weight + + +def _torch_embedding_dense_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, +): + # Some Torch CUDA versions fail for empty frequency-scaling inputs or + # zero feature width. Both have an identically zero/empty dense result. + if indices.numel() == 0 or grad_weight.numel() == 0: + grad_weight.zero_() + return grad_weight + result = torch.ops.aten.embedding_dense_backward.default( + grad_output, + indices.long(), + num_weights, + padding_idx, + scale_grad_by_freq, + ) + assert result.layout == torch.strided + grad_weight.copy_(result) + return grad_weight + + +def _oracle(grad_output, indices, num_weights, padding_idx, scale_grad_by_freq): + # Independent mathematical reference: accumulate rows in CPU FP64. + # This does NOT call embedding_dense_backward, and copies the source storage + # before any possibly overlapping output can be written. + index = indices.detach().to(device="cpu", dtype=torch.int64).reshape(-1) + dim = grad_output.shape[-1] + values = grad_output.detach().to(device="cpu", dtype=torch.float64).clone() + values = values.reshape(index.numel(), dim) + mask = index != padding_idx + index = index[mask] + values = values[mask] + result = torch.zeros((num_weights, dim), dtype=torch.float64) + if index.numel(): + if scale_grad_by_freq: + counts = torch.bincount(index, minlength=num_weights) + values = values / counts[index, None] + result.index_add_(0, index, values) + return result.to(device=grad_output.device, dtype=grad_output.dtype) + + +def _check( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, + implementation_index, + rtol, + atol, +): + expected = _oracle( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + ) + _embedding_dense_backward( + grad_output, + indices, + grad_weight, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + ) + assert grad_weight.layout == torch.strided + assert torch.isfinite(grad_weight).all().item() + torch.testing.assert_close(grad_weight, expected, rtol=rtol, atol=atol) + return grad_weight + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "indices_values,num_weights,embedding_dim,indices_strides," + "grad_output_strides,grad_weight_strides,padding_idx,scale_grad_by_freq", + _TEST_CASES, +) +@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward( + indices_values, + num_weights, + embedding_dim, + indices_strides, + grad_output_strides, + grad_weight_strides, + padding_idx, + scale_grad_by_freq, + indices_dtype, + implementation_index, + dtype, + device, + rtol, + atol, +): + values = torch.tensor(indices_values, dtype=indices_dtype) + indices = empty_strided( + values.shape, indices_strides, dtype=indices_dtype, device=device + ) + indices.copy_(values.to(device)) + grad_output = randn_strided( + (*indices.shape, embedding_dim), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_weight = empty_strided( + (num_weights, embedding_dim), + grad_weight_strides, + dtype=dtype, + device=device, + ) + grad_weight.fill_(float("nan")) + return Payload( + lambda *args: _embedding_dense_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + ), + lambda *args: _torch_embedding_dense_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + ), + (grad_output, indices, grad_weight), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize("padding_idx", (-1, 1, 3)) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_analytic( + padding_idx, + scale_grad_by_freq, + indices_dtype, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], dtype=indices_dtype, device=device) + grad_output = torch.tensor([[2, 4], [8, 12], [3, 9]], dtype=dtype, device=device) + grad_weight = torch.full((5, 2), float("nan"), dtype=dtype, device=device) + expected = torch.zeros_like(grad_weight) + if padding_idx != 1: + expected[1] = torch.tensor( + [5, 8] if scale_grad_by_freq else [10, 16], + dtype=dtype, + device=device, + ) + if padding_idx != 3: + expected[3] = torch.tensor([3, 9], dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + torch.testing.assert_close(grad_weight, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_padding_nan_and_unused_rows( + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 2, 4], device=device) + grad_output = torch.tensor( + [[float("nan"), float("nan")], [float("nan"), float("nan")], [2, -3], [5, 7]], + dtype=dtype, + device=device, + ) + grad_weight = torch.full((6, 2), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=6, + padding_idx=1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + # Padding is skipped entirely, including its NaN upstream gradient. + assert torch.count_nonzero(grad_weight[[0, 1, 3, 5]]).item() == 0 + + +@pytest.mark.parametrize("all_padding", (False, True)) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_empty_or_all_padding_zeroes_output( + all_padding, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.full( + (2, 3) if all_padding else (2, 0), 1, dtype=torch.int64, device=device + ) + grad_output = torch.full( + (*indices.shape, 3), float("nan"), dtype=dtype, device=device + ) + grad_weight = torch.full((4, 3), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=4, + padding_idx=1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + assert torch.count_nonzero(grad_weight).item() == 0 + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +def test_embedding_dense_backward_transposed_layout( + scale_grad_by_freq, + implementation_index, + device, +): + indices = torch.tensor([[0, 1], [1, 2], [1, 3]], device=device).t() + grad_output = torch.randn((3, 2, 5), device=device).transpose(0, 1) + grad_weight = torch.full((5, 4), float("nan"), device=device).t() + _check( + grad_output, + indices, + grad_weight, + num_weights=4, + padding_idx=2, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) + + +@pytest.mark.parametrize("pattern", ("zero", "cancel", "negative")) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_gradient_values( + pattern, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], device=device) + values = { + "zero": [[0, 0], [0, 0], [0, 0]], + "cancel": [[2, -4], [-2, 4], [0, 0]], + "negative": [[-2, -4], [-8, -12], [-3, -9]], + }[pattern] + grad_output = torch.tensor(values, dtype=dtype, device=device) + grad_weight = torch.full((5, 2), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + if pattern in ("zero", "cancel"): + assert torch.count_nonzero(grad_weight).item() == 0 + + +@pytest.mark.parametrize("alias", ("full", "partial", "strided")) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_grad_output_alias( + alias, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([2, 0, 2], device=device) + if alias == "full": + grad_output = torch.arange(1, 7, dtype=dtype, device=device).reshape(3, 2) + grad_weight = grad_output + elif alias == "partial": + backing = torch.arange(1, 13, dtype=dtype, device=device) + grad_output = backing[:6].view(3, 2) + grad_weight = backing[3:9].view(3, 2) + else: + backing = torch.arange(1, 13, dtype=dtype, device=device).reshape(3, 4) + grad_output = backing[:, :2] + grad_weight = backing[:, 1:3] + _check( + grad_output, + indices, + grad_weight, + num_weights=3, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize("partial", (False, True)) +@pytest.mark.parametrize( + "indices_dtype,dtype", + ( + (torch.int32, torch.float32), + (torch.int64, torch.float64), + ), +) +def test_embedding_dense_backward_indices_alias( + partial, + indices_dtype, + dtype, + implementation_index, + device, +): + # Equal element sizes allow index and output views over the same bytes. + backing = torch.zeros((12,), dtype=dtype, device=device) + indices = backing[1:7].view(indices_dtype) + indices.copy_(torch.tensor([2, 0, 2, 1, 0, 2], dtype=indices_dtype, device=device)) + grad_weight = backing[4:10].view(3, 2) if partial else backing[1:7].view(3, 2) + grad_output = torch.arange(1, 13, dtype=dtype, device=device).view(6, 2) + _check( + grad_output, + indices, + grad_weight, + num_weights=3, + padding_idx=-1, + scale_grad_by_freq=True, + implementation_index=implementation_index, + rtol=1e-10 if dtype == torch.float64 else 1e-5, + atol=1e-10 if dtype == torch.float64 else 1e-5, + ) + + +def test_embedding_dense_backward_strided_output_guards(implementation_index, device): + indices = torch.tensor([1, 1, 3], device=device) + grad_output = torch.tensor( + [[2, 4], [8, 12], [3, 9]], + dtype=torch.float32, + device=device, + ) + backing = torch.full((5, 5), 17.0, device=device) + grad_weight = backing[:, 1::2] + expected_backing = backing.clone() + expected_backing[:, 1::2] = _oracle(grad_output, indices, 5, -1, True) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=True, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) + torch.testing.assert_close(backing, expected_backing) + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +def test_embedding_dense_backward_overwrites_previous_result( + scale_grad_by_freq, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], device=device) + grad_output = torch.ones((3, 2), device=device) + grad_weight = torch.full((5, 2), 17.0, device=device) + for values in ([1, 1, 3], [0, 2, 2]): + indices.copy_(torch.tensor(values, device=device)) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) diff --git a/tests/test_generate_torch_ops.py b/tests/test_generate_torch_ops.py index 53a6cde59..615aff6f8 100644 --- a/tests/test_generate_torch_ops.py +++ b/tests/test_generate_torch_ops.py @@ -1,537 +1,537 @@ -import importlib.util -import pathlib -import sys -import types - -import pytest - - -def _load_generator_module(): - path = ( - pathlib.Path(__file__).resolve().parents[1] - / "scripts" - / "generate_torch_ops.py" - ) - spec = importlib.util.spec_from_file_location("generate_torch_ops_under_test", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - sys.modules[spec.name] = module - spec.loader.exec_module(module) - - return module - - -def test_select_op_names_honors_slot_8_and_legacy_headers(): - module = _load_generator_module() - config = { - "default_all": {"headers": None, "implementations": None}, - "default_native": {"headers": None, "implementations": (0,)}, - "explicit_torch": {"headers": None, "implementations": (8,)}, - "legacy": {"headers": ["legacy.h"], "implementations": None}, - } - - assert module._select_op_names( - None, - ["default_all", "default_native"], - config, - ) == ["default_all", "explicit_torch"] - assert module._select_op_names( - ["explicit_torch"], - ["default_all", "default_native"], - config, - ) == ["explicit_torch"] - - -def test_select_op_names_maps_public_names_to_aten_names(): - module = _load_generator_module() - config = { - "div": {"headers": None, "implementations": (8,)}, - "internal_log_softmax": {"headers": None, "implementations": (8,)}, - } - - assert module._select_op_names( - None, - ["div", "div_", "_log_softmax"], - config, - ) == ["div", "div_", "_log_softmax"] - - -def test_load_aten_entries_uses_packaged_torchgen(monkeypatch): - module = _load_generator_module() - entries = [{"func": "relu.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)"}] - monkeypatch.setattr(module, "_load_packaged_aten_entries", lambda: (entries, True)) - - assert module._load_aten_entries("v9.9.9") == entries - - -def test_packaged_aten_entries_include_autogenerated_out_variants(): - module = _load_generator_module() - entries, expanded_autogen = module._load_packaged_aten_entries() - - if not expanded_autogen: - pytest.skip("installed torchgen does not support autogen expansion") - - relu_out_entry = next( - entry - for entry in module._find_out_entries(entries, "relu") - if entry["func"].startswith("relu.out(") - ) - - assert module._parse_func(relu_out_entry["func"]).out_params - - -def test_packaged_aten_entries_fall_back_when_tags_are_missing( - monkeypatch, tmp_path, capsys -): - module = _load_generator_module() - native_dir = tmp_path / "packaged" / "ATen" / "native" - native_dir.mkdir(parents=True) - native_path = native_dir / "native_functions.yaml" - native_path.write_text( - "- func: add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)\n" - ) - spec = types.SimpleNamespace(submodule_search_locations=[str(tmp_path)]) - monkeypatch.setattr(module.importlib.util, "find_spec", lambda name: spec) - - entries, expanded_autogen = module._load_packaged_aten_entries() - - assert expanded_autogen is False - assert entries == [ - {"func": "add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)"} - ] - assert "using explicit schemas only" in capsys.readouterr().err - - -def test_public_op_name_normalizes_aten_internal_and_inplace_names(): - module = _load_generator_module() - - assert module._public_op_name("_softmax") == "internal_softmax" - assert module._public_op_name("add_") == "add" - assert module._public_op_name("_add_relu_") == "internal_add_relu" - - -def test_schema_self_param_renders_as_input_in_public_cpp_api(): - module = _load_generator_module() - op = module._parse_func( - "_softmax(Tensor self, int dim, bool half_to_float, *, " - "Tensor(a!) out) -> Tensor(a!)" - ) - - assert op.params[0].name == "self" - assert op.params[0].api_name == "input" - - base = module._generate_base_header("internal_softmax", [op]) - source = module._generate_torch_method_source("internal_softmax", op) - - assert "Softmax(const Tensor input, const int64_t dim" in base - assert "self_shape_" not in base - assert "input_shape_" in base - assert "auto at_self = ToAtenTensor" in source - assert "input_shape_" in source - assert "at::_softmax_out(at_out, at_self" in source - - -def test_torch_source_uses_existing_c10_stream_guards(): - module = _load_generator_module() - op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") - - method = module._generate_torch_method_source("abs", op) - source = module._generate_torch_source("abs", [op]) - - assert "stream_ == nullptr" not in method - assert "TorchStreamGuard stream_guard{stream_, device_index}" in method - - assert '#include "torch/stream_guard.h"' in source - assert "INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Abs, 8)" in source - assert '#include "torch/nvidia/c10.h"' not in source - - -def test_optional_tensor_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "batch_norm_elemt(Tensor input, Tensor? weight=None, " - "Tensor? bias=None, Tensor mean, Tensor invstd, float eps, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "std::optional", - "std::optional", - "Tensor", - "Tensor", - "double", - "Tensor", - ] - - base = module._generate_base_header("batch_norm_elemt", [op]) - source = module._generate_torch_method_source("batch_norm_elemt", op) - - assert "#include " in base - assert "std::optional weight" in base - assert "std::optional bias" in base - assert "bool has_weight_" in base - assert "bool has_bias_" in base - assert ( - "weight_shape_{weight ? Tensor::Shape{weight->shape()} : Tensor::Shape{}}" - in base - ) - assert ( - "weight_strides_{weight ? Tensor::Strides{weight->strides()} " - ": Tensor::Strides{}}" in base - ) - assert "c10::optional at_weight" in source - assert "c10::optional at_bias" in source - assert "weight->shape()" in source - assert "weight_shape_" not in source - assert "at::batch_norm_elemt_out" in source - assert "at_weight" in source - assert "at_bias" in source - - -def test_tensor_list_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "stack(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "std::vector", - "int64_t", - "Tensor", - ] - assert op.is_testable - - base = module._generate_base_header("stack", [op]) - source = module._generate_torch_method_source("stack", op) - - assert "#include " in base - assert "std::vector tensors" in base - assert "std::vector at_tensors" in source - assert "at_tensors.reserve(tensors.size())" in source - assert "for (const auto& tensor : tensors)" in source - assert "at::stack_out(at_out, at_tensors, dim)" in source - - -def test_optional_tensor_list_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "index.Tensor_out(Tensor self, Tensor?[] indices, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "std::vector>", - "Tensor", - ] - - source = module._generate_torch_method_source("index", op) - - assert "c10::List> at_indices" in source - assert "for (const auto& tensor : indices)" in source - assert "at_indices.push_back(c10::nullopt)" in source - assert "at::index_out(at_out, at_self, at_indices)" in source - - -def test_optional_scalar_and_array_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - quantile = module._parse_func( - "quantile(Tensor input, Tensor q, int? dim=None, bool keepdim=False, " - "str interpolation='linear', *, Tensor(a!) out) -> Tensor(a!)" - ) - upsample = module._parse_func( - "upsample_bicubic2d(Tensor input, SymInt[2] output_size, " - "bool align_corners, float[]? scale_factors=None, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in quantile.visible_params] == [ - "Tensor", - "Tensor", - "std::optional", - "bool", - "std::string", - "Tensor", - ] - assert [param.cpp_type for param in upsample.visible_params] == [ - "Tensor", - "std::vector", - "bool", - "std::optional>", - "Tensor", - ] - - quantile_source = module._generate_torch_method_source("quantile", quantile) - upsample_source = module._generate_torch_method_source( - "upsample_bicubic2d", upsample - ) - - assert "c10::optional at_dim" in quantile_source - assert "at::quantile_out" in quantile_source - assert "at_dim" in quantile_source - assert "c10::optional> at_scale_factors" in upsample_source - assert "at::upsample_bicubic2d_out" in upsample_source - assert "at_scale_factors" in upsample_source - - -def test_required_scalar_type_params_use_public_data_type(): - module = _load_generator_module() - op = module._parse_func( - "_softmax_backward_data(Tensor grad_output, Tensor output, int dim, " - "ScalarType input_dtype, *, Tensor(a!) grad_input) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "Tensor", - "int64_t", - "DataType", - "Tensor", - ] - - source = module._generate_torch_method_source("internal_softmax_backward_data", op) - - assert "at::_softmax_backward_data_out" in source - assert "ToAtenDataType(input_dtype)" in source - - -def test_existing_base_overload_can_omit_optional_schema_params(): - module = _load_generator_module() - op = module._parse_func( - "slow_conv3d(Tensor input, Tensor weight, int[3] kernel_size, " - "Tensor? bias=None, int[3] stride=1, int[3] padding=0, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("Tensor", "weight"), - ("std::vector", "kernel_size"), - ("std::vector", "stride"), - ("std::vector", "padding"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.name for param in bound.visible_params] == [ - "input", - "weight", - "kernel_size", - "stride", - "padding", - "out", - ] - - source = module._generate_torch_method_source("slow_conv3d", bound) - - assert "std::optional bias" not in source - assert "c10::optional{}" in source - assert "at::slow_conv3d_out" in source - - -def test_existing_base_overload_can_omit_defaulted_schema_params(): - module = _load_generator_module() - op = module._parse_func( - "add.Tensor(Tensor self, Tensor other, *, Scalar alpha=1, " - "Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("const Tensor", "input"), - ("const Tensor", "other"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - - source = module._generate_torch_method_source("add", bound) - - assert "double alpha" not in source - assert "const auto device_index = out.device().index();" in source - assert "device_index_)" not in source - assert "at::add_out(at_out, at_self, at_other, 1)" in source - - -def test_existing_base_overload_matches_by_name_when_types_repeat(): - module = _load_generator_module() - op = module._parse_func( - "std(Tensor input, int[1]? dim=None, bool unbiased=True, " - "bool keepdim=False, *, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("bool", "keepdim"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.name for param in bound.visible_params] == [ - "input", - "keepdim", - "out", - ] - - source = module._generate_torch_method_source("std", bound) - - assert "c10::optional{}, true, keepdim" in source - assert "unbiased" not in source - - -def test_existing_base_overload_can_group_tensor_inputs_before_attributes(): - module = _load_generator_module() - op = module._parse_func( - "gather.out(Tensor self, int dim, Tensor index, bool sparse_grad=False, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("Tensor", "index"), - ("int64_t", "dim"), - ("bool", "sparse_grad"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "index", - "dim", - "sparse_grad", - "out", - ] - - assert [ - next( - index - for index, binding in enumerate(bound.param_bindings) - if binding is param - ) - for param in bound.visible_params - ] == [0, 2, 1, 3, 4] - - source = module._generate_torch_method_source("gather", bound) - - assert "at::gather_out(at_out, at_self, dim, at_index, sparse_grad)" in source - - -def test_rrelu_with_noise_base_order_preserves_aten_call_order(): - module = _load_generator_module() - op = module._parse_func( - "rrelu_with_noise.out(Tensor self, Tensor(b!) noise, Scalar lower=0.125, " - "Scalar upper=0.3333333333333333, bool training=False, " - "Generator? generator=None, *, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("double", "lower"), - ("double", "upper"), - ("bool", "training"), - ("Tensor", "out"), - ("Tensor", "noise"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "lower", - "upper", - "training", - "out", - "noise", - ] - assert [ - next( - index - for index, binding in enumerate(bound.param_bindings) - if binding is param - ) - for param in bound.visible_params - ] == [0, 2, 3, 4, 6, 1] - - source = module._generate_torch_method_source("rrelu_with_noise", bound) - - assert "Tensor out, Tensor noise" in source - assert ( - "at::rrelu_with_noise_out(at_out, at_self, at_noise, lower, upper, " - "training, c10::optional{})" in source - ) - - -def test_write_text_if_changed_preserves_unchanged_mtime(tmp_path): - module = _load_generator_module() - path = tmp_path / "generated.cc" - path.write_text("same\n") - before = path.stat().st_mtime_ns - - assert module._write_text_if_changed(path, "same\n") is False - assert path.stat().st_mtime_ns == before - - assert module._write_text_if_changed(path, "different\n") is True - assert path.read_text() == "different\n" - assert path.stat().st_mtime_ns >= before - - -def test_remove_stale_files_keeps_expected_outputs(tmp_path): - module = _load_generator_module() - root = tmp_path / "generated" - keep = root / "torch" / "keep.cc" - stale = root / "torch" / "stale.cc" - nested_stale = root / "base" / "stale.h" - keep.parent.mkdir(parents=True) - nested_stale.parent.mkdir(parents=True) - keep.write_text("keep\n") - stale.write_text("stale\n") - nested_stale.write_text("stale\n") - - module._remove_stale_files(root, {keep}) - - assert keep.exists() - assert not stale.exists() - assert not nested_stale.exists() - assert not (root / "base").exists() - - -def test_inplace_schema_can_bind_to_explicit_output_signature(): - module = _load_generator_module() - op = module._parse_func("fill_.Scalar(Tensor(a!) self, Scalar value) -> Tensor(a!)") - signature = [ - ("Tensor", "input"), - ("double", "value"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert bound.explicit_inplace_out is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "value", - "out", - ] - assert bound.out_params[0].api_name == "out" - - source = module._generate_torch_method_source("fill", bound) - - assert "const auto device_index = out.device().index();" in source - assert "const_cast(input.data())" in source - assert "auto at_out = ToAtenTensor" in source - assert "at_out.copy_(at_self);" in source - assert "at_out.fill_(value);" in source - - implicit_bound = module._bind_base_signature(op, signature[:-1]) - - assert implicit_bound is not None - assert implicit_bound.explicit_inplace_out is None - - implicit_source = module._generate_torch_method_source("fill", implicit_bound) - assert "at_self.fill_(value);" in implicit_source +import importlib.util +import pathlib +import sys +import types + +import pytest + + +def _load_generator_module(): + path = ( + pathlib.Path(__file__).resolve().parents[1] + / "scripts" + / "generate_torch_ops.py" + ) + spec = importlib.util.spec_from_file_location("generate_torch_ops_under_test", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + return module + + +def test_select_op_names_honors_slot_8_and_legacy_headers(): + module = _load_generator_module() + config = { + "default_all": {"headers": None, "implementations": None}, + "default_native": {"headers": None, "implementations": (0,)}, + "explicit_torch": {"headers": None, "implementations": (8,)}, + "legacy": {"headers": ["legacy.h"], "implementations": None}, + } + + assert module._select_op_names( + None, + ["default_all", "default_native"], + config, + ) == ["default_all", "explicit_torch"] + assert module._select_op_names( + ["explicit_torch"], + ["default_all", "default_native"], + config, + ) == ["explicit_torch"] + + +def test_select_op_names_maps_public_names_to_aten_names(): + module = _load_generator_module() + config = { + "div": {"headers": None, "implementations": (8,)}, + "internal_log_softmax": {"headers": None, "implementations": (8,)}, + } + + assert module._select_op_names( + None, + ["div", "div_", "_log_softmax"], + config, + ) == ["div", "div_", "_log_softmax"] + + +def test_load_aten_entries_uses_packaged_torchgen(monkeypatch): + module = _load_generator_module() + entries = [{"func": "relu.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)"}] + monkeypatch.setattr(module, "_load_packaged_aten_entries", lambda: (entries, True)) + + assert module._load_aten_entries("v9.9.9") == entries + + +def test_packaged_aten_entries_include_autogenerated_out_variants(): + module = _load_generator_module() + entries, expanded_autogen = module._load_packaged_aten_entries() + + if not expanded_autogen: + pytest.skip("installed torchgen does not support autogen expansion") + + relu_out_entry = next( + entry + for entry in module._find_out_entries(entries, "relu") + if entry["func"].startswith("relu.out(") + ) + + assert module._parse_func(relu_out_entry["func"]).out_params + + +def test_packaged_aten_entries_fall_back_when_tags_are_missing( + monkeypatch, tmp_path, capsys +): + module = _load_generator_module() + native_dir = tmp_path / "packaged" / "ATen" / "native" + native_dir.mkdir(parents=True) + native_path = native_dir / "native_functions.yaml" + native_path.write_text( + "- func: add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)\n" + ) + spec = types.SimpleNamespace(submodule_search_locations=[str(tmp_path)]) + monkeypatch.setattr(module.importlib.util, "find_spec", lambda name: spec) + + entries, expanded_autogen = module._load_packaged_aten_entries() + + assert expanded_autogen is False + assert entries == [ + {"func": "add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)"} + ] + assert "using explicit schemas only" in capsys.readouterr().err + + +def test_public_op_name_normalizes_aten_internal_and_inplace_names(): + module = _load_generator_module() + + assert module._public_op_name("_softmax") == "internal_softmax" + assert module._public_op_name("add_") == "add" + assert module._public_op_name("_add_relu_") == "internal_add_relu" + + +def test_schema_self_param_renders_as_input_in_public_cpp_api(): + module = _load_generator_module() + op = module._parse_func( + "_softmax(Tensor self, int dim, bool half_to_float, *, " + "Tensor(a!) out) -> Tensor(a!)" + ) + + assert op.params[0].name == "self" + assert op.params[0].api_name == "input" + + base = module._generate_base_header("internal_softmax", [op]) + source = module._generate_torch_method_source("internal_softmax", op) + + assert "Softmax(const Tensor input, const int64_t dim" in base + assert "self_shape_" not in base + assert "input_shape_" in base + assert "auto at_self = ToAtenTensor" in source + assert "input_shape_" in source + assert "at::_softmax_out(at_out, at_self" in source + + +def test_torch_source_uses_existing_c10_stream_guards(): + module = _load_generator_module() + op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") + + method = module._generate_torch_method_source("abs", op) + source = module._generate_torch_source("abs", [op]) + + assert "stream_ == nullptr" not in method + assert "TorchStreamGuard stream_guard{stream_, device_index}" in method + + assert '#include "torch/stream_guard.h"' in source + assert "INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Abs, 8)" in source + assert '#include "torch/nvidia/c10.h"' not in source + + +def test_optional_tensor_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "batch_norm_elemt(Tensor input, Tensor? weight=None, " + "Tensor? bias=None, Tensor mean, Tensor invstd, float eps, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "std::optional", + "std::optional", + "Tensor", + "Tensor", + "double", + "Tensor", + ] + + base = module._generate_base_header("batch_norm_elemt", [op]) + source = module._generate_torch_method_source("batch_norm_elemt", op) + + assert "#include " in base + assert "std::optional weight" in base + assert "std::optional bias" in base + assert "bool has_weight_" in base + assert "bool has_bias_" in base + assert ( + "weight_shape_{weight ? Tensor::Shape{weight->shape()} : Tensor::Shape{}}" + in base + ) + assert ( + "weight_strides_{weight ? Tensor::Strides{weight->strides()} " + ": Tensor::Strides{}}" in base + ) + assert "c10::optional at_weight" in source + assert "c10::optional at_bias" in source + assert "weight->shape()" in source + assert "weight_shape_" not in source + assert "at::batch_norm_elemt_out" in source + assert "at_weight" in source + assert "at_bias" in source + + +def test_tensor_list_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "stack(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "std::vector", + "int64_t", + "Tensor", + ] + assert op.is_testable + + base = module._generate_base_header("stack", [op]) + source = module._generate_torch_method_source("stack", op) + + assert "#include " in base + assert "std::vector tensors" in base + assert "std::vector at_tensors" in source + assert "at_tensors.reserve(tensors.size())" in source + assert "for (const auto& tensor : tensors)" in source + assert "at::stack_out(at_out, at_tensors, dim)" in source + + +def test_optional_tensor_list_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "index.Tensor_out(Tensor self, Tensor?[] indices, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "std::vector>", + "Tensor", + ] + + source = module._generate_torch_method_source("index", op) + + assert "c10::List> at_indices" in source + assert "for (const auto& tensor : indices)" in source + assert "at_indices.push_back(c10::nullopt)" in source + assert "at::index_out(at_out, at_self, at_indices)" in source + + +def test_optional_scalar_and_array_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + quantile = module._parse_func( + "quantile(Tensor input, Tensor q, int? dim=None, bool keepdim=False, " + "str interpolation='linear', *, Tensor(a!) out) -> Tensor(a!)" + ) + upsample = module._parse_func( + "upsample_bicubic2d(Tensor input, SymInt[2] output_size, " + "bool align_corners, float[]? scale_factors=None, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in quantile.visible_params] == [ + "Tensor", + "Tensor", + "std::optional", + "bool", + "std::string", + "Tensor", + ] + assert [param.cpp_type for param in upsample.visible_params] == [ + "Tensor", + "std::vector", + "bool", + "std::optional>", + "Tensor", + ] + + quantile_source = module._generate_torch_method_source("quantile", quantile) + upsample_source = module._generate_torch_method_source( + "upsample_bicubic2d", upsample + ) + + assert "c10::optional at_dim" in quantile_source + assert "at::quantile_out" in quantile_source + assert "at_dim" in quantile_source + assert "c10::optional> at_scale_factors" in upsample_source + assert "at::upsample_bicubic2d_out" in upsample_source + assert "at_scale_factors" in upsample_source + + +def test_required_scalar_type_params_use_public_data_type(): + module = _load_generator_module() + op = module._parse_func( + "_softmax_backward_data(Tensor grad_output, Tensor output, int dim, " + "ScalarType input_dtype, *, Tensor(a!) grad_input) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "Tensor", + "int64_t", + "DataType", + "Tensor", + ] + + source = module._generate_torch_method_source("internal_softmax_backward_data", op) + + assert "at::_softmax_backward_data_out" in source + assert "ToAtenDataType(input_dtype)" in source + + +def test_existing_base_overload_can_omit_optional_schema_params(): + module = _load_generator_module() + op = module._parse_func( + "slow_conv3d(Tensor input, Tensor weight, int[3] kernel_size, " + "Tensor? bias=None, int[3] stride=1, int[3] padding=0, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("Tensor", "weight"), + ("std::vector", "kernel_size"), + ("std::vector", "stride"), + ("std::vector", "padding"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.name for param in bound.visible_params] == [ + "input", + "weight", + "kernel_size", + "stride", + "padding", + "out", + ] + + source = module._generate_torch_method_source("slow_conv3d", bound) + + assert "std::optional bias" not in source + assert "c10::optional{}" in source + assert "at::slow_conv3d_out" in source + + +def test_existing_base_overload_can_omit_defaulted_schema_params(): + module = _load_generator_module() + op = module._parse_func( + "add.Tensor(Tensor self, Tensor other, *, Scalar alpha=1, " + "Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("const Tensor", "input"), + ("const Tensor", "other"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + + source = module._generate_torch_method_source("add", bound) + + assert "double alpha" not in source + assert "const auto device_index = out.device().index();" in source + assert "device_index_)" not in source + assert "at::add_out(at_out, at_self, at_other, 1)" in source + + +def test_existing_base_overload_matches_by_name_when_types_repeat(): + module = _load_generator_module() + op = module._parse_func( + "std(Tensor input, int[1]? dim=None, bool unbiased=True, " + "bool keepdim=False, *, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("bool", "keepdim"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.name for param in bound.visible_params] == [ + "input", + "keepdim", + "out", + ] + + source = module._generate_torch_method_source("std", bound) + + assert "c10::optional{}, true, keepdim" in source + assert "unbiased" not in source + + +def test_existing_base_overload_can_group_tensor_inputs_before_attributes(): + module = _load_generator_module() + op = module._parse_func( + "gather.out(Tensor self, int dim, Tensor index, bool sparse_grad=False, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("Tensor", "index"), + ("int64_t", "dim"), + ("bool", "sparse_grad"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "index", + "dim", + "sparse_grad", + "out", + ] + + assert [ + next( + index + for index, binding in enumerate(bound.param_bindings) + if binding is param + ) + for param in bound.visible_params + ] == [0, 2, 1, 3, 4] + + source = module._generate_torch_method_source("gather", bound) + + assert "at::gather_out(at_out, at_self, dim, at_index, sparse_grad)" in source + + +def test_rrelu_with_noise_base_order_preserves_aten_call_order(): + module = _load_generator_module() + op = module._parse_func( + "rrelu_with_noise.out(Tensor self, Tensor(b!) noise, Scalar lower=0.125, " + "Scalar upper=0.3333333333333333, bool training=False, " + "Generator? generator=None, *, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("double", "lower"), + ("double", "upper"), + ("bool", "training"), + ("Tensor", "out"), + ("Tensor", "noise"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "lower", + "upper", + "training", + "out", + "noise", + ] + assert [ + next( + index + for index, binding in enumerate(bound.param_bindings) + if binding is param + ) + for param in bound.visible_params + ] == [0, 2, 3, 4, 6, 1] + + source = module._generate_torch_method_source("rrelu_with_noise", bound) + + assert "Tensor out, Tensor noise" in source + assert ( + "at::rrelu_with_noise_out(at_out, at_self, at_noise, lower, upper, " + "training, c10::optional{})" in source + ) + + +def test_write_text_if_changed_preserves_unchanged_mtime(tmp_path): + module = _load_generator_module() + path = tmp_path / "generated.cc" + path.write_text("same\n") + before = path.stat().st_mtime_ns + + assert module._write_text_if_changed(path, "same\n") is False + assert path.stat().st_mtime_ns == before + + assert module._write_text_if_changed(path, "different\n") is True + assert path.read_text() == "different\n" + assert path.stat().st_mtime_ns >= before + + +def test_remove_stale_files_keeps_expected_outputs(tmp_path): + module = _load_generator_module() + root = tmp_path / "generated" + keep = root / "torch" / "keep.cc" + stale = root / "torch" / "stale.cc" + nested_stale = root / "base" / "stale.h" + keep.parent.mkdir(parents=True) + nested_stale.parent.mkdir(parents=True) + keep.write_text("keep\n") + stale.write_text("stale\n") + nested_stale.write_text("stale\n") + + module._remove_stale_files(root, {keep}) + + assert keep.exists() + assert not stale.exists() + assert not nested_stale.exists() + assert not (root / "base").exists() + + +def test_inplace_schema_can_bind_to_explicit_output_signature(): + module = _load_generator_module() + op = module._parse_func("fill_.Scalar(Tensor(a!) self, Scalar value) -> Tensor(a!)") + signature = [ + ("Tensor", "input"), + ("double", "value"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert bound.explicit_inplace_out is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "value", + "out", + ] + assert bound.out_params[0].api_name == "out" + + source = module._generate_torch_method_source("fill", bound) + + assert "const auto device_index = out.device().index();" in source + assert "const_cast(input.data())" in source + assert "auto at_out = ToAtenTensor" in source + assert "at_out.copy_(at_self);" in source + assert "at_out.fill_(value);" in source + + implicit_bound = module._bind_base_signature(op, signature[:-1]) + + assert implicit_bound is not None + assert implicit_bound.explicit_inplace_out is None + + implicit_source = module._generate_torch_method_source("fill", implicit_bound) + assert "at_self.fill_(value);" in implicit_source diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py index a3efcd026..bb2801d5d 100644 --- a/tests/test_generate_wrappers.py +++ b/tests/test_generate_wrappers.py @@ -19,6 +19,30 @@ def _load_generator_module(): return module +def test_shared_validation_calls_are_not_parsed_as_constructors(monkeypatch, tmp_path): + module = _load_generator_module() + header = tmp_path / "cross_entropy.h" + header.write_text( + """ +class CrossEntropy : public Operator { + public: + explicit CrossEntropy(const Tensor input, Tensor out) { + ValidateCrossEntropy(input, out); + } + virtual void operator()(const Tensor input, Tensor out) const = 0; +}; +""" + ) + monkeypatch.setattr(module, "_find_base_header", lambda _: header) + parsed = module._parse_operator_header("cross_entropy") + assert len(parsed.constructors) == 1 + assert len(parsed.calls) == 1 + assert [arg.spelling for arg in parsed.constructors[0].get_arguments()] == [ + "input", + "out", + ] + + def test_generated_dispatch_keeps_scalar_and_optional_tensor_overloads_distinct( monkeypatch, tmp_path ): diff --git a/tests/test_masked_fill_backward.py b/tests/test_masked_fill_backward.py index 53f97cc35..07712f899 100644 --- a/tests/test_masked_fill_backward.py +++ b/tests/test_masked_fill_backward.py @@ -1,147 +1,464 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (shape, grad_output_strides, mask_strides, grad_input_strides, mask_mode) -_TEST_CASES = ( - pytest.param( - (17,), - None, - None, - None, - "alternating", - id="one-dimensional", - ), - pytest.param( - (3, 5), - None, - None, - None, - "all-false", - id="all-false-mask", - ), - pytest.param( - (3, 5), - None, - None, - None, - "all-true", - id="all-true-mask", - ), - pytest.param( - (2, 3, 4), - None, - None, - None, - "alternating", - id="three-dimensional", - ), - pytest.param( - (2, 3, 4), - (20, 6, 1), - (18, 5, 1), - (24, 7, 1), - "alternating", - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "shape", - "grad_output_strides", - "mask_strides", - "grad_input_strides", - "mask_mode", - ), - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_masked_fill_backward( - shape, - grad_output_strides, - mask_strides, - grad_input_strides, - mask_mode, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output = randn_strided( - shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - mask = _make_mask(shape, mask_strides, mask_mode, device) - grad_input = empty_strided( - shape, - grad_input_strides, - dtype=dtype, - device=device, - ) - grad_input.fill_(17) - - return Payload( - lambda *args: _masked_fill_backward( - *args, - implementation_index=implementation_index, - ), - _torch_masked_fill_backward, - (grad_output, mask, grad_input), - {}, - rtol=rtol, - atol=atol, - ) - - -def _make_mask(shape, strides, mode, device): - mask = empty_strided(shape, strides, dtype=torch.bool, device=device) - - if mode == "all-false": - mask.fill_(False) - elif mode == "all-true": - mask.fill_(True) - else: - values = torch.arange(mask.numel(), device=device).reshape(shape) % 3 == 0 - mask.copy_(values) - - return mask - - -def _masked_fill_backward( - grad_output, - mask, - grad_input, - *, - implementation_index, -): - infini.ops.masked_fill_backward( - grad_output, - mask, - grad_input, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return grad_input - - -def _torch_masked_fill_backward(grad_output, mask, grad_input): - grad_input.copy_(grad_output.masked_fill(mask, 0)) - - return grad_input +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream + + +# Format: (input_shape, mask_shape, grad_output_strides, mask_strides, +# grad_input_strides, mask_mode) +_TEST_CASES = ( + pytest.param((17,), (17,), None, None, None, "alternating", id="one-dimensional"), + pytest.param((3, 5), (3, 5), None, None, None, "all-false", id="all-false-mask"), + pytest.param((3, 5), (3, 5), None, None, None, "all-true", id="all-true-mask"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + "alternating", + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (20, 6, 1), + (18, 5, 1), + (24, 7, 1), + "alternating", + id="strided-tensors", + ), + pytest.param( + (2, 3), + (3,), + None, + None, + None, + "alternating", + id="column-mask-broadcast", + ), + pytest.param( + (2, 3), + (2, 1), + None, + None, + None, + "alternating", + id="explicit-row-mask-broadcast", + ), + pytest.param( + (2, 3, 4), + (4,), + None, + None, + None, + "alternating", + id="mask-broadcast-across-two-dims", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "alternating", + id="input-broadcast-reduction", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "all-false", + id="input-broadcast-all-false", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "all-true", + id="input-broadcast-all-true", + ), + pytest.param( + (3,), + (2, 3), + None, + None, + None, + "alternating", + id="input-missing-leading-dim", + ), + pytest.param( + (2, 1), + (1, 3), + None, + None, + None, + "alternating", + id="both-operands-broadcast", + ), + pytest.param( + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + "alternating", + id="middle-dim-reduction", + ), + pytest.param( + (1, 3, 1), + (2, 1, 4), + None, + None, + None, + "alternating", + id="multiple-dim-reduction", + ), + pytest.param( + (2, 3), + (1, 2, 3), + None, + None, + None, + "alternating", + id="mask-adds-leading-singleton", + ), + pytest.param( + (1, 3), + (2, 3), + (8, 2), + (7, 2), + (9, 2), + "alternating", + id="strided-reduction", + ), + pytest.param( + (2, 1), + (1, 3), + (8, 2), + (8, 2), + (3, 1), + "alternating", + id="strided-mutual-broadcast", + ), + pytest.param((), (), None, None, None, "all-false", id="scalar-all-false"), + pytest.param((), (), None, None, None, "all-true", id="scalar-all-true"), + pytest.param((2, 3), (), None, None, None, "all-false", id="scalar-mask-all-false"), + pytest.param((2, 3), (), None, None, None, "all-true", id="scalar-mask-all-true"), + pytest.param( + (), + (2, 3), + None, + None, + None, + "alternating", + id="scalar-input-reduction", + ), + pytest.param((0,), (0,), None, None, None, "alternating", id="empty-vector"), + pytest.param( + (0, 3), + (3,), + None, + None, + None, + "alternating", + id="empty-output-with-small-mask", + ), + pytest.param( + (1, 3), + (0, 3), + None, + None, + None, + "alternating", + id="empty-output-reduces-to-nonempty-input", + ), + pytest.param( + (2, 0, 4), + (1, 0, 1), + None, + None, + None, + "alternating", + id="empty-middle-dim", + ), + pytest.param( + (), + (0, 3), + None, + None, + None, + "alternating", + id="empty-output-reduces-to-scalar", + ), +) + +_DTYPES = (torch.float32, torch.float16, torch.bfloat16) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "input_shape", + "mask_shape", + "grad_output_strides", + "mask_strides", + "grad_input_strides", + "mask_mode", + ), + _TEST_CASES, +) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_masked_fill_backward( + input_shape, + mask_shape, + grad_output_strides, + mask_strides, + grad_input_strides, + mask_mode, + implementation_index, + dtype, + device, +): + output_shape = torch.broadcast_shapes(input_shape, mask_shape) + grad_output = _empty(output_shape, grad_output_strides, dtype, device) + grad_output.normal_() + if tuple(output_shape) != tuple(input_shape): + # Small signed integers keep all these sums exactly representable, + # including in FP16/BF16, so reduction tests can use zero tolerance. + values = torch.arange(grad_output.numel(), device=device) % 7 - 3 + grad_output.copy_(values.reshape(output_shape)) + + mask = _make_mask(mask_shape, mask_strides, mask_mode, device) + grad_input = _empty(input_shape, grad_input_strides, dtype, device) + grad_input.fill_(17) + + return Payload( + lambda *args: _masked_fill_backward( + *args, implementation_index=implementation_index + ), + _torch_masked_fill_backward, + (grad_output, mask, grad_input), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +def test_masked_fill_backward_known_reduction(implementation_index, dtype, device): + grad_output = torch.tensor([[1, 2, 3], [4, 5, 6]], dtype=dtype, device=device) + mask = torch.tensor([[False, True, False], [False, False, True]], device=device) + grad_input = torch.full((1, 3), 17, dtype=dtype, device=device) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + expected = torch.tensor([[5, 5, 3]], dtype=dtype, device=device) + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("input_shape", ((2, 3), (1, 3))) +@pytest.mark.parametrize("all_true", (False, True)) +def test_masked_fill_backward_nonfinite_gradients( + implementation_index, dtype, device, input_shape, all_true +): + grad_output = torch.tensor( + [[float("nan"), float("inf"), 3], [4, -float("inf"), float("nan")]], + dtype=dtype, + device=device, + ) + mask = torch.tensor([[True, True, False], [False, False, False]], device=device) + if all_true: + mask.fill_(True) + grad_input = torch.full(input_shape, 17, dtype=dtype, device=device) + expected = _torch_masked_fill_backward( + grad_output, mask, torch.empty_like(grad_input) + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0, equal_nan=True) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("input_shape", ((2, 3), (1, 3))) +def test_masked_fill_backward_expanded_readonly_inputs( + implementation_index, dtype, device, input_shape +): + # Both inputs have zero strides from expand(); output has ordinary storage. + grad_output = torch.tensor([[1, 2, 3]], dtype=dtype, device=device).expand(2, 3) + mask = torch.tensor([[False], [True]], device=device).expand(2, 3) + grad_input = torch.full(input_shape, 17, dtype=dtype, device=device) + expected = _torch_masked_fill_backward( + grad_output, mask, torch.empty_like(grad_input) + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +# Call the implementation directly: a Payload helper may clone arguments and +# destroy the storage overlap that these tests are intended to exercise. +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "alias_mode", + ("exact", "partial", "strided", "reduced", "mask"), +) +def test_masked_fill_backward_overlapping_storage( + implementation_index, dtype, device, alias_mode +): + storage = torch.arange(24, dtype=dtype, device=device) + grad_output = storage[:6].reshape(2, 3) + mask = _make_mask((2, 3), None, "alternating", device) + + if alias_mode == "exact": + grad_input = grad_output + elif alias_mode == "partial": + grad_input = storage[1:7].reshape(2, 3) + elif alias_mode == "strided": + grad_input = storage[1:13:2].reshape(2, 3) + elif alias_mode == "reduced": + grad_input = storage[1:4].reshape(1, 3) + else: + # Different dtypes can still refer to overlapping byte ranges. + grad_input = storage[:6].reshape(2, 3) + grad_input.fill_(17) + mask = storage.view(torch.bool)[:6].reshape(2, 3) + mask.copy_(_make_mask((2, 3), None, "alternating", device)) + grad_output = torch.arange(1, 7, dtype=dtype, device=device).reshape(2, 3) + + # Preserve the original values before the implementation writes its output. + expected = _torch_masked_fill_backward( + grad_output.detach().clone(), + mask.detach().clone(), + torch.empty(grad_input.shape, dtype=dtype, device=device), + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +# Format: (input_shape, mask_shape, supplied_grad_output_shape) +_INVALID_SHAPE_CASES = ( + pytest.param((2, 3), (2,), (2, 3), id="reject-implicit-row-alignment"), + pytest.param((2, 3), (2, 2), (2, 3), id="reject-incompatible-mask-dimension"), + pytest.param((2, 3), (2, 3), (3,), id="reject-missing-output-dimension"), + pytest.param((2, 3), (1, 2, 3), (2, 3), id="reject-missing-leading-singleton"), + pytest.param((1, 3), (3,), (2, 3), id="reject-unexplained-output-expansion"), + pytest.param((2, 3), (2, 3), (1, 3), id="reject-smaller-output"), + pytest.param((2, 3), (0, 3), (0, 3), id="reject-zero-versus-nonsingleton"), +) + + +@pytest.mark.parametrize( + ("input_shape", "mask_shape", "output_shape"), _INVALID_SHAPE_CASES +) +def test_masked_fill_backward_rejects_invalid_shapes( + implementation_index, device, input_shape, mask_shape, output_shape +): + setup = f"device = torch.device({str(device)!r})\n" + setup += f"grad_output = torch.ones({output_shape!r}, device=device)\n" + setup += f"mask = torch.zeros({mask_shape!r}, dtype=torch.bool, device=device)\n" + setup += f"grad_input = torch.empty({input_shape!r}, device=device)\n" + probe = setup + "infini.ops.MaskedFillBackward(grad_output, mask, grad_input)\n" + source = ( + setup + + f"infini.ops.masked_fill_backward(grad_output, mask, grad_input, stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "broadcast shape of grad_input and mask"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +def _empty(shape, strides, dtype, device): + if strides is None: + return torch.empty(shape, dtype=dtype, device=device) + return torch.empty_strided(shape, strides, dtype=dtype, device=device) + + +def _make_mask(shape, strides, mode, device): + mask = _empty(shape, strides, torch.bool, device) + if mode == "all-false": + mask.fill_(False) + elif mode == "all-true": + mask.fill_(True) + else: + values = torch.arange(mask.numel(), device=device).reshape(shape) % 3 == 0 + mask.copy_(values) + return mask + + +def _masked_fill_backward(grad_output, mask, grad_input, *, implementation_index): + infini.ops.masked_fill_backward( + grad_output, + mask, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return grad_input + + +def _torch_masked_fill_backward(grad_output, mask, grad_input): + # Use forward + autograd as the oracle, rather than reimplementing the + # masking/reduction formula under test. Snapshot inputs for alias safety. + saved_grad = grad_output.detach().clone() + saved_mask = mask.detach().clone() + with torch.enable_grad(): + input_tensor = torch.zeros( + grad_input.shape, + dtype=grad_input.dtype, + device=grad_input.device, + requires_grad=True, + ) + output = input_tensor.masked_fill(saved_mask, -7) + (expected,) = torch.autograd.grad(output, input_tensor, grad_outputs=saved_grad) + grad_input.copy_(expected) + return grad_input diff --git a/tests/test_mean_backward.py b/tests/test_mean_backward.py index 95db25fd7..9dc7e20a3 100644 --- a/tests/test_mean_backward.py +++ b/tests/test_mean_backward.py @@ -5,7 +5,7 @@ from tests.utils import Payload, empty_strided, get_stream, randn_strided -# Format: (input_shape, dim, keep_dim, grad_output_strides, grad_input_strides) +# Format: (input_shape, dim, keepdim, grad_output_strides, grad_input_strides) _TEST_CASES = ( pytest.param( (5,), @@ -68,7 +68,7 @@ @pytest.mark.auto_act_and_assert @pytest.mark.parametrize( - "input_shape, dim, keep_dim, grad_output_strides, grad_input_strides", + "input_shape, dim, keepdim, grad_output_strides, grad_input_strides", _TEST_CASES, ) @pytest.mark.parametrize( @@ -82,7 +82,7 @@ def test_mean_backward( input_shape, dim, - keep_dim, + keepdim, grad_output_strides, grad_input_strides, implementation_index, @@ -93,7 +93,7 @@ def test_mean_backward( ): normalized_dim = dim % len(input_shape) grad_output_shape = list(input_shape) - if keep_dim: + if keepdim: grad_output_shape[normalized_dim] = 1 else: del grad_output_shape[normalized_dim] @@ -118,7 +118,7 @@ def test_mean_backward( implementation_index=implementation_index, ), _torch_mean_backward, - (grad_output, dim, keep_dim, grad_input), + (grad_output, dim, keepdim, grad_input), {}, rtol=rtol, atol=atol, @@ -128,7 +128,7 @@ def test_mean_backward( def _mean_backward( grad_output, dim, - keep_dim, + keepdim, grad_input, *, implementation_index, @@ -136,7 +136,7 @@ def _mean_backward( infini.ops.mean_backward( grad_output, dim, - keep_dim, + keepdim, grad_input, stream=get_stream(grad_output.device), implementation_index=implementation_index, @@ -145,12 +145,12 @@ def _mean_backward( return grad_input -def _torch_mean_backward(grad_output, dim, keep_dim, grad_input): +def _torch_mean_backward(grad_output, dim, keepdim, grad_input): normalized_dim = dim % grad_input.ndim reduce_size = grad_input.shape[normalized_dim] expanded_grad = grad_output - if not keep_dim: + if not keepdim: expanded_grad = expanded_grad.unsqueeze(normalized_dim) grad_input.copy_(expanded_grad.expand_as(grad_input) / reduce_size) diff --git a/tests/test_mul_backward.py b/tests/test_mul_backward.py index 888a585dd..08ae1b9d0 100644 --- a/tests/test_mul_backward.py +++ b/tests/test_mul_backward.py @@ -1,135 +1,465 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (shape, grad_output_strides, a_strides, b_strides, grad_a_strides, -# grad_b_strides) -_TEST_CASES = ( - pytest.param( - (17,), - None, - None, - None, - None, - None, - id="one-dimensional", - ), - pytest.param( - (2, 3), - None, - None, - None, - None, - None, - id="two-dimensional", - ), - pytest.param( - (2, 3, 4), - None, - None, - None, - None, - None, - id="three-dimensional", - ), - pytest.param( - (2, 3, 4), - (30, 7, 1), - (28, 6, 1), - (32, 5, 1), - (35, 8, 1), - (36, 9, 1), - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "shape", - "grad_output_strides", - "a_strides", - "b_strides", - "grad_a_strides", - "grad_b_strides", - ), - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-6, 1e-6), - (torch.float16, 1e-3, 1e-3), - (torch.bfloat16, 1e-2, 1e-2), - ), -) -def test_mul_backward( - shape, - grad_output_strides, - a_strides, - b_strides, - grad_a_strides, - grad_b_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output = randn_strided( - shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - a = randn_strided(shape, a_strides, dtype=dtype, device=device) - b = randn_strided(shape, b_strides, dtype=dtype, device=device) - grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) - grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) - grad_a.fill_(17) - grad_b.fill_(17) - - return Payload( - lambda *args: _mul_backward( - *args, - implementation_index=implementation_index, - ), - _torch_mul_backward, - (grad_output, a, b, grad_a, grad_b), - {}, - rtol=rtol, - atol=atol, - ) - - -def _mul_backward( - grad_output, - a, - b, - grad_a, - grad_b, - *, - implementation_index, -): - infini.ops.mul_backward( - grad_output, - a, - b, - grad_a, - grad_b, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) - - -def _torch_mul_backward(grad_output, a, b, grad_a, grad_b): - grad_a.copy_(grad_output * b) - grad_b.copy_(grad_output * a) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_DTYPES = ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + +# Format: (shape, grad_output_strides, a_strides, b_strides, +# grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), None, None, None, None, None, id="one-dimensional"), + pytest.param((2, 3), None, None, None, None, None, id="two-dimensional"), + pytest.param((2, 3, 4), None, None, None, None, None, id="three-dimensional"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (28, 6, 1), + (32, 5, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "grad_output_strides", + "a_strides", + "b_strides", + "grad_a_strides", + "grad_b_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_mul_backward( + shape, + grad_output_strides, + a_strides, + b_strides, + grad_a_strides, + grad_b_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + a = randn_strided(shape, a_strides, dtype=dtype, device=device) + b = randn_strided(shape, b_strides, dtype=dtype, device=device) + grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) + grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _mul_backward( + *args, + implementation_index=implementation_index, + ), + _torch_mul_backward, + (grad_output, a, b, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + *, + implementation_index, +): + infini.ops.mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +def _torch_mul_backward(grad_output, a, b, grad_a, grad_b): + # Finish both computations before writing potentially aliased outputs. + result_a = (grad_output * b).sum_to_size(a.shape) + result_b = (grad_output * a).sum_to_size(b.shape) + grad_a.copy_(result_a) + grad_b.copy_(result_b) + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +# Alias tests deliberately call the operator directly. A harness that clones +# each argument independently may destroy alias relationships. +def _reference_gradients(grad_output, a, b): + # Snapshot BEFORE calling the operator: it may overwrite these inputs. + with torch.enable_grad(): + a_ref = a.detach().clone().requires_grad_(True) + b_ref = b.detach().clone().requires_grad_(True) + return torch.autograd.grad( + a_ref * b_ref, + (a_ref, b_ref), + grad_output.detach().clone(), + ) + + +def _assert_gradients(grad_a, grad_b, expected, rtol, atol): + torch.testing.assert_close(grad_a, expected[0], rtol=rtol, atol=atol) + torch.testing.assert_close(grad_b, expected[1], rtol=rtol, atol=atol) + + +def _make_inputs(shape, dtype, device): + return { + name: randn_strided(shape, None, dtype=dtype, device=device) + for name in ("grad_output", "a", "b") + } + + +def _backing(dtype, device): + # Small, exactly representable values, with space for guard elements. + return (torch.arange(40, dtype=torch.float32, device=device) / 8 - 2).to( + dtype=dtype, + ) + + +def _alias_views(backing, layout): + # All views have shape (2, 3) and internally non-overlapping elements. + # Offset 2 leaves leading guard elements in the backing allocation. + if layout == "full": + x = backing.as_strided((2, 3), (3, 1), 2) + y = backing.as_strided((2, 3), (3, 1), 2) + elif layout == "partial": + x = backing.as_strided((2, 3), (3, 1), 2) + y = backing.as_strided((2, 3), (3, 1), 4) + elif layout == "transposed": + x = backing.as_strided((2, 3), (1, 2), 2) + y = backing.as_strided((2, 3), (3, 1), 2) + elif layout == "strided-partial": + x = backing.as_strided((2, 3), (6, 2), 2) + y = backing.as_strided((2, 3), (6, 2), 4) + else: + raise AssertionError(layout) + return x, y + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_mul_backward_review_regression( + dtype, + rtol, + atol, + device, + implementation_index, +): + a = torch.tensor([2, 3], dtype=dtype, device=device) + b = torch.tensor([5, 7], dtype=dtype, device=device) + grad_output = torch.ones_like(a) + grad_a = a.view_as(a) + grad_b = torch.full_like(b, 17) + + _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + torch.testing.assert_close( + grad_a, + torch.tensor([5, 7], dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + torch.testing.assert_close( + grad_b, + torch.tensor([2, 3], dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("source", ("a", "b", "grad_output")) +@pytest.mark.parametrize("destination", ("grad_a", "grad_b")) +@pytest.mark.parametrize( + "layout", + ("full", "partial", "transposed", "strided-partial"), +) +@pytest.mark.parametrize("reverse", (False, True), ids=("forward", "reverse")) +def test_mul_backward_input_output_alias( + source, + destination, + layout, + reverse, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + backing = _backing(dtype, device) + input_view, output_view = _alias_views(backing, layout) + if reverse: + input_view, output_view = output_view, input_view + inputs[source] = input_view + outputs = { + name: torch.full((2, 3), 17, dtype=dtype, device=device) + for name in ("grad_a", "grad_b") + } + outputs[destination] = output_view + + expected = _reference_gradients(**inputs) + expected_backing = backing.clone() + expected_view = expected_backing.as_strided( + output_view.shape, + output_view.stride(), + output_view.storage_offset(), + ) + expected_view.copy_(expected[0 if destination == "grad_a" else 1]) + unchanged_inputs = { + name: tensor.clone() for name, tensor in inputs.items() if name != source + } + + _mul_backward(**inputs, **outputs, implementation_index=implementation_index) + + _assert_gradients(outputs["grad_a"], outputs["grad_b"], expected, rtol, atol) + # Include gaps and guard elements, not just the logical output elements. + torch.testing.assert_close(backing, expected_backing, rtol=rtol, atol=atol) + for name, snapshot in unchanged_inputs.items(): + torch.testing.assert_close(inputs[name], snapshot, rtol=0, atol=0) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize( + ("source_a", "source_b"), + ( + ("a", "b"), + ("b", "a"), + ("grad_output", "a"), + ("grad_output", "b"), + ("a", "grad_output"), + ("b", "grad_output"), + ), +) +def test_mul_backward_both_outputs_alias_inputs( + source_a, + source_b, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + grad_a = inputs[source_a].view_as(inputs[source_a]) + grad_b = inputs[source_b].view_as(inputs[source_b]) + expected = _reference_gradients(**inputs) + + _mul_backward( + **inputs, + grad_a=grad_a, + grad_b=grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("destination", ("separate", "grad_a", "grad_b")) +def test_mul_backward_all_inputs_alias( + destination, + dtype, + rtol, + atol, + device, + implementation_index, +): + shared = _backing(dtype, device).as_strided((2, 3), (6, 2), 2) + inputs = { + name: shared.as_strided(shared.shape, shared.stride()) + for name in ("grad_output", "a", "b") + } + outputs = { + name: torch.full((2, 3), 17, dtype=dtype, device=device) + for name in ("grad_a", "grad_b") + } + if destination != "separate": + outputs[destination] = shared + expected = _reference_gradients(**inputs) + + _mul_backward(**inputs, **outputs, implementation_index=implementation_index) + + _assert_gradients(outputs["grad_a"], outputs["grad_b"], expected, rtol, atol) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize( + ("a_shape", "b_shape"), + (((2, 3, 4), (4,)), ((2, 1, 4), (1, 3, 1))), +) +@pytest.mark.parametrize("alias", ("none", "a", "b", "both", "grad-to-a", "grad-to-b")) +def test_mul_backward_broadcast_alias( + a_shape, + b_shape, + alias, + dtype, + rtol, + atol, + device, + implementation_index, +): + a = randn_strided(a_shape, None, dtype=dtype, device=device) + b = randn_strided(b_shape, None, dtype=dtype, device=device) + grad_output = randn_strided((2, 3, 4), None, dtype=dtype, device=device) + grad_a = torch.full(a_shape, 17, dtype=dtype, device=device) + grad_b = torch.full(b_shape, 17, dtype=dtype, device=device) + if alias in ("a", "both"): + grad_a = a.view_as(a) + if alias in ("b", "both"): + grad_b = b.view_as(b) + if alias == "grad-to-a": + grad_a = ( + grad_output.view_as(grad_output) + if a_shape == (2, 3, 4) + else grad_output[:, :1, :] + ) + if alias == "grad-to-b": + grad_b = grad_output[0, 0, :] if b_shape == (4,) else grad_output[:1, :, :1] + expected = _reference_gradients(grad_output, a, b) + + _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + + +@pytest.mark.parametrize("layout", ("full", "partial", "strided", "interleaved")) +@pytest.mark.parametrize("reverse", (False, True)) +def test_mul_backward_reject_output_overlap( + layout, reverse, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = ( + setup + "infini.ops.MulBackward(x, x, x, torch.empty(4, device=device), x)\n" + ) + stride = (6, 2) if layout in ("strided", "interleaved") else (3, 1) + offset = {"full": 2, "partial": 4, "strided": 4, "interleaved": 3}[layout] + offsets = (offset, 2) if reverse else (2, offset) + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {stride!r}, {offsets[0]}) +b = backing.as_strided((2, 3), {stride!r}, {offsets[1]}) +infini.ops.mul_backward(x, x, x, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("layout", ("adjacent", "strided-disjoint")) +@pytest.mark.parametrize("reverse", (False, True), ids=("forward", "reverse")) +def test_mul_backward_disjoint_outputs_in_same_storage( + layout, + reverse, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + backing = _backing(dtype, device) + if layout == "adjacent": + # Half-open ranges meet at the boundary; this is not overlap. + grad_a = backing.as_strided((2, 3), (3, 1), 2) + grad_b = backing.as_strided((2, 3), (3, 1), 8) + else: + grad_a = backing.as_strided((2, 3), (6, 2), 2) + grad_b = backing.as_strided((2, 3), (6, 2), 18) + if reverse: + grad_a, grad_b = grad_b, grad_a + expected = _reference_gradients(**inputs) + expected_backing = backing.clone() + for output, result in zip((grad_a, grad_b), expected): + expected_backing.as_strided( + output.shape, + output.stride(), + output.storage_offset(), + ).copy_(result) + + _mul_backward( + **inputs, + grad_a=grad_a, + grad_b=grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + torch.testing.assert_close(backing, expected_backing, rtol=rtol, atol=atol) diff --git a/tests/test_pow_backward.py b/tests/test_pow_backward.py index 74f88a31e..b9a59bb36 100644 --- a/tests/test_pow_backward.py +++ b/tests/test_pow_backward.py @@ -5,62 +5,117 @@ from tests.utils import Payload, empty_strided, get_stream, randn_strided -# Format: (shape, scalar, scalar_is_base, grad_output_strides, input_strides, -# grad_input_strides) _TEST_CASES = ( + pytest.param((17,), 2.0, False, None, None, None, id="one-dimensional-input-base"), + pytest.param((2, 3), 3.0, False, None, None, None, id="two-dimensional-input-base"), + pytest.param((2, 3, 4), 0.5, False, None, None, None, id="fractional-input-base"), + pytest.param((2, 3, 4), 2.0, True, None, None, None, id="scalar-base"), + pytest.param((2, 3, 4), 0.5, True, None, None, None, id="fractional-scalar-base"), pytest.param( - (17,), + (2, 3, 4), 3.0, False, (30, 7, 1), (28, 6, 1), (35, 8, 1), id="strided-tensors" + ), +) + +_DTYPE_CASES = ( + pytest.param(torch.float32, 1e-6, 1e-6, id="float32"), + pytest.param(torch.float16, 1e-3, 1e-3, id="float16"), + pytest.param(torch.bfloat16, 1e-2, 1e-2, id="bfloat16"), +) + +_EDGE_CASES = ( + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 0.0, + False, + id="zero-exponent-including-zero-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 1.0, + False, + id="unit-exponent-including-zero-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), 2.0, False, - None, - None, - None, - id="one-dimensional-input-base", + id="square-negative-and-zero-input", ), pytest.param( - (2, 3), + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), 3.0, False, - None, - None, - None, - id="two-dimensional-input-base", + id="cube-negative-and-zero-input", ), pytest.param( - (2, 3, 4), + (-2.0, -1.0, 1.0, 2.0), + (1.0, -2.0, 0.0, 3.0), + -2.0, + False, + id="negative-integer-exponent", + ), + pytest.param( + (0.25, 1.0, 4.0), + (2.0, 0.0, -2.0), 0.5, False, - None, - None, - None, - id="fractional-input-base", + id="fractional-exponent-positive-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 1.0, + True, + id="unit-scalar-base-zero-derivative", ), pytest.param( - (2, 3, 4), + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), 2.0, True, - None, - None, - None, - id="scalar-base", + id="scalar-base-negative-zero-positive-exponents", ), pytest.param( - (2, 3, 4), + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), 0.5, True, - None, - None, - None, - id="fractional-scalar-base", + id="fractional-scalar-base-negative-derivative", ), +) + +_NUMERICAL_CASES = ( pytest.param( - (2, 3, 4), - 3.0, + (0.5,), + (40000.0,), + 2.0, False, - (30, 7, 1), - (28, 6, 1), - (35, 8, 1), - id="strided-tensors", + id="input-base-large-grad-finite-result", + ), + pytest.param( + (1.0,), + (40000.0,), + 2.0, + True, + id="scalar-base-large-grad-finite-result", + ), + pytest.param( + (-16.0,), + (1.0,), + 0.5, + True, + id="scalar-base-pow-overflow-finite-gradient", + ), + pytest.param( + (0.5,), + (-40000.0,), + 2.0, + False, + id="input-base-large-negative-grad", ), ) @@ -77,14 +132,7 @@ ), _TEST_CASES, ) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 1e-6, 1e-6), - (torch.float16, 1e-3, 1e-3), - (torch.bfloat16, 1e-2, 1e-2), - ), -) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) def test_pow_backward( shape, scalar, @@ -104,12 +152,7 @@ def test_pow_backward( dtype=dtype, device=device, ) - input = randn_strided( - shape, - input_strides, - dtype=dtype, - device=device, - ) + input = randn_strided(shape, input_strides, dtype=dtype, device=device) input.abs_().add_(0.5) grad_input = empty_strided( shape, @@ -132,6 +175,81 @@ def test_pow_backward( ) +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ("input_values", "grad_output_values", "scalar", "scalar_is_base"), + _EDGE_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) +def test_pow_backward_edge_cases( + input_values, + grad_output_values, + scalar, + scalar_is_base, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = torch.tensor(input_values, dtype=dtype, device=device) + grad_output = torch.tensor(grad_output_values, dtype=dtype, device=device) + grad_input = torch.full_like(input, 17) + + return Payload( + lambda *args: _pow_backward( + *args, + implementation_index=implementation_index, + ), + _torch_pow_backward, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ("input_values", "grad_output_values", "scalar", "scalar_is_base"), + _NUMERICAL_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) +def test_pow_backward_numerical_stability( + input_values, + grad_output_values, + scalar, + scalar_is_base, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = torch.tensor(input_values, dtype=dtype, device=device) + grad_output = torch.tensor(grad_output_values, dtype=dtype, device=device) + grad_input = torch.full_like(input, 17) + + def actual(*args): + result = _pow_backward( + *args, + implementation_index=implementation_index, + ) + assert torch.isfinite(result).all().item(), ( + "Expected a finite gradient, but got inf or nan" + ) + return result + + return Payload( + actual, + _torch_pow_backward_high_precision, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + def _pow_backward( grad_output, input, @@ -150,7 +268,6 @@ def _pow_backward( stream=get_stream(grad_output.device), implementation_index=implementation_index, ) - return grad_input @@ -162,7 +279,6 @@ def _torch_pow_backward( grad_input, ): differentiable_input = input.detach().clone().requires_grad_(True) - if scalar_is_base: result = torch.pow(scalar, differentiable_input) else: @@ -170,5 +286,32 @@ def _torch_pow_backward( gradient = torch.autograd.grad(result, differentiable_input, grad_output)[0] grad_input.copy_(gradient) + return grad_input + +def _torch_pow_backward_high_precision( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, +): + reference_input = input.detach().to(device="cpu", dtype=torch.float64) + reference_grad_output = grad_output.detach().to( + device="cpu", + dtype=torch.float64, + ) + reference_grad_input = torch.empty_like(reference_input) + gradient = _torch_pow_backward( + reference_grad_output, + reference_input, + scalar, + scalar_is_base, + reference_grad_input, + ) + expected = gradient.to(device=grad_input.device, dtype=grad_input.dtype) + assert torch.isfinite(expected).all().item(), ( + "The regression case must have a finite expected gradient" + ) + grad_input.copy_(expected) return grad_input diff --git a/tests/test_rsqrt_backward.py b/tests/test_rsqrt_backward.py index f6ec99142..93a19d126 100644 --- a/tests/test_rsqrt_backward.py +++ b/tests/test_rsqrt_backward.py @@ -1,3 +1,5 @@ +import decimal + import infini.ops import pytest import torch @@ -115,9 +117,45 @@ def _rsqrt_backward( def _torch_rsqrt_backward(grad_output, input, grad_input): - differentiable_input = input.detach().clone().requires_grad_(True) + differentiable_input = ( + input.detach().to(device="cpu", dtype=torch.float64).requires_grad_(True) + ) result = torch.rsqrt(differentiable_input) - gradient = torch.autograd.grad(result, differentiable_input, grad_output)[0] + upstream = grad_output.detach().to(device="cpu", dtype=torch.float64) + gradient = torch.autograd.grad(result, differentiable_input, upstream)[0] grad_input.copy_(gradient) return grad_input + + +@pytest.mark.parametrize( + "dtype,value,upstream", + ( + (torch.float16, 2**-12, 0.001), + (torch.float16, 2**-24, 2**-24), + (torch.bfloat16, 2**-120, 2**-100), + (torch.float32, 2**-100, 2**-80), + (torch.float64, 2**-800, 2**-700), + ), +) +def test_rsqrt_backward_finite_intermediates( + dtype, value, upstream, device, implementation_index +): + input = torch.full((2, 3), value, dtype=dtype, device=device) + grad_output = torch.full_like(input, upstream) + backing = torch.full((32,), 17, dtype=dtype, device=device) + output = backing.as_strided((2, 3), (6, 2), 2) + + with decimal.localcontext() as context: + context.prec = 80 + x = decimal.Decimal.from_float(input[0, 0].item()) + g = decimal.Decimal.from_float(grad_output[0, 0].item()) + expected_value = float(-g / (2 * x * x.sqrt())) + + expected = backing.clone() + expected.as_strided((2, 3), (6, 2), 2).fill_(expected_value) + _rsqrt_backward( + grad_output, input, output, implementation_index=implementation_index + ) + assert torch.isfinite(output).all().item() + torch.testing.assert_close(backing, expected, rtol=0, atol=0) diff --git a/tests/test_slice.py b/tests/test_slice.py deleted file mode 100644 index bdee862ba..000000000 --- a/tests/test_slice.py +++ /dev/null @@ -1,134 +0,0 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -_TEST_CASES = ( - pytest.param( - (17,), - (5,), - None, - None, - (1,), - (16,), - (3,), - id="one-dimensional", - ), - pytest.param( - (6, 9), - (3, 3), - (12, 1), - (5, 1), - (1, 0), - (6, 9), - (2, 3), - id="strided-two-dimensional", - ), - pytest.param( - (4, 6, 7), - (2, 3, 3), - (60, 9, 1), - (20, 5, 1), - (-4, 1, 0), - (-1, 6, 7), - (2, 2, 3), - id="negative-indices", - ), - pytest.param( - (2, 3, 4, 5), - (2, 2, 3, 3), - None, - (24, 12, 4, 1), - (0, 0, 1, 0), - (2, 3, 4, 5), - (1, 2, 1, 2), - id="four-dimensional", - ), - pytest.param( - (5, 7), - (0, 7), - None, - None, - (3, 0), - (3, 7), - (1, 1), - id="empty-slice", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "input_shape", - "out_shape", - "input_strides", - "out_strides", - "starts", - "ends", - "steps", - ), - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_slice( - input_shape, - out_shape, - input_strides, - out_strides, - starts, - ends, - steps, - implementation_index, - dtype, - device, - rtol, - atol, -): - input = randn_strided(input_shape, input_strides, dtype=dtype, device=device) - out = empty_strided(out_shape, out_strides, dtype=dtype, device=device) - - return Payload( - lambda *args: _slice(*args, implementation_index=implementation_index), - _torch_slice, - (input, starts, ends, steps, out), - {}, - rtol=rtol, - atol=atol, - ) - - -def _slice(input, starts, ends, steps, out, *, implementation_index): - infini.ops.slice( - input, - starts, - ends, - steps, - out, - stream=get_stream(input.device), - implementation_index=implementation_index, - ) - - return out - - -def _torch_slice(input, starts, ends, steps, out): - out.copy_(input[_slice_index(starts, ends, steps)]) - - return out - - -def _slice_index(starts, ends, steps): - return tuple( - slice(start, end, step) - for start, end, step in zip(starts, ends, steps, strict=True) - ) diff --git a/tests/test_slice_backward.py b/tests/test_slice_backward.py index 8f90bd349..1a41dbcd2 100644 --- a/tests/test_slice_backward.py +++ b/tests/test_slice_backward.py @@ -1,155 +1,92 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -_TEST_CASES = ( - pytest.param( - (17,), - (5,), - None, - None, - (1,), - (16,), - (3,), - id="one-dimensional", - ), - pytest.param( - (6, 9), - (3, 3), - (12, 1), - (5, 1), - (1, 0), - (6, 9), - (2, 3), - id="strided-two-dimensional", - ), - pytest.param( - (4, 6, 7), - (2, 3, 3), - (60, 9, 1), - (20, 5, 1), - (-4, 1, 0), - (-1, 6, 7), - (2, 2, 3), - id="negative-indices", - ), - pytest.param( - (2, 3, 4, 5), - (2, 2, 3, 3), - (80, 25, 6, 1), - None, - (0, 0, 1, 0), - (2, 3, 4, 5), - (1, 2, 1, 2), - id="four-dimensional", - ), - pytest.param( - (5, 7), - (0, 7), - None, - None, - (3, 0), - (3, 7), - (1, 1), - id="empty-slice", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "grad_input_shape", - "grad_output_shape", - "grad_input_strides", - "grad_output_strides", - "starts", - "ends", - "steps", - ), - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_slice_backward( - grad_input_shape, - grad_output_shape, - grad_input_strides, - grad_output_strides, - starts, - ends, - steps, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output = randn_strided( - grad_output_shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - grad_input = empty_strided( - grad_input_shape, - grad_input_strides, - dtype=dtype, - device=device, - ) - grad_input.fill_(17) - - return Payload( - lambda *args: _slice_backward(*args, implementation_index=implementation_index), - _torch_slice_backward, - (grad_output, starts, ends, steps, grad_input), - {}, - rtol=rtol, - atol=atol, - ) - - -def _slice_backward( - grad_output, - starts, - ends, - steps, - grad_input, - *, - implementation_index, -): - infini.ops.slice_backward( - grad_output, - starts, - ends, - steps, - grad_input, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return grad_input - - -def _torch_slice_backward(grad_output, starts, ends, steps, grad_input): - grad_input.zero_() - grad_input[_slice_index(starts, ends, steps)] = grad_output - - return grad_input - - -def _slice_index(starts, ends, steps): - assert len(starts) == len(ends) == len(steps) - - return tuple( - slice(start, end, step) for start, end, step in zip(starts, ends, steps) - ) +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_CASES = ( + ((17,), None, 0, 1, 16, 3), + ((6, 9), (12, 1), 0, 1, 6, 2), + ((6, 9), (12, 1), 1, 0, 9, 3), + ((4, 6, 7), (60, 9, 1), -2, -5, 6, 2), + ((2, 3, 4, 5), None, 3, 0, 5, 2), + ((5, 7), None, 0, 3, 3, 1), + ((3, 5), None, -1, -100, 100, 2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strides,dim,start,end,step", _CASES) +@pytest.mark.parametrize("strided_upstream", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_slice_backward( + shape, + strides, + dim, + start, + end, + step, + strided_upstream, + dtype, + device, + implementation_index, +): + grad_input = empty_strided(shape, strides, dtype=dtype, device=device) + index = [slice(None)] * len(shape) + index[dim] = slice(start, end, step) + output_shape = grad_input[tuple(index)].shape + output_strides = ( + tuple(stride * 2 for stride in torch.empty(output_shape).stride()) + if strided_upstream + else None + ) + grad_output = randn_strided( + output_shape, output_strides, dtype=dtype, device=device + ) + grad_input.fill_(17) + + def actual(grad_output, grad_input): + infini.ops.slice_backward( + grad_output, + shape, + dim, + start, + end, + step, + grad_input, + stream=get_stream(device), + implementation_index=implementation_index, + ) + return grad_input + + def reference(grad_output, grad_input): + x = torch.zeros(shape, dtype=dtype, device=device, requires_grad=True) + with torch.enable_grad(): + result = x[tuple(index)] + gradient = torch.autograd.grad(result, x, grad_output)[0] + grad_input.copy_(gradient) + return grad_input + + return Payload(actual, reference, (grad_output, grad_input), {}, rtol=0, atol=0) + + +@pytest.mark.parametrize("offset", (0, 2)) +def test_slice_backward_alias(offset, device, implementation_index): + backing = torch.arange(32, dtype=torch.float32, device=device) + grad_output = backing[offset : offset + 6].reshape(3, 2) + out = backing[:12].reshape(3, 4) + expected = torch.zeros_like(out) + expected[:, ::2] = grad_output.clone() + infini.ops.slice_backward( + grad_output, + (3, 4), + 1, + 0, + 4, + 2, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) diff --git a/tests/test_slice_copy.py b/tests/test_slice_copy.py new file mode 100644 index 000000000..116838fc5 --- /dev/null +++ b/tests/test_slice_copy.py @@ -0,0 +1,84 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +_CASES = ( + ((17,), None, 0, 1, 16, 3), + ((6, 9), (12, 1), 0, 1, 6, 2), + ((6, 9), (12, 1), 1, 0, 9, 3), + ((4, 6, 7), (60, 9, 1), -2, -5, 6, 2), + ((2, 3, 4, 5), None, 3, 0, 5, 2), + ((5, 7), None, 0, 3, 3, 1), + ((3, 5), None, -1, -100, 100, 2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strides,dim,start,end,step", _CASES) +@pytest.mark.parametrize("optional_bounds", (False, True)) +@pytest.mark.parametrize("strided_output", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_slice_copy( + shape, + strides, + dim, + start, + end, + step, + optional_bounds, + strided_output, + dtype, + device, + implementation_index, +): + input = randn_strided(shape, strides, dtype=dtype, device=device) + start, end = (None, None) if optional_bounds else (start, end) + index = [slice(None)] * len(shape) + index[dim] = slice(start, end, step) + expected_shape = input[tuple(index)].shape + out_strides = tuple( + stride * (2 if strided_output else 1) + for stride in torch.empty(expected_shape).stride() + ) + out = torch.empty_strided(expected_shape, out_strides, dtype=dtype, device=device) + + def actual(input, out): + infini.ops.slice_copy( + input, + dim, + start, + end, + step, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + return out + + def reference(input, out): + out.copy_(input[tuple(index)].clone()) + return out + + return Payload(actual, reference, (input, out), {}, rtol=0, atol=0) + + +@pytest.mark.parametrize("offset", (0, 2)) +def test_slice_copy_alias(offset, device, implementation_index): + backing = torch.arange(32, dtype=torch.float32, device=device) + input = backing[:12].reshape(3, 4) + out = backing[offset : offset + 6].reshape(3, 2) + expected = input[:, ::2].clone() + infini.ops.slice_copy( + input, + 1, + None, + None, + 2, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) diff --git a/tests/test_split.py b/tests/test_split.py index 01b7b18fd..c23ece814 100644 --- a/tests/test_split.py +++ b/tests/test_split.py @@ -1,3 +1,7 @@ +import json +import subprocess +import sys + import infini.ops import pytest import torch @@ -5,6 +9,64 @@ from tests.utils import Payload, empty_strided, get_stream, randn_strided +def test_split_alias(device, implementation_index): + input = torch.arange(12, dtype=torch.float32, device=device).reshape(4, 3) + expected = input.clone() + outputs = [input[2:], input[:2]] + infini.ops.split( + input, + 2, + 0, + outputs, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(outputs[0], expected[:2], rtol=0, atol=0) + torch.testing.assert_close(outputs[1], expected[2:], rtol=0, atol=0) + + +def test_split_rejects_output_overlap(device, implementation_index): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((4, 3), device=device)\n" + probe = setup + ( + "bad = torch.empty((2, 3), dtype=torch.int32, device=device)\n" + "infini.ops.Split(x, 2, 0, [bad, bad])\n" + ) + source = ( + setup + + f"infini.ops.split(x, 2, 0, [x[:2], x[:2]], stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "Output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + _TEST_CASES = ( pytest.param( (12,), diff --git a/tests/test_stack_backward.py b/tests/test_stack_backward.py index c50931b85..98a5069d9 100644 --- a/tests/test_stack_backward.py +++ b/tests/test_stack_backward.py @@ -1,3 +1,7 @@ +import json +import subprocess +import sys + import infini.ops import pytest import torch @@ -5,6 +9,63 @@ from tests.utils import Payload, empty_strided, get_stream, randn_strided +def test_stack_backward_alias(device, implementation_index): + grad_output = torch.arange(12, dtype=torch.float32, device=device).reshape(2, 2, 3) + expected = grad_output.clone() + outputs = [grad_output[1], grad_output[0]] + infini.ops.stack_backward( + grad_output, + 0, + outputs, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(outputs[0], expected[0], rtol=0, atol=0) + torch.testing.assert_close(outputs[1], expected[1], rtol=0, atol=0) + + +def test_stack_backward_rejects_output_overlap(device, implementation_index): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 2, 3), device=device)\n" + probe = setup + ( + "bad = torch.empty((2, 3), dtype=torch.int32, device=device)\n" + "infini.ops.StackBackward(x, 0, [bad, bad])\n" + ) + source = ( + setup + + f"infini.ops.stack_backward(x, 0, [x[0], x[0]], stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "Output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + # Format: (input_shape, num_inputs, dim, grad_output_strides, grad_input_strides) _TEST_CASES = ( pytest.param( diff --git a/tests/test_sub_backward.py b/tests/test_sub_backward.py index 868dda678..9007713ed 100644 --- a/tests/test_sub_backward.py +++ b/tests/test_sub_backward.py @@ -1,87 +1,414 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (shape, grad_output_strides, grad_a_strides, grad_b_strides) -_TEST_CASES = ( - pytest.param((17,), None, None, None, id="one-dimensional"), - pytest.param((2, 3), None, None, None, id="two-dimensional"), - pytest.param((2, 3, 4), None, None, None, id="three-dimensional"), - pytest.param( - (2, 3, 4), - (30, 7, 1), - (35, 8, 1), - (36, 9, 1), - id="strided-tensors", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "shape, grad_output_strides, grad_a_strides, grad_b_strides", - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_sub_backward( - shape, - grad_output_strides, - grad_a_strides, - grad_b_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output = randn_strided( - shape, - grad_output_strides, - dtype=dtype, - device=device, - ) - grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) - grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) - grad_a.fill_(17) - grad_b.fill_(17) - - return Payload( - lambda *args: _sub_backward( - *args, - implementation_index=implementation_index, - ), - _torch_sub_backward, - (grad_output, grad_a, grad_b), - {}, - rtol=rtol, - atol=atol, - ) - - -def _sub_backward(grad_output, grad_a, grad_b, *, implementation_index): - infini.ops.sub_backward( - grad_output, - grad_a, - grad_b, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) - - -def _torch_sub_backward(grad_output, grad_a, grad_b): - grad_a.copy_(grad_output) - grad_b.copy_(-grad_output) - - return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (grad_output_shape, grad_a_shape, grad_b_shape, +# grad_output_strides, grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), (17,), (17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), (2, 3), (2, 3), None, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), + pytest.param( + (2, 3), + (2, 3), + (2, 3), + (1, 2), + (1, 2), + (1, 2), + id="transposed-layout", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (4,), + None, + None, + None, + id="broadcast-b-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (4,), + (2, 3, 4), + None, + None, + None, + id="broadcast-a-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + id="broadcast-both-inputs", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + (30, 7, 1), + (11, 7, 2), + (15, 3, 1), + id="broadcast-strided-tensors", + ), + pytest.param( + (2, 3), + (), + (2, 3), + None, + None, + None, + id="broadcast-scalar-a", + ), + pytest.param( + (2, 3), + (2, 3), + (), + None, + None, + None, + id="broadcast-scalar-b", + ), + pytest.param((), (), (), None, None, None, id="scalar-output"), + pytest.param( + (1, 3, 1), + (3, 1), + (1, 3, 1), + None, + None, + None, + id="singleton-dimensions", + ), + pytest.param( + (0, 3), + (0, 3), + (0, 3), + None, + None, + None, + id="empty-output", + ), + pytest.param( + (0, 3), + (1, 3), + (0, 3), + None, + None, + None, + id="empty-output-nonempty-grad-a", + ), + pytest.param( + (0, 3), + (0, 3), + (1, 3), + None, + None, + None, + id="empty-output-nonempty-grad-b", + ), +) + +_ALPHAS = ( + pytest.param(1.0, id="alpha-one"), + pytest.param(0.0, id="alpha-zero"), + pytest.param(2.0, id="alpha-two"), + pytest.param(-1.0, id="alpha-negative"), + pytest.param(0.5, id="alpha-fractional"), +) + +_DTYPES = ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "grad_output_shape, grad_a_shape, grad_b_shape, " + "grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_sub_backward( + grad_output_shape, + grad_a_shape, + grad_b_shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + alpha, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided( + grad_a_shape, + grad_a_strides, + dtype=dtype, + device=device, + ) + grad_b = empty_strided( + grad_b_shape, + grad_b_strides, + dtype=dtype, + device=device, + ) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _sub_backward( + *args, + implementation_index=implementation_index, + ), + _torch_sub_backward, + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pack_result(grad_output, grad_a, grad_b): + # Also compare grad_output: ordinary, non-alias calls must not modify it. + return torch.cat( + (grad_a.reshape(-1), grad_b.reshape(-1), grad_output.reshape(-1)), + ) + + +def _sub_backward(grad_output, alpha, grad_a, grad_b, *, implementation_index): + infini.ops.sub_backward( + grad_output, + alpha, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return _pack_result(grad_output, grad_a, grad_b) + + +def _torch_sub_backward(grad_output, alpha, grad_a, grad_b): + # Use canonical forward + autograd, independently of the hand-written formula. + with torch.enable_grad(): + a = torch.zeros_like(grad_a, requires_grad=True) + b = torch.zeros_like(grad_b, requires_grad=True) + out = torch.sub(a, b, alpha=alpha) + expected_a, expected_b = torch.autograd.grad( + out, + (a, b), + grad_outputs=grad_output, + ) + + # Autograd may return a view of grad_output. Snapshot BOTH results first. + expected_a = expected_a.clone() + expected_b = expected_b.clone() + with torch.no_grad(): + grad_a.copy_(expected_a) + grad_b.copy_(expected_b) + return _pack_result(grad_output, grad_a, grad_b) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("target", ("grad_a", "grad_b")) +@pytest.mark.parametrize("layout", ("exact", "partial", "strided", "broadcast")) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_sub_backward_alias( + target, + layout, + alpha, + implementation_index, + dtype, + device, +): + # Small integers and power-of-two alpha make these cases exactly representable. + source = torch.arange(-16, 16, dtype=dtype, device=device) + + return Payload( + lambda values: _run_alias( + values, + alpha, + target, + layout, + implementation_index=implementation_index, + ), + lambda values: _run_alias(values, alpha, target, layout), + (source,), + {}, + rtol=0, + atol=0, + ) + + +def _run_alias(source, alpha, target, layout, *, implementation_index=None): + # Construct views INSIDE each invocation so Payload copying cannot remove aliasing. + storage = source.clone() + grad_output = storage[:6].view(2, 3) + if layout == "exact": + shared_output = grad_output + elif layout == "partial": + shared_output = storage[1:7].view(2, 3) + elif layout == "strided": + shared_output = storage[:6].view(3, 2).transpose(0, 1) + elif layout == "broadcast": + shared_output = storage[:3].view(1, 3) + else: + raise ValueError(f"Unknown alias layout: {layout}") + + independent_output = torch.full_like(grad_output, 17) + if target == "grad_a": + grad_a, grad_b = shared_output, independent_output + else: + grad_a, grad_b = independent_output, shared_output + + if implementation_index is None: + _torch_sub_backward(grad_output, alpha, grad_a, grad_b) + else: + _sub_backward( + grad_output, + alpha, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + # Include the whole backing buffer to detect writes outside the output view. + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1), storage)) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +@pytest.mark.parametrize( + "alpha, grad_value, grad_a_shape, grad_b_shape, expected_a, expected_b", + ( + pytest.param(2.0, 1.0, (2, 3), (2, 3), 1.0, -2.0, id="review-example"), + pytest.param(2.0, 1.0, (2, 3), (3,), 1.0, -4.0, id="scaled-reduction"), + pytest.param(0.0, 1.0, (2, 3), (3,), 1.0, 0.0, id="zero-alpha"), + pytest.param(-1.0, 1.0, (2, 3), (3,), 1.0, 2.0, id="negative-alpha"), + pytest.param(0.5, 1.0, (2, 3), (3,), 1.0, -1.0, id="fractional-alpha"), + pytest.param(2.0, 1.0, (1, 3), (2, 1), 2.0, -6.0, id="both-reduced"), + pytest.param( + 0.5, 40000.0, (2, 3), (2, 3), 40000.0, -20000.0, id="large-finite-gradient" + ), + pytest.param( + 0.0, 40000.0, (2, 3), (3,), 40000.0, 0.0, id="zero-alpha-before-reduction" + ), + ), +) +def test_sub_backward_known_values( + alpha, + grad_value, + grad_a_shape, + grad_b_shape, + expected_a, + expected_b, + implementation_index, + dtype, + device, +): + grad_output = torch.full((2, 3), grad_value, dtype=dtype, device=device) + grad_a = torch.full(grad_a_shape, 17.0, dtype=dtype, device=device) + grad_b = torch.full(grad_b_shape, 17.0, dtype=dtype, device=device) + expected = _pack_result( + grad_output, + torch.full_like(grad_a, expected_a), + torch.full_like(grad_b, expected_b), + ) + + return Payload( + lambda *args: _sub_backward( + *args, + implementation_index=implementation_index, + ), + lambda *args: expected.clone(), + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("offset", (0, 2)) +@pytest.mark.parametrize("strides", ((3, 1), (6, 2))) +def test_sub_backward_rejects_output_overlap( + offset, strides, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = setup + "infini.ops.SubBackward(x, 2.0, torch.empty(4, device=device), x)\n" + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {strides!r}, 0) +b = backing.as_strided((2, 3), {strides!r}, {offset}) +infini.ops.sub_backward(x, 2.0, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_transpose.py b/tests/test_transpose.py deleted file mode 100644 index aeecc5430..000000000 --- a/tests/test_transpose.py +++ /dev/null @@ -1,128 +0,0 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (input_shape, input_strides, dim0, dim1, out_strides) -_TEST_CASES = ( - pytest.param( - (2, 3), - None, - 0, - 1, - None, - id="two-dimensional", - ), - pytest.param( - (2, 3, 4), - None, - 0, - 2, - None, - id="three-dimensional-outer-dimensions", - ), - pytest.param( - (2, 3, 4), - None, - -1, - -2, - None, - id="negative-dimensions", - ), - pytest.param( - (2, 3, 4), - None, - 1, - 1, - None, - id="same-dimension", - ), - pytest.param( - (2, 3, 4), - (20, 6, 1), - 0, - 2, - (10, 3, 1), - id="strided-input-and-output", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - "input_shape, input_strides, dim0, dim1, out_strides", - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_transpose( - input_shape, - input_strides, - dim0, - dim1, - out_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - input = randn_strided( - input_shape, - input_strides, - dtype=dtype, - device=device, - ) - out_shape = list(input_shape) - normalized_dim0 = dim0 % len(input_shape) - normalized_dim1 = dim1 % len(input_shape) - out_shape[normalized_dim0], out_shape[normalized_dim1] = ( - out_shape[normalized_dim1], - out_shape[normalized_dim0], - ) - out = empty_strided( - tuple(out_shape), - out_strides, - dtype=dtype, - device=device, - ) - out.fill_(17) - - return Payload( - lambda *args: _transpose( - *args, - implementation_index=implementation_index, - ), - _torch_transpose, - (input, dim0, dim1, out), - {}, - rtol=rtol, - atol=atol, - ) - - -def _transpose(input, dim0, dim1, out, *, implementation_index): - infini.ops.transpose( - input, - dim0, - dim1, - out, - stream=get_stream(input.device), - implementation_index=implementation_index, - ) - - return out - - -def _torch_transpose(input, dim0, dim1, out): - out.copy_(torch.transpose(input, dim0, dim1)) - - return out diff --git a/tests/test_transpose_backward.py b/tests/test_transpose_backward.py index 78248f07b..dfcc96300 100644 --- a/tests/test_transpose_backward.py +++ b/tests/test_transpose_backward.py @@ -1,141 +1,156 @@ -import infini.ops -import pytest -import torch - -from tests.utils import Payload, empty_strided, get_stream, randn_strided - - -# Format: (grad_input_shape, dim0, dim1, grad_output_strides, grad_input_strides) -_TEST_CASES = ( - pytest.param( - (2, 3), - 0, - 1, - None, - None, - id="two-dimensional", - ), - pytest.param( - (2, 3, 4), - 0, - 2, - None, - None, - id="three-dimensional-outer-dimensions", - ), - pytest.param( - (2, 3, 4), - -1, - -2, - None, - None, - id="negative-dimensions", - ), - pytest.param( - (2, 3, 4), - 1, - 1, - None, - None, - id="same-dimension", - ), - pytest.param( - (2, 3, 4), - 0, - 2, - (10, 3, 1), - (20, 6, 1), - id="strided-gradients", - ), -) - - -@pytest.mark.auto_act_and_assert -@pytest.mark.parametrize( - ( - "grad_input_shape", - "dim0", - "dim1", - "grad_output_strides", - "grad_input_strides", - ), - _TEST_CASES, -) -@pytest.mark.parametrize( - ("dtype", "rtol", "atol"), - ( - (torch.float32, 0, 0), - (torch.float16, 0, 0), - (torch.bfloat16, 0, 0), - ), -) -def test_transpose_backward( - grad_input_shape, - dim0, - dim1, - grad_output_strides, - grad_input_strides, - implementation_index, - dtype, - device, - rtol, - atol, -): - grad_output_shape = list(grad_input_shape) - normalized_dim0 = dim0 % len(grad_input_shape) - normalized_dim1 = dim1 % len(grad_input_shape) - grad_output_shape[normalized_dim0], grad_output_shape[normalized_dim1] = ( - grad_output_shape[normalized_dim1], - grad_output_shape[normalized_dim0], - ) - grad_output = randn_strided( - tuple(grad_output_shape), - grad_output_strides, - dtype=dtype, - device=device, - ) - grad_input = empty_strided( - grad_input_shape, - grad_input_strides, - dtype=dtype, - device=device, - ) - grad_input.fill_(17) - - return Payload( - lambda *args: _transpose_backward( - *args, - implementation_index=implementation_index, - ), - _torch_transpose_backward, - (grad_output, dim0, dim1, grad_input), - {}, - rtol=rtol, - atol=atol, - ) - - -def _transpose_backward( - grad_output, - dim0, - dim1, - grad_input, - *, - implementation_index, -): - infini.ops.transpose_backward( - grad_output, - dim0, - dim1, - grad_input, - stream=get_stream(grad_output.device), - implementation_index=implementation_index, - ) - - return grad_input - - -def _torch_transpose_backward(grad_output, dim0, dim1, grad_input): - grad_input.copy_(torch.transpose(grad_output, dim0, dim1)) - - return grad_input +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (grad_input_shape, dim0, dim1, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (2, 3), + 0, + 1, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + None, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param( + (2, 3, 4), + -1, + -2, + None, + None, + id="negative-dimensions", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="same-dimension", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + (10, 3, 1), + (20, 6, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "grad_input_shape", + "dim0", + "dim1", + "grad_output_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_transpose_backward( + grad_input_shape, + dim0, + dim1, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output_shape = list(grad_input_shape) + normalized_dim0 = dim0 % len(grad_input_shape) + normalized_dim1 = dim1 % len(grad_input_shape) + grad_output_shape[normalized_dim0], grad_output_shape[normalized_dim1] = ( + grad_output_shape[normalized_dim1], + grad_output_shape[normalized_dim0], + ) + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + grad_input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _transpose_backward( + *args, + implementation_index=implementation_index, + ), + _torch_transpose_backward, + (grad_output, dim0, dim1, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + *, + implementation_index, +): + infini.ops.transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_transpose_backward(grad_output, dim0, dim1, grad_input): + grad_input.copy_(torch.transpose(grad_output, dim0, dim1)) + + return grad_input + + +@pytest.mark.parametrize("layout", ("contiguous", "strided")) +@pytest.mark.parametrize("offsets", ((0, 0), (0, 2), (2, 0))) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_transpose_backward_alias(layout, offsets, dtype, device, implementation_index): + backing = torch.arange(64, dtype=torch.float32, device=device).to(dtype) + strides = (6, 2) if layout == "strided" else (3, 1) + grad_output = backing.as_strided((3, 3), strides, offsets[0]) + grad_input = backing.as_strided((3, 3), strides, offsets[1]) + expected = grad_output.transpose(0, 1).clone() + _transpose_backward( + grad_output, 0, 1, grad_input, implementation_index=implementation_index + ) + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) diff --git a/tests/test_transpose_copy.py b/tests/test_transpose_copy.py new file mode 100644 index 000000000..01381f603 --- /dev/null +++ b/tests/test_transpose_copy.py @@ -0,0 +1,416 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_DTYPES = (torch.float32, torch.float16, torch.bfloat16) + +_TEST_CASES = ( + pytest.param((2, 3), None, 0, 1, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + None, + 0, + 2, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param((2, 3, 4), None, -1, -2, None, id="negative-dimensions"), + pytest.param((2, 3, 4), None, 1, 1, None, id="same-dimension"), + pytest.param( + (2, 3, 4), + (20, 6, 1), + 0, + 2, + (10, 3, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + tuple((dtype, 0, 0) for dtype in _DTYPES), +) +def test_transpose_copy( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + out = empty_strided( + _out_shape(input_shape, dim0, dim1), + out_strides, + dtype=dtype, + device=device, + ) + out.fill_(17) + + return Payload( + lambda *args: _transpose_copy( + *args, + implementation_index=implementation_index, + ), + _torch_transpose_copy, + (input, dim0, dim1, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose_copy(input, dim0, dim1, out, *, implementation_index): + infini.ops.transpose_copy( + input, + dim0, + dim1, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + return out + + +def _torch_transpose_copy(input, dim0, dim1, out): + result = input.transpose_copy(dim0, dim1).clone() + out.copy_(result) + return out + + +def _out_shape(shape, dim0, dim1): + result = list(shape) + dim0 %= len(shape) + dim1 %= len(shape) + result[dim0], result[dim1] = result[dim1], result[dim0] + return tuple(result) + + +def _span(shape, strides): + return 1 + sum((size - 1) * stride for size, stride in zip(shape, strides)) + + +def _offsets(placement, input_span, output_span): + if placement == "same-start": + return 8, 8 + if placement == "partial-output-after": + return 8, 10 + if placement == "partial-output-before": + return 10, 8 + if placement == "adjacent-output-after": + return 8, 8 + input_span + if placement == "adjacent-output-before": + return 8 + output_span, 8 + if placement == "separated-output-after": + return 8, 8 + input_span + 5 + if placement == "separated-output-before": + return 8 + output_span + 5, 8 + raise AssertionError(placement) + + +def _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + *, + dtype, + device, + implementation_index, + repeats=1, + special_values=None, +): + + backing = torch.arange(256, dtype=torch.float32, device=device).to(dtype) + input = backing.as_strided(input_shape, input_strides, input_offset) + out = backing.as_strided( + _out_shape(input_shape, dim0, dim1), + out_strides, + output_offset, + ) + if special_values is not None: + input.copy_( + torch.tensor(special_values, dtype=dtype, device=device).reshape( + input_shape + ) + ) + + input_metadata = (tuple(input.shape), input.stride(), input.data_ptr()) + output_metadata = (tuple(out.shape), out.stride(), out.data_ptr()) + + for _ in range(repeats): + expected = input.detach().clone().transpose_copy(dim0, dim1).clone() + expected_backing = backing.clone() + expected_backing.as_strided( + out.shape, + out.stride(), + out.storage_offset(), + ).copy_(expected) + + _transpose_copy( + input, + dim0, + dim1, + out, + implementation_index=implementation_index, + ) + + torch.testing.assert_close(out, expected, rtol=0, atol=0, equal_nan=True) + torch.testing.assert_close( + backing, + expected_backing, + rtol=0, + atol=0, + equal_nan=True, + ) + if special_values is not None: + torch.testing.assert_close( + torch.signbit(backing), + torch.signbit(expected_backing), + rtol=0, + atol=0, + ) + assert (tuple(input.shape), input.stride(), input.data_ptr()) == input_metadata + assert (tuple(out.shape), out.stride(), out.data_ptr()) == output_metadata + + +_ALIAS_LAYOUTS = ( + pytest.param((3, 3), (3, 1), 0, 1, (3, 1), id="square"), + pytest.param((3, 3), (3, 1), -1, -2, (3, 1), id="square-negative-dims"), + pytest.param((3, 3), (1, 3), 0, 1, (3, 1), id="transpose_copyd-input"), + pytest.param((3, 3), (3, 1), 0, 1, (1, 3), id="transpose_copyd-output"), + pytest.param((3, 3), (8, 2), 0, 1, (9, 2), id="gapped-input-and-output"), + pytest.param((2, 3), (3, 1), 0, 1, (2, 1), id="rectangular"), + pytest.param((2, 3), (10, 2), 0, 1, (7, 2), id="rectangular-gapped"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + 0, + 2, + (20, 5, 1), + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (30, 7, 1), + -1, + -3, + (20, 5, 1), + id="three-dimensional-negative-dims", + ), + pytest.param( + (2, 3), + (1, 2), + 1, + 1, + (3, 1), + id="same-dim-different-layouts", + ), + pytest.param( + (2, 3), + (8, 2), + -1, + 1, + (9, 2), + id="same-dim-negative-equivalent", + ), + pytest.param((1, 4), (9, 2), 0, 1, (2, 11), id="singleton-dimension"), + pytest.param((7,), (2,), 0, -1, (3,), id="one-dimensional"), + pytest.param( + (2, 2, 3, 4), + (50, 22, 6, 1), + 0, + 3, + (18, 8, 2, 1), + id="four-dimensional", + ), +) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + _ALIAS_LAYOUTS, +) +@pytest.mark.parametrize( + "placement", + ( + "same-start", + "partial-output-after", + "partial-output-before", + "adjacent-output-after", + "adjacent-output-before", + "separated-output-after", + "separated-output-before", + ), +) +def test_transpose_copy_shared_storage( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + placement, + dtype, + device, + implementation_index, +): + input_offset, output_offset = _offsets( + placement, + _span(input_shape, input_strides), + _span(_out_shape(input_shape, dim0, dim1), out_strides), + ) + _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + dtype=dtype, + device=device, + implementation_index=implementation_index, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +def test_transpose_copy_review_regression(dtype, device, implementation_index): + input = torch.tensor( + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + dtype=dtype, + device=device, + ) + out = input.view_as(input) + + _transpose_copy(input, 0, 1, out, implementation_index=implementation_index) + + torch.testing.assert_close( + out, + torch.tensor( + [[1, 4, 7], [2, 5, 8], [3, 6, 9]], + dtype=dtype, + device=device, + ), + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + ( + pytest.param((3, 3), (6, 2), 0, 1, (6, 2), id="square"), + pytest.param((2, 3), (6, 2), 0, 1, (4, 2), id="rectangular"), + pytest.param( + (2, 3, 4), + (24, 8, 2), + 0, + 2, + (12, 4, 2), + id="three-dimensional", + ), + ), +) +@pytest.mark.parametrize("reverse", (False, True)) +def test_transpose_copy_interleaved_views( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + reverse, + dtype, + device, + implementation_index, +): + + input_offset, output_offset = (9, 8) if reverse else (8, 9) + _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + dtype=dtype, + device=device, + implementation_index=implementation_index, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("offsets", ((8, 8), (8, 10), (10, 8))) +def test_transpose_copy_repeated_alias_calls( + offsets, + dtype, + device, + implementation_index, +): + _run_alias_case( + (3, 3), + (3, 1), + 0, + 1, + (3, 1), + *offsets, + dtype=dtype, + device=device, + implementation_index=implementation_index, + repeats=5, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("offsets", ((8, 8), (8, 10), (10, 8))) +def test_transpose_copy_alias_special_values( + offsets, + dtype, + device, + implementation_index, +): + _run_alias_case( + (3, 3), + (3, 1), + -1, + -2, + (3, 1), + *offsets, + dtype=dtype, + device=device, + implementation_index=implementation_index, + special_values=( + float("nan"), + float("inf"), + -float("inf"), + 0.0, + -0.0, + 1.0, + -2.0, + 0.25, + -0.5, + ), + ) diff --git a/tests/test_vocab_parallel_cross_entropy_backward.py b/tests/test_vocab_parallel_cross_entropy_backward.py index 3747ef4f0..5e658d1f8 100644 --- a/tests/test_vocab_parallel_cross_entropy_backward.py +++ b/tests/test_vocab_parallel_cross_entropy_backward.py @@ -173,25 +173,67 @@ def _torch_vocab_parallel_cross_entropy_backward( label_smoothing, grad_input, ): - rows = softmax_local.shape[0] - dloss = grad_output.reshape(-1) + probabilities = softmax_local.detach().to(device="cpu", dtype=torch.float64) + rows = probabilities.shape[0] + dloss = grad_output.detach().to(device="cpu", dtype=torch.float64).reshape(-1) if dloss.numel() == 1: dloss = dloss.expand(rows) - gradient = softmax_local * dloss.reshape(rows, 1) - valid_rows = 1 - target_mask - row_scale = dloss * (1 - label_smoothing) * valid_rows - target_one_hot = torch.zeros_like(gradient) - target_one_hot.scatter_(1, masked_target.reshape(rows, 1), 1) - gradient = gradient - target_one_hot * row_scale.reshape(rows, 1) - - smoothing_term = label_smoothing / vocab_size_original - gradient = gradient - ( - dloss.reshape(rows, 1) - * smoothing_term - * valid_mask_local.reshape(1, vocab_size_local) + valid_rows = 1 - target_mask.detach().to(device="cpu", dtype=torch.float64) + valid_columns = ( + valid_mask_local.detach() + .to(device="cpu", dtype=torch.float64) + .reshape(1, vocab_size_local) + ) + distribution = (label_smoothing / vocab_size_original) * valid_columns.expand_as( + probabilities + ).clone() + target = masked_target.detach().to(device="cpu", dtype=torch.int64).reshape(rows, 1) + distribution.scatter_add_( + 1, target, ((1 - label_smoothing) * valid_rows).reshape(rows, 1) ) - gradient = gradient * valid_mask_local.reshape(1, vocab_size_local) + gradient = (probabilities - distribution) * dloss.reshape(rows, 1) * valid_columns grad_input.copy_(gradient) return grad_input + + +@pytest.mark.parametrize( + "dtype", (torch.float16, torch.bfloat16, torch.float32, torch.float64) +) +@pytest.mark.parametrize("smoothing", (1e-4, 1e-8)) +@pytest.mark.parametrize("alias", (False, True)) +def test_vocab_parallel_cross_entropy_backward_small_smoothing( + dtype, smoothing, alias, device, implementation_index +): + probabilities = torch.tensor([[1.0, 0.0]], dtype=dtype, device=device) + upstream = torch.tensor(60000.0, dtype=dtype, device=device) + target_mask = torch.zeros(1, dtype=dtype, device=device) + target = torch.zeros(1, dtype=torch.int64, device=device) + valid = torch.ones(2, dtype=dtype, device=device) + output = probabilities if alias else torch.empty_like(probabilities) + expected = torch.empty_like(probabilities) + _torch_vocab_parallel_cross_entropy_backward( + upstream, probabilities, target_mask, target, valid, 2, 2, smoothing, expected + ) + _vocab_parallel_cross_entropy_backward( + upstream, + probabilities, + target_mask, + target, + valid, + 2, + 2, + smoothing, + output, + implementation_index=implementation_index, + ) + assert output[0, 0].item() > 0 + assert output[0, 1].item() < 0 + tolerance = { + torch.float16: 1e-3, + torch.bfloat16: 1e-2, + torch.float32: 1e-6, + torch.float64: 1e-7, + }[dtype] + torch.testing.assert_close(output, expected, rtol=tolerance, atol=0) From 4673741615be99a41b01376b1cd0df0b216a57ce Mon Sep 17 00:00:00 2001 From: zhoujf1 <2047484072@qq.com> Date: Wed, 18 Nov 2026 14:04:26 +0800 Subject: [PATCH 4/4] fix: address PR review feedback for ATen operators --- scripts/generate_torch_ops.py | 25 +++++++++++++++++- .../adam_accumulate_grad.cc | 2 +- tests/test_generate_torch_ops.py | 26 +++++++++++++++++++ tests/test_transpose_copy.py | 4 +-- 4 files changed, 53 insertions(+), 4 deletions(-) diff --git a/scripts/generate_torch_ops.py b/scripts/generate_torch_ops.py index a67818d16..8a5c1724b 100644 --- a/scripts/generate_torch_ops.py +++ b/scripts/generate_torch_ops.py @@ -1431,6 +1431,21 @@ def _render_arg(schema_index, p): rendered_args = ", ".join(_render_arg(index, p) for index, p in arg_order) aten_call = f"at::{op.aten_name}_out({rendered_args})" + if op.aten_name == "embedding_dense_backward": + conversion_lines.append( + " assert(at_grad_output.dim() >= 1);\n" + " const auto embedding_dim = at_grad_output.size(-1);\n" + " assert(at_grad_output.numel() == at_indices.numel() * embedding_dim);\n" + " assert(at_out.dim() == 2 && at_out.size(0) == num_weights &&\n" + " at_out.size(1) == embedding_dim &&\n" + " at_out.scalar_type() == at_grad_output.scalar_type());\n" + " // Avoid CUDA kernels with empty indices or zero feature width.\n" + " if (at_indices.numel() == 0 || embedding_dim == 0) {\n" + " at_out.zero_();\n" + " return;\n" + " }" + ) + return _TORCH_METHOD_TEMPLATE.format( op_type=op_type, op_call_signature=_format_signature(op), @@ -1446,13 +1461,21 @@ def _render_arg(schema_index, p): def _generate_torch_source(name: str, ops: list[Op]) -> str: op_type = _op_cpp_type(name) methods = "\n\n".join(_generate_torch_method_source(name, op) for op in ops) - return _TORCH_SOURCE_TEMPLATE.format( + source = _TORCH_SOURCE_TEMPLATE.format( name=name, methods=methods, op_type=op_type, slot=_PYTORCH_SLOT, ) + if any(op.aten_name == "embedding_dense_backward" for op in ops): + source = source.replace( + '#include "torch/instantiate.h"', + '#include \n\n#include "torch/instantiate.h"', + ) + + return source + _BASE_TEMPLATE = """\ #ifndef INFINI_OPS_BASE_{name_uc}_H_ diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc index 6ca6cf04b..ee7b7abc7 100644 --- a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc @@ -42,7 +42,7 @@ void Operator::operator()( const auto gradient = at_grad.to(compute_type); const auto next_m = at::lerp(at_m.to(compute_type), gradient, 1.0 - beta1_); auto next_v = at_v.to(compute_type) * beta2_; - next_v.addcmul_(gradient, gradient, 1.0 - beta2_); + next_v.addcmul_(gradient * (1.0 - beta2_), gradient); const double bcm = 1.0 - std::pow(beta1_, static_cast(step_)); const double bcv = 1.0 - std::pow(beta2_, static_cast(step_)); const auto denominator = at::sqrt(next_v) / std::sqrt(bcv) + eps_; diff --git a/tests/test_generate_torch_ops.py b/tests/test_generate_torch_ops.py index 615aff6f8..30cc7df91 100644 --- a/tests/test_generate_torch_ops.py +++ b/tests/test_generate_torch_ops.py @@ -146,6 +146,32 @@ def test_torch_source_uses_existing_c10_stream_guards(): assert '#include "torch/nvidia/c10.h"' not in source +def test_embedding_dense_backward_handles_empty_inputs_before_aten(): + module = _load_generator_module() + op = module._parse_func( + "embedding_dense_backward.out(Tensor grad_output, Tensor indices, " + "SymInt num_weights, SymInt padding_idx, bool scale_grad_by_freq, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + source = module._generate_torch_source("embedding_dense_backward", [op]) + guard = source.index("if (at_indices.numel() == 0 || embedding_dim == 0)") + zero = source.index("at_out.zero_();", guard) + early_return = source.index("return;", zero) + call = source.index("at::embedding_dense_backward_out(") + assert guard < zero < early_return < call + assert source.index("at_out.size(1) == embedding_dim") < guard + + +def test_copy_guards_do_not_change_unrelated_generated_ops(): + module = _load_generator_module() + op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") + source = module._generate_torch_source("abs", [op]) + assert "at::abs_out(at_out, at_self)" in source + assert "StorageSpansOverlap" not in source + assert "embedding_dim" not in source + assert '#include "torch/storage_overlap.h"' not in source + + def test_optional_tensor_params_are_exposed_and_forwarded_to_aten(): module = _load_generator_module() op = module._parse_func( diff --git a/tests/test_transpose_copy.py b/tests/test_transpose_copy.py index 01381f603..609d1edca 100644 --- a/tests/test_transpose_copy.py +++ b/tests/test_transpose_copy.py @@ -91,7 +91,7 @@ def _transpose_copy(input, dim0, dim1, out, *, implementation_index): def _torch_transpose_copy(input, dim0, dim1, out): - result = input.transpose_copy(dim0, dim1).clone() + result = torch.transpose_copy(input, dim0, dim1).clone() out.copy_(result) return out @@ -160,7 +160,7 @@ def _run_alias_case( output_metadata = (tuple(out.shape), out.stride(), out.data_ptr()) for _ in range(repeats): - expected = input.detach().clone().transpose_copy(dim0, dim1).clone() + expected = torch.transpose_copy(input.detach().clone(), dim0, dim1).clone() expected_backing = backing.clone() expected_backing.as_strided( out.shape,