Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3,617 changes: 1,820 additions & 1,797 deletions scripts/generate_torch_ops.py

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion scripts/generate_wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ def _parse_operator_header(op_name):
constructors = [
_ParsedFunction(_parse_parameter_list(params))
for params in _find_signature_parameters(
class_body, rf"(?:explicit\s+)?{class_name}\s*\("
class_body, rf"\b(?:explicit\s+)?{class_name}\s*\("
)
]
calls = [
Expand Down
4 changes: 4 additions & 0 deletions scripts/torch_ops.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
- dstack
- elu
- elu_backward
- embedding_dense_backward
- empty
- eq
- erf
Expand Down Expand Up @@ -363,6 +364,8 @@
- sin
- sinc
- sinh
- slice_backward
- slice_copy
- slogdet
- slow_conv3d
- slow_conv3d_forward
Expand Down Expand Up @@ -454,6 +457,7 @@
- threshold
- threshold_backward
- topk
- transpose_copy
- triangular_solve
- tril
- triu
Expand Down
55 changes: 55 additions & 0 deletions src/base/accumulate_grad.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#ifndef INFINI_OPS_BASE_ACCUMULATE_GRAD_H_
#define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_

#include <cassert>

#include "base/validation.h"
#include "operator.h"

namespace infini::ops {

class AccumulateGrad : public Operator<AccumulateGrad> {
public:
AccumulateGrad(const Tensor gradient, const double rate, Tensor tensor)
: gradient_shape_{gradient.shape()},
gradient_strides_{gradient.strides()},
gradient_type_{gradient.dtype()},
tensor_shape_{tensor.shape()},
tensor_strides_{tensor.strides()},
tensor_type_{tensor.dtype()},
rate_{rate},
device_index_{tensor.device().index()} {
validation::AssertSameDevice({gradient, tensor});
assert(validation::IsFloatingPoint(tensor_type_) &&
"`AccumulateGrad` requires floating-point tensors.");
assert(gradient.shape() == tensor.shape() &&
"`AccumulateGrad` requires `gradient` and `tensor` to share shape");
assert(gradient_type_ == tensor_type_ &&
"`AccumulateGrad` requires `gradient` and `tensor` to share dtype");
}

// Gradient and tensor may overlap, including partial external storage spans.
virtual void operator()(const Tensor gradient, const double rate,
Tensor tensor) const = 0;

protected:
Tensor::Shape gradient_shape_;

Tensor::Strides gradient_strides_;

DataType gradient_type_;

Tensor::Shape tensor_shape_;

Tensor::Strides tensor_strides_;

DataType tensor_type_;

double rate_{1.0};

int device_index_{0};
};

} // namespace infini::ops

#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_
101 changes: 101 additions & 0 deletions src/base/adam_accumulate_grad.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#ifndef INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_
#define INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_

#include <cassert>
#include <cstdint>

#include "base/validation.h"
#include "operator.h"

namespace infini::ops {

class AdamAccumulateGrad : public Operator<AdamAccumulateGrad> {
public:
AdamAccumulateGrad(const Tensor grad, const double learning_rate,
const double beta1, const double beta2, const double eps,
const int64_t step, Tensor param, Tensor m, Tensor v)
: grad_shape_{grad.shape()},
grad_strides_{grad.strides()},
grad_type_{grad.dtype()},
param_shape_{param.shape()},
param_strides_{param.strides()},
param_type_{param.dtype()},
m_shape_{m.shape()},
m_strides_{m.strides()},
m_type_{m.dtype()},
v_shape_{v.shape()},
v_strides_{v.strides()},
v_type_{v.dtype()},
learning_rate_{learning_rate},
beta1_{beta1},
beta2_{beta2},
eps_{eps},
step_{step},
device_index_{param.device().index()} {
validation::AssertSameDevice({grad, param, m, v});
assert(grad.shape() == param.shape() && param.shape() == m.shape() &&
m.shape() == v.shape() &&
"`AdamAccumulateGrad` requires grad/param/m/v to share shape");
assert(validation::IsFloatingPoint(param_type_) &&
grad_type_ == param_type_ &&
"Adam parameters and gradients require matching floating dtypes.");
assert(m_type_ == v_type_ &&
(m_type_ == param_type_ || (param_type_ != DataType::kFloat64 &&
m_type_ == DataType::kFloat32)) &&
"Adam moments must match the parameter dtype or use float32.");

assert(0.0 <= beta1_ && beta1_ < 1.0 && 0.0 <= beta2_ && beta2_ < 1.0 &&
"Adam beta values must be in [0, 1).");
assert(learning_rate_ >= 0.0 && eps_ > 0.0 &&
"Adam requires non-negative learning rate and positive epsilon.");

assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1");
}

// Grad may alias outputs; param, m and v byte spans must not overlap.
virtual void operator()(const Tensor grad, const double learning_rate,
const double beta1, const double beta2,
const double eps, const int64_t step, Tensor param,
Tensor m, Tensor v) const = 0;

protected:
Tensor::Shape grad_shape_;

Tensor::Strides grad_strides_;

DataType grad_type_;

Tensor::Shape param_shape_;

Tensor::Strides param_strides_;

DataType param_type_;

Tensor::Shape m_shape_;

Tensor::Strides m_strides_;

DataType m_type_;

Tensor::Shape v_shape_;

Tensor::Strides v_strides_;

DataType v_type_;

double learning_rate_{0.0};

double beta1_{0.9};

double beta2_{0.999};

double eps_{1e-8};

int64_t step_{1};

int device_index_{0};
};

} // namespace infini::ops

#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_
63 changes: 63 additions & 0 deletions src/base/add_backward.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#ifndef INFINI_OPS_BASE_ADD_BACKWARD_H_
#define INFINI_OPS_BASE_ADD_BACKWARD_H_

#include <cassert>

#include "base/validation.h"
#include "operator.h"

namespace infini::ops {

class AddBackward : public Operator<AddBackward> {
public:
AddBackward(const Tensor grad_output, const double alpha, Tensor grad_input,
Tensor grad_other)
: grad_output_shape_{grad_output.shape()},
grad_output_strides_{grad_output.strides()},
grad_output_type_{grad_output.dtype()},
grad_input_shape_{grad_input.shape()},
grad_input_strides_{grad_input.strides()},
grad_input_type_{grad_input.dtype()},
grad_other_shape_{grad_other.shape()},
grad_other_strides_{grad_other.strides()},
grad_other_type_{grad_other.dtype()},
device_index_{grad_input.device().index()} {
validation::AssertSameDevice({grad_output, grad_input, grad_other});
assert(grad_output_type_ == grad_input_type_ &&
grad_input_type_ == grad_other_type_ &&
"`AddBackward` requires matching dtypes");

validation::AssertBroadcastsTo(grad_input, grad_output);

validation::AssertBroadcastsTo(grad_other, grad_output);
}

// Inputs may alias outputs; output byte spans must not overlap.
virtual void operator()(const Tensor grad_output, const double alpha,
Tensor grad_input, Tensor grad_other) const = 0;

protected:
Tensor::Shape grad_output_shape_;

Tensor::Strides grad_output_strides_;

DataType grad_output_type_;

Tensor::Shape grad_input_shape_;

Tensor::Strides grad_input_strides_;

DataType grad_input_type_;

Tensor::Shape grad_other_shape_;

Tensor::Strides grad_other_strides_;

DataType grad_other_type_;

int device_index_{0};
};

} // namespace infini::ops

#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_
64 changes: 64 additions & 0 deletions src/base/cross_entropy.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_H_
#define INFINI_OPS_BASE_CROSS_ENTROPY_H_

#include <cassert>
#include <cstdint>
#include <optional>
#include <string>

#include "base/cross_entropy_common.h"
#include "operator.h"

namespace infini::ops {

class CrossEntropy : public Operator<CrossEntropy> {
public:
static constexpr int64_t kIgnoreIndex = -100;

CrossEntropy(const Tensor input, const Tensor target,
std::optional<Tensor> weight, int64_t ignore_index,
std::string reduction, double label_smoothing, Tensor out)
: input_shape_{input.shape()},
input_strides_{input.strides()},
input_type_{input.dtype()},
target_shape_{target.shape()},
target_strides_{target.strides()},
target_type_{target.dtype()},
out_shape_{out.shape()},
out_strides_{out.strides()},
out_type_{out.dtype()},
device_index_{out.device().index()} {
ValidateCrossEntropy(input, target, weight, ignore_index, reduction,
label_smoothing, out);
}

virtual void operator()(const Tensor input, const Tensor target,
std::optional<Tensor> weight, int64_t ignore_index,
std::string reduction, double label_smoothing,
Tensor out) const = 0;

protected:
Tensor::Shape input_shape_;

Tensor::Strides input_strides_;

DataType input_type_;

Tensor::Shape target_shape_;

Tensor::Strides target_strides_;

DataType target_type_;

Tensor::Shape out_shape_;

Tensor::Strides out_strides_;

DataType out_type_;

int device_index_{0};
};

} // namespace infini::ops

#endif // INFINI_OPS_BASE_CROSS_ENTROPY_H_
77 changes: 77 additions & 0 deletions src/base/cross_entropy_backward.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_
#define INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_

#include <cassert>
#include <cstdint>

#include "base/cross_entropy_common.h"
#include "operator.h"

namespace infini::ops {

class CrossEntropyBackward : public Operator<CrossEntropyBackward> {
public:
static constexpr int64_t kIgnoreIndex = -100;

CrossEntropyBackward(const Tensor input, const Tensor target,
const Tensor grad_output, std::optional<Tensor> weight,
int64_t ignore_index, std::string reduction,
double label_smoothing, Tensor grad_input)
: input_shape_{input.shape()},
input_strides_{input.strides()},
input_type_{input.dtype()},
target_shape_{target.shape()},
target_strides_{target.strides()},
target_type_{target.dtype()},
grad_output_shape_{grad_output.shape()},
grad_output_strides_{grad_output.strides()},
grad_output_type_{grad_output.dtype()},
grad_input_shape_{grad_input.shape()},
grad_input_strides_{grad_input.strides()},
grad_input_type_{grad_input.dtype()},
device_index_{grad_input.device().index()} {
ValidateCrossEntropy(input, target, weight, ignore_index, reduction,
label_smoothing, grad_output);
validation::AssertSameDevice({input, grad_input});
assert(grad_input.shape() == input.shape() &&
grad_input.dtype() == input.dtype() &&
"`CrossEntropyBackward` grad_input must match logits metadata.");
}

virtual void operator()(const Tensor input, const Tensor target,
const Tensor grad_output,
std::optional<Tensor> weight, int64_t ignore_index,
std::string reduction, double label_smoothing,
Tensor grad_input) const = 0;

protected:
Tensor::Shape input_shape_;

Tensor::Strides input_strides_;

DataType input_type_;

Tensor::Shape target_shape_;

Tensor::Strides target_strides_;

DataType target_type_;

Tensor::Shape grad_output_shape_;

Tensor::Strides grad_output_strides_;

DataType grad_output_type_;

Tensor::Shape grad_input_shape_;

Tensor::Strides grad_input_strides_;

DataType grad_input_type_;

int device_index_{0};
};

} // namespace infini::ops

#endif // INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_
Loading