diff --git a/scripts/generate_torch_ops.py b/scripts/generate_torch_ops.py index 4fd92024b..8a5c1724b 100644 --- a/scripts/generate_torch_ops.py +++ b/scripts/generate_torch_ops.py @@ -1,1797 +1,1820 @@ -"""Generate InfiniOps PyTorch wrappers from ATen `native_functions.yaml`. - -For each op listed in `scripts/torch_ops.yaml`, this script finds the `.out` -variant in PyTorch's locally installed ATen schemas, including variants declared -through `autogen`, and emits: - - `generated/base/.h` — the InfiniOps base class - `class : public Operator<>`, with constructors and pure-virtual - `operator()` overloads mirroring the selected ATen schemas. - - `generated/torch//.h` and `.cc` — the PyTorch backend - `Operator<, kDev, 8>` that calls `at::_out(out, ...)`. - - `generated/torch_ops_metadata.json` — the kind (`unary` / `binary` / - `binary_alpha`) of every successfully-generated op, consumed by the - parametrized test suite. - -Slot 8 is the reserved convention for PyTorch backends; slots 0-7 are -left for native or vendor implementations. (The slot must also be > 0 -to side-step a partial-specialization-after-instantiation conflict with -the primary template `Operator<>` instantiated at index 0.) - -The generated files are not committed; CMake regenerates them at configure -time when `WITH_TORCH=ON`. -""" - -import argparse -import collections -import dataclasses -import importlib.util -import json -import os -import pathlib -import re -import shutil -import subprocess -import sys - -import yaml - -_SCRIPTS_DIR = pathlib.Path(__file__).resolve().parent -sys.path.insert(0, str(_SCRIPTS_DIR)) -import ops_config # noqa: E402 - -_REPO_ROOT = _SCRIPTS_DIR.parent -_OPS_YAML_PATH = _SCRIPTS_DIR / "torch_ops.yaml" -_BASE_DIR = _REPO_ROOT / "src" / "base" -_GENERATED_DIR = _REPO_ROOT / "generated" -_GENERATED_BASE_DIR = _GENERATED_DIR / "base" -_GENERATED_TORCH_DIR = _GENERATED_DIR / "torch" -_METADATA_PATH = _GENERATED_DIR / "torch_ops_metadata.json" - -# Reserved slot for PyTorch backends. Native and vendor implementations -# claim slots 0-7; PyTorch wrappers always live at 8. -_PYTORCH_SLOT = 8 - -# ATen uses symbolic names for some `int`/`float` defaults (e.g. -# `reduction=Mean`). Map them to C++ identifiers usable in a call. -_ENUM_DEFAULTS = { - "Mean": "at::Reduction::Mean", - "Sum": "at::Reduction::Sum", - "Contiguous": "at::MemoryFormat::Contiguous", -} - -# Default PyTorch schema label used only in diagnostics when CMake does not -# provide the locally installed torch version. Codegen reads the actual schema -# from installed `torchgen/packaged/ATen/native/native_functions.yaml`. -_DEFAULT_PYTORCH_VERSION = "v2.4.0" - -# YAML scalar-type tokens → C++ types. Reference types (e.g. `const Scalar&`) -# are not used so the generated signatures match the existing hand-written -# ones, which pass by value to keep pybind11 binding generation simple. -_SCALAR_TYPE_MAP = { - # `at::Scalar` is implicitly constructible from `double`, so we expose - # scalars as `double` in the base class to keep it torch-independent. - "Scalar": "double", - "int": "int64_t", - "bool": "bool", - "float": "double", - # `SymInt` / `SymInt[]` exist for `torch.compile` internals; at runtime - # they're just `int64`/IntArrayRef. - "SymInt": "int64_t", - # `str` for required string params (e.g. `index_reduce.reduce`). - # `std::string` marshals through pybind11 cleanly and converts - # implicitly to ATen's `c10::string_view`. - "str": "std::string", - "ScalarType": "DataType", -} - -# `Dimname` overloads (named-tensor dim) are skipped — passing them -# from Python to ATen requires a wrapper conversion through -# `at::Dimname::fromSymbol(...)` that doesn't fit the cleanly-rendered -# 1:1 arg model, and named tensors remain experimental in PyTorch. -# The int-dim overload is always emitted alongside, so we lose nothing -# user-visible. - -# Optional ATen types and their typed empty optional values at the call site. -# We use typed `c10::optional{}` values rather than bare `at::nullopt` so -# the compiler can disambiguate ops with multiple `_out` overloads (e.g. -# `clamp_out` accepts both `optional` and `optional` for -# `min`/`max`). -_NULLOPT_BY_TYPE = { - "Scalar?": "c10::optional{}", - "int?": "c10::optional{}", - "bool?": "c10::optional{}", - "float?": "c10::optional{}", - "str?": "c10::optional{}", - "ScalarType?": "c10::optional{}", - "MemoryFormat?": "c10::optional{}", - "Layout?": "c10::optional{}", - "Device?": "c10::optional{}", - "Generator?": "c10::optional{}", - "Tensor?": "c10::optional{}", - "Tensor?[]": "c10::List>{}", - "int[]?": "c10::optional{}", - "int[1]?": "c10::optional{}", - "int[2]?": "c10::optional{}", - "int[3]?": "c10::optional{}", - "SymInt?": "c10::optional{}", - "SymInt[]?": "c10::optional{}", - "SymInt[1]?": "c10::optional{}", - "SymInt[2]?": "c10::optional{}", - "SymInt[3]?": "c10::optional{}", - "float[]?": "c10::optional>{}", -} - -# Optional ATen types that have a stable InfiniOps representation. These are -# exposed in generated base signatures and converted back to ATen optionals in -# the PyTorch backend. PyTorch-internal concepts without a public InfiniOps -# analogue stay hidden and are still passed as typed empty optionals. -_EXPOSED_OPTIONAL_CPP_TYPES = { - "Scalar?": "std::optional", - "int?": "std::optional", - "bool?": "std::optional", - "float?": "std::optional", - "str?": "std::optional", - "ScalarType?": "std::optional", - "Tensor?": "std::optional", - "Tensor?[]": "std::vector>", - "int[]?": "std::optional>", - "int[1]?": "std::optional>", - "int[2]?": "std::optional>", - "int[3]?": "std::optional>", - "SymInt?": "std::optional", - "SymInt[]?": "std::optional>", - "SymInt[1]?": "std::optional>", - "SymInt[2]?": "std::optional>", - "SymInt[3]?": "std::optional>", - "float[]?": "std::optional>", -} -_HARDCODE_NULLOPT_TYPES = frozenset( - set(_NULLOPT_BY_TYPE) - set(_EXPOSED_OPTIONAL_CPP_TYPES) -) - - -def _normalize_cpp_type(cpp_type: str) -> str: - text = cpp_type.strip() - - if text.startswith("const "): - text = text[len("const ") :] - - return text.rstrip("&").strip() - - -def _write_text_if_changed(path: pathlib.Path, content: str) -> bool: - """Write `content` only when the file's bytes would change.""" - path.parent.mkdir(parents=True, exist_ok=True) - - if path.exists() and path.read_text() == content: - return False - - path.write_text(content) - return True - - -def _prune_empty_dirs(root: pathlib.Path) -> None: - if not root.exists(): - return - - for child in sorted(root.iterdir(), reverse=True): - if child.is_dir(): - _prune_empty_dirs(child) - - if root.is_dir() and not any(root.iterdir()): - root.rmdir() - - -def _remove_stale_files(root: pathlib.Path, expected_files: set[pathlib.Path]) -> None: - if not root.exists(): - return - - for path in sorted(root.rglob("*"), reverse=True): - if path.is_file() and path not in expected_files: - path.unlink() - - _prune_empty_dirs(root) - - -@dataclasses.dataclass -class Param: - name: str - aten_type: str - default: str | None - keyword_only: bool - cpp_type_override: str | None = None - - @property - def api_name(self) -> str: - if self.name == "self": - return "input" - - return self.name - - @property - def is_tensor(self) -> bool: - if self.cpp_type_override is not None: - return _normalize_cpp_type(self.cpp_type_override) == "Tensor" - - # Real tensors only. `Tensor?` and `Tensor[]` have separate handling. - if self.aten_type.endswith("[]"): - return False - - return self.aten_type == "Tensor" or self.aten_type.startswith("Tensor(") - - @property - def is_optional_tensor(self) -> bool: - if self.cpp_type_override is not None: - return ( - _normalize_cpp_type(self.cpp_type_override) == "std::optional" - ) - - return self.aten_type == "Tensor?" - - @property - def is_tensor_list(self) -> bool: - if self.cpp_type_override is not None: - return _normalize_cpp_type(self.cpp_type_override) == "std::vector" - - return self.aten_type == "Tensor[]" - - @property - def is_optional_tensor_list(self) -> bool: - if self.cpp_type_override is not None: - return ( - _normalize_cpp_type(self.cpp_type_override) - == "std::vector>" - ) - - return self.aten_type == "Tensor?[]" - - @property - def is_mutable_tensor(self) -> bool: - # Mutable tensors carry `!` in their alias annotation, e.g. - # `Tensor(a!)`. - - return self.is_tensor and "!" in self.aten_type - - @property - def is_out(self) -> bool: - # In ATen `_out` schemas, output tensors are keyword-only mutable tensor - # params. Some mutable tensors are real inputs (`running_mean` / - # `running_var` in `_batch_norm_with_update`) and must stay in schema - # order, so mutability alone is not enough. - - return self.is_mutable_tensor and self.keyword_only - - @property - def is_hardcoded_nullopt(self) -> bool: - """If `True`, the param is omitted from the user-facing API and - passed as `at::nullopt` to ATen.""" - - return self.aten_type in _HARDCODE_NULLOPT_TYPES - - @property - def is_hidden(self) -> bool: - """True if the param is omitted from the user-facing API. - - Default-valued non-optional params (\\`bool\\`, \\`int\\`, \\`float\\`, - \\`str\\`, \\`int[N]\\`, …) used to be hidden as a convenience, but - reviewers consistently flagged the resulting omissions — - \\`bool upper/transpose/unitriangular\\` on \\`triangular_solve\\`, - \\`int diagonal\\` on \\`triu\\`, \\`str ord\\` on \\`linalg_matrix_norm\\`, - \\`int n\\` on the special chebyshev family, etc. — as missing - semantic controls. They are now exposed and forwarded to ATen. - - Optional ATen types with a stable InfiniOps representation - (\\`Tensor?\\`, \\`Scalar?\\`, \\`int?\\`, …) are exposed as - \\`std::optional\\`. ATen-internal types like - \\`Generator?\\`/\\`Layout?\\` still have no InfiniOps analogue, so - they remain hidden. - """ - - return self.is_hardcoded_nullopt - - def hidden_value(self) -> str: - """C++ literal substituted for a hidden param in the ATen call.""" - - if self.is_hardcoded_nullopt: - return _NULLOPT_BY_TYPE[self.aten_type] - - if self.default == "True": - return "true" - - if self.default == "False": - return "false" - - if self.aten_type.startswith(("int[", "SymInt[")) and self.default is not None: - # `int[N]=[a, b, c]` → `{a, b, c}`; `int[N]=0` (scalar default - # for list type) → `{0, 0, ...}` replicated to size N. - if self.default.startswith("["): - return "{" + self.default[1:-1] + "}" - - size_match = re.search(r"\[(\d+)\]", self.aten_type) - n = int(size_match.group(1)) if size_match else 1 - - return "{" + ", ".join([self.default] * n) + "}" - - if self.aten_type == "str" and self.default is not None: - # YAML uses single-quoted strings (e.g. `'none'`); C++ char - # literals also use single quotes, so swap to doubles. - - return '"' + self.default.strip("'\"") + '"' - - if self.aten_type in {"int", "float", "SymInt"} and self.default is not None: - # Translate known ATen enum defaults to their C++ identifiers. - - return _ENUM_DEFAULTS.get(self.default, self.default) - - raise AssertionError( - f"param {self.name!r} of type {self.aten_type!r} with default " - f"{self.default!r} is not hidden" - ) - - @property - def cpp_type(self) -> str: - if self.cpp_type_override is not None: - return self.cpp_type_override - - if self.is_tensor: - return "Tensor" - - if self.is_tensor_list: - return "std::vector" - - if self.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: - return _EXPOSED_OPTIONAL_CPP_TYPES[self.aten_type] - - if self.is_hidden: - # Not exposed — the ATen call substitutes a hardcoded value - # so the `cpp_type` is irrelevant. - - return "void" - - bare = self.aten_type.rstrip("?") - # Required `int[N]` / `SymInt[N]` (no default) — pybind11 accepts - # a Python list of ints into `std::vector`, which ATen - # promotes to `IntArrayRef` implicitly. - if bare.startswith(("int[", "SymInt[")) or bare in {"int[]", "SymInt[]"}: - return "std::vector" - - try: - return _SCALAR_TYPE_MAP[bare] - except KeyError as exc: - raise NotImplementedError( - f"unsupported ATen type {self.aten_type!r} for param {self.name!r}" - ) from exc - - -@dataclasses.dataclass -class Op: - aten_name: str - overload: str - params: list[Param] - signature_params: list[Param] | None = None - param_bindings: list[Param | None] | None = None - explicit_inplace_out: Param | None = None - - @property - def pascal_name(self) -> str: - return _snake_to_pascal(self.infini_name) - - @property - def infini_name(self) -> str: - """InfiniOps public op name. - - ATen disambiguates `_out` overloads with suffixes like `Tensor_Tensor_out`, - `out_x`, `forward_output`, `grad_input`, but reviewers consistently - flag those suffixes as bad public-API naming when they leak into - InfiniOps class names. Different ATen overloads of the same base op - become overloaded `operator()` methods on a single class instead. When - two overloads collapse to the same visible C++ signature after hidden - defaults, `_dedupe_visible_overloads` keeps only one. - - ATen-internal leading underscores and in-place trailing underscores are - also normalized so generated class names do not collide with existing - public ops (`_softmax` → `AtenSoftmax`, `add_` → `AddInplace`). - """ - - return _public_op_name(self.aten_name) - - @property - def is_inplace(self) -> bool: - """True for ATen's single-underscore in-place variants.""" - - return _is_inplace_aten_name(self.aten_name) - - @property - def tensor_params(self) -> list[Param]: - return [p for p in self.params if p.is_tensor] - - @property - def out_params(self) -> list[Param]: - """Mutable tensor outputs. Most ops have one (`Tensor(a!) out`); - multi-output ops like `frexp` or `sort` have several - (`Tensor(a!) values`, `Tensor(b!) indices`).""" - - if self.explicit_inplace_out is not None: - return [self.explicit_inplace_out] - - if self.is_inplace: - return [self.params[0]] - - return [p for p in self.params if p.is_out] - - @property - def out_param(self) -> Param: - """Single-output convenience. Asserts there's exactly one.""" - outs = self.out_params - assert len(outs) == 1, f"op {self.aten_name!r} has {len(outs)} out tensors" - - return outs[0] - - @property - def visible_params(self) -> list[Param]: - """Params the wrapper exposes to the user; hidden ones (hardcoded - optional nullopt, default-`False`/`True` bools) are filtered.""" - - if self.signature_params is not None: - return self.signature_params - - return [p for p in self.params if not p.is_hidden] - - def api_param_for(self, schema_index: int) -> Param | None: - """Return the public signature param bound to a schema param.""" - - if self.param_bindings is not None: - return self.param_bindings[schema_index] - - param = self.params[schema_index] - - if param.is_hidden: - return None - - return param - - @property - def is_testable(self) -> bool: - """Cheap structural check: at least one out tensor, and the first - constructor parameter is a tensor. The latter is needed because - `Operator::Make(Tensor tensor, Args... args)` dispatches on - `tensor.device()`, so an op like `pow.Scalar_out(Scalar self, - Tensor exponent, *, Tensor(a!) out)` cannot be wired up without - a separate dispatch path. Generators like `arange` / `linspace` - also fall under this rule (no input tensors at all).""" - - if not self.out_params: - return False - - if self.is_inplace: - return self.params[0].is_mutable_tensor - - # `params` includes out tensors at the end; check the first - # non-out param. If there are no non-out params (`empty.out`, - # `arange.out`), this op also fails the dispatch precondition. - non_out = [p for p in self.params if not p.is_out] - - if not non_out: - return False - - return non_out[0].is_tensor or non_out[0].is_tensor_list - - -_FUNC_RE = re.compile( - r"^(?P[a-zA-Z_][a-zA-Z0-9_]*)" - r"(?:\.(?P\w+))?" - r"\((?P.*)\)\s*->\s*.+$" -) - -_ARG_RE = re.compile( - r"^(?P\S+(?:\([^)]*\))?\??)" # type with optional alias and `?` - r"\s+(?P\w+)" - r"(?:\s*=\s*(?P.+))?$" -) - - -def _parse_func(func_str: str) -> Op: - m = _FUNC_RE.match(func_str) - - if not m: - raise ValueError(f"could not parse func: {func_str!r}") - - return Op( - aten_name=m.group("name"), - overload=m.group("overload") or "", - params=_parse_args(m.group("args")), - ) - - -def _parse_args(args_str: str) -> list[Param]: - params: list[Param] = [] - keyword_only = False - - for token in _split_args(args_str): - if token == "*": - keyword_only = True - continue - - params.append(_parse_one_arg(token, keyword_only)) - - return params - - -def _split_args(args_str: str) -> list[str]: - """Split on top-level commas, respecting `(...)` and `[...]`.""" - parts: list[str] = [] - depth = 0 - current: list[str] = [] - - for ch in args_str: - if ch in "([": - depth += 1 - current.append(ch) - elif ch in ")]": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - piece = "".join(current).strip() - - if piece: - parts.append(piece) - - current = [] - else: - current.append(ch) - - tail = "".join(current).strip() - - if tail: - parts.append(tail) - - return parts - - -def _parse_one_arg(token: str, keyword_only: bool) -> Param: - m = _ARG_RE.match(token) - - if not m: - raise ValueError(f"could not parse arg: {token!r}") - - name = m.group("name") - return Param( - name=name, - aten_type=m.group("type"), - default=m.group("default"), - keyword_only=keyword_only, - ) - - -_BASE_OPERATOR_RE = re.compile( - r"\bvirtual\s+void\s+operator\(\)\s*\((?P.*?)\)\s*const" - r"(?:\s*=\s*0|\s*\{)", - re.S, -) - - -def _split_cpp_params(args_str: str) -> list[str]: - parts: list[str] = [] - depth = 0 - current: list[str] = [] - - for ch in args_str: - if ch in "([<": - depth += 1 - current.append(ch) - elif ch in ")]>": - depth -= 1 - current.append(ch) - elif ch == "," and depth == 0: - piece = "".join(current).strip() - - if piece: - parts.append(piece) - - current = [] - else: - current.append(ch) - - tail = "".join(current).strip() - - if tail: - parts.append(tail) - - return parts - - -def _parse_base_signature_param(token: str) -> tuple[str, str]: - token = token.split("=", 1)[0].strip() - token = re.sub(r"\s+", " ", token) - match = re.match(r"(?P.+?)\s+(?P\w+)$", token) - - if not match: - raise ValueError(f"could not parse base operator param: {token!r}") - - return _normalize_cpp_type(match.group("type")), match.group("name") - - -def _parse_base_operator_signatures( - base_path: pathlib.Path, -) -> list[list[tuple[str, str]]]: - source = base_path.read_text() - signatures = [] - - for match in _BASE_OPERATOR_RE.finditer(source): - args = match.group("args").strip() - - if not args: - signatures.append([]) - continue - - signatures.append( - [_parse_base_signature_param(token) for token in _split_cpp_params(args)] - ) - - return signatures - - -def _cpp_types_compatible(schema_cpp_type: str, base_cpp_type: str) -> bool: - schema = _normalize_cpp_type(schema_cpp_type) - base = _normalize_cpp_type(base_cpp_type) - - if schema == base: - return True - - compatible_pairs = { - ("double", "float"), - ("int64_t", "int"), - ("std::optional", "std::optional"), - ("std::optional", "std::optional"), - } - - return (schema, base) in compatible_pairs - - -def _cpp_param_compatible( - schema_param: Param, base_cpp_type: str, base_name: str -) -> bool: - if schema_param.api_name != base_name: - return False - - return _cpp_types_compatible(schema_param.cpp_type, base_cpp_type) - - -def _bind_base_signature(op: Op, signature: list[tuple[str, str]]) -> Op | None: - bindings: list[Param | None] = [None] * len(op.params) - signature_params = [] - explicit_inplace_out = None - - if ( - op.is_inplace - and signature - and signature[-1][1] == "out" - and _cpp_types_compatible("Tensor", signature[-1][0]) - ): - out_cpp_type, out_name = signature[-1] - explicit_inplace_out = Param( - name=out_name, - aten_type="Tensor(a!)", - default=None, - keyword_only=True, - cpp_type_override=out_cpp_type, - ) - signature = signature[:-1] - - for base_cpp_type, base_name in signature: - matched_index = None - - for index, schema_param in enumerate(op.params): - if bindings[index] is not None: - continue - - if _cpp_param_compatible(schema_param, base_cpp_type, base_name): - matched_index = index - break - - if matched_index is None: - return None - - schema_param = op.params[matched_index] - api_param = dataclasses.replace( - schema_param, name=base_name, cpp_type_override=base_cpp_type - ) - bindings[matched_index] = api_param - signature_params.append(api_param) - - if any( - binding is None and not _is_omittable_param(param) - for param, binding in zip(op.params, bindings) - ): - return None - - if explicit_inplace_out is not None: - signature_params.append(explicit_inplace_out) - - return dataclasses.replace( - op, - signature_params=signature_params, - param_bindings=bindings, - explicit_inplace_out=explicit_inplace_out, - ) - - -def _bind_existing_base_overloads( - name: str, ops: list[Op] -) -> tuple[list[Op], list[str]]: - base_path = _base_path(name) - signatures = _parse_base_operator_signatures(base_path) - bound_ops = [] - warnings = [] - - for signature in signatures: - matches = [ - bound - for op in ops - if (bound := _bind_base_signature(op, signature)) is not None - ] - - if not matches: - rendered = ", ".join(f"{cpp_type} {param}" for cpp_type, param in signature) - warnings.append( - f"base overload `operator()({rendered})` does not match any " - "usable ATen schema" - ) - continue - - matches.sort( - key=lambda match: sum( - binding is None for binding in match.param_bindings or () - ) - ) - bound_ops.append(matches[0]) - - return bound_ops, warnings - - -def _snake_to_pascal(s: str) -> str: - return "".join(p.capitalize() for p in s.split("_")) - - -_OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") - - -def _is_inplace_aten_name(name: str) -> bool: - """Return whether `name` is an ATen in-place operator name.""" - - return name.endswith("_") and not name.endswith("__") - - -def _public_op_name(aten_name: str) -> str: - """Map ATen-only spelling to stable InfiniOps public names.""" - - public_name = aten_name - file_prefixes = [] - - if public_name.startswith("_"): - file_prefixes.append("internal") - public_name = public_name.lstrip("_") - - if _is_inplace_aten_name(public_name): - public_name = public_name[:-1] - - return "_".join(file_prefixes + [public_name]) - - -def _op_namespace_parts(name: str) -> tuple[str, ...]: - parts = name.split("_") - namespaces = [] - - if parts and parts[0] == "internal": - namespaces.append(parts.pop(0)) - - if parts and parts[0] in _OP_NAMESPACE_PREFIXES: - namespaces.append(parts.pop(0)) - - return tuple(namespaces) - - -def _op_class_stem(name: str) -> str: - parts = name.split("_") - - if parts and parts[0] == "internal": - parts.pop(0) - - if parts and parts[0] in _OP_NAMESPACE_PREFIXES: - parts.pop(0) - - return "_".join(parts) - - -def _op_class_name(name: str) -> str: - return _snake_to_pascal(_op_class_stem(name)) - - -def _op_cpp_type(name: str) -> str: - parts = (*_op_namespace_parts(name), _op_class_name(name)) - - return "::".join(parts) - - -def _op_namespace_open(name: str) -> str: - parts = _op_namespace_parts(name) - - if not parts: - return "namespace infini::ops {" - - return f"namespace infini::ops::{'::'.join(parts)} {{" - - -def _op_namespace_close(name: str) -> str: - parts = _op_namespace_parts(name) - - if not parts: - return "} // namespace infini::ops" - - return f"}} // namespace infini::ops::{'::'.join(parts)}" - - -def _base_path(op_name: str) -> pathlib.Path: - return _BASE_DIR / f"{op_name}.h" - - -def _is_omittable_param(param: Param) -> bool: - return param.aten_type in _NULLOPT_BY_TYPE or param.default is not None - - -def _default_call_value(param: Param) -> str: - if param.aten_type in _NULLOPT_BY_TYPE: - return _NULLOPT_BY_TYPE[param.aten_type] - - if param.default is not None: - return _translate_default(param) - - return param.hidden_value() - - -def _load_aten_entries(version: str) -> list[dict]: - """Return parsed ATen schemas bundled with installed `torchgen`. - - `WITH_TORCH=ON` already requires a local PyTorch installation. PyTorch - wheels ship the matching ATen schema under `torchgen/packaged`, including - vendor forks, so codegen should not depend on fetching PyTorch sources from - GitHub during CI builds. The official parser expands schemas declared via - `autogen`, such as `relu.out`, before InfiniOps filters for out variants. - """ - - try: - packaged, expanded_autogen = _load_packaged_aten_entries() - except RuntimeError as error: - raise RuntimeError(f"{error} for PyTorch schema {version!r}") from error - - print( - "using installed `torchgen` ATen schemas " - f"for PyTorch schema {version} " - f"({'with' if expanded_autogen else 'without'} `autogen` expansion).", - file=sys.stderr, - ) - - return packaged - - -def _load_packaged_aten_entries() -> tuple[list[dict], bool]: - """Parse the ATen schemas bundled with installed `torchgen`. - - PyTorch wheels install `torchgen/packaged/ATen/native/native_functions.yaml`; - using it lets offline CI images generate wrappers against the exact schema - shipped with their installed torch fork. - """ - - spec = importlib.util.find_spec("torchgen") - - if spec is None: - raise RuntimeError("could not find the installed `torchgen` package") - - if spec.submodule_search_locations is None: - raise RuntimeError("installed `torchgen` is not a package") - - for root in spec.submodule_search_locations: - native_dir = pathlib.Path(root) / "packaged" / "ATen" / "native" - native_path = native_dir / "native_functions.yaml" - tags_path = native_dir / "tags.yaml" - - if not native_path.is_file(): - continue - - if not tags_path.is_file(): - print( - "warning: could not expand ATen `autogen` schemas because " - f"`{tags_path}` is missing; using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - try: - from torchgen.gen import parse_native_yaml - except (ImportError, AttributeError) as error: - print( - "warning: could not expand ATen `autogen` schemas because " - f"`torchgen.gen.parse_native_yaml` is unavailable ({error}); " - "using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - try: - parsed = parse_native_yaml(str(native_path), str(tags_path)) - except Exception as error: - print( - "warning: could not expand ATen `autogen` schemas from " - f"`{native_path}` ({type(error).__name__}: {error}); " - "using explicit schemas only.", - file=sys.stderr, - ) - - return _load_explicit_aten_entries(native_path), False - - return ( - [{"func": str(function.func)} for function in parsed.native_functions], - True, - ) - - raise RuntimeError( - "could not find installed `torchgen` packaged `native_functions.yaml`" - ) - - -def _load_explicit_aten_entries(native_path: pathlib.Path) -> list[dict]: - entries = yaml.safe_load(native_path.read_text()) - - if not isinstance(entries, list): - raise RuntimeError( - f"expected a list of ATen schemas in `{native_path}`, " - f"got {type(entries).__name__}" - ) - - return entries - - -def _find_out_entries(entries: list[dict], op_name: str) -> list[dict]: - """Return all out-variant entries for `op_name`, with the bare - `.out(` form first and overload-suffixed variants - (e.g. `pow.Tensor_Tensor_out(`, `kthvalue.values(`) after. An - entry counts as an out-variant when it (a) is named - `.out`, (b) ends in `_out`, or (c) carries a - `Tensor(!)` mutability annotation — that last case covers - multi-output ops named after their output tensors - (`kthvalue.values`, `mode.values`, …).""" - bare_prefix = f"{op_name}.out(" - op_overload = re.compile(rf"^{re.escape(op_name)}\.\w+\(") - mut_tensor = re.compile(r"Tensor\([a-z]!\)") - bare: list[dict] = [] - others: list[dict] = [] - - for entry in entries: - func = entry.get("func", "") - - if func.startswith(bare_prefix): - bare.append(entry) - elif op_overload.match(func) and ( - func.split("(", 1)[0].endswith("_out") or mut_tensor.search(func) - ): - others.append(entry) - - return bare + others - - -def _format_signature(op: Op, *, include_defaults: bool = False) -> str: - parts = [] - - for param in op.visible_params: - prefix = "" if param.is_mutable_tensor else "const " - text = f"{prefix}{param.cpp_type} {param.api_name}" - - if include_defaults and param.default is not None: - text += f" = {_translate_default(param)}" - - parts.append(text) - - return ", ".join(parts) - - -def _visible_signature_key(op: Op) -> tuple[str, ...]: - """C++ overload identity for the user-facing API. - - Parameter names and top-level `const` do not distinguish C++ overloads, so - only the exposed C++ type sequence participates in duplicate detection. - """ - - return tuple(param.cpp_type for param in op.visible_params) - - -def _canonical_overload_score(index: int, op: Op) -> tuple[bool, int, int, str, int]: - """Sort key for duplicate visible signatures. - - Prefer the canonical unsuffixed InfiniOps name, then the schema that hides - fewer ATen-only defaults, then the shorter deterministic name. - """ - - return ( - op.infini_name != op.aten_name, - sum(param.is_hidden for param in op.params), - len(op.infini_name), - op.infini_name, - index, - ) - - -def _dedupe_visible_overloads(ops: list[Op]) -> tuple[list[Op], list[tuple[Op, Op]]]: - """Drop overloads that collapse to the same visible C++ signature. - - Returns the selected overloads in the original schema order plus a list of - `(skipped, kept)` duplicate pairs for diagnostics. - """ - winners: dict[tuple[str, ...], tuple[int, Op]] = {} - duplicates: list[tuple[Op, tuple[str, ...]]] = [] - - for index, op in enumerate(ops): - key = _visible_signature_key(op) - current = winners.get(key) - - if current is None: - winners[key] = (index, op) - continue - - current_index, current_op = current - - if _canonical_overload_score(index, op) < _canonical_overload_score( - current_index, current_op - ): - duplicates.append((current_op, key)) - winners[key] = (index, op) - else: - duplicates.append((op, key)) - - selected_indices = {index for index, _ in winners.values()} - selected = [op for index, op in enumerate(ops) if index in selected_indices] - duplicate_pairs = [ - (skipped, winners[key][1]) - for skipped, key in duplicates - if winners[key][1] is not skipped - ] - - return selected, duplicate_pairs - - -def _translate_default(param: Param) -> str: - """Translate a YAML default literal to a C++ literal.""" - raw = param.default - - if raw == "True": - return "true" - - if raw == "False": - return "false" - - if raw == "None": - return "{}" - - return raw # numeric literals (`0`, `1`, `1.0`) pass through - - -def _generate_base_includes(ops: list[Op]) -> str: - cpp_types = [param.cpp_type for op in ops for param in op.visible_params] - includes = [] - - if any("std::optional" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - if any("std::string" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - if any("std::vector" in cpp_type for cpp_type in cpp_types): - includes.append("#include ") - - includes.append('#include "operator.h"') - - return "\n".join(includes) - - -def _generate_base_header(name: str, ops: list[Op]) -> str: - pascal = _op_class_name(name) - - member_decls = [] - tensor_member_order = [] - seen_tensor_members = set() - optional_tensor_member_order = [] - seen_optional_tensor_members = set() - scalar_member_order = [] - scalar_member_types = {} - - for op in ops: - for param in op.tensor_params: - api_name = param.api_name - - if api_name in seen_tensor_members: - continue - - seen_tensor_members.add(api_name) - tensor_member_order.append(api_name) - member_decls.append(f" Tensor::Shape {api_name}_shape_;") - member_decls.append(f" Tensor::Strides {api_name}_strides_;") - member_decls.append(f" DataType {api_name}_type_;") - - for param in op.visible_params: - api_name = param.api_name - - if not param.is_optional_tensor or api_name in seen_optional_tensor_members: - continue - - seen_optional_tensor_members.add(api_name) - optional_tensor_member_order.append(api_name) - member_decls.append(f" bool has_{api_name}_{{false}};") - member_decls.append(f" Tensor::Shape {api_name}_shape_;") - member_decls.append(f" Tensor::Strides {api_name}_strides_;") - member_decls.append(f" DataType {api_name}_type_{{DataType::kFloat32}};") - - # Visible non-tensor params (scalars, strings, vectors) are also - # stored on the base so backends can dispatch on them later — not - # only at the moment `operator()` is invoked. Reviewers flagged - # this on multiple PRs (e.g. `n` on - # `special_chebyshev_polynomial_v_n_scalar`). Same-named params - # across overloads must share a type; if they conflict, the second - # overload's member is dropped (later constructors leave it - # default-initialised). - for param in op.visible_params: - api_name = param.api_name - - if ( - param.is_tensor - or param.is_optional_tensor - or api_name in scalar_member_types - ): - continue - - scalar_member_order.append(api_name) - scalar_member_types[api_name] = param.cpp_type - member_decls.append(f" {param.cpp_type} {api_name}_{{}};") - - member_decls.append(" int device_index_{0};") - - constructors = [] - calls = [] - - for op in ops: - init_pieces = [] - tensor_params = {param.api_name: param for param in op.tensor_params} - optional_tensor_params = { - param.api_name: param - for param in op.visible_params - if param.is_optional_tensor - } - scalar_params = { - param.api_name: param - for param in op.visible_params - if not param.is_tensor - and not param.is_optional_tensor - and scalar_member_types.get(param.api_name) == param.cpp_type - } - - for param_name in tensor_member_order: - param = tensor_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" {api_name}_shape_{{{api_name}.shape()}}") - init_pieces.append(f" {api_name}_strides_{{{api_name}.strides()}}") - init_pieces.append(f" {api_name}_type_{{{api_name}.dtype()}}") - - for param_name in optional_tensor_member_order: - param = optional_tensor_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" has_{api_name}_{{{api_name}.has_value()}}") - init_pieces.append( - f" {api_name}_shape_{{{api_name} ? " - f"Tensor::Shape{{{api_name}->shape()}} : Tensor::Shape{{}}}}" - ) - init_pieces.append( - f" {api_name}_strides_{{{api_name} ? " - f"Tensor::Strides{{{api_name}->strides()}} : Tensor::Strides{{}}}}" - ) - init_pieces.append( - f" {api_name}_type_{{{api_name} ? " - f"{api_name}->dtype() : DataType::kFloat32}}" - ) - - for param_name in scalar_member_order: - param = scalar_params.get(param_name) - - if param is None: - continue - - api_name = param.api_name - init_pieces.append(f" {api_name}_{{{api_name}}}") - - # All out tensors share a device; use the first one. Keep this last - # so initializer order follows the member declaration order. - init_pieces.append( - f" device_index_{{{op.out_params[0].api_name}.device().index()}}" - ) - - init_list = ",\n".join(init_pieces).lstrip() - constructors.append( - f" {pascal}({_format_signature(op)})\n : {init_list} {{}}" - ) - calls.append(f" virtual void operator()({_format_signature(op)}) const = 0;") - - return _BASE_TEMPLATE.format( - name_uc=name.upper(), - pascal=pascal, - namespace_open=_op_namespace_open(name), - namespace_close=_op_namespace_close(name), - includes=_generate_base_includes(ops), - constructors="\n\n".join(constructors), - op_calls="\n\n".join(calls), - member_decls="\n\n".join(member_decls), - ) - - -def _generate_torch_header(name: str, ops: list[Op]) -> str: - pascal = _op_class_name(name) - op_type = _op_cpp_type(name) - op_calls = "\n\n".join( - f" void operator()({_format_signature(op)}) const override;" for op in ops - ) - - return _TORCH_HEADER_TEMPLATE.format( - name_uc=name.upper(), - name=name, - pascal=pascal, - op_type=op_type, - op_calls=op_calls, - slot=_PYTORCH_SLOT, - ) - - -def _generate_torch_method_source(name: str, op: Op) -> str: - op_type = _op_cpp_type(name) - conversion_lines = [] - out_device_index = f"{op.out_params[0].api_name}.device().index()" - - def _optional_aten_type(param: Param) -> str: - return _NULLOPT_BY_TYPE[param.aten_type].removesuffix("{}") - - def _optional_aten_value(schema_param: Param, api_param: Param) -> str: - api_name = api_param.api_name - - if schema_param.aten_type == "Tensor?": - data_expr = f"const_cast({api_name}->data())" - - return ( - f"ToAtenTensor({data_expr}, {api_name}->shape(), " - f"{api_name}->strides(), {api_name}->dtype(), " - f"device_index)" - ) - - if schema_param.aten_type == "Scalar?": - return f"at::Scalar(*{api_name})" - - if schema_param.aten_type == "ScalarType?": - return f"ToAtenDataType(*{api_name})" - - if schema_param.aten_type == "str?": - return f"c10::string_view(*{api_name})" - - if schema_param.aten_type.startswith( - ("int[", "SymInt[") - ) or schema_param.aten_type in { - "int[]?", - "SymInt[]?", - }: - return f"at::IntArrayRef(*{api_name})" - - if schema_param.aten_type == "float[]?": - return f"at::ArrayRef(*{api_name})" - - return f"*{api_name}" - - def _append_optional_conversion(schema_param: Param, api_param: Param) -> None: - api_name = api_param.api_name - optional_type = _optional_aten_type(schema_param) - conversion_lines.append(f" {optional_type} at_{schema_param.name};") - conversion_lines.append(f" if ({api_name}.has_value()) {{") - conversion_lines.append( - f" at_{schema_param.name} = " - f"{optional_type}{{{_optional_aten_value(schema_param, api_param)}}};" - ) - conversion_lines.append(" }") - - for schema_index, param in enumerate(op.params): - if ( - not param.is_tensor - and not param.is_tensor_list - and not param.is_optional_tensor_list - ): - continue - - api_param = op.api_param_for(schema_index) - - if api_param is None: - continue - - api_name = api_param.api_name - - if param.is_optional_tensor_list: - conversion_lines.append( - f" c10::List> at_{param.name};" - ) - conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") - conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") - conversion_lines.append(" if (tensor.has_value()) {") - conversion_lines.append( - " at_" - f"{param.name}.push_back(ToAtenTensor(" - "const_cast(tensor->data()), tensor->shape(), " - "tensor->strides(), tensor->dtype(), device_index));" - ) - conversion_lines.append(" } else {") - conversion_lines.append(f" at_{param.name}.push_back(c10::nullopt);") - conversion_lines.append(" }") - conversion_lines.append(" }") - continue - - if param.is_tensor_list: - conversion_lines.append(f" std::vector at_{param.name};") - conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") - conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") - conversion_lines.append( - " at_" - f"{param.name}.push_back(ToAtenTensor(" - "const_cast(tensor.data()), tensor.shape(), tensor.strides(), " - "tensor.dtype(), device_index));" - ) - conversion_lines.append(" }") - continue - - use_mutable_data = param.is_mutable_tensor and not ( - op.explicit_inplace_out is not None and schema_index == 0 - ) - data_expr = ( - f"{api_name}.data()" - if use_mutable_data - else f"const_cast({api_name}.data())" - ) - conversion_lines.append( - f" auto at_{param.name} = ToAtenTensor(\n" - f" {data_expr}, {api_name}_shape_, {api_name}_strides_,\n" - f" {api_name}_type_, device_index);" - ) - - if op.explicit_inplace_out is not None: - out_name = op.explicit_inplace_out.api_name - conversion_lines.append( - f" auto at_{out_name} = ToAtenTensor(\n" - f" {out_name}.data(), {out_name}_shape_, {out_name}_strides_,\n" - f" {out_name}_type_, device_index);" - ) - - for schema_index, param in enumerate(op.params): - api_param = op.api_param_for(schema_index) - - if ( - api_param is not None - and param.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES - and not param.is_optional_tensor_list - ): - _append_optional_conversion(param, api_param) - - def _render_arg(schema_index, p): - api_param = op.api_param_for(schema_index) - - if api_param is None: - return _default_call_value(p) - - if p.is_hidden: - return _default_call_value(p) - - if p.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: - return f"at_{p.name}" - - if p.is_tensor or p.is_tensor_list: - return f"at_{p.name}" - - if p.aten_type == "ScalarType": - return f"ToAtenDataType({api_param.api_name})" - - return api_param.api_name - - if op.is_inplace: - # In-place ATen calls keep the mutable input in positional order, - # unlike `_out` calls which place output tensors first. - input_param = op.params[0] - arg_order = op.params[1:] - rendered_args = ", ".join( - _render_arg(index + 1, p) for index, p in enumerate(arg_order) - ) - if op.explicit_inplace_out is not None: - out_name = op.explicit_inplace_out.api_name - aten_call = f"at_{out_name}.copy_(at_{input_param.name});\n" - aten_call += f" at_{out_name}.{op.aten_name}({rendered_args})" - else: - aten_call = f"at_{input_param.name}.{op.aten_name}({rendered_args})" - else: - # ATen `_out` form puts all out tensors first, then non-out params - # in YAML order. Hardcoded-nullopt params become `at::nullopt`. - arg_order = [(index, p) for index, p in enumerate(op.params) if p.is_out] + [ - (index, p) for index, p in enumerate(op.params) if not p.is_out - ] - rendered_args = ", ".join(_render_arg(index, p) for index, p in arg_order) - aten_call = f"at::{op.aten_name}_out({rendered_args})" - - return _TORCH_METHOD_TEMPLATE.format( - op_type=op_type, - op_call_signature=_format_signature(op), - tensor_conversions="\n".join(conversion_lines), - out_device_index=out_device_index, - # The generated call expression resolves the right kernel via C++ - # overload resolution from the argument types we pass. - aten_call=aten_call, - slot=_PYTORCH_SLOT, - ) - - -def _generate_torch_source(name: str, ops: list[Op]) -> str: - op_type = _op_cpp_type(name) - methods = "\n\n".join(_generate_torch_method_source(name, op) for op in ops) - return _TORCH_SOURCE_TEMPLATE.format( - name=name, - methods=methods, - op_type=op_type, - slot=_PYTORCH_SLOT, - ) - - -_BASE_TEMPLATE = """\ -#ifndef INFINI_OPS_BASE_{name_uc}_H_ -#define INFINI_OPS_BASE_{name_uc}_H_ - -{includes} - -{namespace_open} - -class {pascal} : public Operator<{pascal}> {{ - public: -{constructors} - -{op_calls} - - protected: -{member_decls} -}}; - -{namespace_close} - -#endif -""" - - -_TORCH_HEADER_TEMPLATE = """\ -#ifndef INFINI_OPS_TORCH_{name_uc}_H_ -#define INFINI_OPS_TORCH_{name_uc}_H_ - -#include "base/{name}.h" - -namespace infini::ops {{ - -template -class Operator<{op_type}, kDev, {slot}> : public {op_type} {{ - public: - using {op_type}::{pascal}; - -{op_calls} -}}; - -}} // namespace infini::ops - -#endif -""" - - -_TORCH_METHOD_TEMPLATE = """\ -template -void Operator<{op_type}, kDev, {slot}>::operator()({op_call_signature}) const {{ - const auto device_index = {out_device_index}; - const TorchStreamGuard stream_guard{{stream_, device_index}}; -{tensor_conversions} - - {aten_call}; -}} -""" - - -_TORCH_SOURCE_TEMPLATE = """\ -#include "torch/{name}/{name}.h" - -#include "torch/instantiate.h" -#include "torch/stream_guard.h" -#include "torch/tensor_.h" - -namespace infini::ops {{ - -{methods} - -INFINI_OPS_INSTANTIATE_TORCH_OPERATOR({op_type}, {slot}) - -}} // namespace infini::ops -""" - - -def _find_clang_format() -> str | None: - """Return the path to a `clang-format` binary, or `None` if unavailable. - - Generated files live under `generated/`, which is gitignored, so offline CI - containers should not block or try network access just to format them. - """ - - found = shutil.which("clang-format") - - if found: - return found - - print( - "`clang-format` not found on PATH; generated files will be emitted " - "without formatting.", - file=sys.stderr, - ) - - return None - - -def _clang_format(text: str, path: pathlib.Path) -> str: - """Pipe `text` through `clang-format` so generated headers / sources - satisfy the same style check (`clang-format` v21) that CI runs. - `path` informs include sorting (the file's own header should come - first in a `.cc`). If no `clang-format` binary is available, return - the input unchanged.""" - - if _CLANG_FORMAT is None: - return text - - return subprocess.run( - [_CLANG_FORMAT, f"--assume-filename={path}"], - input=text, - capture_output=True, - text=True, - check=True, - ).stdout - - -def _emit(name: str, ops: list[Op], *, emit_base: bool) -> set[pathlib.Path]: - base_path = _GENERATED_BASE_DIR / f"{name}.h" - torch_dir = _GENERATED_TORCH_DIR / name - torch_header_path = torch_dir / f"{name}.h" - torch_source_path = torch_dir / f"{name}.cc" - emitted_paths = set() - - if emit_base: - _write_text_if_changed( - base_path, _clang_format(_generate_base_header(name, ops), base_path) - ) - emitted_paths.add(base_path) - - _write_text_if_changed( - torch_header_path, - _clang_format(_generate_torch_header(name, ops), torch_header_path), - ) - _write_text_if_changed( - torch_source_path, - _clang_format(_generate_torch_source(name, ops), torch_source_path), - ) - emitted_paths.add(torch_header_path) - emitted_paths.add(torch_source_path) - - return emitted_paths - - -def _select_op_names(cli_ops, default_ops, config): - if config is None: - return cli_ops or default_ops - - aten_names_by_public_name = collections.defaultdict(list) - for op_name in default_ops: - aten_names_by_public_name[_public_op_name(op_name)].append(op_name) - - selected_public_names = ops_config.torch_op_names( - config, aten_names_by_public_name, _PYTORCH_SLOT - ) - selected = [ - aten_name - for public_name in selected_public_names - for aten_name in aten_names_by_public_name.get(public_name, (public_name,)) - ] - - if cli_ops: - allowed = set(cli_ops) - selected = [op_name for op_name in selected if op_name in allowed] - - return selected - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument( - "--ops", - nargs="*", - help="Override the op allowlist. If omitted, reads `scripts/torch_ops.yaml`.", - ) - parser.add_argument( - "--ops-config", - type=pathlib.Path, - help="Path to an `ops.json` operator and implementation selection.", - ) - parser.add_argument( - "--pytorch-version", - default=os.environ.get("INFINI_OPS_PYTORCH_VERSION", _DEFAULT_PYTORCH_VERSION), - help=( - "PyTorch version label used in diagnostics. Schemas are read from " - "the locally installed `torchgen` package. Default: `%(default)s`. " - "Can also be set via the `INFINI_OPS_PYTORCH_VERSION` environment " - "variable." - ), - ) - args = parser.parse_args() - - global _CLANG_FORMAT - _CLANG_FORMAT = _find_clang_format() - - default_ops = yaml.safe_load(_OPS_YAML_PATH.read_text()) - config = ops_config.load_ops_config(args.ops_config) if args.ops_config else None - op_names = _select_op_names(args.ops, default_ops, config) - aten_entries = _load_aten_entries(args.pytorch_version) - - skipped: list[tuple[str, str]] = [] - metadata: list[dict] = [] - expected_base_files: set[pathlib.Path] = set() - expected_torch_files: set[pathlib.Path] = set() - ops_by_public_name: dict[str, list[Op]] = collections.defaultdict(list) - - for name in op_names: - candidates = _find_out_entries(aten_entries, name) - - if not candidates: - skipped.append((name, f"no `.out` variant for `{name}` in YAML")) - continue - - usable: list[Op] = [] - last_reason = "" - - for entry in candidates: - try: - op = _parse_func(entry["func"]) - - for param in op.params: - _ = param.cpp_type # eagerly raise on unsupported types - except (NotImplementedError, ValueError) as exc: - last_reason = str(exc) - continue - - if not op.is_testable: - last_reason = "no testable tensor input/output pair" - continue - - usable.append(op) - - if not usable: - skipped.append((name, last_reason or "no usable overload")) - continue - - usable, duplicate_overloads = _dedupe_visible_overloads(usable) - - for skipped_op, kept_op in duplicate_overloads: - skipped.append( - ( - skipped_op.infini_name, - "duplicate visible C++ signature for " - f"`{name}`; using `{kept_op.infini_name}`", - ) - ) - - ops_by_public_name[usable[0].infini_name].extend(usable) - - for public_name, usable in ops_by_public_name.items(): - usable, duplicate_overloads = _dedupe_visible_overloads(usable) - - for skipped_op, kept_op in duplicate_overloads: - skipped.append( - ( - skipped_op.infini_name, - "duplicate visible C++ signature for " - f"`{public_name}`; using `{kept_op.infini_name}`", - ) - ) - - base_exists = _base_path(public_name).exists() - - if base_exists: - usable, base_warnings = _bind_existing_base_overloads(public_name, usable) - - for warning in base_warnings: - skipped.append((public_name, warning)) - - if not usable: - skipped.append( - ( - public_name, - "existing base has no overload compatible with ATen schema", - ) - ) - continue - - # Emit one InfiniOps wrapper per ATen op. Distinct visible overloads - # become overloaded constructors / `operator()` methods on the same - # class (`Pow` exposes both tensor and scalar exponents). Overloads - # that collapse to the same C++ signature after hidden defaults are - # skipped above. When a hand-written `src/base/.h` exists, - # skip emitting `generated/base/.h` so the hand-written one - # wins (the generated torch source's `#include "base/.h"` - # resolves through `src/` first). Signature mismatches surface as - # compile errors with a clear message — drop the op from the YAML - # to suppress. - emitted_paths = _emit(public_name, usable, emit_base=not base_exists) - expected_base_files.update( - path for path in emitted_paths if path.is_relative_to(_GENERATED_BASE_DIR) - ) - expected_torch_files.update( - path for path in emitted_paths if path.is_relative_to(_GENERATED_TORCH_DIR) - ) - - for op in usable: - metadata.append( - { - "name": public_name, - "aten_name": op.aten_name, - "overload_name": op.infini_name, - "params": [ - { - "name": p.api_name, - "type": p.aten_type, - "is_tensor": p.is_tensor, - "is_out": p.is_out, - "schema_index": next( - ( - schema_index - for schema_index in range(len(op.params)) - if op.api_param_for(schema_index) is p - ), - None, - ), - } - for p in op.visible_params - ], - } - ) - - _GENERATED_DIR.mkdir(parents=True, exist_ok=True) - _write_text_if_changed( - _METADATA_PATH, json.dumps({"ops": metadata}, indent=2) + "\n" - ) - _remove_stale_files(_GENERATED_BASE_DIR, expected_base_files) - _remove_stale_files(_GENERATED_TORCH_DIR, expected_torch_files) - - generated_names = sorted({m["name"] for m in metadata}) - print( - f"generated {len(metadata)} overloads across {len(generated_names)} ops: " - f"{generated_names}" - ) - - for name, reason in skipped: - print(f" skipped {name!r}: {reason}", file=sys.stderr) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) +"""Generate InfiniOps PyTorch wrappers from ATen `native_functions.yaml`. + +For each op listed in `scripts/torch_ops.yaml`, this script finds the `.out` +variant in PyTorch's locally installed ATen schemas, including variants declared +through `autogen`, and emits: + - `generated/base/.h` — the InfiniOps base class + `class : public Operator<>`, with constructors and pure-virtual + `operator()` overloads mirroring the selected ATen schemas. + - `generated/torch//.h` and `.cc` — the PyTorch backend + `Operator<, kDev, 8>` that calls `at::_out(out, ...)`. + - `generated/torch_ops_metadata.json` — the kind (`unary` / `binary` / + `binary_alpha`) of every successfully-generated op, consumed by the + parametrized test suite. + +Slot 8 is the reserved convention for PyTorch backends; slots 0-7 are +left for native or vendor implementations. (The slot must also be > 0 +to side-step a partial-specialization-after-instantiation conflict with +the primary template `Operator<>` instantiated at index 0.) + +The generated files are not committed; CMake regenerates them at configure +time when `WITH_TORCH=ON`. +""" + +import argparse +import collections +import dataclasses +import importlib.util +import json +import os +import pathlib +import re +import shutil +import subprocess +import sys + +import yaml + +_SCRIPTS_DIR = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_SCRIPTS_DIR)) +import ops_config # noqa: E402 + +_REPO_ROOT = _SCRIPTS_DIR.parent +_OPS_YAML_PATH = _SCRIPTS_DIR / "torch_ops.yaml" +_BASE_DIR = _REPO_ROOT / "src" / "base" +_GENERATED_DIR = _REPO_ROOT / "generated" +_GENERATED_BASE_DIR = _GENERATED_DIR / "base" +_GENERATED_TORCH_DIR = _GENERATED_DIR / "torch" +_METADATA_PATH = _GENERATED_DIR / "torch_ops_metadata.json" + +# Reserved slot for PyTorch backends. Native and vendor implementations +# claim slots 0-7; PyTorch wrappers always live at 8. +_PYTORCH_SLOT = 8 + +# ATen uses symbolic names for some `int`/`float` defaults (e.g. +# `reduction=Mean`). Map them to C++ identifiers usable in a call. +_ENUM_DEFAULTS = { + "Mean": "at::Reduction::Mean", + "Sum": "at::Reduction::Sum", + "Contiguous": "at::MemoryFormat::Contiguous", +} + +# Default PyTorch schema label used only in diagnostics when CMake does not +# provide the locally installed torch version. Codegen reads the actual schema +# from installed `torchgen/packaged/ATen/native/native_functions.yaml`. +_DEFAULT_PYTORCH_VERSION = "v2.4.0" + +# YAML scalar-type tokens → C++ types. Reference types (e.g. `const Scalar&`) +# are not used so the generated signatures match the existing hand-written +# ones, which pass by value to keep pybind11 binding generation simple. +_SCALAR_TYPE_MAP = { + # `at::Scalar` is implicitly constructible from `double`, so we expose + # scalars as `double` in the base class to keep it torch-independent. + "Scalar": "double", + "int": "int64_t", + "bool": "bool", + "float": "double", + # `SymInt` / `SymInt[]` exist for `torch.compile` internals; at runtime + # they're just `int64`/IntArrayRef. + "SymInt": "int64_t", + # `str` for required string params (e.g. `index_reduce.reduce`). + # `std::string` marshals through pybind11 cleanly and converts + # implicitly to ATen's `c10::string_view`. + "str": "std::string", + "ScalarType": "DataType", +} + +# `Dimname` overloads (named-tensor dim) are skipped — passing them +# from Python to ATen requires a wrapper conversion through +# `at::Dimname::fromSymbol(...)` that doesn't fit the cleanly-rendered +# 1:1 arg model, and named tensors remain experimental in PyTorch. +# The int-dim overload is always emitted alongside, so we lose nothing +# user-visible. + +# Optional ATen types and their typed empty optional values at the call site. +# We use typed `c10::optional{}` values rather than bare `at::nullopt` so +# the compiler can disambiguate ops with multiple `_out` overloads (e.g. +# `clamp_out` accepts both `optional` and `optional` for +# `min`/`max`). +_NULLOPT_BY_TYPE = { + "Scalar?": "c10::optional{}", + "int?": "c10::optional{}", + "bool?": "c10::optional{}", + "float?": "c10::optional{}", + "str?": "c10::optional{}", + "ScalarType?": "c10::optional{}", + "MemoryFormat?": "c10::optional{}", + "Layout?": "c10::optional{}", + "Device?": "c10::optional{}", + "Generator?": "c10::optional{}", + "Tensor?": "c10::optional{}", + "Tensor?[]": "c10::List>{}", + "int[]?": "c10::optional{}", + "int[1]?": "c10::optional{}", + "int[2]?": "c10::optional{}", + "int[3]?": "c10::optional{}", + "SymInt?": "c10::optional{}", + "SymInt[]?": "c10::optional{}", + "SymInt[1]?": "c10::optional{}", + "SymInt[2]?": "c10::optional{}", + "SymInt[3]?": "c10::optional{}", + "float[]?": "c10::optional>{}", +} + +# Optional ATen types that have a stable InfiniOps representation. These are +# exposed in generated base signatures and converted back to ATen optionals in +# the PyTorch backend. PyTorch-internal concepts without a public InfiniOps +# analogue stay hidden and are still passed as typed empty optionals. +_EXPOSED_OPTIONAL_CPP_TYPES = { + "Scalar?": "std::optional", + "int?": "std::optional", + "bool?": "std::optional", + "float?": "std::optional", + "str?": "std::optional", + "ScalarType?": "std::optional", + "Tensor?": "std::optional", + "Tensor?[]": "std::vector>", + "int[]?": "std::optional>", + "int[1]?": "std::optional>", + "int[2]?": "std::optional>", + "int[3]?": "std::optional>", + "SymInt?": "std::optional", + "SymInt[]?": "std::optional>", + "SymInt[1]?": "std::optional>", + "SymInt[2]?": "std::optional>", + "SymInt[3]?": "std::optional>", + "float[]?": "std::optional>", +} +_HARDCODE_NULLOPT_TYPES = frozenset( + set(_NULLOPT_BY_TYPE) - set(_EXPOSED_OPTIONAL_CPP_TYPES) +) + + +def _normalize_cpp_type(cpp_type: str) -> str: + text = cpp_type.strip() + + if text.startswith("const "): + text = text[len("const ") :] + + return text.rstrip("&").strip() + + +def _write_text_if_changed(path: pathlib.Path, content: str) -> bool: + """Write `content` only when the file's bytes would change.""" + path.parent.mkdir(parents=True, exist_ok=True) + + if path.exists() and path.read_text() == content: + return False + + path.write_text(content) + return True + + +def _prune_empty_dirs(root: pathlib.Path) -> None: + if not root.exists(): + return + + for child in sorted(root.iterdir(), reverse=True): + if child.is_dir(): + _prune_empty_dirs(child) + + if root.is_dir() and not any(root.iterdir()): + root.rmdir() + + +def _remove_stale_files(root: pathlib.Path, expected_files: set[pathlib.Path]) -> None: + if not root.exists(): + return + + for path in sorted(root.rglob("*"), reverse=True): + if path.is_file() and path not in expected_files: + path.unlink() + + _prune_empty_dirs(root) + + +@dataclasses.dataclass +class Param: + name: str + aten_type: str + default: str | None + keyword_only: bool + cpp_type_override: str | None = None + + @property + def api_name(self) -> str: + if self.name == "self": + return "input" + + return self.name + + @property + def is_tensor(self) -> bool: + if self.cpp_type_override is not None: + return _normalize_cpp_type(self.cpp_type_override) == "Tensor" + + # Real tensors only. `Tensor?` and `Tensor[]` have separate handling. + if self.aten_type.endswith("[]"): + return False + + return self.aten_type == "Tensor" or self.aten_type.startswith("Tensor(") + + @property + def is_optional_tensor(self) -> bool: + if self.cpp_type_override is not None: + return ( + _normalize_cpp_type(self.cpp_type_override) == "std::optional" + ) + + return self.aten_type == "Tensor?" + + @property + def is_tensor_list(self) -> bool: + if self.cpp_type_override is not None: + return _normalize_cpp_type(self.cpp_type_override) == "std::vector" + + return self.aten_type == "Tensor[]" + + @property + def is_optional_tensor_list(self) -> bool: + if self.cpp_type_override is not None: + return ( + _normalize_cpp_type(self.cpp_type_override) + == "std::vector>" + ) + + return self.aten_type == "Tensor?[]" + + @property + def is_mutable_tensor(self) -> bool: + # Mutable tensors carry `!` in their alias annotation, e.g. + # `Tensor(a!)`. + + return self.is_tensor and "!" in self.aten_type + + @property + def is_out(self) -> bool: + # In ATen `_out` schemas, output tensors are keyword-only mutable tensor + # params. Some mutable tensors are real inputs (`running_mean` / + # `running_var` in `_batch_norm_with_update`) and must stay in schema + # order, so mutability alone is not enough. + + return self.is_mutable_tensor and self.keyword_only + + @property + def is_hardcoded_nullopt(self) -> bool: + """If `True`, the param is omitted from the user-facing API and + passed as `at::nullopt` to ATen.""" + + return self.aten_type in _HARDCODE_NULLOPT_TYPES + + @property + def is_hidden(self) -> bool: + """True if the param is omitted from the user-facing API. + + Default-valued non-optional params (\\`bool\\`, \\`int\\`, \\`float\\`, + \\`str\\`, \\`int[N]\\`, …) used to be hidden as a convenience, but + reviewers consistently flagged the resulting omissions — + \\`bool upper/transpose/unitriangular\\` on \\`triangular_solve\\`, + \\`int diagonal\\` on \\`triu\\`, \\`str ord\\` on \\`linalg_matrix_norm\\`, + \\`int n\\` on the special chebyshev family, etc. — as missing + semantic controls. They are now exposed and forwarded to ATen. + + Optional ATen types with a stable InfiniOps representation + (\\`Tensor?\\`, \\`Scalar?\\`, \\`int?\\`, …) are exposed as + \\`std::optional\\`. ATen-internal types like + \\`Generator?\\`/\\`Layout?\\` still have no InfiniOps analogue, so + they remain hidden. + """ + + return self.is_hardcoded_nullopt + + def hidden_value(self) -> str: + """C++ literal substituted for a hidden param in the ATen call.""" + + if self.is_hardcoded_nullopt: + return _NULLOPT_BY_TYPE[self.aten_type] + + if self.default == "True": + return "true" + + if self.default == "False": + return "false" + + if self.aten_type.startswith(("int[", "SymInt[")) and self.default is not None: + # `int[N]=[a, b, c]` → `{a, b, c}`; `int[N]=0` (scalar default + # for list type) → `{0, 0, ...}` replicated to size N. + if self.default.startswith("["): + return "{" + self.default[1:-1] + "}" + + size_match = re.search(r"\[(\d+)\]", self.aten_type) + n = int(size_match.group(1)) if size_match else 1 + + return "{" + ", ".join([self.default] * n) + "}" + + if self.aten_type == "str" and self.default is not None: + # YAML uses single-quoted strings (e.g. `'none'`); C++ char + # literals also use single quotes, so swap to doubles. + + return '"' + self.default.strip("'\"") + '"' + + if self.aten_type in {"int", "float", "SymInt"} and self.default is not None: + # Translate known ATen enum defaults to their C++ identifiers. + + return _ENUM_DEFAULTS.get(self.default, self.default) + + raise AssertionError( + f"param {self.name!r} of type {self.aten_type!r} with default " + f"{self.default!r} is not hidden" + ) + + @property + def cpp_type(self) -> str: + if self.cpp_type_override is not None: + return self.cpp_type_override + + if self.is_tensor: + return "Tensor" + + if self.is_tensor_list: + return "std::vector" + + if self.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: + return _EXPOSED_OPTIONAL_CPP_TYPES[self.aten_type] + + if self.is_hidden: + # Not exposed — the ATen call substitutes a hardcoded value + # so the `cpp_type` is irrelevant. + + return "void" + + bare = self.aten_type.rstrip("?") + # Required `int[N]` / `SymInt[N]` (no default) — pybind11 accepts + # a Python list of ints into `std::vector`, which ATen + # promotes to `IntArrayRef` implicitly. + if bare.startswith(("int[", "SymInt[")) or bare in {"int[]", "SymInt[]"}: + return "std::vector" + + try: + return _SCALAR_TYPE_MAP[bare] + except KeyError as exc: + raise NotImplementedError( + f"unsupported ATen type {self.aten_type!r} for param {self.name!r}" + ) from exc + + +@dataclasses.dataclass +class Op: + aten_name: str + overload: str + params: list[Param] + signature_params: list[Param] | None = None + param_bindings: list[Param | None] | None = None + explicit_inplace_out: Param | None = None + + @property + def pascal_name(self) -> str: + return _snake_to_pascal(self.infini_name) + + @property + def infini_name(self) -> str: + """InfiniOps public op name. + + ATen disambiguates `_out` overloads with suffixes like `Tensor_Tensor_out`, + `out_x`, `forward_output`, `grad_input`, but reviewers consistently + flag those suffixes as bad public-API naming when they leak into + InfiniOps class names. Different ATen overloads of the same base op + become overloaded `operator()` methods on a single class instead. When + two overloads collapse to the same visible C++ signature after hidden + defaults, `_dedupe_visible_overloads` keeps only one. + + ATen-internal leading underscores and in-place trailing underscores are + also normalized so generated class names do not collide with existing + public ops (`_softmax` → `AtenSoftmax`, `add_` → `AddInplace`). + """ + + return _public_op_name(self.aten_name) + + @property + def is_inplace(self) -> bool: + """True for ATen's single-underscore in-place variants.""" + + return _is_inplace_aten_name(self.aten_name) + + @property + def tensor_params(self) -> list[Param]: + return [p for p in self.params if p.is_tensor] + + @property + def out_params(self) -> list[Param]: + """Mutable tensor outputs. Most ops have one (`Tensor(a!) out`); + multi-output ops like `frexp` or `sort` have several + (`Tensor(a!) values`, `Tensor(b!) indices`).""" + + if self.explicit_inplace_out is not None: + return [self.explicit_inplace_out] + + if self.is_inplace: + return [self.params[0]] + + return [p for p in self.params if p.is_out] + + @property + def out_param(self) -> Param: + """Single-output convenience. Asserts there's exactly one.""" + outs = self.out_params + assert len(outs) == 1, f"op {self.aten_name!r} has {len(outs)} out tensors" + + return outs[0] + + @property + def visible_params(self) -> list[Param]: + """Params the wrapper exposes to the user; hidden ones (hardcoded + optional nullopt, default-`False`/`True` bools) are filtered.""" + + if self.signature_params is not None: + return self.signature_params + + return [p for p in self.params if not p.is_hidden] + + def api_param_for(self, schema_index: int) -> Param | None: + """Return the public signature param bound to a schema param.""" + + if self.param_bindings is not None: + return self.param_bindings[schema_index] + + param = self.params[schema_index] + + if param.is_hidden: + return None + + return param + + @property + def is_testable(self) -> bool: + """Cheap structural check: at least one out tensor, and the first + constructor parameter is a tensor. The latter is needed because + `Operator::Make(Tensor tensor, Args... args)` dispatches on + `tensor.device()`, so an op like `pow.Scalar_out(Scalar self, + Tensor exponent, *, Tensor(a!) out)` cannot be wired up without + a separate dispatch path. Generators like `arange` / `linspace` + also fall under this rule (no input tensors at all).""" + + if not self.out_params: + return False + + if self.is_inplace: + return self.params[0].is_mutable_tensor + + # `params` includes out tensors at the end; check the first + # non-out param. If there are no non-out params (`empty.out`, + # `arange.out`), this op also fails the dispatch precondition. + non_out = [p for p in self.params if not p.is_out] + + if not non_out: + return False + + return non_out[0].is_tensor or non_out[0].is_tensor_list + + +_FUNC_RE = re.compile( + r"^(?P[a-zA-Z_][a-zA-Z0-9_]*)" + r"(?:\.(?P\w+))?" + r"\((?P.*)\)\s*->\s*.+$" +) + +_ARG_RE = re.compile( + r"^(?P\S+(?:\([^)]*\))?\??)" # type with optional alias and `?` + r"\s+(?P\w+)" + r"(?:\s*=\s*(?P.+))?$" +) + + +def _parse_func(func_str: str) -> Op: + m = _FUNC_RE.match(func_str) + + if not m: + raise ValueError(f"could not parse func: {func_str!r}") + + return Op( + aten_name=m.group("name"), + overload=m.group("overload") or "", + params=_parse_args(m.group("args")), + ) + + +def _parse_args(args_str: str) -> list[Param]: + params: list[Param] = [] + keyword_only = False + + for token in _split_args(args_str): + if token == "*": + keyword_only = True + continue + + params.append(_parse_one_arg(token, keyword_only)) + + return params + + +def _split_args(args_str: str) -> list[str]: + """Split on top-level commas, respecting `(...)` and `[...]`.""" + parts: list[str] = [] + depth = 0 + current: list[str] = [] + + for ch in args_str: + if ch in "([": + depth += 1 + current.append(ch) + elif ch in ")]": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + piece = "".join(current).strip() + + if piece: + parts.append(piece) + + current = [] + else: + current.append(ch) + + tail = "".join(current).strip() + + if tail: + parts.append(tail) + + return parts + + +def _parse_one_arg(token: str, keyword_only: bool) -> Param: + m = _ARG_RE.match(token) + + if not m: + raise ValueError(f"could not parse arg: {token!r}") + + name = m.group("name") + return Param( + name=name, + aten_type=m.group("type"), + default=m.group("default"), + keyword_only=keyword_only, + ) + + +_BASE_OPERATOR_RE = re.compile( + r"\bvirtual\s+void\s+operator\(\)\s*\((?P.*?)\)\s*const" + r"(?:\s*=\s*0|\s*\{)", + re.S, +) + + +def _split_cpp_params(args_str: str) -> list[str]: + parts: list[str] = [] + depth = 0 + current: list[str] = [] + + for ch in args_str: + if ch in "([<": + depth += 1 + current.append(ch) + elif ch in ")]>": + depth -= 1 + current.append(ch) + elif ch == "," and depth == 0: + piece = "".join(current).strip() + + if piece: + parts.append(piece) + + current = [] + else: + current.append(ch) + + tail = "".join(current).strip() + + if tail: + parts.append(tail) + + return parts + + +def _parse_base_signature_param(token: str) -> tuple[str, str]: + token = token.split("=", 1)[0].strip() + token = re.sub(r"\s+", " ", token) + match = re.match(r"(?P.+?)\s+(?P\w+)$", token) + + if not match: + raise ValueError(f"could not parse base operator param: {token!r}") + + return _normalize_cpp_type(match.group("type")), match.group("name") + + +def _parse_base_operator_signatures( + base_path: pathlib.Path, +) -> list[list[tuple[str, str]]]: + source = base_path.read_text() + signatures = [] + + for match in _BASE_OPERATOR_RE.finditer(source): + args = match.group("args").strip() + + if not args: + signatures.append([]) + continue + + signatures.append( + [_parse_base_signature_param(token) for token in _split_cpp_params(args)] + ) + + return signatures + + +def _cpp_types_compatible(schema_cpp_type: str, base_cpp_type: str) -> bool: + schema = _normalize_cpp_type(schema_cpp_type) + base = _normalize_cpp_type(base_cpp_type) + + if schema == base: + return True + + compatible_pairs = { + ("double", "float"), + ("int64_t", "int"), + ("std::optional", "std::optional"), + ("std::optional", "std::optional"), + } + + return (schema, base) in compatible_pairs + + +def _cpp_param_compatible( + schema_param: Param, base_cpp_type: str, base_name: str +) -> bool: + if schema_param.api_name != base_name: + return False + + return _cpp_types_compatible(schema_param.cpp_type, base_cpp_type) + + +def _bind_base_signature(op: Op, signature: list[tuple[str, str]]) -> Op | None: + bindings: list[Param | None] = [None] * len(op.params) + signature_params = [] + explicit_inplace_out = None + + if ( + op.is_inplace + and signature + and signature[-1][1] == "out" + and _cpp_types_compatible("Tensor", signature[-1][0]) + ): + out_cpp_type, out_name = signature[-1] + explicit_inplace_out = Param( + name=out_name, + aten_type="Tensor(a!)", + default=None, + keyword_only=True, + cpp_type_override=out_cpp_type, + ) + signature = signature[:-1] + + for base_cpp_type, base_name in signature: + matched_index = None + + for index, schema_param in enumerate(op.params): + if bindings[index] is not None: + continue + + if _cpp_param_compatible(schema_param, base_cpp_type, base_name): + matched_index = index + break + + if matched_index is None: + return None + + schema_param = op.params[matched_index] + api_param = dataclasses.replace( + schema_param, name=base_name, cpp_type_override=base_cpp_type + ) + bindings[matched_index] = api_param + signature_params.append(api_param) + + if any( + binding is None and not _is_omittable_param(param) + for param, binding in zip(op.params, bindings) + ): + return None + + if explicit_inplace_out is not None: + signature_params.append(explicit_inplace_out) + + return dataclasses.replace( + op, + signature_params=signature_params, + param_bindings=bindings, + explicit_inplace_out=explicit_inplace_out, + ) + + +def _bind_existing_base_overloads( + name: str, ops: list[Op] +) -> tuple[list[Op], list[str]]: + base_path = _base_path(name) + signatures = _parse_base_operator_signatures(base_path) + bound_ops = [] + warnings = [] + + for signature in signatures: + matches = [ + bound + for op in ops + if (bound := _bind_base_signature(op, signature)) is not None + ] + + if not matches: + rendered = ", ".join(f"{cpp_type} {param}" for cpp_type, param in signature) + warnings.append( + f"base overload `operator()({rendered})` does not match any " + "usable ATen schema" + ) + continue + + matches.sort( + key=lambda match: sum( + binding is None for binding in match.param_bindings or () + ) + ) + bound_ops.append(matches[0]) + + return bound_ops, warnings + + +def _snake_to_pascal(s: str) -> str: + return "".join(p.capitalize() for p in s.split("_")) + + +_OP_NAMESPACE_PREFIXES = ("special", "linalg", "fft") + + +def _is_inplace_aten_name(name: str) -> bool: + """Return whether `name` is an ATen in-place operator name.""" + + return name.endswith("_") and not name.endswith("__") + + +def _public_op_name(aten_name: str) -> str: + """Map ATen-only spelling to stable InfiniOps public names.""" + + public_name = aten_name + file_prefixes = [] + + if public_name.startswith("_"): + file_prefixes.append("internal") + public_name = public_name.lstrip("_") + + if _is_inplace_aten_name(public_name): + public_name = public_name[:-1] + + return "_".join(file_prefixes + [public_name]) + + +def _op_namespace_parts(name: str) -> tuple[str, ...]: + parts = name.split("_") + namespaces = [] + + if parts and parts[0] == "internal": + namespaces.append(parts.pop(0)) + + if parts and parts[0] in _OP_NAMESPACE_PREFIXES: + namespaces.append(parts.pop(0)) + + return tuple(namespaces) + + +def _op_class_stem(name: str) -> str: + parts = name.split("_") + + if parts and parts[0] == "internal": + parts.pop(0) + + if parts and parts[0] in _OP_NAMESPACE_PREFIXES: + parts.pop(0) + + return "_".join(parts) + + +def _op_class_name(name: str) -> str: + return _snake_to_pascal(_op_class_stem(name)) + + +def _op_cpp_type(name: str) -> str: + parts = (*_op_namespace_parts(name), _op_class_name(name)) + + return "::".join(parts) + + +def _op_namespace_open(name: str) -> str: + parts = _op_namespace_parts(name) + + if not parts: + return "namespace infini::ops {" + + return f"namespace infini::ops::{'::'.join(parts)} {{" + + +def _op_namespace_close(name: str) -> str: + parts = _op_namespace_parts(name) + + if not parts: + return "} // namespace infini::ops" + + return f"}} // namespace infini::ops::{'::'.join(parts)}" + + +def _base_path(op_name: str) -> pathlib.Path: + return _BASE_DIR / f"{op_name}.h" + + +def _is_omittable_param(param: Param) -> bool: + return param.aten_type in _NULLOPT_BY_TYPE or param.default is not None + + +def _default_call_value(param: Param) -> str: + if param.aten_type in _NULLOPT_BY_TYPE: + return _NULLOPT_BY_TYPE[param.aten_type] + + if param.default is not None: + return _translate_default(param) + + return param.hidden_value() + + +def _load_aten_entries(version: str) -> list[dict]: + """Return parsed ATen schemas bundled with installed `torchgen`. + + `WITH_TORCH=ON` already requires a local PyTorch installation. PyTorch + wheels ship the matching ATen schema under `torchgen/packaged`, including + vendor forks, so codegen should not depend on fetching PyTorch sources from + GitHub during CI builds. The official parser expands schemas declared via + `autogen`, such as `relu.out`, before InfiniOps filters for out variants. + """ + + try: + packaged, expanded_autogen = _load_packaged_aten_entries() + except RuntimeError as error: + raise RuntimeError(f"{error} for PyTorch schema {version!r}") from error + + print( + "using installed `torchgen` ATen schemas " + f"for PyTorch schema {version} " + f"({'with' if expanded_autogen else 'without'} `autogen` expansion).", + file=sys.stderr, + ) + + return packaged + + +def _load_packaged_aten_entries() -> tuple[list[dict], bool]: + """Parse the ATen schemas bundled with installed `torchgen`. + + PyTorch wheels install `torchgen/packaged/ATen/native/native_functions.yaml`; + using it lets offline CI images generate wrappers against the exact schema + shipped with their installed torch fork. + """ + + spec = importlib.util.find_spec("torchgen") + + if spec is None: + raise RuntimeError("could not find the installed `torchgen` package") + + if spec.submodule_search_locations is None: + raise RuntimeError("installed `torchgen` is not a package") + + for root in spec.submodule_search_locations: + native_dir = pathlib.Path(root) / "packaged" / "ATen" / "native" + native_path = native_dir / "native_functions.yaml" + tags_path = native_dir / "tags.yaml" + + if not native_path.is_file(): + continue + + if not tags_path.is_file(): + print( + "warning: could not expand ATen `autogen` schemas because " + f"`{tags_path}` is missing; using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + try: + from torchgen.gen import parse_native_yaml + except (ImportError, AttributeError) as error: + print( + "warning: could not expand ATen `autogen` schemas because " + f"`torchgen.gen.parse_native_yaml` is unavailable ({error}); " + "using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + try: + parsed = parse_native_yaml(str(native_path), str(tags_path)) + except Exception as error: + print( + "warning: could not expand ATen `autogen` schemas from " + f"`{native_path}` ({type(error).__name__}: {error}); " + "using explicit schemas only.", + file=sys.stderr, + ) + + return _load_explicit_aten_entries(native_path), False + + return ( + [{"func": str(function.func)} for function in parsed.native_functions], + True, + ) + + raise RuntimeError( + "could not find installed `torchgen` packaged `native_functions.yaml`" + ) + + +def _load_explicit_aten_entries(native_path: pathlib.Path) -> list[dict]: + entries = yaml.safe_load(native_path.read_text()) + + if not isinstance(entries, list): + raise RuntimeError( + f"expected a list of ATen schemas in `{native_path}`, " + f"got {type(entries).__name__}" + ) + + return entries + + +def _find_out_entries(entries: list[dict], op_name: str) -> list[dict]: + """Return all out-variant entries for `op_name`, with the bare + `.out(` form first and overload-suffixed variants + (e.g. `pow.Tensor_Tensor_out(`, `kthvalue.values(`) after. An + entry counts as an out-variant when it (a) is named + `.out`, (b) ends in `_out`, or (c) carries a + `Tensor(!)` mutability annotation — that last case covers + multi-output ops named after their output tensors + (`kthvalue.values`, `mode.values`, …).""" + bare_prefix = f"{op_name}.out(" + op_overload = re.compile(rf"^{re.escape(op_name)}\.\w+\(") + mut_tensor = re.compile(r"Tensor\([a-z]!\)") + bare: list[dict] = [] + others: list[dict] = [] + + for entry in entries: + func = entry.get("func", "") + + if func.startswith(bare_prefix): + bare.append(entry) + elif op_overload.match(func) and ( + func.split("(", 1)[0].endswith("_out") or mut_tensor.search(func) + ): + others.append(entry) + + return bare + others + + +def _format_signature(op: Op, *, include_defaults: bool = False) -> str: + parts = [] + + for param in op.visible_params: + prefix = "" if param.is_mutable_tensor else "const " + text = f"{prefix}{param.cpp_type} {param.api_name}" + + if include_defaults and param.default is not None: + text += f" = {_translate_default(param)}" + + parts.append(text) + + return ", ".join(parts) + + +def _visible_signature_key(op: Op) -> tuple[str, ...]: + """C++ overload identity for the user-facing API. + + Parameter names and top-level `const` do not distinguish C++ overloads, so + only the exposed C++ type sequence participates in duplicate detection. + """ + + return tuple(param.cpp_type for param in op.visible_params) + + +def _canonical_overload_score(index: int, op: Op) -> tuple[bool, int, int, str, int]: + """Sort key for duplicate visible signatures. + + Prefer the canonical unsuffixed InfiniOps name, then the schema that hides + fewer ATen-only defaults, then the shorter deterministic name. + """ + + return ( + op.infini_name != op.aten_name, + sum(param.is_hidden for param in op.params), + len(op.infini_name), + op.infini_name, + index, + ) + + +def _dedupe_visible_overloads(ops: list[Op]) -> tuple[list[Op], list[tuple[Op, Op]]]: + """Drop overloads that collapse to the same visible C++ signature. + + Returns the selected overloads in the original schema order plus a list of + `(skipped, kept)` duplicate pairs for diagnostics. + """ + winners: dict[tuple[str, ...], tuple[int, Op]] = {} + duplicates: list[tuple[Op, tuple[str, ...]]] = [] + + for index, op in enumerate(ops): + key = _visible_signature_key(op) + current = winners.get(key) + + if current is None: + winners[key] = (index, op) + continue + + current_index, current_op = current + + if _canonical_overload_score(index, op) < _canonical_overload_score( + current_index, current_op + ): + duplicates.append((current_op, key)) + winners[key] = (index, op) + else: + duplicates.append((op, key)) + + selected_indices = {index for index, _ in winners.values()} + selected = [op for index, op in enumerate(ops) if index in selected_indices] + duplicate_pairs = [ + (skipped, winners[key][1]) + for skipped, key in duplicates + if winners[key][1] is not skipped + ] + + return selected, duplicate_pairs + + +def _translate_default(param: Param) -> str: + """Translate a YAML default literal to a C++ literal.""" + raw = param.default + + if raw == "True": + return "true" + + if raw == "False": + return "false" + + if raw == "None": + return "{}" + + return raw # numeric literals (`0`, `1`, `1.0`) pass through + + +def _generate_base_includes(ops: list[Op]) -> str: + cpp_types = [param.cpp_type for op in ops for param in op.visible_params] + includes = [] + + if any("std::optional" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + if any("std::string" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + if any("std::vector" in cpp_type for cpp_type in cpp_types): + includes.append("#include ") + + includes.append('#include "operator.h"') + + return "\n".join(includes) + + +def _generate_base_header(name: str, ops: list[Op]) -> str: + pascal = _op_class_name(name) + + member_decls = [] + tensor_member_order = [] + seen_tensor_members = set() + optional_tensor_member_order = [] + seen_optional_tensor_members = set() + scalar_member_order = [] + scalar_member_types = {} + + for op in ops: + for param in op.tensor_params: + api_name = param.api_name + + if api_name in seen_tensor_members: + continue + + seen_tensor_members.add(api_name) + tensor_member_order.append(api_name) + member_decls.append(f" Tensor::Shape {api_name}_shape_;") + member_decls.append(f" Tensor::Strides {api_name}_strides_;") + member_decls.append(f" DataType {api_name}_type_;") + + for param in op.visible_params: + api_name = param.api_name + + if not param.is_optional_tensor or api_name in seen_optional_tensor_members: + continue + + seen_optional_tensor_members.add(api_name) + optional_tensor_member_order.append(api_name) + member_decls.append(f" bool has_{api_name}_{{false}};") + member_decls.append(f" Tensor::Shape {api_name}_shape_;") + member_decls.append(f" Tensor::Strides {api_name}_strides_;") + member_decls.append(f" DataType {api_name}_type_{{DataType::kFloat32}};") + + # Visible non-tensor params (scalars, strings, vectors) are also + # stored on the base so backends can dispatch on them later — not + # only at the moment `operator()` is invoked. Reviewers flagged + # this on multiple PRs (e.g. `n` on + # `special_chebyshev_polynomial_v_n_scalar`). Same-named params + # across overloads must share a type; if they conflict, the second + # overload's member is dropped (later constructors leave it + # default-initialised). + for param in op.visible_params: + api_name = param.api_name + + if ( + param.is_tensor + or param.is_optional_tensor + or api_name in scalar_member_types + ): + continue + + scalar_member_order.append(api_name) + scalar_member_types[api_name] = param.cpp_type + member_decls.append(f" {param.cpp_type} {api_name}_{{}};") + + member_decls.append(" int device_index_{0};") + + constructors = [] + calls = [] + + for op in ops: + init_pieces = [] + tensor_params = {param.api_name: param for param in op.tensor_params} + optional_tensor_params = { + param.api_name: param + for param in op.visible_params + if param.is_optional_tensor + } + scalar_params = { + param.api_name: param + for param in op.visible_params + if not param.is_tensor + and not param.is_optional_tensor + and scalar_member_types.get(param.api_name) == param.cpp_type + } + + for param_name in tensor_member_order: + param = tensor_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" {api_name}_shape_{{{api_name}.shape()}}") + init_pieces.append(f" {api_name}_strides_{{{api_name}.strides()}}") + init_pieces.append(f" {api_name}_type_{{{api_name}.dtype()}}") + + for param_name in optional_tensor_member_order: + param = optional_tensor_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" has_{api_name}_{{{api_name}.has_value()}}") + init_pieces.append( + f" {api_name}_shape_{{{api_name} ? " + f"Tensor::Shape{{{api_name}->shape()}} : Tensor::Shape{{}}}}" + ) + init_pieces.append( + f" {api_name}_strides_{{{api_name} ? " + f"Tensor::Strides{{{api_name}->strides()}} : Tensor::Strides{{}}}}" + ) + init_pieces.append( + f" {api_name}_type_{{{api_name} ? " + f"{api_name}->dtype() : DataType::kFloat32}}" + ) + + for param_name in scalar_member_order: + param = scalar_params.get(param_name) + + if param is None: + continue + + api_name = param.api_name + init_pieces.append(f" {api_name}_{{{api_name}}}") + + # All out tensors share a device; use the first one. Keep this last + # so initializer order follows the member declaration order. + init_pieces.append( + f" device_index_{{{op.out_params[0].api_name}.device().index()}}" + ) + + init_list = ",\n".join(init_pieces).lstrip() + constructors.append( + f" {pascal}({_format_signature(op)})\n : {init_list} {{}}" + ) + calls.append(f" virtual void operator()({_format_signature(op)}) const = 0;") + + return _BASE_TEMPLATE.format( + name_uc=name.upper(), + pascal=pascal, + namespace_open=_op_namespace_open(name), + namespace_close=_op_namespace_close(name), + includes=_generate_base_includes(ops), + constructors="\n\n".join(constructors), + op_calls="\n\n".join(calls), + member_decls="\n\n".join(member_decls), + ) + + +def _generate_torch_header(name: str, ops: list[Op]) -> str: + pascal = _op_class_name(name) + op_type = _op_cpp_type(name) + op_calls = "\n\n".join( + f" void operator()({_format_signature(op)}) const override;" for op in ops + ) + + return _TORCH_HEADER_TEMPLATE.format( + name_uc=name.upper(), + name=name, + pascal=pascal, + op_type=op_type, + op_calls=op_calls, + slot=_PYTORCH_SLOT, + ) + + +def _generate_torch_method_source(name: str, op: Op) -> str: + op_type = _op_cpp_type(name) + conversion_lines = [] + out_device_index = f"{op.out_params[0].api_name}.device().index()" + + def _optional_aten_type(param: Param) -> str: + return _NULLOPT_BY_TYPE[param.aten_type].removesuffix("{}") + + def _optional_aten_value(schema_param: Param, api_param: Param) -> str: + api_name = api_param.api_name + + if schema_param.aten_type == "Tensor?": + data_expr = f"const_cast({api_name}->data())" + + return ( + f"ToAtenTensor({data_expr}, {api_name}->shape(), " + f"{api_name}->strides(), {api_name}->dtype(), " + f"device_index)" + ) + + if schema_param.aten_type == "Scalar?": + return f"at::Scalar(*{api_name})" + + if schema_param.aten_type == "ScalarType?": + return f"ToAtenDataType(*{api_name})" + + if schema_param.aten_type == "str?": + return f"c10::string_view(*{api_name})" + + if schema_param.aten_type.startswith( + ("int[", "SymInt[") + ) or schema_param.aten_type in { + "int[]?", + "SymInt[]?", + }: + return f"at::IntArrayRef(*{api_name})" + + if schema_param.aten_type == "float[]?": + return f"at::ArrayRef(*{api_name})" + + return f"*{api_name}" + + def _append_optional_conversion(schema_param: Param, api_param: Param) -> None: + api_name = api_param.api_name + optional_type = _optional_aten_type(schema_param) + conversion_lines.append(f" {optional_type} at_{schema_param.name};") + conversion_lines.append(f" if ({api_name}.has_value()) {{") + conversion_lines.append( + f" at_{schema_param.name} = " + f"{optional_type}{{{_optional_aten_value(schema_param, api_param)}}};" + ) + conversion_lines.append(" }") + + for schema_index, param in enumerate(op.params): + if ( + not param.is_tensor + and not param.is_tensor_list + and not param.is_optional_tensor_list + ): + continue + + api_param = op.api_param_for(schema_index) + + if api_param is None: + continue + + api_name = api_param.api_name + + if param.is_optional_tensor_list: + conversion_lines.append( + f" c10::List> at_{param.name};" + ) + conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") + conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") + conversion_lines.append(" if (tensor.has_value()) {") + conversion_lines.append( + " at_" + f"{param.name}.push_back(ToAtenTensor(" + "const_cast(tensor->data()), tensor->shape(), " + "tensor->strides(), tensor->dtype(), device_index));" + ) + conversion_lines.append(" } else {") + conversion_lines.append(f" at_{param.name}.push_back(c10::nullopt);") + conversion_lines.append(" }") + conversion_lines.append(" }") + continue + + if param.is_tensor_list: + conversion_lines.append(f" std::vector at_{param.name};") + conversion_lines.append(f" at_{param.name}.reserve({api_name}.size());") + conversion_lines.append(f" for (const auto& tensor : {api_name}) {{") + conversion_lines.append( + " at_" + f"{param.name}.push_back(ToAtenTensor(" + "const_cast(tensor.data()), tensor.shape(), tensor.strides(), " + "tensor.dtype(), device_index));" + ) + conversion_lines.append(" }") + continue + + use_mutable_data = param.is_mutable_tensor and not ( + op.explicit_inplace_out is not None and schema_index == 0 + ) + data_expr = ( + f"{api_name}.data()" + if use_mutable_data + else f"const_cast({api_name}.data())" + ) + conversion_lines.append( + f" auto at_{param.name} = ToAtenTensor(\n" + f" {data_expr}, {api_name}_shape_, {api_name}_strides_,\n" + f" {api_name}_type_, device_index);" + ) + + if op.explicit_inplace_out is not None: + out_name = op.explicit_inplace_out.api_name + conversion_lines.append( + f" auto at_{out_name} = ToAtenTensor(\n" + f" {out_name}.data(), {out_name}_shape_, {out_name}_strides_,\n" + f" {out_name}_type_, device_index);" + ) + + for schema_index, param in enumerate(op.params): + api_param = op.api_param_for(schema_index) + + if ( + api_param is not None + and param.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES + and not param.is_optional_tensor_list + ): + _append_optional_conversion(param, api_param) + + def _render_arg(schema_index, p): + api_param = op.api_param_for(schema_index) + + if api_param is None: + return _default_call_value(p) + + if p.is_hidden: + return _default_call_value(p) + + if p.aten_type in _EXPOSED_OPTIONAL_CPP_TYPES: + return f"at_{p.name}" + + if p.is_tensor or p.is_tensor_list: + return f"at_{p.name}" + + if p.aten_type == "ScalarType": + return f"ToAtenDataType({api_param.api_name})" + + return api_param.api_name + + if op.is_inplace: + # In-place ATen calls keep the mutable input in positional order, + # unlike `_out` calls which place output tensors first. + input_param = op.params[0] + arg_order = op.params[1:] + rendered_args = ", ".join( + _render_arg(index + 1, p) for index, p in enumerate(arg_order) + ) + if op.explicit_inplace_out is not None: + out_name = op.explicit_inplace_out.api_name + aten_call = f"at_{out_name}.copy_(at_{input_param.name});\n" + aten_call += f" at_{out_name}.{op.aten_name}({rendered_args})" + else: + aten_call = f"at_{input_param.name}.{op.aten_name}({rendered_args})" + else: + # ATen `_out` form puts all out tensors first, then non-out params + # in YAML order. Hardcoded-nullopt params become `at::nullopt`. + arg_order = [(index, p) for index, p in enumerate(op.params) if p.is_out] + [ + (index, p) for index, p in enumerate(op.params) if not p.is_out + ] + rendered_args = ", ".join(_render_arg(index, p) for index, p in arg_order) + aten_call = f"at::{op.aten_name}_out({rendered_args})" + + if op.aten_name == "embedding_dense_backward": + conversion_lines.append( + " assert(at_grad_output.dim() >= 1);\n" + " const auto embedding_dim = at_grad_output.size(-1);\n" + " assert(at_grad_output.numel() == at_indices.numel() * embedding_dim);\n" + " assert(at_out.dim() == 2 && at_out.size(0) == num_weights &&\n" + " at_out.size(1) == embedding_dim &&\n" + " at_out.scalar_type() == at_grad_output.scalar_type());\n" + " // Avoid CUDA kernels with empty indices or zero feature width.\n" + " if (at_indices.numel() == 0 || embedding_dim == 0) {\n" + " at_out.zero_();\n" + " return;\n" + " }" + ) + + return _TORCH_METHOD_TEMPLATE.format( + op_type=op_type, + op_call_signature=_format_signature(op), + tensor_conversions="\n".join(conversion_lines), + out_device_index=out_device_index, + # The generated call expression resolves the right kernel via C++ + # overload resolution from the argument types we pass. + aten_call=aten_call, + slot=_PYTORCH_SLOT, + ) + + +def _generate_torch_source(name: str, ops: list[Op]) -> str: + op_type = _op_cpp_type(name) + methods = "\n\n".join(_generate_torch_method_source(name, op) for op in ops) + source = _TORCH_SOURCE_TEMPLATE.format( + name=name, + methods=methods, + op_type=op_type, + slot=_PYTORCH_SLOT, + ) + + if any(op.aten_name == "embedding_dense_backward" for op in ops): + source = source.replace( + '#include "torch/instantiate.h"', + '#include \n\n#include "torch/instantiate.h"', + ) + + return source + + +_BASE_TEMPLATE = """\ +#ifndef INFINI_OPS_BASE_{name_uc}_H_ +#define INFINI_OPS_BASE_{name_uc}_H_ + +{includes} + +{namespace_open} + +class {pascal} : public Operator<{pascal}> {{ + public: +{constructors} + +{op_calls} + + protected: +{member_decls} +}}; + +{namespace_close} + +#endif +""" + + +_TORCH_HEADER_TEMPLATE = """\ +#ifndef INFINI_OPS_TORCH_{name_uc}_H_ +#define INFINI_OPS_TORCH_{name_uc}_H_ + +#include "base/{name}.h" + +namespace infini::ops {{ + +template +class Operator<{op_type}, kDev, {slot}> : public {op_type} {{ + public: + using {op_type}::{pascal}; + +{op_calls} +}}; + +}} // namespace infini::ops + +#endif +""" + + +_TORCH_METHOD_TEMPLATE = """\ +template +void Operator<{op_type}, kDev, {slot}>::operator()({op_call_signature}) const {{ + const auto device_index = {out_device_index}; + const TorchStreamGuard stream_guard{{stream_, device_index}}; +{tensor_conversions} + + {aten_call}; +}} +""" + + +_TORCH_SOURCE_TEMPLATE = """\ +#include "torch/{name}/{name}.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops {{ + +{methods} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR({op_type}, {slot}) + +}} // namespace infini::ops +""" + + +def _find_clang_format() -> str | None: + """Return the path to a `clang-format` binary, or `None` if unavailable. + + Generated files live under `generated/`, which is gitignored, so offline CI + containers should not block or try network access just to format them. + """ + + found = shutil.which("clang-format") + + if found: + return found + + print( + "`clang-format` not found on PATH; generated files will be emitted " + "without formatting.", + file=sys.stderr, + ) + + return None + + +def _clang_format(text: str, path: pathlib.Path) -> str: + """Pipe `text` through `clang-format` so generated headers / sources + satisfy the same style check (`clang-format` v21) that CI runs. + `path` informs include sorting (the file's own header should come + first in a `.cc`). If no `clang-format` binary is available, return + the input unchanged.""" + + if _CLANG_FORMAT is None: + return text + + return subprocess.run( + [_CLANG_FORMAT, f"--assume-filename={path}"], + input=text, + capture_output=True, + text=True, + check=True, + ).stdout + + +def _emit(name: str, ops: list[Op], *, emit_base: bool) -> set[pathlib.Path]: + base_path = _GENERATED_BASE_DIR / f"{name}.h" + torch_dir = _GENERATED_TORCH_DIR / name + torch_header_path = torch_dir / f"{name}.h" + torch_source_path = torch_dir / f"{name}.cc" + emitted_paths = set() + + if emit_base: + _write_text_if_changed( + base_path, _clang_format(_generate_base_header(name, ops), base_path) + ) + emitted_paths.add(base_path) + + _write_text_if_changed( + torch_header_path, + _clang_format(_generate_torch_header(name, ops), torch_header_path), + ) + _write_text_if_changed( + torch_source_path, + _clang_format(_generate_torch_source(name, ops), torch_source_path), + ) + emitted_paths.add(torch_header_path) + emitted_paths.add(torch_source_path) + + return emitted_paths + + +def _select_op_names(cli_ops, default_ops, config): + if config is None: + return cli_ops or default_ops + + aten_names_by_public_name = collections.defaultdict(list) + for op_name in default_ops: + aten_names_by_public_name[_public_op_name(op_name)].append(op_name) + + selected_public_names = ops_config.torch_op_names( + config, aten_names_by_public_name, _PYTORCH_SLOT + ) + selected = [ + aten_name + for public_name in selected_public_names + for aten_name in aten_names_by_public_name.get(public_name, (public_name,)) + ] + + if cli_ops: + allowed = set(cli_ops) + selected = [op_name for op_name in selected if op_name in allowed] + + return selected + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--ops", + nargs="*", + help="Override the op allowlist. If omitted, reads `scripts/torch_ops.yaml`.", + ) + parser.add_argument( + "--ops-config", + type=pathlib.Path, + help="Path to an `ops.json` operator and implementation selection.", + ) + parser.add_argument( + "--pytorch-version", + default=os.environ.get("INFINI_OPS_PYTORCH_VERSION", _DEFAULT_PYTORCH_VERSION), + help=( + "PyTorch version label used in diagnostics. Schemas are read from " + "the locally installed `torchgen` package. Default: `%(default)s`. " + "Can also be set via the `INFINI_OPS_PYTORCH_VERSION` environment " + "variable." + ), + ) + args = parser.parse_args() + + global _CLANG_FORMAT + _CLANG_FORMAT = _find_clang_format() + + default_ops = yaml.safe_load(_OPS_YAML_PATH.read_text()) + config = ops_config.load_ops_config(args.ops_config) if args.ops_config else None + op_names = _select_op_names(args.ops, default_ops, config) + aten_entries = _load_aten_entries(args.pytorch_version) + + skipped: list[tuple[str, str]] = [] + metadata: list[dict] = [] + expected_base_files: set[pathlib.Path] = set() + expected_torch_files: set[pathlib.Path] = set() + ops_by_public_name: dict[str, list[Op]] = collections.defaultdict(list) + + for name in op_names: + candidates = _find_out_entries(aten_entries, name) + + if not candidates: + skipped.append((name, f"no `.out` variant for `{name}` in YAML")) + continue + + usable: list[Op] = [] + last_reason = "" + + for entry in candidates: + try: + op = _parse_func(entry["func"]) + + for param in op.params: + _ = param.cpp_type # eagerly raise on unsupported types + except (NotImplementedError, ValueError) as exc: + last_reason = str(exc) + continue + + if not op.is_testable: + last_reason = "no testable tensor input/output pair" + continue + + usable.append(op) + + if not usable: + skipped.append((name, last_reason or "no usable overload")) + continue + + usable, duplicate_overloads = _dedupe_visible_overloads(usable) + + for skipped_op, kept_op in duplicate_overloads: + skipped.append( + ( + skipped_op.infini_name, + "duplicate visible C++ signature for " + f"`{name}`; using `{kept_op.infini_name}`", + ) + ) + + ops_by_public_name[usable[0].infini_name].extend(usable) + + for public_name, usable in ops_by_public_name.items(): + usable, duplicate_overloads = _dedupe_visible_overloads(usable) + + for skipped_op, kept_op in duplicate_overloads: + skipped.append( + ( + skipped_op.infini_name, + "duplicate visible C++ signature for " + f"`{public_name}`; using `{kept_op.infini_name}`", + ) + ) + + base_exists = _base_path(public_name).exists() + + if base_exists: + usable, base_warnings = _bind_existing_base_overloads(public_name, usable) + + for warning in base_warnings: + skipped.append((public_name, warning)) + + if not usable: + skipped.append( + ( + public_name, + "existing base has no overload compatible with ATen schema", + ) + ) + continue + + # Emit one InfiniOps wrapper per ATen op. Distinct visible overloads + # become overloaded constructors / `operator()` methods on the same + # class (`Pow` exposes both tensor and scalar exponents). Overloads + # that collapse to the same C++ signature after hidden defaults are + # skipped above. When a hand-written `src/base/.h` exists, + # skip emitting `generated/base/.h` so the hand-written one + # wins (the generated torch source's `#include "base/.h"` + # resolves through `src/` first). Signature mismatches surface as + # compile errors with a clear message — drop the op from the YAML + # to suppress. + emitted_paths = _emit(public_name, usable, emit_base=not base_exists) + expected_base_files.update( + path for path in emitted_paths if path.is_relative_to(_GENERATED_BASE_DIR) + ) + expected_torch_files.update( + path for path in emitted_paths if path.is_relative_to(_GENERATED_TORCH_DIR) + ) + + for op in usable: + metadata.append( + { + "name": public_name, + "aten_name": op.aten_name, + "overload_name": op.infini_name, + "params": [ + { + "name": p.api_name, + "type": p.aten_type, + "is_tensor": p.is_tensor, + "is_out": p.is_out, + "schema_index": next( + ( + schema_index + for schema_index in range(len(op.params)) + if op.api_param_for(schema_index) is p + ), + None, + ), + } + for p in op.visible_params + ], + } + ) + + _GENERATED_DIR.mkdir(parents=True, exist_ok=True) + _write_text_if_changed( + _METADATA_PATH, json.dumps({"ops": metadata}, indent=2) + "\n" + ) + _remove_stale_files(_GENERATED_BASE_DIR, expected_base_files) + _remove_stale_files(_GENERATED_TORCH_DIR, expected_torch_files) + + generated_names = sorted({m["name"] for m in metadata}) + print( + f"generated {len(metadata)} overloads across {len(generated_names)} ops: " + f"{generated_names}" + ) + + for name, reason in skipped: + print(f" skipped {name!r}: {reason}", file=sys.stderr) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generate_wrappers.py b/scripts/generate_wrappers.py index a4cb10c62..093c27022 100644 --- a/scripts/generate_wrappers.py +++ b/scripts/generate_wrappers.py @@ -272,7 +272,7 @@ def _parse_operator_header(op_name): constructors = [ _ParsedFunction(_parse_parameter_list(params)) for params in _find_signature_parameters( - class_body, rf"(?:explicit\s+)?{class_name}\s*\(" + class_body, rf"\b(?:explicit\s+)?{class_name}\s*\(" ) ] calls = [ diff --git a/scripts/torch_ops.yaml b/scripts/torch_ops.yaml index 8ab9c4398..3b4608d10 100644 --- a/scripts/torch_ops.yaml +++ b/scripts/torch_ops.yaml @@ -100,6 +100,7 @@ - dstack - elu - elu_backward +- embedding_dense_backward - empty - eq - erf @@ -363,6 +364,8 @@ - sin - sinc - sinh +- slice_backward +- slice_copy - slogdet - slow_conv3d - slow_conv3d_forward @@ -454,6 +457,7 @@ - threshold - threshold_backward - topk +- transpose_copy - triangular_solve - tril - triu diff --git a/src/base/accumulate_grad.h b/src/base/accumulate_grad.h new file mode 100644 index 000000000..eeeafff8e --- /dev/null +++ b/src/base/accumulate_grad.h @@ -0,0 +1,55 @@ +#ifndef INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AccumulateGrad : public Operator { + public: + AccumulateGrad(const Tensor gradient, const double rate, Tensor tensor) + : gradient_shape_{gradient.shape()}, + gradient_strides_{gradient.strides()}, + gradient_type_{gradient.dtype()}, + tensor_shape_{tensor.shape()}, + tensor_strides_{tensor.strides()}, + tensor_type_{tensor.dtype()}, + rate_{rate}, + device_index_{tensor.device().index()} { + validation::AssertSameDevice({gradient, tensor}); + assert(validation::IsFloatingPoint(tensor_type_) && + "`AccumulateGrad` requires floating-point tensors."); + assert(gradient.shape() == tensor.shape() && + "`AccumulateGrad` requires `gradient` and `tensor` to share shape"); + assert(gradient_type_ == tensor_type_ && + "`AccumulateGrad` requires `gradient` and `tensor` to share dtype"); + } + + // Gradient and tensor may overlap, including partial external storage spans. + virtual void operator()(const Tensor gradient, const double rate, + Tensor tensor) const = 0; + + protected: + Tensor::Shape gradient_shape_; + + Tensor::Strides gradient_strides_; + + DataType gradient_type_; + + Tensor::Shape tensor_shape_; + + Tensor::Strides tensor_strides_; + + DataType tensor_type_; + + double rate_{1.0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ACCUMULATE_GRAD_H_ diff --git a/src/base/adam_accumulate_grad.h b/src/base/adam_accumulate_grad.h new file mode 100644 index 000000000..638d7fa3a --- /dev/null +++ b/src/base/adam_accumulate_grad.h @@ -0,0 +1,101 @@ +#ifndef INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AdamAccumulateGrad : public Operator { + public: + AdamAccumulateGrad(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, const double eps, + const int64_t step, Tensor param, Tensor m, Tensor v) + : grad_shape_{grad.shape()}, + grad_strides_{grad.strides()}, + grad_type_{grad.dtype()}, + param_shape_{param.shape()}, + param_strides_{param.strides()}, + param_type_{param.dtype()}, + m_shape_{m.shape()}, + m_strides_{m.strides()}, + m_type_{m.dtype()}, + v_shape_{v.shape()}, + v_strides_{v.strides()}, + v_type_{v.dtype()}, + learning_rate_{learning_rate}, + beta1_{beta1}, + beta2_{beta2}, + eps_{eps}, + step_{step}, + device_index_{param.device().index()} { + validation::AssertSameDevice({grad, param, m, v}); + assert(grad.shape() == param.shape() && param.shape() == m.shape() && + m.shape() == v.shape() && + "`AdamAccumulateGrad` requires grad/param/m/v to share shape"); + assert(validation::IsFloatingPoint(param_type_) && + grad_type_ == param_type_ && + "Adam parameters and gradients require matching floating dtypes."); + assert(m_type_ == v_type_ && + (m_type_ == param_type_ || (param_type_ != DataType::kFloat64 && + m_type_ == DataType::kFloat32)) && + "Adam moments must match the parameter dtype or use float32."); + + assert(0.0 <= beta1_ && beta1_ < 1.0 && 0.0 <= beta2_ && beta2_ < 1.0 && + "Adam beta values must be in [0, 1)."); + assert(learning_rate_ >= 0.0 && eps_ > 0.0 && + "Adam requires non-negative learning rate and positive epsilon."); + + assert(step_ >= 1 && "`AdamAccumulateGrad` requires step >= 1"); + } + + // Grad may alias outputs; param, m and v byte spans must not overlap. + virtual void operator()(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, + const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) const = 0; + + protected: + Tensor::Shape grad_shape_; + + Tensor::Strides grad_strides_; + + DataType grad_type_; + + Tensor::Shape param_shape_; + + Tensor::Strides param_strides_; + + DataType param_type_; + + Tensor::Shape m_shape_; + + Tensor::Strides m_strides_; + + DataType m_type_; + + Tensor::Shape v_shape_; + + Tensor::Strides v_strides_; + + DataType v_type_; + + double learning_rate_{0.0}; + + double beta1_{0.9}; + + double beta2_{0.999}; + + double eps_{1e-8}; + + int64_t step_{1}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADAM_ACCUMULATE_GRAD_H_ diff --git a/src/base/add_backward.h b/src/base/add_backward.h new file mode 100644 index 000000000..230c4b152 --- /dev/null +++ b/src/base/add_backward.h @@ -0,0 +1,63 @@ +#ifndef INFINI_OPS_BASE_ADD_BACKWARD_H_ +#define INFINI_OPS_BASE_ADD_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class AddBackward : public Operator { + public: + AddBackward(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`AddBackward` requires matching dtypes"); + + validation::AssertBroadcastsTo(grad_input, grad_output); + + validation::AssertBroadcastsTo(grad_other, grad_output); + } + + // Inputs may alias outputs; output byte spans must not overlap. + virtual void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_ADD_BACKWARD_H_ diff --git a/src/base/cross_entropy.h b/src/base/cross_entropy.h new file mode 100644 index 000000000..e59123d6e --- /dev/null +++ b/src/base/cross_entropy.h @@ -0,0 +1,64 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_H_ + +#include +#include +#include +#include + +#include "base/cross_entropy_common.h" +#include "operator.h" + +namespace infini::ops { + +class CrossEntropy : public Operator { + public: + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropy(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + device_index_{out.device().index()} { + ValidateCrossEntropy(input, target, weight, ignore_index, reduction, + label_smoothing, out); + } + + virtual void operator()(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, + Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape target_shape_; + + Tensor::Strides target_strides_; + + DataType target_type_; + + Tensor::Shape out_shape_; + + Tensor::Strides out_strides_; + + DataType out_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_H_ diff --git a/src/base/cross_entropy_backward.h b/src/base/cross_entropy_backward.h new file mode 100644 index 000000000..50085b78d --- /dev/null +++ b/src/base/cross_entropy_backward.h @@ -0,0 +1,77 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ + +#include +#include + +#include "base/cross_entropy_common.h" +#include "operator.h" + +namespace infini::ops { + +class CrossEntropyBackward : public Operator { + public: + static constexpr int64_t kIgnoreIndex = -100; + + CrossEntropyBackward(const Tensor input, const Tensor target, + const Tensor grad_output, std::optional weight, + int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + target_shape_{target.shape()}, + target_strides_{target.strides()}, + target_type_{target.dtype()}, + grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + ValidateCrossEntropy(input, target, weight, ignore_index, reduction, + label_smoothing, grad_output); + validation::AssertSameDevice({input, grad_input}); + assert(grad_input.shape() == input.shape() && + grad_input.dtype() == input.dtype() && + "`CrossEntropyBackward` grad_input must match logits metadata."); + } + + virtual void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape target_shape_; + + Tensor::Strides target_strides_; + + DataType target_type_; + + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/base/cross_entropy_common.h b/src/base/cross_entropy_common.h new file mode 100644 index 000000000..c9c4496b0 --- /dev/null +++ b/src/base/cross_entropy_common.h @@ -0,0 +1,78 @@ +#ifndef INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ +#define INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ + +#include +#include + +#include "base/validation.h" + +namespace infini::ops { + +inline DataType PromoteCrossEntropyType(DataType a, DataType b) { + if (a == b) return a; + if (a == DataType::kFloat64 || b == DataType::kFloat64) { + return DataType::kFloat64; + } + return DataType::kFloat32; +} + +inline void ValidateCrossEntropy(const Tensor input, const Tensor target, + std::optional weight, + int64_t ignore_index, + const std::string& reduction, + double label_smoothing, const Tensor loss) { + validation::AssertSameDevice({input, target, loss}); + + assert(input.ndim() >= 1 && "`CrossEntropy` requires input rank >= 1."); + assert(validation::IsFloatingPoint(input.dtype()) && + "`CrossEntropy` requires floating-point logits."); + assert((reduction == "none" || reduction == "mean" || reduction == "sum") && + "`CrossEntropy` reduction must be none, mean or sum."); + assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && + "`CrossEntropy` label_smoothing must be in [0, 1]."); + const auto class_dim = input.ndim() == 1 ? 0 : 1; + assert(input.size(class_dim) > 0 && + "`CrossEntropy` requires a non-empty class dimension."); + Tensor::Shape loss_shape; + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + if (i != class_dim) loss_shape.push_back(input.size(i)); + } + const bool probability_target = target.shape() == input.shape(); + + auto result_type = input.dtype(); + if (probability_target) { + assert(validation::IsFloatingPoint(target.dtype()) && + "`CrossEntropy` probability target must be floating-point."); + assert(ignore_index < 0 && + "`CrossEntropy` probability target does not support ignore_index."); + result_type = PromoteCrossEntropyType(result_type, target.dtype()); + } else { + assert((target.dtype() == DataType::kInt64 || + target.dtype() == DataType::kInt32 || + target.dtype() == DataType::kUInt8) && + "`CrossEntropy` class target must be int64, int32 or uint8."); + assert( + target.shape() == loss_shape && + "`CrossEntropy` class target shape must remove the class dimension."); + } + if (weight) { + validation::AssertSameDevice({input, *weight}); + assert(weight->ndim() == 1 && weight->size(0) == input.size(class_dim) && + "`CrossEntropy` weight must have shape (C,)."); + if (probability_target) { + if (validation::IsFloatingPoint(weight->dtype())) { + result_type = PromoteCrossEntropyType(result_type, weight->dtype()); + } + } else { + assert(weight->dtype() == input.dtype() && + "`CrossEntropy` class-target weight dtype must match logits."); + } + } + assert(loss.shape() == (reduction == "none" ? loss_shape : Tensor::Shape{}) && + "`CrossEntropy` loss shape does not match reduction."); + assert(loss.dtype() == result_type && "`CrossEntropy` loss dtype mismatch."); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_CROSS_ENTROPY_COMMON_H_ diff --git a/src/base/masked_fill_backward.h b/src/base/masked_fill_backward.h new file mode 100644 index 000000000..987ba33b9 --- /dev/null +++ b/src/base/masked_fill_backward.h @@ -0,0 +1,97 @@ +#ifndef INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ +#define INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ + +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MaskedFillBackward : public Operator { + public: + MaskedFillBackward(const Tensor grad_output, const Tensor mask, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + mask_shape_{mask.shape()}, + mask_strides_{mask.strides()}, + mask_type_{mask.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, mask, grad_input}); + assert( + grad_output_type_ == grad_input_type_ && + "`MaskedFillBackward` requires grad_output/grad_input dtype to match"); + assert(mask_type_ == DataType::kBool && + "`MaskedFillBackward` requires a bool `mask`"); + assert(IsValidBroadcastShape(grad_input_shape_, mask_shape_, + grad_output_shape_) && + "`MaskedFillBackward` requires grad_output shape to equal the " + "broadcast shape of grad_input and mask"); + } + + virtual void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const = 0; + + protected: + static bool IsValidBroadcastShape(const Tensor::Shape& input_shape, + const Tensor::Shape& mask_shape, + const Tensor::Shape& output_shape) { + const auto ndim = input_shape.size() > mask_shape.size() + ? input_shape.size() + : mask_shape.size(); + if (output_shape.size() != ndim) { + return false; + } + + const auto input_offset = ndim - input_shape.size(); + + const auto mask_offset = ndim - mask_shape.size(); + + for (std::size_t i = 0; i < ndim; ++i) { + const auto input_dim = + i < input_offset ? 1 : input_shape[i - input_offset]; + const auto mask_dim = i < mask_offset ? 1 : mask_shape[i - mask_offset]; + + if (input_dim != mask_dim && input_dim != 1 && mask_dim != 1) { + return false; + } + + const auto output_dim = input_dim == 1 ? mask_dim : input_dim; + if (output_shape[i] != output_dim) { + return false; + } + } + return true; + } + + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape mask_shape_; + + Tensor::Strides mask_strides_; + + DataType mask_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MASKED_FILL_BACKWARD_H_ diff --git a/src/base/mean_backward.h b/src/base/mean_backward.h new file mode 100644 index 000000000..3e448e18f --- /dev/null +++ b/src/base/mean_backward.h @@ -0,0 +1,79 @@ +#ifndef INFINI_OPS_BASE_MEAN_BACKWARD_H_ +#define INFINI_OPS_BASE_MEAN_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MeanBackward : public Operator { + public: + MeanBackward(const Tensor grad_output, const int64_t dim, const bool keepdim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{validation::NormalizeDim(dim, grad_input.ndim())}, + keepdim_{keepdim}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output_type_ == grad_input_type_ && + "`MeanBackward` requires matching dtypes"); + + assert(grad_input.ndim() >= 1 && "`MeanBackward` requires ndim >= 1"); + if (keepdim_) { + assert(grad_output.ndim() == grad_input.ndim() && + grad_output.size(dim_) == 1 && + "`MeanBackward` keepdim=true requires size-1 dim in grad_output"); + + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); + } + } else { + assert(grad_output.ndim() == grad_input.ndim() - 1 && + "`MeanBackward` keepdim=false requires rank-1 grad_output"); + + for (Tensor::Size i = 0, s = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(s) == grad_input.size(i) && + "`MeanBackward` shape mismatch"); + + ++s; + } + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + const bool keepdim, Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t dim_{0}; + + bool keepdim_{false}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MEAN_BACKWARD_H_ diff --git a/src/base/mul_backward.h b/src/base/mul_backward.h new file mode 100644 index 000000000..cd86d93cf --- /dev/null +++ b/src/base/mul_backward.h @@ -0,0 +1,92 @@ +#ifndef INFINI_OPS_BASE_MUL_BACKWARD_H_ +#define INFINI_OPS_BASE_MUL_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class MulBackward : public Operator { + public: + MulBackward(const Tensor grad_output, const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + other_shape_{other.shape()}, + other_strides_{other.strides()}, + other_type_{other.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice( + {grad_output, input, other, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`MulBackward` requires matching grad dtypes"); + assert( + input_type_ == grad_output_type_ && other_type_ == grad_output_type_ && + "`MulBackward` requires input/other dtype to match grad_output dtype"); + + assert(grad_input.shape() == input.shape() && + grad_other.shape() == other.shape() && + "Multiplication gradients must match their input shapes."); + + validation::AssertBroadcastsTo(input, grad_output); + + validation::AssertBroadcastsTo(other, grad_output); + } + + // Aliasing: inputs may overlap each other and either output. + // The storage byte spans of grad_input and grad_other must not overlap, + // including gaps in non-contiguous layouts. + virtual void operator()(const Tensor grad_output, const Tensor input, + const Tensor other, Tensor grad_input, + Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape other_shape_; + + Tensor::Strides other_strides_; + + DataType other_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_MUL_BACKWARD_H_ diff --git a/src/base/pow_backward.h b/src/base/pow_backward.h new file mode 100644 index 000000000..c184c7393 --- /dev/null +++ b/src/base/pow_backward.h @@ -0,0 +1,73 @@ +#ifndef INFINI_OPS_BASE_POW_BACKWARD_H_ +#define INFINI_OPS_BASE_POW_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class PowBackward : public Operator { + public: + PowBackward(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + scalar_{scalar}, + scalar_is_base_{scalar_is_base}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, input, grad_input}); + assert( + input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`PowBackward` requires input/grad_output/grad_input to share shape"); + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && + "`PowBackward` requires matching dtypes"); + assert((!scalar_is_base_ || scalar_ > 0.0) && + "`PowBackward` scalar base must be positive for log(scalar)."); + assert(validation::IsFloatingPoint(input_type_) && + "Power gradients require floating dtypes."); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + const double scalar, const bool scalar_is_base, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + double scalar_{0.0}; + + bool scalar_is_base_{false}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_POW_BACKWARD_H_ diff --git a/src/base/repeat_interleave.h b/src/base/repeat_interleave.h new file mode 100644 index 000000000..5afae25d8 --- /dev/null +++ b/src/base/repeat_interleave.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleave : public Operator { + public: + RepeatInterleave(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + out_shape_{out.shape()}, + out_strides_{out.strides()}, + out_type_{out.dtype()}, + repeats_{repeats}, + dim_{validation::NormalizeDim(dim, input.ndim())}, + device_index_{out.device().index()} { + validation::AssertSameDevice({input, out}); + assert(repeats_ >= 0 && + "`RepeatInterleave` requires non-negative repeats."); + + assert(input.ndim() == out.ndim() && "RepeatInterleave ranks must match."); + assert(input_type_ == out_type_ && + "`RepeatInterleave` requires same dtype"); + + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + const auto expect = + (i == dim_) ? input.size(i) * repeats_ : input.size(i); + assert(out.size(i) == expect && + "`RepeatInterleave` output shape mismatch"); + } + } + + virtual void operator()(const Tensor input, const int64_t repeats, + const int64_t dim, Tensor out) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape out_shape_; + + Tensor::Strides out_strides_; + + DataType out_type_; + + int64_t repeats_{0}; + + int64_t dim_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_H_ diff --git a/src/base/repeat_interleave_backward.h b/src/base/repeat_interleave_backward.h new file mode 100644 index 000000000..b8d586fb0 --- /dev/null +++ b/src/base/repeat_interleave_backward.h @@ -0,0 +1,72 @@ +#ifndef INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ +#define INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RepeatInterleaveBackward : public Operator { + public: + RepeatInterleaveBackward(const Tensor grad_output, const int64_t dim, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim_{validation::NormalizeDim(dim, grad_input.ndim())}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output_type_ == grad_input_type_ && + "`RepeatInterleaveBackward` requires same dtype"); + assert(grad_output.ndim() == grad_input.ndim() && + "`RepeatInterleaveBackward` requires same rank"); + + for (Tensor::Size i = 0; i < grad_input.ndim(); ++i) { + if (i == dim_) continue; + assert(grad_output.size(i) == grad_input.size(i) && + "`RepeatInterleaveBackward` shape mismatch on non-repeated dims"); + } + if (grad_input.size(dim_) == 0) { + assert(grad_output.size(dim_) == 0 && + "Empty repeated dimensions must match."); + + repeats_ = 0; + } else { + repeats_ = grad_output.size(dim_) / grad_input.size(dim_); + assert(grad_output.size(dim_) == grad_input.size(dim_) * repeats_ && + "RepeatInterleaveBackward grad_output dim must be divisible."); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t repeats_{0}; + + int64_t dim_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_REPEAT_INTERLEAVE_BACKWARD_H_ diff --git a/src/base/rsqrt_backward.h b/src/base/rsqrt_backward.h new file mode 100644 index 000000000..28be8d647 --- /dev/null +++ b/src/base/rsqrt_backward.h @@ -0,0 +1,62 @@ +#ifndef INFINI_OPS_BASE_RSQRT_BACKWARD_H_ +#define INFINI_OPS_BASE_RSQRT_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class RsqrtBackward : public Operator { + public: + RsqrtBackward(const Tensor grad_output, const Tensor input, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, input, grad_input}); + assert(input.shape() == grad_output.shape() && + grad_output.shape() == grad_input.shape() && + "`RsqrtBackward` requires matching shapes"); + assert(input_type_ == grad_output_type_ && + grad_output_type_ == grad_input_type_ && + "`RsqrtBackward` requires matching dtypes"); + assert(validation::IsFloatingPoint(input_type_) && + "`RsqrtBackward` requires floating point dtypes"); + } + + virtual void operator()(const Tensor grad_output, const Tensor input, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_RSQRT_BACKWARD_H_ diff --git a/src/base/split.h b/src/base/split.h new file mode 100644 index 000000000..475e3bbe9 --- /dev/null +++ b/src/base/split.h @@ -0,0 +1,88 @@ +#ifndef INFINI_OPS_BASE_SPLIT_H_ +#define INFINI_OPS_BASE_SPLIT_H_ + +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class Split : public Operator { + public: + Split(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs) + : input_shape_{input.shape()}, + input_strides_{input.strides()}, + input_type_{input.dtype()}, + split_size_{split_size}, + dim_{validation::NormalizeDim(dim, input.ndim())}, + outs_{std::move(outs)}, + device_index_{outs_.empty() ? 0 : outs_.front().device().index()} { + assert(split_size_ > 0 && "`Split` requires positive `split_size`"); + + assert(!outs_.empty() && "`Split` requires at least one output tensor"); + const auto expected = + input.size(dim_) == 0 ? 1 : 1 + (input.size(dim_) - 1) / split_size_; + assert( + static_cast(outs_.size()) == expected && + "`Split` output count must equal ceil(input_size(dim) / split_size)"); + + for (std::size_t i = 0; i < outs_.size(); ++i) { + validation::AssertSameDevice({input, outs_[i]}); + assert(outs_[i].ndim() == input.ndim() && "`Split` output rank mismatch"); + + assert(outs_[i].dtype() == input_type_ && "`Split` dtype mismatch"); + + for (Tensor::Size d = 0; d < input.ndim(); ++d) { + if (d != dim_) { + assert(outs_[i].size(d) == input.size(d) && + "`Split` output shape mismatch"); + } + } + const auto expect_dim = std::min( + split_size_, static_cast(input.size(dim_)) - + static_cast(i) * split_size_); + assert(outs_[i].size(dim_) == expect_dim && + "`Split` output dim size mismatch"); + + out_shapes_.emplace_back(outs_[i].shape().begin(), + outs_[i].shape().end()); + out_strides_.emplace_back(outs_[i].strides().begin(), + outs_[i].strides().end()); + + out_types_.emplace_back(outs_[i].dtype()); + } + } + + virtual void operator()(const Tensor input, const int64_t split_size, + const int64_t dim, + std::vector outs) const = 0; + + protected: + Tensor::Shape input_shape_; + + Tensor::Strides input_strides_; + + DataType input_type_; + + int64_t split_size_{0}; + + int64_t dim_{0}; + + std::vector outs_{}; + + std::vector out_shapes_; + + std::vector out_strides_; + + std::vector out_types_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SPLIT_H_ diff --git a/src/base/stack_backward.h b/src/base/stack_backward.h new file mode 100644 index 000000000..9fcf6b7ec --- /dev/null +++ b/src/base/stack_backward.h @@ -0,0 +1,80 @@ +#ifndef INFINI_OPS_BASE_STACK_BACKWARD_H_ +#define INFINI_OPS_BASE_STACK_BACKWARD_H_ + +#include +#include +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class StackBackward : public Operator { + public: + StackBackward(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + dim_{validation::NormalizeDim(dim, grad_output.ndim())}, + grad_inputs_{std::move(grad_inputs)}, + device_index_{ + grad_inputs_.empty() ? 0 : grad_inputs_.front().device().index()} { + assert(!grad_inputs_.empty() && + "`StackBackward` requires at least one output"); + assert(static_cast(grad_inputs_.size()) == + grad_output.size(dim_) && + "`StackBackward` output count must equal grad_output.size(dim)"); + + for (std::size_t i = 0; i < grad_inputs_.size(); ++i) { + validation::AssertSameDevice({grad_output, grad_inputs_[i]}); + assert(grad_inputs_[i].ndim() == grad_output.ndim() - 1 && + "`StackBackward` output rank must be grad_output rank - 1"); + assert(grad_inputs_[i].dtype() == grad_output_type_ && + "`StackBackward` dtype mismatch"); + + for (Tensor::Size d = 0, s = 0; d < grad_output.ndim(); ++d) { + if (d == dim_) continue; + assert(grad_inputs_[i].size(s) == grad_output.size(d) && + "`StackBackward` shape mismatch"); + + ++s; + } + grad_input_shapes_.emplace_back(grad_inputs_[i].shape().begin(), + grad_inputs_[i].shape().end()); + + grad_input_strides_.emplace_back(grad_inputs_[i].strides().begin(), + grad_inputs_[i].strides().end()); + + grad_input_types_.push_back(grad_inputs_[i].dtype()); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + int64_t dim_{0}; + + std::vector grad_inputs_{}; + + std::vector grad_input_shapes_; + + std::vector grad_input_strides_; + + std::vector grad_input_types_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_STACK_BACKWARD_H_ diff --git a/src/base/sub_backward.h b/src/base/sub_backward.h new file mode 100644 index 000000000..71dcfd37a --- /dev/null +++ b/src/base/sub_backward.h @@ -0,0 +1,63 @@ +#ifndef INFINI_OPS_BASE_SUB_BACKWARD_H_ +#define INFINI_OPS_BASE_SUB_BACKWARD_H_ + +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class SubBackward : public Operator { + public: + SubBackward(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + grad_other_shape_{grad_other.shape()}, + grad_other_strides_{grad_other.strides()}, + grad_other_type_{grad_other.dtype()}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input, grad_other}); + assert(grad_output_type_ == grad_input_type_ && + grad_input_type_ == grad_other_type_ && + "`SubBackward` requires matching dtypes"); + + validation::AssertBroadcastsTo(grad_input, grad_output); + + validation::AssertBroadcastsTo(grad_other, grad_output); + } + + // Inputs may alias outputs; output byte spans must not overlap. + virtual void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + Tensor::Shape grad_other_shape_; + + Tensor::Strides grad_other_strides_; + + DataType grad_other_type_; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_SUB_BACKWARD_H_ diff --git a/src/base/transpose_backward.h b/src/base/transpose_backward.h new file mode 100644 index 000000000..8f80bac68 --- /dev/null +++ b/src/base/transpose_backward.h @@ -0,0 +1,66 @@ +#ifndef INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ +#define INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class TransposeBackward : public Operator { + public: + TransposeBackward(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + dim0_{validation::NormalizeDim(dim0, grad_output.ndim())}, + dim1_{validation::NormalizeDim(dim1, grad_output.ndim())}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, grad_input}); + assert(grad_output.ndim() >= 1 && "Transpose requires a non-scalar input."); + assert(grad_input.ndim() == grad_output.ndim() && + "Transpose ranks must match."); + assert(grad_output_type_ == grad_input_type_ && + "`TransposeBackward` requires same dtype"); + + for (Tensor::Size i = 0; i < grad_output.ndim(); ++i) { + const auto expect = (i == dim0_) ? grad_output.size(dim1_) + : (i == dim1_) ? grad_output.size(dim0_) + : grad_output.size(i); + assert(grad_input.size(i) == expect && + "`TransposeBackward` grad_input shape mismatch"); + } + } + + virtual void operator()(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t dim0_{0}; + + int64_t dim1_{0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_TRANSPOSE_BACKWARD_H_ diff --git a/src/base/validation.h b/src/base/validation.h new file mode 100644 index 000000000..23900b0db --- /dev/null +++ b/src/base/validation.h @@ -0,0 +1,50 @@ +#ifndef INFINI_OPS_BASE_VALIDATION_H_ +#define INFINI_OPS_BASE_VALIDATION_H_ + +#include +#include +#include + +#include "data_type.h" +#include "tensor.h" + +namespace infini::ops::validation { + +inline void AssertSameDevice(std::initializer_list tensors) { + if (tensors.size() == 0) return; + + const auto device = tensors.begin()->device(); + for (const auto& tensor : tensors) { + assert(tensor.device() == device && + "Tensors must share a device type and index."); + } +} + +inline bool IsFloatingPoint(DataType dtype) { + return dtype == DataType::kFloat16 || dtype == DataType::kBFloat16 || + dtype == DataType::kFloat32 || dtype == DataType::kFloat64; +} + +inline int64_t NormalizeDim(int64_t dim, Tensor::Size ndim) { + const auto rank = static_cast(ndim); + + assert(rank > 0 && "Dimension requires a non-scalar tensor."); + + assert(dim >= -rank && dim < rank && "Dimension is out of range."); + + return dim < 0 ? dim + rank : dim; +} + +inline void AssertBroadcastsTo(const Tensor input, const Tensor output) { + assert(input.ndim() <= output.ndim() && + "Input rank must not exceed output rank."); + const auto offset = output.ndim() - input.ndim(); + for (Tensor::Size i = 0; i < input.ndim(); ++i) { + assert((input.size(i) == 1 || input.size(i) == output.size(i + offset)) && + "Input must broadcast to output with right-aligned dimensions."); + } +} + +} // namespace infini::ops::validation + +#endif // INFINI_OPS_BASE_VALIDATION_H_ diff --git a/src/base/vocab_parallel_cross_entropy_backward.h b/src/base/vocab_parallel_cross_entropy_backward.h new file mode 100644 index 000000000..0279d27a3 --- /dev/null +++ b/src/base/vocab_parallel_cross_entropy_backward.h @@ -0,0 +1,113 @@ +#ifndef INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ + +#include +#include + +#include "base/validation.h" +#include "operator.h" + +namespace infini::ops { + +class VocabParallelCrossEntropyBackward + : public Operator { + public: + VocabParallelCrossEntropyBackward( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) + : grad_output_shape_{grad_output.shape()}, + grad_output_strides_{grad_output.strides()}, + grad_output_type_{grad_output.dtype()}, + softmax_shape_{softmax_local.shape()}, + softmax_strides_{softmax_local.strides()}, + softmax_type_{softmax_local.dtype()}, + target_mask_shape_{target_mask.shape()}, + target_mask_strides_{target_mask.strides()}, + target_mask_type_{target_mask.dtype()}, + masked_target_shape_{masked_target.shape()}, + masked_target_strides_{masked_target.strides()}, + masked_target_type_{masked_target.dtype()}, + valid_mask_shape_{valid_mask_local.shape()}, + valid_mask_strides_{valid_mask_local.strides()}, + valid_mask_type_{valid_mask_local.dtype()}, + grad_input_shape_{grad_input.shape()}, + grad_input_strides_{grad_input.strides()}, + grad_input_type_{grad_input.dtype()}, + vocab_size_local_{vocab_size_local}, + vocab_size_original_{vocab_size_original}, + label_smoothing_{label_smoothing}, + device_index_{grad_input.device().index()} { + validation::AssertSameDevice({grad_output, softmax_local, target_mask, + masked_target, valid_mask_local, grad_input}); + assert(validation::IsFloatingPoint(grad_input_type_) && + grad_input_type_ == softmax_type_ && + grad_output_type_ == grad_input_type_ && + "Loss gradients require matching floating dtypes."); + assert(softmax_local.shape() == grad_input.shape() && + "Local softmax and gradient shapes must match."); + assert(vocab_size_local > 0 && vocab_size_original > 0 && + "Vocabulary sizes must be positive."); + assert(label_smoothing >= 0.0 && label_smoothing <= 1.0 && + "Label smoothing must be in [0, 1]."); + } + + virtual void operator()(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, + const int64_t vocab_size_local, + const int64_t vocab_size_original, + const double label_smoothing, + Tensor grad_input) const = 0; + + protected: + Tensor::Shape grad_output_shape_; + + Tensor::Strides grad_output_strides_; + + DataType grad_output_type_; + + Tensor::Shape softmax_shape_; + + Tensor::Strides softmax_strides_; + + DataType softmax_type_; + + Tensor::Shape target_mask_shape_; + + Tensor::Strides target_mask_strides_; + + DataType target_mask_type_; + + Tensor::Shape masked_target_shape_; + + Tensor::Strides masked_target_strides_; + + DataType masked_target_type_; + + Tensor::Shape valid_mask_shape_; + + Tensor::Strides valid_mask_strides_; + + DataType valid_mask_type_; + + Tensor::Shape grad_input_shape_; + + Tensor::Strides grad_input_strides_; + + DataType grad_input_type_; + + int64_t vocab_size_local_{0}; + + int64_t vocab_size_original_{0}; + + double label_smoothing_{0.0}; + + int device_index_{0}; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_BASE_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.cc b/src/torch/ops/accumulate_grad/accumulate_grad.cc new file mode 100644 index 000000000..37a46aac6 --- /dev/null +++ b/src/torch/ops/accumulate_grad/accumulate_grad.cc @@ -0,0 +1,35 @@ +#include "torch/ops/accumulate_grad/accumulate_grad.h" + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor gradient, + const double rate, Tensor tensor) + : AccumulateGrad{gradient, rate, tensor} {} + +template +void Operator::operator()(const Tensor gradient, + const double rate, + Tensor tensor) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_gradient = + ToAtenTensor(const_cast(gradient.data()), gradient_shape_, + gradient_strides_, gradient_type_, device_index_); + auto at_tensor = + ToAtenTensor(tensor.data(), tensor_shape_, tensor_strides_, + tensor_type_, device_index_); + + if (StorageSpansOverlap(at_tensor, at_gradient)) { + at_gradient = at_gradient.clone(); + } + at_tensor.add_(at_gradient, rate); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AccumulateGrad, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/accumulate_grad/accumulate_grad.h b/src/torch/ops/accumulate_grad/accumulate_grad.h new file mode 100644 index 000000000..863186cdb --- /dev/null +++ b/src/torch/ops/accumulate_grad/accumulate_grad.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_TORCH_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_TORCH_ACCUMULATE_GRAD_H_ + +#include "base/accumulate_grad.h" + +namespace infini::ops { + +template +class Operator : public AccumulateGrad { + public: + Operator(const Tensor gradient, const double rate, Tensor tensor); + + void operator()(const Tensor gradient, const double rate, + Tensor tensor) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc new file mode 100644 index 000000000..ee7b7abc7 --- /dev/null +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.cc @@ -0,0 +1,58 @@ +#include "torch/ops/adam_accumulate_grad/adam_accumulate_grad.h" + +#include + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) + : AdamAccumulateGrad{grad, learning_rate, beta1, beta2, eps, + step, param, m, v} {} + +template +void Operator::operator()( + const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, Tensor param, + Tensor m, Tensor v) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad = ToAtenTensor(const_cast(grad.data()), grad_shape_, + grad_strides_, grad_type_, device_index_); + auto at_param = ToAtenTensor(param.data(), param_shape_, param_strides_, + param_type_, device_index_); + auto at_m = ToAtenTensor(m.data(), m_shape_, m_strides_, m_type_, + device_index_); + auto at_v = ToAtenTensor(v.data(), v_shape_, v_strides_, v_type_, + device_index_); + + assert(!StorageSpansOverlap(at_param, at_m) && + !StorageSpansOverlap(at_param, at_v) && + !StorageSpansOverlap(at_m, at_v) && + "`AdamAccumulateGrad` output storage must not overlap."); + + const auto compute_type = + at_param.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + const auto gradient = at_grad.to(compute_type); + const auto next_m = at::lerp(at_m.to(compute_type), gradient, 1.0 - beta1_); + auto next_v = at_v.to(compute_type) * beta2_; + next_v.addcmul_(gradient * (1.0 - beta2_), gradient); + const double bcm = 1.0 - std::pow(beta1_, static_cast(step_)); + const double bcv = 1.0 - std::pow(beta2_, static_cast(step_)); + const auto denominator = at::sqrt(next_v) / std::sqrt(bcv) + eps_; + const auto next_param = at::addcdiv(at_param.to(compute_type), next_m, + denominator, -learning_rate_ / bcm); + at_param.copy_(next_param); + at_m.copy_(next_m); + at_v.copy_(next_v); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AdamAccumulateGrad, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h new file mode 100644 index 000000000..8579c0206 --- /dev/null +++ b/src/torch/ops/adam_accumulate_grad/adam_accumulate_grad.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ +#define INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ + +#include "base/adam_accumulate_grad.h" + +namespace infini::ops { + +template +class Operator : public AdamAccumulateGrad { + public: + Operator(const Tensor grad, const double learning_rate, const double beta1, + const double beta2, const double eps, const int64_t step, + Tensor param, Tensor m, Tensor v); + + void operator()(const Tensor grad, const double learning_rate, + const double beta1, const double beta2, const double eps, + const int64_t step, Tensor param, Tensor m, + Tensor v) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_ADAM_ACCUMULATE_GRAD_H_ diff --git a/src/torch/ops/add_backward/add_backward.cc b/src/torch/ops/add_backward/add_backward.cc new file mode 100644 index 000000000..a59745add --- /dev/null +++ b/src/torch/ops/add_backward/add_backward.cc @@ -0,0 +1,54 @@ +#include "torch/ops/add_backward/add_backward.h" + +#include +#include +#include + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const double alpha, Tensor grad_input, + Tensor grad_other) + : AddBackward{grad_output, alpha, grad_input, grad_other} {} + +template +void Operator::operator()(const Tensor grad_output, + const double alpha, + Tensor grad_input, + Tensor grad_other) const { + const TorchStreamGuard guard{stream_, device_index_}; + + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`AddBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = at_grad.sum_to_size(grad_input_sizes).clone(); + auto grad_other_result = (at_grad * alpha).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(AddBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/add_backward/add_backward.h b/src/torch/ops/add_backward/add_backward.h new file mode 100644 index 000000000..b534455db --- /dev/null +++ b/src/torch/ops/add_backward/add_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_ADD_BACKWARD_H_ +#define INFINI_OPS_TORCH_ADD_BACKWARD_H_ + +#include "base/add_backward.h" + +namespace infini::ops { + +template +class Operator : public AddBackward { + public: + Operator(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other); + + void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_ADD_BACKWARD_H_ diff --git a/src/torch/ops/cross_entropy/cross_entropy.cc b/src/torch/ops/cross_entropy/cross_entropy.cc new file mode 100644 index 000000000..5eabdecc1 --- /dev/null +++ b/src/torch/ops/cross_entropy/cross_entropy.cc @@ -0,0 +1,71 @@ +#include "torch/ops/cross_entropy/cross_entropy.h" + +#include + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, + const Tensor target, + std::optional weight, + int64_t ignore_index, + std::string reduction, + double label_smoothing, Tensor out) + : CrossEntropy{input, target, weight, ignore_index, + reduction, label_smoothing, out} {} + +template +void Operator::operator()( + const Tensor input, const Tensor target, std::optional weight, + int64_t ignore_index, std::string reduction, double label_smoothing, + Tensor out) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_target = + ToAtenTensor(const_cast(target.data()), target_shape_, + target_strides_, target_type_, device_index_); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + std::optional at_weight; + if (weight) { + at_weight = + ToAtenTensor(const_cast(weight->data()), weight->shape(), + weight->strides(), weight->dtype(), device_index_); + } + + const bool probability_target = target_shape_ == input_shape_; + if (!probability_target) at_target = at_target.to(at::kLong); + + const auto promote_low_precision = [](at::Tensor value) { + if (value.scalar_type() == at::kHalf || + value.scalar_type() == at::kBFloat16) { + return value.to(at::kFloat); + } + return value; + }; + at_input = promote_low_precision(at_input); + if (probability_target) at_target = promote_low_precision(at_target); + if (at_weight) *at_weight = promote_low_precision(*at_weight); + + assert((reduction == "none" || reduction == "mean" || reduction == "sum") && + "Invalid CrossEntropy reduction"); + const int64_t at_reduction = reduction == "none" ? at::Reduction::None + : reduction == "sum" ? at::Reduction::Sum + : at::Reduction::Mean; + + auto loss = + at::cross_entropy_loss(at_input, at_target, at_weight, at_reduction, + ignore_index, label_smoothing); + at_out.copy_(loss); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropy, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/cross_entropy/cross_entropy.h b/src/torch/ops/cross_entropy/cross_entropy.h new file mode 100644 index 000000000..87adfb06a --- /dev/null +++ b/src/torch/ops/cross_entropy/cross_entropy.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_H_ +#define INFINI_OPS_TORCH_CROSS_ENTROPY_H_ + +#include "base/cross_entropy.h" + +namespace infini::ops { + +template +class Operator : public CrossEntropy { + public: + Operator(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor out); + + void operator()(const Tensor input, const Tensor target, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, + Tensor out) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_H_ diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc new file mode 100644 index 000000000..3cb92b777 --- /dev/null +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.cc @@ -0,0 +1,94 @@ +#include "torch/ops/cross_entropy_backward/cross_entropy_backward.h" + +#include + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) + : CrossEntropyBackward{input, target, grad_output, + weight, ignore_index, reduction, + label_smoothing, grad_input} {} + +template +void Operator::operator()( + const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto x = ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto target_tensor = + ToAtenTensor(const_cast(target.data()), target_shape_, + target_strides_, target_type_, device_index_); + auto upstream = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto out = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + // Accumulate low-precision derivatives and reduction denominators in FP32. + const auto compute_type = + x.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + x = x.to(compute_type); + if (upstream.scalar_type() == at::kHalf || + upstream.scalar_type() == at::kBFloat16) { + upstream = upstream.to(at::kFloat); + } + const int64_t dim = x.dim() == 1 ? 0 : 1; + const auto classes = x.size(dim); + const bool probability_target = target_shape_ == input_shape_; + std::vector weight_shape(x.dim(), 1); + weight_shape[dim] = classes; + auto class_weight = at::ones({classes}, x.options()); + if (weight) { + class_weight = + ToAtenTensor(const_cast(weight->data()), weight->shape(), + weight->strides(), weight->dtype(), device_index_); + if (class_weight.scalar_type() != at::kDouble) { + class_weight = class_weight.to(compute_type); + } + } + at::Tensor valid; + at::Tensor denominator; + at::Tensor distribution; + if (probability_target) { + if (target_tensor.scalar_type() != at::kDouble) { + target_tensor = target_tensor.to(compute_type); + } + distribution = + target_tensor * (1.0 - label_smoothing) + label_smoothing / classes; + denominator = at::scalar_tensor(x.numel() / classes, x.options()); + } else { + auto indices = target_tensor.to(at::kLong); + valid = indices.ne(ignore_index); + auto safe_indices = indices.masked_fill(~valid, 0); + distribution = at::full_like(x, label_smoothing / classes); + auto selected = distribution.gather(dim, safe_indices.unsqueeze(dim)); + distribution.scatter_(dim, safe_indices.unsqueeze(dim), + selected + (1.0 - label_smoothing)); + const auto flat_indices = safe_indices.reshape({-1}); + const auto flat_weights = class_weight.index_select(0, flat_indices); + auto selected_weight = flat_weights.reshape(indices.sizes()); + denominator = selected_weight.masked_fill(~valid, 0).sum(); + } + auto weighted = distribution * class_weight.reshape(weight_shape); + auto gradient = at::softmax(x, dim) * weighted.sum(dim, true) - weighted; + if (reduction == "mean") gradient = gradient / denominator; + if (reduction == "none") upstream = upstream.unsqueeze(dim); + gradient = gradient * upstream; + // Ignored positions stay zero even when a mean has zero total weight. + if (!probability_target) gradient.masked_fill_(~valid.unsqueeze(dim), 0); + out.copy_(gradient); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(CrossEntropyBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h new file mode 100644 index 000000000..435b08062 --- /dev/null +++ b/src/torch/ops/cross_entropy_backward/cross_entropy_backward.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ + +#include "base/cross_entropy_backward.h" + +namespace infini::ops { + +template +class Operator : public CrossEntropyBackward { + public: + Operator(const Tensor input, const Tensor target, const Tensor grad_output, + std::optional weight, int64_t ignore_index, + std::string reduction, double label_smoothing, Tensor grad_input); + + void operator()(const Tensor input, const Tensor target, + const Tensor grad_output, std::optional weight, + int64_t ignore_index, std::string reduction, + double label_smoothing, Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.cc b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc new file mode 100644 index 000000000..cf549673a --- /dev/null +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.cc @@ -0,0 +1,42 @@ +#include "torch/ops/masked_fill_backward/masked_fill_backward.h" + +#include + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor mask, + Tensor grad_input) + : MaskedFillBackward{grad_output, mask, grad_input} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor mask, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_mask = ToAtenTensor(const_cast(mask.data()), mask_shape_, + mask_strides_, mask_type_, device_index_); + at_mask = at_mask.to(at::kBool); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + auto masked_grad = at::masked_fill(at_grad_output, at_mask, 0); + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + auto result = masked_grad.sum_to_size(grad_input_sizes); + + at_grad_input.copy_(result); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MaskedFillBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/masked_fill_backward/masked_fill_backward.h b/src/torch/ops/masked_fill_backward/masked_fill_backward.h new file mode 100644 index 000000000..d64f03f3c --- /dev/null +++ b/src/torch/ops/masked_fill_backward/masked_fill_backward.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_TORCH_MASKED_FILL_BACKWARD_H_ +#define INFINI_OPS_TORCH_MASKED_FILL_BACKWARD_H_ + +#include "base/masked_fill_backward.h" + +namespace infini::ops { + +template +class Operator : public MaskedFillBackward { + public: + Operator(const Tensor grad_output, const Tensor mask, Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor mask, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/mean_backward/mean_backward.cc b/src/torch/ops/mean_backward/mean_backward.cc new file mode 100644 index 000000000..8fef69645 --- /dev/null +++ b/src/torch/ops/mean_backward/mean_backward.cc @@ -0,0 +1,43 @@ +#include "torch/ops/mean_backward/mean_backward.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, const bool keepdim, + Tensor grad_input) + : MeanBackward{grad_output, dim, keepdim, grad_input} {} + +template +void Operator::operator()(const Tensor grad_output, + const int64_t dim, + const bool keepdim, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (at_grad_input.numel() == 0) { + return; + } + + auto at_g = at_grad_output; + if (!keepdim_) { + at_g = at_g.unsqueeze(dim_); + } + + const double count = static_cast(grad_input_shape_[dim_]); + at_grad_input.copy_(at_g.expand(at_grad_input.sizes()) / count); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MeanBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/mean_backward/mean_backward.h b/src/torch/ops/mean_backward/mean_backward.h new file mode 100644 index 000000000..11b80949b --- /dev/null +++ b/src/torch/ops/mean_backward/mean_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_MEAN_BACKWARD_H_ +#define INFINI_OPS_TORCH_MEAN_BACKWARD_H_ + +#include "base/mean_backward.h" + +namespace infini::ops { + +template +class Operator : public MeanBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, const bool keepdim, + Tensor grad_input); + + void operator()(const Tensor grad_output, const int64_t dim, + const bool keepdim, Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/mul_backward/mul_backward.cc b/src/torch/ops/mul_backward/mul_backward.cc new file mode 100644 index 000000000..a958610e0 --- /dev/null +++ b/src/torch/ops/mul_backward/mul_backward.cc @@ -0,0 +1,62 @@ +#include "torch/ops/mul_backward/mul_backward.h" + +#include +#include +#include +#include + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other) + : MulBackward{grad_output, input, other, grad_input, grad_other} {} + +template +void Operator::operator()(const Tensor grad_output, + const Tensor input, + const Tensor other, + Tensor grad_input, + Tensor grad_other) const { + const TorchStreamGuard guard{stream_, device_index_}; + + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_other = + ToAtenTensor(const_cast(other.data()), other_shape_, + other_strides_, other_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`MulBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = (at_grad * at_other).sum_to_size(grad_input_sizes); + auto grad_other_result = (at_grad * at_input).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(MulBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/mul_backward/mul_backward.h b/src/torch/ops/mul_backward/mul_backward.h new file mode 100644 index 000000000..1334cd99f --- /dev/null +++ b/src/torch/ops/mul_backward/mul_backward.h @@ -0,0 +1,21 @@ +#ifndef INFINI_OPS_TORCH_MUL_BACKWARD_H_ +#define INFINI_OPS_TORCH_MUL_BACKWARD_H_ + +#include "base/mul_backward.h" + +namespace infini::ops { + +template +class Operator : public MulBackward { + public: + Operator(const Tensor grad_output, const Tensor input, const Tensor other, + Tensor grad_input, Tensor grad_other); + + void operator()(const Tensor grad_output, const Tensor input, + const Tensor other, Tensor grad_input, + Tensor grad_other) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_MUL_BACKWARD_H_ diff --git a/src/torch/ops/pow_backward/pow_backward.cc b/src/torch/ops/pow_backward/pow_backward.cc new file mode 100644 index 000000000..a06f303fb --- /dev/null +++ b/src/torch/ops/pow_backward/pow_backward.cc @@ -0,0 +1,60 @@ +#include "torch/ops/pow_backward/pow_backward.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor input, + const double scalar, + const bool scalar_is_base, + Tensor grad_input) + : PowBackward{grad_output, input, scalar, scalar_is_base, grad_input} {} + +template +void Operator::operator()(const Tensor grad_output, + const Tensor input, + const double scalar, + const bool scalar_is_base, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (!scalar_is_base_ && scalar_ == 0.0) { + at_grad_input.zero_(); + return; + } + auto at_compute_input = at_input; + auto at_compute_grad_output = at_grad_output; + if (at_input.scalar_type() == at::kHalf || + at_input.scalar_type() == at::kBFloat16) { + at_compute_input = at_input.to(at::kFloat); + } + if (at_grad_output.scalar_type() == at::kHalf || + at_grad_output.scalar_type() == at::kBFloat16) { + at_compute_grad_output = at_grad_output.to(at::kFloat); + } + + if (scalar_is_base_) { + auto at_local_grad = at::pow(scalar_, at_compute_input) * std::log(scalar_); + at_grad_input.copy_(at_compute_grad_output * at_local_grad); + } else { + auto at_local_grad = scalar_ * at::pow(at_compute_input, scalar_ - 1.0); + at_grad_input.copy_(at_compute_grad_output * at_local_grad); + } +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(PowBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/pow_backward/pow_backward.h b/src/torch/ops/pow_backward/pow_backward.h new file mode 100644 index 000000000..977a3c847 --- /dev/null +++ b/src/torch/ops/pow_backward/pow_backward.h @@ -0,0 +1,21 @@ +#ifndef INFINI_OPS_TORCH_POW_BACKWARD_H_ +#define INFINI_OPS_TORCH_POW_BACKWARD_H_ + +#include "base/pow_backward.h" + +namespace infini::ops { + +template +class Operator : public PowBackward { + public: + Operator(const Tensor grad_output, const Tensor input, const double scalar, + const bool scalar_is_base, Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor input, + const double scalar, const bool scalar_is_base, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.cc b/src/torch/ops/repeat_interleave/repeat_interleave.cc new file mode 100644 index 000000000..14209eeec --- /dev/null +++ b/src/torch/ops/repeat_interleave/repeat_interleave.cc @@ -0,0 +1,32 @@ +#include "torch/ops/repeat_interleave/repeat_interleave.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, + const int64_t repeats, + const int64_t dim, Tensor out) + : RepeatInterleave{input, repeats, dim, out} {} + +template +void Operator::operator()(const Tensor input, + const int64_t repeats, + const int64_t dim, + Tensor out) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_out = ToAtenTensor(out.data(), out_shape_, out_strides_, + out_type_, device_index_); + + at_out.copy_(at::repeat_interleave(at_input, repeats_, dim_)); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RepeatInterleave, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave/repeat_interleave.h b/src/torch/ops/repeat_interleave/repeat_interleave.h new file mode 100644 index 000000000..b11812c6f --- /dev/null +++ b/src/torch/ops/repeat_interleave/repeat_interleave.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_REPEAT_INTERLEAVE_H_ +#define INFINI_OPS_TORCH_REPEAT_INTERLEAVE_H_ + +#include "base/repeat_interleave.h" + +namespace infini::ops { + +template +class Operator : public RepeatInterleave { + public: + Operator(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out); + + void operator()(const Tensor input, const int64_t repeats, const int64_t dim, + Tensor out) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc new file mode 100644 index 000000000..0752fa0e8 --- /dev/null +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.cc @@ -0,0 +1,44 @@ +#include "torch/ops/repeat_interleave_backward/repeat_interleave_backward.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, + Tensor grad_input) + : RepeatInterleaveBackward{grad_output, dim, grad_input} {} + +template +void Operator::operator()( + const Tensor grad_output, const int64_t dim, Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + if (at_grad_input.numel() == 0) return; + if (repeats_ == 0) { + at_grad_input.zero_(); + return; + } + + auto sizes = at_grad_output.sizes().vec(); + sizes[dim_] /= repeats_; + sizes.insert(sizes.begin() + dim_ + 1, repeats_); + + auto grouped = at_grad_output.reshape(sizes); + auto summed = grouped.sum(dim_ + 1); + + at_grad_input.copy_(summed); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RepeatInterleaveBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h new file mode 100644 index 000000000..267ec765e --- /dev/null +++ b/src/torch/ops/repeat_interleave_backward/repeat_interleave_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_REPEAT_INTERLEAVE_BACKWARD_H_ +#define INFINI_OPS_TORCH_REPEAT_INTERLEAVE_BACKWARD_H_ + +#include "base/repeat_interleave_backward.h" + +namespace infini::ops { + +template +class Operator + : public RepeatInterleaveBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, Tensor grad_input); + + void operator()(const Tensor grad_output, const int64_t dim, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.cc b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc new file mode 100644 index 000000000..0a4e07ca1 --- /dev/null +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.cc @@ -0,0 +1,41 @@ +#include "torch/ops/rsqrt_backward/rsqrt_backward.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const Tensor input, + Tensor grad_input) + : RsqrtBackward{grad_output, input, grad_input} {} + +template +void Operator::operator()(const Tensor grad_output, + const Tensor input, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + const auto compute_type = + at_input.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + const auto input_value = at_input.to(compute_type); + const auto upstream = at_grad_output.to(compute_type); + // Avoid cubing rsqrt before a small upstream gradient can scale it down. + const auto result = upstream * (-0.5) * at::rsqrt(input_value) / input_value; + at_grad_input.copy_(result); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(RsqrtBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/rsqrt_backward/rsqrt_backward.h b/src/torch/ops/rsqrt_backward/rsqrt_backward.h new file mode 100644 index 000000000..cae79d160 --- /dev/null +++ b/src/torch/ops/rsqrt_backward/rsqrt_backward.h @@ -0,0 +1,19 @@ +#ifndef INFINI_OPS_TORCH_RSQRT_BACKWARD_H_ +#define INFINI_OPS_TORCH_RSQRT_BACKWARD_H_ + +#include "base/rsqrt_backward.h" + +namespace infini::ops { + +template +class Operator : public RsqrtBackward { + public: + Operator(const Tensor grad_output, const Tensor input, Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor input, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/split/split.cc b/src/torch/ops/split/split.cc new file mode 100644 index 000000000..a8d350cc6 --- /dev/null +++ b/src/torch/ops/split/split.cc @@ -0,0 +1,46 @@ +#include "torch/ops/split/split.h" + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor input, const int64_t split_size, + const int64_t dim, std::vector outs) + : Split{input, split_size, dim, std::move(outs)} {} + +template +void Operator::operator()(const Tensor input, + const int64_t split_size, + const int64_t dim, + std::vector outs) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_input = + ToAtenTensor(const_cast(input.data()), input_shape_, + input_strides_, input_type_, device_index_); + + // Preserve every source slice before any output can overwrite it. + const auto parts = at::split(at_input.clone(), split_size_, dim_); + std::vector outputs; + for (std::size_t i = 0; i < outs.size(); ++i) { + outputs.push_back(ToAtenTensor(outs[i].data(), out_shapes_[i], + out_strides_[i], out_types_[i], + device_index_)); + for (std::size_t j = 0; j < i; ++j) { + assert(!StorageSpansOverlap(outputs[i], outputs[j]) && + "Output storage must not overlap."); + } + } + assert(parts.size() == outs_.size() && "`Split` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + outputs[i].copy_(parts[i]); + } +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Split, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/split/split.h b/src/torch/ops/split/split.h new file mode 100644 index 000000000..442664781 --- /dev/null +++ b/src/torch/ops/split/split.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_SPLIT_H_ +#define INFINI_OPS_TORCH_SPLIT_H_ + +#include "base/split.h" + +namespace infini::ops { + +template +class Operator : public Split { + public: + Operator(const Tensor input, const int64_t split_size, const int64_t dim, + std::vector outs); + + void operator()(const Tensor input, const int64_t split_size, + const int64_t dim, std::vector outs) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/stack_backward/stack_backward.cc b/src/torch/ops/stack_backward/stack_backward.cc new file mode 100644 index 000000000..8c66a5ea6 --- /dev/null +++ b/src/torch/ops/stack_backward/stack_backward.cc @@ -0,0 +1,47 @@ +#include "torch/ops/stack_backward/stack_backward.h" + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim, + std::vector grad_inputs) + : StackBackward{grad_output, dim, std::move(grad_inputs)} {} + +template +void Operator::operator()( + const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + + // Preserve every source slice before any output can overwrite it. + const auto parts = at::unbind(at_grad_output.clone(), dim_); + std::vector outputs; + for (std::size_t i = 0; i < grad_inputs.size(); ++i) { + outputs.push_back(ToAtenTensor( + grad_inputs[i].data(), grad_input_shapes_[i], grad_input_strides_[i], + grad_input_types_[i], device_index_)); + for (std::size_t j = 0; j < i; ++j) { + assert(!StorageSpansOverlap(outputs[i], outputs[j]) && + "Output storage must not overlap."); + } + } + assert(parts.size() == grad_inputs_.size() && + "`StackBackward` part count mismatch"); + + for (std::size_t i = 0; i < parts.size(); ++i) { + outputs[i].copy_(parts[i]); + } +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(StackBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/stack_backward/stack_backward.h b/src/torch/ops/stack_backward/stack_backward.h new file mode 100644 index 000000000..e51b2a393 --- /dev/null +++ b/src/torch/ops/stack_backward/stack_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_STACK_BACKWARD_H_ +#define INFINI_OPS_TORCH_STACK_BACKWARD_H_ + +#include "base/stack_backward.h" + +namespace infini::ops { + +template +class Operator : public StackBackward { + public: + Operator(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs); + + void operator()(const Tensor grad_output, const int64_t dim, + std::vector grad_inputs) const override; +}; + +} // namespace infini::ops + +#endif diff --git a/src/torch/ops/sub_backward/sub_backward.cc b/src/torch/ops/sub_backward/sub_backward.cc new file mode 100644 index 000000000..a539363de --- /dev/null +++ b/src/torch/ops/sub_backward/sub_backward.cc @@ -0,0 +1,54 @@ +#include "torch/ops/sub_backward/sub_backward.h" + +#include +#include +#include + +#include "torch/instantiate.h" +#include "torch/storage_overlap.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const double alpha, Tensor grad_input, + Tensor grad_other) + : SubBackward{grad_output, alpha, grad_input, grad_other} {} + +template +void Operator::operator()(const Tensor grad_output, + const double alpha, + Tensor grad_input, + Tensor grad_other) const { + const TorchStreamGuard guard{stream_, device_index_}; + + auto at_grad = ToAtenTensor(const_cast(grad_output.data()), + grad_output_shape_, grad_output_strides_, + grad_output_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + auto at_grad_other = + ToAtenTensor(grad_other.data(), grad_other_shape_, + grad_other_strides_, grad_other_type_, device_index_); + + assert(!StorageSpansOverlap(at_grad_input, at_grad_other) && + "`SubBackward` output storage must not overlap."); + + const std::vector grad_input_sizes(grad_input_shape_.begin(), + grad_input_shape_.end()); + const std::vector grad_other_sizes(grad_other_shape_.begin(), + grad_other_shape_.end()); + + auto grad_input_result = at_grad.sum_to_size(grad_input_sizes).clone(); + auto grad_other_result = (at_grad * (-alpha)).sum_to_size(grad_other_sizes); + + at_grad_input.copy_(grad_input_result); + at_grad_other.copy_(grad_other_result); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(SubBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/sub_backward/sub_backward.h b/src/torch/ops/sub_backward/sub_backward.h new file mode 100644 index 000000000..46fbe31d8 --- /dev/null +++ b/src/torch/ops/sub_backward/sub_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_SUB_BACKWARD_H_ +#define INFINI_OPS_TORCH_SUB_BACKWARD_H_ + +#include "base/sub_backward.h" + +namespace infini::ops { + +template +class Operator : public SubBackward { + public: + Operator(const Tensor grad_output, const double alpha, Tensor grad_input, + Tensor grad_other); + + void operator()(const Tensor grad_output, const double alpha, + Tensor grad_input, Tensor grad_other) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_SUB_BACKWARD_H_ diff --git a/src/torch/ops/transpose_backward/transpose_backward.cc b/src/torch/ops/transpose_backward/transpose_backward.cc new file mode 100644 index 000000000..03bc408e4 --- /dev/null +++ b/src/torch/ops/transpose_backward/transpose_backward.cc @@ -0,0 +1,35 @@ +#include "torch/ops/transpose_backward/transpose_backward.h" + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator(const Tensor grad_output, + const int64_t dim0, + const int64_t dim1, + Tensor grad_input) + : TransposeBackward{grad_output, dim0, dim1, grad_input} {} + +template +void Operator::operator()(const Tensor grad_output, + const int64_t dim0, + const int64_t dim1, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + at_grad_input.copy_(at_grad_output.transpose(dim0_, dim1_).clone()); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(TransposeBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/transpose_backward/transpose_backward.h b/src/torch/ops/transpose_backward/transpose_backward.h new file mode 100644 index 000000000..b17850c9f --- /dev/null +++ b/src/torch/ops/transpose_backward/transpose_backward.h @@ -0,0 +1,20 @@ +#ifndef INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ +#define INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ + +#include "base/transpose_backward.h" + +namespace infini::ops { + +template +class Operator : public TransposeBackward { + public: + Operator(const Tensor grad_output, const int64_t dim0, const int64_t dim1, + Tensor grad_input); + + void operator()(const Tensor grad_output, const int64_t dim0, + const int64_t dim1, Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_OPS_TRANSPOSE_BACKWARD_H_ diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc new file mode 100644 index 000000000..252867095 --- /dev/null +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.cc @@ -0,0 +1,101 @@ +#include "torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h" + +#include + +#include "torch/instantiate.h" +#include "torch/stream_guard.h" +#include "torch/tensor_.h" + +namespace infini::ops { + +template +Operator::Operator( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) + : VocabParallelCrossEntropyBackward{ + grad_output, softmax_local, target_mask, + masked_target, valid_mask_local, vocab_size_local, + vocab_size_original, label_smoothing, grad_input} {} + +template +void Operator::operator()( + const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input) const { + const TorchStreamGuard guard{stream_, device_index_}; + auto at_grad_output = ToAtenTensor( + const_cast(grad_output.data()), grad_output_shape_, + grad_output_strides_, grad_output_type_, device_index_); + auto at_softmax = ToAtenTensor(const_cast(softmax_local.data()), + softmax_shape_, softmax_strides_, + softmax_type_, device_index_); + auto at_target_mask = ToAtenTensor( + const_cast(target_mask.data()), target_mask_shape_, + target_mask_strides_, target_mask_type_, device_index_); + auto at_masked_target = ToAtenTensor( + const_cast(masked_target.data()), masked_target_shape_, + masked_target_strides_, masked_target_type_, device_index_); + auto at_valid_mask = ToAtenTensor( + const_cast(valid_mask_local.data()), valid_mask_shape_, + valid_mask_strides_, valid_mask_type_, device_index_); + auto at_grad_input = + ToAtenTensor(grad_input.data(), grad_input_shape_, + grad_input_strides_, grad_input_type_, device_index_); + + assert(vocab_size_local > 0 && + "VocabParallelCrossEntropyBackward requires positive local vocab"); + assert(vocab_size_original > 0 && + "VocabParallelCrossEntropyBackward requires positive global vocab"); + assert(at_softmax.numel() % vocab_size_local == 0 && + "softmax_local size is not divisible by vocab_size_local"); + assert(at_target_mask.numel() == at_softmax.numel() / vocab_size_local && + "target_mask size mismatch"); + assert(at_masked_target.numel() == at_target_mask.numel() && + "masked_target size mismatch"); + assert(at_valid_mask.numel() == vocab_size_local && + "valid_mask_local size mismatch"); + + const auto rows = at_softmax.numel() / vocab_size_local; + assert((at_grad_output.numel() == 1 || at_grad_output.numel() == rows) && + "grad_output must be scalar or one value per row"); + + const auto compute_type = + at_grad_input.scalar_type() == at::kDouble ? at::kDouble : at::kFloat; + auto softmax_2d = + at_softmax.reshape({rows, vocab_size_local}).to(compute_type); + auto target_mask_1d = at_target_mask.reshape({rows}).to(compute_type); + auto target_1d = at_masked_target.reshape({rows}).to(at::kLong); + auto valid_1d = at_valid_mask.reshape({vocab_size_local}).to(compute_type); + + auto dloss = at_grad_output.reshape({-1}).to(compute_type); + if (dloss.numel() == 1) { + dloss = dloss.expand({rows}); + } + + const auto valid_rows = 1.0 - target_mask_1d; + + auto target_one_hot = at::zeros_like(softmax_2d); + target_one_hot.scatter_(1, target_1d.reshape({rows, 1}), 1.0); + const auto local_target = target_one_hot * valid_rows.reshape({rows, 1}); + // Preserve small smoothing corrections before scaling by the upstream + // gradient. + auto gradient = softmax_2d - local_target; + gradient.add_(local_target * label_smoothing); + + const auto smoothing_term = + label_smoothing / static_cast(vocab_size_original); + gradient.sub_(smoothing_term * valid_1d.reshape({1, vocab_size_local})); + gradient.mul_(dloss.reshape({rows, 1})); + gradient.mul_(valid_1d.reshape({1, vocab_size_local})); + + at_grad_input.copy_(gradient.reshape(at_grad_input.sizes())); +} + +INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(VocabParallelCrossEntropyBackward, 1) + +} // namespace infini::ops diff --git a/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h new file mode 100644 index 000000000..b17e4748d --- /dev/null +++ b/src/torch/ops/vocab_parallel_cross_entropy_backward/vocab_parallel_cross_entropy_backward.h @@ -0,0 +1,28 @@ +#ifndef INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ +#define INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ + +#include "base/vocab_parallel_cross_entropy_backward.h" + +namespace infini::ops { + +template +class Operator + : public VocabParallelCrossEntropyBackward { + public: + Operator(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, const double label_smoothing, + Tensor grad_input); + + void operator()(const Tensor grad_output, const Tensor softmax_local, + const Tensor target_mask, const Tensor masked_target, + const Tensor valid_mask_local, const int64_t vocab_size_local, + const int64_t vocab_size_original, + const double label_smoothing, + Tensor grad_input) const override; +}; + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_VOCAB_PARALLEL_CROSS_ENTROPY_BACKWARD_H_ diff --git a/src/torch/storage_overlap.h b/src/torch/storage_overlap.h new file mode 100644 index 000000000..064ba5b1d --- /dev/null +++ b/src/torch/storage_overlap.h @@ -0,0 +1,23 @@ +#ifndef INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ +#define INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ + +#include + +#include + +namespace infini::ops { + +// External pointers have independent ATen Storage identities. Include stride +// gaps in the byte span, conservatively treating interleaved views as overlap. +inline bool StorageSpansOverlap(const at::Tensor& a, const at::Tensor& b) { + if (a.numel() == 0 || b.numel() == 0) return false; + const auto ap = reinterpret_cast(a.data_ptr()); + const auto bp = reinterpret_cast(b.data_ptr()); + const auto an = a.storage().nbytes(); + const auto bn = b.storage().nbytes(); + return an && bn && (ap <= bp ? bp - ap < an : ap - bp < bn); +} + +} // namespace infini::ops + +#endif // INFINI_OPS_TORCH_STORAGE_OVERLAP_H_ diff --git a/tests/test_accumulate_grad.py b/tests/test_accumulate_grad.py new file mode 100644 index 000000000..72138b1c9 --- /dev/null +++ b/tests/test_accumulate_grad.py @@ -0,0 +1,163 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +# Format: (shape, rate, gradient_strides, tensor_strides) +_TEST_CASES = ( + pytest.param((17,), 0.5, None, None, id="one-dimensional"), + pytest.param((2, 3), -0.25, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + 1.5, + (30, 7, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, rate, gradient_strides, tensor_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_accumulate_grad( + shape, + rate, + gradient_strides, + tensor_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + gradient = randn_strided( + shape, + gradient_strides, + dtype=dtype, + device=device, + ) + tensor = randn_strided( + shape, + tensor_strides, + dtype=dtype, + device=device, + ) + + return Payload( + lambda *args: _accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_accumulate_grad, + (gradient, rate, tensor), + {}, + rtol=rtol, + atol=atol, + ) + + +def _accumulate_grad(gradient, rate, tensor, *, implementation_index): + infini.ops.accumulate_grad( + gradient, + rate, + tensor, + stream=get_stream(gradient.device), + implementation_index=implementation_index, + ) + + return tensor + + +def _torch_accumulate_grad(gradient, rate, tensor): + tensor.add_(gradient * rate) + + return tensor + + +@pytest.mark.parametrize("rate", (0.0, 0.5, -1.0)) +def test_accumulate_grad_repeated(rate, device, implementation_index): + tensor = torch.ones(4, device=device) + gradient = torch.full_like(tensor, 2) + expected = tensor.clone() + + for _ in range(5): + expected.add_(gradient, alpha=rate) + _accumulate_grad( + gradient, rate, tensor, implementation_index=implementation_index + ) + torch.testing.assert_close(tensor, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize( + "dtype", (torch.float16, torch.bfloat16, torch.float32, torch.float64) +) +@pytest.mark.parametrize("rate", (0.0, 0.5)) +@pytest.mark.parametrize("stride", (1, 2)) +@pytest.mark.parametrize("offset", (-1, 0, 1, 80)) +def test_accumulate_grad_external_overlap( + dtype, rate, stride, offset, device, implementation_index +): + backing = torch.arange(256, dtype=dtype, device=device) + tensor = backing.as_strided((32,), (stride,), 8) + gradient = backing.as_strided((32,), (stride,), 8 + offset) + expected = tensor.clone().add_(gradient.clone(), alpha=rate) + expected_backing = backing.clone() + expected_backing.as_strided((32,), (stride,), 8).copy_(expected) + _accumulate_grad(gradient, rate, tensor, implementation_index=implementation_index) + torch.testing.assert_close(backing, expected_backing, rtol=0, atol=0) + + +def test_accumulate_grad_rejects_integer(device, implementation_index): + setup = f"x = torch.ones(4, dtype=torch.int32, device={str(device)!r})\n" + probe = setup + "infini.ops.AccumulateGrad(x, 0.5, x)\n" + source = ( + setup + + f"infini.ops.accumulate_grad(x, 0.5, x, implementation_index={implementation_index}, stream=get_stream(x.device))\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "requires floating-point tensors"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_adam_accumulate_grad.py b/tests/test_adam_accumulate_grad.py new file mode 100644 index 000000000..c8d33d194 --- /dev/null +++ b/tests/test_adam_accumulate_grad.py @@ -0,0 +1,310 @@ +import json +import os +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +# Format: (shape, learning_rate, beta1, beta2, eps, step, strides) +_TEST_CASES = ( + pytest.param( + (17,), + 1e-2, + 0.9, + 0.999, + 1e-8, + 1, + None, + id="one-dimensional-first-step", + ), + pytest.param( + (2, 3), + 5e-3, + 0.8, + 0.99, + 1e-6, + 3, + None, + id="two-dimensional-third-step", + ), + pytest.param( + (2, 3, 4), + 1e-2, + 0.9, + 0.999, + 1e-8, + 2, + (30, 7, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, learning_rate, beta1, beta2, eps, step, strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 2e-3, 2e-3), + (torch.bfloat16, 2e-2, 2e-2), + ), +) +def test_adam_accumulate_grad( + shape, + learning_rate, + beta1, + beta2, + eps, + step, + strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad = randn_strided(shape, strides, dtype=dtype, device=device) + param = randn_strided(shape, strides, dtype=dtype, device=device) + m = randn_strided(shape, strides, dtype=dtype, device=device) + v = randn_strided(shape, strides, dtype=dtype, device=device).abs_().add_(0.1) + + return Payload( + lambda *args: _adam_accumulate_grad( + *args, + implementation_index=implementation_index, + ), + _torch_adam_accumulate_grad, + (grad, learning_rate, beta1, beta2, eps, step, param, m, v), + {}, + rtol=rtol, + atol=atol, + ) + + +def _adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, + *, + implementation_index, +): + infini.ops.adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, + stream=get_stream(grad.device), + implementation_index=implementation_index, + ) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) + + +def _torch_adam_accumulate_grad( + grad, + learning_rate, + beta1, + beta2, + eps, + step, + param, + m, + v, +): + reference_param = torch.nn.Parameter( + param.detach().to(device="cpu", dtype=torch.float64) + ) + optimizer = torch.optim.Adam( + [reference_param], + lr=learning_rate, + betas=(beta1, beta2), + eps=eps, + foreach=False, + ) + optimizer.state[reference_param] = { + "step": torch.tensor(float(step - 1)), + "exp_avg": m.detach().to(device="cpu", dtype=torch.float64).clone(), + "exp_avg_sq": v.detach().to(device="cpu", dtype=torch.float64).clone(), + } + reference_param.grad = grad.detach().to(device="cpu", dtype=torch.float64).clone() + optimizer.step() + param.copy_(reference_param.detach()) + m.copy_(optimizer.state[reference_param]["exp_avg"]) + v.copy_(optimizer.state[reference_param]["exp_avg_sq"]) + + return torch.cat((param.reshape(-1), m.reshape(-1), v.reshape(-1))) + + +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +@pytest.mark.parametrize("gradient_scale", (0.0, 1e-7, 1.0)) +def test_adam_accumulate_grad_mixed_precision_multistep( + dtype, + gradient_scale, + device, + implementation_index, +): + param = torch.linspace(-1, 1, 12, device=device).reshape(3, 4).to(dtype) + m = torch.zeros((3, 4), dtype=torch.float32, device=device) + v = torch.zeros_like(m) + expected_param, expected_m, expected_v = param.clone(), m.clone(), v.clone() + + for step in range(1, 9): + grad = torch.full_like(param, gradient_scale * (-1 if step % 2 else 1)) + _torch_adam_accumulate_grad( + grad, 0.01, 0.9, 0.999, 1e-8, step, expected_param, expected_m, expected_v + ) + _adam_accumulate_grad( + grad, + 0.01, + 0.9, + 0.999, + 1e-8, + step, + param, + m, + v, + implementation_index=implementation_index, + ) + torch.testing.assert_close(param, expected_param, rtol=0, atol=0) + torch.testing.assert_close(m, expected_m, rtol=1e-5, atol=1e-12) + torch.testing.assert_close(v, expected_v, rtol=1e-5, atol=1e-20) + + +@pytest.mark.parametrize( + "dtype,gradient,moment,variance", + ( + (torch.float32, 1e20, 0.0, 0.0), + (torch.float32, 0.0, 1e19, 1e37), + (torch.float64, 1e155, 0.0, 0.0), + ), +) +def test_adam_accumulate_grad_finite_intermediates( + dtype, gradient, moment, variance, device, implementation_index +): + grad = torch.full((3,), gradient, dtype=dtype, device=device) + param = torch.zeros_like(grad) + m, v = torch.full_like(grad, moment), torch.full_like(grad, variance) + expected = [value.clone() for value in (param, m, v)] + _torch_adam_accumulate_grad(grad, 0.001, 0.9, 0.999, 1e-8, 1, *expected) + _adam_accumulate_grad( + grad, + 0.001, + 0.9, + 0.999, + 1e-8, + 1, + param, + m, + v, + implementation_index=implementation_index, + ) + + for actual, reference in zip((param, m, v), expected): + assert torch.isfinite(actual).all().item() + rtol = 1e-12 if dtype == torch.float64 else 1e-5 + torch.testing.assert_close(actual, reference, rtol=rtol, atol=0) + + +@pytest.mark.parametrize("dtype", (torch.float32, torch.float64)) +@pytest.mark.parametrize("stride", (1, 2)) +@pytest.mark.parametrize( + "alias,offset", + [("none", 0)] + + [(name, shift) for name in ("param", "m", "v") for shift in (-1, 0, 1)], +) +def test_adam_accumulate_grad_input_output_overlap( + dtype, stride, alias, offset, device, implementation_index +): + backing = torch.linspace(0.25, 2.0, 128, dtype=dtype, device=device) + positions = {"param": 8, "m": 40, "v": 72} + param, m, v = [ + backing.as_strided((6,), (stride,), pos) for pos in positions.values() + ] + grad = torch.full_like(param, 0.5) + + if alias != "none": + grad = backing.as_strided((6,), (stride,), positions[alias] + offset) + + expected = [value.clone() for value in (param, m, v)] + _torch_adam_accumulate_grad(grad.clone(), 0.001, 0.9, 0.999, 1e-8, 1, *expected) + expected_backing = backing.clone() + + for pos, result in zip(positions.values(), expected): + expected_backing.as_strided((6,), (stride,), pos).copy_(result) + + _adam_accumulate_grad( + grad, + 0.001, + 0.9, + 0.999, + 1e-8, + 1, + param, + m, + v, + implementation_index=implementation_index, + ) + torch.testing.assert_close(backing, expected_backing, rtol=1e-5, atol=1e-8) + + +@pytest.mark.skipif( + os.environ.get("INFINI_OPS_TEST_ASSERTIONS") != "1", + reason="set INFINI_OPS_TEST_ASSERTIONS=1 with an assertions-enabled build", +) +@pytest.mark.parametrize("pair", (("param", "m"), ("param", "v"), ("m", "v"))) +@pytest.mark.parametrize("layout", ("full", "partial", "strided", "interleaved")) +@pytest.mark.parametrize("reverse", (False, True)) +def test_adam_accumulate_grad_rejects_output_overlap( + pair, layout, reverse, device, implementation_index +): + stride = 2 if layout in ("strided", "interleaved") else 1 + offsets = (8, {"full": 8, "partial": 10, "strided": 10, "interleaved": 9}[layout]) + + if reverse: + offsets = offsets[::-1] + + source = f""" +import sys +sys.path[:] = {json.dumps(sys.path)} +import infini.ops +import torch +from tests.utils import get_stream +device = torch.device({str(device)!r}) +backing = torch.ones(128, device=device) +outputs = {{name: torch.ones(6, device=device) for name in ('param', 'm', 'v')}} +outputs[{pair[0]!r}] = backing.as_strided((6,), ({stride},), {offsets[0]}) +outputs[{pair[1]!r}] = backing.as_strided((6,), ({stride},), {offsets[1]}) +grad = torch.full((6,), 0.5, device=device) +infini.ops.adam_accumulate_grad(grad, 0.001, 0.9, 0.999, 1e-8, 1, + outputs['param'], outputs['m'], outputs['v'], + implementation_index={implementation_index}, stream=get_stream(device)) +""" + result = subprocess.run( + [sys.executable, "-c", source], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + assert result.returncode != 0, "Overlapping Adam outputs were accepted" + assert "output storage must not overlap" in result.stdout + result.stderr diff --git a/tests/test_add_backward.py b/tests/test_add_backward.py new file mode 100644 index 000000000..c2789a373 --- /dev/null +++ b/tests/test_add_backward.py @@ -0,0 +1,406 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param((17,), (17,), (17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), (2, 3), (2, 3), None, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), + pytest.param( + (2, 3), + (2, 3), + (2, 3), + (1, 2), + (1, 2), + (1, 2), + id="transposed-layout", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (4,), + None, + None, + None, + id="broadcast-b-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (4,), + (2, 3, 4), + None, + None, + None, + id="broadcast-a-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + id="broadcast-both-inputs", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + (30, 7, 1), + (11, 7, 2), + (15, 3, 1), + id="broadcast-strided-tensors", + ), + pytest.param( + (2, 3), + (), + (2, 3), + None, + None, + None, + id="broadcast-scalar-a", + ), + pytest.param( + (2, 3), + (2, 3), + (), + None, + None, + None, + id="broadcast-scalar-b", + ), + pytest.param((), (), (), None, None, None, id="scalar-output"), + pytest.param( + (1, 3, 1), + (3, 1), + (1, 3, 1), + None, + None, + None, + id="singleton-dimensions", + ), + pytest.param( + (0, 3), + (0, 3), + (0, 3), + None, + None, + None, + id="empty-output", + ), + pytest.param( + (0, 3), + (1, 3), + (0, 3), + None, + None, + None, + id="empty-output-nonempty-grad-a", + ), + pytest.param( + (0, 3), + (0, 3), + (1, 3), + None, + None, + None, + id="empty-output-nonempty-grad-b", + ), +) + +_ALPHAS = ( + pytest.param(1.0, id="alpha-one"), + pytest.param(0.0, id="alpha-zero"), + pytest.param(2.0, id="alpha-two"), + pytest.param(-1.0, id="alpha-negative"), + pytest.param(0.5, id="alpha-fractional"), +) + +_DTYPES = ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "grad_output_shape, grad_a_shape, grad_b_shape, " + "grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_add_backward( + grad_output_shape, + grad_a_shape, + grad_b_shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + alpha, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided( + grad_a_shape, + grad_a_strides, + dtype=dtype, + device=device, + ) + grad_b = empty_strided( + grad_b_shape, + grad_b_strides, + dtype=dtype, + device=device, + ) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _add_backward( + *args, + implementation_index=implementation_index, + ), + _torch_add_backward, + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pack_result(grad_output, grad_a, grad_b): + return torch.cat( + (grad_a.reshape(-1), grad_b.reshape(-1), grad_output.reshape(-1)), + ) + + +def _add_backward(grad_output, alpha, grad_a, grad_b, *, implementation_index): + infini.ops.add_backward( + grad_output, + alpha, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return _pack_result(grad_output, grad_a, grad_b) + + +def _torch_add_backward(grad_output, alpha, grad_a, grad_b): + with torch.enable_grad(): + a = torch.zeros_like(grad_a, requires_grad=True) + b = torch.zeros_like(grad_b, requires_grad=True) + out = torch.add(a, b, alpha=alpha) + expected_a, expected_b = torch.autograd.grad( + out, + (a, b), + grad_outputs=grad_output, + ) + + expected_a = expected_a.clone() + expected_b = expected_b.clone() + with torch.no_grad(): + grad_a.copy_(expected_a) + grad_b.copy_(expected_b) + return _pack_result(grad_output, grad_a, grad_b) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("target", ("grad_a", "grad_b")) +@pytest.mark.parametrize("layout", ("exact", "partial", "strided", "broadcast")) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_add_backward_alias( + target, + layout, + alpha, + implementation_index, + dtype, + device, +): + source = torch.arange(-16, 16, dtype=dtype, device=device) + + return Payload( + lambda values: _run_alias( + values, + alpha, + target, + layout, + implementation_index=implementation_index, + ), + lambda values: _run_alias(values, alpha, target, layout), + (source,), + {}, + rtol=0, + atol=0, + ) + + +def _run_alias(source, alpha, target, layout, *, implementation_index=None): + storage = source.clone() + grad_output = storage[:6].view(2, 3) + if layout == "exact": + shared_output = grad_output + elif layout == "partial": + shared_output = storage[1:7].view(2, 3) + elif layout == "strided": + shared_output = storage[:6].view(3, 2).transpose(0, 1) + elif layout == "broadcast": + shared_output = storage[:3].view(1, 3) + else: + raise ValueError(f"Unknown alias layout: {layout}") + + independent_output = torch.full_like(grad_output, 17) + if target == "grad_a": + grad_a, grad_b = shared_output, independent_output + else: + grad_a, grad_b = independent_output, shared_output + + if implementation_index is None: + _torch_add_backward(grad_output, alpha, grad_a, grad_b) + else: + _add_backward( + grad_output, + alpha, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1), storage)) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +@pytest.mark.parametrize( + "alpha, grad_value, grad_a_shape, grad_b_shape, expected_a, expected_b", + ( + pytest.param(2.0, 1.0, (2, 3), (2, 3), 1.0, 2.0, id="review-example"), + pytest.param(2.0, 1.0, (2, 3), (3,), 1.0, 4.0, id="scaled-reduction"), + pytest.param(0.0, 1.0, (2, 3), (3,), 1.0, 0.0, id="zero-alpha"), + pytest.param(-1.0, 1.0, (2, 3), (3,), 1.0, -2.0, id="negative-alpha"), + pytest.param(0.5, 1.0, (2, 3), (3,), 1.0, 1.0, id="fractional-alpha"), + pytest.param(2.0, 1.0, (1, 3), (2, 1), 2.0, 6.0, id="both-reduced"), + pytest.param( + 0.5, 40000.0, (2, 3), (2, 3), 40000.0, 20000.0, id="large-finite-gradient" + ), + pytest.param( + 0.0, 40000.0, (2, 3), (3,), 40000.0, 0.0, id="zero-alpha-before-reduction" + ), + ), +) +def test_add_backward_known_values( + alpha, + grad_value, + grad_a_shape, + grad_b_shape, + expected_a, + expected_b, + implementation_index, + dtype, + device, +): + grad_output = torch.full((2, 3), grad_value, dtype=dtype, device=device) + grad_a = torch.full(grad_a_shape, 17.0, dtype=dtype, device=device) + grad_b = torch.full(grad_b_shape, 17.0, dtype=dtype, device=device) + expected = _pack_result( + grad_output, + torch.full_like(grad_a, expected_a), + torch.full_like(grad_b, expected_b), + ) + + return Payload( + lambda *args: _add_backward( + *args, + implementation_index=implementation_index, + ), + lambda *args: expected.clone(), + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("offset", (0, 2)) +@pytest.mark.parametrize("strides", ((3, 1), (6, 2))) +def test_add_backward_rejects_output_overlap( + offset, strides, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = setup + "infini.ops.AddBackward(x, 2.0, torch.empty(4, device=device), x)\n" + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {strides!r}, 0) +b = backing.as_strided((2, 3), {strides!r}, {offset}) +infini.ops.add_backward(x, 2.0, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_cross_entropy.py b/tests/test_cross_entropy.py new file mode 100644 index 000000000..1fd5c6033 --- /dev/null +++ b/tests/test_cross_entropy.py @@ -0,0 +1,631 @@ +import math + +import infini.ops +import pytest +import torch +import torch.nn.functional as F + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_DTYPES = (torch.float32, torch.float64, torch.float16, torch.bfloat16) +_REDUCTIONS = ("none", "mean", "sum") +# Dimension 0 is the class dimension for (C,); otherwise it is dimension 1. +_SHAPES = ( + pytest.param((7,), False, id="unbatched"), + pytest.param((1, 7), False, id="batch-one"), + pytest.param((4, 7), False, id="two-dimensional"), + pytest.param((2, 3, 5), False, id="three-dimensional"), + pytest.param((2, 3, 2, 4), False, id="four-dimensional"), + pytest.param((3, 1), False, id="one-class"), + pytest.param((7,), True, id="unbatched-strided"), + pytest.param((2, 3, 5), True, id="all-tensors-strided"), +) + + +def _tolerance(dtype): + return { + torch.float64: (1e-10, 1e-10), + torch.float32: (1e-5, 1e-5), + torch.float16: (1e-3, 1e-3), + torch.bfloat16: (1e-2, 1e-2), + }[dtype] + + +def _strides(shape, strided): + if not strided or not shape: + return None + result = [] + step = 2 + for size in reversed(shape): + result.append(step) + step *= max(size, 1) + return tuple(reversed(result)) + + +def _class_dim(shape): + return 0 if len(shape) == 1 else 1 + + +def _loss_shape(shape): + dim = _class_dim(shape) + return shape[:dim] + shape[dim + 1 :] + + +def _promote_low_precision(tensor): + if tensor is not None and tensor.dtype in (torch.float16, torch.bfloat16): + return tensor.float() + return tensor + + +def _result_dtype(input, target, weight): + dtype = input.dtype + if target.is_floating_point(): + dtype = torch.promote_types(dtype, target.dtype) + if weight is not None: + dtype = torch.promote_types(dtype, weight.dtype) + return dtype + + +def _expected(input, target, weight, ignore_index, reduction, label_smoothing): + # FP16/BF16 reference uses FP32 intermediates, not same-dtype Torch CE. + target = ( + _promote_low_precision(target) if target.is_floating_point() else target.long() + ) + return F.cross_entropy( + _promote_low_precision(input), + target, + weight=_promote_low_precision(weight), + ignore_index=ignore_index, + reduction=reduction, + label_smoothing=label_smoothing, + ) + + +def _cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + *, + implementation_index, +): + # Requires the binding for the new canonical C++ argument order. + infini.ops.cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + return out + + +def _torch_cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, +): + out.copy_( + _expected( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + ) + ) + return out + + +def _data(shape, probability_target, weighted, strided, dtype, device): + input = randn_strided( + shape, + _strides(shape, strided), + dtype=dtype, + device=device, + ) + dim = _class_dim(shape) + classes = shape[dim] + if probability_target: + target = empty_strided( + shape, + _strides(shape, strided), + dtype=dtype, + device=device, + ) + target.copy_(torch.randn(shape, device=device).softmax(dim=dim)) + else: + target_shape = _loss_shape(shape) + target = empty_strided( + target_shape, + _strides(target_shape, strided), + dtype=torch.int64, + device=device, + ) + target.copy_(torch.randint(classes, target_shape, device=device)) + weight = None + if weighted: + weight = empty_strided( + (classes,), + _strides((classes,), strided), + dtype=dtype, + device=device, + ) + weight.copy_(torch.linspace(0.25, 2.0, classes, device=device)) + return input, target, weight + + +def _out(input, target, weight, reduction, strided=False): + shape = _loss_shape(tuple(input.shape)) if reduction == "none" else () + out = empty_strided( + shape, + _strides(shape, strided), + dtype=_result_dtype(input, target, weight), + device=input.device, + ) + out.fill_(17) + return out + + +def _check( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + implementation_index, + out=None, +): + if out is None: + out = _out(input, target, weight, reduction) + # Compute and detach the reference storage BEFORE the backend can write + # an aliased output. Merely assigning another Tensor variable is not a copy. + expected = ( + _expected( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + ) + .to(out.dtype) + .clone() + ) + _cross_entropy( + input, + target, + weight, + ignore_index, + reduction, + label_smoothing, + out, + implementation_index=implementation_index, + ) + rtol, atol = _tolerance(out.dtype) + torch.testing.assert_close(out, expected, rtol=rtol, atol=atol, equal_nan=True) + return out + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strided", _SHAPES) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2, 1.0)) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy( + shape, + strided, + probability_target, + weighted, + reduction, + label_smoothing, + implementation_index, + dtype, + device, +): + input, target, weight = _data( + shape, + probability_target, + weighted, + strided, + dtype, + device, + ) + out = _out(input, target, weight, reduction, strided) + rtol, atol = _tolerance(out.dtype) + return Payload( + lambda *args: _cross_entropy( + *args, + implementation_index=implementation_index, + ), + _torch_cross_entropy, + (input, target, weight, -100, reduction, label_smoothing, out), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize("pattern", ("partial", "one-valid", "all")) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2)) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy_ignored_targets( + pattern, + weighted, + reduction, + label_smoothing, + implementation_index, + dtype, + device, +): + input, target, weight = _data((2, 3, 5), False, weighted, True, dtype, device) + if pattern == "partial": + target[0, ::2] = -100 + else: + target.fill_(-100) + if pattern == "one-valid": + target[1, 3] = 1 + out = _check( + input, + target, + weight, + -100, + reduction, + label_smoothing, + implementation_index, + ) + if pattern == "all": + if reduction == "mean": + assert torch.isnan(out).item() + else: + assert torch.count_nonzero(out).item() == 0 + elif reduction == "none": + assert torch.count_nonzero(out[target == -100]).item() == 0 + + +@pytest.mark.parametrize("ignore_index", (-7, 1, 255)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_custom_ignore_index( + ignore_index, + reduction, + implementation_index, + device, +): + input = torch.zeros((3, 3), device=device) + target = torch.tensor([ignore_index, 0, 2], device=device) + out = _check( + input, target, None, ignore_index, reduction, 0.0, implementation_index + ) + expected = { + "none": [0.0, math.log(3), math.log(3)], + "mean": math.log(3), + "sum": 2 * math.log(3), + }[reduction] + torch.testing.assert_close(out, torch.tensor(expected, device=device)) + + +@pytest.mark.parametrize("target_dtype", (torch.int32, torch.uint8)) +def test_cross_entropy_integer_target_extensions( + target_dtype, + implementation_index, + device, +): + input = torch.zeros((3, 3), device=device) + target = torch.tensor([0, 1, 255], dtype=target_dtype, device=device) + _check(input, target, None, 255, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("all_zero", (False, True)) +def test_cross_entropy_zero_weights( + probability_target, + reduction, + all_zero, + implementation_index, + device, +): + input, target, _ = _data( + (3, 3), probability_target, False, False, torch.float32, device + ) + if not probability_target: + target.copy_(torch.tensor([0, 1, 2], device=device)) + weight = torch.tensor([0.0, 0.0 if all_zero else 2.0, 0.0], device=device) + out = _check(input, target, weight, -100, reduction, 0.2, implementation_index) + if all_zero: + if not probability_target and reduction == "mean": + assert torch.isnan(out).item() + else: + assert torch.count_nonzero(out).item() == 0 + + +def test_cross_entropy_probability_zero_row(implementation_index, device): + input = torch.zeros((2, 3), device=device) + target = torch.tensor([[1.0, 0.0, 0.0], [0.0, 0.0, 0.0]], device=device) + # A zero probability row does not remove that row from the mean divisor. + out = _check(input, target, None, -100, "mean", 0.0, implementation_index) + torch.testing.assert_close(out, torch.tensor(math.log(3) / 2, device=device)) + + +@pytest.mark.parametrize("probability_target", (False, True)) +def test_cross_entropy_weighted_mean_divisor( + probability_target, + implementation_index, + device, +): + input = torch.zeros((2, 2), device=device) + target = torch.tensor([0, 1], device=device) + weight = torch.tensor([1.0, 3.0], device=device) + if probability_target: + target = F.one_hot(target, num_classes=2).float() + out = _check(input, target, weight, -100, "mean", 0.0, implementation_index) + # Same one-hot labels, different upstream mean denominators: 4 vs 2. + expected = math.log(2) * (2 if probability_target else 1) + torch.testing.assert_close(out, torch.tensor(expected, device=device)) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("label_smoothing", (0.0, 0.2, 1.0)) +def test_cross_entropy_smoothing_analytic( + probability_target, + reduction, + label_smoothing, + implementation_index, + device, +): + # Each row assigns probability 3/4 to its correct class and 1/4 to the + # other class. Calculate the answer without calling Torch CE. + input = torch.tensor( + [[math.log(3), 0.0], [0.0, math.log(3)]], dtype=torch.float64, device=device + ) + target = torch.tensor([0, 1], device=device) + weight = torch.tensor([1.0, 3.0], dtype=torch.float64, device=device) + if probability_target: + target = F.one_hot(target, num_classes=2).double() + a = 1 - label_smoothing / 2 + b = label_smoothing / 2 + losses = [ + -a * math.log(0.75) - 3 * b * math.log(0.25), + -3 * a * math.log(0.75) - b * math.log(0.25), + ] + expected = torch.tensor(losses, dtype=torch.float64, device=device) + if reduction == "sum": + expected = expected.sum() + elif reduction == "mean": + expected = expected.sum() / (2 if probability_target else 4) + out = _check( + input, target, weight, -100, reduction, label_smoothing, implementation_index + ) + torch.testing.assert_close(out, expected, rtol=1e-10, atol=1e-10) + + +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_unbatched_ignored(reduction, implementation_index, device): + input = torch.zeros((3,), device=device) + target = torch.tensor(-100, device=device) + out = _check(input, target, None, -100, reduction, 0.2, implementation_index) + if reduction == "mean": + assert torch.isnan(out).item() + else: + assert out.item() == 0 + + +@pytest.mark.parametrize("shape", ((0, 3), (2, 3, 0))) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_empty( + shape, + probability_target, + reduction, + implementation_index, + device, +): + input, target, weight = _data( + shape, probability_target, True, False, torch.float32, device + ) + out = _check(input, target, weight, -100, reduction, 0.2, implementation_index) + if reduction == "mean": + assert torch.isnan(out).item() + elif reduction == "sum": + assert out.item() == 0 + else: + assert out.numel() == 0 + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_cross_entropy_extreme_logits( + probability_target, + reduction, + dtype, + implementation_index, + device, +): + input = torch.tensor( + [ + [1000.0, -1000.0, 0.0], + [-1000.0, 1000.0, 0.0], + [1000.0, 1000.0, 1000.0], + [-1000.0, -1000.0, -1000.0], + ], + dtype=dtype, + device=device, + ) + target = torch.tensor([1, 1, 0, 2], device=device) + if probability_target: + target = F.one_hot(target, num_classes=3).to(dtype) + out = _check(input, target, None, -100, reduction, 0.0, implementation_index) + losses = torch.tensor( + [2000.0, 0.0, math.log(3), math.log(3)], dtype=torch.float64, device=device + ) + expected = ( + losses + if reduction == "none" + else (losses.mean() if reduction == "mean" else losses.sum()) + ) + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).all().item() + torch.testing.assert_close(out, expected.to(dtype), rtol=rtol, atol=atol) + + +@pytest.mark.parametrize("count", (32768, 65504, 65505, 65520, 65536, 131072)) +@pytest.mark.parametrize("classes", (2, 4)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +def test_cross_entropy_long_sequence( + count, + classes, + weighted, + dtype, + implementation_index, + device, +): + # Shape (1,C,L) deliberately exercises the sequence path. With a constant + # class weight of 2, even count=32768 has total_weight > FP16's maximum. + input = torch.zeros((1, classes, count + 17), dtype=dtype, device=device) + target = torch.full((1, count + 17), -100, dtype=torch.int64, device=device) + target[:, :count] = 0 + weight = ( + torch.full((classes,), 2.0, dtype=dtype, device=device) if weighted else None + ) + out = _check(input, target, weight, -100, "mean", 0.0, implementation_index) + # Independent mathematical oracle: each prediction is uniform. + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).item() + torch.testing.assert_close( + out, + torch.tensor(math.log(classes), dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + + +def test_cross_entropy_sum_output_overflow(implementation_index, device): + # FP32 intermediates cannot make a genuinely unrepresentable FP16 output + # finite. This must remain +inf, unlike the finite long-sequence mean. + input = torch.zeros((1, 4, 65536), dtype=torch.float16, device=device) + target = torch.zeros((1, 65536), dtype=torch.int64, device=device) + out = _check(input, target, None, -100, "sum", 0.0, implementation_index) + assert torch.isposinf(out).item() + + +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +def test_cross_entropy_long_probability_target(dtype, implementation_index, device): + input = torch.zeros((1, 4, 65536), dtype=dtype, device=device) + target = torch.full_like(input, 0.25) + out = _check(input, target, None, -100, "mean", 0.2, implementation_index) + rtol, atol = _tolerance(dtype) + assert torch.isfinite(out).item() + torch.testing.assert_close( + out, + torch.tensor(math.log(4), dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize( + "input_dtype,target_dtype", + ( + (torch.float32, torch.float64), + (torch.float16, torch.float32), + (torch.float16, torch.bfloat16), + ), +) +def test_cross_entropy_probability_dtype_promotion( + input_dtype, + target_dtype, + implementation_index, + device, +): + input, target, weight = _data((2, 3), True, True, True, input_dtype, device) + target = target.to(target_dtype) + _check(input, target, weight, -100, "none", 0.2, implementation_index) + + +def test_cross_entropy_probability_weight_dtype_promotion(implementation_index, device): + input, target, weight = _data((2, 3), True, True, True, torch.float32, device) + _check(input, target, weight.double(), -100, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("reduction", _REDUCTIONS) +def test_cross_entropy_transposed_layout( + probability_target, + reduction, + implementation_index, + device, +): + input = torch.randn((3, 2, 5), device=device).transpose(0, 1) + if probability_target: + target = torch.randn((3, 2, 5), device=device).softmax(0).transpose(0, 1) + else: + target = (torch.arange(10, device=device).reshape(5, 2) % 3).t() + target[0, 0] = -100 + weight = torch.tensor([0.5, 17.0, 2.0, 17.0, 3.0], device=device)[::2] + out = ( + torch.full((5, 2), 17.0, device=device).t() + if reduction == "none" + else torch.full((), 17.0, device=device) + ) + _check(input, target, weight, -100, reduction, 0.2, implementation_index, out=out) + + +@pytest.mark.parametrize( + "alias", + ( + "input-full", + "input-partial", + "target-full", + "target-partial", + "weight", + ), +) +@pytest.mark.parametrize("reduction", ("none", "mean")) +def test_cross_entropy_aliasing(alias, reduction, implementation_index, device): + full = alias in ("input-full", "target-full") + shape = (1,) if full and reduction == "mean" else ((2, 1) if full else (2, 3)) + probability_target = alias.startswith("target") + input, target, weight = _data( + shape, probability_target, True, True, torch.float32, device + ) + if alias == "weight": + out = weight[:2] if reduction == "none" else weight[1] + else: + source = target if probability_target else input + if full: + out = source.squeeze(1) if reduction == "none" else source[0] + else: + out = source[:, 0] if reduction == "none" else source[0, 1] + _check(input, target, weight, -100, reduction, 0.2, implementation_index, out=out) + + +def test_cross_entropy_strided_output_guards(implementation_index, device): + input, target, weight = _data((2, 3), False, True, True, torch.float32, device) + backing = torch.full((7,), 17.0, device=device) + out = backing[1:5:2] + _check(input, target, weight, -100, "none", 0.2, implementation_index, out=out) + untouched = torch.tensor([0, 2, 4, 5, 6], device=device) + torch.testing.assert_close( + backing[untouched], torch.full((5,), 17.0, device=device) + ) diff --git a/tests/test_cross_entropy_backward.py b/tests/test_cross_entropy_backward.py new file mode 100644 index 000000000..5986f984f --- /dev/null +++ b/tests/test_cross_entropy_backward.py @@ -0,0 +1,323 @@ +import infini.ops +import pytest +import torch + +from tests.utils import ( + Payload, + empty_strided, + get_stream, + randint_strided, + randn_strided, +) + + +# Format: (input_shape, input_strides, target_strides, grad_input_strides) +# CrossEntropy uses the PyTorch convention: dimension 1 is the class dimension. +_TEST_CASES = ( + pytest.param( + (4, 7), + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 5), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 5), + (30, 7, 1), + None, + (35, 8, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, target_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("use_ignore_index", (False, True)) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_cross_entropy_backward( + input_shape, + input_strides, + target_strides, + grad_input_strides, + use_ignore_index, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + target = randint_strided( + 0, + input_shape[1], + (input_shape[0],) + input_shape[2:], + target_strides, + dtype=torch.int64, + device=device, + ) + if use_ignore_index: + target.reshape(-1)[0] = -100 + grad_output = torch.randn((), dtype=dtype, device=device) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _cross_entropy_backward( + *args, + implementation_index=implementation_index, + ), + _torch_cross_entropy_backward, + (input, target, grad_output, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _cross_entropy_backward( + input, + target, + grad_output, + grad_input, + *, + implementation_index, +): + infini.ops.cross_entropy_backward( + input, + target, + grad_output, + None, + -100, + "mean", + 0.0, + grad_input, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_cross_entropy_backward(input, target, grad_output, grad_input): + logits = input.detach().clone().requires_grad_(True) + loss = torch.nn.functional.cross_entropy( + logits, + target, + reduction="mean", + ignore_index=-100, + ) + gradient = torch.autograd.grad(loss, logits, grad_output)[0] + grad_input.copy_(gradient) + + return grad_input + + +# Forward API coverage is shared so backward stays aligned with every attribute. +@pytest.mark.parametrize( + "shape,strided", (((7,), False), ((4, 7), False), ((2, 3, 5), True)) +) +@pytest.mark.parametrize("probability_target", (False, True)) +@pytest.mark.parametrize("weighted", (False, True)) +@pytest.mark.parametrize("reduction", ("none", "sum", "mean")) +@pytest.mark.parametrize("smoothing", (0.0, 0.2, 1.0)) +@pytest.mark.parametrize( + "dtype", (torch.float64, torch.float32, torch.float16, torch.bfloat16) +) +def test_cross_entropy_backward_full_contract( + shape, + strided, + probability_target, + weighted, + reduction, + smoothing, + dtype, + device, + implementation_index, +): + from tests.test_cross_entropy import _data + + input, target, weight = _data( + shape, probability_target, weighted, strided, dtype, device + ) + _check_full_backward( + input, target, weight, -100, reduction, smoothing, implementation_index + ) + + +def _check_full_backward( + input, + target, + weight, + ignore_index, + reduction, + smoothing, + implementation_index, + out=None, +): + from tests.test_cross_entropy import _expected, _result_dtype, _tolerance + + with torch.enable_grad(): + logits = input.detach().clone().requires_grad_(True) + loss = _expected( + logits, target.detach().clone(), weight, ignore_index, reduction, smoothing + ) + upstream = torch.full_like( + loss, 0.75, dtype=_result_dtype(input, target, weight) + ) + expected = torch.autograd.grad(loss, logits, upstream)[0].detach().clone() + + if out is None: + out = torch.empty_like(input) + + infini.ops.cross_entropy_backward( + input, + target, + upstream, + weight, + ignore_index, + reduction, + smoothing, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + rtol, atol = _tolerance(input.dtype) + torch.testing.assert_close(out, expected, rtol=rtol, atol=atol, equal_nan=True) + return out + + +@pytest.mark.parametrize("count", (65504, 65536, 131072)) +@pytest.mark.parametrize("dtype", (torch.float16, torch.bfloat16)) +@pytest.mark.parametrize("mode", ("ordinary", "all-ignore", "one-valid", "extreme")) +def test_cross_entropy_backward_long_sequence( + count, dtype, mode, device, implementation_index +): + input = torch.zeros((1, 4, count), dtype=dtype, device=device) + target = torch.zeros((1, count), dtype=torch.int64, device=device) + + if mode == "all-ignore": + target.fill_(-100) + elif mode == "one-valid": + target.fill_(-100) + target[0, 0] = 0 + elif mode == "extreme": + input[:, 1, :] = 1000 + + out = _check_full_backward( + input, target, None, -100, "mean", 0.0, implementation_index + ) + + if mode == "ordinary": + expected = torch.full_like(input, 0.75 * 0.25 / count) + expected[:, 0, :] = -0.75 * 0.75 / count + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + if mode == "all-ignore": + assert torch.count_nonzero(out) == 0 + + +@pytest.mark.parametrize("smoothing", (0.0, 0.2)) +@pytest.mark.parametrize("reduction", ("none", "sum", "mean")) +def test_cross_entropy_backward_zero_weights( + smoothing, reduction, device, implementation_index +): + input = torch.randn((3, 4), device=device) + target = torch.tensor([-7, 1, 2], device=device) + weight = torch.zeros(4, device=device) + _check_full_backward( + input, target, weight, -7, reduction, smoothing, implementation_index + ) + + +@pytest.mark.parametrize("target_dtype", (torch.float16, torch.float32, torch.float64)) +@pytest.mark.parametrize("weight_dtype", (torch.float32, torch.float64)) +def test_cross_entropy_backward_probability_promotion( + target_dtype, + weight_dtype, + device, + implementation_index, +): + input = torch.randn((2, 3), device=device) + target = torch.randn((2, 3), device=device).softmax(1).to(target_dtype) + weight = torch.tensor([0.5, 1.0, 2.0], dtype=weight_dtype, device=device) + _check_full_backward(input, target, weight, -100, "mean", 0.2, implementation_index) + + +@pytest.mark.parametrize("ignore_index", (-7, 1, 255)) +@pytest.mark.parametrize("reduction", ("none", "mean", "sum")) +@pytest.mark.parametrize("smoothing", (0.0, 0.2)) +def test_cross_entropy_backward_custom_ignore_index( + ignore_index, + reduction, + smoothing, + device, + implementation_index, +): + input = torch.randn((2, 3), device=device) + target = torch.tensor([ignore_index, 2], device=device) + weight = torch.tensor([0.5, 1.0, 2.0], device=device) + _check_full_backward( + input, target, weight, ignore_index, reduction, smoothing, implementation_index + ) + + +@pytest.mark.parametrize("alias", ("input", "upstream", "weight")) +def test_cross_entropy_backward_alias(alias, device, implementation_index): + input = torch.randn((1, 3), device=device) + target = torch.tensor([1], device=device) + weight = torch.tensor([0.5, 1.0, 2.0], device=device) + # Input/output and weight/output may share memory; snapshot the oracle first. + out = input if alias == "input" else weight.view(1, 3) + + if alias == "upstream": + upstream = out.reshape(-1)[:1].view(()) + upstream.fill_(0.75) + with torch.enable_grad(): + logits = input.detach().clone().requires_grad_(True) + loss = torch.nn.functional.cross_entropy(logits, target, weight=weight) + expected = torch.autograd.grad(loss, logits, upstream.clone())[0] + infini.ops.cross_entropy_backward( + input, + target, + upstream, + weight, + -100, + "mean", + 0.0, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected) + else: + _check_full_backward( + input, target, weight, -100, "mean", 0.2, implementation_index, out + ) diff --git a/tests/test_embedding_dense_backward.py b/tests/test_embedding_dense_backward.py new file mode 100644 index 000000000..9a6b80c09 --- /dev/null +++ b/tests/test_embedding_dense_backward.py @@ -0,0 +1,590 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Dense-only interface: there is deliberately no sparse argument. +# (indices, num_weights, embedding_dim, indices_strides, +# grad_output_strides, grad_weight_strides, padding_idx, scale_grad_by_freq) +_TEST_CASES = ( + pytest.param( + (1, 2, 1, 4, 2, 1), 6, 4, None, None, None, -1, False, id="repeated-indices" + ), + pytest.param( + ((0, 1, 0), (2, 1, 3)), + 5, + 7, + (5, 1), + (32, 8, 1), + (9, 1), + 1, + False, + id="padding-and-strides", + ), + pytest.param( + (2, 2, 2, 3, 3, 4), + 7, + 5, + None, + None, + None, + -1, + True, + id="scale-grad-by-frequency", + ), + pytest.param( + (1, 2, 1, 1, 2, 4), + 6, + 5, + (2,), + (13, 2), + (12, 2), + 1, + True, + id="padding-with-frequency-and-strides", + ), + pytest.param((), 4, 3, None, None, None, -1, False, id="empty"), + pytest.param( + (), + 4, + 3, + (2,), + (7, 2), + (8, 2), + 0, + True, + id="empty-with-padding-frequency-and-strides", + ), + pytest.param( + ((), ()), 4, 3, None, None, None, -1, True, id="multidimensional-empty" + ), + pytest.param((1, 1, 1, 1), 4, 3, None, None, None, 1, False, id="all-padding"), + pytest.param( + ((2, 2), (2, 2)), + 4, + 3, + (5, 2), + (15, 6, 1), + (8, 2), + 2, + True, + id="all-padding-with-frequency", + ), + pytest.param( + (0, 0, 3, 0), 5, 3, None, None, None, 0, True, id="one-non-padding-occurrence" + ), + pytest.param(2, 4, 5, None, (2,), (12, 2), -1, True, id="scalar-index"), + pytest.param(2, 4, 5, None, None, None, 2, True, id="scalar-padding"), + pytest.param( + (((0, 2), (1, 0)), ((2, 1), (0, 2))), + 4, + 3, + None, + None, + None, + 1, + True, + id="three-dimensional-indices", + ), + pytest.param( + (0, 0, 0), 1, 1, None, None, None, -1, True, id="one-weight-one-feature" + ), + pytest.param( + (0, 5, 0, 5), + 6, + 33, + None, + None, + None, + -1, + False, + id="first-last-rows-and-wide-embedding", + ), + pytest.param((0, 3, 3), 4, 1, None, None, None, 3, True, id="last-row-padding"), + pytest.param( + (0, 3, 3), + 4, + 1, + None, + None, + None, + -1, + True, + id="minus-one-does-not-mean-last-row-padding", + ), + pytest.param((), 0, 3, None, None, None, -1, False, id="empty-vocabulary"), + pytest.param( + (), 0, 3, None, None, None, -1, True, id="empty-vocabulary-with-frequency" + ), + pytest.param( + (0, 1, 1), 4, 0, None, None, None, -1, False, id="zero-embedding-dimension" + ), + pytest.param( + (0, 1, 1), + 4, + 0, + None, + None, + None, + -1, + True, + id="zero-embedding-dimension-with-frequency", + ), +) + +_DTYPES = ( + pytest.param(torch.float32, 1e-5, 1e-5, id="float32"), + pytest.param(torch.float64, 1e-10, 1e-10, id="float64"), + pytest.param(torch.float16, 1e-2, 1e-2, id="float16"), + pytest.param(torch.bfloat16, 5e-2, 5e-2, id="bfloat16"), +) + + +def _embedding_dense_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, + implementation_index, +): + infini.ops.embedding_dense_backward( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + grad_weight, + implementation_index=implementation_index, + stream=get_stream(grad_output.device), + ) + return grad_weight + + +def _torch_embedding_dense_backward( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, +): + # Some Torch CUDA versions fail for empty frequency-scaling inputs or + # zero feature width. Both have an identically zero/empty dense result. + if indices.numel() == 0 or grad_weight.numel() == 0: + grad_weight.zero_() + return grad_weight + result = torch.ops.aten.embedding_dense_backward.default( + grad_output, + indices.long(), + num_weights, + padding_idx, + scale_grad_by_freq, + ) + assert result.layout == torch.strided + grad_weight.copy_(result) + return grad_weight + + +def _oracle(grad_output, indices, num_weights, padding_idx, scale_grad_by_freq): + # Independent mathematical reference: accumulate rows in CPU FP64. + # This does NOT call embedding_dense_backward, and copies the source storage + # before any possibly overlapping output can be written. + index = indices.detach().to(device="cpu", dtype=torch.int64).reshape(-1) + dim = grad_output.shape[-1] + values = grad_output.detach().to(device="cpu", dtype=torch.float64).clone() + values = values.reshape(index.numel(), dim) + mask = index != padding_idx + index = index[mask] + values = values[mask] + result = torch.zeros((num_weights, dim), dtype=torch.float64) + if index.numel(): + if scale_grad_by_freq: + counts = torch.bincount(index, minlength=num_weights) + values = values / counts[index, None] + result.index_add_(0, index, values) + return result.to(device=grad_output.device, dtype=grad_output.dtype) + + +def _check( + grad_output, + indices, + grad_weight, + *, + num_weights, + padding_idx, + scale_grad_by_freq, + implementation_index, + rtol, + atol, +): + expected = _oracle( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + ) + _embedding_dense_backward( + grad_output, + indices, + grad_weight, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + ) + assert grad_weight.layout == torch.strided + assert torch.isfinite(grad_weight).all().item() + torch.testing.assert_close(grad_weight, expected, rtol=rtol, atol=atol) + return grad_weight + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "indices_values,num_weights,embedding_dim,indices_strides," + "grad_output_strides,grad_weight_strides,padding_idx,scale_grad_by_freq", + _TEST_CASES, +) +@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward( + indices_values, + num_weights, + embedding_dim, + indices_strides, + grad_output_strides, + grad_weight_strides, + padding_idx, + scale_grad_by_freq, + indices_dtype, + implementation_index, + dtype, + device, + rtol, + atol, +): + values = torch.tensor(indices_values, dtype=indices_dtype) + indices = empty_strided( + values.shape, indices_strides, dtype=indices_dtype, device=device + ) + indices.copy_(values.to(device)) + grad_output = randn_strided( + (*indices.shape, embedding_dim), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_weight = empty_strided( + (num_weights, embedding_dim), + grad_weight_strides, + dtype=dtype, + device=device, + ) + grad_weight.fill_(float("nan")) + return Payload( + lambda *args: _embedding_dense_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + ), + lambda *args: _torch_embedding_dense_backward( + *args, + num_weights=num_weights, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + ), + (grad_output, indices, grad_weight), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize("padding_idx", (-1, 1, 3)) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("indices_dtype", (torch.int32, torch.int64)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_analytic( + padding_idx, + scale_grad_by_freq, + indices_dtype, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], dtype=indices_dtype, device=device) + grad_output = torch.tensor([[2, 4], [8, 12], [3, 9]], dtype=dtype, device=device) + grad_weight = torch.full((5, 2), float("nan"), dtype=dtype, device=device) + expected = torch.zeros_like(grad_weight) + if padding_idx != 1: + expected[1] = torch.tensor( + [5, 8] if scale_grad_by_freq else [10, 16], + dtype=dtype, + device=device, + ) + if padding_idx != 3: + expected[3] = torch.tensor([3, 9], dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=padding_idx, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + torch.testing.assert_close(grad_weight, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_padding_nan_and_unused_rows( + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 2, 4], device=device) + grad_output = torch.tensor( + [[float("nan"), float("nan")], [float("nan"), float("nan")], [2, -3], [5, 7]], + dtype=dtype, + device=device, + ) + grad_weight = torch.full((6, 2), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=6, + padding_idx=1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + # Padding is skipped entirely, including its NaN upstream gradient. + assert torch.count_nonzero(grad_weight[[0, 1, 3, 5]]).item() == 0 + + +@pytest.mark.parametrize("all_padding", (False, True)) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_empty_or_all_padding_zeroes_output( + all_padding, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.full( + (2, 3) if all_padding else (2, 0), 1, dtype=torch.int64, device=device + ) + grad_output = torch.full( + (*indices.shape, 3), float("nan"), dtype=dtype, device=device + ) + grad_weight = torch.full((4, 3), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=4, + padding_idx=1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + assert torch.count_nonzero(grad_weight).item() == 0 + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +def test_embedding_dense_backward_transposed_layout( + scale_grad_by_freq, + implementation_index, + device, +): + indices = torch.tensor([[0, 1], [1, 2], [1, 3]], device=device).t() + grad_output = torch.randn((3, 2, 5), device=device).transpose(0, 1) + grad_weight = torch.full((5, 4), float("nan"), device=device).t() + _check( + grad_output, + indices, + grad_weight, + num_weights=4, + padding_idx=2, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) + + +@pytest.mark.parametrize("pattern", ("zero", "cancel", "negative")) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_gradient_values( + pattern, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], device=device) + values = { + "zero": [[0, 0], [0, 0], [0, 0]], + "cancel": [[2, -4], [-2, 4], [0, 0]], + "negative": [[-2, -4], [-8, -12], [-3, -9]], + }[pattern] + grad_output = torch.tensor(values, dtype=dtype, device=device) + grad_weight = torch.full((5, 2), float("nan"), dtype=dtype, device=device) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + if pattern in ("zero", "cancel"): + assert torch.count_nonzero(grad_weight).item() == 0 + + +@pytest.mark.parametrize("alias", ("full", "partial", "strided")) +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +@pytest.mark.parametrize("dtype,rtol,atol", _DTYPES) +def test_embedding_dense_backward_grad_output_alias( + alias, + scale_grad_by_freq, + dtype, + rtol, + atol, + implementation_index, + device, +): + indices = torch.tensor([2, 0, 2], device=device) + if alias == "full": + grad_output = torch.arange(1, 7, dtype=dtype, device=device).reshape(3, 2) + grad_weight = grad_output + elif alias == "partial": + backing = torch.arange(1, 13, dtype=dtype, device=device) + grad_output = backing[:6].view(3, 2) + grad_weight = backing[3:9].view(3, 2) + else: + backing = torch.arange(1, 13, dtype=dtype, device=device).reshape(3, 4) + grad_output = backing[:, :2] + grad_weight = backing[:, 1:3] + _check( + grad_output, + indices, + grad_weight, + num_weights=3, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize("partial", (False, True)) +@pytest.mark.parametrize( + "indices_dtype,dtype", + ( + (torch.int32, torch.float32), + (torch.int64, torch.float64), + ), +) +def test_embedding_dense_backward_indices_alias( + partial, + indices_dtype, + dtype, + implementation_index, + device, +): + # Equal element sizes allow index and output views over the same bytes. + backing = torch.zeros((12,), dtype=dtype, device=device) + indices = backing[1:7].view(indices_dtype) + indices.copy_(torch.tensor([2, 0, 2, 1, 0, 2], dtype=indices_dtype, device=device)) + grad_weight = backing[4:10].view(3, 2) if partial else backing[1:7].view(3, 2) + grad_output = torch.arange(1, 13, dtype=dtype, device=device).view(6, 2) + _check( + grad_output, + indices, + grad_weight, + num_weights=3, + padding_idx=-1, + scale_grad_by_freq=True, + implementation_index=implementation_index, + rtol=1e-10 if dtype == torch.float64 else 1e-5, + atol=1e-10 if dtype == torch.float64 else 1e-5, + ) + + +def test_embedding_dense_backward_strided_output_guards(implementation_index, device): + indices = torch.tensor([1, 1, 3], device=device) + grad_output = torch.tensor( + [[2, 4], [8, 12], [3, 9]], + dtype=torch.float32, + device=device, + ) + backing = torch.full((5, 5), 17.0, device=device) + grad_weight = backing[:, 1::2] + expected_backing = backing.clone() + expected_backing[:, 1::2] = _oracle(grad_output, indices, 5, -1, True) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=True, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) + torch.testing.assert_close(backing, expected_backing) + + +@pytest.mark.parametrize("scale_grad_by_freq", (False, True)) +def test_embedding_dense_backward_overwrites_previous_result( + scale_grad_by_freq, + implementation_index, + device, +): + indices = torch.tensor([1, 1, 3], device=device) + grad_output = torch.ones((3, 2), device=device) + grad_weight = torch.full((5, 2), 17.0, device=device) + for values in ([1, 1, 3], [0, 2, 2]): + indices.copy_(torch.tensor(values, device=device)) + _check( + grad_output, + indices, + grad_weight, + num_weights=5, + padding_idx=-1, + scale_grad_by_freq=scale_grad_by_freq, + implementation_index=implementation_index, + rtol=1e-5, + atol=1e-5, + ) diff --git a/tests/test_generate_torch_ops.py b/tests/test_generate_torch_ops.py index 53a6cde59..30cc7df91 100644 --- a/tests/test_generate_torch_ops.py +++ b/tests/test_generate_torch_ops.py @@ -1,537 +1,563 @@ -import importlib.util -import pathlib -import sys -import types - -import pytest - - -def _load_generator_module(): - path = ( - pathlib.Path(__file__).resolve().parents[1] - / "scripts" - / "generate_torch_ops.py" - ) - spec = importlib.util.spec_from_file_location("generate_torch_ops_under_test", path) - module = importlib.util.module_from_spec(spec) - assert spec.loader is not None - sys.modules[spec.name] = module - spec.loader.exec_module(module) - - return module - - -def test_select_op_names_honors_slot_8_and_legacy_headers(): - module = _load_generator_module() - config = { - "default_all": {"headers": None, "implementations": None}, - "default_native": {"headers": None, "implementations": (0,)}, - "explicit_torch": {"headers": None, "implementations": (8,)}, - "legacy": {"headers": ["legacy.h"], "implementations": None}, - } - - assert module._select_op_names( - None, - ["default_all", "default_native"], - config, - ) == ["default_all", "explicit_torch"] - assert module._select_op_names( - ["explicit_torch"], - ["default_all", "default_native"], - config, - ) == ["explicit_torch"] - - -def test_select_op_names_maps_public_names_to_aten_names(): - module = _load_generator_module() - config = { - "div": {"headers": None, "implementations": (8,)}, - "internal_log_softmax": {"headers": None, "implementations": (8,)}, - } - - assert module._select_op_names( - None, - ["div", "div_", "_log_softmax"], - config, - ) == ["div", "div_", "_log_softmax"] - - -def test_load_aten_entries_uses_packaged_torchgen(monkeypatch): - module = _load_generator_module() - entries = [{"func": "relu.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)"}] - monkeypatch.setattr(module, "_load_packaged_aten_entries", lambda: (entries, True)) - - assert module._load_aten_entries("v9.9.9") == entries - - -def test_packaged_aten_entries_include_autogenerated_out_variants(): - module = _load_generator_module() - entries, expanded_autogen = module._load_packaged_aten_entries() - - if not expanded_autogen: - pytest.skip("installed torchgen does not support autogen expansion") - - relu_out_entry = next( - entry - for entry in module._find_out_entries(entries, "relu") - if entry["func"].startswith("relu.out(") - ) - - assert module._parse_func(relu_out_entry["func"]).out_params - - -def test_packaged_aten_entries_fall_back_when_tags_are_missing( - monkeypatch, tmp_path, capsys -): - module = _load_generator_module() - native_dir = tmp_path / "packaged" / "ATen" / "native" - native_dir.mkdir(parents=True) - native_path = native_dir / "native_functions.yaml" - native_path.write_text( - "- func: add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)\n" - ) - spec = types.SimpleNamespace(submodule_search_locations=[str(tmp_path)]) - monkeypatch.setattr(module.importlib.util, "find_spec", lambda name: spec) - - entries, expanded_autogen = module._load_packaged_aten_entries() - - assert expanded_autogen is False - assert entries == [ - {"func": "add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)"} - ] - assert "using explicit schemas only" in capsys.readouterr().err - - -def test_public_op_name_normalizes_aten_internal_and_inplace_names(): - module = _load_generator_module() - - assert module._public_op_name("_softmax") == "internal_softmax" - assert module._public_op_name("add_") == "add" - assert module._public_op_name("_add_relu_") == "internal_add_relu" - - -def test_schema_self_param_renders_as_input_in_public_cpp_api(): - module = _load_generator_module() - op = module._parse_func( - "_softmax(Tensor self, int dim, bool half_to_float, *, " - "Tensor(a!) out) -> Tensor(a!)" - ) - - assert op.params[0].name == "self" - assert op.params[0].api_name == "input" - - base = module._generate_base_header("internal_softmax", [op]) - source = module._generate_torch_method_source("internal_softmax", op) - - assert "Softmax(const Tensor input, const int64_t dim" in base - assert "self_shape_" not in base - assert "input_shape_" in base - assert "auto at_self = ToAtenTensor" in source - assert "input_shape_" in source - assert "at::_softmax_out(at_out, at_self" in source - - -def test_torch_source_uses_existing_c10_stream_guards(): - module = _load_generator_module() - op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") - - method = module._generate_torch_method_source("abs", op) - source = module._generate_torch_source("abs", [op]) - - assert "stream_ == nullptr" not in method - assert "TorchStreamGuard stream_guard{stream_, device_index}" in method - - assert '#include "torch/stream_guard.h"' in source - assert "INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Abs, 8)" in source - assert '#include "torch/nvidia/c10.h"' not in source - - -def test_optional_tensor_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "batch_norm_elemt(Tensor input, Tensor? weight=None, " - "Tensor? bias=None, Tensor mean, Tensor invstd, float eps, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "std::optional", - "std::optional", - "Tensor", - "Tensor", - "double", - "Tensor", - ] - - base = module._generate_base_header("batch_norm_elemt", [op]) - source = module._generate_torch_method_source("batch_norm_elemt", op) - - assert "#include " in base - assert "std::optional weight" in base - assert "std::optional bias" in base - assert "bool has_weight_" in base - assert "bool has_bias_" in base - assert ( - "weight_shape_{weight ? Tensor::Shape{weight->shape()} : Tensor::Shape{}}" - in base - ) - assert ( - "weight_strides_{weight ? Tensor::Strides{weight->strides()} " - ": Tensor::Strides{}}" in base - ) - assert "c10::optional at_weight" in source - assert "c10::optional at_bias" in source - assert "weight->shape()" in source - assert "weight_shape_" not in source - assert "at::batch_norm_elemt_out" in source - assert "at_weight" in source - assert "at_bias" in source - - -def test_tensor_list_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "stack(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "std::vector", - "int64_t", - "Tensor", - ] - assert op.is_testable - - base = module._generate_base_header("stack", [op]) - source = module._generate_torch_method_source("stack", op) - - assert "#include " in base - assert "std::vector tensors" in base - assert "std::vector at_tensors" in source - assert "at_tensors.reserve(tensors.size())" in source - assert "for (const auto& tensor : tensors)" in source - assert "at::stack_out(at_out, at_tensors, dim)" in source - - -def test_optional_tensor_list_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - op = module._parse_func( - "index.Tensor_out(Tensor self, Tensor?[] indices, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "std::vector>", - "Tensor", - ] - - source = module._generate_torch_method_source("index", op) - - assert "c10::List> at_indices" in source - assert "for (const auto& tensor : indices)" in source - assert "at_indices.push_back(c10::nullopt)" in source - assert "at::index_out(at_out, at_self, at_indices)" in source - - -def test_optional_scalar_and_array_params_are_exposed_and_forwarded_to_aten(): - module = _load_generator_module() - quantile = module._parse_func( - "quantile(Tensor input, Tensor q, int? dim=None, bool keepdim=False, " - "str interpolation='linear', *, Tensor(a!) out) -> Tensor(a!)" - ) - upsample = module._parse_func( - "upsample_bicubic2d(Tensor input, SymInt[2] output_size, " - "bool align_corners, float[]? scale_factors=None, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in quantile.visible_params] == [ - "Tensor", - "Tensor", - "std::optional", - "bool", - "std::string", - "Tensor", - ] - assert [param.cpp_type for param in upsample.visible_params] == [ - "Tensor", - "std::vector", - "bool", - "std::optional>", - "Tensor", - ] - - quantile_source = module._generate_torch_method_source("quantile", quantile) - upsample_source = module._generate_torch_method_source( - "upsample_bicubic2d", upsample - ) - - assert "c10::optional at_dim" in quantile_source - assert "at::quantile_out" in quantile_source - assert "at_dim" in quantile_source - assert "c10::optional> at_scale_factors" in upsample_source - assert "at::upsample_bicubic2d_out" in upsample_source - assert "at_scale_factors" in upsample_source - - -def test_required_scalar_type_params_use_public_data_type(): - module = _load_generator_module() - op = module._parse_func( - "_softmax_backward_data(Tensor grad_output, Tensor output, int dim, " - "ScalarType input_dtype, *, Tensor(a!) grad_input) -> Tensor(a!)" - ) - - assert [param.cpp_type for param in op.visible_params] == [ - "Tensor", - "Tensor", - "int64_t", - "DataType", - "Tensor", - ] - - source = module._generate_torch_method_source("internal_softmax_backward_data", op) - - assert "at::_softmax_backward_data_out" in source - assert "ToAtenDataType(input_dtype)" in source - - -def test_existing_base_overload_can_omit_optional_schema_params(): - module = _load_generator_module() - op = module._parse_func( - "slow_conv3d(Tensor input, Tensor weight, int[3] kernel_size, " - "Tensor? bias=None, int[3] stride=1, int[3] padding=0, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("Tensor", "weight"), - ("std::vector", "kernel_size"), - ("std::vector", "stride"), - ("std::vector", "padding"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.name for param in bound.visible_params] == [ - "input", - "weight", - "kernel_size", - "stride", - "padding", - "out", - ] - - source = module._generate_torch_method_source("slow_conv3d", bound) - - assert "std::optional bias" not in source - assert "c10::optional{}" in source - assert "at::slow_conv3d_out" in source - - -def test_existing_base_overload_can_omit_defaulted_schema_params(): - module = _load_generator_module() - op = module._parse_func( - "add.Tensor(Tensor self, Tensor other, *, Scalar alpha=1, " - "Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("const Tensor", "input"), - ("const Tensor", "other"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - - source = module._generate_torch_method_source("add", bound) - - assert "double alpha" not in source - assert "const auto device_index = out.device().index();" in source - assert "device_index_)" not in source - assert "at::add_out(at_out, at_self, at_other, 1)" in source - - -def test_existing_base_overload_matches_by_name_when_types_repeat(): - module = _load_generator_module() - op = module._parse_func( - "std(Tensor input, int[1]? dim=None, bool unbiased=True, " - "bool keepdim=False, *, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("bool", "keepdim"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.name for param in bound.visible_params] == [ - "input", - "keepdim", - "out", - ] - - source = module._generate_torch_method_source("std", bound) - - assert "c10::optional{}, true, keepdim" in source - assert "unbiased" not in source - - -def test_existing_base_overload_can_group_tensor_inputs_before_attributes(): - module = _load_generator_module() - op = module._parse_func( - "gather.out(Tensor self, int dim, Tensor index, bool sparse_grad=False, " - "*, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("Tensor", "index"), - ("int64_t", "dim"), - ("bool", "sparse_grad"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "index", - "dim", - "sparse_grad", - "out", - ] - - assert [ - next( - index - for index, binding in enumerate(bound.param_bindings) - if binding is param - ) - for param in bound.visible_params - ] == [0, 2, 1, 3, 4] - - source = module._generate_torch_method_source("gather", bound) - - assert "at::gather_out(at_out, at_self, dim, at_index, sparse_grad)" in source - - -def test_rrelu_with_noise_base_order_preserves_aten_call_order(): - module = _load_generator_module() - op = module._parse_func( - "rrelu_with_noise.out(Tensor self, Tensor(b!) noise, Scalar lower=0.125, " - "Scalar upper=0.3333333333333333, bool training=False, " - "Generator? generator=None, *, Tensor(a!) out) -> Tensor(a!)" - ) - signature = [ - ("Tensor", "input"), - ("double", "lower"), - ("double", "upper"), - ("bool", "training"), - ("Tensor", "out"), - ("Tensor", "noise"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "lower", - "upper", - "training", - "out", - "noise", - ] - assert [ - next( - index - for index, binding in enumerate(bound.param_bindings) - if binding is param - ) - for param in bound.visible_params - ] == [0, 2, 3, 4, 6, 1] - - source = module._generate_torch_method_source("rrelu_with_noise", bound) - - assert "Tensor out, Tensor noise" in source - assert ( - "at::rrelu_with_noise_out(at_out, at_self, at_noise, lower, upper, " - "training, c10::optional{})" in source - ) - - -def test_write_text_if_changed_preserves_unchanged_mtime(tmp_path): - module = _load_generator_module() - path = tmp_path / "generated.cc" - path.write_text("same\n") - before = path.stat().st_mtime_ns - - assert module._write_text_if_changed(path, "same\n") is False - assert path.stat().st_mtime_ns == before - - assert module._write_text_if_changed(path, "different\n") is True - assert path.read_text() == "different\n" - assert path.stat().st_mtime_ns >= before - - -def test_remove_stale_files_keeps_expected_outputs(tmp_path): - module = _load_generator_module() - root = tmp_path / "generated" - keep = root / "torch" / "keep.cc" - stale = root / "torch" / "stale.cc" - nested_stale = root / "base" / "stale.h" - keep.parent.mkdir(parents=True) - nested_stale.parent.mkdir(parents=True) - keep.write_text("keep\n") - stale.write_text("stale\n") - nested_stale.write_text("stale\n") - - module._remove_stale_files(root, {keep}) - - assert keep.exists() - assert not stale.exists() - assert not nested_stale.exists() - assert not (root / "base").exists() - - -def test_inplace_schema_can_bind_to_explicit_output_signature(): - module = _load_generator_module() - op = module._parse_func("fill_.Scalar(Tensor(a!) self, Scalar value) -> Tensor(a!)") - signature = [ - ("Tensor", "input"), - ("double", "value"), - ("Tensor", "out"), - ] - - bound = module._bind_base_signature(op, signature) - - assert bound is not None - assert bound.explicit_inplace_out is not None - assert [param.api_name for param in bound.visible_params] == [ - "input", - "value", - "out", - ] - assert bound.out_params[0].api_name == "out" - - source = module._generate_torch_method_source("fill", bound) - - assert "const auto device_index = out.device().index();" in source - assert "const_cast(input.data())" in source - assert "auto at_out = ToAtenTensor" in source - assert "at_out.copy_(at_self);" in source - assert "at_out.fill_(value);" in source - - implicit_bound = module._bind_base_signature(op, signature[:-1]) - - assert implicit_bound is not None - assert implicit_bound.explicit_inplace_out is None - - implicit_source = module._generate_torch_method_source("fill", implicit_bound) - assert "at_self.fill_(value);" in implicit_source +import importlib.util +import pathlib +import sys +import types + +import pytest + + +def _load_generator_module(): + path = ( + pathlib.Path(__file__).resolve().parents[1] + / "scripts" + / "generate_torch_ops.py" + ) + spec = importlib.util.spec_from_file_location("generate_torch_ops_under_test", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + + return module + + +def test_select_op_names_honors_slot_8_and_legacy_headers(): + module = _load_generator_module() + config = { + "default_all": {"headers": None, "implementations": None}, + "default_native": {"headers": None, "implementations": (0,)}, + "explicit_torch": {"headers": None, "implementations": (8,)}, + "legacy": {"headers": ["legacy.h"], "implementations": None}, + } + + assert module._select_op_names( + None, + ["default_all", "default_native"], + config, + ) == ["default_all", "explicit_torch"] + assert module._select_op_names( + ["explicit_torch"], + ["default_all", "default_native"], + config, + ) == ["explicit_torch"] + + +def test_select_op_names_maps_public_names_to_aten_names(): + module = _load_generator_module() + config = { + "div": {"headers": None, "implementations": (8,)}, + "internal_log_softmax": {"headers": None, "implementations": (8,)}, + } + + assert module._select_op_names( + None, + ["div", "div_", "_log_softmax"], + config, + ) == ["div", "div_", "_log_softmax"] + + +def test_load_aten_entries_uses_packaged_torchgen(monkeypatch): + module = _load_generator_module() + entries = [{"func": "relu.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)"}] + monkeypatch.setattr(module, "_load_packaged_aten_entries", lambda: (entries, True)) + + assert module._load_aten_entries("v9.9.9") == entries + + +def test_packaged_aten_entries_include_autogenerated_out_variants(): + module = _load_generator_module() + entries, expanded_autogen = module._load_packaged_aten_entries() + + if not expanded_autogen: + pytest.skip("installed torchgen does not support autogen expansion") + + relu_out_entry = next( + entry + for entry in module._find_out_entries(entries, "relu") + if entry["func"].startswith("relu.out(") + ) + + assert module._parse_func(relu_out_entry["func"]).out_params + + +def test_packaged_aten_entries_fall_back_when_tags_are_missing( + monkeypatch, tmp_path, capsys +): + module = _load_generator_module() + native_dir = tmp_path / "packaged" / "ATen" / "native" + native_dir.mkdir(parents=True) + native_path = native_dir / "native_functions.yaml" + native_path.write_text( + "- func: add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)\n" + ) + spec = types.SimpleNamespace(submodule_search_locations=[str(tmp_path)]) + monkeypatch.setattr(module.importlib.util, "find_spec", lambda name: spec) + + entries, expanded_autogen = module._load_packaged_aten_entries() + + assert expanded_autogen is False + assert entries == [ + {"func": "add.out(Tensor self, Tensor other, *, Tensor(a!) out) -> Tensor(a!)"} + ] + assert "using explicit schemas only" in capsys.readouterr().err + + +def test_public_op_name_normalizes_aten_internal_and_inplace_names(): + module = _load_generator_module() + + assert module._public_op_name("_softmax") == "internal_softmax" + assert module._public_op_name("add_") == "add" + assert module._public_op_name("_add_relu_") == "internal_add_relu" + + +def test_schema_self_param_renders_as_input_in_public_cpp_api(): + module = _load_generator_module() + op = module._parse_func( + "_softmax(Tensor self, int dim, bool half_to_float, *, " + "Tensor(a!) out) -> Tensor(a!)" + ) + + assert op.params[0].name == "self" + assert op.params[0].api_name == "input" + + base = module._generate_base_header("internal_softmax", [op]) + source = module._generate_torch_method_source("internal_softmax", op) + + assert "Softmax(const Tensor input, const int64_t dim" in base + assert "self_shape_" not in base + assert "input_shape_" in base + assert "auto at_self = ToAtenTensor" in source + assert "input_shape_" in source + assert "at::_softmax_out(at_out, at_self" in source + + +def test_torch_source_uses_existing_c10_stream_guards(): + module = _load_generator_module() + op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") + + method = module._generate_torch_method_source("abs", op) + source = module._generate_torch_source("abs", [op]) + + assert "stream_ == nullptr" not in method + assert "TorchStreamGuard stream_guard{stream_, device_index}" in method + + assert '#include "torch/stream_guard.h"' in source + assert "INFINI_OPS_INSTANTIATE_TORCH_OPERATOR(Abs, 8)" in source + assert '#include "torch/nvidia/c10.h"' not in source + + +def test_embedding_dense_backward_handles_empty_inputs_before_aten(): + module = _load_generator_module() + op = module._parse_func( + "embedding_dense_backward.out(Tensor grad_output, Tensor indices, " + "SymInt num_weights, SymInt padding_idx, bool scale_grad_by_freq, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + source = module._generate_torch_source("embedding_dense_backward", [op]) + guard = source.index("if (at_indices.numel() == 0 || embedding_dim == 0)") + zero = source.index("at_out.zero_();", guard) + early_return = source.index("return;", zero) + call = source.index("at::embedding_dense_backward_out(") + assert guard < zero < early_return < call + assert source.index("at_out.size(1) == embedding_dim") < guard + + +def test_copy_guards_do_not_change_unrelated_generated_ops(): + module = _load_generator_module() + op = module._parse_func("abs.out(Tensor self, *, Tensor(a!) out) -> Tensor(a!)") + source = module._generate_torch_source("abs", [op]) + assert "at::abs_out(at_out, at_self)" in source + assert "StorageSpansOverlap" not in source + assert "embedding_dim" not in source + assert '#include "torch/storage_overlap.h"' not in source + + +def test_optional_tensor_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "batch_norm_elemt(Tensor input, Tensor? weight=None, " + "Tensor? bias=None, Tensor mean, Tensor invstd, float eps, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "std::optional", + "std::optional", + "Tensor", + "Tensor", + "double", + "Tensor", + ] + + base = module._generate_base_header("batch_norm_elemt", [op]) + source = module._generate_torch_method_source("batch_norm_elemt", op) + + assert "#include " in base + assert "std::optional weight" in base + assert "std::optional bias" in base + assert "bool has_weight_" in base + assert "bool has_bias_" in base + assert ( + "weight_shape_{weight ? Tensor::Shape{weight->shape()} : Tensor::Shape{}}" + in base + ) + assert ( + "weight_strides_{weight ? Tensor::Strides{weight->strides()} " + ": Tensor::Strides{}}" in base + ) + assert "c10::optional at_weight" in source + assert "c10::optional at_bias" in source + assert "weight->shape()" in source + assert "weight_shape_" not in source + assert "at::batch_norm_elemt_out" in source + assert "at_weight" in source + assert "at_bias" in source + + +def test_tensor_list_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "stack(Tensor[] tensors, int dim=0, *, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "std::vector", + "int64_t", + "Tensor", + ] + assert op.is_testable + + base = module._generate_base_header("stack", [op]) + source = module._generate_torch_method_source("stack", op) + + assert "#include " in base + assert "std::vector tensors" in base + assert "std::vector at_tensors" in source + assert "at_tensors.reserve(tensors.size())" in source + assert "for (const auto& tensor : tensors)" in source + assert "at::stack_out(at_out, at_tensors, dim)" in source + + +def test_optional_tensor_list_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + op = module._parse_func( + "index.Tensor_out(Tensor self, Tensor?[] indices, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "std::vector>", + "Tensor", + ] + + source = module._generate_torch_method_source("index", op) + + assert "c10::List> at_indices" in source + assert "for (const auto& tensor : indices)" in source + assert "at_indices.push_back(c10::nullopt)" in source + assert "at::index_out(at_out, at_self, at_indices)" in source + + +def test_optional_scalar_and_array_params_are_exposed_and_forwarded_to_aten(): + module = _load_generator_module() + quantile = module._parse_func( + "quantile(Tensor input, Tensor q, int? dim=None, bool keepdim=False, " + "str interpolation='linear', *, Tensor(a!) out) -> Tensor(a!)" + ) + upsample = module._parse_func( + "upsample_bicubic2d(Tensor input, SymInt[2] output_size, " + "bool align_corners, float[]? scale_factors=None, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in quantile.visible_params] == [ + "Tensor", + "Tensor", + "std::optional", + "bool", + "std::string", + "Tensor", + ] + assert [param.cpp_type for param in upsample.visible_params] == [ + "Tensor", + "std::vector", + "bool", + "std::optional>", + "Tensor", + ] + + quantile_source = module._generate_torch_method_source("quantile", quantile) + upsample_source = module._generate_torch_method_source( + "upsample_bicubic2d", upsample + ) + + assert "c10::optional at_dim" in quantile_source + assert "at::quantile_out" in quantile_source + assert "at_dim" in quantile_source + assert "c10::optional> at_scale_factors" in upsample_source + assert "at::upsample_bicubic2d_out" in upsample_source + assert "at_scale_factors" in upsample_source + + +def test_required_scalar_type_params_use_public_data_type(): + module = _load_generator_module() + op = module._parse_func( + "_softmax_backward_data(Tensor grad_output, Tensor output, int dim, " + "ScalarType input_dtype, *, Tensor(a!) grad_input) -> Tensor(a!)" + ) + + assert [param.cpp_type for param in op.visible_params] == [ + "Tensor", + "Tensor", + "int64_t", + "DataType", + "Tensor", + ] + + source = module._generate_torch_method_source("internal_softmax_backward_data", op) + + assert "at::_softmax_backward_data_out" in source + assert "ToAtenDataType(input_dtype)" in source + + +def test_existing_base_overload_can_omit_optional_schema_params(): + module = _load_generator_module() + op = module._parse_func( + "slow_conv3d(Tensor input, Tensor weight, int[3] kernel_size, " + "Tensor? bias=None, int[3] stride=1, int[3] padding=0, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("Tensor", "weight"), + ("std::vector", "kernel_size"), + ("std::vector", "stride"), + ("std::vector", "padding"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.name for param in bound.visible_params] == [ + "input", + "weight", + "kernel_size", + "stride", + "padding", + "out", + ] + + source = module._generate_torch_method_source("slow_conv3d", bound) + + assert "std::optional bias" not in source + assert "c10::optional{}" in source + assert "at::slow_conv3d_out" in source + + +def test_existing_base_overload_can_omit_defaulted_schema_params(): + module = _load_generator_module() + op = module._parse_func( + "add.Tensor(Tensor self, Tensor other, *, Scalar alpha=1, " + "Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("const Tensor", "input"), + ("const Tensor", "other"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + + source = module._generate_torch_method_source("add", bound) + + assert "double alpha" not in source + assert "const auto device_index = out.device().index();" in source + assert "device_index_)" not in source + assert "at::add_out(at_out, at_self, at_other, 1)" in source + + +def test_existing_base_overload_matches_by_name_when_types_repeat(): + module = _load_generator_module() + op = module._parse_func( + "std(Tensor input, int[1]? dim=None, bool unbiased=True, " + "bool keepdim=False, *, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("bool", "keepdim"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.name for param in bound.visible_params] == [ + "input", + "keepdim", + "out", + ] + + source = module._generate_torch_method_source("std", bound) + + assert "c10::optional{}, true, keepdim" in source + assert "unbiased" not in source + + +def test_existing_base_overload_can_group_tensor_inputs_before_attributes(): + module = _load_generator_module() + op = module._parse_func( + "gather.out(Tensor self, int dim, Tensor index, bool sparse_grad=False, " + "*, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("Tensor", "index"), + ("int64_t", "dim"), + ("bool", "sparse_grad"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "index", + "dim", + "sparse_grad", + "out", + ] + + assert [ + next( + index + for index, binding in enumerate(bound.param_bindings) + if binding is param + ) + for param in bound.visible_params + ] == [0, 2, 1, 3, 4] + + source = module._generate_torch_method_source("gather", bound) + + assert "at::gather_out(at_out, at_self, dim, at_index, sparse_grad)" in source + + +def test_rrelu_with_noise_base_order_preserves_aten_call_order(): + module = _load_generator_module() + op = module._parse_func( + "rrelu_with_noise.out(Tensor self, Tensor(b!) noise, Scalar lower=0.125, " + "Scalar upper=0.3333333333333333, bool training=False, " + "Generator? generator=None, *, Tensor(a!) out) -> Tensor(a!)" + ) + signature = [ + ("Tensor", "input"), + ("double", "lower"), + ("double", "upper"), + ("bool", "training"), + ("Tensor", "out"), + ("Tensor", "noise"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "lower", + "upper", + "training", + "out", + "noise", + ] + assert [ + next( + index + for index, binding in enumerate(bound.param_bindings) + if binding is param + ) + for param in bound.visible_params + ] == [0, 2, 3, 4, 6, 1] + + source = module._generate_torch_method_source("rrelu_with_noise", bound) + + assert "Tensor out, Tensor noise" in source + assert ( + "at::rrelu_with_noise_out(at_out, at_self, at_noise, lower, upper, " + "training, c10::optional{})" in source + ) + + +def test_write_text_if_changed_preserves_unchanged_mtime(tmp_path): + module = _load_generator_module() + path = tmp_path / "generated.cc" + path.write_text("same\n") + before = path.stat().st_mtime_ns + + assert module._write_text_if_changed(path, "same\n") is False + assert path.stat().st_mtime_ns == before + + assert module._write_text_if_changed(path, "different\n") is True + assert path.read_text() == "different\n" + assert path.stat().st_mtime_ns >= before + + +def test_remove_stale_files_keeps_expected_outputs(tmp_path): + module = _load_generator_module() + root = tmp_path / "generated" + keep = root / "torch" / "keep.cc" + stale = root / "torch" / "stale.cc" + nested_stale = root / "base" / "stale.h" + keep.parent.mkdir(parents=True) + nested_stale.parent.mkdir(parents=True) + keep.write_text("keep\n") + stale.write_text("stale\n") + nested_stale.write_text("stale\n") + + module._remove_stale_files(root, {keep}) + + assert keep.exists() + assert not stale.exists() + assert not nested_stale.exists() + assert not (root / "base").exists() + + +def test_inplace_schema_can_bind_to_explicit_output_signature(): + module = _load_generator_module() + op = module._parse_func("fill_.Scalar(Tensor(a!) self, Scalar value) -> Tensor(a!)") + signature = [ + ("Tensor", "input"), + ("double", "value"), + ("Tensor", "out"), + ] + + bound = module._bind_base_signature(op, signature) + + assert bound is not None + assert bound.explicit_inplace_out is not None + assert [param.api_name for param in bound.visible_params] == [ + "input", + "value", + "out", + ] + assert bound.out_params[0].api_name == "out" + + source = module._generate_torch_method_source("fill", bound) + + assert "const auto device_index = out.device().index();" in source + assert "const_cast(input.data())" in source + assert "auto at_out = ToAtenTensor" in source + assert "at_out.copy_(at_self);" in source + assert "at_out.fill_(value);" in source + + implicit_bound = module._bind_base_signature(op, signature[:-1]) + + assert implicit_bound is not None + assert implicit_bound.explicit_inplace_out is None + + implicit_source = module._generate_torch_method_source("fill", implicit_bound) + assert "at_self.fill_(value);" in implicit_source diff --git a/tests/test_generate_wrappers.py b/tests/test_generate_wrappers.py index a3efcd026..bb2801d5d 100644 --- a/tests/test_generate_wrappers.py +++ b/tests/test_generate_wrappers.py @@ -19,6 +19,30 @@ def _load_generator_module(): return module +def test_shared_validation_calls_are_not_parsed_as_constructors(monkeypatch, tmp_path): + module = _load_generator_module() + header = tmp_path / "cross_entropy.h" + header.write_text( + """ +class CrossEntropy : public Operator { + public: + explicit CrossEntropy(const Tensor input, Tensor out) { + ValidateCrossEntropy(input, out); + } + virtual void operator()(const Tensor input, Tensor out) const = 0; +}; +""" + ) + monkeypatch.setattr(module, "_find_base_header", lambda _: header) + parsed = module._parse_operator_header("cross_entropy") + assert len(parsed.constructors) == 1 + assert len(parsed.calls) == 1 + assert [arg.spelling for arg in parsed.constructors[0].get_arguments()] == [ + "input", + "out", + ] + + def test_generated_dispatch_keeps_scalar_and_optional_tensor_overloads_distinct( monkeypatch, tmp_path ): diff --git a/tests/test_masked_fill_backward.py b/tests/test_masked_fill_backward.py new file mode 100644 index 000000000..07712f899 --- /dev/null +++ b/tests/test_masked_fill_backward.py @@ -0,0 +1,464 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream + + +# Format: (input_shape, mask_shape, grad_output_strides, mask_strides, +# grad_input_strides, mask_mode) +_TEST_CASES = ( + pytest.param((17,), (17,), None, None, None, "alternating", id="one-dimensional"), + pytest.param((3, 5), (3, 5), None, None, None, "all-false", id="all-false-mask"), + pytest.param((3, 5), (3, 5), None, None, None, "all-true", id="all-true-mask"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + "alternating", + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (20, 6, 1), + (18, 5, 1), + (24, 7, 1), + "alternating", + id="strided-tensors", + ), + pytest.param( + (2, 3), + (3,), + None, + None, + None, + "alternating", + id="column-mask-broadcast", + ), + pytest.param( + (2, 3), + (2, 1), + None, + None, + None, + "alternating", + id="explicit-row-mask-broadcast", + ), + pytest.param( + (2, 3, 4), + (4,), + None, + None, + None, + "alternating", + id="mask-broadcast-across-two-dims", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "alternating", + id="input-broadcast-reduction", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "all-false", + id="input-broadcast-all-false", + ), + pytest.param( + (1, 3), + (2, 3), + None, + None, + None, + "all-true", + id="input-broadcast-all-true", + ), + pytest.param( + (3,), + (2, 3), + None, + None, + None, + "alternating", + id="input-missing-leading-dim", + ), + pytest.param( + (2, 1), + (1, 3), + None, + None, + None, + "alternating", + id="both-operands-broadcast", + ), + pytest.param( + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + "alternating", + id="middle-dim-reduction", + ), + pytest.param( + (1, 3, 1), + (2, 1, 4), + None, + None, + None, + "alternating", + id="multiple-dim-reduction", + ), + pytest.param( + (2, 3), + (1, 2, 3), + None, + None, + None, + "alternating", + id="mask-adds-leading-singleton", + ), + pytest.param( + (1, 3), + (2, 3), + (8, 2), + (7, 2), + (9, 2), + "alternating", + id="strided-reduction", + ), + pytest.param( + (2, 1), + (1, 3), + (8, 2), + (8, 2), + (3, 1), + "alternating", + id="strided-mutual-broadcast", + ), + pytest.param((), (), None, None, None, "all-false", id="scalar-all-false"), + pytest.param((), (), None, None, None, "all-true", id="scalar-all-true"), + pytest.param((2, 3), (), None, None, None, "all-false", id="scalar-mask-all-false"), + pytest.param((2, 3), (), None, None, None, "all-true", id="scalar-mask-all-true"), + pytest.param( + (), + (2, 3), + None, + None, + None, + "alternating", + id="scalar-input-reduction", + ), + pytest.param((0,), (0,), None, None, None, "alternating", id="empty-vector"), + pytest.param( + (0, 3), + (3,), + None, + None, + None, + "alternating", + id="empty-output-with-small-mask", + ), + pytest.param( + (1, 3), + (0, 3), + None, + None, + None, + "alternating", + id="empty-output-reduces-to-nonempty-input", + ), + pytest.param( + (2, 0, 4), + (1, 0, 1), + None, + None, + None, + "alternating", + id="empty-middle-dim", + ), + pytest.param( + (), + (0, 3), + None, + None, + None, + "alternating", + id="empty-output-reduces-to-scalar", + ), +) + +_DTYPES = (torch.float32, torch.float16, torch.bfloat16) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "input_shape", + "mask_shape", + "grad_output_strides", + "mask_strides", + "grad_input_strides", + "mask_mode", + ), + _TEST_CASES, +) +@pytest.mark.parametrize("dtype", _DTYPES) +def test_masked_fill_backward( + input_shape, + mask_shape, + grad_output_strides, + mask_strides, + grad_input_strides, + mask_mode, + implementation_index, + dtype, + device, +): + output_shape = torch.broadcast_shapes(input_shape, mask_shape) + grad_output = _empty(output_shape, grad_output_strides, dtype, device) + grad_output.normal_() + if tuple(output_shape) != tuple(input_shape): + # Small signed integers keep all these sums exactly representable, + # including in FP16/BF16, so reduction tests can use zero tolerance. + values = torch.arange(grad_output.numel(), device=device) % 7 - 3 + grad_output.copy_(values.reshape(output_shape)) + + mask = _make_mask(mask_shape, mask_strides, mask_mode, device) + grad_input = _empty(input_shape, grad_input_strides, dtype, device) + grad_input.fill_(17) + + return Payload( + lambda *args: _masked_fill_backward( + *args, implementation_index=implementation_index + ), + _torch_masked_fill_backward, + (grad_output, mask, grad_input), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +def test_masked_fill_backward_known_reduction(implementation_index, dtype, device): + grad_output = torch.tensor([[1, 2, 3], [4, 5, 6]], dtype=dtype, device=device) + mask = torch.tensor([[False, True, False], [False, False, True]], device=device) + grad_input = torch.full((1, 3), 17, dtype=dtype, device=device) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + expected = torch.tensor([[5, 5, 3]], dtype=dtype, device=device) + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("input_shape", ((2, 3), (1, 3))) +@pytest.mark.parametrize("all_true", (False, True)) +def test_masked_fill_backward_nonfinite_gradients( + implementation_index, dtype, device, input_shape, all_true +): + grad_output = torch.tensor( + [[float("nan"), float("inf"), 3], [4, -float("inf"), float("nan")]], + dtype=dtype, + device=device, + ) + mask = torch.tensor([[True, True, False], [False, False, False]], device=device) + if all_true: + mask.fill_(True) + grad_input = torch.full(input_shape, 17, dtype=dtype, device=device) + expected = _torch_masked_fill_backward( + grad_output, mask, torch.empty_like(grad_input) + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0, equal_nan=True) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("input_shape", ((2, 3), (1, 3))) +def test_masked_fill_backward_expanded_readonly_inputs( + implementation_index, dtype, device, input_shape +): + # Both inputs have zero strides from expand(); output has ordinary storage. + grad_output = torch.tensor([[1, 2, 3]], dtype=dtype, device=device).expand(2, 3) + mask = torch.tensor([[False], [True]], device=device).expand(2, 3) + grad_input = torch.full(input_shape, 17, dtype=dtype, device=device) + expected = _torch_masked_fill_backward( + grad_output, mask, torch.empty_like(grad_input) + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +# Call the implementation directly: a Payload helper may clone arguments and +# destroy the storage overlap that these tests are intended to exercise. +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "alias_mode", + ("exact", "partial", "strided", "reduced", "mask"), +) +def test_masked_fill_backward_overlapping_storage( + implementation_index, dtype, device, alias_mode +): + storage = torch.arange(24, dtype=dtype, device=device) + grad_output = storage[:6].reshape(2, 3) + mask = _make_mask((2, 3), None, "alternating", device) + + if alias_mode == "exact": + grad_input = grad_output + elif alias_mode == "partial": + grad_input = storage[1:7].reshape(2, 3) + elif alias_mode == "strided": + grad_input = storage[1:13:2].reshape(2, 3) + elif alias_mode == "reduced": + grad_input = storage[1:4].reshape(1, 3) + else: + # Different dtypes can still refer to overlapping byte ranges. + grad_input = storage[:6].reshape(2, 3) + grad_input.fill_(17) + mask = storage.view(torch.bool)[:6].reshape(2, 3) + mask.copy_(_make_mask((2, 3), None, "alternating", device)) + grad_output = torch.arange(1, 7, dtype=dtype, device=device).reshape(2, 3) + + # Preserve the original values before the implementation writes its output. + expected = _torch_masked_fill_backward( + grad_output.detach().clone(), + mask.detach().clone(), + torch.empty(grad_input.shape, dtype=dtype, device=device), + ) + + _masked_fill_backward( + grad_output, mask, grad_input, implementation_index=implementation_index + ) + + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) + + +# Format: (input_shape, mask_shape, supplied_grad_output_shape) +_INVALID_SHAPE_CASES = ( + pytest.param((2, 3), (2,), (2, 3), id="reject-implicit-row-alignment"), + pytest.param((2, 3), (2, 2), (2, 3), id="reject-incompatible-mask-dimension"), + pytest.param((2, 3), (2, 3), (3,), id="reject-missing-output-dimension"), + pytest.param((2, 3), (1, 2, 3), (2, 3), id="reject-missing-leading-singleton"), + pytest.param((1, 3), (3,), (2, 3), id="reject-unexplained-output-expansion"), + pytest.param((2, 3), (2, 3), (1, 3), id="reject-smaller-output"), + pytest.param((2, 3), (0, 3), (0, 3), id="reject-zero-versus-nonsingleton"), +) + + +@pytest.mark.parametrize( + ("input_shape", "mask_shape", "output_shape"), _INVALID_SHAPE_CASES +) +def test_masked_fill_backward_rejects_invalid_shapes( + implementation_index, device, input_shape, mask_shape, output_shape +): + setup = f"device = torch.device({str(device)!r})\n" + setup += f"grad_output = torch.ones({output_shape!r}, device=device)\n" + setup += f"mask = torch.zeros({mask_shape!r}, dtype=torch.bool, device=device)\n" + setup += f"grad_input = torch.empty({input_shape!r}, device=device)\n" + probe = setup + "infini.ops.MaskedFillBackward(grad_output, mask, grad_input)\n" + source = ( + setup + + f"infini.ops.masked_fill_backward(grad_output, mask, grad_input, stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "broadcast shape of grad_input and mask"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +def _empty(shape, strides, dtype, device): + if strides is None: + return torch.empty(shape, dtype=dtype, device=device) + return torch.empty_strided(shape, strides, dtype=dtype, device=device) + + +def _make_mask(shape, strides, mode, device): + mask = _empty(shape, strides, torch.bool, device) + if mode == "all-false": + mask.fill_(False) + elif mode == "all-true": + mask.fill_(True) + else: + values = torch.arange(mask.numel(), device=device).reshape(shape) % 3 == 0 + mask.copy_(values) + return mask + + +def _masked_fill_backward(grad_output, mask, grad_input, *, implementation_index): + infini.ops.masked_fill_backward( + grad_output, + mask, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return grad_input + + +def _torch_masked_fill_backward(grad_output, mask, grad_input): + # Use forward + autograd as the oracle, rather than reimplementing the + # masking/reduction formula under test. Snapshot inputs for alias safety. + saved_grad = grad_output.detach().clone() + saved_mask = mask.detach().clone() + with torch.enable_grad(): + input_tensor = torch.zeros( + grad_input.shape, + dtype=grad_input.dtype, + device=grad_input.device, + requires_grad=True, + ) + output = input_tensor.masked_fill(saved_mask, -7) + (expected,) = torch.autograd.grad(output, input_tensor, grad_outputs=saved_grad) + grad_input.copy_(expected) + return grad_input diff --git a/tests/test_mean_backward.py b/tests/test_mean_backward.py new file mode 100644 index 000000000..9dc7e20a3 --- /dev/null +++ b/tests/test_mean_backward.py @@ -0,0 +1,158 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, dim, keepdim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 0, + False, + None, + None, + id="one-dimensional-no-keep-dim", + ), + pytest.param( + (2, 3), + 1, + False, + None, + None, + id="two-dimensional-no-keep-dim", + ), + pytest.param( + (2, 3), + 1, + True, + None, + None, + id="two-dimensional-keep-dim", + ), + pytest.param( + (2, 3, 4), + -1, + False, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 0, + True, + None, + None, + id="three-dimensional-first-dimension-keep-dim", + ), + pytest.param( + (2, 3, 4), + 1, + False, + (10, 1), + (24, 7, 1), + id="strided-gradients", + ), + pytest.param( + (2, 3, 4), + 1, + True, + (10, 1, 1), + (24, 7, 1), + id="strided-gradients-keep-dim", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, dim, keepdim, grad_output_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_mean_backward( + input_shape, + dim, + keepdim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape) + grad_output_shape = list(input_shape) + if keepdim: + grad_output_shape[normalized_dim] = 1 + else: + del grad_output_shape[normalized_dim] + + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _mean_backward( + *args, + implementation_index=implementation_index, + ), + _torch_mean_backward, + (grad_output, dim, keepdim, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _mean_backward( + grad_output, + dim, + keepdim, + grad_input, + *, + implementation_index, +): + infini.ops.mean_backward( + grad_output, + dim, + keepdim, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_mean_backward(grad_output, dim, keepdim, grad_input): + normalized_dim = dim % grad_input.ndim + reduce_size = grad_input.shape[normalized_dim] + expanded_grad = grad_output + + if not keepdim: + expanded_grad = expanded_grad.unsqueeze(normalized_dim) + + grad_input.copy_(expanded_grad.expand_as(grad_input) / reduce_size) + + return grad_input diff --git a/tests/test_mul_backward.py b/tests/test_mul_backward.py new file mode 100644 index 000000000..08ae1b9d0 --- /dev/null +++ b/tests/test_mul_backward.py @@ -0,0 +1,465 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_DTYPES = ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + +# Format: (shape, grad_output_strides, a_strides, b_strides, +# grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), None, None, None, None, None, id="one-dimensional"), + pytest.param((2, 3), None, None, None, None, None, id="two-dimensional"), + pytest.param((2, 3, 4), None, None, None, None, None, id="three-dimensional"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (28, 6, 1), + (32, 5, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "grad_output_strides", + "a_strides", + "b_strides", + "grad_a_strides", + "grad_b_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_mul_backward( + shape, + grad_output_strides, + a_strides, + b_strides, + grad_a_strides, + grad_b_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + a = randn_strided(shape, a_strides, dtype=dtype, device=device) + b = randn_strided(shape, b_strides, dtype=dtype, device=device) + grad_a = empty_strided(shape, grad_a_strides, dtype=dtype, device=device) + grad_b = empty_strided(shape, grad_b_strides, dtype=dtype, device=device) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _mul_backward( + *args, + implementation_index=implementation_index, + ), + _torch_mul_backward, + (grad_output, a, b, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + *, + implementation_index, +): + infini.ops.mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +def _torch_mul_backward(grad_output, a, b, grad_a, grad_b): + # Finish both computations before writing potentially aliased outputs. + result_a = (grad_output * b).sum_to_size(a.shape) + result_b = (grad_output * a).sum_to_size(b.shape) + grad_a.copy_(result_a) + grad_b.copy_(result_b) + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1))) + + +# Alias tests deliberately call the operator directly. A harness that clones +# each argument independently may destroy alias relationships. +def _reference_gradients(grad_output, a, b): + # Snapshot BEFORE calling the operator: it may overwrite these inputs. + with torch.enable_grad(): + a_ref = a.detach().clone().requires_grad_(True) + b_ref = b.detach().clone().requires_grad_(True) + return torch.autograd.grad( + a_ref * b_ref, + (a_ref, b_ref), + grad_output.detach().clone(), + ) + + +def _assert_gradients(grad_a, grad_b, expected, rtol, atol): + torch.testing.assert_close(grad_a, expected[0], rtol=rtol, atol=atol) + torch.testing.assert_close(grad_b, expected[1], rtol=rtol, atol=atol) + + +def _make_inputs(shape, dtype, device): + return { + name: randn_strided(shape, None, dtype=dtype, device=device) + for name in ("grad_output", "a", "b") + } + + +def _backing(dtype, device): + # Small, exactly representable values, with space for guard elements. + return (torch.arange(40, dtype=torch.float32, device=device) / 8 - 2).to( + dtype=dtype, + ) + + +def _alias_views(backing, layout): + # All views have shape (2, 3) and internally non-overlapping elements. + # Offset 2 leaves leading guard elements in the backing allocation. + if layout == "full": + x = backing.as_strided((2, 3), (3, 1), 2) + y = backing.as_strided((2, 3), (3, 1), 2) + elif layout == "partial": + x = backing.as_strided((2, 3), (3, 1), 2) + y = backing.as_strided((2, 3), (3, 1), 4) + elif layout == "transposed": + x = backing.as_strided((2, 3), (1, 2), 2) + y = backing.as_strided((2, 3), (3, 1), 2) + elif layout == "strided-partial": + x = backing.as_strided((2, 3), (6, 2), 2) + y = backing.as_strided((2, 3), (6, 2), 4) + else: + raise AssertionError(layout) + return x, y + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_mul_backward_review_regression( + dtype, + rtol, + atol, + device, + implementation_index, +): + a = torch.tensor([2, 3], dtype=dtype, device=device) + b = torch.tensor([5, 7], dtype=dtype, device=device) + grad_output = torch.ones_like(a) + grad_a = a.view_as(a) + grad_b = torch.full_like(b, 17) + + _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + torch.testing.assert_close( + grad_a, + torch.tensor([5, 7], dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + torch.testing.assert_close( + grad_b, + torch.tensor([2, 3], dtype=dtype, device=device), + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("source", ("a", "b", "grad_output")) +@pytest.mark.parametrize("destination", ("grad_a", "grad_b")) +@pytest.mark.parametrize( + "layout", + ("full", "partial", "transposed", "strided-partial"), +) +@pytest.mark.parametrize("reverse", (False, True), ids=("forward", "reverse")) +def test_mul_backward_input_output_alias( + source, + destination, + layout, + reverse, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + backing = _backing(dtype, device) + input_view, output_view = _alias_views(backing, layout) + if reverse: + input_view, output_view = output_view, input_view + inputs[source] = input_view + outputs = { + name: torch.full((2, 3), 17, dtype=dtype, device=device) + for name in ("grad_a", "grad_b") + } + outputs[destination] = output_view + + expected = _reference_gradients(**inputs) + expected_backing = backing.clone() + expected_view = expected_backing.as_strided( + output_view.shape, + output_view.stride(), + output_view.storage_offset(), + ) + expected_view.copy_(expected[0 if destination == "grad_a" else 1]) + unchanged_inputs = { + name: tensor.clone() for name, tensor in inputs.items() if name != source + } + + _mul_backward(**inputs, **outputs, implementation_index=implementation_index) + + _assert_gradients(outputs["grad_a"], outputs["grad_b"], expected, rtol, atol) + # Include gaps and guard elements, not just the logical output elements. + torch.testing.assert_close(backing, expected_backing, rtol=rtol, atol=atol) + for name, snapshot in unchanged_inputs.items(): + torch.testing.assert_close(inputs[name], snapshot, rtol=0, atol=0) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize( + ("source_a", "source_b"), + ( + ("a", "b"), + ("b", "a"), + ("grad_output", "a"), + ("grad_output", "b"), + ("a", "grad_output"), + ("b", "grad_output"), + ), +) +def test_mul_backward_both_outputs_alias_inputs( + source_a, + source_b, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + grad_a = inputs[source_a].view_as(inputs[source_a]) + grad_b = inputs[source_b].view_as(inputs[source_b]) + expected = _reference_gradients(**inputs) + + _mul_backward( + **inputs, + grad_a=grad_a, + grad_b=grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("destination", ("separate", "grad_a", "grad_b")) +def test_mul_backward_all_inputs_alias( + destination, + dtype, + rtol, + atol, + device, + implementation_index, +): + shared = _backing(dtype, device).as_strided((2, 3), (6, 2), 2) + inputs = { + name: shared.as_strided(shared.shape, shared.stride()) + for name in ("grad_output", "a", "b") + } + outputs = { + name: torch.full((2, 3), 17, dtype=dtype, device=device) + for name in ("grad_a", "grad_b") + } + if destination != "separate": + outputs[destination] = shared + expected = _reference_gradients(**inputs) + + _mul_backward(**inputs, **outputs, implementation_index=implementation_index) + + _assert_gradients(outputs["grad_a"], outputs["grad_b"], expected, rtol, atol) + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize( + ("a_shape", "b_shape"), + (((2, 3, 4), (4,)), ((2, 1, 4), (1, 3, 1))), +) +@pytest.mark.parametrize("alias", ("none", "a", "b", "both", "grad-to-a", "grad-to-b")) +def test_mul_backward_broadcast_alias( + a_shape, + b_shape, + alias, + dtype, + rtol, + atol, + device, + implementation_index, +): + a = randn_strided(a_shape, None, dtype=dtype, device=device) + b = randn_strided(b_shape, None, dtype=dtype, device=device) + grad_output = randn_strided((2, 3, 4), None, dtype=dtype, device=device) + grad_a = torch.full(a_shape, 17, dtype=dtype, device=device) + grad_b = torch.full(b_shape, 17, dtype=dtype, device=device) + if alias in ("a", "both"): + grad_a = a.view_as(a) + if alias in ("b", "both"): + grad_b = b.view_as(b) + if alias == "grad-to-a": + grad_a = ( + grad_output.view_as(grad_output) + if a_shape == (2, 3, 4) + else grad_output[:, :1, :] + ) + if alias == "grad-to-b": + grad_b = grad_output[0, 0, :] if b_shape == (4,) else grad_output[:1, :, :1] + expected = _reference_gradients(grad_output, a, b) + + _mul_backward( + grad_output, + a, + b, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + + +@pytest.mark.parametrize("layout", ("full", "partial", "strided", "interleaved")) +@pytest.mark.parametrize("reverse", (False, True)) +def test_mul_backward_reject_output_overlap( + layout, reverse, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = ( + setup + "infini.ops.MulBackward(x, x, x, torch.empty(4, device=device), x)\n" + ) + stride = (6, 2) if layout in ("strided", "interleaved") else (3, 1) + offset = {"full": 2, "partial": 4, "strided": 4, "interleaved": 3}[layout] + offsets = (offset, 2) if reverse else (2, offset) + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {stride!r}, {offsets[0]}) +b = backing.as_strided((2, 3), {stride!r}, {offsets[1]}) +infini.ops.mul_backward(x, x, x, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +@pytest.mark.parametrize("layout", ("adjacent", "strided-disjoint")) +@pytest.mark.parametrize("reverse", (False, True), ids=("forward", "reverse")) +def test_mul_backward_disjoint_outputs_in_same_storage( + layout, + reverse, + dtype, + rtol, + atol, + device, + implementation_index, +): + inputs = _make_inputs((2, 3), dtype, device) + backing = _backing(dtype, device) + if layout == "adjacent": + # Half-open ranges meet at the boundary; this is not overlap. + grad_a = backing.as_strided((2, 3), (3, 1), 2) + grad_b = backing.as_strided((2, 3), (3, 1), 8) + else: + grad_a = backing.as_strided((2, 3), (6, 2), 2) + grad_b = backing.as_strided((2, 3), (6, 2), 18) + if reverse: + grad_a, grad_b = grad_b, grad_a + expected = _reference_gradients(**inputs) + expected_backing = backing.clone() + for output, result in zip((grad_a, grad_b), expected): + expected_backing.as_strided( + output.shape, + output.stride(), + output.storage_offset(), + ).copy_(result) + + _mul_backward( + **inputs, + grad_a=grad_a, + grad_b=grad_b, + implementation_index=implementation_index, + ) + + _assert_gradients(grad_a, grad_b, expected, rtol, atol) + torch.testing.assert_close(backing, expected_backing, rtol=rtol, atol=atol) diff --git a/tests/test_pow_backward.py b/tests/test_pow_backward.py new file mode 100644 index 000000000..b9a59bb36 --- /dev/null +++ b/tests/test_pow_backward.py @@ -0,0 +1,317 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_TEST_CASES = ( + pytest.param((17,), 2.0, False, None, None, None, id="one-dimensional-input-base"), + pytest.param((2, 3), 3.0, False, None, None, None, id="two-dimensional-input-base"), + pytest.param((2, 3, 4), 0.5, False, None, None, None, id="fractional-input-base"), + pytest.param((2, 3, 4), 2.0, True, None, None, None, id="scalar-base"), + pytest.param((2, 3, 4), 0.5, True, None, None, None, id="fractional-scalar-base"), + pytest.param( + (2, 3, 4), 3.0, False, (30, 7, 1), (28, 6, 1), (35, 8, 1), id="strided-tensors" + ), +) + +_DTYPE_CASES = ( + pytest.param(torch.float32, 1e-6, 1e-6, id="float32"), + pytest.param(torch.float16, 1e-3, 1e-3, id="float16"), + pytest.param(torch.bfloat16, 1e-2, 1e-2, id="bfloat16"), +) + +_EDGE_CASES = ( + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 0.0, + False, + id="zero-exponent-including-zero-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 1.0, + False, + id="unit-exponent-including-zero-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 2.0, + False, + id="square-negative-and-zero-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 3.0, + False, + id="cube-negative-and-zero-input", + ), + pytest.param( + (-2.0, -1.0, 1.0, 2.0), + (1.0, -2.0, 0.0, 3.0), + -2.0, + False, + id="negative-integer-exponent", + ), + pytest.param( + (0.25, 1.0, 4.0), + (2.0, 0.0, -2.0), + 0.5, + False, + id="fractional-exponent-positive-input", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 1.0, + True, + id="unit-scalar-base-zero-derivative", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 2.0, + True, + id="scalar-base-negative-zero-positive-exponents", + ), + pytest.param( + (-2.0, 0.0, 2.0), + (1.0, -2.0, 3.0), + 0.5, + True, + id="fractional-scalar-base-negative-derivative", + ), +) + +_NUMERICAL_CASES = ( + pytest.param( + (0.5,), + (40000.0,), + 2.0, + False, + id="input-base-large-grad-finite-result", + ), + pytest.param( + (1.0,), + (40000.0,), + 2.0, + True, + id="scalar-base-large-grad-finite-result", + ), + pytest.param( + (-16.0,), + (1.0,), + 0.5, + True, + id="scalar-base-pow-overflow-finite-gradient", + ), + pytest.param( + (0.5,), + (-40000.0,), + 2.0, + False, + id="input-base-large-negative-grad", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "shape", + "scalar", + "scalar_is_base", + "grad_output_strides", + "input_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) +def test_pow_backward( + shape, + scalar, + scalar_is_base, + grad_output_strides, + input_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + input = randn_strided(shape, input_strides, dtype=dtype, device=device) + input.abs_().add_(0.5) + grad_input = empty_strided( + shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _pow_backward( + *args, + implementation_index=implementation_index, + ), + _torch_pow_backward, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ("input_values", "grad_output_values", "scalar", "scalar_is_base"), + _EDGE_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) +def test_pow_backward_edge_cases( + input_values, + grad_output_values, + scalar, + scalar_is_base, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = torch.tensor(input_values, dtype=dtype, device=device) + grad_output = torch.tensor(grad_output_values, dtype=dtype, device=device) + grad_input = torch.full_like(input, 17) + + return Payload( + lambda *args: _pow_backward( + *args, + implementation_index=implementation_index, + ), + _torch_pow_backward, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ("input_values", "grad_output_values", "scalar", "scalar_is_base"), + _NUMERICAL_CASES, +) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPE_CASES) +def test_pow_backward_numerical_stability( + input_values, + grad_output_values, + scalar, + scalar_is_base, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = torch.tensor(input_values, dtype=dtype, device=device) + grad_output = torch.tensor(grad_output_values, dtype=dtype, device=device) + grad_input = torch.full_like(input, 17) + + def actual(*args): + result = _pow_backward( + *args, + implementation_index=implementation_index, + ) + assert torch.isfinite(result).all().item(), ( + "Expected a finite gradient, but got inf or nan" + ) + return result + + return Payload( + actual, + _torch_pow_backward_high_precision, + (grad_output, input, scalar, scalar_is_base, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, + *, + implementation_index, +): + infini.ops.pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return grad_input + + +def _torch_pow_backward( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, +): + differentiable_input = input.detach().clone().requires_grad_(True) + if scalar_is_base: + result = torch.pow(scalar, differentiable_input) + else: + result = torch.pow(differentiable_input, scalar) + + gradient = torch.autograd.grad(result, differentiable_input, grad_output)[0] + grad_input.copy_(gradient) + return grad_input + + +def _torch_pow_backward_high_precision( + grad_output, + input, + scalar, + scalar_is_base, + grad_input, +): + reference_input = input.detach().to(device="cpu", dtype=torch.float64) + reference_grad_output = grad_output.detach().to( + device="cpu", + dtype=torch.float64, + ) + reference_grad_input = torch.empty_like(reference_input) + gradient = _torch_pow_backward( + reference_grad_output, + reference_input, + scalar, + scalar_is_base, + reference_grad_input, + ) + expected = gradient.to(device=grad_input.device, dtype=grad_input.dtype) + assert torch.isfinite(expected).all().item(), ( + "The regression case must have a finite expected gradient" + ) + grad_input.copy_(expected) + return grad_input diff --git a/tests/test_repeat_interleave.py b/tests/test_repeat_interleave.py new file mode 100644 index 000000000..307f216a3 --- /dev/null +++ b/tests/test_repeat_interleave.py @@ -0,0 +1,131 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, repeats, dim, input_strides, out_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 3, + 0, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + 2, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 3), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="repeat-once", + ), + pytest.param( + (2, 3, 4), + 2, + 1, + (20, 6, 1), + (32, 5, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, repeats, dim, input_strides, out_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_repeat_interleave( + input_shape, + repeats, + dim, + input_strides, + out_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + out_shape = list(input_shape) + out_shape[dim % len(input_shape)] *= repeats + out = empty_strided( + tuple(out_shape), + out_strides, + dtype=dtype, + device=device, + ) + out.fill_(17) + + return Payload( + lambda *args: _repeat_interleave( + *args, + implementation_index=implementation_index, + ), + _torch_repeat_interleave, + (input, repeats, dim, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _repeat_interleave(input, repeats, dim, out, *, implementation_index): + infini.ops.repeat_interleave( + input, + repeats, + dim, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return out + + +def _torch_repeat_interleave(input, repeats, dim, out): + out.copy_(torch.repeat_interleave(input, repeats, dim=dim)) + + return out diff --git a/tests/test_repeat_interleave_backward.py b/tests/test_repeat_interleave_backward.py new file mode 100644 index 000000000..b2160d99b --- /dev/null +++ b/tests/test_repeat_interleave_backward.py @@ -0,0 +1,143 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (input_shape, repeats, dim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (5,), + 3, + 0, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + 2, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 3), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="repeat-once", + ), + pytest.param( + (2, 3, 4), + 2, + 1, + (40, 8, 1), + (20, 6, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, repeats, dim, grad_output_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_repeat_interleave_backward( + input_shape, + repeats, + dim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape) + grad_output_shape = list(input_shape) + grad_output_shape[normalized_dim] *= repeats + + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _repeat_interleave_backward( + *args, + implementation_index=implementation_index, + ), + _torch_repeat_interleave_backward, + (grad_output, dim, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _repeat_interleave_backward( + grad_output, + dim, + grad_input, + *, + implementation_index, +): + infini.ops.repeat_interleave_backward( + grad_output, + dim, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_repeat_interleave_backward(grad_output, dim, grad_input): + dim = dim % grad_output.ndim + repeats = grad_output.shape[dim] // grad_input.shape[dim] + reshaped_shape = list(grad_output.shape) + reshaped_shape[dim] = grad_input.shape[dim] + reshaped_shape.insert(dim + 1, repeats) + grad_input.copy_(grad_output.reshape(reshaped_shape).sum(dim=dim + 1)) + + return grad_input diff --git a/tests/test_rsqrt_backward.py b/tests/test_rsqrt_backward.py new file mode 100644 index 000000000..93a19d126 --- /dev/null +++ b/tests/test_rsqrt_backward.py @@ -0,0 +1,161 @@ +import decimal + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (shape, grad_output_strides, input_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (17,), + None, + None, + None, + id="one-dimensional", + ), + pytest.param( + (2, 3), + None, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (30, 7, 1), + (28, 6, 1), + (35, 8, 1), + id="strided-tensors", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "shape, grad_output_strides, input_strides, grad_input_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_rsqrt_backward( + shape, + grad_output_strides, + input_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + input = randn_strided( + shape, + input_strides, + dtype=dtype, + device=device, + ) + input.abs_().add_(0.5) + grad_input = empty_strided( + shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _rsqrt_backward( + *args, + implementation_index=implementation_index, + ), + _torch_rsqrt_backward, + (grad_output, input, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _rsqrt_backward( + grad_output, + input, + grad_input, + *, + implementation_index, +): + infini.ops.rsqrt_backward( + grad_output, + input, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_rsqrt_backward(grad_output, input, grad_input): + differentiable_input = ( + input.detach().to(device="cpu", dtype=torch.float64).requires_grad_(True) + ) + result = torch.rsqrt(differentiable_input) + upstream = grad_output.detach().to(device="cpu", dtype=torch.float64) + gradient = torch.autograd.grad(result, differentiable_input, upstream)[0] + grad_input.copy_(gradient) + + return grad_input + + +@pytest.mark.parametrize( + "dtype,value,upstream", + ( + (torch.float16, 2**-12, 0.001), + (torch.float16, 2**-24, 2**-24), + (torch.bfloat16, 2**-120, 2**-100), + (torch.float32, 2**-100, 2**-80), + (torch.float64, 2**-800, 2**-700), + ), +) +def test_rsqrt_backward_finite_intermediates( + dtype, value, upstream, device, implementation_index +): + input = torch.full((2, 3), value, dtype=dtype, device=device) + grad_output = torch.full_like(input, upstream) + backing = torch.full((32,), 17, dtype=dtype, device=device) + output = backing.as_strided((2, 3), (6, 2), 2) + + with decimal.localcontext() as context: + context.prec = 80 + x = decimal.Decimal.from_float(input[0, 0].item()) + g = decimal.Decimal.from_float(grad_output[0, 0].item()) + expected_value = float(-g / (2 * x * x.sqrt())) + + expected = backing.clone() + expected.as_strided((2, 3), (6, 2), 2).fill_(expected_value) + _rsqrt_backward( + grad_output, input, output, implementation_index=implementation_index + ) + assert torch.isfinite(output).all().item() + torch.testing.assert_close(backing, expected, rtol=0, atol=0) diff --git a/tests/test_slice_backward.py b/tests/test_slice_backward.py new file mode 100644 index 000000000..1a41dbcd2 --- /dev/null +++ b/tests/test_slice_backward.py @@ -0,0 +1,92 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_CASES = ( + ((17,), None, 0, 1, 16, 3), + ((6, 9), (12, 1), 0, 1, 6, 2), + ((6, 9), (12, 1), 1, 0, 9, 3), + ((4, 6, 7), (60, 9, 1), -2, -5, 6, 2), + ((2, 3, 4, 5), None, 3, 0, 5, 2), + ((5, 7), None, 0, 3, 3, 1), + ((3, 5), None, -1, -100, 100, 2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strides,dim,start,end,step", _CASES) +@pytest.mark.parametrize("strided_upstream", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_slice_backward( + shape, + strides, + dim, + start, + end, + step, + strided_upstream, + dtype, + device, + implementation_index, +): + grad_input = empty_strided(shape, strides, dtype=dtype, device=device) + index = [slice(None)] * len(shape) + index[dim] = slice(start, end, step) + output_shape = grad_input[tuple(index)].shape + output_strides = ( + tuple(stride * 2 for stride in torch.empty(output_shape).stride()) + if strided_upstream + else None + ) + grad_output = randn_strided( + output_shape, output_strides, dtype=dtype, device=device + ) + grad_input.fill_(17) + + def actual(grad_output, grad_input): + infini.ops.slice_backward( + grad_output, + shape, + dim, + start, + end, + step, + grad_input, + stream=get_stream(device), + implementation_index=implementation_index, + ) + return grad_input + + def reference(grad_output, grad_input): + x = torch.zeros(shape, dtype=dtype, device=device, requires_grad=True) + with torch.enable_grad(): + result = x[tuple(index)] + gradient = torch.autograd.grad(result, x, grad_output)[0] + grad_input.copy_(gradient) + return grad_input + + return Payload(actual, reference, (grad_output, grad_input), {}, rtol=0, atol=0) + + +@pytest.mark.parametrize("offset", (0, 2)) +def test_slice_backward_alias(offset, device, implementation_index): + backing = torch.arange(32, dtype=torch.float32, device=device) + grad_output = backing[offset : offset + 6].reshape(3, 2) + out = backing[:12].reshape(3, 4) + expected = torch.zeros_like(out) + expected[:, ::2] = grad_output.clone() + infini.ops.slice_backward( + grad_output, + (3, 4), + 1, + 0, + 4, + 2, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) diff --git a/tests/test_slice_copy.py b/tests/test_slice_copy.py new file mode 100644 index 000000000..116838fc5 --- /dev/null +++ b/tests/test_slice_copy.py @@ -0,0 +1,84 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, get_stream, randn_strided + + +_CASES = ( + ((17,), None, 0, 1, 16, 3), + ((6, 9), (12, 1), 0, 1, 6, 2), + ((6, 9), (12, 1), 1, 0, 9, 3), + ((4, 6, 7), (60, 9, 1), -2, -5, 6, 2), + ((2, 3, 4, 5), None, 3, 0, 5, 2), + ((5, 7), None, 0, 3, 3, 1), + ((3, 5), None, -1, -100, 100, 2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("shape,strides,dim,start,end,step", _CASES) +@pytest.mark.parametrize("optional_bounds", (False, True)) +@pytest.mark.parametrize("strided_output", (False, True)) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_slice_copy( + shape, + strides, + dim, + start, + end, + step, + optional_bounds, + strided_output, + dtype, + device, + implementation_index, +): + input = randn_strided(shape, strides, dtype=dtype, device=device) + start, end = (None, None) if optional_bounds else (start, end) + index = [slice(None)] * len(shape) + index[dim] = slice(start, end, step) + expected_shape = input[tuple(index)].shape + out_strides = tuple( + stride * (2 if strided_output else 1) + for stride in torch.empty(expected_shape).stride() + ) + out = torch.empty_strided(expected_shape, out_strides, dtype=dtype, device=device) + + def actual(input, out): + infini.ops.slice_copy( + input, + dim, + start, + end, + step, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + return out + + def reference(input, out): + out.copy_(input[tuple(index)].clone()) + return out + + return Payload(actual, reference, (input, out), {}, rtol=0, atol=0) + + +@pytest.mark.parametrize("offset", (0, 2)) +def test_slice_copy_alias(offset, device, implementation_index): + backing = torch.arange(32, dtype=torch.float32, device=device) + input = backing[:12].reshape(3, 4) + out = backing[offset : offset + 6].reshape(3, 2) + expected = input[:, ::2].clone() + infini.ops.slice_copy( + input, + 1, + None, + None, + 2, + out, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(out, expected, rtol=0, atol=0) diff --git a/tests/test_split.py b/tests/test_split.py new file mode 100644 index 000000000..c23ece814 --- /dev/null +++ b/tests/test_split.py @@ -0,0 +1,178 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +def test_split_alias(device, implementation_index): + input = torch.arange(12, dtype=torch.float32, device=device).reshape(4, 3) + expected = input.clone() + outputs = [input[2:], input[:2]] + infini.ops.split( + input, + 2, + 0, + outputs, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(outputs[0], expected[:2], rtol=0, atol=0) + torch.testing.assert_close(outputs[1], expected[2:], rtol=0, atol=0) + + +def test_split_rejects_output_overlap(device, implementation_index): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((4, 3), device=device)\n" + probe = setup + ( + "bad = torch.empty((2, 3), dtype=torch.int32, device=device)\n" + "infini.ops.Split(x, 2, 0, [bad, bad])\n" + ) + source = ( + setup + + f"infini.ops.split(x, 2, 0, [x[:2], x[:2]], stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "Output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +_TEST_CASES = ( + pytest.param( + (12,), + None, + 3, + 0, + id="one-dimensional-even", + ), + pytest.param( + (10,), + None, + 4, + 0, + id="one-dimensional-uneven", + ), + pytest.param( + (4, 12), + None, + 3, + 1, + id="two-dimensional", + ), + pytest.param( + (2, 5, 8), + None, + 3, + -1, + id="negative-dimension", + ), + pytest.param( + (4, 6), + (12, 2), + 2, + 1, + id="strided-input", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, split_size, dim", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_split( + input_shape, + input_strides, + split_size, + dim, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided(input_shape, input_strides, dtype=dtype, device=device) + output_shapes = tuple( + output.shape for output in torch.split(input, split_size, dim=dim) + ) + outs = [ + empty_strided(shape, None, dtype=dtype, device=device) + for shape in output_shapes + ] + + return Payload( + lambda *args: _split( + *args, + split_size=split_size, + dim=dim, + implementation_index=implementation_index, + ), + lambda *args: _torch_split( + *args, + split_size=split_size, + dim=dim, + ), + (input, outs), + {}, + rtol=rtol, + atol=atol, + ) + + +def _split(input, outs, *, split_size, dim, implementation_index): + infini.ops.split( + input, + split_size, + dim, + outs, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + + return torch.cat(outs, dim=dim) + + +def _torch_split(input, outs, *, split_size, dim): + for out, expected in zip(outs, torch.split(input, split_size, dim=dim)): + out.copy_(expected) + + return torch.cat(outs, dim=dim) diff --git a/tests/test_stack_backward.py b/tests/test_stack_backward.py new file mode 100644 index 000000000..98a5069d9 --- /dev/null +++ b/tests/test_stack_backward.py @@ -0,0 +1,194 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +def test_stack_backward_alias(device, implementation_index): + grad_output = torch.arange(12, dtype=torch.float32, device=device).reshape(2, 2, 3) + expected = grad_output.clone() + outputs = [grad_output[1], grad_output[0]] + infini.ops.stack_backward( + grad_output, + 0, + outputs, + stream=get_stream(device), + implementation_index=implementation_index, + ) + torch.testing.assert_close(outputs[0], expected[0], rtol=0, atol=0) + torch.testing.assert_close(outputs[1], expected[1], rtol=0, atol=0) + + +def test_stack_backward_rejects_output_overlap(device, implementation_index): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 2, 3), device=device)\n" + probe = setup + ( + "bad = torch.empty((2, 3), dtype=torch.int32, device=device)\n" + "infini.ops.StackBackward(x, 0, [bad, bad])\n" + ) + source = ( + setup + + f"infini.ops.stack_backward(x, 0, [x[0], x[0]], stream=get_stream(device), implementation_index={implementation_index})\n" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "Output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic + + +# Format: (input_shape, num_inputs, dim, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (2, 5), + 3, + 0, + None, + None, + id="two-dimensional-first-dimension", + ), + pytest.param( + (2, 5), + 4, + 1, + None, + None, + id="two-dimensional-second-dimension", + ), + pytest.param( + (2, 3, 4), + 2, + -1, + None, + None, + id="three-dimensional-negative-dimension", + ), + pytest.param( + (2, 3), + 3, + 1, + (20, 5, 1), + (5, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "input_shape", + "num_inputs", + "dim", + "grad_output_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), + ), +) +def test_stack_backward( + input_shape, + num_inputs, + dim, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + normalized_dim = dim % len(input_shape + (0,)) + grad_output_shape = ( + input_shape[:normalized_dim] + (num_inputs,) + input_shape[normalized_dim:] + ) + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_inputs = [ + empty_strided( + input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + for _ in range(num_inputs) + ] + for grad_input in grad_inputs: + grad_input.fill_(17) + + return Payload( + lambda *args: _stack_backward( + *args, + implementation_index=implementation_index, + ), + _torch_stack_backward, + (grad_output, dim, grad_inputs), + {}, + rtol=rtol, + atol=atol, + ) + + +def _stack_backward( + grad_output, + dim, + grad_inputs, + *, + implementation_index, +): + infini.ops.stack_backward( + grad_output, + dim, + grad_inputs, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return torch.stack(grad_inputs, dim=dim) + + +def _torch_stack_backward(grad_output, dim, grad_inputs): + for grad_input, expected in zip(grad_inputs, torch.unbind(grad_output, dim=dim)): + grad_input.copy_(expected) + + return torch.stack(grad_inputs, dim=dim) diff --git a/tests/test_sub_backward.py b/tests/test_sub_backward.py new file mode 100644 index 000000000..9007713ed --- /dev/null +++ b/tests/test_sub_backward.py @@ -0,0 +1,414 @@ +import json +import subprocess +import sys + +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (grad_output_shape, grad_a_shape, grad_b_shape, +# grad_output_strides, grad_a_strides, grad_b_strides) +_TEST_CASES = ( + pytest.param((17,), (17,), (17,), None, None, None, id="one-dimensional"), + pytest.param((2, 3), (2, 3), (2, 3), None, None, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + None, + None, + None, + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (2, 3, 4), + (30, 7, 1), + (35, 8, 1), + (36, 9, 1), + id="strided-tensors", + ), + pytest.param( + (2, 3), + (2, 3), + (2, 3), + (1, 2), + (1, 2), + (1, 2), + id="transposed-layout", + ), + pytest.param( + (2, 3, 4), + (2, 3, 4), + (4,), + None, + None, + None, + id="broadcast-b-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (4,), + (2, 3, 4), + None, + None, + None, + id="broadcast-a-leading-dimensions", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + None, + None, + None, + id="broadcast-both-inputs", + ), + pytest.param( + (2, 3, 4), + (2, 1, 4), + (1, 3, 1), + (30, 7, 1), + (11, 7, 2), + (15, 3, 1), + id="broadcast-strided-tensors", + ), + pytest.param( + (2, 3), + (), + (2, 3), + None, + None, + None, + id="broadcast-scalar-a", + ), + pytest.param( + (2, 3), + (2, 3), + (), + None, + None, + None, + id="broadcast-scalar-b", + ), + pytest.param((), (), (), None, None, None, id="scalar-output"), + pytest.param( + (1, 3, 1), + (3, 1), + (1, 3, 1), + None, + None, + None, + id="singleton-dimensions", + ), + pytest.param( + (0, 3), + (0, 3), + (0, 3), + None, + None, + None, + id="empty-output", + ), + pytest.param( + (0, 3), + (1, 3), + (0, 3), + None, + None, + None, + id="empty-output-nonempty-grad-a", + ), + pytest.param( + (0, 3), + (0, 3), + (1, 3), + None, + None, + None, + id="empty-output-nonempty-grad-b", + ), +) + +_ALPHAS = ( + pytest.param(1.0, id="alpha-one"), + pytest.param(0.0, id="alpha-zero"), + pytest.param(2.0, id="alpha-two"), + pytest.param(-1.0, id="alpha-negative"), + pytest.param(0.5, id="alpha-fractional"), +) + +_DTYPES = ( + (torch.float32, 1e-5, 1e-6), + (torch.float16, 1e-3, 1e-3), + (torch.bfloat16, 1e-2, 1e-2), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "grad_output_shape, grad_a_shape, grad_b_shape, " + "grad_output_strides, grad_a_strides, grad_b_strides", + _TEST_CASES, +) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize(("dtype", "rtol", "atol"), _DTYPES) +def test_sub_backward( + grad_output_shape, + grad_a_shape, + grad_b_shape, + grad_output_strides, + grad_a_strides, + grad_b_strides, + alpha, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output = randn_strided( + grad_output_shape, + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_a = empty_strided( + grad_a_shape, + grad_a_strides, + dtype=dtype, + device=device, + ) + grad_b = empty_strided( + grad_b_shape, + grad_b_strides, + dtype=dtype, + device=device, + ) + grad_a.fill_(17) + grad_b.fill_(17) + + return Payload( + lambda *args: _sub_backward( + *args, + implementation_index=implementation_index, + ), + _torch_sub_backward, + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=rtol, + atol=atol, + ) + + +def _pack_result(grad_output, grad_a, grad_b): + # Also compare grad_output: ordinary, non-alias calls must not modify it. + return torch.cat( + (grad_a.reshape(-1), grad_b.reshape(-1), grad_output.reshape(-1)), + ) + + +def _sub_backward(grad_output, alpha, grad_a, grad_b, *, implementation_index): + infini.ops.sub_backward( + grad_output, + alpha, + grad_a, + grad_b, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + return _pack_result(grad_output, grad_a, grad_b) + + +def _torch_sub_backward(grad_output, alpha, grad_a, grad_b): + # Use canonical forward + autograd, independently of the hand-written formula. + with torch.enable_grad(): + a = torch.zeros_like(grad_a, requires_grad=True) + b = torch.zeros_like(grad_b, requires_grad=True) + out = torch.sub(a, b, alpha=alpha) + expected_a, expected_b = torch.autograd.grad( + out, + (a, b), + grad_outputs=grad_output, + ) + + # Autograd may return a view of grad_output. Snapshot BOTH results first. + expected_a = expected_a.clone() + expected_b = expected_b.clone() + with torch.no_grad(): + grad_a.copy_(expected_a) + grad_b.copy_(expected_b) + return _pack_result(grad_output, grad_a, grad_b) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("target", ("grad_a", "grad_b")) +@pytest.mark.parametrize("layout", ("exact", "partial", "strided", "broadcast")) +@pytest.mark.parametrize("alpha", _ALPHAS) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_sub_backward_alias( + target, + layout, + alpha, + implementation_index, + dtype, + device, +): + # Small integers and power-of-two alpha make these cases exactly representable. + source = torch.arange(-16, 16, dtype=dtype, device=device) + + return Payload( + lambda values: _run_alias( + values, + alpha, + target, + layout, + implementation_index=implementation_index, + ), + lambda values: _run_alias(values, alpha, target, layout), + (source,), + {}, + rtol=0, + atol=0, + ) + + +def _run_alias(source, alpha, target, layout, *, implementation_index=None): + # Construct views INSIDE each invocation so Payload copying cannot remove aliasing. + storage = source.clone() + grad_output = storage[:6].view(2, 3) + if layout == "exact": + shared_output = grad_output + elif layout == "partial": + shared_output = storage[1:7].view(2, 3) + elif layout == "strided": + shared_output = storage[:6].view(3, 2).transpose(0, 1) + elif layout == "broadcast": + shared_output = storage[:3].view(1, 3) + else: + raise ValueError(f"Unknown alias layout: {layout}") + + independent_output = torch.full_like(grad_output, 17) + if target == "grad_a": + grad_a, grad_b = shared_output, independent_output + else: + grad_a, grad_b = independent_output, shared_output + + if implementation_index is None: + _torch_sub_backward(grad_output, alpha, grad_a, grad_b) + else: + _sub_backward( + grad_output, + alpha, + grad_a, + grad_b, + implementation_index=implementation_index, + ) + + # Include the whole backing buffer to detect writes outside the output view. + return torch.cat((grad_a.reshape(-1), grad_b.reshape(-1), storage)) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +@pytest.mark.parametrize( + "alpha, grad_value, grad_a_shape, grad_b_shape, expected_a, expected_b", + ( + pytest.param(2.0, 1.0, (2, 3), (2, 3), 1.0, -2.0, id="review-example"), + pytest.param(2.0, 1.0, (2, 3), (3,), 1.0, -4.0, id="scaled-reduction"), + pytest.param(0.0, 1.0, (2, 3), (3,), 1.0, 0.0, id="zero-alpha"), + pytest.param(-1.0, 1.0, (2, 3), (3,), 1.0, 2.0, id="negative-alpha"), + pytest.param(0.5, 1.0, (2, 3), (3,), 1.0, -1.0, id="fractional-alpha"), + pytest.param(2.0, 1.0, (1, 3), (2, 1), 2.0, -6.0, id="both-reduced"), + pytest.param( + 0.5, 40000.0, (2, 3), (2, 3), 40000.0, -20000.0, id="large-finite-gradient" + ), + pytest.param( + 0.0, 40000.0, (2, 3), (3,), 40000.0, 0.0, id="zero-alpha-before-reduction" + ), + ), +) +def test_sub_backward_known_values( + alpha, + grad_value, + grad_a_shape, + grad_b_shape, + expected_a, + expected_b, + implementation_index, + dtype, + device, +): + grad_output = torch.full((2, 3), grad_value, dtype=dtype, device=device) + grad_a = torch.full(grad_a_shape, 17.0, dtype=dtype, device=device) + grad_b = torch.full(grad_b_shape, 17.0, dtype=dtype, device=device) + expected = _pack_result( + grad_output, + torch.full_like(grad_a, expected_a), + torch.full_like(grad_b, expected_b), + ) + + return Payload( + lambda *args: _sub_backward( + *args, + implementation_index=implementation_index, + ), + lambda *args: expected.clone(), + (grad_output, alpha, grad_a, grad_b), + {}, + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("offset", (0, 2)) +@pytest.mark.parametrize("strides", ((3, 1), (6, 2))) +def test_sub_backward_rejects_output_overlap( + offset, strides, device, implementation_index +): + setup = f"device = torch.device({str(device)!r})\nx = torch.ones((2, 3), device=device)\n" + probe = setup + "infini.ops.SubBackward(x, 2.0, torch.empty(4, device=device), x)\n" + source = ( + setup + + f""" +backing = torch.ones(64, device=device) +a = backing.as_strided((2, 3), {strides!r}, 0) +b = backing.as_strided((2, 3), {strides!r}, {offset}) +infini.ops.sub_backward(x, 2.0, a, b, + stream=get_stream(device), implementation_index={implementation_index}) +""" + ) + # Run the constructor probe first; isolate C++ assertions from pytest. + prelude = ( + f"import sys; sys.path[:] = {json.dumps(sys.path)}\n" + "import infini.ops\nimport torch\n" + "from tests.utils import get_stream\n" + ) + + for code, expected_message in ( + (probe, None), + (source, "output storage must not overlap"), + ): + result = subprocess.run( + [sys.executable, "-c", prelude + code], + capture_output=True, + text=True, + errors="replace", + timeout=60, + ) + diagnostic = result.stdout + result.stderr + + if expected_message is None: + if result.returncode == 0: + pytest.skip( + "descriptor validation requires an assertions-enabled build" + ) + + assert "assert" in diagnostic.lower(), diagnostic + else: + assert result.returncode != 0, "Invalid parameters were accepted" + assert expected_message in diagnostic, diagnostic diff --git a/tests/test_transpose_backward.py b/tests/test_transpose_backward.py new file mode 100644 index 000000000..dfcc96300 --- /dev/null +++ b/tests/test_transpose_backward.py @@ -0,0 +1,156 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +# Format: (grad_input_shape, dim0, dim1, grad_output_strides, grad_input_strides) +_TEST_CASES = ( + pytest.param( + (2, 3), + 0, + 1, + None, + None, + id="two-dimensional", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + None, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param( + (2, 3, 4), + -1, + -2, + None, + None, + id="negative-dimensions", + ), + pytest.param( + (2, 3, 4), + 1, + 1, + None, + None, + id="same-dimension", + ), + pytest.param( + (2, 3, 4), + 0, + 2, + (10, 3, 1), + (20, 6, 1), + id="strided-gradients", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "grad_input_shape", + "dim0", + "dim1", + "grad_output_strides", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 0, 0), + (torch.float16, 0, 0), + (torch.bfloat16, 0, 0), + ), +) +def test_transpose_backward( + grad_input_shape, + dim0, + dim1, + grad_output_strides, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + grad_output_shape = list(grad_input_shape) + normalized_dim0 = dim0 % len(grad_input_shape) + normalized_dim1 = dim1 % len(grad_input_shape) + grad_output_shape[normalized_dim0], grad_output_shape[normalized_dim1] = ( + grad_output_shape[normalized_dim1], + grad_output_shape[normalized_dim0], + ) + grad_output = randn_strided( + tuple(grad_output_shape), + grad_output_strides, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + grad_input_shape, + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _transpose_backward( + *args, + implementation_index=implementation_index, + ), + _torch_transpose_backward, + (grad_output, dim0, dim1, grad_input), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + *, + implementation_index, +): + infini.ops.transpose_backward( + grad_output, + dim0, + dim1, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_transpose_backward(grad_output, dim0, dim1, grad_input): + grad_input.copy_(torch.transpose(grad_output, dim0, dim1)) + + return grad_input + + +@pytest.mark.parametrize("layout", ("contiguous", "strided")) +@pytest.mark.parametrize("offsets", ((0, 0), (0, 2), (2, 0))) +@pytest.mark.parametrize("dtype", (torch.float32, torch.float16, torch.bfloat16)) +def test_transpose_backward_alias(layout, offsets, dtype, device, implementation_index): + backing = torch.arange(64, dtype=torch.float32, device=device).to(dtype) + strides = (6, 2) if layout == "strided" else (3, 1) + grad_output = backing.as_strided((3, 3), strides, offsets[0]) + grad_input = backing.as_strided((3, 3), strides, offsets[1]) + expected = grad_output.transpose(0, 1).clone() + _transpose_backward( + grad_output, 0, 1, grad_input, implementation_index=implementation_index + ) + torch.testing.assert_close(grad_input, expected, rtol=0, atol=0) diff --git a/tests/test_transpose_copy.py b/tests/test_transpose_copy.py new file mode 100644 index 000000000..609d1edca --- /dev/null +++ b/tests/test_transpose_copy.py @@ -0,0 +1,416 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream, randn_strided + + +_DTYPES = (torch.float32, torch.float16, torch.bfloat16) + +_TEST_CASES = ( + pytest.param((2, 3), None, 0, 1, None, id="two-dimensional"), + pytest.param( + (2, 3, 4), + None, + 0, + 2, + None, + id="three-dimensional-outer-dimensions", + ), + pytest.param((2, 3, 4), None, -1, -2, None, id="negative-dimensions"), + pytest.param((2, 3, 4), None, 1, 1, None, id="same-dimension"), + pytest.param( + (2, 3, 4), + (20, 6, 1), + 0, + 2, + (10, 3, 1), + id="strided-input-and-output", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + tuple((dtype, 0, 0) for dtype in _DTYPES), +) +def test_transpose_copy( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + input = randn_strided( + input_shape, + input_strides, + dtype=dtype, + device=device, + ) + out = empty_strided( + _out_shape(input_shape, dim0, dim1), + out_strides, + dtype=dtype, + device=device, + ) + out.fill_(17) + + return Payload( + lambda *args: _transpose_copy( + *args, + implementation_index=implementation_index, + ), + _torch_transpose_copy, + (input, dim0, dim1, out), + {}, + rtol=rtol, + atol=atol, + ) + + +def _transpose_copy(input, dim0, dim1, out, *, implementation_index): + infini.ops.transpose_copy( + input, + dim0, + dim1, + out, + stream=get_stream(input.device), + implementation_index=implementation_index, + ) + return out + + +def _torch_transpose_copy(input, dim0, dim1, out): + result = torch.transpose_copy(input, dim0, dim1).clone() + out.copy_(result) + return out + + +def _out_shape(shape, dim0, dim1): + result = list(shape) + dim0 %= len(shape) + dim1 %= len(shape) + result[dim0], result[dim1] = result[dim1], result[dim0] + return tuple(result) + + +def _span(shape, strides): + return 1 + sum((size - 1) * stride for size, stride in zip(shape, strides)) + + +def _offsets(placement, input_span, output_span): + if placement == "same-start": + return 8, 8 + if placement == "partial-output-after": + return 8, 10 + if placement == "partial-output-before": + return 10, 8 + if placement == "adjacent-output-after": + return 8, 8 + input_span + if placement == "adjacent-output-before": + return 8 + output_span, 8 + if placement == "separated-output-after": + return 8, 8 + input_span + 5 + if placement == "separated-output-before": + return 8 + output_span + 5, 8 + raise AssertionError(placement) + + +def _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + *, + dtype, + device, + implementation_index, + repeats=1, + special_values=None, +): + + backing = torch.arange(256, dtype=torch.float32, device=device).to(dtype) + input = backing.as_strided(input_shape, input_strides, input_offset) + out = backing.as_strided( + _out_shape(input_shape, dim0, dim1), + out_strides, + output_offset, + ) + if special_values is not None: + input.copy_( + torch.tensor(special_values, dtype=dtype, device=device).reshape( + input_shape + ) + ) + + input_metadata = (tuple(input.shape), input.stride(), input.data_ptr()) + output_metadata = (tuple(out.shape), out.stride(), out.data_ptr()) + + for _ in range(repeats): + expected = torch.transpose_copy(input.detach().clone(), dim0, dim1).clone() + expected_backing = backing.clone() + expected_backing.as_strided( + out.shape, + out.stride(), + out.storage_offset(), + ).copy_(expected) + + _transpose_copy( + input, + dim0, + dim1, + out, + implementation_index=implementation_index, + ) + + torch.testing.assert_close(out, expected, rtol=0, atol=0, equal_nan=True) + torch.testing.assert_close( + backing, + expected_backing, + rtol=0, + atol=0, + equal_nan=True, + ) + if special_values is not None: + torch.testing.assert_close( + torch.signbit(backing), + torch.signbit(expected_backing), + rtol=0, + atol=0, + ) + assert (tuple(input.shape), input.stride(), input.data_ptr()) == input_metadata + assert (tuple(out.shape), out.stride(), out.data_ptr()) == output_metadata + + +_ALIAS_LAYOUTS = ( + pytest.param((3, 3), (3, 1), 0, 1, (3, 1), id="square"), + pytest.param((3, 3), (3, 1), -1, -2, (3, 1), id="square-negative-dims"), + pytest.param((3, 3), (1, 3), 0, 1, (3, 1), id="transpose_copyd-input"), + pytest.param((3, 3), (3, 1), 0, 1, (1, 3), id="transpose_copyd-output"), + pytest.param((3, 3), (8, 2), 0, 1, (9, 2), id="gapped-input-and-output"), + pytest.param((2, 3), (3, 1), 0, 1, (2, 1), id="rectangular"), + pytest.param((2, 3), (10, 2), 0, 1, (7, 2), id="rectangular-gapped"), + pytest.param( + (2, 3, 4), + (30, 7, 1), + 0, + 2, + (20, 5, 1), + id="three-dimensional", + ), + pytest.param( + (2, 3, 4), + (30, 7, 1), + -1, + -3, + (20, 5, 1), + id="three-dimensional-negative-dims", + ), + pytest.param( + (2, 3), + (1, 2), + 1, + 1, + (3, 1), + id="same-dim-different-layouts", + ), + pytest.param( + (2, 3), + (8, 2), + -1, + 1, + (9, 2), + id="same-dim-negative-equivalent", + ), + pytest.param((1, 4), (9, 2), 0, 1, (2, 11), id="singleton-dimension"), + pytest.param((7,), (2,), 0, -1, (3,), id="one-dimensional"), + pytest.param( + (2, 2, 3, 4), + (50, 22, 6, 1), + 0, + 3, + (18, 8, 2, 1), + id="four-dimensional", + ), +) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + _ALIAS_LAYOUTS, +) +@pytest.mark.parametrize( + "placement", + ( + "same-start", + "partial-output-after", + "partial-output-before", + "adjacent-output-after", + "adjacent-output-before", + "separated-output-after", + "separated-output-before", + ), +) +def test_transpose_copy_shared_storage( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + placement, + dtype, + device, + implementation_index, +): + input_offset, output_offset = _offsets( + placement, + _span(input_shape, input_strides), + _span(_out_shape(input_shape, dim0, dim1), out_strides), + ) + _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + dtype=dtype, + device=device, + implementation_index=implementation_index, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +def test_transpose_copy_review_regression(dtype, device, implementation_index): + input = torch.tensor( + [[1, 2, 3], [4, 5, 6], [7, 8, 9]], + dtype=dtype, + device=device, + ) + out = input.view_as(input) + + _transpose_copy(input, 0, 1, out, implementation_index=implementation_index) + + torch.testing.assert_close( + out, + torch.tensor( + [[1, 4, 7], [2, 5, 8], [3, 6, 9]], + dtype=dtype, + device=device, + ), + rtol=0, + atol=0, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize( + "input_shape, input_strides, dim0, dim1, out_strides", + ( + pytest.param((3, 3), (6, 2), 0, 1, (6, 2), id="square"), + pytest.param((2, 3), (6, 2), 0, 1, (4, 2), id="rectangular"), + pytest.param( + (2, 3, 4), + (24, 8, 2), + 0, + 2, + (12, 4, 2), + id="three-dimensional", + ), + ), +) +@pytest.mark.parametrize("reverse", (False, True)) +def test_transpose_copy_interleaved_views( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + reverse, + dtype, + device, + implementation_index, +): + + input_offset, output_offset = (9, 8) if reverse else (8, 9) + _run_alias_case( + input_shape, + input_strides, + dim0, + dim1, + out_strides, + input_offset, + output_offset, + dtype=dtype, + device=device, + implementation_index=implementation_index, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("offsets", ((8, 8), (8, 10), (10, 8))) +def test_transpose_copy_repeated_alias_calls( + offsets, + dtype, + device, + implementation_index, +): + _run_alias_case( + (3, 3), + (3, 1), + 0, + 1, + (3, 1), + *offsets, + dtype=dtype, + device=device, + implementation_index=implementation_index, + repeats=5, + ) + + +@pytest.mark.parametrize("dtype", _DTYPES) +@pytest.mark.parametrize("offsets", ((8, 8), (8, 10), (10, 8))) +def test_transpose_copy_alias_special_values( + offsets, + dtype, + device, + implementation_index, +): + _run_alias_case( + (3, 3), + (3, 1), + -1, + -2, + (3, 1), + *offsets, + dtype=dtype, + device=device, + implementation_index=implementation_index, + special_values=( + float("nan"), + float("inf"), + -float("inf"), + 0.0, + -0.0, + 1.0, + -2.0, + 0.25, + -0.5, + ), + ) diff --git a/tests/test_vocab_parallel_cross_entropy_backward.py b/tests/test_vocab_parallel_cross_entropy_backward.py new file mode 100644 index 000000000..5e658d1f8 --- /dev/null +++ b/tests/test_vocab_parallel_cross_entropy_backward.py @@ -0,0 +1,239 @@ +import infini.ops +import pytest +import torch + +from tests.utils import Payload, empty_strided, get_stream + + +# Format: (rows, vocab_size_local, vocab_size_original, label_smoothing, +# grad_output_mode, grad_input_strides) +_TEST_CASES = ( + pytest.param( + 3, + 4, + 6, + 0.0, + "scalar", + None, + id="scalar-grad-no-smoothing", + ), + pytest.param( + 3, + 4, + 8, + 0.1, + "per-row", + None, + id="per-row-grad-with-smoothing", + ), + pytest.param( + 4, + 5, + 8, + 0.2, + "scalar", + (28, 5), + id="strided-grad-input", + ), +) + + +@pytest.mark.auto_act_and_assert +@pytest.mark.parametrize( + ( + "rows", + "vocab_size_local", + "vocab_size_original", + "label_smoothing", + "grad_output_mode", + "grad_input_strides", + ), + _TEST_CASES, +) +@pytest.mark.parametrize( + ("dtype", "rtol", "atol"), + ( + (torch.float32, 1e-6, 1e-6), + (torch.float16, 2e-3, 2e-3), + (torch.bfloat16, 2e-2, 2e-2), + ), +) +def test_vocab_parallel_cross_entropy_backward( + rows, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_output_mode, + grad_input_strides, + implementation_index, + dtype, + device, + rtol, + atol, +): + softmax_local = torch.rand( + (rows, vocab_size_local), + dtype=dtype, + device=device, + ).softmax(dim=-1) + target_mask = torch.tensor( + [0, 1, 0, 0][:rows], + dtype=dtype, + device=device, + ) + masked_target = torch.tensor( + [1, 2, 0, 3][:rows], + dtype=torch.int64, + device=device, + ) + valid_mask_local = torch.tensor( + [1, 1, 0, 1, 1][:vocab_size_local], + dtype=dtype, + device=device, + ) + if grad_output_mode == "scalar": + grad_output = torch.tensor(1.25, dtype=dtype, device=device) + else: + grad_output = torch.linspace( + 0.5, + 1.5, + rows, + dtype=dtype, + device=device, + ) + grad_input = empty_strided( + (rows, vocab_size_local), + grad_input_strides, + dtype=dtype, + device=device, + ) + grad_input.fill_(17) + + return Payload( + lambda *args: _vocab_parallel_cross_entropy_backward( + *args, + implementation_index=implementation_index, + ), + _torch_vocab_parallel_cross_entropy_backward, + ( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + ), + {}, + rtol=rtol, + atol=atol, + ) + + +def _vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + *, + implementation_index, +): + infini.ops.vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, + stream=get_stream(grad_output.device), + implementation_index=implementation_index, + ) + + return grad_input + + +def _torch_vocab_parallel_cross_entropy_backward( + grad_output, + softmax_local, + target_mask, + masked_target, + valid_mask_local, + vocab_size_local, + vocab_size_original, + label_smoothing, + grad_input, +): + probabilities = softmax_local.detach().to(device="cpu", dtype=torch.float64) + rows = probabilities.shape[0] + dloss = grad_output.detach().to(device="cpu", dtype=torch.float64).reshape(-1) + if dloss.numel() == 1: + dloss = dloss.expand(rows) + + valid_rows = 1 - target_mask.detach().to(device="cpu", dtype=torch.float64) + valid_columns = ( + valid_mask_local.detach() + .to(device="cpu", dtype=torch.float64) + .reshape(1, vocab_size_local) + ) + distribution = (label_smoothing / vocab_size_original) * valid_columns.expand_as( + probabilities + ).clone() + target = masked_target.detach().to(device="cpu", dtype=torch.int64).reshape(rows, 1) + distribution.scatter_add_( + 1, target, ((1 - label_smoothing) * valid_rows).reshape(rows, 1) + ) + gradient = (probabilities - distribution) * dloss.reshape(rows, 1) * valid_columns + grad_input.copy_(gradient) + + return grad_input + + +@pytest.mark.parametrize( + "dtype", (torch.float16, torch.bfloat16, torch.float32, torch.float64) +) +@pytest.mark.parametrize("smoothing", (1e-4, 1e-8)) +@pytest.mark.parametrize("alias", (False, True)) +def test_vocab_parallel_cross_entropy_backward_small_smoothing( + dtype, smoothing, alias, device, implementation_index +): + probabilities = torch.tensor([[1.0, 0.0]], dtype=dtype, device=device) + upstream = torch.tensor(60000.0, dtype=dtype, device=device) + target_mask = torch.zeros(1, dtype=dtype, device=device) + target = torch.zeros(1, dtype=torch.int64, device=device) + valid = torch.ones(2, dtype=dtype, device=device) + output = probabilities if alias else torch.empty_like(probabilities) + expected = torch.empty_like(probabilities) + _torch_vocab_parallel_cross_entropy_backward( + upstream, probabilities, target_mask, target, valid, 2, 2, smoothing, expected + ) + _vocab_parallel_cross_entropy_backward( + upstream, + probabilities, + target_mask, + target, + valid, + 2, + 2, + smoothing, + output, + implementation_index=implementation_index, + ) + assert output[0, 0].item() > 0 + assert output[0, 1].item() < 0 + tolerance = { + torch.float16: 1e-3, + torch.bfloat16: 1e-2, + torch.float32: 1e-6, + torch.float64: 1e-7, + }[dtype] + torch.testing.assert_close(output, expected, rtol=tolerance, atol=0)