Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/linked/torch/metax/mcoplib.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
python_distribution_package: mcoplib
library_glob: mcoplib/_C*.so
2 changes: 2 additions & 0 deletions src/linked/torch/metax/mcoplib_moe.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
python_distribution_package: mcoplib
library_glob: mcoplib/_moe_C*.so
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ std::vector<at::Tensor> mha_varlen_fwd(
float dropout_p, float softmax_scale, bool zero_tensors, bool causal,
int window_size_left, int window_size_right, float softcap,
bool return_softmax, std::optional<at::Generator> generator,
std::optional<at::Tensor>& flash_attn_mars_ext);
std::optional<at::Tensor>& flash_attn_mars_ext, bool return_max_logit);

namespace infini::ops::linked::torch::metax {

Expand All @@ -36,7 +36,8 @@ struct FlashAttnVarlen : C10<Device::Type::kMetax> {
max_seqlen_k, dropout_p, softmax_scale,
zero_tensors, causal, window_size_left,
window_size_right, softcap, return_softmax,
generator, flash_attn_mars_ext);
generator, flash_attn_mars_ext,
/*return_max_logit=*/false);
}
};

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
library: flash_attn
required_symbols:
- >-
mha_varlen_fwd(at::Tensor&, at::Tensor const&, at::Tensor const&, std::optional<at::Tensor>&, at::Tensor const&, at::Tensor const&, std::optional<at::Tensor>&, std::optional<at::Tensor const>&, std::optional<at::Tensor>&, std::optional<at::Tensor>&, int, int, float, float, bool, bool, int, int, float, bool, std::optional<at::Generator>, std::optional<at::Tensor>&)
mha_varlen_fwd(at::Tensor&, at::Tensor const&, at::Tensor const&, std::optional<at::Tensor>&, at::Tensor const&, at::Tensor const&, std::optional<at::Tensor>&, std::optional<at::Tensor const>&, std::optional<at::Tensor>&, std::optional<at::Tensor>&, int, int, float, float, bool, bool, int, int, float, bool, std::optional<at::Generator>, std::optional<at::Tensor>&, bool)
8 changes: 8 additions & 0 deletions src/linked/torch/metax/ops/moe_align_block_size/vllm.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#include "linked/torch/metax/ops/moe_align_block_size/vllm.h"

namespace infini::ops::linked::torch {

template class TorchMoeAlignBlockSize<
::infini::ops::linked::torch::metax::VllmMoeAlignBlockSize>;

} // namespace infini::ops::linked::torch
66 changes: 66 additions & 0 deletions src/linked/torch/metax/ops/moe_align_block_size/vllm.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_
#define INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_

#include <ATen/core/dispatch/Dispatcher.h>
#include <ATen/core/stack.h>

#include <cassert>
#include <utility>

#include "linked/torch/ops/moe_align_block_size.h"
#include "torch/metax/c10.h"

namespace infini::ops::linked::torch::metax {

struct VllmMoeAlignBlockSize : C10<Device::Type::kMetax> {
static void Call(at::Tensor topk_ids, int64_t num_experts, int64_t block_size,
at::Tensor sorted_token_ids, at::Tensor experts_ids,
at::Tensor num_tokens_post_pad,
std::optional<at::Tensor> maybe_expert_map) {
static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow(
"_moe_C::moe_align_block_size", "");
c10::Stack stack;
stack.reserve(7);
stack.emplace_back(std::move(topk_ids));
stack.emplace_back(num_experts);
stack.emplace_back(block_size);
stack.emplace_back(std::move(sorted_token_ids));
stack.emplace_back(std::move(experts_ids));
stack.emplace_back(std::move(num_tokens_post_pad));
if (maybe_expert_map.has_value()) {
stack.emplace_back(std::move(*maybe_expert_map));
} else {
stack.emplace_back();
}
op.callBoxed(&stack);

assert(stack.empty() &&
"`moe_align_block_size` returned an unexpected number of values");
}
};

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

extern template class TorchMoeAlignBlockSize<
::infini::ops::linked::torch::metax::VllmMoeAlignBlockSize>;

} // namespace infini::ops::linked::torch

namespace infini::ops {

template <>
class Operator<MoeAlignBlockSize, Device::Type::kMetax, 16>
: public linked::torch::TorchMoeAlignBlockSize<
linked::torch::metax::VllmMoeAlignBlockSize> {
public:
using linked::torch::TorchMoeAlignBlockSize<
linked::torch::metax::VllmMoeAlignBlockSize>::TorchMoeAlignBlockSize;
using linked::torch::TorchMoeAlignBlockSize<
linked::torch::metax::VllmMoeAlignBlockSize>::operator();
};

} // namespace infini::ops

#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_MOE_ALIGN_BLOCK_SIZE_VLLM_H_
5 changes: 5 additions & 0 deletions src/linked/torch/metax/ops/moe_align_block_size/vllm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
library: mcoplib_moe
required_symbols:
- >-
moe_align_block_size(at::Tensor, long, long, at::Tensor, at::Tensor,
at::Tensor, std::optional<at::Tensor>)
7 changes: 7 additions & 0 deletions src/linked/torch/metax/ops/pad/aten.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#include "linked/torch/metax/ops/pad/aten.h"

namespace infini::ops::linked::torch {

template class TorchPad<::infini::ops::linked::torch::metax::AtenPad>;

} // namespace infini::ops::linked::torch
53 changes: 53 additions & 0 deletions src/linked/torch/metax/ops/pad/aten.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_
#define INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_

#include <ATen/core/dispatch/Dispatcher.h>
#include <ATen/core/stack.h>

#include <cassert>
#include <optional>
#include <vector>

#include "linked/torch/ops/pad.h"
#include "torch/metax/c10.h"

namespace infini::ops::linked::torch::metax {

struct AtenPad : C10<Device::Type::kMetax> {
static void Call(at::Tensor out, at::Tensor input, std::vector<int64_t> pad,
double value) {
static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow(
"aten::constant_pad_nd", "");
c10::Stack stack;
stack.reserve(4);
stack.emplace_back(std::move(input));
stack.emplace_back(c10::List<int64_t>(pad));
stack.emplace_back(value);
op.callBoxed(&stack);
TORCH_CHECK(stack.size() == 1,
"`constant_pad_nd` returned unexpected values.");
out.copy_(stack.front().toTensor());
}
};

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

extern template class TorchPad< ::infini::ops::linked::torch::metax::AtenPad>;

} // namespace infini::ops::linked::torch

namespace infini::ops {

template <>
class Operator<Pad, Device::Type::kMetax, 16>
: public linked::torch::TorchPad<linked::torch::metax::AtenPad> {
public:
using linked::torch::TorchPad<linked::torch::metax::AtenPad>::TorchPad;
using linked::torch::TorchPad<linked::torch::metax::AtenPad>::operator();
};

} // namespace infini::ops

#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_PAD_ATEN_H_
4 changes: 4 additions & 0 deletions src/linked/torch/metax/ops/pad/aten.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
library: mcoplib
operator_schema: >-
aten::constant_pad_nd(Tensor self, SymInt[] pad, Scalar value=0) -> Tensor
dispatch_key: CompositeExplicitAutograd
19 changes: 19 additions & 0 deletions src/linked/torch/metax/ops/quick_gelu/vllm.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#include "linked/torch/metax/ops/quick_gelu/vllm.h"

// MetaX vLLM `_C` exports this global C++ symbol.
void gelu_quick(at::Tensor& out, at::Tensor& input);

namespace infini::ops::linked::torch::metax {

void VllmQuickGelu::Call(at::Tensor& out, at::Tensor& input) {
::gelu_quick(out, input);
}

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

template class TorchQuickGelu<
::infini::ops::linked::torch::metax::VllmQuickGelu>;

} // namespace infini::ops::linked::torch
37 changes: 37 additions & 0 deletions src/linked/torch/metax/ops/quick_gelu/vllm.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_
#define INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_

#include "linked/torch/ops/quick_gelu.h"
#include "torch/metax/c10.h"

namespace infini::ops::linked::torch::metax {

struct VllmQuickGelu : C10<Device::Type::kMetax> {
static void Call(at::Tensor& out, at::Tensor& input);
};

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

extern template class TorchQuickGelu<
::infini::ops::linked::torch::metax::VllmQuickGelu>;

} // namespace infini::ops::linked::torch

namespace infini::ops {

template <>
class Operator<QuickGelu, Device::Type::kMetax, 16>
: public linked::torch::TorchQuickGelu<
linked::torch::metax::VllmQuickGelu> {
public:
using linked::torch::TorchQuickGelu<
linked::torch::metax::VllmQuickGelu>::TorchQuickGelu;
using linked::torch::TorchQuickGelu<
linked::torch::metax::VllmQuickGelu>::operator();
};

} // namespace infini::ops

#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_QUICK_GELU_VLLM_H_
3 changes: 3 additions & 0 deletions src/linked/torch/metax/ops/quick_gelu/vllm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
library: mcoplib
required_symbols:
- gelu_quick(at::Tensor&, at::Tensor&)
2 changes: 1 addition & 1 deletion src/linked/torch/metax/ops/rms_norm/vllm.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
library: vllm
library: mcoplib
required_symbols:
- rms_norm(at::Tensor&, at::Tensor&, at::Tensor&, double)
2 changes: 1 addition & 1 deletion src/linked/torch/metax/ops/silu_and_mul/vllm.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
library: vllm
library: mcoplib
required_symbols:
- silu_and_mul(at::Tensor&, at::Tensor&)
50 changes: 50 additions & 0 deletions src/linked/torch/metax/ops/topk_sigmoid/vllm.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#include "linked/torch/metax/ops/topk_sigmoid/vllm.h"

#include <ATen/core/dispatch/Dispatcher.h>
#include <ATen/core/stack.h>
#include <c10/util/Exception.h>

#include <utility>

namespace infini::ops::linked::torch::metax {

void VllmTopkSigmoid::Validate(bool has_is_padding,
double routed_scaling_factor) {
TORCH_CHECK(!has_is_padding,
"Linked `topk_sigmoid` does not support `is_padding`.");
TORCH_CHECK(routed_scaling_factor == 1.0,
"Linked `topk_sigmoid` requires "
"`routed_scaling_factor == 1.0`.");
}

void VllmTopkSigmoid::Call(at::Tensor topk_weights, at::Tensor topk_indices,
at::Tensor token_expert_indices,
at::Tensor gating_output, bool renormalize,
std::optional<at::Tensor> bias) {
static const auto op = c10::Dispatcher::singleton().findSchemaOrThrow(
"_moe_C::topk_sigmoid", "");
c10::Stack stack;
stack.emplace_back(std::move(topk_weights));
stack.emplace_back(std::move(topk_indices));
stack.emplace_back(std::move(token_expert_indices));
stack.emplace_back(std::move(gating_output));
stack.emplace_back(renormalize);
if (bias.has_value()) {
stack.emplace_back(std::move(*bias));
} else {
stack.emplace_back();
}
op.callBoxed(&stack);

TORCH_CHECK(stack.empty(),
"Linked `topk_sigmoid` returned unexpected values.");
}

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

template class TorchTopkSigmoid<
::infini::ops::linked::torch::metax::VllmTopkSigmoid>;

} // namespace infini::ops::linked::torch
43 changes: 43 additions & 0 deletions src/linked/torch/metax/ops/topk_sigmoid/vllm.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
#ifndef INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_
#define INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_

#include <optional>

#include "linked/torch/ops/topk_sigmoid.h"
#include "torch/metax/c10.h"

namespace infini::ops::linked::torch::metax {

struct VllmTopkSigmoid : C10<Device::Type::kMetax> {
static void Validate(bool has_is_padding, double routed_scaling_factor);

static void Call(at::Tensor topk_weights, at::Tensor topk_indices,
at::Tensor token_expert_indices, at::Tensor gating_output,
bool renormalize, std::optional<at::Tensor> bias);
};

} // namespace infini::ops::linked::torch::metax

namespace infini::ops::linked::torch {

extern template class TorchTopkSigmoid<
::infini::ops::linked::torch::metax::VllmTopkSigmoid>;

} // namespace infini::ops::linked::torch

namespace infini::ops {

template <>
class Operator<TopkSigmoid, Device::Type::kMetax, 16>
: public linked::torch::TorchTopkSigmoid<
linked::torch::metax::VllmTopkSigmoid> {
public:
using linked::torch::TorchTopkSigmoid<
linked::torch::metax::VllmTopkSigmoid>::TorchTopkSigmoid;
using linked::torch::TorchTopkSigmoid<
linked::torch::metax::VllmTopkSigmoid>::operator();
};

} // namespace infini::ops

#endif // INFINI_OPS_LINKED_TORCH_METAX_OPS_TOPK_SIGMOID_VLLM_H_
6 changes: 6 additions & 0 deletions src/linked/torch/metax/ops/topk_sigmoid/vllm.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
library: mcoplib_moe
operator_schema: >-
_moe_C::topk_sigmoid(Tensor! topk_weights, Tensor! topk_indices,
Tensor! token_expert_indices, Tensor gating_output, bool renormalize,
Tensor? bias) -> ()
dispatch_key: CUDA
2 changes: 2 additions & 0 deletions src/linked/torch/metax/vllm.yaml.disabled
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
python_distribution_package: vllm
library_glob: vllm/_C*.so
Loading
Loading