Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions ci/testcases/tensor_mx.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,32 @@ tests:
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI
args: "-k32"

# Block scales far below 1 (-u): the first K block's products underflow fp32.
- id: sgemm_tcu_mx-mxfp8-nt8-underflow
via: blackbox
drivers:
- simx
- rtlsim
app: sgemm_tcu_mx
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI
args: "-k64 -u"
- id: sgemm_tcu_mx-mxbf8-nt8-underflow
via: blackbox
drivers:
- simx
- rtlsim
app: sgemm_tcu_mx
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxbf8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI
args: "-k64 -u"
- id: sgemm_tcu_mx-mxfp4-nt8-underflow
via: blackbox
drivers:
- simx
- rtlsim
app: sgemm_tcu_mx
configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE -DITYPE=mxfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI
args: "-k64 -u"

# SimX<->rtlsim cycle parity on the MX datapath: same config on both drivers,
# cycles must agree within tolerance (see core.yaml).
- id: model_parity-mxfp8
Expand Down
32 changes: 14 additions & 18 deletions hw/rtl/tcu/dpi/VX_tcu_fedp_dpi.sv
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,14 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #(
`UNUSED_VAR (fmt_d);

`ifdef VX_CFG_TCU_MX_ENABLE
function automatic [7:0] mx_scale_exp_sum(
input logic [7:0] scale_a,
input logic [7:0] scale_b
);
logic [7:0] raw_scale_a, raw_scale_b;
begin
raw_scale_a = scale_a - 8'd127;
raw_scale_b = scale_b - 8'd127;
mx_scale_exp_sum = raw_scale_a + raw_scale_b;
// E8M0 scale as the fp32 value 2^(scale-127): 0 is subnormal, and 0xFF
// is NaN, so every element it scales is NaN
function automatic [63:0] mx_scale_f32(input logic [7:0] scale);
mx_scale_f32 = {32'hffffffff, 1'b0, scale, 23'h0};
if (scale == 8'd0) begin
mx_scale_f32[31:0] = 32'h00400000;
end else if (scale == 8'hff) begin
mx_scale_f32[31:0] = 32'h7fc00000;
end
endfunction
`endif
Expand Down Expand Up @@ -136,13 +135,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #(
if ((((i * 4 + j) * SF) / (N * 4)) == s) begin
dpi_f2f(enable, int'(0), int'(4), {56'hffffffffffffff, a_row[i][j * 8 +: 8]}, 3'b0, a_f, fflags);
dpi_f2f(enable, int'(0), int'(4), {56'hffffffffffffff, b_col[i][j * 8 +: 8]}, 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags);
dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags);
dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags);
end
end
if (group_prod[30:0] != 0) begin
group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]);
end
dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags);
end
end
Expand All @@ -154,13 +152,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #(
if ((((i * 4 + j) * SF) / (N * 4)) == s) begin
dpi_f2f(enable, int'(0), int'(5), {56'hffffffffffffff, a_row[i][j * 8 +: 8]}, 3'b0, a_f, fflags);
dpi_f2f(enable, int'(0), int'(5), {56'hffffffffffffff, b_col[i][j * 8 +: 8]}, 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags);
dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags);
dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags);
end
end
if (group_prod[30:0] != 0) begin
group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]);
end
dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags);
end
end
Expand All @@ -175,13 +172,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #(
if ((((i * 8 + j) * SF) / (N * 8)) == s) begin
dpi_f2f(enable, int'(0), int'(7), {60'hfffffffffffffff, a_row[i][j * 4 +: 4]}, 3'b0, a_f, fflags);
dpi_f2f(enable, int'(0), int'(7), {60'hfffffffffffffff, b_col[i][j * 4 +: 4]}, 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags);
dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags);
dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags);
dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags);
end
end
if (group_prod[30:0] != 0) begin
group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]);
end
dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags);
end
end
Expand Down
20 changes: 12 additions & 8 deletions sw/common/softfloat_ext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -929,13 +929,19 @@ tfloat32_t f32_to_tf32(float32_t a) {
return res;
}

// An E8M0 block scale, 2^(sf - 127). 0xFF is NaN, and so is every element it
// scales.
static float e8m0_scale(uint8_t sf) {
if (sf == 0xff) {
return std::numeric_limits<float>::quiet_NaN();
}
return std::ldexp(1.0f, static_cast<int32_t>(sf) - 127);
}

float32_t mxfp8_to_f32(mxfloat8_t a) {
uint32_t fflags = 0;
auto base_value = cvt_custom_to_f32(a.v, 4, 3, softfloat_roundingMode, &fflags);
// e8m0 scale factor: unbiased exponent (bias = 127)
int32_t scale_exp = (int32_t)a.sf - 127;
float scale_factor = std::ldexp(1.0f, scale_exp);
float out = base_value * scale_factor;
float out = base_value * e8m0_scale(a.sf);
softfloat_exceptionFlags |= fflags;
float32_t res;
res.v = vortex::bit_cast<uint32_t>(out);
Expand All @@ -959,8 +965,7 @@ mxfloat8_t f32_to_mxfp8(float32_t a, sfexp8_t scale_factor) {
float32_t mxbf8_to_f32(mxbfloat8_t a) {
uint32_t fflags = 0;
auto base_value = cvt_custom_to_f32(a.v, 5, 2, softfloat_roundingMode, &fflags);
int32_t scale_exp = static_cast<int32_t>(a.sf) - 127;
float out = base_value * std::ldexp(1.0f, scale_exp);
float out = base_value * e8m0_scale(a.sf);
softfloat_exceptionFlags |= fflags;
float32_t res;
res.v = vortex::bit_cast<uint32_t>(out);
Expand All @@ -980,9 +985,8 @@ mxbfloat8_t f32_to_mxbf8(float32_t a, sfexp8_t scale_factor) {
float32_t mxfp4_to_f32(mxfloat4_t a) {
float4_t base = {static_cast<uint8_t>(a.v & 0x0f)};
float base_value = vortex::bit_cast<float>(f4e2m1_to_f32(base).v);
int32_t scale_exp = static_cast<int32_t>(a.sf) - 127;
float32_t res;
res.v = vortex::bit_cast<uint32_t>(base_value * std::ldexp(1.0f, scale_exp));
res.v = vortex::bit_cast<uint32_t>(base_value * e8m0_scale(a.sf));
return res;
}

Expand Down
17 changes: 15 additions & 2 deletions tests/regression/sgemm_tcu_mx/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ const char *kernel_file = "kernel.vxbin";
uint32_t xm = 32;
uint32_t xn = 32;
uint32_t xk = 32;
bool tiny_block = false;

vx_device_h device = nullptr;
vx_queue_h queue = nullptr;
Expand All @@ -218,16 +219,18 @@ kernel_arg_t kernel_arg = {};

static void show_usage() {
std::cout << "Vortex SGEMM TCU MX Test." << std::endl;
std::cout << "Usage: [-m M] [-n N] [-k K] [-h]" << std::endl;
std::cout << "Usage: [-m M] [-n N] [-k K] [-u] [-h]" << std::endl;
std::cout << " -u: scale the first K block of A and B by 2^-70 (its products underflow fp32)" << std::endl;
}

static void parse_args(int argc, char **argv) {
int c;
while ((c = getopt(argc, argv, "m:n:k:h")) != -1) {
while ((c = getopt(argc, argv, "m:n:k:uh")) != -1) {
switch (c) {
case 'm': xm = atoi(optarg); break;
case 'n': xn = atoi(optarg); break;
case 'k': xk = atoi(optarg); break;
case 'u': tiny_block = true; break;
case 'h':
show_usage();
exit(0);
Expand Down Expand Up @@ -332,6 +335,16 @@ int main(int argc, char *argv[]) {
for (auto &v : h_B_dense) {
v = (static_cast<float>(std::rand()) / RAND_MAX) * 2.0f - 1.0f;
}
if (tiny_block) {
for (uint32_t k = 0; k < vt::ITYPE::ele_block; ++k) {
for (uint32_t m = 0; m < M; ++m) {
h_A_dense[m * K_logical + k] *= 0x1p-70f;
}
for (uint32_t n = 0; n < N; ++n) {
h_B_dense[k * N + n] *= 0x1p-70f;
}
}
}

std::vector<itype_t> h_A(sizeA);
std::vector<itype_t> h_B(sizeB);
Expand Down
Loading