diff --git a/ci/testcases/tensor_mx.yaml b/ci/testcases/tensor_mx.yaml index b8498676eb..1c4c68eded 100644 --- a/ci/testcases/tensor_mx.yaml +++ b/ci/testcases/tensor_mx.yaml @@ -263,6 +263,32 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI args: "-k32" +# Block scales far below 1 (-u): the first K block's products underflow fp32. +- id: sgemm_tcu_mx-mxfp8-nt8-underflow + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k64 -u" +- id: sgemm_tcu_mx-mxbf8-nt8-underflow + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxbf8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k64 -u" +- id: sgemm_tcu_mx-mxfp4-nt8-underflow + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE -DITYPE=mxfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k64 -u" + # SimX<->rtlsim cycle parity on the MX datapath: same config on both drivers, # cycles must agree within tolerance (see core.yaml). - id: model_parity-mxfp8 diff --git a/hw/rtl/tcu/dpi/VX_tcu_fedp_dpi.sv b/hw/rtl/tcu/dpi/VX_tcu_fedp_dpi.sv index 1d3b8c94da..890c7bdc10 100644 --- a/hw/rtl/tcu/dpi/VX_tcu_fedp_dpi.sv +++ b/hw/rtl/tcu/dpi/VX_tcu_fedp_dpi.sv @@ -50,15 +50,14 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #( `UNUSED_VAR (fmt_d); `ifdef VX_CFG_TCU_MX_ENABLE - function automatic [7:0] mx_scale_exp_sum( - input logic [7:0] scale_a, - input logic [7:0] scale_b - ); - logic [7:0] raw_scale_a, raw_scale_b; - begin - raw_scale_a = scale_a - 8'd127; - raw_scale_b = scale_b - 8'd127; - mx_scale_exp_sum = raw_scale_a + raw_scale_b; + // E8M0 scale as the fp32 value 2^(scale-127): 0 is subnormal, and 0xFF + // is NaN, so every element it scales is NaN + function automatic [63:0] mx_scale_f32(input logic [7:0] scale); + mx_scale_f32 = {32'hffffffff, 1'b0, scale, 23'h0}; + if (scale == 8'd0) begin + mx_scale_f32[31:0] = 32'h00400000; + end else if (scale == 8'hff) begin + mx_scale_f32[31:0] = 32'h7fc00000; end endfunction `endif @@ -136,13 +135,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #( if ((((i * 4 + j) * SF) / (N * 4)) == s) begin dpi_f2f(enable, int'(0), int'(4), {56'hffffffffffffff, a_row[i][j * 8 +: 8]}, 3'b0, a_f, fflags); dpi_f2f(enable, int'(0), int'(4), {56'hffffffffffffff, b_col[i][j * 8 +: 8]}, 3'b0, b_f, fflags); + dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags); + dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags); dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags); dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags); end end - if (group_prod[30:0] != 0) begin - group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]); - end dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags); end end @@ -154,13 +152,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #( if ((((i * 4 + j) * SF) / (N * 4)) == s) begin dpi_f2f(enable, int'(0), int'(5), {56'hffffffffffffff, a_row[i][j * 8 +: 8]}, 3'b0, a_f, fflags); dpi_f2f(enable, int'(0), int'(5), {56'hffffffffffffff, b_col[i][j * 8 +: 8]}, 3'b0, b_f, fflags); + dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags); + dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags); dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags); dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags); end end - if (group_prod[30:0] != 0) begin - group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]); - end dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags); end end @@ -175,13 +172,12 @@ module VX_tcu_fedp_dpi import VX_tcu_pkg::*; #( if ((((i * 8 + j) * SF) / (N * 8)) == s) begin dpi_f2f(enable, int'(0), int'(7), {60'hfffffffffffffff, a_row[i][j * 4 +: 4]}, 3'b0, a_f, fflags); dpi_f2f(enable, int'(0), int'(7), {60'hfffffffffffffff, b_col[i][j * 4 +: 4]}, 3'b0, b_f, fflags); + dpi_fmul(enable, int'(0), a_f, mx_scale_f32(sf_a[s]), 3'b0, a_f, fflags); + dpi_fmul(enable, int'(0), b_f, mx_scale_f32(sf_b[s]), 3'b0, b_f, fflags); dpi_fmul(enable, int'(0), a_f, b_f, 3'b0, temp, fflags); dpi_fadd(enable, int'(0), temp, group_prod, 3'b0, group_prod, fflags); end end - if (group_prod[30:0] != 0) begin - group_prod[30:23] = group_prod[30:23] + mx_scale_exp_sum(sf_a[s], sf_b[s]); - end dpi_fadd(enable, int'(0), group_prod, prod, 3'b0, prod, fflags); end end diff --git a/sw/common/softfloat_ext.cpp b/sw/common/softfloat_ext.cpp index fdb002def0..3c56b25a3e 100644 --- a/sw/common/softfloat_ext.cpp +++ b/sw/common/softfloat_ext.cpp @@ -929,13 +929,19 @@ tfloat32_t f32_to_tf32(float32_t a) { return res; } +// An E8M0 block scale, 2^(sf - 127). 0xFF is NaN, and so is every element it +// scales. +static float e8m0_scale(uint8_t sf) { + if (sf == 0xff) { + return std::numeric_limits::quiet_NaN(); + } + return std::ldexp(1.0f, static_cast(sf) - 127); +} + float32_t mxfp8_to_f32(mxfloat8_t a) { uint32_t fflags = 0; auto base_value = cvt_custom_to_f32(a.v, 4, 3, softfloat_roundingMode, &fflags); - // e8m0 scale factor: unbiased exponent (bias = 127) - int32_t scale_exp = (int32_t)a.sf - 127; - float scale_factor = std::ldexp(1.0f, scale_exp); - float out = base_value * scale_factor; + float out = base_value * e8m0_scale(a.sf); softfloat_exceptionFlags |= fflags; float32_t res; res.v = vortex::bit_cast(out); @@ -959,8 +965,7 @@ mxfloat8_t f32_to_mxfp8(float32_t a, sfexp8_t scale_factor) { float32_t mxbf8_to_f32(mxbfloat8_t a) { uint32_t fflags = 0; auto base_value = cvt_custom_to_f32(a.v, 5, 2, softfloat_roundingMode, &fflags); - int32_t scale_exp = static_cast(a.sf) - 127; - float out = base_value * std::ldexp(1.0f, scale_exp); + float out = base_value * e8m0_scale(a.sf); softfloat_exceptionFlags |= fflags; float32_t res; res.v = vortex::bit_cast(out); @@ -980,9 +985,8 @@ mxbfloat8_t f32_to_mxbf8(float32_t a, sfexp8_t scale_factor) { float32_t mxfp4_to_f32(mxfloat4_t a) { float4_t base = {static_cast(a.v & 0x0f)}; float base_value = vortex::bit_cast(f4e2m1_to_f32(base).v); - int32_t scale_exp = static_cast(a.sf) - 127; float32_t res; - res.v = vortex::bit_cast(base_value * std::ldexp(1.0f, scale_exp)); + res.v = vortex::bit_cast(base_value * e8m0_scale(a.sf)); return res; } diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index 83a566e5b3..83b6524704 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -200,6 +200,7 @@ const char *kernel_file = "kernel.vxbin"; uint32_t xm = 32; uint32_t xn = 32; uint32_t xk = 32; +bool tiny_block = false; vx_device_h device = nullptr; vx_queue_h queue = nullptr; @@ -218,16 +219,18 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex SGEMM TCU MX Test." << std::endl; - std::cout << "Usage: [-m M] [-n N] [-k K] [-h]" << std::endl; + std::cout << "Usage: [-m M] [-n N] [-k K] [-u] [-h]" << std::endl; + std::cout << " -u: scale the first K block of A and B by 2^-70 (its products underflow fp32)" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "m:n:k:h")) != -1) { + while ((c = getopt(argc, argv, "m:n:k:uh")) != -1) { switch (c) { case 'm': xm = atoi(optarg); break; case 'n': xn = atoi(optarg); break; case 'k': xk = atoi(optarg); break; + case 'u': tiny_block = true; break; case 'h': show_usage(); exit(0); @@ -332,6 +335,16 @@ int main(int argc, char *argv[]) { for (auto &v : h_B_dense) { v = (static_cast(std::rand()) / RAND_MAX) * 2.0f - 1.0f; } + if (tiny_block) { + for (uint32_t k = 0; k < vt::ITYPE::ele_block; ++k) { + for (uint32_t m = 0; m < M; ++m) { + h_A_dense[m * K_logical + k] *= 0x1p-70f; + } + for (uint32_t n = 0; n < N; ++n) { + h_B_dense[k * N + n] *= 0x1p-70f; + } + } + } std::vector h_A(sizeA); std::vector h_B(sizeB);