diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp index 7b7ad3f0ef03..c2d2fb4ab2ba 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp @@ -15,12 +15,14 @@ #include "RISCVSubtarget.h" #include "llvm/ADT/SmallSet.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/Module.h" @@ -39,6 +41,15 @@ static cl::opt cl::desc("Disable two address hints for register " "allocation")); +// Vortex's operand collector reads source registers from four banks selected +// by the register number's low two bits; two sources in one bank cost an extra +// read cycle. +static cl::opt VortexBankHints( + "riscv-vortex-bank-hints", cl::Hidden, cl::init(true), + cl::desc("On XVortex, prefer registers in a different bank from the other " + "source registers of the instructions that read them")); +static constexpr unsigned VortexNumRegBanks = 4; + static_assert(RISCV::X1 == RISCV::X0 + 1, "Register list not consecutive"); static_assert(RISCV::X31 == RISCV::X0 + 31, "Register list not consecutive"); static_assert(RISCV::F1_H == RISCV::F0_H + 1, "Register list not consecutive"); @@ -1033,5 +1044,69 @@ bool RISCVRegisterInfo::getRegAllocationHints( if (TwoAddrHints.count(OrderReg)) Hints.push_back(OrderReg); + if (VortexBankHints && !BaseImplRetVal && Subtarget.hasVendorXVortex()) + addVortexBankHints(VirtReg, Order, Hints, MF, *VRM, Matrix); + return BaseImplRetVal; } + +static bool isVortexBankedReg(MCRegister Reg) { + return RISCV::GPRRegClass.contains(Reg) || + RISCV::FPR16RegClass.contains(Reg) || + RISCV::FPR32RegClass.contains(Reg) || + RISCV::FPR64RegClass.contains(Reg); +} + +void RISCVRegisterInfo::addVortexBankHints(Register VirtReg, + ArrayRef Order, + SmallVectorImpl &Hints, + const MachineFunction &MF, + const VirtRegMap &VRM, + const LiveRegMatrix *Matrix) const { + if (Order.empty() || !isVortexBankedReg(Order.front())) + return; + const MachineRegisterInfo &MRI = MF.getRegInfo(); + + // Count, per bank, the already-assigned registers read alongside VirtReg. + unsigned Conflicts[VortexNumRegBanks] = {}; + bool AnyConflict = false; + for (const MachineOperand &MO : MRI.use_nodbg_operands(VirtReg)) { + if (MO.getSubReg()) + continue; + for (const MachineOperand &Src : MO.getParent()->explicit_uses()) { + if (!Src.isReg() || Src.getSubReg() || Src.getReg() == VirtReg) + continue; + Register Reg = Src.getReg(); + MCRegister Phys = Reg.isPhysical() ? Reg.asMCReg() : VRM.getPhys(Reg); + // x0 is never read from the register file. + if (!Phys || Phys == RISCV::X0 || !isVortexBankedReg(Phys)) + continue; + ++Conflicts[getEncodingValue(Phys) % VortexNumRegBanks]; + AnyConflict = true; + } + } + if (!AnyConflict) + return; + + // Reorder only registers that cost nothing extra to take: a callee-saved + // register no other value uses yet costs a save and a restore, so those keep + // their place after the hints. + const MCPhysReg *CSRs = MRI.getCalleeSavedRegs(); + auto IsCheap = [&](MCPhysReg Reg) { + if (Matrix && Matrix->isPhysRegUsed(Reg)) + return true; + for (const MCPhysReg *CSR = CSRs; CSR && *CSR; ++CSR) + if (*CSR == Reg) + return false; + return true; + }; + SmallVector Cheap; + for (MCPhysReg Reg : Order) + if (!MRI.isReserved(Reg) && !is_contained(Hints, Reg) && IsCheap(Reg)) + Cheap.push_back(Reg); + llvm::stable_sort(Cheap, [&](MCPhysReg A, MCPhysReg B) { + return Conflicts[getEncodingValue(A) % VortexNumRegBanks] < + Conflicts[getEncodingValue(B) % VortexNumRegBanks]; + }); + Hints.append(Cheap.begin(), Cheap.end()); +} diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h index 3ab79694e175..84346e6f2219 100644 --- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.h +++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.h @@ -132,6 +132,13 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo { const MachineFunction &MF, const VirtRegMap *VRM, const LiveRegMatrix *Matrix) const override; + // Appends the free registers ordered so that VirtReg avoids the register + // banks of the other sources read with it. + void addVortexBankHints(Register VirtReg, ArrayRef Order, + SmallVectorImpl &Hints, + const MachineFunction &MF, const VirtRegMap &VRM, + const LiveRegMatrix *Matrix) const; + static bool isVRRegClass(const TargetRegisterClass *RC) { return RISCVRI::isVRegClass(RC->TSFlags) && RISCVRI::getNF(RC->TSFlags) == 1; diff --git a/llvm/test/CodeGen/RISCV/xvortex-bank-hints.ll b/llvm/test/CodeGen/RISCV/xvortex-bank-hints.ll new file mode 100644 index 000000000000..d3cdc51b268f --- /dev/null +++ b/llvm/test/CodeGen/RISCV/xvortex-bank-hints.ll @@ -0,0 +1,67 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mtriple=riscv32 -mattr=+f,+xvortex -verify-machineinstrs < %s \ +; RUN: | FileCheck %s --check-prefix=HINTS +; RUN: llc -mtriple=riscv32 -mattr=+f,+xvortex -verify-machineinstrs \ +; RUN: -riscv-vortex-bank-hints=false < %s | FileCheck %s --check-prefix=NOHINTS + +; Vortex's operand collector reads an instruction's sources from four register +; banks (register number mod 4). With the hints, the loaded values land in +; banks apart from the other sources of the fmadd/fadd that read them. + +define void @stencil(ptr %in, ptr %out, float %c0, float %c1) { +; HINTS-LABEL: stencil: +; HINTS: # %bb.0: +; HINTS-NEXT: flw fa5, 12(a0) +; HINTS-NEXT: flw fa3, 0(a0) +; HINTS-NEXT: flw fa2, 4(a0) +; HINTS-NEXT: flw ft0, 16(a0) +; HINTS-NEXT: flw fa4, 8(a0) +; HINTS-NEXT: flw ft1, 20(a0) +; HINTS-NEXT: fmadd.s fa2, fa3, fa0, fa2 +; HINTS-NEXT: fmadd.s fa0, fa5, fa0, ft0 +; HINTS-NEXT: fmadd.s fa5, fa4, fa1, fa2 +; HINTS-NEXT: fmadd.s fa4, ft1, fa1, fa0 +; HINTS-NEXT: fadd.s fa5, fa5, fa4 +; HINTS-NEXT: fmul.s fa4, fa5, fa3 +; HINTS-NEXT: fsw fa4, 0(a1) +; HINTS-NEXT: ret +; +; NOHINTS-LABEL: stencil: +; NOHINTS: # %bb.0: +; NOHINTS-NEXT: flw fa5, 12(a0) +; NOHINTS-NEXT: flw fa4, 0(a0) +; NOHINTS-NEXT: flw fa3, 4(a0) +; NOHINTS-NEXT: flw fa2, 16(a0) +; NOHINTS-NEXT: flw ft0, 8(a0) +; NOHINTS-NEXT: flw ft1, 20(a0) +; NOHINTS-NEXT: fmadd.s fa3, fa4, fa0, fa3 +; NOHINTS-NEXT: fmadd.s fa5, fa5, fa0, fa2 +; NOHINTS-NEXT: fmadd.s fa3, ft0, fa1, fa3 +; NOHINTS-NEXT: fmadd.s fa5, ft1, fa1, fa5 +; NOHINTS-NEXT: fadd.s fa5, fa3, fa5 +; NOHINTS-NEXT: fmul.s fa5, fa5, fa4 +; NOHINTS-NEXT: fsw fa5, 0(a1) +; NOHINTS-NEXT: ret + %p1 = getelementptr float, ptr %in, i32 1 + %p2 = getelementptr float, ptr %in, i32 2 + %p3 = getelementptr float, ptr %in, i32 3 + %p4 = getelementptr float, ptr %in, i32 4 + %p5 = getelementptr float, ptr %in, i32 5 + %a = load float, ptr %in + %b = load float, ptr %p1 + %c = load float, ptr %p2 + %d = load float, ptr %p3 + %e = load float, ptr %p4 + %f = load float, ptr %p5 + %s0 = call float @llvm.fma.f32(float %a, float %c0, float %b) + %s1 = call float @llvm.fma.f32(float %c, float %c1, float %s0) + %s2 = call float @llvm.fma.f32(float %d, float %c0, float %e) + %s3 = call float @llvm.fma.f32(float %f, float %c1, float %s2) + %s4 = fadd float %s1, %s3 + %s5 = fmul float %s4, %a + store float %s5, ptr %out + ret void +} + + +declare float @llvm.fma.f32(float, float, float) diff --git a/llvm/test/CodeGen/RISCV/xvortex-group-bitcast.ll b/llvm/test/CodeGen/RISCV/xvortex-group-bitcast.ll index b7882526146b..6752398a6838 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-group-bitcast.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-group-bitcast.ll @@ -153,15 +153,15 @@ define void @f2i_sign(ptr %p, ptr %q, ptr %r) { ; RV64-NEXT: fsw fa5, 12(sp) ; RV64-NEXT: lw a1, 0(sp) ; RV64-NEXT: lw a3, 4(sp) -; RV64-NEXT: lw a4, 8(sp) -; RV64-NEXT: lw a5, 12(sp) +; RV64-NEXT: lw a5, 8(sp) +; RV64-NEXT: lw a6, 12(sp) ; RV64-NEXT: xor a1, a1, a0 ; RV64-NEXT: xor a3, a3, a0 -; RV64-NEXT: xor a4, a4, a0 -; RV64-NEXT: xor a0, a5, a0 +; RV64-NEXT: xor a5, a5, a0 +; RV64-NEXT: xor a0, a6, a0 ; RV64-NEXT: sw a1, 0(a2) ; RV64-NEXT: sw a3, 4(a2) -; RV64-NEXT: sw a4, 8(a2) +; RV64-NEXT: sw a5, 8(a2) ; RV64-NEXT: sw a0, 12(a2) ; RV64-NEXT: addi sp, sp, 16 ; RV64-NEXT: .cfi_def_cfa_offset 0 diff --git a/llvm/test/CodeGen/RISCV/xvortex-group-extract-subvector.ll b/llvm/test/CodeGen/RISCV/xvortex-group-extract-subvector.ll index fec32a516c4f..5c3f2c4abf6a 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-group-extract-subvector.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-group-extract-subvector.ll @@ -196,7 +196,7 @@ define void @v8i32_hi(ptr %p, ptr %q) { ; RV64-NEXT: add t2, a5, a5 ; RV64-NEXT: add t3, a4, a4 ; RV64-NEXT: add t4, a3, a3 -; RV64-NEXT: add t5, a2, a2 +; RV64-NEXT: add t6, a2, a2 ; RV64-NEXT: add t1, t1, t1 ; RV64-NEXT: add t0, t0, t0 ; RV64-NEXT: add a7, a7, a7 @@ -205,7 +205,7 @@ define void @v8i32_hi(ptr %p, ptr %q) { ; RV64-NEXT: sw a7, 4(a0) ; RV64-NEXT: sw t0, 8(a0) ; RV64-NEXT: sw t1, 12(a0) -; RV64-NEXT: sw t5, 16(a0) +; RV64-NEXT: sw t6, 16(a0) ; RV64-NEXT: sw t4, 20(a0) ; RV64-NEXT: sw t3, 24(a0) ; RV64-NEXT: sw t2, 28(a0) diff --git a/llvm/test/CodeGen/RISCV/xvortex-group-i1-mem.ll b/llvm/test/CodeGen/RISCV/xvortex-group-i1-mem.ll index 76e8eee203ed..637d201be17b 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-group-i1-mem.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-group-i1-mem.ll @@ -15,13 +15,13 @@ define void @store_v4i1(ptr %p, <4 x i32> %a, <4 x i32> %b) { ; RV32-NEXT: lw a4, 4(a2) ; RV32-NEXT: lw a5, 8(a2) ; RV32-NEXT: lw a2, 12(a2) -; RV32-NEXT: lw a6, 12(a1) -; RV32-NEXT: lw a7, 8(a1) -; RV32-NEXT: lw t0, 0(a1) +; RV32-NEXT: lw a7, 12(a1) +; RV32-NEXT: lw a6, 8(a1) +; RV32-NEXT: lw t1, 0(a1) ; RV32-NEXT: lw a1, 4(a1) -; RV32-NEXT: slt a2, a6, a2 -; RV32-NEXT: slt a5, a7, a5 -; RV32-NEXT: slt a3, t0, a3 +; RV32-NEXT: slt a2, a7, a2 +; RV32-NEXT: slt a5, a6, a5 +; RV32-NEXT: slt a3, t1, a3 ; RV32-NEXT: slt a1, a1, a4 ; RV32-NEXT: slli a2, a2, 3 ; RV32-NEXT: slli a5, a5, 2 @@ -38,13 +38,13 @@ define void @store_v4i1(ptr %p, <4 x i32> %a, <4 x i32> %b) { ; RV64-NEXT: lw a4, 8(a2) ; RV64-NEXT: lw a5, 16(a2) ; RV64-NEXT: lw a2, 24(a2) -; RV64-NEXT: lw a6, 24(a1) -; RV64-NEXT: lw a7, 16(a1) -; RV64-NEXT: lw t0, 0(a1) +; RV64-NEXT: lw a7, 24(a1) +; RV64-NEXT: lw a6, 16(a1) +; RV64-NEXT: lw t1, 0(a1) ; RV64-NEXT: lw a1, 8(a1) -; RV64-NEXT: slt a2, a6, a2 -; RV64-NEXT: slt a5, a7, a5 -; RV64-NEXT: slt a3, t0, a3 +; RV64-NEXT: slt a2, a7, a2 +; RV64-NEXT: slt a5, a6, a5 +; RV64-NEXT: slt a3, t1, a3 ; RV64-NEXT: slt a1, a1, a4 ; RV64-NEXT: slli a2, a2, 3 ; RV64-NEXT: slli a5, a5, 2 diff --git a/llvm/test/CodeGen/RISCV/xvortex-group-lanes.ll b/llvm/test/CodeGen/RISCV/xvortex-group-lanes.ll index 958860ac48d4..09a8799e84cc 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-group-lanes.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-group-lanes.ll @@ -232,8 +232,8 @@ define void @insert_v4i32(ptr %p, i32 %s) { ; RV64-NEXT: lw zero, 4(a0) ; RV64-NEXT: lw a2, 0(a0) ; RV64-NEXT: lw a3, 8(a0) -; RV64-NEXT: lw a4, 12(a0) -; RV64-NEXT: sw a4, 12(a0) +; RV64-NEXT: lw a5, 12(a0) +; RV64-NEXT: sw a5, 12(a0) ; RV64-NEXT: sw a3, 8(a0) ; RV64-NEXT: sw a1, 4(a0) ; RV64-NEXT: sw a2, 0(a0) @@ -312,8 +312,8 @@ define void @insert_v2i64(ptr %p, i64 %s) { ; RV32-NEXT: lw zero, 4(a0) ; RV32-NEXT: lw zero, 0(a0) ; RV32-NEXT: lw a3, 12(a0) -; RV32-NEXT: lw a4, 8(a0) -; RV32-NEXT: sw a4, 8(a0) +; RV32-NEXT: lw a5, 8(a0) +; RV32-NEXT: sw a5, 8(a0) ; RV32-NEXT: sw a3, 12(a0) ; RV32-NEXT: sw a1, 0(a0) ; RV32-NEXT: sw a2, 4(a0) diff --git a/llvm/test/CodeGen/RISCV/xvortex-wide-dag-merge.ll b/llvm/test/CodeGen/RISCV/xvortex-wide-dag-merge.ll index 82c57c784bc3..34cc933c63b3 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-wide-dag-merge.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-wide-dag-merge.ll @@ -20,13 +20,13 @@ define void @copy4(ptr %d, ptr %s) { ; SCALAR-LABEL: copy4: ; SCALAR: # %bb.0: ; SCALAR-NEXT: flw fa5, 0(a1) -; SCALAR-NEXT: flw fa4, 4(a1) -; SCALAR-NEXT: flw fa3, 8(a1) -; SCALAR-NEXT: flw fa2, 12(a1) +; SCALAR-NEXT: flw fa3, 4(a1) +; SCALAR-NEXT: flw fa2, 8(a1) +; SCALAR-NEXT: flw fa1, 12(a1) ; SCALAR-NEXT: fsw fa5, 0(a0) -; SCALAR-NEXT: fsw fa4, 4(a0) -; SCALAR-NEXT: fsw fa3, 8(a0) -; SCALAR-NEXT: fsw fa2, 12(a0) +; SCALAR-NEXT: fsw fa3, 4(a0) +; SCALAR-NEXT: fsw fa2, 8(a0) +; SCALAR-NEXT: fsw fa1, 12(a0) ; SCALAR-NEXT: ret %s1 = getelementptr inbounds float, ptr %s, i32 1 %s2 = getelementptr inbounds float, ptr %s, i32 2 diff --git a/llvm/test/CodeGen/RISCV/xvortex-wide-lsv-llc.ll b/llvm/test/CodeGen/RISCV/xvortex-wide-lsv-llc.ll index 45c174395215..69da84e268fe 100644 --- a/llvm/test/CodeGen/RISCV/xvortex-wide-lsv-llc.ll +++ b/llvm/test/CodeGen/RISCV/xvortex-wide-lsv-llc.ll @@ -542,8 +542,8 @@ define float @local_float4_spill(ptr addrspace(3) align 16 %p) { ; VX-NEXT: flw ft9, 16(sp) # 4-byte Folded Reload ; VX-NEXT: flw ft10, 20(sp) # 4-byte Folded Reload ; VX-NEXT: flw ft11, 24(sp) # 4-byte Folded Reload -; VX-NEXT: fadd.s fa1, ft11, fs11 -; VX-NEXT: fadd.s fa0, ft10, fs10 +; VX-NEXT: fadd.s fa0, ft11, fs11 +; VX-NEXT: fadd.s fa1, ft10, fs10 ; VX-NEXT: fadd.s ft0, ft9, fs9 ; VX-NEXT: fadd.s ft1, ft8, fs8 ; VX-NEXT: flw ft8, 44(sp) # 4-byte Folded Reload @@ -562,26 +562,26 @@ define float @local_float4_spill(ptr addrspace(3) align 16 %p) { ; VX-NEXT: flw ft9, 80(sp) # 4-byte Folded Reload ; VX-NEXT: flw ft10, 84(sp) # 4-byte Folded Reload ; VX-NEXT: flw ft11, 88(sp) # 4-byte Folded Reload -; VX-NEXT: flw fs4, 60(sp) # 4-byte Folded Reload -; VX-NEXT: flw fs5, 64(sp) # 4-byte Folded Reload -; VX-NEXT: flw fs6, 68(sp) # 4-byte Folded Reload -; VX-NEXT: flw fs7, 72(sp) # 4-byte Folded Reload -; VX-NEXT: fadd.s ft6, ft11, fs7 -; VX-NEXT: fadd.s ft7, ft10, fs6 -; VX-NEXT: fadd.s fa6, ft9, fs5 -; VX-NEXT: fadd.s fa7, ft8, fs4 +; VX-NEXT: flw fa6, 60(sp) # 4-byte Folded Reload +; VX-NEXT: flw fa7, 64(sp) # 4-byte Folded Reload +; VX-NEXT: flw fs2, 68(sp) # 4-byte Folded Reload +; VX-NEXT: flw fs3, 72(sp) # 4-byte Folded Reload +; VX-NEXT: fadd.s ft7, ft11, fs3 +; VX-NEXT: fadd.s ft6, ft10, fs2 +; VX-NEXT: fadd.s fa7, ft9, fa7 +; VX-NEXT: fadd.s fa6, ft8, fa6 ; VX-NEXT: fmul.s fa2, fa2, ft1 ; VX-NEXT: fmul.s fa3, fa3, ft0 -; VX-NEXT: fmul.s ft0, ft5, fa7 -; VX-NEXT: fmul.s ft1, ft4, fa6 -; VX-NEXT: fsub.s fa3, fa3, ft1 -; VX-NEXT: fsub.s fa2, fa2, ft0 +; VX-NEXT: fmul.s ft1, ft5, fa6 +; VX-NEXT: fmul.s ft0, ft4, fa7 +; VX-NEXT: fsub.s fa3, fa3, ft0 +; VX-NEXT: fsub.s fa2, fa2, ft1 ; VX-NEXT: fadd.s fa3, fa2, fa3 -; VX-NEXT: fmul.s fa4, fa4, fa0 -; VX-NEXT: fmul.s fa5, fa5, fa1 -; VX-NEXT: fmul.s fa2, ft3, ft7 -; VX-NEXT: fmul.s fa1, ft2, ft6 -; VX-NEXT: fsub.s fa5, fa5, fa1 +; VX-NEXT: fmul.s fa4, fa4, fa1 +; VX-NEXT: fmul.s fa5, fa5, fa0 +; VX-NEXT: fmul.s fa2, ft3, ft6 +; VX-NEXT: fmul.s fa0, ft2, ft7 +; VX-NEXT: fsub.s fa5, fa5, fa0 ; VX-NEXT: fsub.s fa4, fa4, fa2 ; VX-NEXT: fadd.s fa5, fa4, fa5 ; VX-NEXT: fadd.s fa0, fa3, fa5