Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 75 additions & 0 deletions llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
#include "RISCVSubtarget.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/BinaryFormat/Dwarf.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/VirtRegMap.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/Module.h"
Expand All @@ -39,6 +41,15 @@ static cl::opt<bool>
cl::desc("Disable two address hints for register "
"allocation"));

// Vortex's operand collector reads source registers from four banks selected
// by the register number's low two bits; two sources in one bank cost an extra
// read cycle.
static cl::opt<bool> VortexBankHints(
"riscv-vortex-bank-hints", cl::Hidden, cl::init(true),
cl::desc("On XVortex, prefer registers in a different bank from the other "
"source registers of the instructions that read them"));
static constexpr unsigned VortexNumRegBanks = 4;

static_assert(RISCV::X1 == RISCV::X0 + 1, "Register list not consecutive");
static_assert(RISCV::X31 == RISCV::X0 + 31, "Register list not consecutive");
static_assert(RISCV::F1_H == RISCV::F0_H + 1, "Register list not consecutive");
Expand Down Expand Up @@ -1033,5 +1044,69 @@ bool RISCVRegisterInfo::getRegAllocationHints(
if (TwoAddrHints.count(OrderReg))
Hints.push_back(OrderReg);

if (VortexBankHints && !BaseImplRetVal && Subtarget.hasVendorXVortex())
addVortexBankHints(VirtReg, Order, Hints, MF, *VRM, Matrix);

return BaseImplRetVal;
}

static bool isVortexBankedReg(MCRegister Reg) {
return RISCV::GPRRegClass.contains(Reg) ||
RISCV::FPR16RegClass.contains(Reg) ||
RISCV::FPR32RegClass.contains(Reg) ||
RISCV::FPR64RegClass.contains(Reg);
}

void RISCVRegisterInfo::addVortexBankHints(Register VirtReg,
ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &Hints,
const MachineFunction &MF,
const VirtRegMap &VRM,
const LiveRegMatrix *Matrix) const {
if (Order.empty() || !isVortexBankedReg(Order.front()))
return;
const MachineRegisterInfo &MRI = MF.getRegInfo();

// Count, per bank, the already-assigned registers read alongside VirtReg.
unsigned Conflicts[VortexNumRegBanks] = {};
bool AnyConflict = false;
for (const MachineOperand &MO : MRI.use_nodbg_operands(VirtReg)) {
if (MO.getSubReg())
continue;
for (const MachineOperand &Src : MO.getParent()->explicit_uses()) {
if (!Src.isReg() || Src.getSubReg() || Src.getReg() == VirtReg)
continue;
Register Reg = Src.getReg();
MCRegister Phys = Reg.isPhysical() ? Reg.asMCReg() : VRM.getPhys(Reg);
// x0 is never read from the register file.
if (!Phys || Phys == RISCV::X0 || !isVortexBankedReg(Phys))
continue;
++Conflicts[getEncodingValue(Phys) % VortexNumRegBanks];
AnyConflict = true;
}
}
if (!AnyConflict)
return;

// Reorder only registers that cost nothing extra to take: a callee-saved
// register no other value uses yet costs a save and a restore, so those keep
// their place after the hints.
const MCPhysReg *CSRs = MRI.getCalleeSavedRegs();
auto IsCheap = [&](MCPhysReg Reg) {
if (Matrix && Matrix->isPhysRegUsed(Reg))
return true;
for (const MCPhysReg *CSR = CSRs; CSR && *CSR; ++CSR)
if (*CSR == Reg)
return false;
return true;
};
SmallVector<MCPhysReg, 32> Cheap;
for (MCPhysReg Reg : Order)
if (!MRI.isReserved(Reg) && !is_contained(Hints, Reg) && IsCheap(Reg))
Cheap.push_back(Reg);
llvm::stable_sort(Cheap, [&](MCPhysReg A, MCPhysReg B) {
return Conflicts[getEncodingValue(A) % VortexNumRegBanks] <
Conflicts[getEncodingValue(B) % VortexNumRegBanks];
});
Hints.append(Cheap.begin(), Cheap.end());
}
7 changes: 7 additions & 0 deletions llvm/lib/Target/RISCV/RISCVRegisterInfo.h
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,13 @@ struct RISCVRegisterInfo : public RISCVGenRegisterInfo {
const MachineFunction &MF, const VirtRegMap *VRM,
const LiveRegMatrix *Matrix) const override;

// Appends the free registers ordered so that VirtReg avoids the register
// banks of the other sources read with it.
void addVortexBankHints(Register VirtReg, ArrayRef<MCPhysReg> Order,
SmallVectorImpl<MCPhysReg> &Hints,
const MachineFunction &MF, const VirtRegMap &VRM,
const LiveRegMatrix *Matrix) const;

static bool isVRRegClass(const TargetRegisterClass *RC) {
return RISCVRI::isVRegClass(RC->TSFlags) &&
RISCVRI::getNF(RC->TSFlags) == 1;
Expand Down
67 changes: 67 additions & 0 deletions llvm/test/CodeGen/RISCV/xvortex-bank-hints.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=riscv32 -mattr=+f,+xvortex -verify-machineinstrs < %s \
; RUN: | FileCheck %s --check-prefix=HINTS
; RUN: llc -mtriple=riscv32 -mattr=+f,+xvortex -verify-machineinstrs \
; RUN: -riscv-vortex-bank-hints=false < %s | FileCheck %s --check-prefix=NOHINTS

; Vortex's operand collector reads an instruction's sources from four register
; banks (register number mod 4). With the hints, the loaded values land in
; banks apart from the other sources of the fmadd/fadd that read them.

define void @stencil(ptr %in, ptr %out, float %c0, float %c1) {
; HINTS-LABEL: stencil:
; HINTS: # %bb.0:
; HINTS-NEXT: flw fa5, 12(a0)
; HINTS-NEXT: flw fa3, 0(a0)
; HINTS-NEXT: flw fa2, 4(a0)
; HINTS-NEXT: flw ft0, 16(a0)
; HINTS-NEXT: flw fa4, 8(a0)
; HINTS-NEXT: flw ft1, 20(a0)
; HINTS-NEXT: fmadd.s fa2, fa3, fa0, fa2
; HINTS-NEXT: fmadd.s fa0, fa5, fa0, ft0
; HINTS-NEXT: fmadd.s fa5, fa4, fa1, fa2
; HINTS-NEXT: fmadd.s fa4, ft1, fa1, fa0
; HINTS-NEXT: fadd.s fa5, fa5, fa4
; HINTS-NEXT: fmul.s fa4, fa5, fa3
; HINTS-NEXT: fsw fa4, 0(a1)
; HINTS-NEXT: ret
;
; NOHINTS-LABEL: stencil:
; NOHINTS: # %bb.0:
; NOHINTS-NEXT: flw fa5, 12(a0)
; NOHINTS-NEXT: flw fa4, 0(a0)
; NOHINTS-NEXT: flw fa3, 4(a0)
; NOHINTS-NEXT: flw fa2, 16(a0)
; NOHINTS-NEXT: flw ft0, 8(a0)
; NOHINTS-NEXT: flw ft1, 20(a0)
; NOHINTS-NEXT: fmadd.s fa3, fa4, fa0, fa3
; NOHINTS-NEXT: fmadd.s fa5, fa5, fa0, fa2
; NOHINTS-NEXT: fmadd.s fa3, ft0, fa1, fa3
; NOHINTS-NEXT: fmadd.s fa5, ft1, fa1, fa5
; NOHINTS-NEXT: fadd.s fa5, fa3, fa5
; NOHINTS-NEXT: fmul.s fa5, fa5, fa4
; NOHINTS-NEXT: fsw fa5, 0(a1)
; NOHINTS-NEXT: ret
%p1 = getelementptr float, ptr %in, i32 1
%p2 = getelementptr float, ptr %in, i32 2
%p3 = getelementptr float, ptr %in, i32 3
%p4 = getelementptr float, ptr %in, i32 4
%p5 = getelementptr float, ptr %in, i32 5
%a = load float, ptr %in
%b = load float, ptr %p1
%c = load float, ptr %p2
%d = load float, ptr %p3
%e = load float, ptr %p4
%f = load float, ptr %p5
%s0 = call float @llvm.fma.f32(float %a, float %c0, float %b)
%s1 = call float @llvm.fma.f32(float %c, float %c1, float %s0)
%s2 = call float @llvm.fma.f32(float %d, float %c0, float %e)
%s3 = call float @llvm.fma.f32(float %f, float %c1, float %s2)
%s4 = fadd float %s1, %s3
%s5 = fmul float %s4, %a
store float %s5, ptr %out
ret void
}


declare float @llvm.fma.f32(float, float, float)
10 changes: 5 additions & 5 deletions llvm/test/CodeGen/RISCV/xvortex-group-bitcast.ll
Original file line number Diff line number Diff line change
Expand Up @@ -153,15 +153,15 @@ define void @f2i_sign(ptr %p, ptr %q, ptr %r) {
; RV64-NEXT: fsw fa5, 12(sp)
; RV64-NEXT: lw a1, 0(sp)
; RV64-NEXT: lw a3, 4(sp)
; RV64-NEXT: lw a4, 8(sp)
; RV64-NEXT: lw a5, 12(sp)
; RV64-NEXT: lw a5, 8(sp)
; RV64-NEXT: lw a6, 12(sp)
; RV64-NEXT: xor a1, a1, a0
; RV64-NEXT: xor a3, a3, a0
; RV64-NEXT: xor a4, a4, a0
; RV64-NEXT: xor a0, a5, a0
; RV64-NEXT: xor a5, a5, a0
; RV64-NEXT: xor a0, a6, a0
; RV64-NEXT: sw a1, 0(a2)
; RV64-NEXT: sw a3, 4(a2)
; RV64-NEXT: sw a4, 8(a2)
; RV64-NEXT: sw a5, 8(a2)
; RV64-NEXT: sw a0, 12(a2)
; RV64-NEXT: addi sp, sp, 16
; RV64-NEXT: .cfi_def_cfa_offset 0
Expand Down
4 changes: 2 additions & 2 deletions llvm/test/CodeGen/RISCV/xvortex-group-extract-subvector.ll
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ define void @v8i32_hi(ptr %p, ptr %q) {
; RV64-NEXT: add t2, a5, a5
; RV64-NEXT: add t3, a4, a4
; RV64-NEXT: add t4, a3, a3
; RV64-NEXT: add t5, a2, a2
; RV64-NEXT: add t6, a2, a2
; RV64-NEXT: add t1, t1, t1
; RV64-NEXT: add t0, t0, t0
; RV64-NEXT: add a7, a7, a7
Expand All @@ -205,7 +205,7 @@ define void @v8i32_hi(ptr %p, ptr %q) {
; RV64-NEXT: sw a7, 4(a0)
; RV64-NEXT: sw t0, 8(a0)
; RV64-NEXT: sw t1, 12(a0)
; RV64-NEXT: sw t5, 16(a0)
; RV64-NEXT: sw t6, 16(a0)
; RV64-NEXT: sw t4, 20(a0)
; RV64-NEXT: sw t3, 24(a0)
; RV64-NEXT: sw t2, 28(a0)
Expand Down
24 changes: 12 additions & 12 deletions llvm/test/CodeGen/RISCV/xvortex-group-i1-mem.ll
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@ define void @store_v4i1(ptr %p, <4 x i32> %a, <4 x i32> %b) {
; RV32-NEXT: lw a4, 4(a2)
; RV32-NEXT: lw a5, 8(a2)
; RV32-NEXT: lw a2, 12(a2)
; RV32-NEXT: lw a6, 12(a1)
; RV32-NEXT: lw a7, 8(a1)
; RV32-NEXT: lw t0, 0(a1)
; RV32-NEXT: lw a7, 12(a1)
; RV32-NEXT: lw a6, 8(a1)
; RV32-NEXT: lw t1, 0(a1)
; RV32-NEXT: lw a1, 4(a1)
; RV32-NEXT: slt a2, a6, a2
; RV32-NEXT: slt a5, a7, a5
; RV32-NEXT: slt a3, t0, a3
; RV32-NEXT: slt a2, a7, a2
; RV32-NEXT: slt a5, a6, a5
; RV32-NEXT: slt a3, t1, a3
; RV32-NEXT: slt a1, a1, a4
; RV32-NEXT: slli a2, a2, 3
; RV32-NEXT: slli a5, a5, 2
Expand All @@ -38,13 +38,13 @@ define void @store_v4i1(ptr %p, <4 x i32> %a, <4 x i32> %b) {
; RV64-NEXT: lw a4, 8(a2)
; RV64-NEXT: lw a5, 16(a2)
; RV64-NEXT: lw a2, 24(a2)
; RV64-NEXT: lw a6, 24(a1)
; RV64-NEXT: lw a7, 16(a1)
; RV64-NEXT: lw t0, 0(a1)
; RV64-NEXT: lw a7, 24(a1)
; RV64-NEXT: lw a6, 16(a1)
; RV64-NEXT: lw t1, 0(a1)
; RV64-NEXT: lw a1, 8(a1)
; RV64-NEXT: slt a2, a6, a2
; RV64-NEXT: slt a5, a7, a5
; RV64-NEXT: slt a3, t0, a3
; RV64-NEXT: slt a2, a7, a2
; RV64-NEXT: slt a5, a6, a5
; RV64-NEXT: slt a3, t1, a3
; RV64-NEXT: slt a1, a1, a4
; RV64-NEXT: slli a2, a2, 3
; RV64-NEXT: slli a5, a5, 2
Expand Down
8 changes: 4 additions & 4 deletions llvm/test/CodeGen/RISCV/xvortex-group-lanes.ll
Original file line number Diff line number Diff line change
Expand Up @@ -232,8 +232,8 @@ define void @insert_v4i32(ptr %p, i32 %s) {
; RV64-NEXT: lw zero, 4(a0)
; RV64-NEXT: lw a2, 0(a0)
; RV64-NEXT: lw a3, 8(a0)
; RV64-NEXT: lw a4, 12(a0)
; RV64-NEXT: sw a4, 12(a0)
; RV64-NEXT: lw a5, 12(a0)
; RV64-NEXT: sw a5, 12(a0)
; RV64-NEXT: sw a3, 8(a0)
; RV64-NEXT: sw a1, 4(a0)
; RV64-NEXT: sw a2, 0(a0)
Expand Down Expand Up @@ -312,8 +312,8 @@ define void @insert_v2i64(ptr %p, i64 %s) {
; RV32-NEXT: lw zero, 4(a0)
; RV32-NEXT: lw zero, 0(a0)
; RV32-NEXT: lw a3, 12(a0)
; RV32-NEXT: lw a4, 8(a0)
; RV32-NEXT: sw a4, 8(a0)
; RV32-NEXT: lw a5, 8(a0)
; RV32-NEXT: sw a5, 8(a0)
; RV32-NEXT: sw a3, 12(a0)
; RV32-NEXT: sw a1, 0(a0)
; RV32-NEXT: sw a2, 4(a0)
Expand Down
12 changes: 6 additions & 6 deletions llvm/test/CodeGen/RISCV/xvortex-wide-dag-merge.ll
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@ define void @copy4(ptr %d, ptr %s) {
; SCALAR-LABEL: copy4:
; SCALAR: # %bb.0:
; SCALAR-NEXT: flw fa5, 0(a1)
; SCALAR-NEXT: flw fa4, 4(a1)
; SCALAR-NEXT: flw fa3, 8(a1)
; SCALAR-NEXT: flw fa2, 12(a1)
; SCALAR-NEXT: flw fa3, 4(a1)
; SCALAR-NEXT: flw fa2, 8(a1)
; SCALAR-NEXT: flw fa1, 12(a1)
; SCALAR-NEXT: fsw fa5, 0(a0)
; SCALAR-NEXT: fsw fa4, 4(a0)
; SCALAR-NEXT: fsw fa3, 8(a0)
; SCALAR-NEXT: fsw fa2, 12(a0)
; SCALAR-NEXT: fsw fa3, 4(a0)
; SCALAR-NEXT: fsw fa2, 8(a0)
; SCALAR-NEXT: fsw fa1, 12(a0)
; SCALAR-NEXT: ret
%s1 = getelementptr inbounds float, ptr %s, i32 1
%s2 = getelementptr inbounds float, ptr %s, i32 2
Expand Down
38 changes: 19 additions & 19 deletions llvm/test/CodeGen/RISCV/xvortex-wide-lsv-llc.ll
Original file line number Diff line number Diff line change
Expand Up @@ -542,8 +542,8 @@ define float @local_float4_spill(ptr addrspace(3) align 16 %p) {
; VX-NEXT: flw ft9, 16(sp) # 4-byte Folded Reload
; VX-NEXT: flw ft10, 20(sp) # 4-byte Folded Reload
; VX-NEXT: flw ft11, 24(sp) # 4-byte Folded Reload
; VX-NEXT: fadd.s fa1, ft11, fs11
; VX-NEXT: fadd.s fa0, ft10, fs10
; VX-NEXT: fadd.s fa0, ft11, fs11
; VX-NEXT: fadd.s fa1, ft10, fs10
; VX-NEXT: fadd.s ft0, ft9, fs9
; VX-NEXT: fadd.s ft1, ft8, fs8
; VX-NEXT: flw ft8, 44(sp) # 4-byte Folded Reload
Expand All @@ -562,26 +562,26 @@ define float @local_float4_spill(ptr addrspace(3) align 16 %p) {
; VX-NEXT: flw ft9, 80(sp) # 4-byte Folded Reload
; VX-NEXT: flw ft10, 84(sp) # 4-byte Folded Reload
; VX-NEXT: flw ft11, 88(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs4, 60(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs5, 64(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs6, 68(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs7, 72(sp) # 4-byte Folded Reload
; VX-NEXT: fadd.s ft6, ft11, fs7
; VX-NEXT: fadd.s ft7, ft10, fs6
; VX-NEXT: fadd.s fa6, ft9, fs5
; VX-NEXT: fadd.s fa7, ft8, fs4
; VX-NEXT: flw fa6, 60(sp) # 4-byte Folded Reload
; VX-NEXT: flw fa7, 64(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs2, 68(sp) # 4-byte Folded Reload
; VX-NEXT: flw fs3, 72(sp) # 4-byte Folded Reload
; VX-NEXT: fadd.s ft7, ft11, fs3
; VX-NEXT: fadd.s ft6, ft10, fs2
; VX-NEXT: fadd.s fa7, ft9, fa7
; VX-NEXT: fadd.s fa6, ft8, fa6
; VX-NEXT: fmul.s fa2, fa2, ft1
; VX-NEXT: fmul.s fa3, fa3, ft0
; VX-NEXT: fmul.s ft0, ft5, fa7
; VX-NEXT: fmul.s ft1, ft4, fa6
; VX-NEXT: fsub.s fa3, fa3, ft1
; VX-NEXT: fsub.s fa2, fa2, ft0
; VX-NEXT: fmul.s ft1, ft5, fa6
; VX-NEXT: fmul.s ft0, ft4, fa7
; VX-NEXT: fsub.s fa3, fa3, ft0
; VX-NEXT: fsub.s fa2, fa2, ft1
; VX-NEXT: fadd.s fa3, fa2, fa3
; VX-NEXT: fmul.s fa4, fa4, fa0
; VX-NEXT: fmul.s fa5, fa5, fa1
; VX-NEXT: fmul.s fa2, ft3, ft7
; VX-NEXT: fmul.s fa1, ft2, ft6
; VX-NEXT: fsub.s fa5, fa5, fa1
; VX-NEXT: fmul.s fa4, fa4, fa1
; VX-NEXT: fmul.s fa5, fa5, fa0
; VX-NEXT: fmul.s fa2, ft3, ft6
; VX-NEXT: fmul.s fa0, ft2, ft7
; VX-NEXT: fsub.s fa5, fa5, fa0
; VX-NEXT: fsub.s fa4, fa4, fa2
; VX-NEXT: fadd.s fa5, fa4, fa5
; VX-NEXT: fadd.s fa0, fa3, fa5
Expand Down