[llvm] [RISCV] RISC-V split register allocation and move vsetvl pass in between (PR #70549)
Piyou Chen via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 28 04:29:58 PDT 2023
https://github.com/BeMg created https://github.com/llvm/llvm-project/pull/70549
This patch try to get rid of vsetvl implict vl/vtype def-use chain and improve the register allocation quality by
1. move the vsetvl insertion pass after RVV register allocation
2. split register allocation into RVV and other
It will gain the benefit for the following optimization from
1. unblock scheduler's constraints by removing vl/vtype def-use chain
2. Support RVV re-materialization
3. Support partial spill
This patch add a new option `-riscv-split-RA=<1|0>` to control this feature and default set as disable.
It can compile the the local testsuite and some relate big benchmark, but it still generate worse code in some case. (llvm regression test)
---
# Note
## Register allocation
- Allocator be split by pass the filter function to control which VReg should be allocated.
## VSETVL insertion pass
- Because it no longer SSA form, there are some place need to update.
- The LiveInterval need to maintain the correctness because this pass will insert/remove instruction.
- LiveDebugVariables also should to perserve, otherwise it will trigger the assertion fail.
>From 19f36869db25f6ac0f4af71930f93eab2e3e5da3 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Fri, 27 Oct 2023 01:49:40 -0700
Subject: [PATCH 1/4] [RISCV] Support basic version split regalloc between RVV
and other
Enable this flow by -riscv-split-RA=1 (default disable)
Use the RegClass filter function to control the register allocator.
---
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 105 +++++++++++++++++++
1 file changed, 105 insertions(+)
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 953ac097b915044..92150ed4c224366 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -27,6 +27,7 @@
#include "llvm/CodeGen/MIRParser/MIParser.h"
#include "llvm/CodeGen/MIRYamlMapping.h"
#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegAllocRegistry.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/InitializePasses.h"
@@ -83,6 +84,10 @@ static cl::opt<bool>
cl::desc("Enable sinking and folding of instruction copies"),
cl::init(false), cl::Hidden);
+static cl::opt<bool> EnableSplitRA("riscv-split-RA", cl::Hidden,
+ cl::desc("Enable Split RA for RVV"),
+ cl::init(false));
+
extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
@@ -245,6 +250,74 @@ bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
}
namespace {
+
+class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
+public:
+ RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
+ : RegisterRegAllocBase(N, D, C) {}
+};
+
+static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
+ const TargetRegisterClass &RC) {
+ return RISCV::VRRegClass.hasSubClassEq(&RC) ||
+ RISCV::VRM2RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRM4RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRM8RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN2M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN2M2RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN2M4RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN3M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN3M2RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN4M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN4M2RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN5M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN6M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN7M1RegClass.hasSubClassEq(&RC) ||
+ RISCV::VRN8M1RegClass.hasSubClassEq(&RC);
+}
+
+static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
+
+static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
+
+/// -riscv-splitRA-rvv-regalloc=... command line option.
+static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
+ RegisterPassParser<RVVRegisterRegAlloc>>
+ RVVRegAlloc("riscv-splitRA-rvv-regalloc", cl::Hidden,
+ cl::init(&useDefaultRegisterAllocator),
+ cl::desc("Register allocator to use for RVV register."));
+
+static void initializeDefaultRVVRegisterAllocatorOnce() {
+ RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
+
+ if (!Ctor) {
+ Ctor = RVVRegAlloc;
+ RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
+ }
+}
+
+static FunctionPass *createBasicRVVRegisterAllocator() {
+ return createBasicRegisterAllocator(onlyAllocateRVVReg);
+}
+
+static FunctionPass *createGreedyRVVRegisterAllocator() {
+ return createGreedyRegisterAllocator(onlyAllocateRVVReg);
+}
+
+static FunctionPass *createFastRVVRegisterAllocator() {
+ return createFastRegisterAllocator(onlyAllocateRVVReg, false);
+}
+
+static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
+ "basic register allocator",
+ createBasicRVVRegisterAllocator);
+static RVVRegisterRegAlloc
+ greedyRegAllocRVVReg("greedy", "greedy register allocator",
+ createGreedyRVVRegisterAllocator);
+
+static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
+ createFastRVVRegisterAllocator);
+
class RISCVPassConfig : public TargetPassConfig {
public:
RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
@@ -293,6 +366,9 @@ class RISCVPassConfig : public TargetPassConfig {
void addPreEmitPass2() override;
void addPreSched2() override;
void addMachineSSAOptimization() override;
+ FunctionPass *createRVVRegAllocPass(bool Optimized);
+ bool addRegAssignAndRewriteFast() override;
+ bool addRegAssignAndRewriteOptimized() override;
void addPreRegAlloc() override;
void addPostRegAlloc() override;
void addOptimizedRegAlloc() override;
@@ -304,6 +380,35 @@ TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
return new RISCVPassConfig(*this, PM);
}
+FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
+ // Initialize the global default.
+ llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
+ initializeDefaultRVVRegisterAllocatorOnce);
+
+ RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
+ if (Ctor != useDefaultRegisterAllocator)
+ return Ctor();
+
+ if (Optimized)
+ return createGreedyRVVRegisterAllocator();
+
+ return createFastRVVRegisterAllocator();
+}
+
+bool RISCVPassConfig::addRegAssignAndRewriteFast() {
+ if (EnableSplitRA)
+ addPass(createRVVRegAllocPass(false));
+ return TargetPassConfig::addRegAssignAndRewriteFast();
+}
+
+bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
+ if (EnableSplitRA) {
+ addPass(createRVVRegAllocPass(true));
+ addPass(createVirtRegRewriter(false));
+ }
+ return TargetPassConfig::addRegAssignAndRewriteOptimized();
+}
+
void RISCVPassConfig::addIRPasses() {
addPass(createAtomicExpandPass());
>From b123472c2e6cbe709265e58e61cbb53f41257a8a Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Fri, 27 Oct 2023 05:02:41 -0700
Subject: [PATCH 2/4] [RISCV] Move LiveDebugVariables into
llvm/include/llvm/CodeGen
---
llvm/{lib => include/llvm}/CodeGen/LiveDebugVariables.h | 0
llvm/lib/CodeGen/LiveDebugVariables.cpp | 2 +-
llvm/lib/CodeGen/RegAllocBasic.cpp | 2 +-
llvm/lib/CodeGen/RegAllocGreedy.cpp | 2 +-
llvm/lib/CodeGen/VirtRegMap.cpp | 2 +-
5 files changed, 4 insertions(+), 4 deletions(-)
rename llvm/{lib => include/llvm}/CodeGen/LiveDebugVariables.h (100%)
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.h b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
similarity index 100%
rename from llvm/lib/CodeGen/LiveDebugVariables.h
rename to llvm/include/llvm/CodeGen/LiveDebugVariables.h
diff --git a/llvm/lib/CodeGen/LiveDebugVariables.cpp b/llvm/lib/CodeGen/LiveDebugVariables.cpp
index 9603c1f01e08569..5f5a858c7bade6e 100644
--- a/llvm/lib/CodeGen/LiveDebugVariables.cpp
+++ b/llvm/lib/CodeGen/LiveDebugVariables.cpp
@@ -18,7 +18,7 @@
//
//===----------------------------------------------------------------------===//
-#include "LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/IntervalMap.h"
diff --git a/llvm/lib/CodeGen/RegAllocBasic.cpp b/llvm/lib/CodeGen/RegAllocBasic.cpp
index 6661991396302ec..a4d4a218fea5584 100644
--- a/llvm/lib/CodeGen/RegAllocBasic.cpp
+++ b/llvm/lib/CodeGen/RegAllocBasic.cpp
@@ -12,7 +12,7 @@
//===----------------------------------------------------------------------===//
#include "AllocationOrder.h"
-#include "LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "RegAllocBase.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/CalcSpillWeights.h"
diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp
index 349d8b0975f3a10..4c25fcfeba89073 100644
--- a/llvm/lib/CodeGen/RegAllocGreedy.cpp
+++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp
@@ -14,7 +14,7 @@
#include "RegAllocGreedy.h"
#include "AllocationOrder.h"
#include "InterferenceCache.h"
-#include "LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "RegAllocBase.h"
#include "RegAllocEvictionAdvisor.h"
#include "RegAllocPriorityAdvisor.h"
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index 48f4ee29fbe95d4..a2b046a59c9d773 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -16,7 +16,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/VirtRegMap.h"
-#include "LiveDebugVariables.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/LiveInterval.h"
>From 1d3de4bc6a8dcd40c338164cef5c18e62f809d20 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Fri, 27 Oct 2023 05:03:17 -0700
Subject: [PATCH 3/4] [RISCV] Remove LLVM_LIBRARY_VISIBILITY in
LiveDebugVariables
---
llvm/include/llvm/CodeGen/LiveDebugVariables.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/LiveDebugVariables.h b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
index 9998ce9e8dad861..3643a2cd0981f19 100644
--- a/llvm/include/llvm/CodeGen/LiveDebugVariables.h
+++ b/llvm/include/llvm/CodeGen/LiveDebugVariables.h
@@ -29,7 +29,7 @@ template <typename T> class ArrayRef;
class LiveIntervals;
class VirtRegMap;
-class LLVM_LIBRARY_VISIBILITY LiveDebugVariables : public MachineFunctionPass {
+class LiveDebugVariables : public MachineFunctionPass {
void *pImpl = nullptr;
public:
>From 30122916a6e0e732984e941b89ebbddd056e32c8 Mon Sep 17 00:00:00 2001
From: Piyou Chen <piyou.chen at sifive.com>
Date: Fri, 27 Oct 2023 09:01:08 -0700
Subject: [PATCH 4/4] [RISCV] Support vsetvl pass for splitRA
During the splitRA situation, it need to
1. Maintain the LiveInterval for the following register allocation stuff
2. Due to exit SSA, we can't use the getVRegDef to retrieve reaching definition
---
llvm/lib/Target/RISCV/RISCV.h | 2 +-
llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 252 +++++++++++++++---
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 8 +-
llvm/test/CodeGen/RISCV/rvv/splitRA-vsetvl.ll | 116 ++++++++
4 files changed, 331 insertions(+), 47 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/rvv/splitRA-vsetvl.ll
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index 3d8e33dc716ea44..9c9906ff0bc6430 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -60,7 +60,7 @@ void initializeRISCVPreRAExpandPseudoPass(PassRegistry &);
FunctionPass *createRISCVExpandAtomicPseudoPass();
void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
-FunctionPass *createRISCVInsertVSETVLIPass();
+FunctionPass *createRISCVInsertVSETVLIPass(bool IsSplitRA = false);
void initializeRISCVInsertVSETVLIPass(PassRegistry &);
FunctionPass *createRISCVPostRAExpandPseudoPass();
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 9d2b2c3b3f59264..03c8d72a32dc714 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -26,6 +26,7 @@
#include "RISCV.h"
#include "RISCVSubtarget.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include <queue>
@@ -44,6 +45,86 @@ static cl::opt<bool> UseStrictAsserts(
namespace {
+// Return true when CandidateDefMI can arrive UserMI without reach other defMI
+// thought CFG. Otherwise, if it:
+// 1. Encounter the DefMI again
+// 2. Seen the all BasicBlock can reach
+// return false
+static bool isReachingDef(const MachineInstr &UserMI,
+ const MachineInstr &CandidateDefMI, Register Reg,
+ const MachineRegisterInfo *MRI) {
+ std::queue<std::pair<const MachineBasicBlock *, bool>> MBBQ;
+ llvm::DenseMap<int, bool> SeenMBB;
+
+ MBBQ.push(std::make_pair(CandidateDefMI.getParent(), false));
+ while (!MBBQ.empty()) {
+
+ const MachineBasicBlock *CurrMBB = MBBQ.front().first;
+ bool SeenDef = MBBQ.front().second;
+ MBBQ.pop();
+ SeenMBB[CurrMBB->getNumber()] = true;
+ bool NeedSkip = false;
+ for (auto &MI : *CurrMBB) {
+ // If we encounter DefMI more than once, this CandidateDefMI is not
+ // reaching definition for UserMI.
+ if (SeenDef && llvm::any_of(MRI->def_instructions(Reg),
+ [&MI](const MachineInstr &DefMI) {
+ return DefMI.isIdenticalTo(MI);
+ })) {
+ NeedSkip = true;
+ break;
+ }
+
+ if (MI.isIdenticalTo(CandidateDefMI))
+ SeenDef = true;
+
+ if (SeenDef && MI.isIdenticalTo(UserMI))
+ return true;
+ }
+
+ if (NeedSkip)
+ continue;
+
+ for (auto *Succ : CurrMBB->successors()) {
+ if (SeenMBB[Succ->getNumber()])
+ continue;
+ MBBQ.push(std::make_pair(Succ, SeenDef));
+ }
+ }
+
+ return false;
+}
+
+// Return MachineInstr* when we found the unique reaching definition.
+// Otherwise, return nullptr.
+// FIXME: Could it detect the PHI node situation?
+static MachineInstr *findReachingDef(Register Reg, const MachineInstr *MI,
+ const MachineRegisterInfo *MRI) {
+
+ MachineInstr *ReachingDefMI = nullptr;
+ for (auto &DefMI : MRI->def_instructions(Reg)) {
+ if (isReachingDef(*MI, DefMI, Reg, MRI)) {
+ if (!ReachingDefMI)
+ ReachingDefMI = &DefMI;
+ else
+ return nullptr;
+ }
+ }
+
+ return ReachingDefMI;
+}
+
+// For the SSA form, we could just use the getVRegDef to take Reaching
+// definition. For the non-SSA, we create this pass own reaching definition
+// searching function.
+static MachineInstr *getReachingDefMI(Register VReg, const MachineInstr *MI,
+ const MachineRegisterInfo *MRI) {
+ if (MRI->isSSA())
+ return MRI->getVRegDef(VReg);
+
+ return findReachingDef(VReg, MI, MRI);
+}
+
static unsigned getVLOpNum(const MachineInstr &MI) {
return RISCVII::getVLOpNum(MI.getDesc());
}
@@ -187,13 +268,17 @@ static bool hasUndefinedMergeOp(const MachineInstr &MI,
if (UseMO.getReg() == RISCV::NoRegister)
return true;
- if (MachineInstr *UseMI = MRI.getVRegDef(UseMO.getReg())) {
+ if (!MRI.isSSA() && UseMO.isUndef())
+ return true;
+
+ if (MachineInstr *UseMI = getReachingDefMI(UseMO.getReg(), &MI, &MRI)) {
if (UseMI->isImplicitDef())
return true;
if (UseMI->isRegSequence()) {
for (unsigned i = 1, e = UseMI->getNumOperands(); i < e; i += 2) {
- MachineInstr *SourceMI = MRI.getVRegDef(UseMI->getOperand(i).getReg());
+ MachineInstr *SourceMI =
+ getReachingDefMI(UseMI->getOperand(i).getReg(), UseMI, &MRI);
if (!SourceMI || !SourceMI->isImplicitDef())
return false;
}
@@ -494,9 +579,9 @@ class VSETVLIInfo {
if (hasAVLReg()) {
if (getAVLReg() == RISCV::X0)
return true;
- if (MachineInstr *MI = MRI.getVRegDef(getAVLReg());
- MI && MI->getOpcode() == RISCV::ADDI &&
- MI->getOperand(1).isReg() && MI->getOperand(2).isImm() &&
+ if (MachineInstr *MI = MRI.getUniqueVRegDef(getAVLReg());
+ MI && MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
+ MI->getOperand(2).isImm() &&
MI->getOperand(1).getReg() == RISCV::X0 &&
MI->getOperand(2).getImm() != 0)
return true;
@@ -727,20 +812,32 @@ class RISCVInsertVSETVLI : public MachineFunctionPass {
const RISCVSubtarget *ST;
const TargetInstrInfo *TII;
MachineRegisterInfo *MRI;
+ LiveIntervals *LIS = nullptr;
std::vector<BlockData> BlockInfo;
std::queue<const MachineBasicBlock *> WorkList;
+ bool IsSplitRA = false;
public:
static char ID;
- RISCVInsertVSETVLI() : MachineFunctionPass(ID) {
+ RISCVInsertVSETVLI(bool IsSplitRA = false)
+ : MachineFunctionPass(ID), IsSplitRA(IsSplitRA) {
initializeRISCVInsertVSETVLIPass(*PassRegistry::getPassRegistry());
}
bool runOnMachineFunction(MachineFunction &MF) override;
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
+
+ if (IsSplitRA) {
+ AU.addRequired<LiveIntervals>();
+ AU.addPreserved<LiveIntervals>();
+ AU.addRequired<SlotIndexes>();
+ AU.addPreserved<SlotIndexes>();
+ AU.addPreserved<LiveDebugVariables>();
+ }
+
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -864,6 +961,30 @@ static VSETVLIInfo getInfoForVSETVLI(const MachineInstr &MI) {
return NewInfo;
}
+static void fixupModifyVRegLI(Register VReg, LiveIntervals *LIS) {
+ if (!LIS)
+ return;
+
+ if (LIS->hasInterval(VReg))
+ LIS->removeInterval(VReg);
+ LIS->createAndComputeVirtRegInterval(VReg);
+}
+
+static void fixupModifyVRegLIFromVSETVL(MachineInstr *MI, LiveIntervals *LIS) {
+
+ if (!LIS)
+ return;
+
+ if (LIS->isNotInMIMap(*MI))
+ LIS->InsertMachineInstrInMaps(*MI);
+ for (auto &MO : MI->operands()) {
+ if (!MO.isReg() || MO.getReg() == 0 || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ fixupModifyVRegLI(Reg, LIS);
+ }
+}
+
void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
MachineBasicBlock::iterator InsertPt, DebugLoc DL,
const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo) {
@@ -872,11 +993,13 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
// Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
// VLMAX.
if (Info.hasSameAVL(PrevInfo) && Info.hasSameVLMAX(PrevInfo)) {
- BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
- .addReg(RISCV::X0, RegState::Define | RegState::Dead)
- .addReg(RISCV::X0, RegState::Kill)
- .addImm(Info.encodeVTYPE())
- .addReg(RISCV::VL, RegState::Implicit);
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+ .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+ .addReg(RISCV::X0, RegState::Kill)
+ .addImm(Info.encodeVTYPE())
+ .addReg(RISCV::VL, RegState::Implicit);
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
return;
}
@@ -885,15 +1008,18 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
// same, we can use the X0, X0 form.
if (Info.hasSameVLMAX(PrevInfo) && Info.hasAVLReg() &&
Info.getAVLReg().isVirtual()) {
- if (MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg())) {
+ if (MachineInstr *DefMI =
+ getReachingDefMI(Info.getAVLReg(), &(*InsertPt), MRI)) {
if (isVectorConfigInstr(*DefMI)) {
VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
- BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
- .addReg(RISCV::X0, RegState::Define | RegState::Dead)
- .addReg(RISCV::X0, RegState::Kill)
- .addImm(Info.encodeVTYPE())
- .addReg(RISCV::VL, RegState::Implicit);
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+ .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+ .addReg(RISCV::X0, RegState::Kill)
+ .addImm(Info.encodeVTYPE())
+ .addReg(RISCV::VL, RegState::Implicit);
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
return;
}
}
@@ -902,10 +1028,12 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
}
if (Info.hasAVLImm()) {
- BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
- .addReg(RISCV::X0, RegState::Define | RegState::Dead)
- .addImm(Info.getAVLImm())
- .addImm(Info.encodeVTYPE());
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
+ .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+ .addImm(Info.getAVLImm())
+ .addImm(Info.encodeVTYPE());
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
return;
}
@@ -915,18 +1043,22 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
// the previous vl to become invalid.
if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
Info.hasSameVLMAX(PrevInfo)) {
- BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
- .addReg(RISCV::X0, RegState::Define | RegState::Dead)
- .addReg(RISCV::X0, RegState::Kill)
- .addImm(Info.encodeVTYPE())
- .addReg(RISCV::VL, RegState::Implicit);
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETVLIX0))
+ .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+ .addReg(RISCV::X0, RegState::Kill)
+ .addImm(Info.encodeVTYPE())
+ .addReg(RISCV::VL, RegState::Implicit);
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
return;
}
// Otherwise use an AVL of 1 to avoid depending on previous vl.
- BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
- .addReg(RISCV::X0, RegState::Define | RegState::Dead)
- .addImm(1)
- .addImm(Info.encodeVTYPE());
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(RISCV::PseudoVSETIVLI))
+ .addReg(RISCV::X0, RegState::Define | RegState::Dead)
+ .addImm(1)
+ .addImm(Info.encodeVTYPE());
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
return;
}
@@ -942,10 +1074,13 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
DestReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
Opcode = RISCV::PseudoVSETVLIX0;
}
- BuildMI(MBB, InsertPt, DL, TII->get(Opcode))
- .addReg(DestReg, RegState::Define | RegState::Dead)
- .addReg(AVLReg)
- .addImm(Info.encodeVTYPE());
+ auto NeedFixupMI =
+ BuildMI(MBB, InsertPt, DL, TII->get(Opcode))
+ .addReg(DestReg, RegState::Define | RegState::Dead)
+ .addReg(AVLReg,
+ (IsSplitRA && MRI->def_empty(AVLReg)) ? RegState::Undef : 0)
+ .addImm(Info.encodeVTYPE());
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
}
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL) {
@@ -1007,7 +1142,7 @@ bool RISCVInsertVSETVLI::needVSETVLI(const MachineInstr &MI,
// VSETVLI here.
if (Require.hasAVLReg() && Require.getAVLReg().isVirtual() &&
CurInfo.hasCompatibleVTYPE(Used, Require)) {
- if (MachineInstr *DefMI = MRI->getVRegDef(Require.getAVLReg())) {
+ if (MachineInstr *DefMI = getReachingDefMI(Require.getAVLReg(), &MI, MRI)) {
if (isVectorConfigInstr(*DefMI)) {
VSETVLIInfo DefInfo = getInfoForVSETVLI(*DefMI);
if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
@@ -1062,7 +1197,7 @@ void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
// without being sure we can kill the original source reg entirely.
if (!Info.hasAVLReg() || !Info.getAVLReg().isVirtual())
return;
- MachineInstr *DefMI = MRI->getVRegDef(Info.getAVLReg());
+ const MachineInstr *DefMI = getReachingDefMI(Info.getAVLReg(), &MI, MRI);
if (!DefMI || !isVectorConfigInstr(*DefMI))
return;
@@ -1187,7 +1322,7 @@ bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
return true;
// We need the AVL to be produce by a PHI node in this basic block.
- MachineInstr *PHI = MRI->getVRegDef(AVLReg);
+ MachineInstr *PHI = MRI->getUniqueVRegDef(AVLReg);
if (!PHI || PHI->getOpcode() != RISCV::PHI || PHI->getParent() != &MBB)
return true;
@@ -1202,7 +1337,7 @@ bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
return true;
// We need the PHI input to the be the output of a VSET(I)VLI.
- MachineInstr *DefMI = MRI->getVRegDef(InReg);
+ MachineInstr *DefMI = MRI->getUniqueVRegDef(InReg);
if (!DefMI || !isVectorConfigInstr(*DefMI))
return true;
@@ -1258,8 +1393,12 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
if (VLOp.isReg()) {
// Erase the AVL operand from the instruction.
+ Register VLOpReg = VLOp.getReg();
+ bool IsVirtVLOpReg = VLOp.getReg().isVirtual();
VLOp.setReg(RISCV::NoRegister);
VLOp.setIsKill(false);
+ if (IsVirtVLOpReg)
+ fixupModifyVRegLI(VLOpReg, LIS);
}
MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,
/*isImp*/ true));
@@ -1509,8 +1648,23 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) {
Used = getDemanded(MI, MRI, ST);
}
- for (auto *MI : ToDelete)
+ std::vector<Register> NeedFixup;
+
+ for (auto *MI : ToDelete) {
+ for (auto &MO : MI->operands()) {
+ if (!MO.isReg() || MO.getReg() == 0 || !MO.getReg().isVirtual())
+ continue;
+ Register Reg = MO.getReg();
+ NeedFixup.push_back(Reg);
+ }
MI->eraseFromParent();
+ if (IsSplitRA)
+ LIS->RemoveMachineInstrFromMaps(*MI);
+ }
+
+ for (auto Reg : NeedFixup) {
+ fixupModifyVRegLI(Reg, LIS);
+ }
}
void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
@@ -1518,11 +1672,16 @@ void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
MachineInstr &MI = *I++;
if (RISCV::isFaultFirstLoad(MI)) {
Register VLOutput = MI.getOperand(1).getReg();
- if (!MRI->use_nodbg_empty(VLOutput))
- BuildMI(MBB, I, MI.getDebugLoc(), TII->get(RISCV::PseudoReadVL),
- VLOutput);
+ bool IsVirtual = MI.getOperand(1).getReg().isVirtual();
+ if (!MRI->use_nodbg_empty(VLOutput)) {
+ auto NeedFixupMI = BuildMI(MBB, I, MI.getDebugLoc(),
+ TII->get(RISCV::PseudoReadVL), VLOutput);
+ fixupModifyVRegLIFromVSETVL(NeedFixupMI, LIS);
+ }
// We don't use the vl output of the VLEFF/VLSEGFF anymore.
MI.getOperand(1).setReg(RISCV::X0);
+ if (IsVirtual)
+ fixupModifyVRegLI(VLOutput, LIS);
}
}
}
@@ -1538,6 +1697,11 @@ bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
TII = ST->getInstrInfo();
MRI = &MF.getRegInfo();
+ if (IsSplitRA)
+ LIS = &getAnalysis<LiveIntervals>();
+ else
+ LIS = nullptr;
+
assert(BlockInfo.empty() && "Expect empty block infos");
BlockInfo.resize(MF.getNumBlockIDs());
@@ -1604,6 +1768,6 @@ bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
}
/// Returns an instance of the Insert VSETVLI pass.
-FunctionPass *llvm::createRISCVInsertVSETVLIPass() {
- return new RISCVInsertVSETVLI();
+FunctionPass *llvm::createRISCVInsertVSETVLIPass(bool IsSplitRA) {
+ return new RISCVInsertVSETVLI(IsSplitRA);
}
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 92150ed4c224366..a3d5def678f745d 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -396,8 +396,10 @@ FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
}
bool RISCVPassConfig::addRegAssignAndRewriteFast() {
- if (EnableSplitRA)
+ if (EnableSplitRA) {
addPass(createRVVRegAllocPass(false));
+ addPass(createRISCVInsertVSETVLIPass(true));
+ }
return TargetPassConfig::addRegAssignAndRewriteFast();
}
@@ -405,6 +407,7 @@ bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
if (EnableSplitRA) {
addPass(createRVVRegAllocPass(true));
addPass(createVirtRegRewriter(false));
+ addPass(createRISCVInsertVSETVLIPass(true));
}
return TargetPassConfig::addRegAssignAndRewriteOptimized();
}
@@ -532,7 +535,8 @@ void RISCVPassConfig::addPreRegAlloc() {
addPass(createRISCVPreRAExpandPseudoPass());
if (TM->getOptLevel() != CodeGenOptLevel::None)
addPass(createRISCVMergeBaseOffsetOptPass());
- addPass(createRISCVInsertVSETVLIPass());
+ if (!EnableSplitRA)
+ addPass(createRISCVInsertVSETVLIPass());
if (TM->getOptLevel() != CodeGenOptLevel::None &&
EnableRISCVDeadRegisterElimination)
addPass(createRISCVDeadRegisterDefinitionsPass());
diff --git a/llvm/test/CodeGen/RISCV/rvv/splitRA-vsetvl.ll b/llvm/test/CodeGen/RISCV/rvv/splitRA-vsetvl.ll
new file mode 100644
index 000000000000000..68140cd9939e3e6
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/splitRA-vsetvl.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+d,+zfh,+zvfh,+v,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b -target-abi=lp64d \
+; RUN: --riscv-split-RA=1 -verify-machineinstrs < %s | FileCheck %s
+
+declare <vscale x 2 x i1> @llvm.experimental.constrained.fcmp.nxv2f16(<vscale x 2 x half>, <vscale x 2 x half>, metadata, metadata)
+
+define <vscale x 2 x i1> @fcmp_ole_vv_nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb) nounwind strictfp {
+; CHECK-LABEL: fcmp_ole_vv_nxv2f16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vsetvli a0, zero, e16, mf2, ta, mu
+; CHECK-NEXT: vmfeq.vv v10, v9, v9
+; CHECK-NEXT: vmfeq.vv v11, v8, v8
+; CHECK-NEXT: vmand.mm v0, v11, v10
+; CHECK-NEXT: vmfle.vv v0, v8, v9, v0.t
+; CHECK-NEXT: ret
+ %1 = call <vscale x 2 x i1> @llvm.experimental.constrained.fcmp.nxv2f16(<vscale x 2 x half> %va, <vscale x 2 x half> %vb, metadata !"ole", metadata !"fpexcept.strict") strictfp
+ ret <vscale x 2 x i1> %1
+}
+
+declare <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32>, ptr nocapture, i64)
+
+declare <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32>, ptr nocapture, i64)
+
+declare <vscale x 16 x i32> @llvm.riscv.vadd.nxv16i32.nxv16i32.i64(<vscale x 16 x i32>, <vscale x 16 x i32>, <vscale x 16 x i32>, i64)
+
+declare <vscale x 8 x i32> @llvm.riscv.vadd.nxv8i32.nxv8i32.i64(<vscale x 8 x i32>, <vscale x 8 x i32>, <vscale x 8 x i32>, i64)
+
+declare void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32>, ptr nocapture, i64)
+
+declare void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32>, ptr nocapture, i64)
+
+define dso_local void @test_interleave_cause_spill(ptr nocapture noundef %in) local_unnamed_addr #0 {
+; CHECK-LABEL: test_interleave_cause_spill:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: addi sp, sp, -16
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: csrr a1, vlenb
+; CHECK-NEXT: slli a1, a1, 4
+; CHECK-NEXT: sub sp, sp, a1
+; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x72, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0xa2, 0x38, 0x00, 0x1e, 0x22 # sp + 16 + 16 * vlenb
+; CHECK-NEXT: addi a1, a0, 4
+; CHECK-NEXT: li a2, 32
+; CHECK-NEXT: vsetvli zero, a2, e32, m8, ta, ma
+; CHECK-NEXT: vle32.v v8, (a1)
+; CHECK-NEXT: addi a1, a0, 8
+; CHECK-NEXT: vle32.v v24, (a1)
+; CHECK-NEXT: addi a1, a0, 12
+; CHECK-NEXT: vle32.v v16, (a1)
+; CHECK-NEXT: addi a1, sp, 16
+; CHECK-NEXT: vs8r.v v16, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT: addi a1, a0, 16
+; CHECK-NEXT: vsetvli zero, a2, e32, m4, ta, ma
+; CHECK-NEXT: vle32.v v0, (a1)
+; CHECK-NEXT: addi a1, a0, 20
+; CHECK-NEXT: vle32.v v4, (a1)
+; CHECK-NEXT: addi a1, a0, 24
+; CHECK-NEXT: vle32.v v16, (a1)
+; CHECK-NEXT: vsetvli zero, a2, e32, m8, ta, ma
+; CHECK-NEXT: vadd.vv v24, v8, v24
+; CHECK-NEXT: csrr a1, vlenb
+; CHECK-NEXT: slli a1, a1, 3
+; CHECK-NEXT: add a1, sp, a1
+; CHECK-NEXT: addi a1, a1, 16
+; CHECK-NEXT: vs8r.v v24, (a1) # Unknown-size Folded Spill
+; CHECK-NEXT: addi a1, sp, 16
+; CHECK-NEXT: vl8r.v v24, (a1) # Unknown-size Folded Reload
+; CHECK-NEXT: vadd.vv v8, v8, v24
+; CHECK-NEXT: vsetvli zero, a2, e32, m4, ta, ma
+; CHECK-NEXT: vadd.vv v20, v0, v4
+; CHECK-NEXT: vadd.vv v16, v0, v16
+; CHECK-NEXT: addi a1, a0, 40
+; CHECK-NEXT: vse32.v v20, (a1)
+; CHECK-NEXT: addi a1, a0, 44
+; CHECK-NEXT: vse32.v v16, (a1)
+; CHECK-NEXT: addi a1, a0, 48
+; CHECK-NEXT: csrr a3, vlenb
+; CHECK-NEXT: slli a3, a3, 3
+; CHECK-NEXT: add a3, sp, a3
+; CHECK-NEXT: addi a3, a3, 16
+; CHECK-NEXT: vl8r.v v16, (a3) # Unknown-size Folded Reload
+; CHECK-NEXT: vsetvli zero, a2, e32, m8, ta, ma
+; CHECK-NEXT: vse32.v v16, (a1)
+; CHECK-NEXT: addi a0, a0, 52
+; CHECK-NEXT: vse32.v v8, (a0)
+; CHECK-NEXT: csrr a0, vlenb
+; CHECK-NEXT: slli a0, a0, 4
+; CHECK-NEXT: add sp, sp, a0
+; CHECK-NEXT: addi sp, sp, 16
+; CHECK-NEXT: ret
+entry:
+ %add.ptr = getelementptr inbounds i32, ptr %in, i64 1
+ %0 = tail call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32> poison, ptr nonnull %add.ptr, i64 32)
+ %add.ptr1 = getelementptr inbounds i32, ptr %in, i64 2
+ %1 = tail call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32> poison, ptr nonnull %add.ptr1, i64 32)
+ %add.ptr2 = getelementptr inbounds i32, ptr %in, i64 3
+ %2 = tail call <vscale x 16 x i32> @llvm.riscv.vle.nxv16i32.i64(<vscale x 16 x i32> poison, ptr nonnull %add.ptr2, i64 32)
+ %add.ptr3 = getelementptr inbounds i32, ptr %in, i64 4
+ %3 = tail call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32> poison, ptr nonnull %add.ptr3, i64 32)
+ %add.ptr4 = getelementptr inbounds i32, ptr %in, i64 5
+ %4 = tail call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32> poison, ptr nonnull %add.ptr4, i64 32)
+ %add.ptr5 = getelementptr inbounds i32, ptr %in, i64 6
+ %5 = tail call <vscale x 8 x i32> @llvm.riscv.vle.nxv8i32.i64(<vscale x 8 x i32> poison, ptr nonnull %add.ptr5, i64 32)
+ %6 = tail call <vscale x 16 x i32> @llvm.riscv.vadd.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> %0, <vscale x 16 x i32> %1, i64 32)
+ %7 = tail call <vscale x 8 x i32> @llvm.riscv.vadd.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> %3, <vscale x 8 x i32> %4, i64 32)
+ %8 = tail call <vscale x 16 x i32> @llvm.riscv.vadd.nxv16i32.nxv16i32.i64(<vscale x 16 x i32> poison, <vscale x 16 x i32> %0, <vscale x 16 x i32> %2, i64 32)
+ %9 = tail call <vscale x 8 x i32> @llvm.riscv.vadd.nxv8i32.nxv8i32.i64(<vscale x 8 x i32> poison, <vscale x 8 x i32> %3, <vscale x 8 x i32> %5, i64 32)
+ %add.ptr6 = getelementptr inbounds i32, ptr %in, i64 10
+ tail call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> %7, ptr nonnull %add.ptr6, i64 32)
+ %add.ptr7 = getelementptr inbounds i32, ptr %in, i64 11
+ tail call void @llvm.riscv.vse.nxv8i32.i64(<vscale x 8 x i32> %9, ptr nonnull %add.ptr7, i64 32)
+ %add.ptr8 = getelementptr inbounds i32, ptr %in, i64 12
+ tail call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> %6, ptr nonnull %add.ptr8, i64 32)
+ %add.ptr9 = getelementptr inbounds i32, ptr %in, i64 13
+ tail call void @llvm.riscv.vse.nxv16i32.i64(<vscale x 16 x i32> %8, ptr nonnull %add.ptr9, i64 32)
+ ret void
+}
More information about the llvm-commits
mailing list