[llvm] [MacroFusion] Add IsPostRA to indicate whether running in post-ra scheduler (PR #77567)
Wang Pengcheng via llvm-commits
llvm-commits at lists.llvm.org
Wed Jan 10 00:18:09 PST 2024
https://github.com/wangpc-pp created https://github.com/llvm/llvm-project/pull/77567
This can save some time to know whether MacroFusion mutation is
running in post-ra scheduler.
And this can be used in #77461.
>From 7ec70396ec85205669bca1e4481ba5c22ebaacff Mon Sep 17 00:00:00 2001
From: wangpc <wangpengcheng.pp at bytedance.com>
Date: Wed, 10 Jan 2024 14:57:43 +0800
Subject: [PATCH] [MacroFusion] Add IsPostRA to indicate whether running in
post-ra scheduler
This can save some time to know whether MacroFusion mutation is
running in post-ra scheduler.
And this can be used in #77461.
---
llvm/include/llvm/CodeGen/MacroFusion.h | 5 +-
llvm/lib/CodeGen/MacroFusion.cpp | 14 ++---
.../lib/Target/AArch64/AArch64MacroFusion.cpp | 9 ++--
llvm/lib/Target/AArch64/AArch64MacroFusion.h | 3 +-
.../Target/AArch64/AArch64TargetMachine.cpp | 4 +-
llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp | 8 +--
llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h | 3 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 4 +-
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 7 +--
llvm/lib/Target/ARM/ARMMacroFusion.cpp | 10 ++--
llvm/lib/Target/ARM/ARMMacroFusion.h | 3 +-
llvm/lib/Target/ARM/ARMTargetMachine.cpp | 4 +-
llvm/lib/Target/PowerPC/PPCMacroFusion.cpp | 8 +--
llvm/lib/Target/PowerPC/PPCMacroFusion.h | 3 +-
llvm/lib/Target/PowerPC/PPCTargetMachine.cpp | 4 +-
llvm/lib/Target/RISCV/RISCVMacroFusion.cpp | 54 ++++++++++---------
llvm/lib/Target/RISCV/RISCVMacroFusion.h | 3 +-
llvm/lib/Target/RISCV/RISCVSubtarget.cpp | 2 +-
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 4 +-
llvm/lib/Target/X86/X86MacroFusion.cpp | 8 +--
llvm/lib/Target/X86/X86MacroFusion.h | 2 +-
llvm/lib/Target/X86/X86Subtarget.cpp | 2 +-
llvm/lib/Target/X86/X86TargetMachine.cpp | 4 +-
llvm/test/TableGen/MacroFusion.td | 12 ++---
.../TableGen/MacroFusionPredicatorEmitter.cpp | 20 ++++---
25 files changed, 110 insertions(+), 90 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MacroFusion.h b/llvm/include/llvm/CodeGen/MacroFusion.h
index 191c906e9ef6c6..e418cbb4e7d5f6 100644
--- a/llvm/include/llvm/CodeGen/MacroFusion.h
+++ b/llvm/include/llvm/CodeGen/MacroFusion.h
@@ -32,7 +32,7 @@ class SUnit;
using MacroFusionPredTy = bool (*)(const TargetInstrInfo &TII,
const TargetSubtargetInfo &STI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI);
+ const MachineInstr &SecondMI, bool IsPostRA);
/// Checks if the number of cluster edges between SU and its predecessors is
/// less than FuseLimit
@@ -50,11 +50,12 @@ bool fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU,
/// for instructions that benefit according to the target-specific
/// predicate functions. shouldScheduleAdjacent will be true if any of the
/// provided predicates are true.
+/// If IsPostRA is true, this mutation will run in post RA scheduler.
/// If BranchOnly is true, only branch instructions with one of their
/// predecessors will be fused.
std::unique_ptr<ScheduleDAGMutation>
createMacroFusionDAGMutation(ArrayRef<MacroFusionPredTy> Predicates,
- bool BranchOnly = false);
+ bool IsPostRA, bool BranchOnly = false);
} // end namespace llvm
diff --git a/llvm/lib/CodeGen/MacroFusion.cpp b/llvm/lib/CodeGen/MacroFusion.cpp
index 5bd6ca0978a4b1..b7b39c4fbdb698 100644
--- a/llvm/lib/CodeGen/MacroFusion.cpp
+++ b/llvm/lib/CodeGen/MacroFusion.cpp
@@ -138,13 +138,15 @@ namespace {
/// be fused by the processor into a single operation.
class MacroFusion : public ScheduleDAGMutation {
std::vector<MacroFusionPredTy> Predicates;
+ bool IsPostRA;
bool FuseBlock;
bool scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU);
public:
- MacroFusion(ArrayRef<MacroFusionPredTy> Predicates, bool FuseBlock)
- : Predicates(Predicates.begin(), Predicates.end()), FuseBlock(FuseBlock) {
- }
+ MacroFusion(ArrayRef<MacroFusionPredTy> Predicates, bool IsPostRA,
+ bool FuseBlock)
+ : Predicates(Predicates.begin(), Predicates.end()), IsPostRA(IsPostRA),
+ FuseBlock(FuseBlock) {}
void apply(ScheduleDAGInstrs *DAGInstrs) override;
@@ -161,7 +163,7 @@ bool MacroFusion::shouldScheduleAdjacent(const TargetInstrInfo &TII,
const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
return llvm::any_of(Predicates, [&](MacroFusionPredTy Predicate) {
- return Predicate(TII, STI, FirstMI, SecondMI);
+ return Predicate(TII, STI, FirstMI, SecondMI, IsPostRA);
});
}
@@ -213,8 +215,8 @@ bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU)
std::unique_ptr<ScheduleDAGMutation>
llvm::createMacroFusionDAGMutation(ArrayRef<MacroFusionPredTy> Predicates,
- bool BranchOnly) {
+ bool IsPostRA, bool BranchOnly) {
if (EnableMacroFusion)
- return std::make_unique<MacroFusion>(Predicates, !BranchOnly);
+ return std::make_unique<MacroFusion>(Predicates, IsPostRA, !BranchOnly);
return nullptr;
}
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
index 05d60872bf51ac..bc67488abcaea8 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -443,8 +443,9 @@ static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
- const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
+ const AArch64Subtarget &ST = static_cast<const AArch64Subtarget &>(TSI);
// All checking functions assume that the 1st instr is a wildcard if it is
// unspecified.
@@ -477,6 +478,6 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
}
std::unique_ptr<ScheduleDAGMutation>
-llvm::createAArch64MacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent);
+llvm::createAArch64MacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA);
}
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.h b/llvm/lib/Target/AArch64/AArch64MacroFusion.h
index 2999e7a8aa909c..807db002c762e0 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.h
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.h
@@ -21,7 +21,8 @@ namespace llvm {
/// Note that you have to add:
/// DAG.addMutation(createAArch64MacroFusionDAGMutation());
/// to AArch64PassConfig::createMachineScheduler() to have an effect.
-std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation();
+std::unique_ptr<ScheduleDAGMutation>
+createAArch64MacroFusionDAGMutation(bool IsPostRA);
} // llvm
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 144610e021c58e..a5648f2856888d 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -496,7 +496,7 @@ class AArch64PassConfig : public TargetPassConfig {
DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.hasFusion())
- DAG->addMutation(createAArch64MacroFusionDAGMutation());
+ DAG->addMutation(createAArch64MacroFusionDAGMutation(/*IsPostRA=*/false));
return DAG;
}
@@ -509,7 +509,7 @@ class AArch64PassConfig : public TargetPassConfig {
if (ST.hasFusion()) {
// Run the Macro Fusion after RA again since literals are expanded from
// pseudos then (v. addPreSched2()).
- DAG->addMutation(createAArch64MacroFusionDAGMutation());
+ DAG->addMutation(createAArch64MacroFusionDAGMutation(/*IsPostRA=*/true));
return DAG;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
index 0cbabf3895a67e..08d764650ece29 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
@@ -26,7 +26,8 @@ namespace {
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII_,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
const SIInstrInfo &TII = static_cast<const SIInstrInfo&>(TII_);
switch (SecondMI.getOpcode()) {
@@ -59,8 +60,9 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII_,
namespace llvm {
-std::unique_ptr<ScheduleDAGMutation> createAMDGPUMacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent);
+std::unique_ptr<ScheduleDAGMutation>
+createAMDGPUMacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA);
}
} // end namespace llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
index ad198a301dbe40..6366f97cbcc804 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUMacroFusion.h
@@ -17,7 +17,8 @@ namespace llvm {
/// Note that you have to add:
/// DAG.addMutation(createAMDGPUMacroFusionDAGMutation());
/// to AMDGPUPassConfig::createMachineScheduler() to have an effect.
-std::unique_ptr<ScheduleDAGMutation> createAMDGPUMacroFusionDAGMutation();
+std::unique_ptr<ScheduleDAGMutation>
+createAMDGPUMacroFusionDAGMutation(bool IsPostRA);
} // llvm
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 0f3bb3e7b0d8d0..1a790ac2070425 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -461,7 +461,7 @@ createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
if (ST.shouldClusterStores())
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
DAG->addMutation(createIGroupLPDAGMutation(/*IsPostRA=*/false));
- DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
+ DAG->addMutation(createAMDGPUMacroFusionDAGMutation(/*IsPostRA=*/false));
DAG->addMutation(createAMDGPUExportClusteringDAGMutation());
return DAG;
}
@@ -498,7 +498,7 @@ createIterativeILPMachineScheduler(MachineSchedContext *C) {
DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.shouldClusterStores())
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
- DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
+ DAG->addMutation(createAMDGPUMacroFusionDAGMutation(/*IsPostRA=*/false));
return DAG;
}
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index 33c208495c500e..6328edd70feede 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -123,7 +123,8 @@ bool llvm::checkVOPDRegConstraints(const SIInstrInfo &TII,
static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
const SIInstrInfo &STII = static_cast<const SIInstrInfo &>(TII);
unsigned Opc2 = SecondMI.getOpcode();
auto SecondCanBeVOPD = AMDGPU::getCanBeVOPD(Opc2);
@@ -165,7 +166,7 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
std::vector<SUnit>::iterator ISUI, JSUI;
for (ISUI = DAG->SUnits.begin(); ISUI != DAG->SUnits.end(); ++ISUI) {
const MachineInstr *IMI = ISUI->getInstr();
- if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI))
+ if (!shouldScheduleAdjacent(TII, ST, nullptr, *IMI, /*IsPostRA=*/true))
continue;
if (!hasLessThanNumFused(*ISUI, 2))
continue;
@@ -175,7 +176,7 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
continue;
const MachineInstr *JMI = JSUI->getInstr();
if (!hasLessThanNumFused(*JSUI, 2) ||
- !shouldScheduleAdjacent(TII, ST, IMI, *JMI))
+ !shouldScheduleAdjacent(TII, ST, IMI, *JMI, /*IsPostRA=*/true))
continue;
if (fuseInstructionPair(*DAG, *ISUI, *JSUI))
break;
diff --git a/llvm/lib/Target/ARM/ARMMacroFusion.cpp b/llvm/lib/Target/ARM/ARMMacroFusion.cpp
index 5aeb7abe92a38c..cbe0742560a134 100644
--- a/llvm/lib/Target/ARM/ARMMacroFusion.cpp
+++ b/llvm/lib/Target/ARM/ARMMacroFusion.cpp
@@ -51,8 +51,9 @@ static bool isLiteralsPair(const MachineInstr *FirstMI,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
- const ARMSubtarget &ST = static_cast<const ARMSubtarget&>(TSI);
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
+ const ARMSubtarget &ST = static_cast<const ARMSubtarget &>(TSI);
if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
return true;
@@ -62,8 +63,9 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
return false;
}
-std::unique_ptr<ScheduleDAGMutation> createARMMacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent);
+std::unique_ptr<ScheduleDAGMutation>
+createARMMacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA);
}
} // end namespace llvm
diff --git a/llvm/lib/Target/ARM/ARMMacroFusion.h b/llvm/lib/Target/ARM/ARMMacroFusion.h
index 4896a4a2544dbf..3fea3d7671865d 100644
--- a/llvm/lib/Target/ARM/ARMMacroFusion.h
+++ b/llvm/lib/Target/ARM/ARMMacroFusion.h
@@ -21,7 +21,8 @@ namespace llvm {
/// Note that you have to add:
/// DAG.addMutation(createARMMacroFusionDAGMutation());
/// to ARMPassConfig::createMachineScheduler() to have an effect.
-std::unique_ptr<ScheduleDAGMutation> createARMMacroFusionDAGMutation();
+std::unique_ptr<ScheduleDAGMutation>
+createARMMacroFusionDAGMutation(bool IsPostRA);
} // llvm
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index a99773691df123..6aeea12f8dcae4 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -358,7 +358,7 @@ class ARMPassConfig : public TargetPassConfig {
// add DAG Mutations here.
const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
if (ST.hasFusion())
- DAG->addMutation(createARMMacroFusionDAGMutation());
+ DAG->addMutation(createARMMacroFusionDAGMutation(/*IsPostRA=*/false));
return DAG;
}
@@ -368,7 +368,7 @@ class ARMPassConfig : public TargetPassConfig {
// add DAG Mutations here.
const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>();
if (ST.hasFusion())
- DAG->addMutation(createARMMacroFusionDAGMutation());
+ DAG->addMutation(createARMMacroFusionDAGMutation(/*IsPostRA=*/true));
return DAG;
}
diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.cpp b/llvm/lib/Target/PowerPC/PPCMacroFusion.cpp
index 7ad6ef8c39286d..4638879e15eb18 100644
--- a/llvm/lib/Target/PowerPC/PPCMacroFusion.cpp
+++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.cpp
@@ -234,7 +234,8 @@ static bool checkOpConstraints(FusionFeature::FusionKind Kd,
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
// We use the PPC namespace to avoid the need to prefix opcodes with PPC:: in
// the def file.
using namespace PPC;
@@ -286,8 +287,9 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
namespace llvm {
-std::unique_ptr<ScheduleDAGMutation> createPowerPCMacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent);
+std::unique_ptr<ScheduleDAGMutation>
+createPowerPCMacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA);
}
} // end namespace llvm
diff --git a/llvm/lib/Target/PowerPC/PPCMacroFusion.h b/llvm/lib/Target/PowerPC/PPCMacroFusion.h
index cbf49ee779ceb7..2ef4a6d690cc19 100644
--- a/llvm/lib/Target/PowerPC/PPCMacroFusion.h
+++ b/llvm/lib/Target/PowerPC/PPCMacroFusion.h
@@ -21,7 +21,8 @@ namespace llvm {
/// Note that you have to add:
/// DAG.addMutation(createPowerPCMacroFusionDAGMutation());
/// to PPCPassConfig::createMachineScheduler() to have an effect.
-std::unique_ptr<ScheduleDAGMutation> createPowerPCMacroFusionDAGMutation();
+std::unique_ptr<ScheduleDAGMutation>
+createPowerPCMacroFusionDAGMutation(bool IsPostRA);
} // llvm
#endif // LLVM_LIB_TARGET_POWERPC_PPCMACROFUSION_H
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index d676fa86a10e77..2bfad35fe6ddaf 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -317,7 +317,7 @@ static ScheduleDAGInstrs *createPPCMachineScheduler(MachineSchedContext *C) {
if (ST.hasStoreFusion())
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.hasFusion())
- DAG->addMutation(createPowerPCMacroFusionDAGMutation());
+ DAG->addMutation(createPowerPCMacroFusionDAGMutation(/*IsPostRA=*/false));
return DAG;
}
@@ -333,7 +333,7 @@ static ScheduleDAGInstrs *createPPCPostMachineScheduler(
if (ST.hasStoreFusion())
DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
if (ST.hasFusion())
- DAG->addMutation(createPowerPCMacroFusionDAGMutation());
+ DAG->addMutation(createPowerPCMacroFusionDAGMutation(/*IsPostRA=*/true));
return DAG;
}
diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
index f948f05b22f772..c1174503039fb2 100644
--- a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
+++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp
@@ -18,7 +18,8 @@
using namespace llvm;
-static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) {
+static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI,
+ bool IsPostRA) {
if (!SecondMI.getOperand(1).isReg())
return false;
@@ -26,7 +27,7 @@ static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) {
return false;
// If the input is virtual make sure this is the only user.
- if (FirstDest.isVirtual()) {
+ if (!IsPostRA) {
auto &MRI = SecondMI.getMF()->getRegInfo();
return MRI.hasOneNonDBGUse(FirstDest);
}
@@ -37,7 +38,8 @@ static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) {
// Fuse load with add:
// add rd, rs1, rs2
// ld rd, 0(rd)
-static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
+static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
+ bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::LD)
return false;
@@ -55,13 +57,14 @@ static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOpcode() != RISCV::ADD)
return true;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
// Fuse zero extension of halfword:
// slli rd, rs1, 48
// srli rd, rd, 48
-static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
+static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
+ bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;
@@ -82,13 +85,14 @@ static bool isZExtH(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOperand(2).getImm() != 48)
return false;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
// Fuse zero extension of word:
// slli rd, rs1, 32
// srli rd, rd, 32
-static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
+static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
+ bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;
@@ -109,7 +113,7 @@ static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
if (FirstMI->getOperand(2).getImm() != 32)
return false;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
// Fuse shifted zero extension of word:
@@ -117,7 +121,7 @@ static bool isZExtW(const MachineInstr *FirstMI, const MachineInstr &SecondMI) {
// srli rd, rd, x
// where 0 <= x < 32
static bool isShiftedZExtW(const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI, bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::SRLI)
return false;
@@ -139,14 +143,14 @@ static bool isShiftedZExtW(const MachineInstr *FirstMI,
if (FirstMI->getOperand(2).getImm() != 32)
return false;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
// Fuse AUIPC followed by ADDI
// auipc rd, imm20
// addi rd, rd, imm12
static bool isAUIPCADDI(const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI, bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::ADDI)
return false;
// Assume the 1st instr to be a wildcard if it is unspecified.
@@ -156,15 +160,15 @@ static bool isAUIPCADDI(const MachineInstr *FirstMI,
if (FirstMI->getOpcode() != RISCV::AUIPC)
return false;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
// Fuse LUI followed by ADDI or ADDIW.
// rd = imm[31:0] which decomposes to
// lui rd, imm[31:12]
// addi(w) rd, rd, imm[11:0]
-static bool isLUIADDI(const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+static bool isLUIADDI(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
+ bool IsPostRA) {
if (SecondMI.getOpcode() != RISCV::ADDI &&
SecondMI.getOpcode() != RISCV::ADDIW)
return false;
@@ -175,36 +179,38 @@ static bool isLUIADDI(const MachineInstr *FirstMI,
if (FirstMI->getOpcode() != RISCV::LUI)
return false;
- return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI);
+ return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI, IsPostRA);
}
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
const RISCVSubtarget &ST = static_cast<const RISCVSubtarget &>(TSI);
- if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI))
+ if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI, IsPostRA))
return true;
- if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI))
+ if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI, IsPostRA))
return true;
- if (ST.hasZExtHFusion() && isZExtH(FirstMI, SecondMI))
+ if (ST.hasZExtHFusion() && isZExtH(FirstMI, SecondMI, IsPostRA))
return true;
- if (ST.hasZExtWFusion() && isZExtW(FirstMI, SecondMI))
+ if (ST.hasZExtWFusion() && isZExtW(FirstMI, SecondMI, IsPostRA))
return true;
- if (ST.hasShiftedZExtWFusion() && isShiftedZExtW(FirstMI, SecondMI))
+ if (ST.hasShiftedZExtWFusion() && isShiftedZExtW(FirstMI, SecondMI, IsPostRA))
return true;
- if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI))
+ if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI, IsPostRA))
return true;
return false;
}
-std::unique_ptr<ScheduleDAGMutation> llvm::createRISCVMacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent);
+std::unique_ptr<ScheduleDAGMutation>
+llvm::createRISCVMacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA);
}
diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.h b/llvm/lib/Target/RISCV/RISCVMacroFusion.h
index 7598db3f8fe143..14088e0d30a0a4 100644
--- a/llvm/lib/Target/RISCV/RISCVMacroFusion.h
+++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.h
@@ -21,7 +21,8 @@ namespace llvm {
/// Note that you have to add:
/// DAG.addMutation(createRISCVMacroFusionDAGMutation());
/// to RISCVPassConfig::createMachineScheduler() to have an effect.
-std::unique_ptr<ScheduleDAGMutation> createRISCVMacroFusionDAGMutation();
+std::unique_ptr<ScheduleDAGMutation>
+createRISCVMacroFusionDAGMutation(bool IsPostRA);
} // namespace llvm
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
index 7b64d3cee9c800..6319cd3134c438 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -187,7 +187,7 @@ bool RISCVSubtarget::enableSubRegLiveness() const {
void RISCVSubtarget::getPostRAMutations(
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
- Mutations.push_back(createRISCVMacroFusionDAGMutation());
+ Mutations.push_back(createRISCVMacroFusionDAGMutation(/*IsPostRA=*/true));
}
/// Enable use of alias analysis during code generation (during MI
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index 3abdb6003659fa..fe341387a47295 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -352,7 +352,7 @@ class RISCVPassConfig : public TargetPassConfig {
}
if (ST.hasMacroFusion()) {
DAG = DAG ? DAG : createGenericSchedLive(C);
- DAG->addMutation(createRISCVMacroFusionDAGMutation());
+ DAG->addMutation(createRISCVMacroFusionDAGMutation(/*IsPostRA=*/false));
}
return DAG;
}
@@ -362,7 +362,7 @@ class RISCVPassConfig : public TargetPassConfig {
const RISCVSubtarget &ST = C->MF->getSubtarget<RISCVSubtarget>();
if (ST.hasMacroFusion()) {
ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
- DAG->addMutation(createRISCVMacroFusionDAGMutation());
+ DAG->addMutation(createRISCVMacroFusionDAGMutation(/*IsPostRA=*/true));
return DAG;
}
return nullptr;
diff --git a/llvm/lib/Target/X86/X86MacroFusion.cpp b/llvm/lib/Target/X86/X86MacroFusion.cpp
index c0fa9aa7032437..f8330d57353d0e 100644
--- a/llvm/lib/Target/X86/X86MacroFusion.cpp
+++ b/llvm/lib/Target/X86/X86MacroFusion.cpp
@@ -35,7 +35,8 @@ static X86::SecondMacroFusionInstKind classifySecond(const MachineInstr &MI) {
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
const TargetSubtargetInfo &TSI,
const MachineInstr *FirstMI,
- const MachineInstr &SecondMI) {
+ const MachineInstr &SecondMI,
+ bool IsPostRA) {
const X86Subtarget &ST = static_cast<const X86Subtarget &>(TSI);
// Check if this processor supports any kind of fusion.
@@ -67,8 +68,9 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
namespace llvm {
-std::unique_ptr<ScheduleDAGMutation> createX86MacroFusionDAGMutation() {
- return createMacroFusionDAGMutation(shouldScheduleAdjacent,
+std::unique_ptr<ScheduleDAGMutation>
+createX86MacroFusionDAGMutation(bool IsPostRA) {
+ return createMacroFusionDAGMutation(shouldScheduleAdjacent, IsPostRA,
/*BranchOnly=*/true);
}
diff --git a/llvm/lib/Target/X86/X86MacroFusion.h b/llvm/lib/Target/X86/X86MacroFusion.h
index 05388b275ca3ff..6c88f5d243acfe 100644
--- a/llvm/lib/Target/X86/X86MacroFusion.h
+++ b/llvm/lib/Target/X86/X86MacroFusion.h
@@ -24,7 +24,7 @@ class ScheduleDAGMutation;
/// DAG.addMutation(createX86MacroFusionDAGMutation());
/// to X86PassConfig::createMachineScheduler() to have an effect.
std::unique_ptr<ScheduleDAGMutation>
-createX86MacroFusionDAGMutation();
+createX86MacroFusionDAGMutation(bool IsPostRA);
} // end namespace llvm
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 07f535685e8f97..867b8156663dd5 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -376,7 +376,7 @@ bool X86Subtarget::enableEarlyIfConversion() const {
void X86Subtarget::getPostRAMutations(
std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
- Mutations.push_back(createX86MacroFusionDAGMutation());
+ Mutations.push_back(createX86MacroFusionDAGMutation(/*IsPostRA=*/true));
}
bool X86Subtarget::isPositionIndependent() const {
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index b92bffbe6239bb..4f8c95dfaf5c6d 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -374,14 +374,14 @@ class X86PassConfig : public TargetPassConfig {
ScheduleDAGInstrs *
createMachineScheduler(MachineSchedContext *C) const override {
ScheduleDAGMILive *DAG = createGenericSchedLive(C);
- DAG->addMutation(createX86MacroFusionDAGMutation());
+ DAG->addMutation(createX86MacroFusionDAGMutation(/*IsPostRA=*/false));
return DAG;
}
ScheduleDAGInstrs *
createPostMachineScheduler(MachineSchedContext *C) const override {
ScheduleDAGMI *DAG = createGenericSchedPostRA(C);
- DAG->addMutation(createX86MacroFusionDAGMutation());
+ DAG->addMutation(createX86MacroFusionDAGMutation(/*IsPostRA=*/true));
return DAG;
}
diff --git a/llvm/test/TableGen/MacroFusion.td b/llvm/test/TableGen/MacroFusion.td
index f984a142839c95..6fe0f62fc544cc 100644
--- a/llvm/test/TableGen/MacroFusion.td
+++ b/llvm/test/TableGen/MacroFusion.td
@@ -43,7 +43,7 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: #undef GET_Test_MACRO_FUSION_PRED_DECL
// CHECK-PREDICATOR-EMPTY:
// CHECK-PREDICATOR-NEXT: namespace llvm {
-// CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &);
+// CHECK-PREDICATOR-NEXT: bool isTestFusion(const TargetInstrInfo &, const TargetSubtargetInfo &, const MachineInstr *, const MachineInstr &, bool);
// CHECK-PREDICATOR-NEXT: } // end namespace llvm
// CHECK-PREDICATOR-EMPTY:
// CHECK-PREDICATOR-NEXT: #endif
@@ -56,7 +56,8 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: const TargetInstrInfo &TII,
// CHECK-PREDICATOR-NEXT: const TargetSubtargetInfo &STI,
// CHECK-PREDICATOR-NEXT: const MachineInstr *FirstMI,
-// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI) {
+// CHECK-PREDICATOR-NEXT: const MachineInstr &SecondMI,
+// CHECK-PREDICATOR-NEXT: bool IsPostRA) {
// CHECK-PREDICATOR-NEXT: auto &MRI = SecondMI.getMF()->getRegInfo();
// CHECK-PREDICATOR-NEXT: {
// CHECK-PREDICATOR-NEXT: const MachineInstr *MI = &SecondMI;
@@ -81,11 +82,8 @@ def TestFusion: SimpleFusion<CheckOpcode<[Inst0]>,
// CHECK-PREDICATOR-NEXT: ))
// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: }
-// CHECK-PREDICATOR-NEXT: {
-// CHECK-PREDICATOR-NEXT: Register FirstDest = FirstMI->getOperand(0).getReg();
-// CHECK-PREDICATOR-NEXT: if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))
-// CHECK-PREDICATOR-NEXT: return false;
-// CHECK-PREDICATOR-NEXT: }
+// CHECK-PREDICATOR-NEXT: if (!IsPostRA && !MRI.hasOneNonDBGUse(FirstMI->getOperand(0).getReg()))
+// CHECK-PREDICATOR-NEXT: return false;
// CHECK-PREDICATOR-NEXT: if (!(FirstMI->getOperand(0).isReg() &&
// CHECK-PREDICATOR-NEXT: SecondMI.getOperand(1).isReg() &&
// CHECK-PREDICATOR-NEXT: FirstMI->getOperand(0).getReg() == SecondMI.getOperand(1).getReg()))
diff --git a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
index 78dcd4471ae747..e99e930cbcf2bf 100644
--- a/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
+++ b/llvm/utils/TableGen/MacroFusionPredicatorEmitter.cpp
@@ -25,7 +25,8 @@
// bool isNAME(const TargetInstrInfo &TII,
// const TargetSubtargetInfo &STI,
// const MachineInstr *FirstMI,
-// const MachineInstr &SecondMI) {
+// const MachineInstr &SecondMI,
+// bool IsPostRA) {
// auto &MRI = SecondMI.getMF()->getRegInfo();
// /* Predicates */
// return true;
@@ -85,9 +86,8 @@ void MacroFusionPredicatorEmitter::emitMacroFusionDecl(
for (Record *Fusion : Fusions) {
OS << "bool is" << Fusion->getName() << "(const TargetInstrInfo &, "
- << "const TargetSubtargetInfo &, "
- << "const MachineInstr *, "
- << "const MachineInstr &);\n";
+ << "const TargetSubtargetInfo &, " << "const MachineInstr *, "
+ << "const MachineInstr &, bool);\n";
}
OS << "} // end namespace llvm\n";
@@ -108,7 +108,8 @@ void MacroFusionPredicatorEmitter::emitMacroFusionImpl(
OS.indent(4) << "const TargetInstrInfo &TII,\n";
OS.indent(4) << "const TargetSubtargetInfo &STI,\n";
OS.indent(4) << "const MachineInstr *FirstMI,\n";
- OS.indent(4) << "const MachineInstr &SecondMI) {\n";
+ OS.indent(4) << "const MachineInstr &SecondMI,\n";
+ OS.indent(4) << "bool IsPostRA) {\n";
OS.indent(2) << "auto &MRI = SecondMI.getMF()->getRegInfo();\n";
emitPredicates(Predicates, PE, OS);
@@ -146,12 +147,9 @@ void MacroFusionPredicatorEmitter::emitFirstPredicate(Record *Predicate,
<< (Predicate->getValueAsBit("ReturnValue") ? "true" : "false")
<< ";\n";
} else if (Predicate->isSubClassOf("OneUsePred")) {
- OS.indent(2) << "{\n";
- OS.indent(4) << "Register FirstDest = FirstMI->getOperand(0).getReg();\n";
- OS.indent(4)
- << "if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))\n";
- OS.indent(4) << " return false;\n";
- OS.indent(2) << "}\n";
+ OS.indent(2) << "if (!IsPostRA && "
+ "!MRI.hasOneNonDBGUse(FirstMI->getOperand(0).getReg()))\n";
+ OS.indent(2) << " return false;\n";
} else if (Predicate->isSubClassOf(
"FirstFusionPredicateWithMCInstPredicate")) {
OS.indent(2) << "{\n";
More information about the llvm-commits
mailing list