[llvm] 7c13ae6 - Give option to use isCopyInstr to determine which MI is
Adrian Tong via llvm-commits
llvm-commits at lists.llvm.org
Thu May 26 11:48:19 PDT 2022
Author: Adrian Tong
Date: 2022-05-26T18:43:16Z
New Revision: 7c13ae6490b102a7d14d75c8f502f3e003118f49
URL: https://github.com/llvm/llvm-project/commit/7c13ae6490b102a7d14d75c8f502f3e003118f49
DIFF: https://github.com/llvm/llvm-project/commit/7c13ae6490b102a7d14d75c8f502f3e003118f49.diff
LOG: Give option to use isCopyInstr to determine which MI is
treated as Copy instruction in MCP.
This is then used in AArch64 to remove copy instructions after taildup
ran in machine block placement
Differential Revision: https://reviews.llvm.org/D125335
Added:
llvm/test/CodeGen/AArch64/copyprop.ll
Modified:
llvm/include/llvm/CodeGen/Passes.h
llvm/lib/CodeGen/MachineCopyPropagation.cpp
llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
llvm/test/CodeGen/AArch64/O3-pipeline.ll
llvm/test/CodeGen/AArch64/copyprop.mir
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 7eb82cd1609de..bfd774f578ac8 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -331,6 +331,8 @@ namespace llvm {
/// machine instructions.
extern char &MachineCopyPropagationID;
+ MachineFunctionPass *createMachineCopyPropagationPass(bool UseCopyInstr);
+
/// PeepholeOptimizer - This pass performs peephole optimizations -
/// like extension and comparison eliminations.
extern char &PeepholeOptimizerID;
diff --git a/llvm/lib/CodeGen/MachineCopyPropagation.cpp b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
index 0535332a833ec..a8909c20d7a20 100644
--- a/llvm/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/llvm/lib/CodeGen/MachineCopyPropagation.cpp
@@ -61,6 +61,7 @@
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/InitializePasses.h"
@@ -82,8 +83,24 @@ STATISTIC(NumCopyBackwardPropagated, "Number of copy defs backward propagated");
DEBUG_COUNTER(FwdCounter, "machine-cp-fwd",
"Controls which register COPYs are forwarded");
+static cl::opt<bool> MCPUseCopyInstr("mcp-use-is-copy-instr", cl::init(false),
+ cl::Hidden);
+
namespace {
+static Optional<DestSourcePair> isCopyInstr(const MachineInstr &MI,
+ const TargetInstrInfo &TII,
+ bool UseCopyInstr) {
+ if (UseCopyInstr)
+ return TII.isCopyInstr(MI);
+
+ if (MI.isCopy())
+ return Optional<DestSourcePair>(
+ DestSourcePair{MI.getOperand(0), MI.getOperand(1)});
+
+ return None;
+}
+
class CopyTracker {
struct CopyInfo {
MachineInstr *MI;
@@ -109,7 +126,8 @@ class CopyTracker {
}
/// Remove register from copy maps.
- void invalidateRegister(MCRegister Reg, const TargetRegisterInfo &TRI) {
+ void invalidateRegister(MCRegister Reg, const TargetRegisterInfo &TRI,
+ const TargetInstrInfo &TII, bool UseCopyInstr) {
// Since Reg might be a subreg of some registers, only invalidate Reg is not
// enough. We have to find the COPY defines Reg or registers defined by Reg
// and invalidate all of them.
@@ -119,8 +137,13 @@ class CopyTracker {
auto I = Copies.find(*RUI);
if (I != Copies.end()) {
if (MachineInstr *MI = I->second.MI) {
- RegsToInvalidate.insert(MI->getOperand(0).getReg().asMCReg());
- RegsToInvalidate.insert(MI->getOperand(1).getReg().asMCReg());
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*MI, TII, UseCopyInstr);
+ assert(CopyOperands && "Expect copy");
+
+ RegsToInvalidate.insert(
+ CopyOperands->Destination->getReg().asMCReg());
+ RegsToInvalidate.insert(CopyOperands->Source->getReg().asMCReg());
}
RegsToInvalidate.insert(I->second.DefRegs.begin(),
I->second.DefRegs.end());
@@ -132,7 +155,8 @@ class CopyTracker {
}
/// Clobber a single register, removing it from the tracker's copy maps.
- void clobberRegister(MCRegister Reg, const TargetRegisterInfo &TRI) {
+ void clobberRegister(MCRegister Reg, const TargetRegisterInfo &TRI,
+ const TargetInstrInfo &TII, bool UseCopyInstr) {
for (MCRegUnitIterator RUI(Reg, &TRI); RUI.isValid(); ++RUI) {
auto I = Copies.find(*RUI);
if (I != Copies.end()) {
@@ -141,8 +165,12 @@ class CopyTracker {
markRegsUnavailable(I->second.DefRegs, TRI);
// When we clobber the destination of a copy, we need to clobber the
// whole register it defined.
- if (MachineInstr *MI = I->second.MI)
- markRegsUnavailable({MI->getOperand(0).getReg().asMCReg()}, TRI);
+ if (MachineInstr *MI = I->second.MI) {
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*MI, TII, UseCopyInstr);
+ markRegsUnavailable({CopyOperands->Destination->getReg().asMCReg()},
+ TRI);
+ }
// Now we can erase the copy.
Copies.erase(I);
}
@@ -150,11 +178,13 @@ class CopyTracker {
}
/// Add this copy's registers into the tracker's copy maps.
- void trackCopy(MachineInstr *MI, const TargetRegisterInfo &TRI) {
- assert(MI->isCopy() && "Tracking non-copy?");
+ void trackCopy(MachineInstr *MI, const TargetRegisterInfo &TRI,
+ const TargetInstrInfo &TII, bool UseCopyInstr) {
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(*MI, TII, UseCopyInstr);
+ assert(CopyOperands && "Tracking non-copy?");
- MCRegister Def = MI->getOperand(0).getReg().asMCReg();
- MCRegister Src = MI->getOperand(1).getReg().asMCReg();
+ MCRegister Src = CopyOperands->Source->getReg().asMCReg();
+ MCRegister Def = CopyOperands->Destination->getReg().asMCReg();
// Remember Def is defined by the copy.
for (MCRegUnitIterator RUI(Def, &TRI); RUI.isValid(); ++RUI)
@@ -197,15 +227,22 @@ class CopyTracker {
}
MachineInstr *findAvailBackwardCopy(MachineInstr &I, MCRegister Reg,
- const TargetRegisterInfo &TRI) {
+ const TargetRegisterInfo &TRI,
+ const TargetInstrInfo &TII,
+ bool UseCopyInstr) {
MCRegUnitIterator RUI(Reg, &TRI);
MachineInstr *AvailCopy = findCopyDefViaUnit(*RUI, TRI);
- if (!AvailCopy ||
- !TRI.isSubRegisterEq(AvailCopy->getOperand(1).getReg(), Reg))
+
+ if (!AvailCopy)
+ return nullptr;
+
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*AvailCopy, TII, UseCopyInstr);
+ Register AvailSrc = CopyOperands->Source->getReg();
+ Register AvailDef = CopyOperands->Destination->getReg();
+ if (!TRI.isSubRegisterEq(AvailSrc, Reg))
return nullptr;
- Register AvailSrc = AvailCopy->getOperand(1).getReg();
- Register AvailDef = AvailCopy->getOperand(0).getReg();
for (const MachineInstr &MI :
make_range(AvailCopy->getReverseIterator(), I.getReverseIterator()))
for (const MachineOperand &MO : MI.operands())
@@ -218,20 +255,26 @@ class CopyTracker {
}
MachineInstr *findAvailCopy(MachineInstr &DestCopy, MCRegister Reg,
- const TargetRegisterInfo &TRI) {
+ const TargetRegisterInfo &TRI,
+ const TargetInstrInfo &TII, bool UseCopyInstr) {
// We check the first RegUnit here, since we'll only be interested in the
// copy if it copies the entire register anyway.
MCRegUnitIterator RUI(Reg, &TRI);
MachineInstr *AvailCopy =
findCopyForUnit(*RUI, TRI, /*MustBeAvailable=*/true);
- if (!AvailCopy ||
- !TRI.isSubRegisterEq(AvailCopy->getOperand(0).getReg(), Reg))
+
+ if (!AvailCopy)
+ return nullptr;
+
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*AvailCopy, TII, UseCopyInstr);
+ Register AvailSrc = CopyOperands->Source->getReg();
+ Register AvailDef = CopyOperands->Destination->getReg();
+ if (!TRI.isSubRegisterEq(AvailDef, Reg))
return nullptr;
// Check that the available copy isn't clobbered by any regmasks between
// itself and the destination.
- Register AvailSrc = AvailCopy->getOperand(1).getReg();
- Register AvailDef = AvailCopy->getOperand(0).getReg();
for (const MachineInstr &MI :
make_range(AvailCopy->getIterator(), DestCopy.getIterator()))
for (const MachineOperand &MO : MI.operands())
@@ -252,10 +295,14 @@ class MachineCopyPropagation : public MachineFunctionPass {
const TargetInstrInfo *TII;
const MachineRegisterInfo *MRI;
+ // Return true if this is a copy instruction and false otherwise.
+ bool UseCopyInstr;
+
public:
static char ID; // Pass identification, replacement for typeid
- MachineCopyPropagation() : MachineFunctionPass(ID) {
+ MachineCopyPropagation(bool CopyInstr = false)
+ : MachineFunctionPass(ID), UseCopyInstr(CopyInstr || MCPUseCopyInstr) {
initializeMachineCopyPropagationPass(*PassRegistry::getPassRegistry());
}
@@ -333,9 +380,13 @@ void MachineCopyPropagation::ReadRegister(MCRegister Reg, MachineInstr &Reader,
/// isNopCopy("ecx = COPY eax", AX, CX) == true
/// isNopCopy("ecx = COPY eax", AH, CL) == false
static bool isNopCopy(const MachineInstr &PreviousCopy, MCRegister Src,
- MCRegister Def, const TargetRegisterInfo *TRI) {
- MCRegister PreviousSrc = PreviousCopy.getOperand(1).getReg().asMCReg();
- MCRegister PreviousDef = PreviousCopy.getOperand(0).getReg().asMCReg();
+ MCRegister Def, const TargetRegisterInfo *TRI,
+ const TargetInstrInfo *TII, bool UseCopyInstr) {
+
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(PreviousCopy, *TII, UseCopyInstr);
+ MCRegister PreviousSrc = CopyOperands->Source->getReg().asMCReg();
+ MCRegister PreviousDef = CopyOperands->Destination->getReg().asMCReg();
if (Src == PreviousSrc && Def == PreviousDef)
return true;
if (!TRI->isSubRegister(PreviousSrc, Src))
@@ -355,22 +406,26 @@ bool MachineCopyPropagation::eraseIfRedundant(MachineInstr &Copy,
return false;
// Search for an existing copy.
- MachineInstr *PrevCopy = Tracker.findAvailCopy(Copy, Def, *TRI);
+ MachineInstr *PrevCopy =
+ Tracker.findAvailCopy(Copy, Def, *TRI, *TII, UseCopyInstr);
if (!PrevCopy)
return false;
+ auto PrevCopyOperands = isCopyInstr(*PrevCopy, *TII, UseCopyInstr);
// Check that the existing copy uses the correct sub registers.
- if (PrevCopy->getOperand(0).isDead())
+ if (PrevCopyOperands->Destination->isDead())
return false;
- if (!isNopCopy(*PrevCopy, Src, Def, TRI))
+ if (!isNopCopy(*PrevCopy, Src, Def, TRI, TII, UseCopyInstr))
return false;
LLVM_DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; Copy.dump());
// Copy was redundantly redefining either Src or Def. Remove earlier kill
// flags between Copy and PrevCopy because the value will be reused now.
- assert(Copy.isCopy());
- Register CopyDef = Copy.getOperand(0).getReg();
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr);
+ assert(CopyOperands);
+
+ Register CopyDef = CopyOperands->Destination->getReg();
assert(CopyDef == Src || CopyDef == Def);
for (MachineInstr &MI :
make_range(PrevCopy->getIterator(), Copy.getIterator()))
@@ -384,7 +439,9 @@ bool MachineCopyPropagation::eraseIfRedundant(MachineInstr &Copy,
bool MachineCopyPropagation::isBackwardPropagatableRegClassCopy(
const MachineInstr &Copy, const MachineInstr &UseI, unsigned UseIdx) {
- Register Def = Copy.getOperand(0).getReg();
+
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr);
+ Register Def = CopyOperands->Destination->getReg();
if (const TargetRegisterClass *URC =
UseI.getRegClassConstraint(UseIdx, TII, TRI))
@@ -402,7 +459,8 @@ bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy,
const MachineInstr &UseI,
unsigned UseIdx) {
- Register CopySrcReg = Copy.getOperand(1).getReg();
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(Copy, *TII, UseCopyInstr);
+ Register CopySrcReg = CopyOperands->Source->getReg();
// If the new register meets the opcode register constraints, then allow
// forwarding.
@@ -410,7 +468,8 @@ bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy,
UseI.getRegClassConstraint(UseIdx, TII, TRI))
return URC->contains(CopySrcReg);
- if (!UseI.isCopy())
+ auto UseICopyOperands = isCopyInstr(UseI, *TII, UseCopyInstr);
+ if (!UseICopyOperands)
return false;
/// COPYs don't have register class constraints, so if the user instruction
@@ -433,7 +492,7 @@ bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy,
// Allow forwarding if src and dst belong to any common class, so long as they
// don't belong to any (possibly smaller) common class that requires copies to
// go via a
diff erent class.
- Register UseDstReg = UseI.getOperand(0).getReg();
+ Register UseDstReg = UseICopyOperands->Destination->getReg();
bool Found = false;
bool IsCrossClass = false;
for (const TargetRegisterClass *RC : TRI->regclasses()) {
@@ -451,7 +510,7 @@ bool MachineCopyPropagation::isForwardableRegClassCopy(const MachineInstr &Copy,
return true;
// The forwarded copy would be cross-class. Only do this if the original copy
// was also cross-class.
- Register CopyDstReg = Copy.getOperand(0).getReg();
+ Register CopyDstReg = CopyOperands->Destination->getReg();
for (const TargetRegisterClass *RC : TRI->regclasses()) {
if (RC->contains(CopySrcReg) && RC->contains(CopyDstReg) &&
TRI->getCrossCopyRegClass(RC) != RC)
@@ -523,13 +582,15 @@ void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
if (!MOUse.isRenamable())
continue;
- MachineInstr *Copy =
- Tracker.findAvailCopy(MI, MOUse.getReg().asMCReg(), *TRI);
+ MachineInstr *Copy = Tracker.findAvailCopy(MI, MOUse.getReg().asMCReg(),
+ *TRI, *TII, UseCopyInstr);
if (!Copy)
continue;
- Register CopyDstReg = Copy->getOperand(0).getReg();
- const MachineOperand &CopySrc = Copy->getOperand(1);
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*Copy, *TII, UseCopyInstr);
+ Register CopyDstReg = CopyOperands->Destination->getReg();
+ const MachineOperand &CopySrc = *CopyOperands->Source;
Register CopySrcReg = CopySrc.getReg();
// FIXME: Don't handle partial uses of wider COPYs yet.
@@ -553,7 +614,8 @@ void MachineCopyPropagation::forwardUses(MachineInstr &MI) {
// Check that the instruction is not a copy that partially overwrites the
// original copy source that we are about to use. The tracker mechanism
// cannot cope with that.
- if (MI.isCopy() && MI.modifiesRegister(CopySrcReg, TRI) &&
+ if (isCopyInstr(MI, *TII, UseCopyInstr) &&
+ MI.modifiesRegister(CopySrcReg, TRI) &&
!MI.definesRegister(CopySrcReg)) {
LLVM_DEBUG(dbgs() << "MCP: Copy source overlap with dest in " << MI);
continue;
@@ -592,14 +654,20 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
// Analyze copies (which don't overlap themselves).
- if (MI.isCopy() && !TRI->regsOverlap(MI.getOperand(0).getReg(),
- MI.getOperand(1).getReg())) {
- assert(MI.getOperand(0).getReg().isPhysical() &&
- MI.getOperand(1).getReg().isPhysical() &&
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
+ if (CopyOperands) {
+
+ Register RegSrc = CopyOperands->Source->getReg();
+ Register RegDef = CopyOperands->Destination->getReg();
+
+ if (TRI->regsOverlap(RegDef, RegSrc))
+ continue;
+
+ assert(RegDef.isPhysical() && RegSrc.isPhysical() &&
"MachineCopyPropagation should be run after register allocation!");
- MCRegister Def = MI.getOperand(0).getReg().asMCReg();
- MCRegister Src = MI.getOperand(1).getReg().asMCReg();
+ MCRegister Def = RegDef.asMCReg();
+ MCRegister Src = RegSrc.asMCReg();
// The two copies cancel out and the source of the first copy
// hasn't been overridden, eliminate the second one. e.g.
@@ -622,7 +690,8 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
forwardUses(MI);
// Src may have been changed by forwardUses()
- Src = MI.getOperand(1).getReg().asMCReg();
+ CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
+ Src = CopyOperands->Source->getReg().asMCReg();
// If Src is defined by a previous copy, the previous copy cannot be
// eliminated.
@@ -649,17 +718,17 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
// %xmm2 = copy %xmm0
// ...
// %xmm2 = copy %xmm9
- Tracker.clobberRegister(Def, *TRI);
+ Tracker.clobberRegister(Def, *TRI, *TII, UseCopyInstr);
for (const MachineOperand &MO : MI.implicit_operands()) {
if (!MO.isReg() || !MO.isDef())
continue;
MCRegister Reg = MO.getReg().asMCReg();
if (!Reg)
continue;
- Tracker.clobberRegister(Reg, *TRI);
+ Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
}
- Tracker.trackCopy(&MI, *TRI);
+ Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr);
continue;
}
@@ -673,7 +742,7 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
// later.
if (MO.isTied())
ReadRegister(Reg, MI, RegularUse);
- Tracker.clobberRegister(Reg, *TRI);
+ Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
}
forwardUses(MI);
@@ -709,7 +778,9 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
MaybeDeadCopies.begin();
DI != MaybeDeadCopies.end();) {
MachineInstr *MaybeDead = *DI;
- MCRegister Reg = MaybeDead->getOperand(0).getReg().asMCReg();
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*MaybeDead, *TII, UseCopyInstr);
+ MCRegister Reg = CopyOperands->Destination->getReg().asMCReg();
assert(!MRI->isReserved(Reg));
if (!RegMask->clobbersPhysReg(Reg)) {
@@ -722,7 +793,7 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
// Make sure we invalidate any entries in the copy maps before erasing
// the instruction.
- Tracker.clobberRegister(Reg, *TRI);
+ Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
// erase() will return the next valid iterator pointing to the next
// element after the erased one.
@@ -735,7 +806,7 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
// Any previous copy definition or reading the Defs is no longer available.
for (MCRegister Reg : Defs)
- Tracker.clobberRegister(Reg, *TRI);
+ Tracker.clobberRegister(Reg, *TRI, *TII, UseCopyInstr);
}
// If MBB doesn't have successors, delete the copies whose defs are not used.
@@ -745,12 +816,16 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
for (MachineInstr *MaybeDead : MaybeDeadCopies) {
LLVM_DEBUG(dbgs() << "MCP: Removing copy due to no live-out succ: ";
MaybeDead->dump());
- assert(!MRI->isReserved(MaybeDead->getOperand(0).getReg()));
+
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*MaybeDead, *TII, UseCopyInstr);
+ assert(CopyOperands);
+
+ Register SrcReg = CopyOperands->Source->getReg();
+ Register DestReg = CopyOperands->Destination->getReg();
+ assert(!MRI->isReserved(DestReg));
// Update matching debug values, if any.
- assert(MaybeDead->isCopy());
- Register SrcReg = MaybeDead->getOperand(1).getReg();
- Register DestReg = MaybeDead->getOperand(0).getReg();
SmallVector<MachineInstr *> MaybeDeadDbgUsers(
CopyDbgUsers[MaybeDead].begin(), CopyDbgUsers[MaybeDead].end());
MRI->updateDbgUsersToReg(DestReg.asMCReg(), SrcReg.asMCReg(),
@@ -768,10 +843,14 @@ void MachineCopyPropagation::ForwardCopyPropagateBlock(MachineBasicBlock &MBB) {
}
static bool isBackwardPropagatableCopy(MachineInstr &MI,
- const MachineRegisterInfo &MRI) {
- assert(MI.isCopy() && "MI is expected to be a COPY");
- Register Def = MI.getOperand(0).getReg();
- Register Src = MI.getOperand(1).getReg();
+ const MachineRegisterInfo &MRI,
+ const TargetInstrInfo &TII,
+ bool UseCopyInstr) {
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(MI, TII, UseCopyInstr);
+ assert(CopyOperands && "MI is expected to be a COPY");
+
+ Register Def = CopyOperands->Destination->getReg();
+ Register Src = CopyOperands->Source->getReg();
if (!Def || !Src)
return false;
@@ -779,7 +858,7 @@ static bool isBackwardPropagatableCopy(MachineInstr &MI,
if (MRI.isReserved(Def) || MRI.isReserved(Src))
return false;
- return MI.getOperand(1).isRenamable() && MI.getOperand(1).isKill();
+ return CopyOperands->Source->isRenamable() && CopyOperands->Source->isKill();
}
void MachineCopyPropagation::propagateDefs(MachineInstr &MI) {
@@ -804,13 +883,15 @@ void MachineCopyPropagation::propagateDefs(MachineInstr &MI) {
if (!MODef.isRenamable())
continue;
- MachineInstr *Copy =
- Tracker.findAvailBackwardCopy(MI, MODef.getReg().asMCReg(), *TRI);
+ MachineInstr *Copy = Tracker.findAvailBackwardCopy(
+ MI, MODef.getReg().asMCReg(), *TRI, *TII, UseCopyInstr);
if (!Copy)
continue;
- Register Def = Copy->getOperand(0).getReg();
- Register Src = Copy->getOperand(1).getReg();
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*Copy, *TII, UseCopyInstr);
+ Register Def = CopyOperands->Destination->getReg();
+ Register Src = CopyOperands->Source->getReg();
if (MODef.getReg() != Src)
continue;
@@ -829,7 +910,7 @@ void MachineCopyPropagation::propagateDefs(MachineInstr &MI) {
<< MI << " from " << *Copy);
MODef.setReg(Def);
- MODef.setIsRenamable(Copy->getOperand(0).isRenamable());
+ MODef.setIsRenamable(CopyOperands->Destination->isRenamable());
LLVM_DEBUG(dbgs() << "MCP: After replacement: " << MI << "\n");
MaybeDeadCopies.insert(Copy);
@@ -845,19 +926,23 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
for (MachineInstr &MI : llvm::make_early_inc_range(llvm::reverse(MBB))) {
// Ignore non-trivial COPYs.
- if (MI.isCopy() && MI.getNumOperands() == 2 &&
- !TRI->regsOverlap(MI.getOperand(0).getReg(),
- MI.getOperand(1).getReg())) {
+ Optional<DestSourcePair> CopyOperands = isCopyInstr(MI, *TII, UseCopyInstr);
+ if (CopyOperands) {
+ Register DefReg = CopyOperands->Destination->getReg();
+ Register SrcReg = CopyOperands->Source->getReg();
- MCRegister Def = MI.getOperand(0).getReg().asMCReg();
- MCRegister Src = MI.getOperand(1).getReg().asMCReg();
+ if (TRI->regsOverlap(DefReg, SrcReg))
+ continue;
+
+ MCRegister Def = DefReg.asMCReg();
+ MCRegister Src = SrcReg.asMCReg();
// Unlike forward cp, we don't invoke propagateDefs here,
// just let forward cp do COPY-to-COPY propagation.
- if (isBackwardPropagatableCopy(MI, *MRI)) {
- Tracker.invalidateRegister(Src, *TRI);
- Tracker.invalidateRegister(Def, *TRI);
- Tracker.trackCopy(&MI, *TRI);
+ if (isBackwardPropagatableCopy(MI, *MRI, *TII, UseCopyInstr)) {
+ Tracker.invalidateRegister(Src, *TRI, *TII, UseCopyInstr);
+ Tracker.invalidateRegister(Def, *TRI, *TII, UseCopyInstr);
+ Tracker.trackCopy(&MI, *TRI, *TII, UseCopyInstr);
continue;
}
}
@@ -868,7 +953,7 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
MCRegister Reg = MO.getReg().asMCReg();
if (!Reg)
continue;
- Tracker.invalidateRegister(Reg, *TRI);
+ Tracker.invalidateRegister(Reg, *TRI, *TII, UseCopyInstr);
}
propagateDefs(MI);
@@ -880,7 +965,8 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
continue;
if (MO.isDef())
- Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI);
+ Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI, *TII,
+ UseCopyInstr);
if (MO.readsReg()) {
if (MO.isDebug()) {
@@ -894,7 +980,8 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
}
}
} else {
- Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI);
+ Tracker.invalidateRegister(MO.getReg().asMCReg(), *TRI, *TII,
+ UseCopyInstr);
}
}
}
@@ -902,8 +989,10 @@ void MachineCopyPropagation::BackwardCopyPropagateBlock(
for (auto *Copy : MaybeDeadCopies) {
- Register Src = Copy->getOperand(1).getReg();
- Register Def = Copy->getOperand(0).getReg();
+ Optional<DestSourcePair> CopyOperands =
+ isCopyInstr(*Copy, *TII, UseCopyInstr);
+ Register Src = CopyOperands->Source->getReg();
+ Register Def = CopyOperands->Destination->getReg();
SmallVector<MachineInstr *> MaybeDeadDbgUsers(CopyDbgUsers[Copy].begin(),
CopyDbgUsers[Copy].end());
@@ -934,3 +1023,8 @@ bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) {
return Changed;
}
+
+MachineFunctionPass *
+llvm::createMachineCopyPropagationPass(bool UseCopyInstr = false) {
+ return new MachineCopyPropagation(UseCopyInstr);
+}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index bcfc350805547..3c205fd5c319b 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -34,6 +34,7 @@
#include "llvm/CodeGen/MIRParser/MIParser.h"
#include "llvm/CodeGen/MachineScheduler.h"
#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Function.h"
@@ -62,6 +63,11 @@ static cl::opt<bool>
cl::desc("Enable the conditional branch tuning pass"),
cl::init(true), cl::Hidden);
+static cl::opt<bool> EnableAArch64CopyPropagation(
+ "aarch64-enable-copy-propagation",
+ cl::desc("Enable the copy propagation with AArch64 copy instr"),
+ cl::init(true), cl::Hidden);
+
static cl::opt<bool> EnableMCR("aarch64-enable-mcr",
cl::desc("Enable the machine combiner pass"),
cl::init(true), cl::Hidden);
@@ -772,6 +778,10 @@ void AArch64PassConfig::addPreEmitPass() {
if (TM->getOptLevel() >= CodeGenOpt::Aggressive && EnableLoadStoreOpt)
addPass(createAArch64LoadStoreOptimizationPass());
+ if (TM->getOptLevel() >= CodeGenOpt::Aggressive &&
+ EnableAArch64CopyPropagation)
+ addPass(createMachineCopyPropagationPass(true));
+
addPass(createAArch64A53Fix835769());
if (EnableBranchTargets)
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 2f205f9ab3735..708c8bef290a1 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -200,6 +200,7 @@
; CHECK-NEXT: Insert XRay ops
; CHECK-NEXT: Implement the 'patchable-function' attribute
; CHECK-NEXT: AArch64 load / store optimization pass
+; CHECK-NEXT: Machine Copy Propagation Pass
; CHECK-NEXT: Workaround A53 erratum 835769 pass
; CHECK-NEXT: AArch64 Branch Targets
; CHECK-NEXT: Branch relaxation pass
diff --git a/llvm/test/CodeGen/AArch64/copyprop.ll b/llvm/test/CodeGen/AArch64/copyprop.ll
new file mode 100644
index 0000000000000..a605f23baba21
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/copyprop.ll
@@ -0,0 +1,40 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -O3 -mtriple=aarch64-- | FileCheck %s
+
+define void @copyprop_after_mbp(i32 %v, i32* %a, i32* %b, i32* %c, i32* %d) {
+; CHECK-LABEL: copyprop_after_mbp:
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp w0, #10
+; CHECK-NEXT: b.ne .LBB0_2
+; CHECK-NEXT: // %bb.1: // %bb.0
+; CHECK-NEXT: mov w9, #15
+; CHECK-NEXT: mov w8, #1
+; CHECK-NEXT: str w9, [x2]
+; CHECK-NEXT: mov w9, #12
+; CHECK-NEXT: str w8, [x1]
+; CHECK-NEXT: str w9, [x4]
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB0_2: // %bb.1
+; CHECK-NEXT: mov w9, #25
+; CHECK-NEXT: str w9, [x3]
+; CHECK-NEXT: mov w9, #12
+; CHECK-NEXT: str wzr, [x1]
+; CHECK-NEXT: str w9, [x4]
+; CHECK-NEXT: ret
+ %1 = icmp eq i32 %v, 10
+ br i1 %1, label %bb.0, label %bb.1
+
+bb.0:
+ store i32 15, i32* %b, align 4
+ br label %bb.2
+
+bb.1:
+ store i32 25, i32* %c, align 4
+ br label %bb.2
+
+bb.2:
+ %2 = phi i32 [ 1, %bb.0 ], [ 0, %bb.1 ]
+ store i32 %2, i32* %a, align 4
+ store i32 12, i32* %d, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/copyprop.mir b/llvm/test/CodeGen/AArch64/copyprop.mir
index e23002c569070..2454509d80862 100644
--- a/llvm/test/CodeGen/AArch64/copyprop.mir
+++ b/llvm/test/CodeGen/AArch64/copyprop.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=aarch64-linux-gnu -run-pass machine-cp -o - %s | FileCheck %s
+# RUN: llc -mtriple=aarch64-linux-gnu -run-pass machine-cp -mcp-use-is-copy-instr -o - %s | FileCheck %s
# Tests for MachineCopyPropagation copy forwarding.
---
# Simple forwarding.
@@ -102,3 +102,14 @@ body: |
$x0 = ADDXri $x0, 1, 0
$x2 = SUBXri renamable $x1, 1, 0
...
+---
+# CHECK-LABEL: name: test1_orr_as_copy
+# CHECK: STRBBui $wzr, killed renamable $x8, 36
+name: test1_orr_as_copy
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $x8
+ $w9 = ORRWrs $wzr, $wzr, 0
+ STRBBui killed renamable $w9, killed renamable $x8, 36
+...
More information about the llvm-commits
mailing list