[llvm] r357170 - AMDGPU: Make exec mask optimzations more resistant to block splits
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 28 07:01:39 PDT 2019
Author: arsenm
Date: Thu Mar 28 07:01:39 2019
New Revision: 357170
URL: http://llvm.org/viewvc/llvm-project?rev=357170&view=rev
Log:
AMDGPU: Make exec mask optimzations more resistant to block splits
Also improve the check for SALU instructions to also ignore
implicit_def and other fake instructions.
Modified:
llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp
llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h
llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
llvm/trunk/test/CodeGen/AMDGPU/collapse-endcf.mir
Modified: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp?rev=357170&r1=357169&r2=357170&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.cpp Thu Mar 28 07:01:39 2019
@@ -2453,6 +2453,27 @@ bool SIInstrInfo::hasUnwantedEffectsWhen
return false;
}
+bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI,
+ const MachineInstr &MI) const {
+ if (MI.isMetaInstruction())
+ return false;
+
+ // This won't read exec if this is an SGPR->SGPR copy.
+ if (MI.isCopyLike()) {
+ if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
+ return true;
+
+ // Make sure this isn't copying exec as a normal operand
+ return MI.readsRegister(AMDGPU::EXEC, &RI);
+ }
+
+ // Be conservative with any unhandled generic opcodes.
+ if (!isTargetSpecificOpcode(MI.getOpcode()))
+ return true;
+
+ return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
+}
+
bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
switch (Imm.getBitWidth()) {
case 32:
Modified: llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h?rev=357170&r1=357169&r2=357170&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIInstrInfo.h Thu Mar 28 07:01:39 2019
@@ -624,6 +624,10 @@ public:
/// Whether we must prevent this instruction from executing with EXEC = 0.
bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const;
+ /// Returns true if the instruction could potentially depend on the value of
+ /// exec. If false, exec dependencies may safely be ignored.
+ bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const;
+
bool isInlineConstant(const APInt &Imm) const;
bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const;
Modified: llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp?rev=357170&r1=357169&r2=357170&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp (original)
+++ llvm/trunk/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp Thu Mar 28 07:01:39 2019
@@ -33,10 +33,22 @@ using namespace llvm;
namespace {
class SIOptimizeExecMaskingPreRA : public MachineFunctionPass {
+private:
+ const SIRegisterInfo *TRI;
+ const SIInstrInfo *TII;
+ MachineRegisterInfo *MRI;
+
public:
- static char ID;
+ MachineBasicBlock::iterator skipIgnoreExecInsts(
+ MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const;
+
+ MachineBasicBlock::iterator skipIgnoreExecInstsTrivialSucc(
+ MachineBasicBlock *&MBB,
+ MachineBasicBlock::iterator It) const;
public:
+ static char ID;
+
SIOptimizeExecMaskingPreRA() : MachineFunctionPass(ID) {
initializeSIOptimizeExecMaskingPreRAPass(*PassRegistry::getPassRegistry());
}
@@ -102,6 +114,45 @@ static MachineInstr* getOrExecSource(con
return SaveExecInst;
}
+/// Skip over instructions that don't care about the exec mask.
+MachineBasicBlock::iterator SIOptimizeExecMaskingPreRA::skipIgnoreExecInsts(
+ MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const {
+ for ( ; I != E; ++I) {
+ if (TII->mayReadEXEC(*MRI, *I))
+ break;
+ }
+
+ return I;
+}
+
+// Skip to the next instruction, ignoring debug instructions, and trivial block
+// boundaries (blocks that have one (typically fallthrough) successor, and the
+// successor has one predecessor.
+MachineBasicBlock::iterator
+SIOptimizeExecMaskingPreRA::skipIgnoreExecInstsTrivialSucc(
+ MachineBasicBlock *&MBB,
+ MachineBasicBlock::iterator It) const {
+
+ do {
+ It = skipIgnoreExecInsts(It, MBB->end());
+ if (It != MBB->end() || MBB->succ_size() != 1)
+ break;
+
+ // If there is one trivial successor, advance to the next block.
+ MachineBasicBlock *Succ = *MBB->succ_begin();
+
+ // TODO: Is this really necessary?
+ if (!MBB->isLayoutSuccessor(Succ))
+ break;
+
+ It = Succ->begin();
+ MBB = Succ;
+ } while (true);
+
+ return It;
+}
+
+
// Optimize sequence
// %sel = V_CNDMASK_B32_e64 0, 1, %cc
// %cmp = V_CMP_NE_U32 1, %1
@@ -227,8 +278,10 @@ bool SIOptimizeExecMaskingPreRA::runOnMa
return false;
const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
- const SIRegisterInfo *TRI = ST.getRegisterInfo();
- const SIInstrInfo *TII = ST.getInstrInfo();
+ TRI = ST.getRegisterInfo();
+ TII = ST.getInstrInfo();
+ MRI = &MF.getRegInfo();
+
MachineRegisterInfo &MRI = MF.getRegInfo();
LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI});
@@ -313,25 +366,9 @@ bool SIOptimizeExecMaskingPreRA::runOnMa
if (MBB.succ_size() != 1 || Lead == E || !isEndCF(*Lead, TRI))
continue;
- const MachineBasicBlock* Succ = *MBB.succ_begin();
- if (!MBB.isLayoutSuccessor(Succ))
- continue;
-
- auto I = std::next(Lead);
-
- for ( ; I != E; ++I) {
- if (I->isDebugInstr())
- continue;
-
- if (!TII->isSALU(*I) || I->readsRegister(AMDGPU::EXEC, TRI))
- break;
- }
-
- if (I != E)
- continue;
-
- auto NextLead = skipDebugInstructionsForward(Succ->begin(), Succ->end());
- if (NextLead == Succ->end() || !isEndCF(*NextLead, TRI) ||
+ MachineBasicBlock *TmpMBB = &MBB;
+ auto NextLead = skipIgnoreExecInstsTrivialSucc(TmpMBB, std::next(Lead));
+ if (NextLead == TmpMBB->end() || !isEndCF(*NextLead, TRI) ||
!getOrExecSource(*NextLead, *TII, MRI))
continue;
Modified: llvm/trunk/test/CodeGen/AMDGPU/collapse-endcf.mir
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AMDGPU/collapse-endcf.mir?rev=357170&r1=357169&r2=357170&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/AMDGPU/collapse-endcf.mir (original)
+++ llvm/trunk/test/CodeGen/AMDGPU/collapse-endcf.mir Thu Mar 28 07:01:39 2019
@@ -143,8 +143,7 @@ body: |
; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0
; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
- ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
- ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+ ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]]
; GCN: SI_MASK_BRANCH %bb.3, implicit $exec
; GCN: S_BRANCH %bb.2
@@ -156,7 +155,6 @@ body: |
; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: bb.3:
; GCN: successors: %bb.4(0x80000000)
- ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
; GCN: bb.4:
; GCN: successors: %bb.5(0x80000000)
; GCN: bb.5:
@@ -253,8 +251,7 @@ body: |
; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0
; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
- ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
- ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+ ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]]
; GCN: SI_MASK_BRANCH %bb.3, implicit $exec
; GCN: S_BRANCH %bb.2
@@ -266,7 +263,6 @@ body: |
; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: bb.3:
; GCN: successors: %bb.4(0x80000000)
- ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
; GCN: bb.4:
; GCN: successors: %bb.5(0x80000000)
; GCN: DBG_VALUE
@@ -477,8 +473,7 @@ body: |
; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0
; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
- ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
- ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+ ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]]
; GCN: SI_MASK_BRANCH %bb.3, implicit $exec
; GCN: S_BRANCH %bb.2
@@ -490,7 +485,6 @@ body: |
; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
; GCN: bb.3:
; GCN: successors: %bb.4(0x80000000)
- ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
; GCN: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
; GCN: [[S_BREV_B32_:%[0-9]+]]:sgpr_32 = S_BREV_B32 [[DEF]]
; GCN: KILL [[DEF]]
@@ -773,3 +767,116 @@ body: |
...
+# There's no real reason this can't be handled, but isn't now.
+---
+name: simple_nested_if_not_layout_successor
+tracksRegLiveness: true
+liveins:
+ - { reg: '$vgpr0', virtual-reg: '%0' }
+ - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' }
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ ; GCN-LABEL: name: simple_nested_if_not_layout_successor
+ ; GCN: bb.0:
+ ; GCN: successors: %bb.1(0x40000000), %bb.4(0x40000000)
+ ; GCN: liveins: $vgpr0, $sgpr0_sgpr1
+ ; GCN: [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1
+ ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GCN: [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec
+ ; GCN: [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+ ; GCN: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc
+ ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_]]
+ ; GCN: SI_MASK_BRANCH %bb.4, implicit $exec
+ ; GCN: S_BRANCH %bb.1
+ ; GCN: bb.1:
+ ; GCN: successors: %bb.2(0x40000000), %bb.3(0x40000000)
+ ; GCN: undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+ ; GCN: undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec
+ ; GCN: %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN: [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1
+ ; GCN: undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+ ; GCN: %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
+ ; GCN: %5.sub3:sgpr_128 = S_MOV_B32 61440
+ ; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0
+ ; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+ ; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
+ ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
+ ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+ ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]]
+ ; GCN: SI_MASK_BRANCH %bb.3, implicit $exec
+ ; GCN: S_BRANCH %bb.2
+ ; GCN: bb.2:
+ ; GCN: successors: %bb.3(0x80000000)
+ ; GCN: %5.sub0:sgpr_128 = COPY %5.sub2
+ ; GCN: %5.sub1:sgpr_128 = COPY %5.sub2
+ ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ ; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+ ; GCN: bb.3:
+ ; GCN: successors: %bb.5(0x80000000)
+ ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
+ ; GCN: S_BRANCH %bb.5
+ ; GCN: bb.4:
+ ; GCN: $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc
+ ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+ ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; GCN: $m0 = S_MOV_B32 -1
+ ; GCN: DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+ ; GCN: S_ENDPGM 0
+ ; GCN: bb.5:
+ ; GCN: successors: %bb.4(0x80000000)
+ ; GCN: S_BRANCH %bb.4
+ bb.0:
+ successors: %bb.1, %bb.4
+ liveins: $vgpr0, $sgpr0_sgpr1
+
+ %1:sgpr_64 = COPY $sgpr0_sgpr1
+ %0:vgpr_32 = COPY $vgpr0
+ %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec
+ %3:sreg_64 = COPY $exec, implicit-def $exec
+ %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc
+ $exec = S_MOV_B64_term %4
+ SI_MASK_BRANCH %bb.4, implicit $exec
+ S_BRANCH %bb.1
+
+ bb.1:
+ successors: %bb.2, %bb.3
+
+ undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4)
+ undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec
+ %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+ %7:vgpr_32 = COPY %5.sub1
+ undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec
+ %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
+ %5.sub3:sgpr_128 = S_MOV_B32 61440
+ %5.sub2:sgpr_128 = S_MOV_B32 0
+ BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+ %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
+ %12:sreg_64 = COPY $exec, implicit-def $exec
+ %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
+ $exec = S_MOV_B64_term %13
+ SI_MASK_BRANCH %bb.3, implicit $exec
+ S_BRANCH %bb.2
+
+ bb.2:
+ %5.sub0:sgpr_128 = COPY %5.sub2
+ %5.sub1:sgpr_128 = COPY %5.sub2
+ %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+ BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+
+ bb.3:
+ $exec = S_OR_B64 $exec, %12, implicit-def $scc
+ S_BRANCH %bb.5
+
+ bb.4:
+ $exec = S_OR_B64 $exec, %3, implicit-def $scc
+ %15:vgpr_32 = V_MOV_B32_e32 3, implicit $exec
+ %16:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ $m0 = S_MOV_B32 -1
+ DS_WRITE_B32 %16, %15, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
+ S_ENDPGM 0
+
+ bb.5:
+ S_BRANCH %bb.4
+
+...
More information about the llvm-commits
mailing list