[llvm] [AMDGPU] Add MachinePipeliner support for AMDGPU (PR #212536)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 29 09:51:33 PDT 2026
https://github.com/hjagasiaAMD updated https://github.com/llvm/llvm-project/pull/212536
>From 6f79f56806d859fb20f7507f19ed726b45323d4c Mon Sep 17 00:00:00 2001
From: Harsha Jagasia <harsha.jagasia at amd.com>
Date: Thu, 18 Jun 2026 19:24:21 +0000
Subject: [PATCH] [AMDGPU] Add MachinePipeliner support for AMDGPU
Implement the target hooks to enable MachinePipeliner for AMDGPU. The
pass is off by default and can be enabled with -amdgpu-enable-pipeliner
at -O2 and above.
Only uniform, single-basic-block counted loops with a scalar (SCC)
back-edge are pipelined; loops with a divergent (VCC/EXEC) back-edge, or
containing calls or inline asm, are rejected. Code is generated by the
default modulo schedule expander (DFA, window scheduler, and MVE
expansion are disabled for AMDGPU).
Validated on gfx942 and gfx950 with Composable Kernel and Triton workloads.
---
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 7 +
llvm/lib/Target/AMDGPU/GCNSubtarget.h | 4 +
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 93 +++++++++
llvm/lib/Target/AMDGPU/SIInstrInfo.h | 3 +
.../AMDGPU/swp-amdgpu-pipeline-epilog-phi.mir | 134 +++++++++++++
.../swp-amdgpu-pipeline-gen-structure.mir | 71 +++++++
.../swp-amdgpu-pipeline-loop-carried-mem.mir | 58 ++++++
.../AMDGPU/swp-amdgpu-pipeline-max-mii.ll | 32 ++++
.../AMDGPU/swp-amdgpu-pipeline-minimal.ll | 177 ++++++++++++++++++
...p-amdgpu-pipeline-neg-cond-not-in-loop.mir | 36 ++++
...amdgpu-pipeline-neg-divergent-backedge.mir | 28 +++
.../swp-amdgpu-pipeline-neg-inlineasm.mir | 36 ++++
.../swp-amdgpu-pipeline-neg-multi-bb.ll | 25 +++
.../swp-amdgpu-pipeline-neg-no-exit.mir | 33 ++++
...wp-amdgpu-pipeline-neg-not-cond-branch.mir | 33 ++++
.../swp-amdgpu-pipeline-resmii-issue-bound.ll | 41 ++++
.../AMDGPU/swp-amdgpu-pipeline-resmii-mfma.ll | 32 ++++
.../swp-amdgpu-pipeline-scalar-accumulate.ll | 33 ++++
...swp-amdgpu-pipeline-unpipeline-compare.mir | 48 +++++
.../swp-amdgpu-pipeline-vector-accumulate.ll | 43 +++++
20 files changed, 967 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-epilog-phi.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-gen-structure.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-loop-carried-mem.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-max-mii.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-minimal.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-cond-not-in-loop.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-divergent-backedge.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-inlineasm.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-multi-bb.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-no-exit.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-not-cond-branch.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-issue-bound.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-mfma.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-scalar-accumulate.ll
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-unpipeline-compare.mir
create mode 100644 llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-vector-accumulate.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 30ea659cbf322..14809a10ebad2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -653,6 +653,11 @@ static cl::opt<bool> EnableUniformIntrinsicCombine(
cl::desc("Enable/Disable the Uniform Intrinsic Combine Pass"),
cl::init(true), cl::Hidden);
+static cl::opt<bool>
+ EnableMachinePipeliner("amdgpu-enable-pipeliner",
+ cl::desc("Enable Machine Pipeliner for AMDGCN"),
+ cl::init(false), cl::Hidden);
+
extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
// Register the target
RegisterTargetMachine<R600TargetMachine> X(getTheR600Target());
@@ -1832,6 +1837,8 @@ void GCNPassConfig::addFastRegAlloc() {
void GCNPassConfig::addPreRegAlloc() {
if (getOptLevel() != CodeGenOptLevel::None)
addPass(&AMDGPUPrepareAGPRAllocLegacyID);
+ if (getOptLevel() >= CodeGenOptLevel::Default && EnableMachinePipeliner)
+ addPass(&MachinePipelinerID);
}
void GCNPassConfig::addOptimizedRegAlloc() {
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 8557d7a218d60..ecbd2c80332cf 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1040,6 +1040,10 @@ class GCNSubtarget final : public AMDGPUGenSubtargetInfo,
bool requiresWaitOnWorkgroupReleaseFence(bool TgSplit) const {
return getGeneration() >= GFX10 || TgSplit;
}
+
+ bool useDFAforSMS() const override { return false; }
+
+ bool enableWindowScheduler() const override { return false; }
};
class GCNUserSGPRUsageInfo {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index b2c3772cb0041..be49e5285be60 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -3290,6 +3290,99 @@ bool SIInstrInfo::reverseBranchCondition(
return true;
}
+namespace {
+class AMDGPUPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
+private:
+ /// The compare instruction for loop control
+ const MachineInstr *CmpInst;
+ /// The normalized condition used by createTripCountGreaterCondition()
+ SmallVector<MachineOperand, 2> Cond;
+
+public:
+ AMDGPUPipelinerLoopInfo(MachineInstr *CmpInst,
+ const SmallVectorImpl<MachineOperand> &Cond)
+ : CmpInst(CmpInst), Cond(Cond.begin(), Cond.end()) {}
+
+ bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
+ return CmpInst && MI == CmpInst;
+ }
+
+ std::optional<bool> createTripCountGreaterCondition(
+ int TC, MachineBasicBlock &MBB,
+ SmallVectorImpl<MachineOperand> &CondParam) override {
+ CondParam = this->Cond;
+ return {};
+ }
+
+ void adjustTripCount(int TripCountAdjust) override {}
+
+ void setPreheader(MachineBasicBlock *NewPreheader) override {}
+};
+} // namespace
+
+std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+SIInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+ MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+ SmallVector<MachineOperand, 2> Cond;
+ // Unanalyzable terminator.
+ if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
+ return nullptr;
+
+ // Infinite loop.
+ if (TBB == LoopBB && FBB == LoopBB)
+ return nullptr;
+
+ // Must be conditional branch
+ if (FBB == nullptr)
+ return nullptr;
+
+ assert((TBB == LoopBB || FBB == LoopBB) &&
+ "The Loop must be a single-basic-block loop");
+
+ // Divergent (VCC/EXEC) back-edge; only uniform scalar loops are supported.
+ BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
+ if (Pred != SCC_TRUE && Pred != SCC_FALSE)
+ return nullptr;
+
+ // Convergent ops with an opaque EXEC context.
+ for (const MachineInstr &MI : *LoopBB)
+ if (MI.isCall() || MI.isInlineAsm())
+ return nullptr;
+
+ // Normalization for createTripCountGreaterCondition(): make Cond mean
+ // "exit the loop" so the expander emits correct prolog guard branches.
+ if (TBB == LoopBB)
+ reverseBranchCondition(Cond);
+
+ MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
+ MachineBasicBlock::const_instr_iterator E = LoopBB->instr_begin();
+ MachineInstr *CmpInst = nullptr;
+ while (--I != E) {
+ for (MachineInstr::const_mop_iterator MOI = I->operands_begin(),
+ MOE = I->operands_end();
+ MOI != MOE; ++MOI) {
+ if (MOI->isReg() && MOI->isDef() && MOI->getReg() == Cond[1].getReg()) {
+ CmpInst = &*I;
+ break;
+ }
+ }
+ if (CmpInst)
+ break;
+ }
+
+ // No in-loop compare defining the branch condition.
+ if (!CmpInst)
+ return nullptr;
+ // Condition defined by a PHI, not a compare.
+ if (CmpInst->isPHI())
+ return nullptr;
+
+ assert(CmpInst->modifiesRegister(AMDGPU::SCC, &RI) &&
+ "uniform (SCC) loop condition must be defined by an in-block compare");
+
+ return std::make_unique<AMDGPUPipelinerLoopInfo>(CmpInst, Cond);
+}
+
bool SIInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond,
Register DstReg, Register TrueReg,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 4c8641a6091d7..ab79aaa9a6d62 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -425,6 +425,9 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
bool reverseBranchCondition(
SmallVectorImpl<MachineOperand> &Cond) const override;
+ std::unique_ptr<PipelinerLoopInfo>
+ analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
+
bool canInsertSelect(const MachineBasicBlock &MBB,
ArrayRef<MachineOperand> Cond, Register DstReg,
Register TrueReg, Register FalseReg, int &CondCycles,
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-epilog-phi.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-epilog-phi.mir
new file mode 100644
index 0000000000000..1c1590eeb9822
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-epilog-phi.mir
@@ -0,0 +1,134 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -pipeliner-force-ii=12 %s -o - | FileCheck %s
+# Two independent recurrences (a sum and a product) must be carried across the stage
+# boundary without being swapped: each accumulator's kernel result has to flow through
+# its own epilog PHI and drain op into its own store.
+
+---
+name: swp_amdgpu_pipeline_epilog_phi
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr4_sgpr5' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+body: |
+ ; CHECK-LABEL: name: swp_amdgpu_pipeline_epilog_phi
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: successors: %bb.3(0x80000000)
+ ; CHECK-NEXT: liveins: $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s128), addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4)
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[S_LOAD_DWORDX4_IMM]].sub0, %subreg.sub0, [[S_LOAD_DWORDX4_IMM]].sub1, %subreg.sub1
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE [[S_LOAD_DWORDX4_IMM]].sub2, %subreg.sub0, [[S_LOAD_DWORDX4_IMM]].sub3, %subreg.sub1
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; CHECK-NEXT: [[AV_MOV_:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec
+ ; CHECK-NEXT: [[AV_MOV_1:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.3:
+ ; CHECK-NEXT: successors: %bb.4(0x80000000), %bb.5(0x00000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[S_ASHR_I32_:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_MOV_B32_]], 31, implicit-def dead $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[S_ASHR_I32_]], %subreg.sub1
+ ; CHECK-NEXT: [[S_LSHL_B64_:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[REG_SEQUENCE2]], 2, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE]].sub0, [[S_LSHL_B64_]].sub0, implicit-def $scc
+ ; CHECK-NEXT: [[S_ADDC_U32_:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE]].sub1, [[S_LSHL_B64_]].sub1, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_]], %subreg.sub0, [[S_ADDC_U32_]], %subreg.sub1
+ ; CHECK-NEXT: [[S_ADD_U32_1:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE1]].sub0, [[S_LSHL_B64_]].sub0, implicit-def $scc
+ ; CHECK-NEXT: [[S_ADDC_U32_1:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE1]].sub1, [[S_LSHL_B64_]].sub1, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_1]], %subreg.sub0, [[S_ADDC_U32_1]], %subreg.sub1
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE3]], 0, 0 :: (load (s32), addrspace 1)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE4]], 0, 0 :: (load (s32), addrspace 1)
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_MOV_B32_]], 1, implicit-def dead $scc
+ ; CHECK-NEXT: S_CMP_LT_I32 [[S_ADD_I32_]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC0 %bb.5, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.4
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.4:
+ ; CHECK-NEXT: successors: %bb.5(0x40000000), %bb.4(0x40000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI:%[0-9]+]]:sreg_32 = PHI [[S_ADD_I32_]], %bb.3, %46, %bb.4
+ ; CHECK-NEXT: [[PHI1:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_1]], %bb.3, %51, %bb.4
+ ; CHECK-NEXT: [[PHI2:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_]], %bb.3, %49, %bb.4
+ ; CHECK-NEXT: [[PHI3:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM1]], %bb.3, %50, %bb.4
+ ; CHECK-NEXT: [[PHI4:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM2]], %bb.3, %48, %bb.4
+ ; CHECK-NEXT: [[S_ASHR_I32_1:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[PHI]], 31, implicit-def dead $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE5:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[PHI]], %subreg.sub0, [[S_ASHR_I32_1]], %subreg.sub1
+ ; CHECK-NEXT: [[S_LSHL_B64_1:%[0-9]+]]:sreg_64 = S_LSHL_B64 [[REG_SEQUENCE5]], 2, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_ADD_U32_2:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE]].sub0, [[S_LSHL_B64_1]].sub0, implicit-def $scc
+ ; CHECK-NEXT: [[S_ADDC_U32_2:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE]].sub1, [[S_LSHL_B64_1]].sub1, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]]:sreg_32 = S_ADD_U32 [[REG_SEQUENCE1]].sub0, [[S_LSHL_B64_1]].sub0, implicit-def $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE6:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_2]], %subreg.sub0, [[S_ADDC_U32_2]], %subreg.sub1
+ ; CHECK-NEXT: [[S_ADDC_U32_3:%[0-9]+]]:sreg_32 = S_ADDC_U32 [[REG_SEQUENCE1]].sub1, [[S_LSHL_B64_1]].sub1, implicit-def $scc, implicit $scc
+ ; CHECK-NEXT: [[S_ADD_I32_1:%[0-9]+]]:sreg_32 = S_ADD_I32 [[PHI]], 1, implicit-def dead $scc
+ ; CHECK-NEXT: [[REG_SEQUENCE7:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[S_ADD_U32_3]], %subreg.sub0, [[S_ADDC_U32_3]], %subreg.sub1
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM3:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE7]], 0, 0 :: (load (s32), addrspace 1)
+ ; CHECK-NEXT: [[V_MUL_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e32 [[PHI4]], [[PHI2]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[REG_SEQUENCE6]], 0, 0 :: (load (s32), addrspace 1)
+ ; CHECK-NEXT: [[V_ADD_F32_e32_:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e32 [[PHI3]], [[PHI1]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_CMP_LT_I32 [[S_ADD_I32_1]], [[S_LOAD_DWORD_IMM]], implicit-def $scc
+ ; CHECK-NEXT: S_CBRANCH_SCC1 %bb.4, implicit $scc
+ ; CHECK-NEXT: S_BRANCH %bb.5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.5:
+ ; CHECK-NEXT: successors: %bb.2(0x80000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[PHI5:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_1]], %bb.3, [[V_ADD_F32_e32_]], %bb.4
+ ; CHECK-NEXT: [[PHI6:%[0-9]+]]:vgpr_32 = PHI [[AV_MOV_]], %bb.3, [[V_MUL_F32_e32_]], %bb.4
+ ; CHECK-NEXT: [[PHI7:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM1]], %bb.3, [[S_LOAD_DWORD_IMM4]], %bb.4
+ ; CHECK-NEXT: [[PHI8:%[0-9]+]]:sgpr_32 = PHI [[S_LOAD_DWORD_IMM2]], %bb.3, [[S_LOAD_DWORD_IMM3]], %bb.4
+ ; CHECK-NEXT: [[V_ADD_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_ADD_F32_e32 [[PHI7]], [[PHI5]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MUL_F32_e32_1:%[0-9]+]]:vgpr_32 = nofpexcept V_MUL_F32_e32 [[PHI8]], [[PHI6]], implicit $mode, implicit $exec
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: [[AV_MOV_2:%[0-9]+]]:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[AV_MOV_2]], [[V_ADD_F32_e32_1]], [[REG_SEQUENCE]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR [[AV_MOV_2]], [[V_MUL_F32_e32_1]], [[REG_SEQUENCE1]], 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+ bb.0:
+ liveins: $sgpr4_sgpr5
+
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sgpr_128 = S_LOAD_DWORDX4_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s128), addrspace 4)
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0(p4), 16, 0 :: (dereferenceable invariant load (s32), align 16, addrspace 4)
+ %3:sreg_64_xexec_xnull = REG_SEQUENCE %1.sub0, %subreg.sub0, %1.sub1, %subreg.sub1
+ %4:sreg_64_xexec_xnull = REG_SEQUENCE %1.sub2, %subreg.sub0, %1.sub3, %subreg.sub1
+ %5:sreg_32 = S_MOV_B32 0
+ %6:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 1065353216, implicit $exec
+ %7:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %8:sreg_32 = PHI %5, %bb.0, %9, %bb.1
+ %10:vgpr_32 = PHI %7, %bb.0, %11, %bb.1
+ %12:vgpr_32 = PHI %6, %bb.0, %13, %bb.1
+ %14:sreg_32_xm0 = S_ASHR_I32 %8, 31, implicit-def dead $scc
+ %15:sreg_64 = REG_SEQUENCE %8, %subreg.sub0, %14, %subreg.sub1
+ %16:sreg_64 = S_LSHL_B64 killed %15, 2, implicit-def dead $scc
+ %17:sreg_32 = S_ADD_U32 %3.sub0, %16.sub0, implicit-def $scc
+ %18:sreg_32 = S_ADDC_U32 %3.sub1, %16.sub1, implicit-def $scc, implicit $scc
+ %19:sreg_64 = REG_SEQUENCE %17, %subreg.sub0, %18, %subreg.sub1
+ %20:sreg_32 = S_ADD_U32 %4.sub0, %16.sub0, implicit-def $scc
+ %21:sreg_32 = S_ADDC_U32 %4.sub1, %16.sub1, implicit-def $scc, implicit $scc
+ %22:sreg_64 = REG_SEQUENCE %20, %subreg.sub0, %21, %subreg.sub1
+ %23:sgpr_32 = S_LOAD_DWORD_IMM killed %19, 0, 0 :: (load (s32), addrspace 1)
+ %24:sgpr_32 = S_LOAD_DWORD_IMM killed %22, 0, 0 :: (load (s32), addrspace 1)
+ %11:vgpr_32 = nofpexcept V_ADD_F32_e32 %23, %10, implicit $mode, implicit $exec
+ %13:vgpr_32 = nofpexcept V_MUL_F32_e32 %24, %12, implicit $mode, implicit $exec
+ %9:sreg_32 = S_ADD_I32 %8, 1, implicit-def dead $scc
+ S_CMP_LT_I32 %9, %2, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ %25:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %25, %11, %3, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ GLOBAL_STORE_DWORD_SADDR %25, %13, %4, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-gen-structure.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-gen-structure.mir
new file mode 100644
index 0000000000000..7bad3042e571a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-gen-structure.mir
@@ -0,0 +1,71 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -pipeliner-force-ii=8 %s -o - | FileCheck %s
+# Verify the generated prolog / steady-state kernel / epilog structure of a pipelined loop.
+
+# CHECK-LABEL: name: swp_amdgpu_pipeline_gen_structure
+
+# Prolog: peeled first-iteration load, then a guard that skips the kernel to the epilog.
+# CHECK: bb.3:
+# CHECK: S_LOAD_DWORD_IMM
+# CHECK: S_CBRANCH_SCC0 %bb.5
+
+# Steady-state kernel: loop-carried PHIs, with the next-iteration load overlapping the
+# current-iteration accumulate.
+# CHECK: bb.4:
+# CHECK-DAG: {{%[0-9]+}}:sreg_32 = PHI {{%[0-9]+}}, %bb.3, {{%[0-9]+}}, %bb.4
+# CHECK-DAG: {{%[0-9]+}}:vgpr_32 = PHI {{%[0-9]+}}, %bb.3, {{%[0-9]+}}, %bb.4
+# CHECK-DAG: {{%[0-9]+}}:sgpr_32 = PHI {{%[0-9]+}}, %bb.3, {{%[0-9]+}}, %bb.4
+# CHECK: S_LOAD_DWORD_IMM
+# CHECK: V_ADD_F32_e32
+# CHECK: S_CBRANCH_SCC1 %bb.4
+
+# Epilog: drains the final accumulate for the last in-flight iteration.
+# CHECK: bb.5:
+# CHECK: {{%[0-9]+}}:vgpr_32 = PHI {{%[0-9]+}}, %bb.3, {{%[0-9]+}}, %bb.4
+# CHECK: V_ADD_F32_e32
+# CHECK: bb.2:
+# CHECK: GLOBAL_STORE_DWORD_SADDR
+# CHECK: S_ENDPGM
+
+---
+name: swp_amdgpu_pipeline_gen_structure
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr4_sgpr5' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0(p4), 8, 0 :: (dereferenceable invariant load (s32), align 8, addrspace 4)
+ %3:sreg_32 = S_MOV_B32 0
+ %4:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %5:sreg_32 = PHI %3, %bb.0, %6, %bb.1
+ %7:vgpr_32 = PHI %4, %bb.0, %8, %bb.1
+ %9:sreg_32_xm0 = S_ASHR_I32 %5, 31, implicit-def dead $scc
+ %10:sreg_64 = REG_SEQUENCE %5, %subreg.sub0, %9, %subreg.sub1
+ %11:sreg_64 = S_LSHL_B64 %10, 2, implicit-def dead $scc
+ %12:sreg_32 = S_ADD_U32 %1.sub0, %11.sub0, implicit-def $scc
+ %13:sreg_32 = S_ADDC_U32 %1.sub1, %11.sub1, implicit-def $scc, implicit $scc
+ %14:sreg_64 = REG_SEQUENCE %12, %subreg.sub0, %13, %subreg.sub1
+ %15:sgpr_32 = S_LOAD_DWORD_IMM %14, 0, 0 :: (load (s32), addrspace 1)
+ %8:vgpr_32 = nofpexcept V_ADD_F32_e32 %15, %7, implicit $mode, implicit $exec
+ %6:sreg_32 = S_ADD_I32 %5, 1, implicit-def dead $scc
+ S_CMP_LT_I32 %6, %2, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ %16:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ GLOBAL_STORE_DWORD_SADDR %16, %8, %1, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-loop-carried-mem.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-loop-carried-mem.mir
new file mode 100644
index 0000000000000..9dc50350a8d6c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-loop-carried-mem.mir
@@ -0,0 +1,58 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# A loop that loads a[i] and stores a[i+1] must get a loop-carried memory dependence
+# between the load and the store, so they are not reordered across iterations.
+
+# CHECK: SU([[LOAD:[0-9]+]]): {{.*}}GLOBAL_LOAD_DWORD_SADDR
+# CHECK: SU([[STORE:[0-9]+]]): GLOBAL_STORE_DWORD_SADDR
+# CHECK: ===== Loop Carried Edges Begin =====
+# CHECK: Loop carried edges from SU([[LOAD]])
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU([[STORE]])
+# CHECK: Loop carried edges from SU([[STORE]])
+# CHECK-NEXT: Order
+# CHECK-NEXT: SU([[LOAD]])
+# CHECK: ===== Loop Carried Edges End =====
+
+---
+name: swp_amdgpu_pipeline_loop_carried_mem
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr4_sgpr5' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0(p4), 8, 0 :: (dereferenceable invariant load (s32), align 8, addrspace 4)
+ %3:sreg_32 = S_ADD_U32 %1.sub0, 4, implicit-def $scc
+ %4:sreg_32 = S_ADDC_U32 %1.sub1, 0, implicit-def $scc, implicit $scc
+ %5:sreg_64 = REG_SEQUENCE %3, %subreg.sub0, %4, %subreg.sub1
+ %6:sreg_32 = S_MOV_B32 0
+ %7:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %8:sreg_64_xexec_xnull = PHI %5, %bb.0, %9, %bb.1
+ %10:sreg_32 = PHI %6, %bb.0, %11, %bb.1
+ %12:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %8, %7, -4, 0, implicit $exec :: (load (s32), addrspace 1)
+ %13:vgpr_32 = nofpexcept V_ADD_F32_e32 1065353216, killed %12, implicit $mode, implicit $exec
+ %11:sreg_32 = S_ADD_I32 %10, 1, implicit-def dead $scc
+ GLOBAL_STORE_DWORD_SADDR %7, killed %13, %8, 0, 0, implicit $exec :: (store (s32), addrspace 1)
+ %14:sreg_32 = S_ADD_U32 %8.sub0, 4, implicit-def $scc
+ %15:sreg_32 = S_ADDC_U32 %8.sub1, 0, implicit-def $scc, implicit $scc
+ %9:sreg_64 = REG_SEQUENCE %14, %subreg.sub0, %15, %subreg.sub1
+ S_CMP_LT_I32 %11, %2, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-max-mii.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-max-mii.ll
new file mode 100644
index 0000000000000..f00a50d1d429e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-max-mii.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -pass-remarks-analysis=pipeliner %s -filetype=null 2>&1 | FileCheck %s --check-prefix=DEFAULT
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -pipeliner-max-mii=64 -pass-remarks-analysis=pipeliner %s -filetype=null 2>&1 | FileCheck %s --check-prefix=RAISED
+
+; This loop's MII is 32: two independent accumulators each issue one MFMA per
+; iteration, and each MFMA holds the XDL pipe for 16 cycles (res=32). That
+; exceeds the generic default cap of 27. At default, pipeliner aborts and
+; only a raised cap lets it schedule.
+; DEFAULT: Minimal Initiation Interval too large: 32 > 27
+; RAISED: Schedule found with Initiation Interval
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_max_mii(i32 %arg, ptr addrspace(3) %p) {
+bb:
+ br label %bb1
+
+bb1: ; preds = %bb1, %bb
+ %phiA = phi <16 x float> [ zeroinitializer, %bb ], [ %mfmaA, %bb1 ]
+ %phiB = phi <16 x float> [ zeroinitializer, %bb ], [ %mfmaB, %bb1 ]
+ %iv = phi i32 [ 0, %bb ], [ %arg, %bb1 ]
+ %load = load <4 x float>, ptr addrspace(3) %p, align 16
+ %elt = extractelement <4 x float> %load, i64 0
+ %mfmaA = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %elt, float 0.000000e+00, <16 x float> %phiA, i32 0, i32 0, i32 0)
+ %mfmaB = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %elt, float 0.000000e+00, <16 x float> %phiB, i32 0, i32 0, i32 0)
+ %icmp = icmp eq i32 %iv, 0
+ br i1 %icmp, label %exit, label %bb1
+
+exit: ; preds = %bb1
+ %lcssaA = phi <16 x float> [ %mfmaA, %bb1 ]
+ %lcssaB = phi <16 x float> [ %mfmaB, %bb1 ]
+ store <16 x float> %lcssaA, ptr addrspace(3) %p, align 64
+ store <16 x float> %lcssaB, ptr addrspace(3) %p, align 64
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-minimal.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-minimal.ll
new file mode 100644
index 0000000000000..3ce8d528b34b1
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-minimal.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-UNPIPELINED
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-PIPELINED
+; Minimal loop: software-pipelined only under -amdgpu-enable-pipeliner.
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_minimal(i1 %arg) {
+; CHECK-UNPIPELINED-LABEL: swp_amdgpu_pipeline_minimal:
+; CHECK-UNPIPELINED: ; %bb.0: ; %bb
+; CHECK-UNPIPELINED-NEXT: s_load_dword s0, s[4:5], 0x0
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s4, 0
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-UNPIPELINED-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-UNPIPELINED-NEXT: s_bitcmp1_b32 s0, 0
+; CHECK-UNPIPELINED-NEXT: s_cselect_b64 vcc, -1, 0
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s5, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s6, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s7, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s2, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s9, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s3, s4
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s8, s4
+; CHECK-UNPIPELINED-NEXT: .LBB0_1: ; %bb1
+; CHECK-UNPIPELINED-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-UNPIPELINED-NEXT: s_cmp_lt_i32 s9, 1
+; CHECK-UNPIPELINED-NEXT: s_cselect_b64 s[0:1], -1, 0
+; CHECK-UNPIPELINED-NEXT: s_or_b32 s9, s3, 1
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v4, s2
+; CHECK-UNPIPELINED-NEXT: v_mov_b32_e32 v5, s9
+; CHECK-UNPIPELINED-NEXT: buffer_load_dword v4, v4, s[4:7], 0 offen
+; CHECK-UNPIPELINED-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
+; CHECK-UNPIPELINED-NEXT: buffer_load_dword v5, v5, s[4:7], 0 offen
+; CHECK-UNPIPELINED-NEXT: s_add_i32 s3, s3, 1
+; CHECK-UNPIPELINED-NEXT: s_add_i32 s2, s2, 2
+; CHECK-UNPIPELINED-NEXT: v_pk_add_f32 v[2:3], v[2:3], 1.0 op_sel_hi:[1,0]
+; CHECK-UNPIPELINED-NEXT: s_movk_i32 s9, 0x100
+; CHECK-UNPIPELINED-NEXT: s_cmp_lg_u32 s8, 0
+; CHECK-UNPIPELINED-NEXT: s_mov_b32 s8, 1
+; CHECK-UNPIPELINED-NEXT: s_waitcnt vmcnt(1)
+; CHECK-UNPIPELINED-NEXT: v_cndmask_b32_e64 v4, 0, v4, s[0:1]
+; CHECK-UNPIPELINED-NEXT: s_waitcnt vmcnt(0)
+; CHECK-UNPIPELINED-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc
+; CHECK-UNPIPELINED-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
+; CHECK-UNPIPELINED-NEXT: s_cbranch_scc1 .LBB0_1
+; CHECK-UNPIPELINED-NEXT: ; %bb.2: ; %bb12
+; CHECK-UNPIPELINED-NEXT: s_endpgm
+;
+; CHECK-PIPELINED-LABEL: swp_amdgpu_pipeline_minimal:
+; CHECK-PIPELINED: ; %bb.0: ; %bb
+; CHECK-PIPELINED-NEXT: s_load_dword s1, s[4:5], 0x0
+; CHECK-PIPELINED-NEXT: s_mov_b32 s0, 0
+; CHECK-PIPELINED-NEXT: s_mov_b32 s2, s0
+; CHECK-PIPELINED-NEXT: s_mov_b32 s3, s0
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v6, s0
+; CHECK-PIPELINED-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-PIPELINED-NEXT: s_bitcmp1_b32 s1, 0
+; CHECK-PIPELINED-NEXT: s_cselect_b64 vcc, -1, 0
+; CHECK-PIPELINED-NEXT: s_cmp_lt_i32 s0, 1
+; CHECK-PIPELINED-NEXT: s_mov_b32 s1, s0
+; CHECK-PIPELINED-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-PIPELINED-NEXT: s_or_b32 s6, s0, 1
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v7, s6
+; CHECK-PIPELINED-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-PIPELINED-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
+; CHECK-PIPELINED-NEXT: s_add_i32 s9, s0, 1
+; CHECK-PIPELINED-NEXT: s_add_i32 s10, s0, 2
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v0, 0
+; CHECK-PIPELINED-NEXT: s_mov_b32 s8, 1
+; CHECK-PIPELINED-NEXT: v_pk_add_f32 v[2:3], v[2:3], 1.0 op_sel_hi:[1,0]
+; CHECK-PIPELINED-NEXT: s_cmp_lg_u32 s0, 0
+; CHECK-PIPELINED-NEXT: s_movk_i32 s6, 0x100
+; CHECK-PIPELINED-NEXT: s_cbranch_scc0 .LBB0_4
+; CHECK-PIPELINED-NEXT: ; %bb.1: ; %bb1
+; CHECK-PIPELINED-NEXT: s_cmp_lt_i32 s6, 1
+; CHECK-PIPELINED-NEXT: s_cselect_b64 s[6:7], -1, 0
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v8, s10
+; CHECK-PIPELINED-NEXT: s_or_b32 s11, s9, 1
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v9, s11
+; CHECK-PIPELINED-NEXT: buffer_load_dword v6, v8, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: buffer_load_dword v7, v9, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: s_and_b64 s[6:7], vcc, s[6:7]
+; CHECK-PIPELINED-NEXT: s_add_i32 s11, s9, 1
+; CHECK-PIPELINED-NEXT: s_add_i32 s12, s10, 2
+; CHECK-PIPELINED-NEXT: s_mov_b32 s13, 1
+; CHECK-PIPELINED-NEXT: v_pk_add_f32 v[2:3], v[2:3], 1.0 op_sel_hi:[1,0]
+; CHECK-PIPELINED-NEXT: s_cmp_lg_u32 s8, 0
+; CHECK-PIPELINED-NEXT: s_movk_i32 s8, 0x100
+; CHECK-PIPELINED-NEXT: s_cbranch_scc0 .LBB0_4
+; CHECK-PIPELINED-NEXT: ; %bb.2: ; %bb1
+; CHECK-PIPELINED-NEXT: s_cmp_lt_i32 s8, 1
+; CHECK-PIPELINED-NEXT: s_cselect_b64 s[8:9], -1, 0
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v10, s12
+; CHECK-PIPELINED-NEXT: s_or_b32 s10, s11, 1
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v11, s10
+; CHECK-PIPELINED-NEXT: buffer_load_dword v8, v10, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: buffer_load_dword v9, v11, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: s_and_b64 s[8:9], vcc, s[8:9]
+; CHECK-PIPELINED-NEXT: s_add_i32 s11, s11, 1
+; CHECK-PIPELINED-NEXT: s_add_i32 s12, s12, 2
+; CHECK-PIPELINED-NEXT: s_mov_b32 s10, 1
+; CHECK-PIPELINED-NEXT: v_pk_add_f32 v[2:3], v[2:3], 1.0 op_sel_hi:[1,0]
+; CHECK-PIPELINED-NEXT: s_cmp_lg_u32 s13, 0
+; CHECK-PIPELINED-NEXT: s_movk_i32 s13, 0x100
+; CHECK-PIPELINED-NEXT: s_cbranch_scc0 .LBB0_4
+; CHECK-PIPELINED-NEXT: .LBB0_3: ; %bb1
+; CHECK-PIPELINED-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-PIPELINED-NEXT: s_cmp_lt_i32 s13, 1
+; CHECK-PIPELINED-NEXT: s_cselect_b64 s[14:15], -1, 0
+; CHECK-PIPELINED-NEXT: s_or_b32 s13, s11, 1
+; CHECK-PIPELINED-NEXT: s_waitcnt vmcnt(4)
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v11, v5
+; CHECK-PIPELINED-NEXT: s_waitcnt vmcnt(2)
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v5, v7
+; CHECK-PIPELINED-NEXT: s_waitcnt vmcnt(0)
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v7, v9
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v10, v4
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v4, v6
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v6, v8
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v8, s12
+; CHECK-PIPELINED-NEXT: v_mov_b32_e32 v9, s13
+; CHECK-PIPELINED-NEXT: buffer_load_dword v8, v8, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: s_mov_b64 s[16:17], s[6:7]
+; CHECK-PIPELINED-NEXT: buffer_load_dword v9, v9, s[0:3], 0 offen
+; CHECK-PIPELINED-NEXT: s_mov_b64 s[6:7], s[8:9]
+; CHECK-PIPELINED-NEXT: s_add_i32 s12, s12, 2
+; CHECK-PIPELINED-NEXT: s_add_i32 s11, s11, 1
+; CHECK-PIPELINED-NEXT: v_cndmask_b32_e64 v10, 0, v10, s[4:5]
+; CHECK-PIPELINED-NEXT: v_cndmask_b32_e32 v11, 0, v11, vcc
+; CHECK-PIPELINED-NEXT: s_and_b64 s[8:9], vcc, s[14:15]
+; CHECK-PIPELINED-NEXT: v_pk_add_f32 v[2:3], v[2:3], 1.0 op_sel_hi:[1,0]
+; CHECK-PIPELINED-NEXT: s_movk_i32 s13, 0x100
+; CHECK-PIPELINED-NEXT: s_mov_b64 s[4:5], s[16:17]
+; CHECK-PIPELINED-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[10:11]
+; CHECK-PIPELINED-NEXT: s_cmp_lg_u32 s10, 0
+; CHECK-PIPELINED-NEXT: s_mov_b32 s10, 1
+; CHECK-PIPELINED-NEXT: s_cbranch_scc1 .LBB0_3
+; CHECK-PIPELINED-NEXT: .LBB0_4: ; %bb12
+; CHECK-PIPELINED-NEXT: s_endpgm
+bb:
+ br label %bb1
+
+bb1: ; preds = %bb1, %bb
+ %phi = phi i32 [ 0, %bb ], [ 256, %bb1 ]
+ %phi2 = phi i32 [ 0, %bb ], [ %add, %bb1 ]
+ %phi3 = phi <2 x float> [ zeroinitializer, %bb ], [ %fadd, %bb1 ]
+ %phi4 = phi <2 x float> [ zeroinitializer, %bb ], [ %fadd10, %bb1 ]
+ %phi5 = phi i32 [ 0, %bb ], [ 1, %bb1 ]
+ %icmp = icmp slt i32 %phi, 1
+ %shl = shl i32 %phi2, 1
+ %call = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %shl, i32 0, i32 0)
+ %insertelement = insertelement <2 x i32> zeroinitializer, i32 %call, i64 0
+ %or = or i32 %phi2, 1
+ %call6 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %or, i32 0, i32 0)
+ %and = and i1 %arg, %icmp
+ %insertelement7 = insertelement <2 x i1> zeroinitializer, i1 %and, i64 0
+ %insertelement8 = insertelement <2 x i1> %insertelement7, i1 %arg, i64 1
+ %insertelement9 = insertelement <2 x i32> %insertelement, i32 %call6, i64 1
+ %bitcast = bitcast <2 x i32> %insertelement9 to <2 x float>
+ %select = select <2 x i1> %insertelement8, <2 x float> %bitcast, <2 x float> zeroinitializer
+ %fadd = fadd <2 x float> %phi3, %select
+ %fadd10 = fadd <2 x float> %phi4, splat (float 1.000000e+00)
+ %add = add i32 %phi2, 1
+ %icmp11 = icmp eq i32 %phi5, 0
+ br i1 %icmp11, label %bb12, label %bb1
+
+bb12: ; preds = %bb1
+ ret void
+}
+
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-cond-not-in-loop.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-cond-not-in-loop.mir
new file mode 100644
index 0000000000000..dfee4b3180d1f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-cond-not-in-loop.mir
@@ -0,0 +1,36 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# Reject a loop whose branch condition ($scc) is defined outside the loop, not by
+# an in-loop compare. analyzeLoopForPipelining returns nullptr.
+# CHECK: Unable to analyzeLoop, can NOT pipeline Loop
+
+---
+name: swp_amdgpu_pipeline_neg_cond_not_in_loop
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr8' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ firstKernArgPreloadReg: { reg: '$sgpr8' }
+body: |
+ bb.0:
+ liveins: $sgpr8
+
+ %0:sreg_32 = COPY $sgpr8
+ %1:sreg_32 = S_MOV_B32 0
+ S_CMP_LG_U32 %0, %1, implicit-def $scc
+
+ bb.1:
+ successors: %bb.2, %bb.1
+ liveins: $scc
+
+ %2:sreg_32 = PHI %1, %bb.0, %3, %bb.1
+ %3:sreg_32 = S_MOV_B32 1
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-divergent-backedge.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-divergent-backedge.mir
new file mode 100644
index 0000000000000..00c02d3666b82
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-divergent-backedge.mir
@@ -0,0 +1,28 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# Reject a loop with a divergent (VCC/EXEC) back-edge (non-scalar predicate).
+# analyzeLoopForPipelining returns nullptr.
+# CHECK: Unable to analyzeLoop, can NOT pipeline Loop
+
+---
+name: swp_amdgpu_pipeline_neg_divergent_backedge
+tracksRegLiveness: true
+isSSA: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+ successors: %bb.1
+
+ %0:sreg_64 = S_AND_B64 $exec, -1, implicit-def dead $scc
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ $vcc = COPY %0
+ S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-inlineasm.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-inlineasm.mir
new file mode 100644
index 0000000000000..dfd68c8edbabc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-inlineasm.mir
@@ -0,0 +1,36 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# Reject a loop whose body contains inline asm.
+# analyzeLoopForPipelining returns nullptr.
+# CHECK: Unable to analyzeLoop, can NOT pipeline Loop
+
+---
+name: swp_amdgpu_pipeline_neg_inlineasm
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr8' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ firstKernArgPreloadReg: { reg: '$sgpr8' }
+body: |
+ bb.0:
+ liveins: $sgpr8
+
+ %0:sreg_32 = COPY $sgpr8
+ %1:sreg_32 = S_MOV_B32 0
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %2:sreg_32 = PHI %1, %bb.0, %3, %bb.1
+ INLINEASM &"s_nop 0", 1 /* sideeffect attdialect */
+ %3:sreg_32 = S_ADD_I32 %2, 1, implicit-def dead $scc
+ S_CMP_LG_U32 %0, %3, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-multi-bb.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-multi-bb.ll
new file mode 100644
index 0000000000000..da8cb965b49a0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-multi-bb.ll
@@ -0,0 +1,25 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -pass-remarks-analysis=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+; Reject a loop whose body lowers to more than one basic block: the runtime-masked
+; load is scalarized into cond.load blocks, so the pipeliner bails (single-block only).
+; CHECK: Not a single basic block
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_neg_multi_bb(ptr addrspace(1) %p, i32 inreg %n) {
+entry:
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+ %acc = phi <8 x half> [ zeroinitializer, %entry ], [ %v, %loop ]
+ %m = icmp slt i32 %i, %n
+ %mv = insertelement <8 x i1> poison, i1 %m, i64 0
+ %mask = shufflevector <8 x i1> %mv, <8 x i1> poison, <8 x i32> zeroinitializer
+ %gep = getelementptr half, ptr addrspace(1) %p, i32 %i
+ %v = tail call <8 x half> @llvm.masked.load.v8f16.p1(ptr addrspace(1) %gep, i32 16, <8 x i1> %mask, <8 x half> %acc)
+ %i.next = add i32 %i, 1
+ %cmp = icmp slt i32 %i.next, %n
+ br i1 %cmp, label %loop, label %exit
+
+exit: ; preds = %loop
+ store <8 x half> %v, ptr addrspace(1) %p
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-no-exit.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-no-exit.mir
new file mode 100644
index 0000000000000..b6f713d1e9b72
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-no-exit.mir
@@ -0,0 +1,33 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# Reject a loop with no exit: the conditional latch has both edges targeting the
+# loop block (infinite loop). analyzeLoopForPipelining returns nullptr.
+# CHECK: Unable to analyzeLoop, can NOT pipeline Loop
+
+---
+name: swp_amdgpu_pipeline_neg_no_exit
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr8' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ firstKernArgPreloadReg: { reg: '$sgpr8' }
+body: |
+ bb.0:
+ liveins: $sgpr8
+
+ %0:sreg_32 = COPY $sgpr8
+ %1:sreg_32 = S_MOV_B32 0
+
+ bb.1:
+ %2:sreg_32 = PHI %1, %bb.0, %3, %bb.1
+ %3:sreg_32 = S_ADD_I32 %2, 1, implicit-def dead $scc
+ S_CMP_LG_U32 %0, %3, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.1
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-not-cond-branch.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-not-cond-branch.mir
new file mode 100644
index 0000000000000..bcab4695259a3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-neg-not-cond-branch.mir
@@ -0,0 +1,33 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# Reject a uniform loop with a fall-through exit latch.
+# CHECK: Unable to analyzeLoop, can NOT pipeline Loop
+
+---
+name: swp_amdgpu_pipeline_neg_not_cond_branch
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr8' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ firstKernArgPreloadReg: { reg: '$sgpr8' }
+body: |
+ bb.0:
+ liveins: $sgpr8
+
+ %0:sreg_32 = COPY $sgpr8
+ %1:sreg_32 = S_MOV_B32 0
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %2:sreg_32 = PHI %1, %bb.0, %3, %bb.1
+ %3:sreg_32 = S_ADD_I32 %2, 1, implicit-def dead $scc
+ S_CMP_LG_U32 %0, %3, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-issue-bound.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-issue-bound.ll
new file mode 100644
index 0000000000000..1fe846e7dc63c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-issue-bound.ll
@@ -0,0 +1,41 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Resource-bound II set by the aggregate micro-op issue width, not by any single
+; functional unit. The body issues 13 single-micro-op instructions across three
+; unit classes (VALU=6, SALU=4, VMEM=3), so no one unit reaches 13, but at an
+; issue width of 1 the 13 micro-ops force res=13. That dominates the short
+; recurrence (rec=3), so the II is resource-bound on issue, not recurrence-bound.
+; CHECK: MII = 13 MAX_II = 23 (rec=3, res=13)
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_resmii_issue_bound(i1 %arg, ptr addrspace(1) %out) {
+bb:
+ br label %bb1
+
+bb1: ; preds = %bb1, %bb
+ %iv = phi i32 [ 0, %bb ], [ %iv.next, %bb1 ]
+ %acc = phi float [ 0.000000e+00, %bb ], [ %fadd3, %bb1 ]
+ %off0 = shl i32 %iv, 2
+ %off1 = or i32 %off0, 1
+ %off2 = or i32 %off0, 2
+ %load0 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %off0, i32 0, i32 0)
+ %load1 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %off1, i32 0, i32 0)
+ %load2 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %off2, i32 0, i32 0)
+ %f0 = bitcast i32 %load0 to float
+ %f1 = bitcast i32 %load1 to float
+ %f2 = bitcast i32 %load2 to float
+ %sel0 = select i1 %arg, float %f0, float 0.000000e+00
+ %sel1 = select i1 %arg, float %f1, float 0.000000e+00
+ %sel2 = select i1 %arg, float %f2, float 0.000000e+00
+ %fadd1 = fadd float %acc, %sel0
+ %fadd2 = fadd float %fadd1, %sel1
+ %fadd3 = fadd float %fadd2, %sel2
+ %iv.next = add i32 %iv, 1
+ %icmp = icmp eq i32 %iv, 0
+ br i1 %icmp, label %exit, label %bb1
+
+exit: ; preds = %bb1
+ %lcssa = phi float [ %fadd3, %bb1 ]
+ store float %lcssa, ptr addrspace(1) %out, align 4
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-mfma.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-mfma.ll
new file mode 100644
index 0000000000000..8b7c9b053728f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-resmii-mfma.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; Resource-bound II set by a single functional unit, the MFMA (XDL) matrix pipe.
+; Two independent accumulators each issue one MFMA per iteration; each MFMA holds
+; the XDL pipe for 16 cycles, so res=32 on that one unit. That exceeds both the
+; 3-micro-op issue bound and the single-MFMA recurrence (rec=17), so the II is
+; resource-bound on the XDL pipe rather than on issue width or recurrence.
+; CHECK: MII = 32 MAX_II = 42 (rec=17, res=32)
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_resmii_mfma(i32 %arg, ptr addrspace(3) %p) {
+bb:
+ br label %bb1
+
+bb1: ; preds = %bb1, %bb
+ %phiA = phi <16 x float> [ zeroinitializer, %bb ], [ %mfmaA, %bb1 ]
+ %phiB = phi <16 x float> [ zeroinitializer, %bb ], [ %mfmaB, %bb1 ]
+ %iv = phi i32 [ 0, %bb ], [ %arg, %bb1 ]
+ %load = load <4 x float>, ptr addrspace(3) %p, align 16
+ %elt = extractelement <4 x float> %load, i64 0
+ %mfmaA = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %elt, float 0.000000e+00, <16 x float> %phiA, i32 0, i32 0, i32 0)
+ %mfmaB = tail call <16 x float> @llvm.amdgcn.mfma.f32.32x32x2f32(float %elt, float 0.000000e+00, <16 x float> %phiB, i32 0, i32 0, i32 0)
+ %icmp = icmp eq i32 %iv, 0
+ br i1 %icmp, label %exit, label %bb1
+
+exit: ; preds = %bb1
+ %lcssaA = phi <16 x float> [ %mfmaA, %bb1 ]
+ %lcssaB = phi <16 x float> [ %mfmaB, %bb1 ]
+ store <16 x float> %lcssaA, ptr addrspace(3) %p, align 64
+ store <16 x float> %lcssaB, ptr addrspace(3) %p, align 64
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-scalar-accumulate.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-scalar-accumulate.ll
new file mode 100644
index 0000000000000..d1b790b784c9c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-scalar-accumulate.ll
@@ -0,0 +1,33 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -pass-remarks-analysis=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+; Reduced from Composable Kernels. Scalar fp reduction: an i16 buffer
+; load feeding a transcendental exp() with a NaN-guarded accumulate. Verifies
+; such a loop software-pipelines.
+; CHECK: Schedule found with Initiation Interval
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_scalar_accumulate(i1 %arg) {
+bb:
+ %call = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb1
+
+bb1: ; preds = %bb1, %bb
+ %phi = phi float [ 0.000000e+00, %bb ], [ %select6, %bb1 ]
+ %phi2 = phi i32 [ 0, %bb ], [ 1, %bb1 ]
+ %phi3 = phi i32 [ %call, %bb ], [ %add, %bb1 ]
+ %add = add i32 %phi3, -8
+ %call4 = tail call i16 @llvm.amdgcn.raw.ptr.buffer.load.i16(ptr addrspace(8) null, i32 %phi2, i32 0, i32 0)
+ %icmp = icmp slt i32 %add, 0
+ %and = and i1 %arg, %icmp
+ %bitcast = bitcast i16 %call4 to half
+ %fpext = fpext half %bitcast to float
+ %select = select i1 %and, float %fpext, float 0.000000e+00
+ %fsub = fsub float %select, 1.000000e+00
+ %call5 = tail call float @llvm.exp.f32(float %fsub)
+ %fcmp = fcmp uno float %call5, 0.000000e+00
+ %fadd = fadd float %phi, 1.000000e+00
+ %select6 = select i1 %fcmp, float %phi, float %fadd
+ %icmp7 = icmp eq i32 %phi2, 0
+ br i1 %icmp7, label %bb8, label %bb1
+
+bb8: ; preds = %bb1
+ ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-unpipeline-compare.mir b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-unpipeline-compare.mir
new file mode 100644
index 0000000000000..e4ce77c691919
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-unpipeline-compare.mir
@@ -0,0 +1,48 @@
+# RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -run-pass=pipeliner -debug-only=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+# REQUIRES: asserts
+# The controlling compare must be kept out of the pipelined steady state.
+
+# CHECK: SU([[CMP:[0-9]+]]): S_CMP_LT_I32 {{.*}}implicit-def $scc
+# CHECK: Do not pipeline SU([[CMP]])
+
+---
+name: swp_amdgpu_pipeline_unpipeline_compare
+tracksRegLiveness: true
+isSSA: true
+liveins:
+ - { reg: '$sgpr4_sgpr5' }
+machineFunctionInfo:
+ isEntryFunction: true
+ argumentInfo:
+ kernargSegmentPtr: { reg: '$sgpr4_sgpr5' }
+body: |
+ bb.0:
+ liveins: $sgpr4_sgpr5
+
+ %0:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ %1:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+ %2:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM %0(p4), 8, 0 :: (dereferenceable invariant load (s32), align 8, addrspace 4)
+ %3:sreg_32 = S_MOV_B32 0
+ %4:vgpr_32 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+
+ bb.1:
+ successors: %bb.2, %bb.1
+
+ %5:sreg_32 = PHI %3, %bb.0, %6, %bb.1
+ %7:vgpr_32 = PHI %4, %bb.0, %8, %bb.1
+ %9:sreg_32_xm0 = S_ASHR_I32 %5, 31, implicit-def dead $scc
+ %10:sreg_64 = REG_SEQUENCE %5, %subreg.sub0, %9, %subreg.sub1
+ %11:sreg_64 = S_LSHL_B64 %10, 2, implicit-def dead $scc
+ %12:sreg_32 = S_ADD_U32 %1.sub0, %11.sub0, implicit-def $scc
+ %13:sreg_32 = S_ADDC_U32 %1.sub1, %11.sub1, implicit-def $scc, implicit $scc
+ %14:sreg_64 = REG_SEQUENCE %12, %subreg.sub0, %13, %subreg.sub1
+ %15:sgpr_32 = S_LOAD_DWORD_IMM %14, 0, 0 :: (load (s32), addrspace 1)
+ %8:vgpr_32 = nofpexcept V_ADD_F32_e32 %15, %7, implicit $mode, implicit $exec
+ %6:sreg_32 = S_ADD_I32 %5, 1, implicit-def dead $scc
+ S_CMP_LT_I32 %6, %2, implicit-def $scc
+ S_CBRANCH_SCC1 %bb.1, implicit $scc
+ S_BRANCH %bb.2
+
+ bb.2:
+ S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-vector-accumulate.ll b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-vector-accumulate.ll
new file mode 100644
index 0000000000000..be758d4638aa3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/swp-amdgpu-pipeline-vector-accumulate.ll
@@ -0,0 +1,43 @@
+; RUN: llc -mtriple=amdgpu9.50-amd-amdhsa -amdgpu-enable-pipeliner -pass-remarks-analysis=pipeliner %s -filetype=null 2>&1 | FileCheck %s
+; Reduced from Composable Kernels. Vector <2 x float> reduction with
+; two accumulators fed by two buffer loads and vector-predicated selects.
+; Verifies such a loop software-pipelines.
+; CHECK: Schedule found with Initiation Interval
+
+define amdgpu_kernel void @swp_amdgpu_pipeline_vector_accumulate(i32 %arg, ptr addrspace(8) %arg1, i1 %arg2, i1 %arg3) {
+bb:
+ %call = tail call i32 @llvm.amdgcn.workitem.id.x()
+ br label %bb4
+
+bb4: ; preds = %bb4, %bb
+ %phi = phi i32 [ %call, %bb ], [ %or, %bb4 ]
+ %phi5 = phi i32 [ %call, %bb ], [ %add, %bb4 ]
+ %phi6 = phi <2 x float> [ zeroinitializer, %bb ], [ %fadd, %bb4 ]
+ %phi7 = phi <2 x float> [ zeroinitializer, %bb ], [ %fadd19, %bb4 ]
+ %phi8 = phi i32 [ 0, %bb ], [ 2, %bb4 ]
+ %shl = shl i32 %phi5, 1
+ %call9 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) %arg1, i32 %shl, i32 0, i32 0)
+ %insertelement = insertelement <2 x i32> zeroinitializer, i32 %call9, i64 0
+ %icmp = icmp slt i32 %phi, 0
+ %call10 = tail call i32 @llvm.amdgcn.raw.ptr.buffer.load.i32(ptr addrspace(8) null, i32 %shl, i32 0, i32 0)
+ %and = and i1 %arg2, %icmp
+ %insertelement11 = insertelement <2 x i1> zeroinitializer, i1 %and, i64 0
+ %and12 = and i1 %arg3, %icmp
+ %insertelement13 = insertelement <2 x i1> %insertelement11, i1 %and12, i64 1
+ %insertelement14 = insertelement <2 x i32> %insertelement, i32 1, i64 1
+ %bitcast = bitcast <2 x i32> %insertelement14 to <2 x float>
+ %select = select <2 x i1> %insertelement13, <2 x float> %bitcast, <2 x float> zeroinitializer
+ %fadd = fadd <2 x float> %phi6, %select
+ %insertelement15 = insertelement <2 x i32> zeroinitializer, i32 %call10, i64 0
+ %insertelement16 = insertelement <2 x i32> %insertelement15, i32 %arg, i64 1
+ %bitcast17 = bitcast <2 x i32> %insertelement16 to <2 x float>
+ %select18 = select <2 x i1> %insertelement13, <2 x float> %bitcast17, <2 x float> zeroinitializer
+ %fadd19 = fadd <2 x float> %phi7, %select18
+ %or = or i32 %phi, 1
+ %add = add i32 %phi5, 1
+ %icmp20 = icmp eq i32 %phi8, 0
+ br i1 %icmp20, label %bb21, label %bb4
+
+bb21: ; preds = %bb4
+ ret void
+}
More information about the llvm-commits
mailing list