[llvm-branch-commits] [llvm] 0431144 - [LoopSimplify] Don't separate nested loops on targets with branch divergence (#235627)
via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Nov 11 01:51:52 PST 2026
Author: tianhbai
Date: 2026-11-11T11:09:03+08:00
New Revision: 04311443d9df399925bcafdc6b75071155c26618
URL: https://github.com/llvm/llvm-project/commit/04311443d9df399925bcafdc6b75071155c26618
DIFF: https://github.com/llvm/llvm-project/commit/04311443d9df399925bcafdc6b75071155c26618.diff
LOG: [LoopSimplify] Don't separate nested loops on targets with branch divergence (#235627)
When a loop header has multiple backedges, LoopSimplify may call
`separateNestedLoop` to move some of them into a new inner loop. On
targets
with branch divergence, if those backedges are selected by a divergent
condition (e.g. a divergent `continue`), the new inner loop gets a
divergent
exit, and loop-carried values that were uniform (e.g. the loop counter)
can
become divergent. On AMDGPU this turns scalar values into VGPRs and adds
an
extra exec-mask loop level.
Skip `separateNestedLoop` when `TTI->hasBranchDivergence(F)` is true and
fall
back to `insertUniqueBackedgeBlock`. Degenerate header PHIs are now also
folded before inserting the unique backedge block, as
`separateNestedLoop`
used to do.
This is a per-function check rather than a per-branch uniformity check,
since `UniformityAnalysis` is not available in most places LoopSimplify
runs.
`SimpleLoopUnswitch` uses the same check to disable non-trivial
unswitching.
On 954 internal compute pipelines on gfx1201, 28 change; none increase
VGPR
count or scratch usage, and total code size of the changed pipelines
decreases by 0.7%. On internal graphics workloads on gfx1201 we measured
about 1-1.6% performance improvement.
Assisted-by: Claude Opus 5.5
Added:
llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll
Modified:
llvm/include/llvm/Transforms/Utils/LoopSimplify.h
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
llvm/lib/Transforms/Utils/LoopSimplify.cpp
llvm/lib/Transforms/Utils/LoopUnroll.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
index 58dbbf6b5f75a5..7b151d3ba5139e 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
@@ -49,6 +49,7 @@ class Loop;
class LoopInfo;
class MemorySSAUpdater;
class ScalarEvolution;
+class TargetTransformInfo;
/// This pass is responsible for loop canonicalization.
class LoopSimplifyPass : public RequiredPassInfoMixin<LoopSimplifyPass> {
@@ -62,9 +63,12 @@ class LoopSimplifyPass : public RequiredPassInfoMixin<LoopSimplifyPass> {
/// it into a simplified loop nest with preheaders and single backedges. It will
/// update \c DominatorTree, \c LoopInfo, \c ScalarEvolution and \c MemorySSA
/// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true.
+/// If \p TTI is non-null and the target has branch divergence, loops with
+/// multiple backedges are not split into nested loops.
LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ const TargetTransformInfo *TTI = nullptr);
} // end namespace llvm
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index a26a5249cdfc01..4c93f3146bf94a 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1670,8 +1670,8 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
// will simplify all loops, regardless of whether anything end up being
// unrolled.
for (const auto &L : LI) {
- Changed |=
- simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
+ Changed |= simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr,
+ false /* PreserveLCSSA */, &TTI);
Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
}
diff --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index 7cce2329077699..f54ba31f17492b 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -54,6 +54,7 @@
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/CFG.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
@@ -468,11 +469,33 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
return BEBlock;
}
+/// Replace PHI nodes in the loop header that simplify to a single value.
+static bool simplifyHeaderPHIs(Loop *L, DominatorTree *DT, LoopInfo *LI,
+ ScalarEvolution *SE, AssumptionCache *AC,
+ bool PreserveLCSSA) {
+ bool Changed = false;
+ const DataLayout &DL = L->getHeader()->getDataLayout();
+ PHINode *PN;
+ for (BasicBlock::iterator I = L->getHeader()->begin();
+ (PN = dyn_cast<PHINode>(I++));)
+ if (Value *V = simplifyInstruction(PN, {DL, nullptr, DT, AC})) {
+ if (SE)
+ SE->forgetValue(PN);
+ if (!PreserveLCSSA || LI->replacementPreservesLCSSAForm(PN, V)) {
+ PN->replaceAllUsesWith(V);
+ PN->eraseFromParent();
+ Changed = true;
+ }
+ }
+ return Changed;
+}
+
/// Simplify one loop and queue further loops for simplification.
static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ const TargetTransformInfo *TTI) {
bool Changed = false;
if (MSSAU && VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
@@ -554,7 +577,12 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
// If this is really a nested loop, rip it out into a child loop. Don't do
// this for loops with a giant number of backedges, just factor them into a
// common backedge instead.
- if (L->getNumBackEdges() < 8) {
+ // On targets with branch divergence, a backedge taken by only some threads
+ // (e.g. a divergent "continue") would become a divergent inner loop, which
+ // can turn otherwise uniform loop state into per-thread state.
+ bool AllowSeparate =
+ !TTI || !TTI->hasBranchDivergence(L->getHeader()->getParent());
+ if (AllowSeparate && L->getNumBackEdges() < 8) {
if (Loop *OuterL = separateNestedLoop(L, Preheader, DT, LI, SE,
PreserveLCSSA, AC, MSSAU)) {
++NumNested;
@@ -572,7 +600,10 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
// If we either couldn't, or didn't want to, identify nesting of the loops,
// insert a new block that all backedges target, then make it jump to the
- // loop header.
+ // loop header. Fold degenerate header PHIs first, so they don't turn into
+ // a PHI cycle through the new backedge block.
+ if (simplifyHeaderPHIs(L, DT, LI, SE, AC, PreserveLCSSA))
+ Changed = true;
LoopLatch = insertUniqueBackedgeBlock(L, Preheader, DT, LI, MSSAU);
if (LoopLatch)
Changed = true;
@@ -581,22 +612,11 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
if (MSSAU && VerifyMemorySSA)
MSSAU->getMemorySSA()->verifyMemorySSA();
- const DataLayout &DL = L->getHeader()->getDataLayout();
-
// Scan over the PHI nodes in the loop header. Since they now have only two
// incoming values (the loop is canonicalized), we may have simplified the PHI
// down to 'X = phi [X, Y]', which should be replaced with 'Y'.
- PHINode *PN;
- for (BasicBlock::iterator I = L->getHeader()->begin();
- (PN = dyn_cast<PHINode>(I++)); )
- if (Value *V = simplifyInstruction(PN, {DL, nullptr, DT, AC})) {
- if (SE) SE->forgetValue(PN);
- if (!PreserveLCSSA || LI->replacementPreservesLCSSAForm(PN, V)) {
- PN->replaceAllUsesWith(V);
- PN->eraseFromParent();
- Changed = true;
- }
- }
+ if (simplifyHeaderPHIs(L, DT, LI, SE, AC, PreserveLCSSA))
+ Changed = true;
// If this loop has multiple exits and the exits all go to the same
// block, attempt to merge the exits. This helps several passes, such
@@ -691,7 +711,8 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
- MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+ MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+ const TargetTransformInfo *TTI) {
bool Changed = false;
#ifndef NDEBUG
@@ -719,7 +740,7 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
while (!Worklist.empty())
Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE,
- AC, MSSAU, PreserveLCSSA);
+ AC, MSSAU, PreserveLCSSA, TTI);
// Changing exit conditions for blocks may affect exit counts of this loop and
// any of its parents, so we must invalidate the entire subtree if we've made
@@ -798,11 +819,14 @@ bool LoopSimplify::runOnFunction(Function &F) {
MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
}
+ auto *TTIWP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>();
+ const TargetTransformInfo *TTI = TTIWP ? &TTIWP->getTTI(F) : nullptr;
+
bool PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
// Simplify each loop nest in the function.
for (auto *L : *LI)
- Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA);
+ Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA, TTI);
#ifndef NDEBUG
if (PreserveLCSSA) {
@@ -821,6 +845,7 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
ScalarEvolution *SE = AM.getCachedResult<ScalarEvolutionAnalysis>(F);
AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
+ const TargetTransformInfo *TTI = &AM.getResult<TargetIRAnalysis>(F);
auto *MSSAAnalysis = AM.getCachedResult<MemorySSAAnalysis>(F);
std::unique_ptr<MemorySSAUpdater> MSSAU;
if (MSSAAnalysis) {
@@ -832,8 +857,8 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
// Note that we don't preserve LCSSA in the new PM, if you need it run LCSSA
// after simplifying the loops. MemorySSA is preserved if it exists.
for (auto *L : *LI)
- Changed |=
- simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), /*PreserveLCSSA*/ false);
+ Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(),
+ /*PreserveLCSSA*/ false, TTI);
if (!Changed)
return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index db44f074359ac6..20aa2f105c553b 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1738,11 +1738,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
// TODO: That potentially might be compile-time expensive. We should try
// to fix the loop-simplified form incrementally.
- simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+ simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA, TTI);
} else {
// Simplify loops for which we might've broken loop-simplify form.
for (Loop *SubLoop : LoopsToSimplify)
- simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+ simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA, TTI);
}
return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0f0ecbf896c3f4..616afab41ec54a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7983,8 +7983,8 @@ LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
// will simplify all loops, regardless of whether anything end up being
// vectorized.
for (const auto &L : *LI)
- Changed |= CFGChanged |=
- simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
+ Changed |= CFGChanged |= simplifyLoop(L, DT, LI, SE, AC, nullptr,
+ false /* PreserveLCSSA */, TTI);
// Build up a worklist of inner-loops to vectorize. This is necessary as
// the act of vectorizing or partially unrolling a loop creates new loops
diff --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000000..7c492428aec761
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AMDGPU" in config.root.targets:
+ config.unsupported = True
diff --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
new file mode 100644
index 00000000000000..752d8b86012e4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="loop-simplify,simplifycfg<no-keep-loops>,loop-unroll" < %s | FileCheck %s --check-prefix=UNROLL
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="loop-simplify,simplifycfg<no-keep-loops>,loop-vectorize" < %s | FileCheck %s --check-prefix=VECTORIZE
+
+; SimplifyCFG without canonical loops folds the unique backedge block inserted
+; by LoopSimplify. The loop unroller and the vectorizer re-simplify the loop
+; and must not separate it into a nested loop on a target with branch
+; divergence either.
+
+define i32 @continue_loop(i32 %n, ptr %p) {
+; UNROLL-LABEL: define i32 @continue_loop(
+; UNROLL-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; UNROLL-NEXT: [[ENTRY:.*]]:
+; UNROLL-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; UNROLL-NEXT: br label %[[HEADER:.*]]
+; UNROLL: [[HEADER]]:
+; UNROLL-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; UNROLL-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; UNROLL-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; UNROLL-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[I]]
+; UNROLL-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; UNROLL-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; UNROLL-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; UNROLL-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; UNROLL-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; UNROLL: [[CONTINUE]]:
+; UNROLL-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; UNROLL-NEXT: br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; UNROLL: [[HEADER_BACKEDGE]]:
+; UNROLL-NEXT: [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; UNROLL-NEXT: br label %[[HEADER]]
+; UNROLL: [[UPDATE]]:
+; UNROLL-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; UNROLL-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; UNROLL-NEXT: br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; UNROLL: [[EXIT]]:
+; UNROLL-NEXT: [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; UNROLL-NEXT: ret i32 [[R]]
+;
+; VECTORIZE-LABEL: define i32 @continue_loop(
+; VECTORIZE-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; VECTORIZE-NEXT: [[ENTRY:.*]]:
+; VECTORIZE-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; VECTORIZE-NEXT: br label %[[HEADER:.*]]
+; VECTORIZE: [[HEADER]]:
+; VECTORIZE-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; VECTORIZE-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; VECTORIZE-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; VECTORIZE-NEXT: [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[I]]
+; VECTORIZE-NEXT: [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; VECTORIZE-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; VECTORIZE-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; VECTORIZE-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; VECTORIZE-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; VECTORIZE: [[CONTINUE]]:
+; VECTORIZE-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; VECTORIZE-NEXT: br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; VECTORIZE: [[HEADER_BACKEDGE]]:
+; VECTORIZE-NEXT: [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; VECTORIZE-NEXT: br label %[[HEADER]]
+; VECTORIZE: [[UPDATE]]:
+; VECTORIZE-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; VECTORIZE-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; VECTORIZE-NEXT: br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; VECTORIZE: [[EXIT]]:
+; VECTORIZE-NEXT: [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; VECTORIZE-NEXT: ret i32 [[R]]
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+ %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+ %i.next = add i32 %i, 1
+ %gep = getelementptr i32, ptr %p, i32 %i
+ %v = load i32, ptr %gep, align 4
+ %bits = lshr i32 %v, %tid
+ %bit = and i32 %bits, 1
+ %is.zero = icmp eq i32 %bit, 0
+ br i1 %is.zero, label %continue, label %update
+
+continue:
+ %c1 = icmp slt i32 %i.next, %n
+ br i1 %c1, label %header, label %exit
+
+update:
+ %acc.next = add i32 %acc, %v
+ %c2 = icmp slt i32 %i.next, %n
+ br i1 %c2, label %header, label %exit
+
+exit:
+ %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+ ret i32 %r
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll
new file mode 100644
index 00000000000000..f8d8ec49464d52
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-simplify -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=AMDGPU
+; RUN: opt -S -passes=loop-simplify < %s | FileCheck %s --check-prefix=GENERIC
+
+; A loop with two backedges where the "continue" backedge is selected by a
+; per-lane condition. On a target with branch divergence the loop is not
+; separated into a nested loop; a unique backedge block is inserted instead.
+
+define amdgpu_kernel void @continue_loop(i32 %n, ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; AMDGPU-LABEL: define amdgpu_kernel void @continue_loop(
+; AMDGPU-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; AMDGPU-NEXT: [[ENTRY:.*]]:
+; AMDGPU-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT: br label %[[HEADER:.*]]
+; AMDGPU: [[HEADER]]:
+; AMDGPU-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; AMDGPU-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; AMDGPU-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; AMDGPU-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGPU-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; AMDGPU-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; AMDGPU-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; AMDGPU-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; AMDGPU: [[CONTINUE]]:
+; AMDGPU-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; AMDGPU: [[HEADER_BACKEDGE]]:
+; AMDGPU-NEXT: [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; AMDGPU-NEXT: br label %[[HEADER]]
+; AMDGPU: [[UPDATE]]:
+; AMDGPU-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; AMDGPU-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; AMDGPU: [[EXIT]]:
+; AMDGPU-NEXT: [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; AMDGPU-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; AMDGPU-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT_GEP]], align 4
+; AMDGPU-NEXT: ret void
+;
+; GENERIC-LABEL: define amdgpu_kernel void @continue_loop(
+; GENERIC-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GENERIC-NEXT: [[ENTRY:.*]]:
+; GENERIC-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT: br label %[[HEADER_OUTER:.*]]
+; GENERIC: [[HEADER_OUTER]]:
+; GENERIC-NEXT: [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT: [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT: br label %[[HEADER:.*]]
+; GENERIC: [[HEADER]]:
+; GENERIC-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; GENERIC-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; GENERIC-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; GENERIC-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; GENERIC-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; GENERIC-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; GENERIC-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; GENERIC: [[CONTINUE]]:
+; GENERIC-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; GENERIC: [[UPDATE]]:
+; GENERIC-NEXT: [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; GENERIC-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; GENERIC: [[EXIT_LOOPEXIT]]:
+; GENERIC-NEXT: br label %[[EXIT:.*]]
+; GENERIC: [[EXIT_LOOPEXIT1]]:
+; GENERIC-NEXT: br label %[[EXIT]]
+; GENERIC: [[EXIT]]:
+; GENERIC-NEXT: [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; GENERIC-NEXT: [[OUT_GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; GENERIC-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT_GEP]], align 4
+; GENERIC-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+ %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+ %i.next = add i32 %i, 1
+ %gep = getelementptr i32, ptr addrspace(1) %p, i32 %i
+ %v = load i32, ptr addrspace(1) %gep, align 4
+ %bits = lshr i32 %v, %tid
+ %bit = and i32 %bits, 1
+ %is.zero = icmp eq i32 %bit, 0
+ br i1 %is.zero, label %continue, label %update
+
+continue:
+ %c1 = icmp slt i32 %i.next, %n
+ br i1 %c1, label %header, label %exit
+
+update:
+ %acc.next = add i32 %acc, %v
+ %c2 = icmp slt i32 %i.next, %n
+ br i1 %c2, label %header, label %exit
+
+exit:
+ %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+ %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+ store i32 %r, ptr addrspace(1) %out.gep, align 4
+ ret void
+}
+
+; A function that executes with a single lane has no branch divergence, so
+; the nested loop is still separated.
+
+define amdgpu_kernel void @continue_loop_single_lane(i32 %n, ptr addrspace(1) %p, ptr addrspace(1) %out) #0 {
+; AMDGPU-LABEL: define amdgpu_kernel void @continue_loop_single_lane(
+; AMDGPU-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; AMDGPU-NEXT: [[ENTRY:.*]]:
+; AMDGPU-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT: br label %[[HEADER_OUTER:.*]]
+; AMDGPU: [[HEADER_OUTER]]:
+; AMDGPU-NEXT: [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; AMDGPU-NEXT: [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; AMDGPU-NEXT: br label %[[HEADER:.*]]
+; AMDGPU: [[HEADER]]:
+; AMDGPU-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; AMDGPU-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; AMDGPU-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGPU-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; AMDGPU-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; AMDGPU-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; AMDGPU-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; AMDGPU: [[CONTINUE]]:
+; AMDGPU-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; AMDGPU: [[UPDATE]]:
+; AMDGPU-NEXT: [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; AMDGPU-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; AMDGPU: [[EXIT_LOOPEXIT]]:
+; AMDGPU-NEXT: br label %[[EXIT:.*]]
+; AMDGPU: [[EXIT_LOOPEXIT1]]:
+; AMDGPU-NEXT: br label %[[EXIT]]
+; AMDGPU: [[EXIT]]:
+; AMDGPU-NEXT: [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; AMDGPU-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; AMDGPU-NEXT: ret void
+;
+; GENERIC-LABEL: define amdgpu_kernel void @continue_loop_single_lane(
+; GENERIC-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; GENERIC-NEXT: [[ENTRY:.*]]:
+; GENERIC-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT: br label %[[HEADER_OUTER:.*]]
+; GENERIC: [[HEADER_OUTER]]:
+; GENERIC-NEXT: [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT: [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT: br label %[[HEADER:.*]]
+; GENERIC: [[HEADER]]:
+; GENERIC-NEXT: [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; GENERIC-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; GENERIC-NEXT: [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; GENERIC-NEXT: [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; GENERIC-NEXT: [[BIT:%.*]] = and i32 [[BITS]], 1
+; GENERIC-NEXT: [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; GENERIC-NEXT: br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; GENERIC: [[CONTINUE]]:
+; GENERIC-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; GENERIC: [[UPDATE]]:
+; GENERIC-NEXT: [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; GENERIC-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; GENERIC: [[EXIT_LOOPEXIT]]:
+; GENERIC-NEXT: br label %[[EXIT:.*]]
+; GENERIC: [[EXIT_LOOPEXIT1]]:
+; GENERIC-NEXT: br label %[[EXIT]]
+; GENERIC: [[EXIT]]:
+; GENERIC-NEXT: [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; GENERIC-NEXT: store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; GENERIC-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+ %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+ %i.next = add i32 %i, 1
+ %gep = getelementptr i32, ptr addrspace(1) %p, i32 %i
+ %v = load i32, ptr addrspace(1) %gep, align 4
+ %bits = lshr i32 %v, %tid
+ %bit = and i32 %bits, 1
+ %is.zero = icmp eq i32 %bit, 0
+ br i1 %is.zero, label %continue, label %update
+
+continue:
+ %c1 = icmp slt i32 %i.next, %n
+ br i1 %c1, label %header, label %exit
+
+update:
+ %acc.next = add i32 %acc, %v
+ %c2 = icmp slt i32 %i.next, %n
+ br i1 %c2, label %header, label %exit
+
+exit:
+ %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+ store i32 %r, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+; A degenerate header PHI (%p) is folded before the backedges are merged, so
+; it does not turn into a PHI cycle through the unique backedge block.
+
+define i32 @degenerate_header_phi(i32 %n) {
+; AMDGPU-LABEL: define i32 @degenerate_header_phi(
+; AMDGPU-SAME: i32 [[N:%.*]]) {
+; AMDGPU-NEXT: [[ENTRY:.*]]:
+; AMDGPU-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT: br label %[[HEADER:.*]]
+; AMDGPU: [[HEADER]]:
+; AMDGPU-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; AMDGPU-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT: [[COND:%.*]] = icmp eq i32 [[I]], [[TID]]
+; AMDGPU-NEXT: br i1 [[COND]], label %[[L1:.*]], label %[[L2:.*]]
+; AMDGPU: [[L1]]:
+; AMDGPU-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; AMDGPU: [[HEADER_BACKEDGE]]:
+; AMDGPU-NEXT: br label %[[HEADER]]
+; AMDGPU: [[L2]]:
+; AMDGPU-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT: br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; AMDGPU: [[EXIT]]:
+; AMDGPU-NEXT: [[R:%.*]] = add i32 [[I]], 0
+; AMDGPU-NEXT: ret i32 [[R]]
+;
+; GENERIC-LABEL: define i32 @degenerate_header_phi(
+; GENERIC-SAME: i32 [[N:%.*]]) {
+; GENERIC-NEXT: [[ENTRY:.*]]:
+; GENERIC-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT: br label %[[HEADER:.*]]
+; GENERIC: [[HEADER]]:
+; GENERIC-NEXT: [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; GENERIC-NEXT: [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT: [[COND:%.*]] = icmp eq i32 [[I]], [[TID]]
+; GENERIC-NEXT: br i1 [[COND]], label %[[L1:.*]], label %[[L2:.*]]
+; GENERIC: [[L1]]:
+; GENERIC-NEXT: [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; GENERIC: [[HEADER_BACKEDGE]]:
+; GENERIC-NEXT: br label %[[HEADER]]
+; GENERIC: [[L2]]:
+; GENERIC-NEXT: [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT: br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; GENERIC: [[EXIT]]:
+; GENERIC-NEXT: [[R:%.*]] = add i32 [[I]], 0
+; GENERIC-NEXT: ret i32 [[R]]
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %header
+
+header:
+ %i = phi i32 [ 0, %entry ], [ %i.next, %l1 ], [ %i.next, %l2 ]
+ %p = phi i32 [ 0, %entry ], [ %p, %l1 ], [ 0, %l2 ]
+ %i.next = add i32 %i, 1
+ %cond = icmp eq i32 %i, %tid
+ br i1 %cond, label %l1, label %l2
+
+l1:
+ %c1 = icmp slt i32 %i.next, %n
+ br i1 %c1, label %header, label %exit
+
+l2:
+ %c2 = icmp slt i32 %i.next, %n
+ br i1 %c2, label %header, label %exit
+
+exit:
+ %r = add i32 %i, %p
+ ret i32 %r
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,1" "amdgpu-no-wwm" }
More information about the llvm-branch-commits
mailing list