[llvm-branch-commits] [llvm] 0431144 - [LoopSimplify] Don't separate nested loops on targets with branch divergence (#235627)

via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Wed Nov 11 01:51:52 PST 2026


Author: tianhbai
Date: 2026-11-11T11:09:03+08:00
New Revision: 04311443d9df399925bcafdc6b75071155c26618

URL: https://github.com/llvm/llvm-project/commit/04311443d9df399925bcafdc6b75071155c26618
DIFF: https://github.com/llvm/llvm-project/commit/04311443d9df399925bcafdc6b75071155c26618.diff

LOG: [LoopSimplify] Don't separate nested loops on targets with branch divergence (#235627)

When a loop header has multiple backedges, LoopSimplify may call
`separateNestedLoop` to move some of them into a new inner loop. On
targets
with branch divergence, if those backedges are selected by a divergent
condition (e.g. a divergent `continue`), the new inner loop gets a
divergent
exit, and loop-carried values that were uniform (e.g. the loop counter)
can
become divergent. On AMDGPU this turns scalar values into VGPRs and adds
an
extra exec-mask loop level.

Skip `separateNestedLoop` when `TTI->hasBranchDivergence(F)` is true and
fall
back to `insertUniqueBackedgeBlock`. Degenerate header PHIs are now also
folded before inserting the unique backedge block, as
`separateNestedLoop`
used to do.

This is a per-function check rather than a per-branch uniformity check,
since `UniformityAnalysis` is not available in most places LoopSimplify
runs.
`SimpleLoopUnswitch` uses the same check to disable non-trivial
unswitching.

On 954 internal compute pipelines on gfx1201, 28 change; none increase
VGPR
count or scratch usage, and total code size of the changed pipelines
decreases by 0.7%. On internal graphics workloads on gfx1201 we measured
about 1-1.6% performance improvement.

Assisted-by: Claude Opus 5.5

Added: 
    llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
    llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
    llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll

Modified: 
    llvm/include/llvm/Transforms/Utils/LoopSimplify.h
    llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
    llvm/lib/Transforms/Utils/LoopSimplify.cpp
    llvm/lib/Transforms/Utils/LoopUnroll.cpp
    llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
index 58dbbf6b5f75a5..7b151d3ba5139e 100644
--- a/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
+++ b/llvm/include/llvm/Transforms/Utils/LoopSimplify.h
@@ -49,6 +49,7 @@ class Loop;
 class LoopInfo;
 class MemorySSAUpdater;
 class ScalarEvolution;
+class TargetTransformInfo;
 
 /// This pass is responsible for loop canonicalization.
 class LoopSimplifyPass : public RequiredPassInfoMixin<LoopSimplifyPass> {
@@ -62,9 +63,12 @@ class LoopSimplifyPass : public RequiredPassInfoMixin<LoopSimplifyPass> {
 /// it into a simplified loop nest with preheaders and single backedges. It will
 /// update \c DominatorTree, \c LoopInfo, \c ScalarEvolution and \c MemorySSA
 /// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true.
+/// If \p TTI is non-null and the target has branch divergence, loops with
+/// multiple backedges are not split into nested loops.
 LLVM_ABI bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
                            ScalarEvolution *SE, AssumptionCache *AC,
-                           MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
+                           MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+                           const TargetTransformInfo *TTI = nullptr);
 
 } // end namespace llvm
 

diff  --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index a26a5249cdfc01..4c93f3146bf94a 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1670,8 +1670,8 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
   // will simplify all loops, regardless of whether anything end up being
   // unrolled.
   for (const auto &L : LI) {
-    Changed |=
-        simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
+    Changed |= simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr,
+                            false /* PreserveLCSSA */, &TTI);
     Changed |= formLCSSARecursively(*L, DT, &LI, &SE);
   }
 

diff  --git a/llvm/lib/Transforms/Utils/LoopSimplify.cpp b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
index 7cce2329077699..f54ba31f17492b 100644
--- a/llvm/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/llvm/lib/Transforms/Utils/LoopSimplify.cpp
@@ -54,6 +54,7 @@
 #include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Dominators.h"
@@ -468,11 +469,33 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
   return BEBlock;
 }
 
+/// Replace PHI nodes in the loop header that simplify to a single value.
+static bool simplifyHeaderPHIs(Loop *L, DominatorTree *DT, LoopInfo *LI,
+                               ScalarEvolution *SE, AssumptionCache *AC,
+                               bool PreserveLCSSA) {
+  bool Changed = false;
+  const DataLayout &DL = L->getHeader()->getDataLayout();
+  PHINode *PN;
+  for (BasicBlock::iterator I = L->getHeader()->begin();
+       (PN = dyn_cast<PHINode>(I++));)
+    if (Value *V = simplifyInstruction(PN, {DL, nullptr, DT, AC})) {
+      if (SE)
+        SE->forgetValue(PN);
+      if (!PreserveLCSSA || LI->replacementPreservesLCSSAForm(PN, V)) {
+        PN->replaceAllUsesWith(V);
+        PN->eraseFromParent();
+        Changed = true;
+      }
+    }
+  return Changed;
+}
+
 /// Simplify one loop and queue further loops for simplification.
 static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
                             DominatorTree *DT, LoopInfo *LI,
                             ScalarEvolution *SE, AssumptionCache *AC,
-                            MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+                            MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+                            const TargetTransformInfo *TTI) {
   bool Changed = false;
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
@@ -554,7 +577,12 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
     // If this is really a nested loop, rip it out into a child loop.  Don't do
     // this for loops with a giant number of backedges, just factor them into a
     // common backedge instead.
-    if (L->getNumBackEdges() < 8) {
+    // On targets with branch divergence, a backedge taken by only some threads
+    // (e.g. a divergent "continue") would become a divergent inner loop, which
+    // can turn otherwise uniform loop state into per-thread state.
+    bool AllowSeparate =
+        !TTI || !TTI->hasBranchDivergence(L->getHeader()->getParent());
+    if (AllowSeparate && L->getNumBackEdges() < 8) {
       if (Loop *OuterL = separateNestedLoop(L, Preheader, DT, LI, SE,
                                             PreserveLCSSA, AC, MSSAU)) {
         ++NumNested;
@@ -572,7 +600,10 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
 
     // If we either couldn't, or didn't want to, identify nesting of the loops,
     // insert a new block that all backedges target, then make it jump to the
-    // loop header.
+    // loop header. Fold degenerate header PHIs first, so they don't turn into
+    // a PHI cycle through the new backedge block.
+    if (simplifyHeaderPHIs(L, DT, LI, SE, AC, PreserveLCSSA))
+      Changed = true;
     LoopLatch = insertUniqueBackedgeBlock(L, Preheader, DT, LI, MSSAU);
     if (LoopLatch)
       Changed = true;
@@ -581,22 +612,11 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
 
-  const DataLayout &DL = L->getHeader()->getDataLayout();
-
   // Scan over the PHI nodes in the loop header.  Since they now have only two
   // incoming values (the loop is canonicalized), we may have simplified the PHI
   // down to 'X = phi [X, Y]', which should be replaced with 'Y'.
-  PHINode *PN;
-  for (BasicBlock::iterator I = L->getHeader()->begin();
-       (PN = dyn_cast<PHINode>(I++)); )
-    if (Value *V = simplifyInstruction(PN, {DL, nullptr, DT, AC})) {
-      if (SE) SE->forgetValue(PN);
-      if (!PreserveLCSSA || LI->replacementPreservesLCSSAForm(PN, V)) {
-        PN->replaceAllUsesWith(V);
-        PN->eraseFromParent();
-        Changed = true;
-      }
-    }
+  if (simplifyHeaderPHIs(L, DT, LI, SE, AC, PreserveLCSSA))
+    Changed = true;
 
   // If this loop has multiple exits and the exits all go to the same
   // block, attempt to merge the exits. This helps several passes, such
@@ -691,7 +711,8 @@ static bool simplifyOneLoop(Loop *L, SmallVectorImpl<Loop *> &Worklist,
 
 bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
                         ScalarEvolution *SE, AssumptionCache *AC,
-                        MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
+                        MemorySSAUpdater *MSSAU, bool PreserveLCSSA,
+                        const TargetTransformInfo *TTI) {
   bool Changed = false;
 
 #ifndef NDEBUG
@@ -719,7 +740,7 @@ bool llvm::simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI,
 
   while (!Worklist.empty())
     Changed |= simplifyOneLoop(Worklist.pop_back_val(), Worklist, DT, LI, SE,
-                               AC, MSSAU, PreserveLCSSA);
+                               AC, MSSAU, PreserveLCSSA, TTI);
 
   // Changing exit conditions for blocks may affect exit counts of this loop and
   // any of its parents, so we must invalidate the entire subtree if we've made
@@ -798,11 +819,14 @@ bool LoopSimplify::runOnFunction(Function &F) {
     MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
   }
 
+  auto *TTIWP = getAnalysisIfAvailable<TargetTransformInfoWrapperPass>();
+  const TargetTransformInfo *TTI = TTIWP ? &TTIWP->getTTI(F) : nullptr;
+
   bool PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
 
   // Simplify each loop nest in the function.
   for (auto *L : *LI)
-    Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA);
+    Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), PreserveLCSSA, TTI);
 
 #ifndef NDEBUG
   if (PreserveLCSSA) {
@@ -821,6 +845,7 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
   DominatorTree *DT = &AM.getResult<DominatorTreeAnalysis>(F);
   ScalarEvolution *SE = AM.getCachedResult<ScalarEvolutionAnalysis>(F);
   AssumptionCache *AC = &AM.getResult<AssumptionAnalysis>(F);
+  const TargetTransformInfo *TTI = &AM.getResult<TargetIRAnalysis>(F);
   auto *MSSAAnalysis = AM.getCachedResult<MemorySSAAnalysis>(F);
   std::unique_ptr<MemorySSAUpdater> MSSAU;
   if (MSSAAnalysis) {
@@ -832,8 +857,8 @@ PreservedAnalyses LoopSimplifyPass::run(Function &F,
   // Note that we don't preserve LCSSA in the new PM, if you need it run LCSSA
   // after simplifying the loops. MemorySSA is preserved if it exists.
   for (auto *L : *LI)
-    Changed |=
-        simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(), /*PreserveLCSSA*/ false);
+    Changed |= simplifyLoop(L, DT, LI, SE, AC, MSSAU.get(),
+                            /*PreserveLCSSA*/ false, TTI);
 
   if (!Changed)
     return PreservedAnalyses::all();

diff  --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index db44f074359ac6..20aa2f105c553b 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1738,11 +1738,11 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
 
     // TODO: That potentially might be compile-time expensive. We should try
     // to fix the loop-simplified form incrementally.
-    simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+    simplifyLoop(OuterL, DT, LI, SE, AC, nullptr, PreserveLCSSA, TTI);
   } else {
     // Simplify loops for which we might've broken loop-simplify form.
     for (Loop *SubLoop : LoopsToSimplify)
-      simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA);
+      simplifyLoop(SubLoop, DT, LI, SE, AC, nullptr, PreserveLCSSA, TTI);
   }
 
   return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled

diff  --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 0f0ecbf896c3f4..616afab41ec54a 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7983,8 +7983,8 @@ LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
   // will simplify all loops, regardless of whether anything end up being
   // vectorized.
   for (const auto &L : *LI)
-    Changed |= CFGChanged |=
-        simplifyLoop(L, DT, LI, SE, AC, nullptr, false /* PreserveLCSSA */);
+    Changed |= CFGChanged |= simplifyLoop(L, DT, LI, SE, AC, nullptr,
+                                          false /* PreserveLCSSA */, TTI);
 
   // Build up a worklist of inner-loops to vectorize. This is necessary as
   // the act of vectorizing or partially unrolling a loop creates new loops

diff  --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg b/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
new file mode 100644
index 00000000000000..7c492428aec761
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/lit.local.cfg
@@ -0,0 +1,2 @@
+if not "AMDGPU" in config.root.targets:
+    config.unsupported = True

diff  --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
new file mode 100644
index 00000000000000..752d8b86012e4d
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent-resimplify.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="loop-simplify,simplifycfg<no-keep-loops>,loop-unroll" < %s | FileCheck %s --check-prefix=UNROLL
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes="loop-simplify,simplifycfg<no-keep-loops>,loop-vectorize" < %s | FileCheck %s --check-prefix=VECTORIZE
+
+; SimplifyCFG without canonical loops folds the unique backedge block inserted
+; by LoopSimplify. The loop unroller and the vectorizer re-simplify the loop
+; and must not separate it into a nested loop on a target with branch
+; divergence either.
+
+define i32 @continue_loop(i32 %n, ptr %p) {
+; UNROLL-LABEL: define i32 @continue_loop(
+; UNROLL-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; UNROLL-NEXT:  [[ENTRY:.*]]:
+; UNROLL-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; UNROLL-NEXT:    br label %[[HEADER:.*]]
+; UNROLL:       [[HEADER]]:
+; UNROLL-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; UNROLL-NEXT:    [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; UNROLL-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; UNROLL-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[I]]
+; UNROLL-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; UNROLL-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; UNROLL-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; UNROLL-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; UNROLL-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; UNROLL:       [[CONTINUE]]:
+; UNROLL-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; UNROLL-NEXT:    br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; UNROLL:       [[HEADER_BACKEDGE]]:
+; UNROLL-NEXT:    [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; UNROLL-NEXT:    br label %[[HEADER]]
+; UNROLL:       [[UPDATE]]:
+; UNROLL-NEXT:    [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; UNROLL-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; UNROLL-NEXT:    br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; UNROLL:       [[EXIT]]:
+; UNROLL-NEXT:    [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; UNROLL-NEXT:    ret i32 [[R]]
+;
+; VECTORIZE-LABEL: define i32 @continue_loop(
+; VECTORIZE-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) {
+; VECTORIZE-NEXT:  [[ENTRY:.*]]:
+; VECTORIZE-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; VECTORIZE-NEXT:    br label %[[HEADER:.*]]
+; VECTORIZE:       [[HEADER]]:
+; VECTORIZE-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; VECTORIZE-NEXT:    [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; VECTORIZE-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; VECTORIZE-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[P]], i32 [[I]]
+; VECTORIZE-NEXT:    [[V:%.*]] = load i32, ptr [[GEP]], align 4
+; VECTORIZE-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; VECTORIZE-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; VECTORIZE-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; VECTORIZE-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; VECTORIZE:       [[CONTINUE]]:
+; VECTORIZE-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; VECTORIZE-NEXT:    br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; VECTORIZE:       [[HEADER_BACKEDGE]]:
+; VECTORIZE-NEXT:    [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; VECTORIZE-NEXT:    br label %[[HEADER]]
+; VECTORIZE:       [[UPDATE]]:
+; VECTORIZE-NEXT:    [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; VECTORIZE-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; VECTORIZE-NEXT:    br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; VECTORIZE:       [[EXIT]]:
+; VECTORIZE-NEXT:    [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; VECTORIZE-NEXT:    ret i32 [[R]]
+;
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  br label %header
+
+header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+  %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+  %i.next = add i32 %i, 1
+  %gep = getelementptr i32, ptr %p, i32 %i
+  %v = load i32, ptr %gep, align 4
+  %bits = lshr i32 %v, %tid
+  %bit = and i32 %bits, 1
+  %is.zero = icmp eq i32 %bit, 0
+  br i1 %is.zero, label %continue, label %update
+
+continue:
+  %c1 = icmp slt i32 %i.next, %n
+  br i1 %c1, label %header, label %exit
+
+update:
+  %acc.next = add i32 %acc, %v
+  %c2 = icmp slt i32 %i.next, %n
+  br i1 %c2, label %header, label %exit
+
+exit:
+  %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+  ret i32 %r
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()

diff  --git a/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll
new file mode 100644
index 00000000000000..f8d8ec49464d52
--- /dev/null
+++ b/llvm/test/Transforms/LoopSimplify/AMDGPU/separate-nested-loop-divergent.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-simplify -mtriple=amdgcn-amd-amdhsa < %s | FileCheck %s --check-prefix=AMDGPU
+; RUN: opt -S -passes=loop-simplify < %s | FileCheck %s --check-prefix=GENERIC
+
+; A loop with two backedges where the "continue" backedge is selected by a
+; per-lane condition. On a target with branch divergence the loop is not
+; separated into a nested loop; a unique backedge block is inserted instead.
+
+define amdgpu_kernel void @continue_loop(i32 %n, ptr addrspace(1) %p, ptr addrspace(1) %out) {
+; AMDGPU-LABEL: define amdgpu_kernel void @continue_loop(
+; AMDGPU-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; AMDGPU-NEXT:  [[ENTRY:.*]]:
+; AMDGPU-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT:    br label %[[HEADER:.*]]
+; AMDGPU:       [[HEADER]]:
+; AMDGPU-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; AMDGPU-NEXT:    [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_BE:%.*]], %[[HEADER_BACKEDGE]] ]
+; AMDGPU-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; AMDGPU-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGPU-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; AMDGPU-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; AMDGPU-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; AMDGPU-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE:.*]], label %[[UPDATE:.*]]
+; AMDGPU:       [[CONTINUE]]:
+; AMDGPU-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; AMDGPU:       [[HEADER_BACKEDGE]]:
+; AMDGPU-NEXT:    [[ACC_BE]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT:%.*]], %[[UPDATE]] ]
+; AMDGPU-NEXT:    br label %[[HEADER]]
+; AMDGPU:       [[UPDATE]]:
+; AMDGPU-NEXT:    [[ACC_NEXT]] = add i32 [[ACC]], [[V]]
+; AMDGPU-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; AMDGPU:       [[EXIT]]:
+; AMDGPU-NEXT:    [[R:%.*]] = phi i32 [ [[ACC]], %[[CONTINUE]] ], [ [[ACC_NEXT]], %[[UPDATE]] ]
+; AMDGPU-NEXT:    [[OUT_GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; AMDGPU-NEXT:    store i32 [[R]], ptr addrspace(1) [[OUT_GEP]], align 4
+; AMDGPU-NEXT:    ret void
+;
+; GENERIC-LABEL: define amdgpu_kernel void @continue_loop(
+; GENERIC-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; GENERIC-NEXT:  [[ENTRY:.*]]:
+; GENERIC-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT:    br label %[[HEADER_OUTER:.*]]
+; GENERIC:       [[HEADER_OUTER]]:
+; GENERIC-NEXT:    [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT:    [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT:    br label %[[HEADER:.*]]
+; GENERIC:       [[HEADER]]:
+; GENERIC-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; GENERIC-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; GENERIC-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; GENERIC-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; GENERIC-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; GENERIC-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; GENERIC-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; GENERIC:       [[CONTINUE]]:
+; GENERIC-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; GENERIC:       [[UPDATE]]:
+; GENERIC-NEXT:    [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; GENERIC-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; GENERIC:       [[EXIT_LOOPEXIT]]:
+; GENERIC-NEXT:    br label %[[EXIT:.*]]
+; GENERIC:       [[EXIT_LOOPEXIT1]]:
+; GENERIC-NEXT:    br label %[[EXIT]]
+; GENERIC:       [[EXIT]]:
+; GENERIC-NEXT:    [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; GENERIC-NEXT:    [[OUT_GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 [[TID]]
+; GENERIC-NEXT:    store i32 [[R]], ptr addrspace(1) [[OUT_GEP]], align 4
+; GENERIC-NEXT:    ret void
+;
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  br label %header
+
+header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+  %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+  %i.next = add i32 %i, 1
+  %gep = getelementptr i32, ptr addrspace(1) %p, i32 %i
+  %v = load i32, ptr addrspace(1) %gep, align 4
+  %bits = lshr i32 %v, %tid
+  %bit = and i32 %bits, 1
+  %is.zero = icmp eq i32 %bit, 0
+  br i1 %is.zero, label %continue, label %update
+
+continue:
+  %c1 = icmp slt i32 %i.next, %n
+  br i1 %c1, label %header, label %exit
+
+update:
+  %acc.next = add i32 %acc, %v
+  %c2 = icmp slt i32 %i.next, %n
+  br i1 %c2, label %header, label %exit
+
+exit:
+  %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+  %out.gep = getelementptr i32, ptr addrspace(1) %out, i32 %tid
+  store i32 %r, ptr addrspace(1) %out.gep, align 4
+  ret void
+}
+
+; A function that executes with a single lane has no branch divergence, so
+; the nested loop is still separated.
+
+define amdgpu_kernel void @continue_loop_single_lane(i32 %n, ptr addrspace(1) %p, ptr addrspace(1) %out) #0 {
+; AMDGPU-LABEL: define amdgpu_kernel void @continue_loop_single_lane(
+; AMDGPU-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; AMDGPU-NEXT:  [[ENTRY:.*]]:
+; AMDGPU-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT:    br label %[[HEADER_OUTER:.*]]
+; AMDGPU:       [[HEADER_OUTER]]:
+; AMDGPU-NEXT:    [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; AMDGPU-NEXT:    [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; AMDGPU-NEXT:    br label %[[HEADER:.*]]
+; AMDGPU:       [[HEADER]]:
+; AMDGPU-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; AMDGPU-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; AMDGPU-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; AMDGPU-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; AMDGPU-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; AMDGPU-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; AMDGPU-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; AMDGPU:       [[CONTINUE]]:
+; AMDGPU-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; AMDGPU:       [[UPDATE]]:
+; AMDGPU-NEXT:    [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; AMDGPU-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; AMDGPU:       [[EXIT_LOOPEXIT]]:
+; AMDGPU-NEXT:    br label %[[EXIT:.*]]
+; AMDGPU:       [[EXIT_LOOPEXIT1]]:
+; AMDGPU-NEXT:    br label %[[EXIT]]
+; AMDGPU:       [[EXIT]]:
+; AMDGPU-NEXT:    [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; AMDGPU-NEXT:    store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; AMDGPU-NEXT:    ret void
+;
+; GENERIC-LABEL: define amdgpu_kernel void @continue_loop_single_lane(
+; GENERIC-SAME: i32 [[N:%.*]], ptr addrspace(1) [[P:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; GENERIC-NEXT:  [[ENTRY:.*]]:
+; GENERIC-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT:    br label %[[HEADER_OUTER:.*]]
+; GENERIC:       [[HEADER_OUTER]]:
+; GENERIC-NEXT:    [[I_PH:%.*]] = phi i32 [ [[I_NEXT:%.*]], %[[UPDATE:.*]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT:    [[ACC_PH:%.*]] = phi i32 [ [[ACC_NEXT:%.*]], %[[UPDATE]] ], [ 0, %[[ENTRY]] ]
+; GENERIC-NEXT:    br label %[[HEADER:.*]]
+; GENERIC:       [[HEADER]]:
+; GENERIC-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT]], %[[CONTINUE:.*]] ], [ [[I_PH]], %[[HEADER_OUTER]] ]
+; GENERIC-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[I]]
+; GENERIC-NEXT:    [[V:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; GENERIC-NEXT:    [[BITS:%.*]] = lshr i32 [[V]], [[TID]]
+; GENERIC-NEXT:    [[BIT:%.*]] = and i32 [[BITS]], 1
+; GENERIC-NEXT:    [[IS_ZERO:%.*]] = icmp eq i32 [[BIT]], 0
+; GENERIC-NEXT:    br i1 [[IS_ZERO]], label %[[CONTINUE]], label %[[UPDATE]]
+; GENERIC:       [[CONTINUE]]:
+; GENERIC-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C1]], label %[[HEADER]], label %[[EXIT_LOOPEXIT:.*]]
+; GENERIC:       [[UPDATE]]:
+; GENERIC-NEXT:    [[ACC_NEXT]] = add i32 [[ACC_PH]], [[V]]
+; GENERIC-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C2]], label %[[HEADER_OUTER]], label %[[EXIT_LOOPEXIT1:.*]]
+; GENERIC:       [[EXIT_LOOPEXIT]]:
+; GENERIC-NEXT:    br label %[[EXIT:.*]]
+; GENERIC:       [[EXIT_LOOPEXIT1]]:
+; GENERIC-NEXT:    br label %[[EXIT]]
+; GENERIC:       [[EXIT]]:
+; GENERIC-NEXT:    [[R:%.*]] = phi i32 [ [[ACC_PH]], %[[EXIT_LOOPEXIT]] ], [ [[ACC_NEXT]], %[[EXIT_LOOPEXIT1]] ]
+; GENERIC-NEXT:    store i32 [[R]], ptr addrspace(1) [[OUT]], align 4
+; GENERIC-NEXT:    ret void
+;
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  br label %header
+
+header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %continue ], [ %i.next, %update ]
+  %acc = phi i32 [ 0, %entry ], [ %acc, %continue ], [ %acc.next, %update ]
+  %i.next = add i32 %i, 1
+  %gep = getelementptr i32, ptr addrspace(1) %p, i32 %i
+  %v = load i32, ptr addrspace(1) %gep, align 4
+  %bits = lshr i32 %v, %tid
+  %bit = and i32 %bits, 1
+  %is.zero = icmp eq i32 %bit, 0
+  br i1 %is.zero, label %continue, label %update
+
+continue:
+  %c1 = icmp slt i32 %i.next, %n
+  br i1 %c1, label %header, label %exit
+
+update:
+  %acc.next = add i32 %acc, %v
+  %c2 = icmp slt i32 %i.next, %n
+  br i1 %c2, label %header, label %exit
+
+exit:
+  %r = phi i32 [ %acc, %continue ], [ %acc.next, %update ]
+  store i32 %r, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; A degenerate header PHI (%p) is folded before the backedges are merged, so
+; it does not turn into a PHI cycle through the unique backedge block.
+
+define i32 @degenerate_header_phi(i32 %n) {
+; AMDGPU-LABEL: define i32 @degenerate_header_phi(
+; AMDGPU-SAME: i32 [[N:%.*]]) {
+; AMDGPU-NEXT:  [[ENTRY:.*]]:
+; AMDGPU-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; AMDGPU-NEXT:    br label %[[HEADER:.*]]
+; AMDGPU:       [[HEADER]]:
+; AMDGPU-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; AMDGPU-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; AMDGPU-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], [[TID]]
+; AMDGPU-NEXT:    br i1 [[COND]], label %[[L1:.*]], label %[[L2:.*]]
+; AMDGPU:       [[L1]]:
+; AMDGPU-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; AMDGPU:       [[HEADER_BACKEDGE]]:
+; AMDGPU-NEXT:    br label %[[HEADER]]
+; AMDGPU:       [[L2]]:
+; AMDGPU-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; AMDGPU-NEXT:    br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; AMDGPU:       [[EXIT]]:
+; AMDGPU-NEXT:    [[R:%.*]] = add i32 [[I]], 0
+; AMDGPU-NEXT:    ret i32 [[R]]
+;
+; GENERIC-LABEL: define i32 @degenerate_header_phi(
+; GENERIC-SAME: i32 [[N:%.*]]) {
+; GENERIC-NEXT:  [[ENTRY:.*]]:
+; GENERIC-NEXT:    [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; GENERIC-NEXT:    br label %[[HEADER:.*]]
+; GENERIC:       [[HEADER]]:
+; GENERIC-NEXT:    [[I:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[HEADER_BACKEDGE:.*]] ]
+; GENERIC-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; GENERIC-NEXT:    [[COND:%.*]] = icmp eq i32 [[I]], [[TID]]
+; GENERIC-NEXT:    br i1 [[COND]], label %[[L1:.*]], label %[[L2:.*]]
+; GENERIC:       [[L1]]:
+; GENERIC-NEXT:    [[C1:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C1]], label %[[HEADER_BACKEDGE]], label %[[EXIT:.*]]
+; GENERIC:       [[HEADER_BACKEDGE]]:
+; GENERIC-NEXT:    br label %[[HEADER]]
+; GENERIC:       [[L2]]:
+; GENERIC-NEXT:    [[C2:%.*]] = icmp slt i32 [[I_NEXT]], [[N]]
+; GENERIC-NEXT:    br i1 [[C2]], label %[[HEADER_BACKEDGE]], label %[[EXIT]]
+; GENERIC:       [[EXIT]]:
+; GENERIC-NEXT:    [[R:%.*]] = add i32 [[I]], 0
+; GENERIC-NEXT:    ret i32 [[R]]
+;
+entry:
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  br label %header
+
+header:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %l1 ], [ %i.next, %l2 ]
+  %p = phi i32 [ 0, %entry ], [ %p, %l1 ], [ 0, %l2 ]
+  %i.next = add i32 %i, 1
+  %cond = icmp eq i32 %i, %tid
+  br i1 %cond, label %l1, label %l2
+
+l1:
+  %c1 = icmp slt i32 %i.next, %n
+  br i1 %c1, label %header, label %exit
+
+l2:
+  %c2 = icmp slt i32 %i.next, %n
+  br i1 %c2, label %header, label %exit
+
+exit:
+  %r = add i32 %i, %p
+  ret i32 %r
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+attributes #0 = { "amdgpu-flat-work-group-size"="1,1" "amdgpu-no-wwm" }


        


More information about the llvm-branch-commits mailing list