[llvm] [LoopUnroll] Don't runtime-unroll multi-exit loops with a divergent side exit (PR #224924)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 20 05:33:05 PDT 2026
https://github.com/xgxanq created https://github.com/llvm/llvm-project/pull/224924
Multi-exit runtime unrolling uses branch probability to decide whether a non-latch side exit is worth unrolling. But on a divergent target a low-probability branch can still be divergent, and unrolling duplicates that divergent control flow across every copy, so branch probability is not a safe proxy there.
Thread UniformityInfo into UnrollLoop / UnrollRuntimeLoopRemainder; when the side exit's terminator is divergent, skip the branch-probability check and fall through to the deopt heuristic.
>From ffe94dbf9651dc8ec6fddbb86b4afb481ee45e11 Mon Sep 17 00:00:00 2001
From: anqfu <anqfu at amd.com>
Date: Sun, 20 Sep 2026 12:22:48 +0000
Subject: [PATCH] [LoopUnroll] Don't runtime-unroll multi-exit loops with a
divergent side exit
Multi-exit runtime unrolling uses branch probability to decide whether a
non-latch side exit is worth unrolling. But on a divergent target a
low-probability branch can still be divergent, and unrolling duplicates
that divergent control flow across every copy, so branch probability is
not a safe proxy there.
Thread UniformityInfo into UnrollLoop / UnrollRuntimeLoopRemainder; when
the side exit's terminator is divergent, skip the branch-probability
check and fall through to the deopt heuristic.
---
.../llvm/Transforms/Utils/UnrollLoop.h | 18 +-
llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 5 +-
llvm/lib/Transforms/Utils/LoopUnroll.cpp | 14 +-
.../Transforms/Utils/LoopUnrollRuntime.cpp | 27 +-
.../unroll-runtime-multi-exit-divergent.ll | 246 ++++++++++++++++++
5 files changed, 283 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-multi-exit-divergent.ll
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index ad52b9f57d0fb..4fb56e2073e14 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/DenseMap.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/InstructionCost.h"
@@ -82,14 +83,12 @@ struct UnrollLoopOptions {
bool AddAdditionalAccumulators = false;
};
-LLVM_ABI LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO,
- LoopInfo *LI, ScalarEvolution *SE,
- DominatorTree *DT, AssumptionCache *AC,
- const llvm::TargetTransformInfo *TTI,
- OptimizationRemarkEmitter *ORE,
- bool PreserveLCSSA,
- Loop **RemainderLoop = nullptr,
- AAResults *AA = nullptr);
+LLVM_ABI LoopUnrollResult
+UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, ScalarEvolution *SE,
+ DominatorTree *DT, AssumptionCache *AC,
+ const llvm::TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE,
+ bool PreserveLCSSA, Loop **RemainderLoop = nullptr,
+ AAResults *AA = nullptr, UniformityInfo *UI = nullptr);
LLVM_ABI bool UnrollRuntimeLoopRemainder(
Loop *L, unsigned Count, bool AllowExpensiveTripCount,
@@ -99,7 +98,8 @@ LLVM_ABI bool UnrollRuntimeLoopRemainder(
unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
Loop **ResultLoop = nullptr,
std::optional<unsigned> OriginalTripCount = std::nullopt,
- BranchProbability OriginalLoopProb = BranchProbability::getUnknown());
+ BranchProbability OriginalLoopProb = BranchProbability::getUnknown(),
+ UniformityInfo *UI = nullptr);
LLVM_ABI LoopUnrollResult UnrollAndJamLoop(
Loop *L, unsigned Count, unsigned TripCount, unsigned TripMultiple,
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index ad3b123f3327c..9da455eb65719 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1457,8 +1457,9 @@ tryToUnrollLoop(Loop *L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
ULO.SCEVExpansionBudget = UP.SCEVExpansionBudget;
ULO.RuntimeUnrollMultiExit = UP.RuntimeUnrollMultiExit;
ULO.AddAdditionalAccumulators = UP.AddAdditionalAccumulators;
- LoopUnrollResult UnrollResult = UnrollLoop(
- L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop, AA);
+ LoopUnrollResult UnrollResult =
+ UnrollLoop(L, ULO, LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA,
+ &RemainderLoop, AA, UI);
if (UnrollResult == LoopUnrollResult::Unmodified) {
if (PInfo.ExplicitUnroll) {
LLVM_DEBUG(dbgs().indent(1)
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index e80dd18a34fb7..726d7dda31159 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -925,11 +925,13 @@ static void fixProbContradiction(Loop *L, UnrollLoopOptions ULO,
///
/// If RemainderLoop is non-null, it will receive the remainder loop (if
/// required and not fully unrolled).
-LoopUnrollResult
-llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
- ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC,
- const TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE,
- bool PreserveLCSSA, Loop **RemainderLoop, AAResults *AA) {
+LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
+ ScalarEvolution *SE, DominatorTree *DT,
+ AssumptionCache *AC,
+ const TargetTransformInfo *TTI,
+ OptimizationRemarkEmitter *ORE,
+ bool PreserveLCSSA, Loop **RemainderLoop,
+ AAResults *AA, UniformityInfo *UI) {
assert(DT && "DomTree is required");
if (!L->getLoopPreheader()) {
@@ -1063,7 +1065,7 @@ llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI,
L, ULO.Count, ULO.AllowExpensiveTripCount, EpilogProfitability,
ULO.UnrollRemainder, ULO.ForgetAllSCEV, LI, SE, DT, AC, TTI,
PreserveLCSSA, ULO.SCEVExpansionBudget, ULO.RuntimeUnrollMultiExit,
- RemainderLoop, OriginalTripCount, OriginalLoopProb)) {
+ RemainderLoop, OriginalTripCount, OriginalLoopProb, UI)) {
if (ULO.Force)
ULO.Runtime = false;
else {
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 92776cd1472bf..9687b4ff72b2b 100644
--- a/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -25,6 +25,7 @@
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/UniformityAnalysis.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Dominators.h"
@@ -537,7 +538,7 @@ static Loop *CloneLoopBlocks(Loop *L, Value *NewIter,
/// Returns true if we can profitably unroll the multi-exit loop L.
static bool canProfitablyRuntimeUnrollMultiExitLoop(
- Loop *L, const TargetTransformInfo *TTI,
+ Loop *L, const TargetTransformInfo *TTI, UniformityInfo *UI,
SmallVectorImpl<BasicBlock *> &OtherExits, BasicBlock *LatchExit,
bool UseEpilogRemainder) {
@@ -580,13 +581,19 @@ static bool canProfitablyRuntimeUnrollMultiExitLoop(
assert(LatchBB && "Expected loop to have a latch");
BasicBlock *NonLatchExitingBlock =
(ExitingBlocks[0] == LatchBB) ? ExitingBlocks[1] : ExitingBlocks[0];
- auto BranchProb =
- llvm::getBranchProbability(NonLatchExitingBlock, OtherExits[0]);
- // If BranchProbability could not be extracted (returns unknown), then
- // don't return and do the check for deopt block.
- if (!BranchProb.isUnknown()) {
- auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
- return BranchProb < Threshold;
+ // On divergent targets a rarely-taken branch can still be divergent, and
+ // unrolling duplicates that divergent control flow across every copy. So
+ // branch probability isn't a safe proxy here; fall through to the deopt
+ // check.
+ if (!UI || !UI->hasDivergentTerminator(*NonLatchExitingBlock)) {
+ auto BranchProb =
+ llvm::getBranchProbability(NonLatchExitingBlock, OtherExits[0]);
+ // If BranchProbability could not be extracted (returns unknown), then
+ // don't return and do the check for deopt block.
+ if (!BranchProb.isUnknown()) {
+ auto Threshold = TTI->getPredictableBranchThreshold().getCompl();
+ return BranchProb < Threshold;
+ }
}
}
@@ -676,7 +683,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
const TargetTransformInfo *TTI, bool PreserveLCSSA,
unsigned SCEVExpansionBudget, bool RuntimeUnrollMultiExit,
Loop **ResultLoop, std::optional<unsigned> OriginalTripCount,
- BranchProbability OriginalLoopProb) {
+ BranchProbability OriginalLoopProb, UniformityInfo *UI) {
LLVM_DEBUG(dbgs() << "Trying runtime unrolling on Loop: \n");
LLVM_DEBUG(L->dump());
LLVM_DEBUG(UseEpilogRemainder ? dbgs() << "Using epilog remainder.\n"
@@ -734,7 +741,7 @@ bool llvm::UnrollRuntimeLoopRemainder(
// it is profitable or the general profitability heuristics apply.
if (!RuntimeUnrollMultiExit &&
!canProfitablyRuntimeUnrollMultiExitLoop(
- L, TTI, OtherExits, LatchExit, UseEpilogRemainder)) {
+ L, TTI, UI, OtherExits, LatchExit, UseEpilogRemainder)) {
LLVM_DEBUG(dbgs() << "Multiple exit/exiting blocks in loop and "
"multi-exit unrolling not enabled!\n");
return false;
diff --git a/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-multi-exit-divergent.ll b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-multi-exit-divergent.ll
new file mode 100644
index 0000000000000..d1eaf7109e364
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/AMDGPU/unroll-runtime-multi-exit-divergent.ll
@@ -0,0 +1,246 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mcpu=gfx950 -passes=loop-unroll \
+; RUN: -unroll-runtime -S < %s | FileCheck %s
+
+; Multi-exit runtime unrolling gates a non-latch side exit on the branch-
+; probability heuristic, but on a divergent target a rarely-taken branch can
+; still be divergent, and unrolling duplicates that divergent control flow
+; across every copy. So UniformityInfo is consulted: a divergent side exit must
+; NOT be unrolled, a uniform side exit still may.
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+
+; The side-exit condition mixes in the (divergent) workitem id, so the exit is
+; divergent even though its branch weights make it very unlikely. The loop must
+; be left rolled: no epilog remainder is created.
+;
+define amdgpu_kernel void @divergent_side_exit(ptr addrspace(1) %p, i32 %n, i32 %bound) {
+; CHECK-LABEL: define amdgpu_kernel void @divergent_side_exit(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[N:%.*]], i32 [[BOUND:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TID:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ACC]], [[TID]]
+; CHECK-NEXT: [[SIDECOND:%.*]] = icmp sgt i32 [[VAL]], [[BOUND]]
+; CHECK-NEXT: br i1 [[SIDECOND]], label %[[SIDEEXIT:.*]], label %[[BODY:.*]], !prof [[PROF0:![0-9]+]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; CHECK-NEXT: [[ACC_NEXT]] = add i32 [[ACC]], [[LD]]
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT:.*]], label %[[HEADER]]
+; CHECK: [[SIDEEXIT]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC]], %[[HEADER]] ]
+; CHECK-NEXT: store i32 [[ACC_LCSSA]], ptr addrspace(1) [[P]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %tid = call i32 @llvm.amdgcn.workitem.id.x()
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %latch ]
+ %val = add i32 %acc, %tid
+ %sidecond = icmp sgt i32 %val, %bound
+ br i1 %sidecond, label %sideexit, label %body, !prof !0
+
+body:
+ %gep = getelementptr i32, ptr addrspace(1) %p, i32 %iv
+ %ld = load i32, ptr addrspace(1) %gep
+ %acc.next = add i32 %acc, %ld
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %header
+
+sideexit:
+ store i32 %acc, ptr addrspace(1) %p
+ ret void
+
+exit:
+ ret void
+}
+
+; The side-exit condition only depends on uniform kernel arguments, so the exit
+; is uniform. The predictability heuristic remains valid and the loop is
+; runtime-unrolled with an epilog remainder.
+;
+define amdgpu_kernel void @uniform_side_exit(ptr addrspace(1) %p, i32 %n, i32 %bound) {
+; CHECK-LABEL: define amdgpu_kernel void @uniform_side_exit(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[N:%.*]], i32 [[BOUND:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = freeze i32 [[N]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], -1
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP0]], 7
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 7
+; CHECK-NEXT: br i1 [[TMP2]], label %[[HEADER_EPIL_PREHEADER:.*]], label %[[ENTRY_NEW:.*]]
+; CHECK: [[ENTRY_NEW]]:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i32 [[TMP0]], [[XTRAITER]]
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[IV_NEXT_7:%.*]], %[[LATCH_7:.*]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[ACC_NEXT_7:%.*]], %[[LATCH_7]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i32 [ 0, %[[ENTRY_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[LATCH_7]] ]
+; CHECK-NEXT: [[SIDECOND:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND]], label %[[SIDEEXIT_LOOPEXIT:.*]], label %[[BODY:.*]], !prof [[PROF0]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV]]
+; CHECK-NEXT: [[LD:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
+; CHECK-NEXT: [[ACC_NEXT:%.*]] = add i32 [[ACC]], [[LD]]
+; CHECK-NEXT: br label %[[LATCH:.*]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT:%.*]] = add nuw nsw i32 [[IV]], 1
+; CHECK-NEXT: [[SIDECOND_1:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_1]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_1:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_1]]:
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT]]
+; CHECK-NEXT: [[LD_1:%.*]] = load i32, ptr addrspace(1) [[GEP_1]], align 4
+; CHECK-NEXT: [[ACC_NEXT_1:%.*]] = add i32 [[ACC_NEXT]], [[LD_1]]
+; CHECK-NEXT: br label %[[LATCH_1:.*]]
+; CHECK: [[LATCH_1]]:
+; CHECK-NEXT: [[IV_NEXT_1:%.*]] = add nuw nsw i32 [[IV]], 2
+; CHECK-NEXT: [[SIDECOND_2:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_2]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_2:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_2]]:
+; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_1]]
+; CHECK-NEXT: [[LD_2:%.*]] = load i32, ptr addrspace(1) [[GEP_2]], align 4
+; CHECK-NEXT: [[ACC_NEXT_2:%.*]] = add i32 [[ACC_NEXT_1]], [[LD_2]]
+; CHECK-NEXT: br label %[[LATCH_2:.*]]
+; CHECK: [[LATCH_2]]:
+; CHECK-NEXT: [[IV_NEXT_2:%.*]] = add nuw nsw i32 [[IV]], 3
+; CHECK-NEXT: [[SIDECOND_3:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_3]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_3:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_3]]:
+; CHECK-NEXT: [[GEP_3:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_2]]
+; CHECK-NEXT: [[LD_3:%.*]] = load i32, ptr addrspace(1) [[GEP_3]], align 4
+; CHECK-NEXT: [[ACC_NEXT_3:%.*]] = add i32 [[ACC_NEXT_2]], [[LD_3]]
+; CHECK-NEXT: br label %[[LATCH_3:.*]]
+; CHECK: [[LATCH_3]]:
+; CHECK-NEXT: [[IV_NEXT_3:%.*]] = add nuw nsw i32 [[IV]], 4
+; CHECK-NEXT: [[SIDECOND_4:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_4]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_4:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_4]]:
+; CHECK-NEXT: [[GEP_4:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_3]]
+; CHECK-NEXT: [[LD_4:%.*]] = load i32, ptr addrspace(1) [[GEP_4]], align 4
+; CHECK-NEXT: [[ACC_NEXT_4:%.*]] = add i32 [[ACC_NEXT_3]], [[LD_4]]
+; CHECK-NEXT: br label %[[LATCH_4:.*]]
+; CHECK: [[LATCH_4]]:
+; CHECK-NEXT: [[IV_NEXT_4:%.*]] = add nuw nsw i32 [[IV]], 5
+; CHECK-NEXT: [[SIDECOND_5:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_5]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_5:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_5]]:
+; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_4]]
+; CHECK-NEXT: [[LD_5:%.*]] = load i32, ptr addrspace(1) [[GEP_5]], align 4
+; CHECK-NEXT: [[ACC_NEXT_5:%.*]] = add i32 [[ACC_NEXT_4]], [[LD_5]]
+; CHECK-NEXT: br label %[[LATCH_5:.*]]
+; CHECK: [[LATCH_5]]:
+; CHECK-NEXT: [[IV_NEXT_5:%.*]] = add nuw nsw i32 [[IV]], 6
+; CHECK-NEXT: [[SIDECOND_6:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_6]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_6:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_6]]:
+; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_5]]
+; CHECK-NEXT: [[LD_6:%.*]] = load i32, ptr addrspace(1) [[GEP_6]], align 4
+; CHECK-NEXT: [[ACC_NEXT_6:%.*]] = add i32 [[ACC_NEXT_5]], [[LD_6]]
+; CHECK-NEXT: br label %[[LATCH_6:.*]]
+; CHECK: [[LATCH_6]]:
+; CHECK-NEXT: [[IV_NEXT_6:%.*]] = add nuw nsw i32 [[IV]], 7
+; CHECK-NEXT: [[SIDECOND_7:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_7]], label %[[SIDEEXIT_LOOPEXIT]], label %[[BODY_7:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_7]]:
+; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_NEXT_6]]
+; CHECK-NEXT: [[LD_7:%.*]] = load i32, ptr addrspace(1) [[GEP_7]], align 4
+; CHECK-NEXT: [[ACC_NEXT_7]] = add i32 [[ACC_NEXT_6]], [[LD_7]]
+; CHECK-NEXT: br label %[[LATCH_7]]
+; CHECK: [[LATCH_7]]:
+; CHECK-NEXT: [[IV_NEXT_7]] = add nuw nsw i32 [[IV]], 8
+; CHECK-NEXT: [[NITER_NEXT_7]] = add i32 [[NITER]], 8
+; CHECK-NEXT: [[NITER_NCMP_7:%.*]] = icmp eq i32 [[NITER_NEXT_7]], [[UNROLL_ITER]]
+; CHECK-NEXT: br i1 [[NITER_NCMP_7]], label %[[EXIT_UNR_LCSSA:.*]], label %[[HEADER]]
+; CHECK: [[SIDEEXIT_LOOPEXIT]]:
+; CHECK-NEXT: [[ACC_LCSSA_PH:%.*]] = phi i32 [ [[ACC]], %[[HEADER]] ], [ [[ACC_NEXT]], %[[LATCH]] ], [ [[ACC_NEXT_1]], %[[LATCH_1]] ], [ [[ACC_NEXT_2]], %[[LATCH_2]] ], [ [[ACC_NEXT_3]], %[[LATCH_3]] ], [ [[ACC_NEXT_4]], %[[LATCH_4]] ], [ [[ACC_NEXT_5]], %[[LATCH_5]] ], [ [[ACC_NEXT_6]], %[[LATCH_6]] ]
+; CHECK-NEXT: br label %[[SIDEEXIT:.*]]
+; CHECK: [[SIDEEXIT_LOOPEXIT3:.*]]:
+; CHECK-NEXT: [[ACC_LCSSA_PH4:%.*]] = phi i32 [ [[ACC_EPIL:%.*]], %[[HEADER_EPIL:.*]] ]
+; CHECK-NEXT: br label %[[SIDEEXIT]]
+; CHECK: [[SIDEEXIT]]:
+; CHECK-NEXT: [[ACC_LCSSA:%.*]] = phi i32 [ [[ACC_LCSSA_PH]], %[[SIDEEXIT_LOOPEXIT]] ], [ [[ACC_LCSSA_PH4]], %[[SIDEEXIT_LOOPEXIT3]] ]
+; CHECK-NEXT: store i32 [[ACC_LCSSA]], ptr addrspace(1) [[P]], align 4
+; CHECK-NEXT: ret void
+; CHECK: [[EXIT_UNR_LCSSA]]:
+; CHECK-NEXT: [[IV_UNR:%.*]] = phi i32 [ [[IV_NEXT_7]], %[[LATCH_7]] ]
+; CHECK-NEXT: [[ACC_UNR:%.*]] = phi i32 [ [[ACC_NEXT_7]], %[[LATCH_7]] ]
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[HEADER_EPIL_PREHEADER]], label %[[EXIT:.*]]
+; CHECK: [[HEADER_EPIL_PREHEADER]]:
+; CHECK-NEXT: [[IV_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[ACC_EPIL_INIT:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ACC_UNR]], %[[EXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i32 [[XTRAITER]], 0
+; CHECK-NEXT: call void @llvm.assume(i1 [[LCMP_MOD2]])
+; CHECK-NEXT: br label %[[HEADER_EPIL]]
+; CHECK: [[HEADER_EPIL]]:
+; CHECK-NEXT: [[IV_EPIL:%.*]] = phi i32 [ [[IV_EPIL_INIT]], %[[HEADER_EPIL_PREHEADER]] ], [ [[IV_NEXT_EPIL:%.*]], %[[LATCH_EPIL:.*]] ]
+; CHECK-NEXT: [[ACC_EPIL]] = phi i32 [ [[ACC_EPIL_INIT]], %[[HEADER_EPIL_PREHEADER]] ], [ [[ACC_NEXT_EPIL:%.*]], %[[LATCH_EPIL]] ]
+; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, %[[HEADER_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LATCH_EPIL]] ]
+; CHECK-NEXT: [[SIDECOND_EPIL:%.*]] = icmp sgt i32 [[BOUND]], [[N]]
+; CHECK-NEXT: br i1 [[SIDECOND_EPIL]], label %[[SIDEEXIT_LOOPEXIT3]], label %[[BODY_EPIL:.*]], !prof [[PROF0]]
+; CHECK: [[BODY_EPIL]]:
+; CHECK-NEXT: [[GEP_EPIL:%.*]] = getelementptr i32, ptr addrspace(1) [[P]], i32 [[IV_EPIL]]
+; CHECK-NEXT: [[LD_EPIL:%.*]] = load i32, ptr addrspace(1) [[GEP_EPIL]], align 4
+; CHECK-NEXT: [[ACC_NEXT_EPIL]] = add i32 [[ACC_EPIL]], [[LD_EPIL]]
+; CHECK-NEXT: br label %[[LATCH_EPIL]]
+; CHECK: [[LATCH_EPIL]]:
+; CHECK-NEXT: [[IV_NEXT_EPIL]] = add nuw nsw i32 [[IV_EPIL]], 1
+; CHECK-NEXT: [[EXITCOND_EPIL:%.*]] = icmp eq i32 [[IV_NEXT_EPIL]], [[N]]
+; CHECK-NEXT: [[EPIL_ITER_NEXT]] = add i32 [[EPIL_ITER]], 1
+; CHECK-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_NEXT]], [[XTRAITER]]
+; CHECK-NEXT: br i1 [[EPIL_ITER_CMP]], label %[[HEADER_EPIL]], label %[[EXIT_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP1:![0-9]+]]
+; CHECK: [[EXIT_EPILOG_LCSSA]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %header
+
+header:
+ %iv = phi i32 [ 0, %entry ], [ %iv.next, %latch ]
+ %acc = phi i32 [ 0, %entry ], [ %acc.next, %latch ]
+ %sidecond = icmp sgt i32 %bound, %n
+ br i1 %sidecond, label %sideexit, label %body, !prof !0
+
+body:
+ %gep = getelementptr i32, ptr addrspace(1) %p, i32 %iv
+ %ld = load i32, ptr addrspace(1) %gep
+ %acc.next = add i32 %acc, %ld
+ br label %latch
+
+latch:
+ %iv.next = add nuw nsw i32 %iv, 1
+ %exitcond = icmp eq i32 %iv.next, %n
+ br i1 %exitcond, label %exit, label %header
+
+sideexit:
+ store i32 %acc, ptr addrspace(1) %p
+ ret void
+
+exit:
+ ret void
+}
+
+!0 = !{!"branch_weights", i32 1, i32 1000000}
+;.
+; CHECK: [[PROF0]] = !{!"branch_weights", i32 1, i32 1000000}
+; CHECK: [[LOOP1]] = distinct !{[[LOOP1]], [[META2:![0-9]+]]}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.disable"}
+;.
More information about the llvm-commits
mailing list