[llvm] [LoopInterchange] Always create a new latch when interchanging (PR #194574)
Ryotaro Kasuga via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 28 02:06:15 PDT 2026
https://github.com/kasuga-fj created https://github.com/llvm/llvm-project/pull/194574
In the transformation phase of loop-interchange, when the inner loop to be interchanged is the innermost one, it creates a new latch BB and move several instructions from the original BB to the new one to generate the valid IR. I don't know why this process was performed only when the target loop is innermost one, but as exposed in #163954, this process is necessary when interchanging non-innermost loops as well, e.g., when a reduction exists and the reduction operation is in the latch BB of the inner loop.
This patch removes the conditional branch and make transformation always create a new latch.
Fix #163954
>From 1e3351964a63a422d9e0f9c8055a8c60a81eaa5e Mon Sep 17 00:00:00 2001
From: Ryotaro Kasuga <kasuga.ryotaro at fujitsu.com>
Date: Tue, 28 Apr 2026 08:37:19 +0000
Subject: [PATCH] [LoopInterchange] Always create a new latch when
interchanging
---
.../lib/Transforms/Scalar/LoopInterchange.cpp | 130 +++++++++---------
.../interchanged-loop-nest-3.ll | 16 ++-
.../Transforms/LoopInterchange/pr57148.ll | 118 ++++++++--------
.../reduction-not-involve-innermost.ll | 81 +++++++++++
4 files changed, 212 insertions(+), 133 deletions(-)
create mode 100644 llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index 49a9e77ef8deb..7dac4f19dbad3 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -1934,75 +1934,72 @@ bool LoopInterchangeTransform::transform(
if (InnerReductions.size() == 1)
reduction2Memory();
- if (InnerLoop->getSubLoops().empty()) {
- BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
- LLVM_DEBUG(dbgs() << "Splitting the inner loop latch\n");
- auto &InductionPHIs = LIL.getInnerLoopInductions();
- if (InductionPHIs.empty()) {
- LLVM_DEBUG(dbgs() << "Failed to find the point to split loop latch \n");
- return false;
- }
-
- SmallVector<Instruction *, 8> InnerIndexVarList;
- for (PHINode *CurInductionPHI : InductionPHIs) {
- if (CurInductionPHI->getIncomingBlock(0) == InnerLoopPreHeader)
- InnerIndexVarList.push_back(
- dyn_cast<Instruction>(CurInductionPHI->getIncomingValue(1)));
- else
- InnerIndexVarList.push_back(
- dyn_cast<Instruction>(CurInductionPHI->getIncomingValue(0)));
- }
+ BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
+ LLVM_DEBUG(dbgs() << "Splitting the inner loop latch\n");
+ auto &InductionPHIs = LIL.getInnerLoopInductions();
+ if (InductionPHIs.empty()) {
+ LLVM_DEBUG(dbgs() << "Failed to find the point to split loop latch \n");
+ return false;
+ }
- // Create a new latch block for the inner loop. We split at the
- // current latch's terminator and then move the condition and all
- // operands that are not either loop-invariant or the induction PHI into the
- // new latch block.
- BasicBlock *NewLatch =
- SplitBlock(InnerLoop->getLoopLatch(),
- InnerLoop->getLoopLatch()->getTerminator(), DT, LI);
-
- SmallSetVector<Instruction *, 4> WorkList;
- unsigned i = 0;
- auto MoveInstructions = [&i, &WorkList, this, &InductionPHIs, NewLatch]() {
- for (; i < WorkList.size(); i++) {
- // Duplicate instruction and move it the new latch. Update uses that
- // have been moved.
- Instruction *NewI = WorkList[i]->clone();
- NewI->insertBefore(NewLatch->getFirstNonPHIIt());
- assert(!NewI->mayHaveSideEffects() &&
- "Moving instructions with side-effects may change behavior of "
- "the loop nest!");
- for (Use &U : llvm::make_early_inc_range(WorkList[i]->uses())) {
- Instruction *UserI = cast<Instruction>(U.getUser());
- if (!InnerLoop->contains(UserI->getParent()) ||
- UserI->getParent() == NewLatch ||
- llvm::is_contained(InductionPHIs, UserI))
- U.set(NewI);
- }
- // Add operands of moved instruction to the worklist, except if they are
- // outside the inner loop or are the induction PHI.
- for (Value *Op : WorkList[i]->operands()) {
- Instruction *OpI = dyn_cast<Instruction>(Op);
- if (!OpI ||
- this->LI->getLoopFor(OpI->getParent()) != this->InnerLoop ||
- llvm::is_contained(InductionPHIs, OpI))
- continue;
- WorkList.insert(OpI);
- }
+ SmallVector<Instruction *, 8> InnerIndexVarList;
+ for (PHINode *CurInductionPHI : InductionPHIs) {
+ if (CurInductionPHI->getIncomingBlock(0) == InnerLoopPreHeader)
+ InnerIndexVarList.push_back(
+ dyn_cast<Instruction>(CurInductionPHI->getIncomingValue(1)));
+ else
+ InnerIndexVarList.push_back(
+ dyn_cast<Instruction>(CurInductionPHI->getIncomingValue(0)));
+ }
+
+ // Create a new latch block for the inner loop. We split at the
+ // current latch's terminator and then move the condition and all
+ // operands that are not either loop-invariant or the induction PHI into the
+ // new latch block.
+ BasicBlock *NewLatch =
+ SplitBlock(InnerLoop->getLoopLatch(),
+ InnerLoop->getLoopLatch()->getTerminator(), DT, LI);
+
+ SmallSetVector<Instruction *, 4> WorkList;
+ unsigned i = 0;
+ auto MoveInstructions = [&i, &WorkList, this, &InductionPHIs, NewLatch]() {
+ for (; i < WorkList.size(); i++) {
+ // Duplicate instruction and move it the new latch. Update uses that
+ // have been moved.
+ Instruction *NewI = WorkList[i]->clone();
+ NewI->insertBefore(NewLatch->getFirstNonPHIIt());
+ assert(!NewI->mayHaveSideEffects() &&
+ "Moving instructions with side-effects may change behavior of "
+ "the loop nest!");
+ for (Use &U : llvm::make_early_inc_range(WorkList[i]->uses())) {
+ Instruction *UserI = cast<Instruction>(U.getUser());
+ if (!InnerLoop->contains(UserI->getParent()) ||
+ UserI->getParent() == NewLatch ||
+ llvm::is_contained(InductionPHIs, UserI))
+ U.set(NewI);
}
- };
+ // Add operands of moved instruction to the worklist, except if they are
+ // outside the inner loop or are the induction PHI.
+ for (Value *Op : WorkList[i]->operands()) {
+ Instruction *OpI = dyn_cast<Instruction>(Op);
+ if (!OpI || this->LI->getLoopFor(OpI->getParent()) != this->InnerLoop ||
+ llvm::is_contained(InductionPHIs, OpI))
+ continue;
+ WorkList.insert(OpI);
+ }
+ }
+ };
- // FIXME: Should we interchange when we have a constant condition?
- Instruction *CondI = dyn_cast<Instruction>(
- cast<CondBrInst>(InnerLoop->getLoopLatch()->getTerminator())
- ->getCondition());
- if (CondI)
- WorkList.insert(CondI);
- MoveInstructions();
- for (Instruction *InnerIndexVar : InnerIndexVarList)
- WorkList.insert(cast<Instruction>(InnerIndexVar));
- MoveInstructions();
- }
+ // FIXME: Should we interchange when we have a constant condition?
+ Instruction *CondI = dyn_cast<Instruction>(
+ cast<CondBrInst>(InnerLoop->getLoopLatch()->getTerminator())
+ ->getCondition());
+ if (CondI)
+ WorkList.insert(CondI);
+ MoveInstructions();
+ for (Instruction *InnerIndexVar : InnerIndexVarList)
+ WorkList.insert(cast<Instruction>(InnerIndexVar));
+ MoveInstructions();
// Ensure the inner loop phi nodes have a separate basic block.
BasicBlock *InnerLoopHeader = InnerLoop->getHeader();
@@ -2017,7 +2014,6 @@ bool LoopInterchangeTransform::transform(
// inner loop preheader will become the entry into the interchanged loop nest.
// Currently we move all instructions and rely on LICM to move invariant
// instructions outside the loop nest.
- BasicBlock *InnerLoopPreHeader = InnerLoop->getLoopPreheader();
BasicBlock *OuterLoopHeader = OuterLoop->getHeader();
if (InnerLoopPreHeader != OuterLoopHeader) {
for (Instruction &I :
diff --git a/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll b/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll
index 6be86f1a8fdcf..3cf3b02e8fd21 100644
--- a/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll
+++ b/llvm/test/Transforms/LoopInterchange/interchanged-loop-nest-3.ll
@@ -22,7 +22,7 @@ define void @interchange_08(i32 %t){
; CHECK: [[FOR_COND1_PREHEADER_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_COND1_PREHEADER:.*]]
; CHECK: [[FOR_COND1_PREHEADER]]:
-; CHECK-NEXT: [[I_028:%.*]] = phi i64 [ [[INC16:%.*]], %[[FOR_INC15:.*]] ], [ 0, %[[FOR_COND1_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: [[I_028:%.*]] = phi i64 [ [[INC16:%.*]], %[[FOR_INC16:.*]] ], [ 0, %[[FOR_COND1_PREHEADER_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_BODY6_SPLIT1:.*]]
; CHECK: [[FOR_COND4_PREHEADER_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_COND4_PREHEADER:.*]]
@@ -32,7 +32,7 @@ define void @interchange_08(i32 %t){
; CHECK: [[FOR_BODY6_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_BODY6:.*]]
; CHECK: [[FOR_BODY6]]:
-; CHECK-NEXT: [[K_026:%.*]] = phi i64 [ [[TMP1:%.*]], %[[FOR_BODY6_SPLIT:.*]] ], [ 0, %[[FOR_BODY6_PREHEADER]] ]
+; CHECK-NEXT: [[K_026:%.*]] = phi i64 [ [[TMP5:%.*]], %[[FOR_BODY6_SPLIT:.*]] ], [ 0, %[[FOR_BODY6_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_COND4_PREHEADER_PREHEADER]]
; CHECK: [[FOR_BODY6_SPLIT1]]:
; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds [100 x [100 x [100 x i32]]], ptr @D, i64 0, i64 [[K_026]], i64 [[J_027]], i64 [[I_028]]
@@ -42,10 +42,14 @@ define void @interchange_08(i32 %t){
; CHECK-NEXT: [[INC:%.*]] = add nuw nsw i64 [[K_026]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INC]], 100
; CHECK-NEXT: br label %[[FOR_INC12:.*]]
-; CHECK: [[FOR_BODY6_SPLIT]]:
-; CHECK-NEXT: [[TMP1]] = add nuw nsw i64 [[K_026]], 1
+; CHECK: [[FOR_INC15:.*]]:
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[K_026]], 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 100
-; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_END17:.*]], label %[[FOR_BODY6]]
+; CHECK-NEXT: br label %[[FOR_INC16]]
+; CHECK: [[FOR_BODY6_SPLIT]]:
+; CHECK-NEXT: [[TMP5]] = add nuw nsw i64 [[K_026]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 100
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_END17:.*]], label %[[FOR_BODY6]]
; CHECK: [[FOR_INC12]]:
; CHECK-NEXT: [[INC13:%.*]] = add nuw nsw i64 [[J_027]], 1
; CHECK-NEXT: [[EXITCOND29:%.*]] = icmp eq i64 [[INC13]], 100
@@ -54,7 +58,7 @@ define void @interchange_08(i32 %t){
; CHECK-NEXT: [[TMP3]] = add nuw nsw i64 [[J_027]], 1
; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[TMP3]], 100
; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY6_SPLIT]], label %[[FOR_COND4_PREHEADER]]
-; CHECK: [[FOR_INC15]]:
+; CHECK: [[FOR_INC16]]:
; CHECK-NEXT: [[INC16]] = add nuw nsw i64 [[I_028]], 1
; CHECK-NEXT: [[EXITCOND30:%.*]] = icmp eq i64 [[INC16]], 100
; CHECK-NEXT: br i1 [[EXITCOND30]], label %[[FOR_INC12_SPLIT]], label %[[FOR_COND1_PREHEADER]]
diff --git a/llvm/test/Transforms/LoopInterchange/pr57148.ll b/llvm/test/Transforms/LoopInterchange/pr57148.ll
index 809c41c1713c2..fcafde2bb2688 100644
--- a/llvm/test/Transforms/LoopInterchange/pr57148.ll
+++ b/llvm/test/Transforms/LoopInterchange/pr57148.ll
@@ -12,46 +12,41 @@ target triple = "x86_64-unknown-linux-gnu"
define void @test1() {
; CHECK-LABEL: define void @test1() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER_PREHEADER:.*]]
-; CHECK: [[FOR_COND33_PREHEADER_PREHEADER:.*]]:
-; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER:.*]]
-; CHECK: [[FOR_COND33_PREHEADER]]:
-; CHECK-NEXT: [[I_011:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: [[FOR_COND33_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY42_SPLIT1:.*]]
-; CHECK: [[FOR_BODY42_PREHEADER:.*]]:
+; CHECK: [[FOR_COND33_PREHEADER_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY42:.*]]
-; CHECK: [[FOR_COND37_PREHEADER_PREHEADER]]:
-; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]]
-; CHECK: [[FOR_COND37_PREHEADER]]:
-; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[INC66:%.*]], %[[FOR_END64:.*]] ], [ 0, %[[FOR_COND37_PREHEADER_PREHEADER]] ]
-; CHECK-NEXT: br label %[[FOR_BODY42_PREHEADER]]
; CHECK: [[FOR_BODY42]]:
-; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ [[TMP1:%.*]], %[[FOR_BODY42_SPLIT:.*]] ], [ -512, %[[FOR_BODY42_PREHEADER]] ]
-; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER_PREHEADER]]
+; CHECK-NEXT: [[I_011:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_END68:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]]
; CHECK: [[FOR_BODY42_SPLIT1]]:
+; CHECK-NEXT: br label %[[FOR_BODY42_PREHEADER:.*]]
+; CHECK: [[FOR_BODY42_PREHEADER]]:
+; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[TMP3:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_BODY42_SPLIT1]] ]
+; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER_PREHEADER]]
+; CHECK: [[FOR_COND37_PREHEADER]]:
+; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_BODY42]] ], [ [[TMP1:%.*]], %[[FOR_COND37_PREHEADER]] ]
; CHECK-NEXT: [[SUB51:%.*]] = add nsw i16 [[K_09]], 512
; CHECK-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds [512 x [4 x i32]], ptr @b, i16 0, i16 [[SUB51]], i16 [[J_010]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX55]], align 1
; CHECK-NEXT: [[ADD61:%.*]] = add i32 undef, undef
-; CHECK-NEXT: [[INC63:%.*]] = add nsw i16 [[K_09]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[K_09]], 42
-; CHECK-NEXT: br label %[[FOR_END67]]
-; CHECK: [[FOR_BODY42_SPLIT]]:
-; CHECK-NEXT: [[ADD61_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_END67]] ]
; CHECK-NEXT: [[TMP1]] = add nsw i16 [[K_09]], 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i16 [[K_09]], 42
-; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_BODY42]], label %[[FOR_END64]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_END64:.*]]
; CHECK: [[FOR_END64]]:
-; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61_LCSSA]], %[[FOR_BODY42_SPLIT]] ]
+; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_COND37_PREHEADER]] ]
; CHECK-NEXT: store i32 [[ADD61_LCSSA_LCSSA]], ptr undef, align 1
-; CHECK-NEXT: [[INC66]] = add nuw nsw i16 [[J_010]], 1
+; CHECK-NEXT: [[INC66:%.*]] = add nuw nsw i16 [[J_010]], 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i16 [[J_010]], 43
-; CHECK-NEXT: br i1 [[CMP2]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
+; CHECK-NEXT: br label %[[FOR_END68]]
; CHECK: [[FOR_END67]]:
+; CHECK-NEXT: [[TMP3]] = add nuw nsw i16 [[J_010]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i16 [[J_010]], 43
+; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_BODY42_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
+; CHECK: [[FOR_END68]]:
; CHECK-NEXT: [[INC69]] = add nuw nsw i16 [[I_011]], 1
; CHECK-NEXT: [[EXITCOND13_NOT:%.*]] = icmp eq i16 [[INC69]], 2
-; CHECK-NEXT: br i1 [[EXITCOND13_NOT]], label %[[FOR_BODY42_SPLIT]], label %[[FOR_COND33_PREHEADER]]
+; CHECK-NEXT: br i1 [[EXITCOND13_NOT]], label %[[FOR_END67]], label %[[FOR_BODY42]]
; CHECK: [[FOR_COND75_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_COND75:.*]]
; CHECK: [[FOR_COND75]]:
@@ -106,13 +101,13 @@ define void @test2() {
; CHECK: [[FOR_COND33_PREHEADER_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER1:.*]]
; CHECK: [[FOR_COND33_PREHEADER1]]:
-; CHECK-NEXT: [[I_166:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_INC68:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: [[I_166:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_INC69:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
; CHECK-NEXT: [[ARRAYIDX60:%.*]] = getelementptr inbounds [2 x [4 x i32]], ptr @c, i16 0, i16 [[I_166]], i16 [[J_165:%.*]]
; CHECK-NEXT: br label %[[VECTOR_BODY85_SPLIT1:.*]]
; CHECK: [[FOR_COND33_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]]
; CHECK: [[FOR_COND37_PREHEADER]]:
-; CHECK-NEXT: [[J_165]] = phi i16 [ [[INC66:%.*]], %[[MIDDLE_BLOCK80:.*]] ], [ 0, %[[FOR_COND33_PREHEADER]] ]
+; CHECK-NEXT: [[J_165]] = phi i16 [ [[TMP7:%.*]], %[[MIDDLE_BLOCK80:.*]] ], [ 0, %[[FOR_COND33_PREHEADER]] ]
; CHECK-NEXT: br label %[[VECTOR_BODY85:.*]]
; CHECK: [[VECTOR_BODY85]]:
; CHECK-NEXT: br label %[[VECTOR_BODY86:.*]]
@@ -125,16 +120,20 @@ define void @test2() {
; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 1
; CHECK-NEXT: [[TMP3:%.*]] = add nuw i16 [[INDEX86]], 4
; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i16 [[INDEX86]], 42
-; CHECK-NEXT: br label %[[FOR_INC68]]
+; CHECK-NEXT: br label %[[FOR_INC68:.*]]
; CHECK: [[VECTOR_BODY85_SPLIT]]:
; CHECK-NEXT: [[TMP5]] = add nuw i16 [[INDEX86]], 4
; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i16 [[INDEX86]], 42
; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_BODY86]], label %[[MIDDLE_BLOCK80]]
-; CHECK: [[MIDDLE_BLOCK80]]:
-; CHECK-NEXT: [[INC66]] = add nuw nsw i16 [[J_165]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[INC66]], 42
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
; CHECK: [[FOR_INC68]]:
+; CHECK-NEXT: [[INC66:%.*]] = add nuw nsw i16 [[J_165]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[INC66]], 42
+; CHECK-NEXT: br label %[[FOR_INC69]]
+; CHECK: [[MIDDLE_BLOCK80]]:
+; CHECK-NEXT: [[TMP7]] = add nuw nsw i16 [[J_165]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i16 [[TMP7]], 42
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
+; CHECK: [[FOR_INC69]]:
; CHECK-NEXT: [[INC69]] = add nuw nsw i16 [[I_166]], 1
; CHECK-NEXT: [[EXITCOND77_NOT:%.*]] = icmp slt i16 [[INC69]], 24
; CHECK-NEXT: br i1 [[EXITCOND77_NOT]], label %[[FOR_COND33_PREHEADER1]], label %[[VECTOR_BODY85_SPLIT]]
@@ -179,55 +178,54 @@ for.cond75.preheader: ; preds = %for.inc68
; Same as test1, but with a third index on the GEP
define void @test3() {
; CHECK-LABEL: define void @test3() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER_PREHEADER:.*]]
-; CHECK: [[FOR_COND33_PREHEADER_PREHEADER:.*]]:
-; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER:.*]]
-; CHECK: [[FOR_COND33_PREHEADER]]:
-; CHECK-NEXT: [[I_011:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
+; CHECK-NEXT: [[FOR_COND33_PREHEADER:.*:]]
; CHECK-NEXT: br label %[[FOR_BODY42_SPLIT1:.*]]
-; CHECK: [[FOR_BODY42_PREHEADER:.*]]:
+; CHECK: [[FOR_COND33_PREHEADER_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_BODY42:.*]]
-; CHECK: [[FOR_COND38_PREHEADER_PREHEADER:.*]]:
+; CHECK: [[FOR_BODY42]]:
+; CHECK-NEXT: [[I_011:%.*]] = phi i16 [ [[INC69:%.*]], %[[FOR_END68:.*]] ], [ 0, %[[FOR_COND33_PREHEADER_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_COND38_PREHEADER:.*]]
-; CHECK: [[FOR_COND37_PREHEADER_PREHEADER]]:
+; CHECK: [[FOR_BODY42_PREHEADER:.*]]:
; CHECK-NEXT: br label %[[FOR_COND37_PREHEADER:.*]]
-; CHECK: [[FOR_COND37_PREHEADER]]:
-; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[INC66:%.*]], %[[FOR_END64:.*]] ], [ 0, %[[FOR_COND37_PREHEADER_PREHEADER]] ]
-; CHECK-NEXT: br label %[[FOR_COND38_PREHEADER_PREHEADER]]
-; CHECK: [[FOR_COND38_PREHEADER]]:
-; CHECK-NEXT: [[K_010:%.*]] = phi i16 [ [[INC67:%.*]], %[[FOR_END65:.*]] ], [ 0, %[[FOR_COND38_PREHEADER_PREHEADER]] ]
+; CHECK: [[FOR_BODY42_SPLIT1]]:
+; CHECK-NEXT: br label %[[FOR_COND38_PREHEADER_PREHEADER:.*]]
+; CHECK: [[FOR_COND38_PREHEADER_PREHEADER]]:
+; CHECK-NEXT: [[J_010:%.*]] = phi i16 [ [[TMP5:%.*]], %[[FOR_END67:.*]] ], [ 0, %[[FOR_BODY42_SPLIT1]] ]
; CHECK-NEXT: br label %[[FOR_BODY42_PREHEADER]]
-; CHECK: [[FOR_BODY42]]:
-; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ [[TMP3:%.*]], %[[FOR_BODY42_SPLIT:.*]] ], [ -512, %[[FOR_BODY42_PREHEADER]] ]
+; CHECK: [[FOR_COND37_PREHEADER]]:
+; CHECK-NEXT: [[K_010:%.*]] = phi i16 [ [[TMP1:%.*]], %[[FOR_END64:.*]] ], [ 0, %[[FOR_BODY42_PREHEADER]] ]
; CHECK-NEXT: br label %[[FOR_COND33_PREHEADER_PREHEADER]]
-; CHECK: [[FOR_BODY42_SPLIT1]]:
+; CHECK: [[FOR_COND38_PREHEADER]]:
+; CHECK-NEXT: [[K_09:%.*]] = phi i16 [ -512, %[[FOR_BODY42]] ], [ [[TMP3:%.*]], %[[FOR_COND38_PREHEADER]] ]
; CHECK-NEXT: [[SUB51:%.*]] = add nsw i16 [[K_09]], 512
; CHECK-NEXT: [[ARRAYIDX55:%.*]] = getelementptr inbounds [1024 x [512 x [4 x i32]]], ptr @d, i16 0, i16 [[SUB51]], i16 [[J_010]], i16 [[K_010]]
; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[ARRAYIDX55]], align 1
; CHECK-NEXT: [[ADD61:%.*]] = add i32 undef, undef
-; CHECK-NEXT: [[TMP1:%.*]] = add nsw i16 [[K_09]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i16 [[K_09]], 42
-; CHECK-NEXT: br label %[[FOR_END67]]
-; CHECK: [[FOR_BODY42_SPLIT]]:
-; CHECK-NEXT: [[ADD61_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_END67]] ]
; CHECK-NEXT: [[TMP3]] = add nsw i16 [[K_09]], 1
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt i16 [[K_09]], 42
-; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_BODY42]], label %[[FOR_END65]]
+; CHECK-NEXT: br i1 [[TMP2]], label %[[FOR_COND38_PREHEADER]], label %[[FOR_END65:.*]]
; CHECK: [[FOR_END65]]:
-; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61_LCSSA]], %[[FOR_BODY42_SPLIT]] ]
+; CHECK-NEXT: [[ADD61_LCSSA_LCSSA:%.*]] = phi i32 [ [[ADD61]], %[[FOR_COND38_PREHEADER]] ]
; CHECK-NEXT: store i32 [[ADD61_LCSSA_LCSSA]], ptr undef, align 1
-; CHECK-NEXT: [[INC67]] = add nuw nsw i16 [[K_010]], 1
+; CHECK-NEXT: [[INC67:%.*]] = add nuw nsw i16 [[K_010]], 1
; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i16 [[K_010]], 44
-; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_COND38_PREHEADER]], label %[[FOR_END64]]
+; CHECK-NEXT: br label %[[FOR_END66:.*]]
; CHECK: [[FOR_END64]]:
-; CHECK-NEXT: [[INC66]] = add nuw nsw i16 [[J_010]], 1
+; CHECK-NEXT: [[TMP1]] = add nuw nsw i16 [[K_010]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp slt i16 [[K_010]], 44
+; CHECK-NEXT: br i1 [[TMP6]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_END67]]
+; CHECK: [[FOR_END66]]:
+; CHECK-NEXT: [[INC66:%.*]] = add nuw nsw i16 [[J_010]], 1
; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i16 [[J_010]], 43
-; CHECK-NEXT: br i1 [[CMP2]], label %[[FOR_COND37_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
+; CHECK-NEXT: br label %[[FOR_END68]]
; CHECK: [[FOR_END67]]:
+; CHECK-NEXT: [[TMP5]] = add nuw nsw i16 [[J_010]], 1
+; CHECK-NEXT: [[TMP4:%.*]] = icmp slt i16 [[J_010]], 43
+; CHECK-NEXT: br i1 [[TMP4]], label %[[FOR_COND38_PREHEADER_PREHEADER]], label %[[FOR_COND75_PREHEADER:.*]]
+; CHECK: [[FOR_END68]]:
; CHECK-NEXT: [[INC69]] = add nuw nsw i16 [[I_011]], 1
; CHECK-NEXT: [[EXITCOND13_NOT:%.*]] = icmp eq i16 [[INC69]], 2
-; CHECK-NEXT: br i1 [[EXITCOND13_NOT]], label %[[FOR_BODY42_SPLIT]], label %[[FOR_COND33_PREHEADER]]
+; CHECK-NEXT: br i1 [[EXITCOND13_NOT]], label %[[FOR_END64]], label %[[FOR_BODY42]]
; CHECK: [[FOR_COND75_PREHEADER]]:
; CHECK-NEXT: br label %[[FOR_COND75:.*]]
; CHECK: [[FOR_COND75]]:
diff --git a/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll b/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll
new file mode 100644
index 0000000000000..094e73db54ff3
--- /dev/null
+++ b/llvm/test/Transforms/LoopInterchange/reduction-not-involve-innermost.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -S -passes=loop-interchange \
+; RUN: -loop-interchange-profitabilities=ignore | FileCheck %s
+
+; This is a 3-level nested loop (i, j, k) with a reduction variable. The
+; reduction variable involves the outer loop (i) and the middle loop (j), but
+; not the innermost loop (k). Previously, the loop interchange pass crashed
+; when trying to interchange the i-loop and the j-loop. This test ensures that
+; the loop interchange pass can handle such a case.
+
+define void @f() {
+; CHECK-LABEL: define void @f() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: br label %[[LOOP_J_HEADER_PREHEADER:.*]]
+; CHECK: [[LOOP_I_HEADER_PREHEADER:.*]]:
+; CHECK-NEXT: br label %[[LOOP_I_HEADER:.*]]
+; CHECK: [[LOOP_I_HEADER]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_INC:%.*]], %[[LOOP_I_LATCH:.*]] ], [ 0, %[[LOOP_I_HEADER_PREHEADER]] ]
+; CHECK-NEXT: [[RED_J:%.*]] = phi i8 [ [[RED_J_NEXT:%.*]], %[[LOOP_I_LATCH]] ], [ [[RED_I:%.*]], %[[LOOP_I_HEADER_PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_K:.*]]
+; CHECK: [[LOOP_J_HEADER_PREHEADER]]:
+; CHECK-NEXT: br label %[[LOOP_J_HEADER:.*]]
+; CHECK: [[LOOP_J_HEADER]]:
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[TMP0:%.*]], %[[LOOP_J_LATCH_SPLIT:.*]] ], [ 0, %[[LOOP_J_HEADER_PREHEADER]] ]
+; CHECK-NEXT: [[RED_I]] = phi i8 [ [[RED_J_NEXT_LCSSA:%.*]], %[[LOOP_J_LATCH_SPLIT]] ], [ 0, %[[LOOP_J_HEADER_PREHEADER]] ]
+; CHECK-NEXT: br label %[[LOOP_I_HEADER_PREHEADER]]
+; CHECK: [[LOOP_K]]:
+; CHECK-NEXT: [[K:%.*]] = phi i64 [ 0, %[[LOOP_I_HEADER]] ], [ [[K_INC:%.*]], %[[LOOP_K]] ]
+; CHECK-NEXT: [[K_INC]] = add i64 [[K]], 1
+; CHECK-NEXT: [[EC_K:%.*]] = icmp eq i64 [[K_INC]], 10
+; CHECK-NEXT: br i1 [[EC_K]], label %[[LOOP_J_LATCH:.*]], label %[[LOOP_K]]
+; CHECK: [[LOOP_J_LATCH]]:
+; CHECK-NEXT: [[RED_J_NEXT]] = or i8 [[RED_J]], 42
+; CHECK-NEXT: [[J_INC:%.*]] = add i64 [[J]], 1
+; CHECK-NEXT: [[EC_J:%.*]] = icmp eq i64 [[J_INC]], 20
+; CHECK-NEXT: br label %[[LOOP_I_LATCH]]
+; CHECK: [[LOOP_J_LATCH_SPLIT]]:
+; CHECK-NEXT: [[RED_J_NEXT_LCSSA]] = phi i8 [ [[RED_J_NEXT]], %[[LOOP_I_LATCH]] ]
+; CHECK-NEXT: [[TMP0]] = add i64 [[J]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[TMP0]], 20
+; CHECK-NEXT: br i1 [[TMP1]], label %[[EXIT:.*]], label %[[LOOP_J_HEADER]]
+; CHECK: [[LOOP_I_LATCH]]:
+; CHECK-NEXT: [[I_INC]] = add i64 [[I]], 1
+; CHECK-NEXT: [[EC_I:%.*]] = icmp eq i64 [[I_INC]], 30
+; CHECK-NEXT: br i1 [[EC_I]], label %[[LOOP_J_LATCH_SPLIT]], label %[[LOOP_I_HEADER]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop.i.header
+
+loop.i.header:
+ %i = phi i64 [ 0, %entry ], [ %i.inc, %loop.i.latch ]
+ %red.i = phi i8 [ 0, %entry ], [ %red.j.next, %loop.i.latch ]
+ br label %loop.j.header
+
+loop.j.header:
+ %j = phi i64 [ 0, %loop.i.header ], [ %j.inc, %loop.j.latch ]
+ %red.j = phi i8 [ %red.i, %loop.i.header ], [ %red.j.next, %loop.j.latch ]
+ br label %loop.k
+
+loop.k:
+ %k = phi i64 [ 0, %loop.j.header ], [ %k.inc, %loop.k ]
+ %k.inc = add i64 %k, 1
+ %ec.k = icmp eq i64 %k.inc, 10
+ br i1 %ec.k, label %loop.j.latch, label %loop.k
+
+loop.j.latch:
+ %red.j.next = or i8 %red.j, 42
+ %j.inc = add i64 %j, 1
+ %ec.j = icmp eq i64 %j.inc, 20
+ br i1 %ec.j, label %loop.i.latch, label %loop.j.header
+
+loop.i.latch:
+ %i.inc = add i64 %i, 1
+ %ec.i = icmp eq i64 %i.inc, 30
+ br i1 %ec.i, label %exit, label %loop.i.header
+
+exit:
+ ret void
+}
More information about the llvm-commits
mailing list