[llvm] [LoopFusion] Fix sink instructions (PR #147501)
Madhur Amilkanthwar via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 21 21:46:00 PDT 2025
https://github.com/madhur13490 updated https://github.com/llvm/llvm-project/pull/147501
>From 997b6ea88d21612de4c85ce08b4e86d6a9a6327e Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Tue, 8 Jul 2025 01:48:33 -0700
Subject: [PATCH 1/4] [LoopFusion] Fix sink instructions
If we have instructions in second loop's preheader
which can be sunk, we should also be adjusting
PHI nodes to receive values from the new loop's latch block.
Fixes #128600
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 34 ++++++--
.../Transforms/LoopFusion/sunk-phi-nodes.ll | 86 +++++++++++++++++++
2 files changed, 115 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index d6bd92d520e28..6e1556a4d90b4 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -988,8 +988,8 @@ struct LoopFuser {
// If it is not safe to hoist/sink all instructions in the
// pre-header, we cannot fuse these loops.
- if (!collectMovablePreheaderInsts(*FC0, *FC1, SafeToHoist,
- SafeToSink)) {
+ if (!collectAndFixMovablePreheaderInsts(*FC0, *FC1, SafeToHoist,
+ SafeToSink)) {
LLVM_DEBUG(dbgs() << "Could not hoist/sink all instructions in "
"Fusion Candidate Pre-header.\n"
<< "Not Fusing.\n");
@@ -1033,8 +1033,8 @@ struct LoopFuser {
FuseCounter);
FusionCandidate FusedCand(
- performFusion((Peel ? FC0Copy : *FC0), *FC1), DT, &PDT, ORE,
- FC0Copy.PP);
+ performFusion((Peel ? FC0Copy : *FC0), *FC1, SafeToSink), DT,
+ &PDT, ORE, FC0Copy.PP);
FusedCand.verify();
assert(FusedCand.isEligibleForFusion(SE) &&
"Fused candidate should be eligible for fusion!");
@@ -1176,9 +1176,31 @@ struct LoopFuser {
return true;
}
+ void fixPHINodes(SmallVector<Instruction *, 4> &SafeToSink,
+ const FusionCandidate &FC0,
+ const FusionCandidate &FC1) const {
+ // Iterate over SafeToSink instructions and update PHI nodes
+ // to take values from the latch block of FC0 if they are taking
+ // from the latch block of FC1.
+ for (Instruction *Inst : SafeToSink) {
+ LLVM_DEBUG(dbgs() << "UPDATING: Instruction: " << *Inst << "\n");
+ // Continue if the instruction is not a PHI node.
+ if (!isa<PHINode>(Inst))
+ continue;
+ PHINode *Phi = dyn_cast<PHINode>(Inst);
+ LLVM_DEBUG(dbgs() << "UPDATING: PHI node: " << *Phi << "\n");
+ for (unsigned I = 0; I < Phi->getNumIncomingValues(); I++) {
+ if (Phi->getIncomingBlock(I) != FC0.Latch)
+ continue;
+ assert(FC1.Latch && "FC1 latch is not set");
+ Phi->setIncomingBlock(I, FC1.Latch);
+ }
+ }
+ }
+
/// Collect instructions in the \p FC1 Preheader that can be hoisted
/// to the \p FC0 Preheader or sunk into the \p FC1 Body
- bool collectMovablePreheaderInsts(
+ bool collectAndFixMovablePreheaderInsts(
const FusionCandidate &FC0, const FusionCandidate &FC1,
SmallVector<Instruction *, 4> &SafeToHoist,
SmallVector<Instruction *, 4> &SafeToSink) const {
@@ -1226,6 +1248,8 @@ struct LoopFuser {
}
LLVM_DEBUG(
dbgs() << "All preheader instructions could be sunk or hoisted!\n");
+
+ fixPHINodes(SafeToSink, FC0, FC1);
return true;
}
diff --git a/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
new file mode 100644
index 0000000000000..3c72df8ae19fb
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=mem2reg,loop-rotate,loop-fusion < %s 2>&1 | FileCheck %s
+define i32 @main() {
+; CHECK-LABEL: define i32 @main() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; CHECK: [[FOR_BODY]]:
+; CHECK-NEXT: [[SUM1_02:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_INC6:.*]] ]
+; CHECK-NEXT: [[I_01:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[I1_04:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC7:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[SUM2_03:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD5:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM1_02]], [[I_01]]
+; CHECK-NEXT: br label %[[FOR_INC:.*]]
+; CHECK: [[FOR_INC]]:
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[I1_04]], [[I1_04]]
+; CHECK-NEXT: [[ADD5]] = add nsw i32 [[SUM2_03]], [[MUL]]
+; CHECK-NEXT: br label %[[FOR_INC6]]
+; CHECK: [[FOR_INC6]]:
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_01]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 10
+; CHECK-NEXT: [[INC7]] = add nsw i32 [[I1_04]], 1
+; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i32 [[INC7]], 10
+; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY]], label %[[FOR_END8:.*]]
+; CHECK: [[FOR_END8]]:
+; CHECK-NEXT: ret i32 0
+;
+entry:
+ %retval = alloca i32, align 4
+ %sum1 = alloca i32, align 4
+ %sum2 = alloca i32, align 4
+ %i = alloca i32, align 4
+ %i1 = alloca i32, align 4
+ store i32 0, ptr %retval, align 4
+ store i32 0, ptr %sum1, align 4
+ store i32 0, ptr %sum2, align 4
+ store i32 0, ptr %i, align 4
+ br label %for.cond
+
+for.cond:
+ %0 = load i32, ptr %i, align 4
+ %cmp = icmp slt i32 %0, 10
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+ %1 = load i32, ptr %i, align 4
+ %2 = load i32, ptr %sum1, align 4
+ %add = add nsw i32 %2, %1
+ store i32 %add, ptr %sum1, align 4
+ br label %for.inc
+
+for.inc:
+ %3 = load i32, ptr %i, align 4
+ %inc = add nsw i32 %3, 1
+ store i32 %inc, ptr %i, align 4
+ br label %for.cond
+
+for.end:
+ store i32 0, ptr %i1, align 4
+ br label %for.cond2
+
+for.cond2:
+ %4 = load i32, ptr %i1, align 4
+ %cmp3 = icmp slt i32 %4, 10
+ br i1 %cmp3, label %for.body4, label %for.end8
+
+for.body4:
+ %5 = load i32, ptr %i1, align 4
+ %6 = load i32, ptr %i1, align 4
+ %mul = mul nsw i32 %5, %6
+ %7 = load i32, ptr %sum2, align 4
+ %add5 = add nsw i32 %7, %mul
+ store i32 %add5, ptr %sum2, align 4
+ br label %for.inc6
+
+for.inc6:
+ %8 = load i32, ptr %i1, align 4
+ %inc7 = add nsw i32 %8, 1
+ store i32 %inc7, ptr %i1, align 4
+ br label %for.cond2
+
+for.end8:
+ %9 = load i32, ptr %sum1, align 4
+ %10 = load i32, ptr %sum2, align 4
+ ret i32 0
+}
+
>From 8fe7a5e2009fdf206d3a948540356b16c1300084 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Fri, 11 Jul 2025 05:18:59 -0700
Subject: [PATCH 2/4] address review comments
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 13 ++--
.../Transforms/LoopFusion/sunk-phi-nodes.ll | 65 ++++++++++++++-----
2 files changed, 55 insertions(+), 23 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 6e1556a4d90b4..e17156660068f 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -988,8 +988,8 @@ struct LoopFuser {
// If it is not safe to hoist/sink all instructions in the
// pre-header, we cannot fuse these loops.
- if (!collectAndFixMovablePreheaderInsts(*FC0, *FC1, SafeToHoist,
- SafeToSink)) {
+ if (!collectMovablePreheaderInsts(*FC0, *FC1, SafeToHoist,
+ SafeToSink)) {
LLVM_DEBUG(dbgs() << "Could not hoist/sink all instructions in "
"Fusion Candidate Pre-header.\n"
<< "Not Fusing.\n");
@@ -1200,7 +1200,7 @@ struct LoopFuser {
/// Collect instructions in the \p FC1 Preheader that can be hoisted
/// to the \p FC0 Preheader or sunk into the \p FC1 Body
- bool collectAndFixMovablePreheaderInsts(
+ bool collectMovablePreheaderInsts(
const FusionCandidate &FC0, const FusionCandidate &FC1,
SmallVector<Instruction *, 4> &SafeToHoist,
SmallVector<Instruction *, 4> &SafeToSink) const {
@@ -1249,7 +1249,6 @@ struct LoopFuser {
LLVM_DEBUG(
dbgs() << "All preheader instructions could be sunk or hoisted!\n");
- fixPHINodes(SafeToSink, FC0, FC1);
return true;
}
@@ -1593,7 +1592,8 @@ struct LoopFuser {
/// two loops could also be fused into a single block. This will require
/// analysis to prove it is safe to move the contents of the block past
/// existing code, which currently has not been implemented.
- Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1) {
+ Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1,
+ SmallVector<Instruction *, 4> &SafeToSink) {
assert(FC0.isValid() && FC1.isValid() &&
"Expecting valid fusion candidates");
@@ -1735,6 +1735,9 @@ struct LoopFuser {
TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
FC1.Latch, FC1.Header));
+ // Fix PHI nodes that are sunk into the body of the loop.
+ fixPHINodes(SafeToSink, FC0, FC1);
+
// Update DT/PDT
DTU.applyUpdates(TreeUpdates);
diff --git a/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
index 3c72df8ae19fb..45b812855e7eb 100644
--- a/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
+++ b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
@@ -1,27 +1,56 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=mem2reg,loop-rotate,loop-fusion < %s 2>&1 | FileCheck %s
-define i32 @main() {
-; CHECK-LABEL: define i32 @main() {
-; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: br label %[[FOR_BODY:.*]]
+; RUN: opt -S -passes=loop-fusion < %s 2>&1 | FileCheck %s
+define i32 @foo() {
+; CHECK-LABEL: define i32 @foo() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[SUM1:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[SUM2:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT: [[I1:%.*]] = alloca i32, align 4
+; CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
+; CHECK-NEXT: store i32 0, ptr [[SUM1]], align 4
+; CHECK-NEXT: store i32 0, ptr [[SUM2]], align 4
+; CHECK-NEXT: store i32 0, ptr [[I]], align 4
+; CHECK-NEXT: br label %[[FOR_COND:.*]]
+; CHECK: [[FOR_COND]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
+; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[SUM1_02:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_INC6:.*]] ]
-; CHECK-NEXT: [[I_01:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_INC6]] ]
-; CHECK-NEXT: [[I1_04:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC7:%.*]], %[[FOR_INC6]] ]
-; CHECK-NEXT: [[SUM2_03:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD5:%.*]], %[[FOR_INC6]] ]
-; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM1_02]], [[I_01]]
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SUM1]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
+; CHECK-NEXT: store i32 [[ADD]], ptr [[SUM1]], align 4
; CHECK-NEXT: br label %[[FOR_INC:.*]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[I1_04]], [[I1_04]]
-; CHECK-NEXT: [[ADD5]] = add nsw i32 [[SUM2_03]], [[MUL]]
-; CHECK-NEXT: br label %[[FOR_INC6]]
+; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I]], align 4
+; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP3]], 1
+; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4
+; CHECK-NEXT: br label %[[FOR_COND]]
+; CHECK: [[FOR_END]]:
+; CHECK-NEXT: store i32 0, ptr [[I1]], align 4
+; CHECK-NEXT: br label %[[FOR_COND2:.*]]
+; CHECK: [[FOR_COND2]]:
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I1]], align 4
+; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i32 [[TMP4]], 10
+; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END8:.*]]
+; CHECK: [[FOR_BODY4]]:
+; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I1]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[I1]], align 4
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[SUM2]], align 4
+; CHECK-NEXT: [[ADD5:%.*]] = add nsw i32 [[TMP7]], [[MUL]]
+; CHECK-NEXT: store i32 [[ADD5]], ptr [[SUM2]], align 4
+; CHECK-NEXT: br label %[[FOR_INC6:.*]]
; CHECK: [[FOR_INC6]]:
-; CHECK-NEXT: [[INC]] = add nsw i32 [[I_01]], 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 10
-; CHECK-NEXT: [[INC7]] = add nsw i32 [[I1_04]], 1
-; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i32 [[INC7]], 10
-; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY]], label %[[FOR_END8:.*]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[I1]], align 4
+; CHECK-NEXT: [[INC7:%.*]] = add nsw i32 [[TMP8]], 1
+; CHECK-NEXT: store i32 [[INC7]], ptr [[I1]], align 4
+; CHECK-NEXT: br label %[[FOR_COND2]]
; CHECK: [[FOR_END8]]:
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[SUM1]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[SUM2]], align 4
; CHECK-NEXT: ret i32 0
;
entry:
>From 35ea738df05f7e7bd79dad5ef0bd80b332281e75 Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Fri, 11 Jul 2025 11:36:23 -0700
Subject: [PATCH 3/4] address review comments
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 4 +-
.../Transforms/LoopFusion/sunk-phi-nodes.ll | 144 ++++++------------
2 files changed, 48 insertions(+), 100 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index e17156660068f..0ad85c5b338b3 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1176,6 +1176,7 @@ struct LoopFuser {
return true;
}
+ // This function fixes sunk PHI nodes after fusion.
void fixPHINodes(SmallVector<Instruction *, 4> &SafeToSink,
const FusionCandidate &FC0,
const FusionCandidate &FC1) const {
@@ -1183,12 +1184,10 @@ struct LoopFuser {
// to take values from the latch block of FC0 if they are taking
// from the latch block of FC1.
for (Instruction *Inst : SafeToSink) {
- LLVM_DEBUG(dbgs() << "UPDATING: Instruction: " << *Inst << "\n");
// Continue if the instruction is not a PHI node.
if (!isa<PHINode>(Inst))
continue;
PHINode *Phi = dyn_cast<PHINode>(Inst);
- LLVM_DEBUG(dbgs() << "UPDATING: PHI node: " << *Phi << "\n");
for (unsigned I = 0; I < Phi->getNumIncomingValues(); I++) {
if (Phi->getIncomingBlock(I) != FC0.Latch)
continue;
@@ -1248,7 +1247,6 @@ struct LoopFuser {
}
LLVM_DEBUG(
dbgs() << "All preheader instructions could be sunk or hoisted!\n");
-
return true;
}
diff --git a/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
index 45b812855e7eb..36c6bdde781c7 100644
--- a/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
+++ b/llvm/test/Transforms/LoopFusion/sunk-phi-nodes.ll
@@ -1,115 +1,65 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S -passes=loop-fusion < %s 2>&1 | FileCheck %s
-define i32 @foo() {
-; CHECK-LABEL: define i32 @foo() {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[RETVAL:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[SUM1:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[SUM2:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[I:%.*]] = alloca i32, align 4
-; CHECK-NEXT: [[I1:%.*]] = alloca i32, align 4
-; CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
-; CHECK-NEXT: store i32 0, ptr [[SUM1]], align 4
-; CHECK-NEXT: store i32 0, ptr [[SUM2]], align 4
-; CHECK-NEXT: store i32 0, ptr [[I]], align 4
-; CHECK-NEXT: br label %[[FOR_COND:.*]]
-; CHECK: [[FOR_COND]]:
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-; CHECK-NEXT: br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+; RUN: opt -passes=loop-fusion -S < %s 2>&1 | FileCheck %s
+define dso_local i32 @check_sunk_phi_nodes() {
+; CHECK-LABEL: define dso_local i32 @check_sunk_phi_nodes() {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[FOR_BODY:.*]]
; CHECK: [[FOR_BODY]]:
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[SUM1]], align 4
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP1]]
-; CHECK-NEXT: store i32 [[ADD]], ptr [[SUM1]], align 4
+; CHECK-NEXT: [[SUM1_02:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD:%.*]], %[[FOR_INC6:.*]] ]
+; CHECK-NEXT: [[I_01:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[I1_04:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[INC7:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[SUM2_03:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[ADD5:%.*]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[ADD]] = add nsw i32 [[SUM1_02]], [[I_01]]
; CHECK-NEXT: br label %[[FOR_INC:.*]]
; CHECK: [[FOR_INC]]:
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[I]], align 4
-; CHECK-NEXT: [[INC:%.*]] = add nsw i32 [[TMP3]], 1
-; CHECK-NEXT: store i32 [[INC]], ptr [[I]], align 4
-; CHECK-NEXT: br label %[[FOR_COND]]
-; CHECK: [[FOR_END]]:
-; CHECK-NEXT: store i32 0, ptr [[I1]], align 4
-; CHECK-NEXT: br label %[[FOR_COND2:.*]]
-; CHECK: [[FOR_COND2]]:
-; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[I1]], align 4
-; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i32 [[TMP4]], 10
-; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY4:.*]], label %[[FOR_END8:.*]]
-; CHECK: [[FOR_BODY4]]:
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[I1]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[I1]], align 4
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP5]], [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[SUM2]], align 4
-; CHECK-NEXT: [[ADD5:%.*]] = add nsw i32 [[TMP7]], [[MUL]]
-; CHECK-NEXT: store i32 [[ADD5]], ptr [[SUM2]], align 4
-; CHECK-NEXT: br label %[[FOR_INC6:.*]]
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[I1_04]], [[I1_04]]
+; CHECK-NEXT: [[ADD5]] = add nsw i32 [[SUM2_03]], [[MUL]]
+; CHECK-NEXT: br label %[[FOR_INC6]]
; CHECK: [[FOR_INC6]]:
-; CHECK-NEXT: [[TMP8:%.*]] = load i32, ptr [[I1]], align 4
-; CHECK-NEXT: [[INC7:%.*]] = add nsw i32 [[TMP8]], 1
-; CHECK-NEXT: store i32 [[INC7]], ptr [[I1]], align 4
-; CHECK-NEXT: br label %[[FOR_COND2]]
+; CHECK-NEXT: [[INC]] = add nsw i32 [[I_01]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[INC]], 10
+; CHECK-NEXT: [[INC7]] = add nsw i32 [[I1_04]], 1
+; CHECK-NEXT: [[CMP3:%.*]] = icmp slt i32 [[INC7]], 10
+; CHECK-NEXT: br i1 [[CMP3]], label %[[FOR_BODY]], label %[[FOR_END8:.*]]
; CHECK: [[FOR_END8]]:
-; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[SUM1]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[SUM2]], align 4
-; CHECK-NEXT: ret i32 0
+; CHECK-NEXT: [[SUM2_0_LCSSA:%.*]] = phi i32 [ [[ADD5]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[SUM1_0_LCSSA:%.*]] = phi i32 [ [[ADD]], %[[FOR_INC6]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SUM1_0_LCSSA]], [[SUM2_0_LCSSA]]
+; CHECK-NEXT: ret i32 [[TMP0]]
;
entry:
- %retval = alloca i32, align 4
- %sum1 = alloca i32, align 4
- %sum2 = alloca i32, align 4
- %i = alloca i32, align 4
- %i1 = alloca i32, align 4
- store i32 0, ptr %retval, align 4
- store i32 0, ptr %sum1, align 4
- store i32 0, ptr %sum2, align 4
- store i32 0, ptr %i, align 4
- br label %for.cond
+ br label %for.body
-for.cond:
- %0 = load i32, ptr %i, align 4
- %cmp = icmp slt i32 %0, 10
- br i1 %cmp, label %for.body, label %for.end
-
-for.body:
- %1 = load i32, ptr %i, align 4
- %2 = load i32, ptr %sum1, align 4
- %add = add nsw i32 %2, %1
- store i32 %add, ptr %sum1, align 4
+for.body: ; preds = %entry, %for.inc
+ %sum1.02 = phi i32 [ 0, %entry ], [ %add, %for.inc ]
+ %i.01 = phi i32 [ 0, %entry ], [ %inc, %for.inc ]
+ %add = add nsw i32 %sum1.02, %i.01
br label %for.inc
-for.inc:
- %3 = load i32, ptr %i, align 4
- %inc = add nsw i32 %3, 1
- store i32 %inc, ptr %i, align 4
- br label %for.cond
-
-for.end:
- store i32 0, ptr %i1, align 4
- br label %for.cond2
+for.inc: ; preds = %for.body
+ %inc = add nsw i32 %i.01, 1
+ %cmp = icmp slt i32 %inc, 10
+ br i1 %cmp, label %for.body, label %for.end
-for.cond2:
- %4 = load i32, ptr %i1, align 4
- %cmp3 = icmp slt i32 %4, 10
- br i1 %cmp3, label %for.body4, label %for.end8
+for.end: ; preds = %for.inc
+ %sum1.0.lcssa = phi i32 [ %add, %for.inc ]
+ br label %for.body4
-for.body4:
- %5 = load i32, ptr %i1, align 4
- %6 = load i32, ptr %i1, align 4
- %mul = mul nsw i32 %5, %6
- %7 = load i32, ptr %sum2, align 4
- %add5 = add nsw i32 %7, %mul
- store i32 %add5, ptr %sum2, align 4
+for.body4: ; preds = %for.end, %for.inc6
+ %i1.04 = phi i32 [ 0, %for.end ], [ %inc7, %for.inc6 ]
+ %sum2.03 = phi i32 [ 0, %for.end ], [ %add5, %for.inc6 ]
+ %mul = mul nsw i32 %i1.04, %i1.04
+ %add5 = add nsw i32 %sum2.03, %mul
br label %for.inc6
-for.inc6:
- %8 = load i32, ptr %i1, align 4
- %inc7 = add nsw i32 %8, 1
- store i32 %inc7, ptr %i1, align 4
- br label %for.cond2
+for.inc6: ; preds = %for.body4
+ %inc7 = add nsw i32 %i1.04, 1
+ %cmp3 = icmp slt i32 %inc7, 10
+ br i1 %cmp3, label %for.body4, label %for.end8
-for.end8:
- %9 = load i32, ptr %sum1, align 4
- %10 = load i32, ptr %sum2, align 4
- ret i32 0
+for.end8: ; preds = %for.inc6
+ %sum2.0.lcssa = phi i32 [ %add5, %for.inc6 ]
+ %0 = add i32 %sum1.0.lcssa, %sum2.0.lcssa
+ ret i32 %0
}
>From b6262e062f6e0d0ee4d651b6cd64b4b78fa9667a Mon Sep 17 00:00:00 2001
From: Madhur Amilkanthwar <madhura at nvidia.com>
Date: Mon, 21 Jul 2025 21:31:01 -0700
Subject: [PATCH 4/4] address review comments; more comments
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 30 ++++++++++++-------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 0ad85c5b338b3..b5eb647a042b9 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1033,8 +1033,8 @@ struct LoopFuser {
FuseCounter);
FusionCandidate FusedCand(
- performFusion((Peel ? FC0Copy : *FC0), *FC1, SafeToSink), DT,
- &PDT, ORE, FC0Copy.PP);
+ performFusion((Peel ? FC0Copy : *FC0), *FC1), DT, &PDT, ORE,
+ FC0Copy.PP);
FusedCand.verify();
assert(FusedCand.isEligibleForFusion(SE) &&
"Fused candidate should be eligible for fusion!");
@@ -1176,18 +1176,19 @@ struct LoopFuser {
return true;
}
- // This function fixes sunk PHI nodes after fusion.
- void fixPHINodes(SmallVector<Instruction *, 4> &SafeToSink,
+ /// This function fixes PHI nodes after fusion in \p SafeToSink.
+ /// \p SafeToSink instructions are the instructions that are to be moved past
+ /// the fused loop. Thus, the PHI nodes in \p SafeToSink should be updated to
+ /// receive values from the fused loop if they are currently taking values
+ /// from the first loop (i.e. FC0)'s latch.
+ void fixPHINodes(ArrayRef<Instruction *> SafeToSink,
const FusionCandidate &FC0,
const FusionCandidate &FC1) const {
- // Iterate over SafeToSink instructions and update PHI nodes
- // to take values from the latch block of FC0 if they are taking
- // from the latch block of FC1.
for (Instruction *Inst : SafeToSink) {
- // Continue if the instruction is not a PHI node.
- if (!isa<PHINode>(Inst))
- continue;
+ // No update needed for non-PHI nodes.
PHINode *Phi = dyn_cast<PHINode>(Inst);
+ if (!Phi)
+ continue;
for (unsigned I = 0; I < Phi->getNumIncomingValues(); I++) {
if (Phi->getIncomingBlock(I) != FC0.Latch)
continue;
@@ -1502,6 +1503,9 @@ struct LoopFuser {
assert(I->getParent() == FC1.Preheader);
I->moveBefore(*FC1.ExitBlock, FC1.ExitBlock->getFirstInsertionPt());
}
+ // PHI nodes in SinkInsts need to be updated to receive values from the
+ // fused loop.
+ fixPHINodes(SinkInsts, FC0, FC1);
}
/// Determine if two fusion candidates have identical guards
@@ -1590,8 +1594,7 @@ struct LoopFuser {
/// two loops could also be fused into a single block. This will require
/// analysis to prove it is safe to move the contents of the block past
/// existing code, which currently has not been implemented.
- Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1,
- SmallVector<Instruction *, 4> &SafeToSink) {
+ Loop *performFusion(const FusionCandidate &FC0, const FusionCandidate &FC1) {
assert(FC0.isValid() && FC1.isValid() &&
"Expecting valid fusion candidates");
@@ -1733,9 +1736,6 @@ struct LoopFuser {
TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
FC1.Latch, FC1.Header));
- // Fix PHI nodes that are sunk into the body of the loop.
- fixPHINodes(SafeToSink, FC0, FC1);
-
// Update DT/PDT
DTU.applyUpdates(TreeUpdates);
More information about the llvm-commits
mailing list