[llvm] [LoopFusion] Allow fusing adjacent loops with same-iteration scalar deps (PR #216242)
Anchu Rajendran S via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 13:51:02 PDT 2026
https://github.com/anchuraj updated https://github.com/llvm/llvm-project/pull/216242
>From 80094b0cd0a0832b712cb8967eb8073841abe19c Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Fri, 14 Aug 2026 09:42:10 +0530
Subject: [PATCH 1/2] [LoopFusion] Allow fusing adjacent loops with
same-iteration scalar deps
The scalar dependence branch rejected all non-invariant flow deps, while
the array branch already accepted EQ-only direction vectors. Treat
same-iteration scalar store-then-load patterns as safe to fuse when the
dependence direction has no GT/LT component, matching the array logic.
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 11 +++
.../LoopFusion/store_then_accumulate.ll | 80 +++++++++++++++++++
2 files changed, 91 insertions(+)
create mode 100644 llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 6ce82a0dd7684..1fe45ed1fe48e 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1165,6 +1165,17 @@ struct LoopFuser {
NumDA++;
return true;
}
+ // Same-iteration scalar flow/anti dependences between adjacent loops are
+ // preserved by placing FC0's body before FC1's body in the fused loop.
+ // This enables fusing accumulation chains such as:
+ // for (i) A[i] = ...; for (i) A[i] += ...;
+ unsigned CurDir = DepResult->getDirection(CurLoopLevel, true);
+ if (!(CurDir & Dependence::DVEntry::GT) &&
+ !(CurDir & Dependence::DVEntry::LT)) {
+ LLVM_DEBUG(dbgs() << "Safe to fuse same-iteration scalar dependence\n");
+ NumDA++;
+ return true;
+ }
LLVM_DEBUG(
dbgs() << "Not safe to fuse due to a scalar flow dependency\n");
return false;
diff --git a/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
new file mode 100644
index 0000000000000..2745ce7b4d7aa
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-simplify,loop-fusion -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STAT
+; RUN: opt -S -passes=loop-simplify,loop-fusion < %s | FileCheck %s
+; STAT: 1 loop-fusion{{.*}} - Loops fused
+
+; Two adjacent loops: store then read-modify-write on the same array element.
+
+define void @store_then_accumulate(ptr noalias nocapture %hist, ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %C, i64 %n, float %w0, float %w1) {
+; CHECK-LABEL: define void @store_then_accumulate(
+; CHECK-SAME: ptr noalias captures(none) [[HIST:%.*]], ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i64 [[N:%.*]], float [[W0:%.*]], float [[W1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[I1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I1_NEXT:%.*]], %[[LOOP2_LATCH:.*]] ]
+; CHECK-NEXT: [[I2:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I2_NEXT:%.*]], %[[LOOP2_LATCH]] ]
+; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I1]]
+; CHECK-NEXT: [[A_VAL:%.*]] = load float, ptr [[A_GEP]], align 4
+; CHECK-NEXT: [[W0_PROD:%.*]] = fmul float [[W0]], [[A_VAL]]
+; CHECK-NEXT: [[H1_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I1]]
+; CHECK-NEXT: store float [[W0_PROD]], ptr [[H1_GEP]], align 4
+; CHECK-NEXT: br label %[[LOOP1_LATCH:.*]]
+; CHECK: [[LOOP1_LATCH]]:
+; CHECK-NEXT: [[C_GEP:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[I2]]
+; CHECK-NEXT: [[C_VAL:%.*]] = load float, ptr [[C_GEP]], align 4
+; CHECK-NEXT: [[H2_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I2]]
+; CHECK-NEXT: [[OLD2:%.*]] = load float, ptr [[H2_GEP]], align 4
+; CHECK-NEXT: [[W1_PROD:%.*]] = fmul float [[W1]], [[C_VAL]]
+; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[OLD2]], [[W1_PROD]]
+; CHECK-NEXT: store float [[SUM2]], ptr [[H2_GEP]], align 4
+; CHECK-NEXT: br label %[[LOOP2_LATCH]]
+; CHECK: [[LOOP2_LATCH]]:
+; CHECK-NEXT: [[I1_NEXT]] = add nuw nsw i64 [[I1]], 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[I1_NEXT]], [[N]]
+; CHECK-NEXT: [[I2_NEXT]] = add nuw nsw i64 [[I2]], 1
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i64 [[I2_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP2]], label %[[LOOP1]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop1
+
+loop1:
+ %i1 = phi i64 [ 0, %entry ], [ %i1.next, %loop1.latch ]
+ %a.gep = getelementptr inbounds float, ptr %A, i64 %i1
+ %a.val = load float, ptr %a.gep, align 4
+ %w0.prod = fmul float %w0, %a.val
+ %h1.gep = getelementptr inbounds float, ptr %hist, i64 %i1
+ store float %w0.prod, ptr %h1.gep, align 4
+ br label %loop1.latch
+
+loop1.latch:
+ %i1.next = add nuw nsw i64 %i1, 1
+ %cmp1 = icmp ult i64 %i1.next, %n
+ br i1 %cmp1, label %loop1, label %loop2.preheader
+
+loop2.preheader:
+ br label %loop2
+
+loop2:
+ %i2 = phi i64 [ 0, %loop2.preheader ], [ %i2.next, %loop2.latch ]
+ %c.gep = getelementptr inbounds float, ptr %C, i64 %i2
+ %c.val = load float, ptr %c.gep, align 4
+ %h2.gep = getelementptr inbounds float, ptr %hist, i64 %i2
+ %old2 = load float, ptr %h2.gep, align 4
+ %w1.prod = fmul float %w1, %c.val
+ %sum2 = fadd float %old2, %w1.prod
+ store float %sum2, ptr %h2.gep, align 4
+ br label %loop2.latch
+
+loop2.latch:
+ %i2.next = add nuw nsw i64 %i2, 1
+ %cmp2 = icmp ult i64 %i2.next, %n
+ br i1 %cmp2, label %loop2, label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; STAT: {{.*}}
>From 6a75159dda225fc87468fec01ba46493a3f1ce71 Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Sat, 15 Aug 2026 02:14:25 +0530
Subject: [PATCH 2/2] R2: Addressing review comments
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 5 +++-
.../LoopFusion/store_then_accumulate.ll | 29 ++++++++++---------
2 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 1fe45ed1fe48e..e7a790283dca0 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1168,7 +1168,10 @@ struct LoopFuser {
// Same-iteration scalar flow/anti dependences between adjacent loops are
// preserved by placing FC0's body before FC1's body in the fused loop.
// This enables fusing accumulation chains such as:
- // for (i) A[i] = ...; for (i) A[i] += ...;
+ // for (i)
+ // A[i] = ...;
+ // for (i)
+ // A[i] += ...;
unsigned CurDir = DepResult->getDirection(CurLoopLevel, true);
if (!(CurDir & Dependence::DVEntry::GT) &&
!(CurDir & Dependence::DVEntry::LT)) {
diff --git a/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
index 2745ce7b4d7aa..750c78d91515c 100644
--- a/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
+++ b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
@@ -1,13 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -passes=loop-simplify,loop-fusion -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STAT
; RUN: opt -S -passes=loop-simplify,loop-fusion < %s | FileCheck %s
-; STAT: 1 loop-fusion{{.*}} - Loops fused
; Two adjacent loops: store then read-modify-write on the same array element.
+; The same-iteration flow dependence is preserved by fusion, since the first
+; loop's body is placed before the second loop's body.
-define void @store_then_accumulate(ptr noalias nocapture %hist, ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %C, i64 %n, float %w0, float %w1) {
+; C Code
+;
+;; for (long int i = 0; i < n; i++)
+;; hist[i] = A[i];
+;; for (long int i = 0; i < n; i++)
+;; hist[i] += C[i];
+
+define void @store_then_accumulate(ptr noalias %hist, ptr %A, ptr %C, i64 %n) {
; CHECK-LABEL: define void @store_then_accumulate(
-; CHECK-SAME: ptr noalias captures(none) [[HIST:%.*]], ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i64 [[N:%.*]], float [[W0:%.*]], float [[W1:%.*]]) {
+; CHECK-SAME: ptr noalias [[HIST:%.*]], ptr [[A:%.*]], ptr [[C:%.*]], i64 [[N:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
; CHECK-NEXT: br label %[[LOOP1:.*]]
; CHECK: [[LOOP1]]:
@@ -15,17 +22,15 @@ define void @store_then_accumulate(ptr noalias nocapture %hist, ptr noalias noca
; CHECK-NEXT: [[I2:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I2_NEXT:%.*]], %[[LOOP2_LATCH]] ]
; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I1]]
; CHECK-NEXT: [[A_VAL:%.*]] = load float, ptr [[A_GEP]], align 4
-; CHECK-NEXT: [[W0_PROD:%.*]] = fmul float [[W0]], [[A_VAL]]
; CHECK-NEXT: [[H1_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I1]]
-; CHECK-NEXT: store float [[W0_PROD]], ptr [[H1_GEP]], align 4
+; CHECK-NEXT: store float [[A_VAL]], ptr [[H1_GEP]], align 4
; CHECK-NEXT: br label %[[LOOP1_LATCH:.*]]
; CHECK: [[LOOP1_LATCH]]:
; CHECK-NEXT: [[C_GEP:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[I2]]
; CHECK-NEXT: [[C_VAL:%.*]] = load float, ptr [[C_GEP]], align 4
; CHECK-NEXT: [[H2_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I2]]
; CHECK-NEXT: [[OLD2:%.*]] = load float, ptr [[H2_GEP]], align 4
-; CHECK-NEXT: [[W1_PROD:%.*]] = fmul float [[W1]], [[C_VAL]]
-; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[OLD2]], [[W1_PROD]]
+; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[OLD2]], [[C_VAL]]
; CHECK-NEXT: store float [[SUM2]], ptr [[H2_GEP]], align 4
; CHECK-NEXT: br label %[[LOOP2_LATCH]]
; CHECK: [[LOOP2_LATCH]]:
@@ -44,9 +49,8 @@ loop1:
%i1 = phi i64 [ 0, %entry ], [ %i1.next, %loop1.latch ]
%a.gep = getelementptr inbounds float, ptr %A, i64 %i1
%a.val = load float, ptr %a.gep, align 4
- %w0.prod = fmul float %w0, %a.val
%h1.gep = getelementptr inbounds float, ptr %hist, i64 %i1
- store float %w0.prod, ptr %h1.gep, align 4
+ store float %a.val, ptr %h1.gep, align 4
br label %loop1.latch
loop1.latch:
@@ -63,8 +67,7 @@ loop2:
%c.val = load float, ptr %c.gep, align 4
%h2.gep = getelementptr inbounds float, ptr %hist, i64 %i2
%old2 = load float, ptr %h2.gep, align 4
- %w1.prod = fmul float %w1, %c.val
- %sum2 = fadd float %old2, %w1.prod
+ %sum2 = fadd float %old2, %c.val
store float %sum2, ptr %h2.gep, align 4
br label %loop2.latch
@@ -76,5 +79,3 @@ loop2.latch:
exit:
ret void
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; STAT: {{.*}}
More information about the llvm-commits
mailing list