[llvm] [LoopFusion] Allow fusing adjacent loops with same-iteration scalar deps (PR #216242)
Anchu Rajendran S via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 13 21:14:18 PDT 2026
https://github.com/anchuraj created https://github.com/llvm/llvm-project/pull/216242
The scalar dependence branch rejected all non-invariant flow deps, while the array branch already accepted EQ-only direction vectors. Treat same-iteration scalar store-then-load patterns as safe to fuse when the dependence direction has no GT/LT component, matching the array logic.
>From 80094b0cd0a0832b712cb8967eb8073841abe19c Mon Sep 17 00:00:00 2001
From: Anchu Rajendran <asudhaku at amd.com>
Date: Fri, 14 Aug 2026 09:42:10 +0530
Subject: [PATCH] [LoopFusion] Allow fusing adjacent loops with same-iteration
scalar deps
The scalar dependence branch rejected all non-invariant flow deps, while
the array branch already accepted EQ-only direction vectors. Treat
same-iteration scalar store-then-load patterns as safe to fuse when the
dependence direction has no GT/LT component, matching the array logic.
---
llvm/lib/Transforms/Scalar/LoopFuse.cpp | 11 +++
.../LoopFusion/store_then_accumulate.ll | 80 +++++++++++++++++++
2 files changed, 91 insertions(+)
create mode 100644 llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index 6ce82a0dd7684..1fe45ed1fe48e 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -1165,6 +1165,17 @@ struct LoopFuser {
NumDA++;
return true;
}
+ // Same-iteration scalar flow/anti dependences between adjacent loops are
+ // preserved by placing FC0's body before FC1's body in the fused loop.
+ // This enables fusing accumulation chains such as:
+ // for (i) A[i] = ...; for (i) A[i] += ...;
+ unsigned CurDir = DepResult->getDirection(CurLoopLevel, true);
+ if (!(CurDir & Dependence::DVEntry::GT) &&
+ !(CurDir & Dependence::DVEntry::LT)) {
+ LLVM_DEBUG(dbgs() << "Safe to fuse same-iteration scalar dependence\n");
+ NumDA++;
+ return true;
+ }
LLVM_DEBUG(
dbgs() << "Not safe to fuse due to a scalar flow dependency\n");
return false;
diff --git a/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
new file mode 100644
index 0000000000000..2745ce7b4d7aa
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/store_then_accumulate.ll
@@ -0,0 +1,80 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=loop-simplify,loop-fusion -stats -disable-output < %s 2>&1 | FileCheck %s --check-prefix=STAT
+; RUN: opt -S -passes=loop-simplify,loop-fusion < %s | FileCheck %s
+; STAT: 1 loop-fusion{{.*}} - Loops fused
+
+; Two adjacent loops: store then read-modify-write on the same array element.
+
+define void @store_then_accumulate(ptr noalias nocapture %hist, ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %C, i64 %n, float %w0, float %w1) {
+; CHECK-LABEL: define void @store_then_accumulate(
+; CHECK-SAME: ptr noalias captures(none) [[HIST:%.*]], ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[C:%.*]], i64 [[N:%.*]], float [[W0:%.*]], float [[W1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[LOOP1:.*]]
+; CHECK: [[LOOP1]]:
+; CHECK-NEXT: [[I1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I1_NEXT:%.*]], %[[LOOP2_LATCH:.*]] ]
+; CHECK-NEXT: [[I2:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I2_NEXT:%.*]], %[[LOOP2_LATCH]] ]
+; CHECK-NEXT: [[A_GEP:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[I1]]
+; CHECK-NEXT: [[A_VAL:%.*]] = load float, ptr [[A_GEP]], align 4
+; CHECK-NEXT: [[W0_PROD:%.*]] = fmul float [[W0]], [[A_VAL]]
+; CHECK-NEXT: [[H1_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I1]]
+; CHECK-NEXT: store float [[W0_PROD]], ptr [[H1_GEP]], align 4
+; CHECK-NEXT: br label %[[LOOP1_LATCH:.*]]
+; CHECK: [[LOOP1_LATCH]]:
+; CHECK-NEXT: [[C_GEP:%.*]] = getelementptr inbounds float, ptr [[C]], i64 [[I2]]
+; CHECK-NEXT: [[C_VAL:%.*]] = load float, ptr [[C_GEP]], align 4
+; CHECK-NEXT: [[H2_GEP:%.*]] = getelementptr inbounds float, ptr [[HIST]], i64 [[I2]]
+; CHECK-NEXT: [[OLD2:%.*]] = load float, ptr [[H2_GEP]], align 4
+; CHECK-NEXT: [[W1_PROD:%.*]] = fmul float [[W1]], [[C_VAL]]
+; CHECK-NEXT: [[SUM2:%.*]] = fadd float [[OLD2]], [[W1_PROD]]
+; CHECK-NEXT: store float [[SUM2]], ptr [[H2_GEP]], align 4
+; CHECK-NEXT: br label %[[LOOP2_LATCH]]
+; CHECK: [[LOOP2_LATCH]]:
+; CHECK-NEXT: [[I1_NEXT]] = add nuw nsw i64 [[I1]], 1
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ult i64 [[I1_NEXT]], [[N]]
+; CHECK-NEXT: [[I2_NEXT]] = add nuw nsw i64 [[I2]], 1
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ult i64 [[I2_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[CMP2]], label %[[LOOP1]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ br label %loop1
+
+loop1:
+ %i1 = phi i64 [ 0, %entry ], [ %i1.next, %loop1.latch ]
+ %a.gep = getelementptr inbounds float, ptr %A, i64 %i1
+ %a.val = load float, ptr %a.gep, align 4
+ %w0.prod = fmul float %w0, %a.val
+ %h1.gep = getelementptr inbounds float, ptr %hist, i64 %i1
+ store float %w0.prod, ptr %h1.gep, align 4
+ br label %loop1.latch
+
+loop1.latch:
+ %i1.next = add nuw nsw i64 %i1, 1
+ %cmp1 = icmp ult i64 %i1.next, %n
+ br i1 %cmp1, label %loop1, label %loop2.preheader
+
+loop2.preheader:
+ br label %loop2
+
+loop2:
+ %i2 = phi i64 [ 0, %loop2.preheader ], [ %i2.next, %loop2.latch ]
+ %c.gep = getelementptr inbounds float, ptr %C, i64 %i2
+ %c.val = load float, ptr %c.gep, align 4
+ %h2.gep = getelementptr inbounds float, ptr %hist, i64 %i2
+ %old2 = load float, ptr %h2.gep, align 4
+ %w1.prod = fmul float %w1, %c.val
+ %sum2 = fadd float %old2, %w1.prod
+ store float %sum2, ptr %h2.gep, align 4
+ br label %loop2.latch
+
+loop2.latch:
+ %i2.next = add nuw nsw i64 %i2, 1
+ %cmp2 = icmp ult i64 %i2.next, %n
+ br i1 %cmp2, label %loop2, label %exit
+
+exit:
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; STAT: {{.*}}
More information about the llvm-commits
mailing list