[llvm] [DA] Refine identical SameSD addrec dependences (PR #206425)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 01:06:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-analysis
Author: XiaShark
<details>
<summary>Changes</summary>
LoopFusion now uses DependenceAnalysis by default. In a case with two adjacent loops over the same iteration space, DA can fail to classify a store/load pair as SIV even though both subscripts are identical addrecs on SameSD loops.
The missed case has equivalent address recurrences in sibling loops:
SrcSCEV = {A,+,4}<L0>
DstSCEV = {A,+,4}<L1>
where L0 and L1 are sibling loops with the same trip count and depth. The generic subscript classifier requires addrecs accepted by checkSubscript(). Depending on how SCEV derives nowrap flags, one side may not carry a signed no-wrap flag even when the corresponding induction increment has nsw. As a result, the pair is classified as NonLinear before DA can recognize that the two SameSD recurrences are identical, and LoopFusion rejects a safe fusion.
Teach DA to recognize identical addrecs in SameSD loops as SIV directly when their starts are known equal, their steps match, and their start/step expressions are loop-invariant for the corresponding loop nests.
Once classified as SIV, refine identical recurrences in Strong SIV to an equal direction. They only have same-iteration dependences, so after fusion the store still precedes the load in the fused iteration and there is no backward loop-carried dependence.
Add a LoopFusion regression test mirroring the motivating shape, with an additional pointer recurrence and readonly call in the first loop. The checks are generated with update_test_checks.py.
See https://godbolt.org/z/hvK347er9
---
Full diff: https://github.com/llvm/llvm-project/pull/206425.diff
2 Files Affected:
- (modified) llvm/lib/Analysis/DependenceAnalysis.cpp (+29)
- (added) llvm/test/Transforms/LoopFusion/da-conservative-direction.ll (+76)
``````````diff
diff --git a/llvm/lib/Analysis/DependenceAnalysis.cpp b/llvm/lib/Analysis/DependenceAnalysis.cpp
index 9d5a555fb8998..d731ef91578f3 100644
--- a/llvm/lib/Analysis/DependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/DependenceAnalysis.cpp
@@ -842,6 +842,26 @@ DependenceInfo::classifyPair(const SCEV *Src, const Loop *SrcLoopNest,
SmallBitVector &Loops) {
SmallBitVector SrcLoops(MaxLevels + 1);
SmallBitVector DstLoops(MaxLevels + 1);
+ const SCEVAddRecExpr *SrcAddRec = dyn_cast<SCEVAddRecExpr>(Src);
+ const SCEVAddRecExpr *DstAddRec = dyn_cast<SCEVAddRecExpr>(Dst);
+ if (SrcAddRec && DstAddRec &&
+ SE->isKnownPredicate(CmpInst::ICMP_EQ, SrcAddRec->getStart(),
+ DstAddRec->getStart()) &&
+ SrcAddRec->getStepRecurrence(*SE) == DstAddRec->getStepRecurrence(*SE) &&
+ SrcAddRec->getLoop() != DstAddRec->getLoop() &&
+ haveSameSD(SrcAddRec->getLoop(), DstAddRec->getLoop()) &&
+ isLoopInvariant(SrcAddRec->getStart(), SrcLoopNest) &&
+ isLoopInvariant(DstAddRec->getStart(), DstLoopNest) &&
+ isLoopInvariant(SrcAddRec->getStepRecurrence(*SE), SrcLoopNest) &&
+ isLoopInvariant(DstAddRec->getStepRecurrence(*SE), DstLoopNest)) {
+ SrcLoops.set(mapSrcLoop(SrcAddRec->getLoop()));
+ DstLoops.set(mapDstLoop(DstAddRec->getLoop()));
+ Loops = SrcLoops;
+ Loops |= DstLoops;
+ assert(Loops.count() == 1 && "Expected one SameSD level");
+ return Subscript::SIV;
+ }
+
if (!checkSrcSubscript(Src, SrcLoopNest, SrcLoops))
return Subscript::NonLinear;
if (!checkDstSubscript(Dst, DstLoopNest, DstLoops))
@@ -986,6 +1006,15 @@ bool DependenceInfo::strongSIVtest(const SCEVAddRecExpr *Src,
assert(0 < Level && Level <= CommonLevels && "level out of range");
Level--;
+ // Identical recurrences only have same-iteration dependences. This is useful
+ // for SameSD loops, where the addrecs are in different loops but share the
+ // same start and step.
+ if (SE->isKnownPredicate(CmpInst::ICMP_EQ, SrcConst, DstConst)) {
+ Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
+ ++StrongSIVsuccesses;
+ return false;
+ }
+
const SCEV *Delta = minusSCEVNoSignedOverflow(SrcConst, DstConst, *SE);
if (!Delta)
return false;
diff --git a/llvm/test/Transforms/LoopFusion/da-conservative-direction.ll b/llvm/test/Transforms/LoopFusion/da-conservative-direction.ll
new file mode 100644
index 0000000000000..83c49186d9d68
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/da-conservative-direction.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=loop-fusion < %s | FileCheck %s
+
+; DA used to classify the store/load dependence as non-linear even though the
+; subscripts are identical addrecs in SameSD loops. This shape intentionally
+; mirrors a loop nest where the first loop has another pointer recurrence and a
+; readonly call; simply adding nsw to the induction increments is not enough for
+; trunk to fuse it.
+define i64 @fuse_identical_samesd_addrecs(ptr %a, ptr %x, ptr %y, i64 %stride, i64 %n) {
+; CHECK-LABEL: define i64 @fuse_identical_samesd_addrecs(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]], i64 [[STRIDE:%.*]], i64 [[N:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP0:%.*]] = icmp eq i64 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[FOR_1_PREHEADER:.*]]
+; CHECK: [[FOR_1_PREHEADER]]:
+; CHECK-NEXT: br label %[[FOR_1:.*]]
+; CHECK: [[FOR_1]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], %[[FOR_1]] ], [ 0, %[[FOR_1_PREHEADER]] ]
+; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P_NEXT:%.*]], %[[FOR_1]] ], [ [[Y]], %[[FOR_1_PREHEADER]] ]
+; CHECK-NEXT: [[J:%.*]] = phi i64 [ [[J_NEXT:%.*]], %[[FOR_1]] ], [ 0, %[[FOR_1_PREHEADER]] ]
+; CHECK-NEXT: [[BEST:%.*]] = phi float [ [[BEST_NEXT:%.*]], %[[FOR_1]] ], [ +inf, %[[FOR_1_PREHEADER]] ]
+; CHECK-NEXT: [[IDX:%.*]] = phi i64 [ [[IDX_NEXT:%.*]], %[[FOR_1]] ], [ 0, %[[FOR_1_PREHEADER]] ]
+; CHECK-NEXT: [[CALL:%.*]] = tail call float @readonly_call(ptr [[X]], ptr [[P]], i64 [[STRIDE]])
+; CHECK-NEXT: [[GEP_A_1:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: store float [[CALL]], ptr [[GEP_A_1]], align 4
+; CHECK-NEXT: [[P_NEXT]] = getelementptr inbounds nuw float, ptr [[P]], i64 [[STRIDE]]
+; CHECK-NEXT: [[I_NEXT]] = add nsw i64 [[I]], 1
+; CHECK-NEXT: [[DONE_1:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: [[GEP_A_2:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[J]]
+; CHECK-NEXT: [[V:%.*]] = load float, ptr [[GEP_A_2]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = fcmp olt float [[V]], [[BEST]]
+; CHECK-NEXT: [[IDX_NEXT]] = select i1 [[CMP]], i64 [[J]], i64 [[IDX]]
+; CHECK-NEXT: [[BEST_NEXT]] = select i1 [[CMP]], float [[V]], float [[BEST]]
+; CHECK-NEXT: [[J_NEXT]] = add nsw i64 [[J]], 1
+; CHECK-NEXT: [[DONE_2:%.*]] = icmp eq i64 [[J_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[DONE_2]], label %[[EXIT_LOOPEXIT:.*]], label %[[FOR_1]]
+; CHECK: [[EXIT_LOOPEXIT]]:
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IDX_NEXT]], %[[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT: ret i64 [[RES]]
+;
+entry:
+ %cmp0 = icmp eq i64 %n, 0
+ br i1 %cmp0, label %exit, label %for.1
+
+for.1:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %for.1 ]
+ %p = phi ptr [ %y, %entry ], [ %p.next, %for.1 ]
+ %call = tail call float @readonly_call(ptr %x, ptr %p, i64 %stride)
+ %gep.a.1 = getelementptr inbounds nuw float, ptr %a, i64 %i
+ store float %call, ptr %gep.a.1, align 4
+ %p.next = getelementptr inbounds nuw float, ptr %p, i64 %stride
+ %i.next = add nsw i64 %i, 1
+ %done.1 = icmp eq i64 %i.next, %n
+ br i1 %done.1, label %for.2, label %for.1
+
+for.2:
+ %j = phi i64 [ 0, %for.1 ], [ %j.next, %for.2 ]
+ %best = phi float [ 0x7FF0000000000000, %for.1 ], [ %best.next, %for.2 ]
+ %idx = phi i64 [ 0, %for.1 ], [ %idx.next, %for.2 ]
+ %gep.a.2 = getelementptr inbounds nuw float, ptr %a, i64 %j
+ %v = load float, ptr %gep.a.2, align 4
+ %cmp = fcmp olt float %v, %best
+ %idx.next = select i1 %cmp, i64 %j, i64 %idx
+ %best.next = select i1 %cmp, float %v, float %best
+ %j.next = add nsw i64 %j, 1
+ %done.2 = icmp eq i64 %j.next, %n
+ br i1 %done.2, label %exit, label %for.2
+
+exit:
+ %res = phi i64 [ 0, %entry ], [ %idx.next, %for.2 ]
+ ret i64 %res
+}
+
+declare float @readonly_call(ptr, ptr, i64) nounwind readonly
``````````
</details>
https://github.com/llvm/llvm-project/pull/206425
More information about the llvm-commits
mailing list