[llvm] [SLP]Fix loop-aware cost for gathers crossing sibling-loop boundaries (PR #224700)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 18 11:04:08 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/224700
Scale such gathers by the common parent nest, not by the sibling's
trip count.
Fixes #209325
>From 1756fc48a0afa70606a4c0a7d3067df2e10c2bc8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 18 Sep 2026 11:03:55 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 21 +++++++++++--
.../sibling-loops-different-trip-counts.ll | 30 ++++++-------------
2 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d865401cae6b2..1a839118c31a3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -9189,9 +9189,8 @@ static const Loop *findInnermostNonInvariantLoop(const Loop *L,
ArrayRef<Value *> VL) {
assert(L && "Expected valid loop");
auto IsLoopInvariant = [&](const Loop *L, ArrayRef<Value *> VL) {
- return all_of(VL, [&](Value *V) {
- return isa<Constant>(V) || !isa<Instruction>(V) || L->isLoopInvariant(V);
- });
+ return all_of(make_isa_range<Instruction>(VL),
+ [L](Instruction *I) { return L->isLoopInvariant(I); });
};
while (L && IsLoopInvariant(L, VL))
L = L->getParentLoop();
@@ -15729,6 +15728,22 @@ uint64_t BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
EI.UserTE->getOpcode() == Instruction::PHI) {
auto *PH = cast<PHINode>(EI.UserTE->getMainOp());
Parent = PH->getIncomingBlock(EI.EdgeIdx);
+ const Loop *PhiL = LI->getLoopFor(PH->getParent());
+ const Loop *InL = LI->getLoopFor(Parent);
+ if (PhiL && InL && !PhiL->contains(InL) && !InL->contains(PhiL)) {
+ const SCEV *PhiBTC = SE->getBackedgeTakenCount(PhiL);
+ const SCEV *InBTC = SE->getBackedgeTakenCount(InL);
+ if (isa<SCEVCouldNotCompute>(PhiBTC) || PhiBTC != InBTC) {
+ // If all gathered elements are invariant in the phi's nest below
+ // the common ancestor of the two nests, the gather is a pure
+ // nest-crossing transfer, paid once per crossing: scale it by
+ // the common parent nest.
+ const Loop *FirstVariantL =
+ findInnermostNonInvariantLoop(PhiL, TE.Scalars);
+ if (FirstVariantL && FirstVariantL->contains(InL))
+ Parent = FirstVariantL->getHeader();
+ }
+ }
} else {
Parent = EI.UserTE->getMainOp()->getParent();
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/sibling-loops-different-trip-counts.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/sibling-loops-different-trip-counts.ll
index bb81dc03712ed..38cebf755563b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/sibling-loops-different-trip-counts.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/sibling-loops-different-trip-counts.ll
@@ -13,7 +13,7 @@ define void @test(ptr %arr, ptr %out) {
; CHECK-NEXT: br label %[[OUTER_HEADER:.*]]
; CHECK: [[OUTER_HEADER]]:
; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[OUTER_NEXT:%.*]], %[[OUTER_LATCH:.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP9:%.*]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP11:%.*]], %[[OUTER_LATCH]] ]
; CHECK-NEXT: br label %[[FIRST_HEADER:.*]]
; CHECK: [[FIRST_HEADER]]:
; CHECK-NEXT: [[I_FIRST:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[I_FIRST_NEXT:%.*]], %[[FIRST_HEADER]] ]
@@ -25,38 +25,26 @@ define void @test(ptr %arr, ptr %out) {
; CHECK-NEXT: [[TMP5]] = fadd <2 x double> [[TMP4]], [[TMP1]]
; CHECK-NEXT: [[I_FIRST_NEXT]] = add nuw nsw i64 [[I_FIRST]], 1
; CHECK-NEXT: [[FIRST_CONT:%.*]] = icmp ult i64 [[I_FIRST_NEXT]], 32
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
; CHECK-NEXT: br i1 [[FIRST_CONT]], label %[[FIRST_HEADER]], label %[[SECOND_HEADER:.*]]
; CHECK: [[SECOND_HEADER]]:
; CHECK-NEXT: [[I_SECOND:%.*]] = phi i64 [ [[I_SECOND_NEXT:%.*]], %[[SECOND_HEADER]] ], [ 0, %[[FIRST_HEADER]] ]
-; CHECK-NEXT: [[ACC1_SECOND:%.*]] = phi double [ [[ACC1_AFTER_SECOND:%.*]], %[[SECOND_HEADER]] ], [ [[TMP7]], %[[FIRST_HEADER]] ]
-; CHECK-NEXT: [[ACC0_SECOND:%.*]] = phi double [ [[ACC0_AFTER_SECOND:%.*]], %[[SECOND_HEADER]] ], [ [[TMP6]], %[[FIRST_HEADER]] ]
+; CHECK-NEXT: [[TMP6:%.*]] = phi <2 x double> [ [[TMP10:%.*]], %[[SECOND_HEADER]] ], [ [[TMP5]], %[[FIRST_HEADER]] ]
; CHECK-NEXT: [[Q0:%.*]] = getelementptr inbounds nuw double, ptr [[ARR]], i64 [[I_SECOND]]
-; CHECK-NEXT: [[W0:%.*]] = load double, ptr [[Q0]], align 8
-; CHECK-NEXT: [[W0_BIAS:%.*]] = fadd double [[W0]], 1.000000e+00
-; CHECK-NEXT: [[W0_ACC:%.*]] = fadd double [[W0_BIAS]], [[W0_BIAS]]
-; CHECK-NEXT: [[ACC0_AFTER_SECOND]] = fadd double [[W0_ACC]], [[ACC0_SECOND]]
-; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds nuw i8, ptr [[Q0]], i64 8
-; CHECK-NEXT: [[W1:%.*]] = load double, ptr [[Q1]], align 8
-; CHECK-NEXT: [[W1_BIAS:%.*]] = fadd double [[W1]], 1.000000e+00
-; CHECK-NEXT: [[W1_ACC:%.*]] = fadd double [[W1_BIAS]], [[W1_BIAS]]
-; CHECK-NEXT: [[ACC1_AFTER_SECOND]] = fadd double [[W1_ACC]], [[ACC1_SECOND]]
+; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[Q0]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = fadd <2 x double> [[TMP7]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <2 x double> [[TMP8]], [[TMP8]]
+; CHECK-NEXT: [[TMP10]] = fadd <2 x double> [[TMP9]], [[TMP6]]
; CHECK-NEXT: [[I_SECOND_NEXT]] = add nuw nsw i64 [[I_SECOND]], 1
; CHECK-NEXT: [[SECOND_CONT:%.*]] = icmp ult i64 [[I_SECOND_NEXT]], 2
; CHECK-NEXT: br i1 [[SECOND_CONT]], label %[[SECOND_HEADER]], label %[[OUTER_LATCH]]
; CHECK: [[OUTER_LATCH]]:
-; CHECK-NEXT: [[ACC0_AFTER_SECOND_LCSSA:%.*]] = phi double [ [[ACC0_AFTER_SECOND]], %[[SECOND_HEADER]] ]
-; CHECK-NEXT: [[ACC1_AFTER_SECOND_LCSSA:%.*]] = phi double [ [[ACC1_AFTER_SECOND]], %[[SECOND_HEADER]] ]
+; CHECK-NEXT: [[TMP11]] = phi <2 x double> [ [[TMP10]], %[[SECOND_HEADER]] ]
; CHECK-NEXT: [[OUTER_NEXT]] = add nuw nsw i32 [[OUTER_IV]], 1
; CHECK-NEXT: [[OUTER_CONT:%.*]] = icmp ult i32 [[OUTER_NEXT]], 2
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> poison, double [[ACC0_AFTER_SECOND_LCSSA]], i64 0
-; CHECK-NEXT: [[TMP9]] = insertelement <2 x double> [[TMP8]], double [[ACC1_AFTER_SECOND_LCSSA]], i64 1
; CHECK-NEXT: br i1 [[OUTER_CONT]], label %[[OUTER_HEADER]], label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[ACC0_AFTER_SECOND_LCSSA_LCSSA:%.*]] = phi double [ [[ACC0_AFTER_SECOND_LCSSA]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[ACC1_AFTER_SECOND_LCSSA_LCSSA:%.*]] = phi double [ [[ACC1_AFTER_SECOND_LCSSA]], %[[OUTER_LATCH]] ]
-; CHECK-NEXT: [[TMP13:%.*]] = fadd double [[ACC0_AFTER_SECOND_LCSSA_LCSSA]], [[ACC1_AFTER_SECOND_LCSSA_LCSSA]]
+; CHECK-NEXT: [[TMP12:%.*]] = phi <2 x double> [ [[TMP11]], %[[OUTER_LATCH]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = call reassoc double @llvm.vector.reduce.fadd.v2f64(double -0.000000e+00, <2 x double> [[TMP12]])
; CHECK-NEXT: store double [[TMP13]], ptr [[OUT]], align 8
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list