[llvm] [SLP] Account for GEP pointer-chain cost when root scalars feed load/store indices (PR #192726)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 06:10:01 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/192726
>From 4b338164e2374e6f48f569d26bfa1132effa5a42 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 17 Apr 2026 13:05:08 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 45 +++++++++++++++++++
.../SLPVectorizer/AArch64/getelementptr.ll | 18 ++------
.../SLPVectorizer/X86/minimum-sizes.ll | 30 +++++--------
3 files changed, 60 insertions(+), 33 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6625384616c26..627cc37d44c40 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18543,9 +18543,29 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
// On AArch64, this helps in fusing a mov instruction, associated with
// extractelement, with fmul in the backend so that extractelement is free.
SmallVector<std::tuple<Value *, User *, int>, 4> ScalarUserAndIdx;
+ bool AllUsersGEPSWithStoresLoads = true;
+ SmallBitVector UsedLanes(VectorizableTree.front()->getVectorFactor());
+ SmallVector<const Value *> Pointers;
+ Type *UserScalarTy = nullptr;
for (ExternalUser &EU : ExternalUses) {
ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
+ if (EU.E.Idx == 0) {
+ UsedLanes.set(EU.Lane);
+ auto *User = dyn_cast_if_present<GetElementPtrInst>(EU.User);
+ if (User && User->hasOneUse() &&
+ isa<LoadInst, StoreInst>(User->user_back())) {
+ Type *LocalTy = getValueType(User->user_back());
+ if (!UserScalarTy)
+ UserScalarTy = LocalTy;
+ else if (UserScalarTy != LocalTy)
+ AllUsersGEPSWithStoresLoads = false;
+ Pointers.push_back(User);
+ } else {
+ AllUsersGEPSWithStoresLoads = false;
+ }
+ }
}
+ AllUsersGEPSWithStoresLoads &= UsedLanes.all();
SmallDenseSet<std::pair<Value *, Value *>, 8> CheckedScalarUser;
for (ExternalUser &EU : ExternalUses) {
LLVM_DEBUG(dbgs() << "SLP: Computing cost for external use of TreeEntry "
@@ -18818,6 +18838,31 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ExtractCost += ExtraCost;
}
+ // Charge the pointer-chain cost difference once for the root entry when
+ // every external use of its scalars is a GEP feeding a single load/store
+ // (see the detection loop above). Vectorizing the root in this pattern
+ // forces lane extracts (or a vector GEP with unknown stride) to drive the
+ // address computation, which is typically more expensive than keeping the
+ // indices scalar in a unit-stride address chain. Add the delta once rather
+ // than per external use.
+ if (AllUsersGEPSWithStoresLoads && !Pointers.empty()) {
+ const TreeEntry &RootEntry = *VectorizableTree.front();
+ const bool AnyRootKeptAsScalar = any_of(RootEntry.Scalars, [&](Value *V) {
+ return ExternalUsesAsOriginalScalar.contains(V);
+ });
+ if (!AnyRootKeptAsScalar) {
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ Type *RootScalarTy = RootEntry.Scalars.front()->getType();
+ auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
+ InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
+ Pointers, Pointers.front(), TTI::PointersChainInfo::getUnitStride(),
+ RootScalarTy, CostKind);
+ InstructionCost VectorGEPCost = TTI->getPointersChainCost(
+ Pointers, Pointers.front(),
+ TTI::PointersChainInfo::getUnknownStride(), VecTy, CostKind);
+ ExtractCost += ScaleCost(VectorGEPCost - ScalarGEPCost, RootEntry);
+ }
+ }
// Insert externals for extract of operands of casts to be emitted as scalars
// instead of extractelement.
for (Value *V : ScalarOpsFromCasts) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 6848ca42e064b..956e41005f5ef 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -30,15 +30,7 @@
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '1'
-; YAML: --- !Passed
-; YAML-NEXT: Pass: slp-vectorizer
-; YAML-NEXT: Name: VectorizedList
-; YAML-NEXT: Function: getelementptr_4x32
-; YAML-NEXT: Args:
-; YAML-NEXT: - String: 'SLP vectorized with cost '
-; YAML-NEXT: - Cost: '12'
-; YAML-NEXT: - String: ' and with tree size '
-; YAML-NEXT: - TreeSize: '3'
+; YAML: --- !Missed
define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
; CHECK-LABEL: @getelementptr_4x32(
@@ -46,7 +38,6 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
; CHECK-NEXT: br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
; CHECK: for.body.preheader:
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.cond.cleanup.loopexit:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
@@ -57,13 +48,10 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD16]], [[FOR_BODY]] ]
; CHECK-NEXT: [[T4:%.*]] = shl nsw i32 [[TMP15]], 1
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[T4]], 0
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP12]]
; CHECK-NEXT: [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = add nsw i32 [[T4]], [[X:%.*]]
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
; CHECK-NEXT: [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = add nsw i32 [[T4]], [[Y:%.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 58fb5f772207d..3b085ee821001 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -17,15 +17,12 @@ target triple = "x86_64-unknown-linux-gnu"
define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
; SSE-LABEL: @PR31243_zext(
; SSE-NEXT: entry:
-; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
-; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
-; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
-; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
-; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
-; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
+; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3:%.*]] to i32
+; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5:%.*]] to i32
+; SSE-NEXT: [[T2:%.*]] = or i32 [[TMP4]], 1
+; SSE-NEXT: [[T3:%.*]] = or i32 [[TMP6]], 1
+; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[T2]]
+; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[T3]]
; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1
; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1
; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -76,15 +73,12 @@ entry:
define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
; SSE-LABEL: @PR31243_sext(
; SSE-NEXT: entry:
-; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
-; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
-; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
-; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
-; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
-; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
+; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3:%.*]] to i32
+; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5:%.*]] to i32
+; SSE-NEXT: [[T2:%.*]] = or i32 [[TMP4]], 1
+; SSE-NEXT: [[T3:%.*]] = or i32 [[TMP6]], 1
+; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[T2]]
+; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[T3]]
; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1
; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1
; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]]
>From 891b9e53bf59c8a47e3ea4502a9072b76be55aa8 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 17 Apr 2026 13:41:40 -0700
Subject: [PATCH 2/2] Address comments
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 22 ++++++++++++++-----
.../SLPVectorizer/AArch64/getelementptr.ll | 18 ++++++++++++---
2 files changed, 31 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 627cc37d44c40..2a3db89a4c473 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18850,16 +18850,26 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
const bool AnyRootKeptAsScalar = any_of(RootEntry.Scalars, [&](Value *V) {
return ExternalUsesAsOriginalScalar.contains(V);
});
- if (!AnyRootKeptAsScalar) {
+ const Value *CommonBase = nullptr;
+ bool HaveCommonBase = true;
+ for (const Value *P : Pointers) {
+ const Value *Op = cast<GetElementPtrInst>(P)->getPointerOperand();
+ if (!CommonBase)
+ CommonBase = Op;
+ else if (CommonBase != Op) {
+ HaveCommonBase = false;
+ break;
+ }
+ }
+ if (!AnyRootKeptAsScalar && HaveCommonBase) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
- Type *RootScalarTy = RootEntry.Scalars.front()->getType();
auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
- Pointers, Pointers.front(), TTI::PointersChainInfo::getUnitStride(),
- RootScalarTy, CostKind);
+ Pointers, CommonBase, TTI::PointersChainInfo::getUnitStride(),
+ UserScalarTy, CostKind);
InstructionCost VectorGEPCost = TTI->getPointersChainCost(
- Pointers, Pointers.front(),
- TTI::PointersChainInfo::getUnknownStride(), VecTy, CostKind);
+ Pointers, CommonBase, TTI::PointersChainInfo::getUnknownStride(),
+ VecTy, CostKind);
ExtractCost += ScaleCost(VectorGEPCost - ScalarGEPCost, RootEntry);
}
}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 956e41005f5ef..6848ca42e064b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -30,7 +30,15 @@
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '1'
-; YAML: --- !Missed
+; YAML: --- !Passed
+; YAML-NEXT: Pass: slp-vectorizer
+; YAML-NEXT: Name: VectorizedList
+; YAML-NEXT: Function: getelementptr_4x32
+; YAML-NEXT: Args:
+; YAML-NEXT: - String: 'SLP vectorized with cost '
+; YAML-NEXT: - Cost: '12'
+; YAML-NEXT: - String: ' and with tree size '
+; YAML-NEXT: - TreeSize: '3'
define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
; CHECK-LABEL: @getelementptr_4x32(
@@ -38,6 +46,7 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
; CHECK-NEXT: br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
; CHECK: for.body.preheader:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.cond.cleanup.loopexit:
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
@@ -48,10 +57,13 @@ define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y
; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD16]], [[FOR_BODY]] ]
; CHECK-NEXT: [[T4:%.*]] = shl nsw i32 [[TMP15]], 1
-; CHECK-NEXT: [[TMP12:%.*]] = add nsw i32 [[T4]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP12]]
; CHECK-NEXT: [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = add nsw i32 [[T4]], [[X:%.*]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
; CHECK-NEXT: [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
; CHECK-NEXT: [[TMP13:%.*]] = add nsw i32 [[T4]], [[Y:%.*]]
More information about the llvm-commits
mailing list