[llvm] 6b4f024 - [SLP] Account for GEP pointer-chain cost when root scalars feed load/store indices
via llvm-commits
llvm-commits at lists.llvm.org
Fri May 8 06:10:30 PDT 2026
Author: Alexey Bataev
Date: 2026-05-08T09:10:25-04:00
New Revision: 6b4f02426d53578871a11b6ce94553bc9931223f
URL: https://github.com/llvm/llvm-project/commit/6b4f02426d53578871a11b6ce94553bc9931223f
DIFF: https://github.com/llvm/llvm-project/commit/6b4f02426d53578871a11b6ce94553bc9931223f.diff
LOG: [SLP] Account for GEP pointer-chain cost when root scalars feed load/store indices
When every external use of the root TreeEntry's scalars is a GEP with a
single load or store user (sharing one access type) and all lanes are
consumed this way, charge the delta between the vector (unknown stride)
and scalar (unit stride) pointer-chain costs once via
TTI::getPointersChainCost, scaled for the root entry. Vectorizing such
a root forces lane extracts or a vector GEP to drive address
computation, which is typically more expensive than keeping the indices
scalar in a unit-stride address chain.
Reviewers: hiraditya, bababuck, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/192726
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9cfeae0b8bee7..eec6499c7b724 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19173,9 +19173,32 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
// On AArch64, this helps in fusing a mov instruction, associated with
// extractelement, with fmul in the backend so that extractelement is free.
SmallVector<std::tuple<Value *, User *, int>, 4> ScalarUserAndIdx;
+ bool AllUsersGEPSWithStoresLoads = true;
+ SmallBitVector UsedLanes(VectorizableTree.front()->getVectorFactor());
+ SmallVector<const Value *> Pointers;
+ Type *UserScalarTy = nullptr;
for (ExternalUser &EU : ExternalUses) {
ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
+ if (EU.E.Idx == 0) {
+ UsedLanes.set(EU.Lane);
+ auto *User = dyn_cast_if_present<GetElementPtrInst>(EU.User);
+ if (User && User->hasOneUse() &&
+ isa<LoadInst, StoreInst>(User->user_back())) {
+ Type *LocalTy = getValueType(User->user_back());
+ if (!UserScalarTy) {
+ UserScalarTy = LocalTy;
+ } else if (UserScalarTy != LocalTy) {
+ AllUsersGEPSWithStoresLoads = false;
+ break;
+ }
+ Pointers.push_back(User);
+ } else {
+ AllUsersGEPSWithStoresLoads = false;
+ break;
+ }
+ }
}
+ AllUsersGEPSWithStoresLoads &= UsedLanes.all();
SmallDenseSet<std::pair<Value *, Value *>, 8> CheckedScalarUser;
for (ExternalUser &EU : ExternalUses) {
LLVM_DEBUG(dbgs() << "SLP: Computing cost for external use of TreeEntry "
@@ -19447,6 +19470,41 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ExtractCost += ExtraCost;
}
+ // Charge the pointer-chain cost
diff erence once for the root entry when
+ // every external use of its scalars is a GEP feeding a single load/store
+ // (see the detection loop above). Vectorizing the root in this pattern
+ // forces lane extracts (or a vector GEP with unknown stride) to drive the
+ // address computation, which is typically more expensive than keeping the
+ // indices scalar in a unit-stride address chain. Add the delta once rather
+ // than per external use.
+ if (AllUsersGEPSWithStoresLoads && !Pointers.empty()) {
+ const TreeEntry &RootEntry = *VectorizableTree.front();
+ const bool AnyRootKeptAsScalar = any_of(RootEntry.Scalars, [&](Value *V) {
+ return ExternalUsesAsOriginalScalar.contains(V);
+ });
+ const Value *CommonBase = nullptr;
+ bool HaveCommonBase = true;
+ for (const Value *P : Pointers) {
+ const Value *Op = getUnderlyingObject(P);
+ if (!CommonBase)
+ CommonBase = Op;
+ else if (CommonBase != Op) {
+ HaveCommonBase = false;
+ break;
+ }
+ }
+ if (!AnyRootKeptAsScalar && HaveCommonBase) {
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
+ InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
+ Pointers, CommonBase, TTI::PointersChainInfo::getUnitStride(),
+ UserScalarTy, CostKind);
+ InstructionCost VectorGEPCost = TTI->getPointersChainCost(
+ Pointers, CommonBase, TTI::PointersChainInfo::getUnknownStride(),
+ VecTy, CostKind);
+ ExtractCost += ScaleCost(VectorGEPCost - ScalarGEPCost, RootEntry);
+ }
+ }
// Insert externals for extract of operands of casts to be emitted as scalars
// instead of extractelement.
for (Value *V : ScalarOpsFromCasts) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
index 58fb5f772207d..3b085ee821001 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/minimum-sizes.ll
@@ -17,15 +17,12 @@ target triple = "x86_64-unknown-linux-gnu"
define i8 @PR31243_zext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
; SSE-LABEL: @PR31243_zext(
; SSE-NEXT: entry:
-; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
-; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
-; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3]] to i32
-; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
-; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5]] to i32
-; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
+; SSE-NEXT: [[TMP4:%.*]] = zext i8 [[TMP3:%.*]] to i32
+; SSE-NEXT: [[TMP6:%.*]] = zext i8 [[TMP5:%.*]] to i32
+; SSE-NEXT: [[T2:%.*]] = or i32 [[TMP4]], 1
+; SSE-NEXT: [[T3:%.*]] = or i32 [[TMP6]], 1
+; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[T2]]
+; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[T3]]
; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1
; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1
; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]]
@@ -76,15 +73,12 @@ entry:
define i8 @PR31243_sext(i8 %v0, i8 %v1, i8 %v2, i8 %v3, ptr %ptr) {
; SSE-LABEL: @PR31243_sext(
; SSE-NEXT: entry:
-; SSE-NEXT: [[TMP0:%.*]] = insertelement <2 x i8> poison, i8 [[V0:%.*]], i32 0
-; SSE-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> [[TMP0]], i8 [[V1:%.*]], i32 1
-; SSE-NEXT: [[TMP2:%.*]] = or <2 x i8> [[TMP1]], splat (i8 1)
-; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3]] to i32
-; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[TMP4]]
-; SSE-NEXT: [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5]] to i32
-; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[TMP6]]
+; SSE-NEXT: [[TMP4:%.*]] = sext i8 [[TMP3:%.*]] to i32
+; SSE-NEXT: [[TMP6:%.*]] = sext i8 [[TMP5:%.*]] to i32
+; SSE-NEXT: [[T2:%.*]] = or i32 [[TMP4]], 1
+; SSE-NEXT: [[T3:%.*]] = or i32 [[TMP6]], 1
+; SSE-NEXT: [[T4:%.*]] = getelementptr inbounds i8, ptr [[PTR:%.*]], i32 [[T2]]
+; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 [[T3]]
; SSE-NEXT: [[T6:%.*]] = load i8, ptr [[T4]], align 1
; SSE-NEXT: [[T7:%.*]] = load i8, ptr [[T5]], align 1
; SSE-NEXT: [[T8:%.*]] = add i8 [[T6]], [[T7]]
More information about the llvm-commits
mailing list