[llvm] [SLP] Fix GEP cost computation for load vectorization cost estimates (PR #191728)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 12 09:36:03 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/191728
Pass Instruction::Load instead of Instruction::GetElementPtr to
getGEPCosts in isMaskedLoadCompress and CheckForShuffledLoads.
These call sites estimate costs for wide contiguous loads and sub-vector
load patterns, not for masked gather pointer vector formation. Using
Instruction::GetElementPtr incorrectly triggered the gather-style cost
path, which computes vector GEP formation costs. Since the call sites
already add scalarization overhead for pointer vector building
separately, this led to double-counting of pointer costs and inaccurate
vectorization decisions.
>From 0a3b764a207970e48e85998d68ead3a6a82e3fea Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 12 Apr 2026 09:35:51 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 12 +++++-----
.../X86/phi-operand-gathered-loads.ll | 22 ++++++++++++++-----
2 files changed, 21 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9fac57f3aca2a..0e647fa235916 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7084,7 +7084,7 @@ static bool isMaskedLoadCompress(
reorderScalars(OrderedPointerOps, Mask);
auto [ScalarGEPCost, VectorGEPCost] =
getGEPCosts(TTI, OrderedPointerOps, OrderedPointerOps.front(),
- Instruction::GetElementPtr, CostKind, ScalarTy, LoadVecTy);
+ Instruction::Load, CostKind, ScalarTy, LoadVecTy);
// The cost of scalar loads.
InstructionCost ScalarLoadsCost =
std::accumulate(VL.begin(), VL.end(), InstructionCost(),
@@ -7621,8 +7621,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
// Compare masked gather cost and loads + insert subvector costs.
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
auto [ScalarGEPCost, VectorGEPCost] =
- getGEPCosts(TTI, PointerOps, PointerOps.front(),
- Instruction::GetElementPtr, CostKind, ScalarTy, VecTy);
+ getGEPCosts(TTI, PointerOps, PointerOps.front(), Instruction::Load,
+ CostKind, ScalarTy, VecTy);
// Estimate the cost of masked gather GEP. If not a splat, roughly
// estimate as a buildvector, otherwise estimate as splat.
APInt DemandedElts = APInt::getAllOnes(Sz);
@@ -7732,9 +7732,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
(LS == LoadsState::ScatterVectorize && ProfitableGatherPointers)
? 0
: getGEPCosts(TTI, ArrayRef(PointerOps).slice(I * VF, VF),
- LI0->getPointerOperand(),
- Instruction::GetElementPtr, CostKind, ScalarTy,
- SubVecTy)
+ LI0->getPointerOperand(), Instruction::Load,
+ CostKind, ScalarTy, SubVecTy)
.second;
if (LS == LoadsState::ScatterVectorize) {
if (static_cast<unsigned>(
@@ -16737,7 +16736,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
Scalars, PointerOps, E->ReorderIndices, *TTI, *DL, *SE, *AC, *DT,
*TLI, [](Value *) { return true; }, IsMasked, InterleaveFactor,
CompressMask, LoadVecTy);
- assert(IsVectorized && "Failed to vectorize load");
CompressEntryToData.try_emplace(E, CompressMask, LoadVecTy,
InterleaveFactor, IsMasked);
Align CommonAlignment = LI0->getAlign();
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
index 1fe1217d361a8..1680600a0fc0e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
@@ -14,13 +14,23 @@ define void @test(ptr %this, i1 %cmp4.not) {
; CHECK: [[IF_ELSE37]]:
; CHECK-NEXT: br label %[[IF_END46]]
; CHECK: [[IF_END46]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 160, i64 1, i64 0, i64 1>, %[[IF_ELSE37]] ], [ <i64 0, i64 0, i64 1, i64 0>, %[[ENTRY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x ptr> poison, ptr [[THIS]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x ptr> [[TMP1]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, <4 x ptr> [[TMP2]], <4 x i64> [[TMP0]]
+; CHECK-NEXT: [[DOTSINK264:%.*]] = phi i64 [ 160, %[[IF_ELSE37]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOTSINK262:%.*]] = phi i64 [ 0, %[[IF_ELSE37]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOTSINK261:%.*]] = phi i64 [ 1, %[[IF_ELSE37]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[M_PARTID038:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK264]]
+; CHECK-NEXT: [[M_INDEX042:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK262]]
+; CHECK-NEXT: [[M_INDEX144:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK261]]
+; CHECK-NEXT: [[DOTSINK:%.*]] = load i32, ptr [[M_INDEX144]], align 4
+; CHECK-NEXT: [[DOTSINK186:%.*]] = load i32, ptr [[M_INDEX042]], align 4
+; CHECK-NEXT: [[DOTSINK188:%.*]] = load i32, ptr [[M_PARTID038]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 92
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store i32 [[DOTSINK188]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 96
+; CHECK-NEXT: store i32 [[DOTSINK]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 100
+; CHECK-NEXT: store i32 [[DOTSINK186]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 104
+; CHECK-NEXT: store i32 [[DOTSINK]], ptr [[TMP3]], align 8
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list