[llvm] 183660d - [SLP] Fix GEP cost computation for load vectorization cost estimates
via llvm-commits
llvm-commits at lists.llvm.org
Sun Apr 12 09:36:23 PDT 2026
Author: Alexey Bataev
Date: 2026-04-12T12:36:19-04:00
New Revision: 183660de3a29cbd69f51730ee2044437edcaf32a
URL: https://github.com/llvm/llvm-project/commit/183660de3a29cbd69f51730ee2044437edcaf32a
DIFF: https://github.com/llvm/llvm-project/commit/183660de3a29cbd69f51730ee2044437edcaf32a.diff
LOG: [SLP] Fix GEP cost computation for load vectorization cost estimates
Pass Instruction::Load instead of Instruction::GetElementPtr to
getGEPCosts in isMaskedLoadCompress and CheckForShuffledLoads.
These call sites estimate costs for wide contiguous loads and sub-vector
load patterns, not for masked gather pointer vector formation. Using
Instruction::GetElementPtr incorrectly triggered the gather-style cost
path, which computes vector GEP formation costs. Since the call sites
already add scalarization overhead for pointer vector building
separately, this led to double-counting of pointer costs and inaccurate
vectorization decisions.
Reviewers: hiraditya, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/191728
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9fac57f3aca2a..0e647fa235916 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -7084,7 +7084,7 @@ static bool isMaskedLoadCompress(
reorderScalars(OrderedPointerOps, Mask);
auto [ScalarGEPCost, VectorGEPCost] =
getGEPCosts(TTI, OrderedPointerOps, OrderedPointerOps.front(),
- Instruction::GetElementPtr, CostKind, ScalarTy, LoadVecTy);
+ Instruction::Load, CostKind, ScalarTy, LoadVecTy);
// The cost of scalar loads.
InstructionCost ScalarLoadsCost =
std::accumulate(VL.begin(), VL.end(), InstructionCost(),
@@ -7621,8 +7621,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
// Compare masked gather cost and loads + insert subvector costs.
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
auto [ScalarGEPCost, VectorGEPCost] =
- getGEPCosts(TTI, PointerOps, PointerOps.front(),
- Instruction::GetElementPtr, CostKind, ScalarTy, VecTy);
+ getGEPCosts(TTI, PointerOps, PointerOps.front(), Instruction::Load,
+ CostKind, ScalarTy, VecTy);
// Estimate the cost of masked gather GEP. If not a splat, roughly
// estimate as a buildvector, otherwise estimate as splat.
APInt DemandedElts = APInt::getAllOnes(Sz);
@@ -7732,9 +7732,8 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads(
(LS == LoadsState::ScatterVectorize && ProfitableGatherPointers)
? 0
: getGEPCosts(TTI, ArrayRef(PointerOps).slice(I * VF, VF),
- LI0->getPointerOperand(),
- Instruction::GetElementPtr, CostKind, ScalarTy,
- SubVecTy)
+ LI0->getPointerOperand(), Instruction::Load,
+ CostKind, ScalarTy, SubVecTy)
.second;
if (LS == LoadsState::ScatterVectorize) {
if (static_cast<unsigned>(
@@ -16737,7 +16736,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
Scalars, PointerOps, E->ReorderIndices, *TTI, *DL, *SE, *AC, *DT,
*TLI, [](Value *) { return true; }, IsMasked, InterleaveFactor,
CompressMask, LoadVecTy);
- assert(IsVectorized && "Failed to vectorize load");
CompressEntryToData.try_emplace(E, CompressMask, LoadVecTy,
InterleaveFactor, IsMasked);
Align CommonAlignment = LI0->getAlign();
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
index 1fe1217d361a8..1680600a0fc0e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-operand-gathered-loads.ll
@@ -14,13 +14,23 @@ define void @test(ptr %this, i1 %cmp4.not) {
; CHECK: [[IF_ELSE37]]:
; CHECK-NEXT: br label %[[IF_END46]]
; CHECK: [[IF_END46]]:
-; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i64> [ <i64 160, i64 1, i64 0, i64 1>, %[[IF_ELSE37]] ], [ <i64 0, i64 0, i64 1, i64 0>, %[[ENTRY]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x ptr> poison, ptr [[THIS]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x ptr> [[TMP1]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, <4 x ptr> [[TMP2]], <4 x i64> [[TMP0]]
+; CHECK-NEXT: [[DOTSINK264:%.*]] = phi i64 [ 160, %[[IF_ELSE37]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOTSINK262:%.*]] = phi i64 [ 0, %[[IF_ELSE37]] ], [ 1, %[[ENTRY]] ]
+; CHECK-NEXT: [[DOTSINK261:%.*]] = phi i64 [ 1, %[[IF_ELSE37]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[M_PARTID038:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK264]]
+; CHECK-NEXT: [[M_INDEX042:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK262]]
+; CHECK-NEXT: [[M_INDEX144:%.*]] = getelementptr i8, ptr [[THIS]], i64 [[DOTSINK261]]
+; CHECK-NEXT: [[DOTSINK:%.*]] = load i32, ptr [[M_INDEX144]], align 4
+; CHECK-NEXT: [[DOTSINK186:%.*]] = load i32, ptr [[M_INDEX042]], align 4
+; CHECK-NEXT: [[DOTSINK188:%.*]] = load i32, ptr [[M_PARTID038]], align 4
; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 92
-; CHECK-NEXT: [[TMP5:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 4 [[TMP3]], <4 x i1> splat (i1 true), <4 x i32> poison)
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[TMP4]], align 4
+; CHECK-NEXT: store i32 [[DOTSINK188]], ptr [[TMP4]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 96
+; CHECK-NEXT: store i32 [[DOTSINK]], ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 100
+; CHECK-NEXT: store i32 [[DOTSINK186]], ptr [[TMP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[NEWPT]], i64 104
+; CHECK-NEXT: store i32 [[DOTSINK]], ptr [[TMP3]], align 8
; CHECK-NEXT: ret void
;
entry:
More information about the llvm-commits
mailing list