[llvm] 5d97ece - Revert "[SLP]Extend GEP pointer-chain cost to casts and non-root external uses" (#217546)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 01:22:08 PDT 2026
Author: Madhur Amilkanthwar
Date: 2026-08-20T08:22:02Z
New Revision: 5d97ecebb0de2d20db209a88f2a3489078c1773a
URL: https://github.com/llvm/llvm-project/commit/5d97ecebb0de2d20db209a88f2a3489078c1773a
DIFF: https://github.com/llvm/llvm-project/commit/5d97ecebb0de2d20db209a88f2a3489078c1773a.diff
LOG: Revert "[SLP]Extend GEP pointer-chain cost to casts and non-root external uses" (#217546)
Reverts llvm/llvm-project#216520 as it is causing regressing in one of
the SPEC'26 benchmark.
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e6e792573a958..fc17f40b49e7c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19863,33 +19863,32 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
// On AArch64, this helps in fusing a mov instruction, associated with
// extractelement, with fmul in the backend so that extractelement is free.
SmallVector<std::tuple<Value *, User *, int>, 4> ScalarUserAndIdx;
- // Detect external uses that drive address computations: the scalar (through
- // an optional single-use index-promotion cast) is used as a GEP index.
bool AllUsersGEPSWithStoresLoads = true;
+ SmallBitVector UsedLanes(getRootNode().getVectorFactor());
SmallVector<const Value *> Pointers;
Type *UserScalarTy = nullptr;
for (ExternalUser &EU : ExternalUses) {
ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
- Value *Usr = EU.User;
- if (Usr && match(Usr, m_OneUse(m_ZExtOrSExt(m_Value()))))
- Usr = cast<Instruction>(Usr)->user_back();
- auto *User = dyn_cast_if_present<GetElementPtrInst>(Usr);
- // Only a GEP that feeds a single load/store of a fixed access type drives
- // a real memory address computation.
- Type *AccessTy = nullptr;
- if (User && User->hasOneUse() &&
- isa<LoadInst, StoreInst>(User->user_back()))
- AccessTy = getValueType(User->user_back());
- if (!AccessTy || isa<ScalableVectorType>(AccessTy))
- User = nullptr;
- if (User && (!UserScalarTy || UserScalarTy == AccessTy)) {
- UserScalarTy = AccessTy;
- Pointers.push_back(User);
- } else {
- AllUsersGEPSWithStoresLoads = false;
- break;
+ if (EU.E.Idx == 0) {
+ UsedLanes.set(EU.Lane);
+ auto *User = dyn_cast_if_present<GetElementPtrInst>(EU.User);
+ if (User && User->hasOneUse() &&
+ isa<LoadInst, StoreInst>(User->user_back())) {
+ Type *LocalTy = getValueType(User->user_back());
+ if (!UserScalarTy && !isa<ScalableVectorType>(LocalTy)) {
+ UserScalarTy = LocalTy;
+ } else if (UserScalarTy != LocalTy) {
+ AllUsersGEPSWithStoresLoads = false;
+ break;
+ }
+ Pointers.push_back(User);
+ } else {
+ AllUsersGEPSWithStoresLoads = false;
+ break;
+ }
}
}
+ AllUsersGEPSWithStoresLoads &= UsedLanes.all();
// Pre-pass: for each externally-used scalar, find the basic block at which
// the extractelement will be placed by codegen. This mirrors what
@@ -20265,14 +20264,18 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ExtractCost += ExtraCost;
}
- // Charge the pointer-chain cost
diff erence once when every escaped scalar
- // is used only to drive an address computation (see the detection loop
- // above). Vectorizing the tree in this pattern forces lane extracts (or a
- // vector GEP with unknown stride) to drive the address computation, which is
- // typically more expensive than keeping the indices scalar in a unit-stride
- // address chain. Add the delta once rather than per external use.
+ // Charge the pointer-chain cost
diff erence once for the root entry when
+ // every external use of its scalars is a GEP feeding a single load/store
+ // (see the detection loop above). Vectorizing the root in this pattern
+ // forces lane extracts (or a vector GEP with unknown stride) to drive the
+ // address computation, which is typically more expensive than keeping the
+ // indices scalar in a unit-stride address chain. Add the delta once rather
+ // than per external use.
if (AllUsersGEPSWithStoresLoads && !Pointers.empty()) {
const TreeEntry &RootEntry = getRootNode();
+ const bool AnyRootKeptAsScalar = any_of(RootEntry.Scalars, [&](Value *V) {
+ return ExternalUsesAsOriginalScalar.contains(V);
+ });
const Value *CommonBase = nullptr;
bool HaveCommonBase = true;
for (const Value *P : Pointers) {
@@ -20284,7 +20287,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
break;
}
}
- if (HaveCommonBase) {
+ if (!AnyRootKeptAsScalar && HaveCommonBase) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
index c39be0ed960ce..dcb93453b5520 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
@@ -13,45 +13,21 @@ define i32 @test(ptr %this, i32 %a, i32 %b) {
; CHECK-NEXT: tail call void @deopt()
; CHECK-NEXT: unreachable
; CHECK: [[IF_END]]:
-; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[B]], [[A]]
; CHECK-NEXT: [[LENGTH:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LENGTH]], align 8
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[B]], [[A]]
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[ADD]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = lshr <8 x i32> [[TMP3]], <i32 0, i32 4, i32 8, i32 12, i32 16, i32 20, i32 24, i32 0>
+; CHECK-NEXT: [[TMP5:%.*]] = and <8 x i32> [[TMP4]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 28>
+; CHECK-NEXT: [[TMP6:%.*]] = lshr <8 x i32> [[TMP4]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 28>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
; CHECK-NEXT: [[AND:%.*]] = and i32 [[ADD]], 15
-; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 4
-; CHECK-NEXT: [[AND1:%.*]] = and i32 [[SHR]], 15
-; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i32 [[AND1]], 16
-; CHECK-NEXT: [[SHR2:%.*]] = lshr i32 [[ADD]], 8
-; CHECK-NEXT: [[AND3:%.*]] = and i32 [[SHR2]], 15
-; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i32 [[AND3]], 32
-; CHECK-NEXT: [[SHR5:%.*]] = lshr i32 [[ADD]], 12
-; CHECK-NEXT: [[AND6:%.*]] = and i32 [[SHR5]], 15
-; CHECK-NEXT: [[TMP18:%.*]] = or disjoint i32 [[AND6]], 48
-; CHECK-NEXT: [[SHR8:%.*]] = lshr i32 [[ADD]], 16
-; CHECK-NEXT: [[AND9:%.*]] = and i32 [[SHR8]], 15
-; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i32 [[AND9]], 64
-; CHECK-NEXT: [[SHR11:%.*]] = lshr i32 [[ADD]], 20
-; CHECK-NEXT: [[AND12:%.*]] = and i32 [[SHR11]], 15
-; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i32 [[AND12]], 80
-; CHECK-NEXT: [[SHR14:%.*]] = lshr i32 [[ADD]], 24
-; CHECK-NEXT: [[AND15:%.*]] = and i32 [[SHR14]], 15
-; CHECK-NEXT: [[TMP24:%.*]] = or disjoint i32 [[AND15]], 96
-; CHECK-NEXT: [[SHR17:%.*]] = lshr i32 [[ADD]], 28
-; CHECK-NEXT: [[TMP26:%.*]] = or disjoint i32 [[SHR17]], 112
-; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[AND]], [[TMP1]]
-; CHECK-NEXT: [[CMP19:%.*]] = icmp ult i32 [[TMP14]], [[TMP1]]
-; CHECK-NEXT: [[AND23167:%.*]] = and i1 [[CMP]], [[CMP19]]
-; CHECK-NEXT: [[CMP26:%.*]] = icmp ult i32 [[TMP16]], [[TMP1]]
-; CHECK-NEXT: [[AND33168:%.*]] = and i1 [[CMP26]], [[AND23167]]
-; CHECK-NEXT: [[CMP36:%.*]] = icmp ult i32 [[TMP18]], [[TMP1]]
-; CHECK-NEXT: [[AND43169:%.*]] = and i1 [[CMP36]], [[AND33168]]
-; CHECK-NEXT: [[CMP46:%.*]] = icmp ult i32 [[TMP20]], [[TMP1]]
-; CHECK-NEXT: [[AND53170:%.*]] = and i1 [[CMP46]], [[AND43169]]
-; CHECK-NEXT: [[CMP56:%.*]] = icmp ult i32 [[TMP22]], [[TMP1]]
-; CHECK-NEXT: [[AND63171:%.*]] = and i1 [[CMP56]], [[AND53170]]
-; CHECK-NEXT: [[CMP66:%.*]] = icmp ult i32 [[TMP24]], [[TMP1]]
-; CHECK-NEXT: [[AND73172:%.*]] = and i1 [[CMP66]], [[AND63171]]
-; CHECK-NEXT: [[CMP76:%.*]] = icmp ult i32 [[TMP26]], [[TMP1]]
-; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP76]], [[AND73172]]
+; CHECK-NEXT: [[TMP8:%.*]] = or disjoint <8 x i32> [[TMP7]], <i32 0, i32 16, i32 32, i32 48, i32 64, i32 80, i32 96, i32 112>
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i32> [[TMP9]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ult <8 x i32> [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP11]])
; CHECK-NEXT: br i1 [[TMP12]], label %[[IF_END88:.*]], label %[[IF_THEN87:.*]]
; CHECK: [[IF_THEN87]]:
; CHECK-NEXT: tail call void @deopt()
@@ -62,42 +38,49 @@ define i32 @test(ptr %this, i32 %a, i32 %b) {
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM]]
; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
; CHECK-NEXT: [[CONV89:%.*]] = sext i8 [[TMP13]] to i32
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[TMP8]], i64 1
; CHECK-NEXT: [[IDXPROM90:%.*]] = zext nneg i32 [[TMP14]] to i64
; CHECK-NEXT: [[ARRAYIDX91:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM90]]
; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX91]], align 1
; CHECK-NEXT: [[CONV92:%.*]] = sext i8 [[TMP15]] to i32
; CHECK-NEXT: [[SHL93:%.*]] = shl nsw i32 [[CONV92]], 4
; CHECK-NEXT: [[ADD94:%.*]] = add nsw i32 [[SHL93]], [[CONV89]]
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i32> [[TMP8]], i64 2
; CHECK-NEXT: [[IDXPROM95:%.*]] = zext nneg i32 [[TMP16]] to i64
; CHECK-NEXT: [[ARRAYIDX96:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM95]]
; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX96]], align 1
; CHECK-NEXT: [[CONV97:%.*]] = sext i8 [[TMP17]] to i32
; CHECK-NEXT: [[SHL98:%.*]] = shl nsw i32 [[CONV97]], 8
; CHECK-NEXT: [[ADD99:%.*]] = add nsw i32 [[ADD94]], [[SHL98]]
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[TMP8]], i64 3
; CHECK-NEXT: [[IDXPROM100:%.*]] = zext nneg i32 [[TMP18]] to i64
; CHECK-NEXT: [[ARRAYIDX101:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM100]]
; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX101]], align 1
; CHECK-NEXT: [[CONV102:%.*]] = sext i8 [[TMP19]] to i32
; CHECK-NEXT: [[SHL103:%.*]] = shl nsw i32 [[CONV102]], 12
; CHECK-NEXT: [[ADD104:%.*]] = add nsw i32 [[ADD99]], [[SHL103]]
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <8 x i32> [[TMP8]], i64 4
; CHECK-NEXT: [[IDXPROM105:%.*]] = zext nneg i32 [[TMP20]] to i64
; CHECK-NEXT: [[ARRAYIDX106:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM105]]
; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX106]], align 1
; CHECK-NEXT: [[CONV107:%.*]] = sext i8 [[TMP21]] to i32
; CHECK-NEXT: [[SHL108:%.*]] = shl nsw i32 [[CONV107]], 16
; CHECK-NEXT: [[ADD109:%.*]] = add nsw i32 [[ADD104]], [[SHL108]]
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[TMP8]], i64 5
; CHECK-NEXT: [[IDXPROM110:%.*]] = zext nneg i32 [[TMP22]] to i64
; CHECK-NEXT: [[ARRAYIDX111:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM110]]
; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX111]], align 1
; CHECK-NEXT: [[CONV112:%.*]] = sext i8 [[TMP23]] to i32
; CHECK-NEXT: [[SHL113:%.*]] = shl nsw i32 [[CONV112]], 20
; CHECK-NEXT: [[ADD114:%.*]] = add nsw i32 [[ADD109]], [[SHL113]]
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <8 x i32> [[TMP8]], i64 6
; CHECK-NEXT: [[IDXPROM115:%.*]] = zext nneg i32 [[TMP24]] to i64
; CHECK-NEXT: [[ARRAYIDX116:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM115]]
; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX116]], align 1
; CHECK-NEXT: [[CONV117:%.*]] = sext i8 [[TMP25]] to i32
; CHECK-NEXT: [[SHL118:%.*]] = shl nsw i32 [[CONV117]], 24
; CHECK-NEXT: [[ADD119:%.*]] = add nsw i32 [[ADD114]], [[SHL118]]
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP8]], i64 7
; CHECK-NEXT: [[IDXPROM120:%.*]] = zext nneg i32 [[TMP26]] to i64
; CHECK-NEXT: [[ARRAYIDX121:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM120]]
; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX121]], align 1
More information about the llvm-commits
mailing list