[llvm-branch-commits] [llvm] 4dea21e - [SLP]Extend GEP pointer-chain cost to casts and non-root external uses
Tobias Hieta via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri Aug 28 02:05:44 PDT 2026
Author: Karthika Devi C
Date: 2026-08-28T11:05:16+02:00
New Revision: 4dea21e892d795318cfac64a914c72ea7be6ffe5
URL: https://github.com/llvm/llvm-project/commit/4dea21e892d795318cfac64a914c72ea7be6ffe5
DIFF: https://github.com/llvm/llvm-project/commit/4dea21e892d795318cfac64a914c72ea7be6ffe5.diff
LOG: [SLP]Extend GEP pointer-chain cost to casts and non-root external uses
Cherry-pick of 084c5507ee0146a7506f3117868082162760b689 to release/23.x.
Fixes AArch64 regression introduced by 376311097a27a6eb99ec2614e7c39c97ce33172f.
Original Pull Request: #216520
Recommit after perf regression fixes: #217683
Added:
llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9bf48c489d949..3cba5769e0047 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19960,32 +19960,35 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
// On AArch64, this helps in fusing a mov instruction, associated with
// extractelement, with fmul in the backend so that extractelement is free.
SmallVector<std::tuple<Value *, User *, int>, 4> ScalarUserAndIdx;
+ // Record every external use: a missing entry is indistinguishable from
+ // lane 0 and is priced as a free extract by the extract-fusion cost model.
+ for (ExternalUser &EU : ExternalUses)
+ ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
+ // Detect external uses that drive address computations: the scalar (through
+ // an optional single-use index-promotion cast) is used as a GEP index.
bool AllUsersGEPSWithStoresLoads = true;
- SmallBitVector UsedLanes(VectorizableTree.front()->getVectorFactor());
SmallVector<const Value *> Pointers;
Type *UserScalarTy = nullptr;
for (ExternalUser &EU : ExternalUses) {
- ScalarUserAndIdx.emplace_back(EU.Scalar, EU.User, EU.Lane);
- if (EU.E.Idx == 0) {
- UsedLanes.set(EU.Lane);
- auto *User = dyn_cast_if_present<GetElementPtrInst>(EU.User);
- if (User && User->hasOneUse() &&
- isa<LoadInst, StoreInst>(User->user_back())) {
- Type *LocalTy = getValueType(User->user_back());
- if (!UserScalarTy && !isa<ScalableVectorType>(LocalTy)) {
- UserScalarTy = LocalTy;
- } else if (UserScalarTy != LocalTy) {
- AllUsersGEPSWithStoresLoads = false;
- break;
- }
- Pointers.push_back(User);
- } else {
- AllUsersGEPSWithStoresLoads = false;
- break;
- }
+ Value *Usr = EU.User;
+ if (Usr && match(Usr, m_OneUse(m_ZExtOrSExt(m_Value()))))
+ Usr = cast<Instruction>(Usr)->user_back();
+ auto *User = dyn_cast_if_present<GetElementPtrInst>(Usr);
+ // Only a GEP that feeds a single load/store of a fixed access type drives
+ // a real memory address computation.
+ Type *AccessTy = nullptr;
+ if (User && User->hasOneUse() &&
+ isa<LoadInst, StoreInst>(User->user_back()))
+ AccessTy = getValueType(User->user_back());
+ if (AccessTy && !isa<ScalableVectorType>(AccessTy) &&
+ (!UserScalarTy || UserScalarTy == AccessTy)) {
+ UserScalarTy = AccessTy;
+ Pointers.push_back(User);
+ } else {
+ AllUsersGEPSWithStoresLoads = false;
+ break;
}
}
- AllUsersGEPSWithStoresLoads &= UsedLanes.all();
// Pre-pass: for each externally-used scalar, find the basic block at which
// the extractelement will be placed by codegen. This mirrors what
@@ -20362,30 +20365,18 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ExtractCost += ExtraCost;
}
- // Charge the pointer-chain cost
diff erence once for the root entry when
- // every external use of its scalars is a GEP feeding a single load/store
- // (see the detection loop above). Vectorizing the root in this pattern
- // forces lane extracts (or a vector GEP with unknown stride) to drive the
- // address computation, which is typically more expensive than keeping the
- // indices scalar in a unit-stride address chain. Add the delta once rather
- // than per external use.
+ // Charge the pointer-chain cost
diff erence once when every escaped scalar
+ // is used only to drive an address computation (see the detection loop
+ // above). Vectorizing the tree in this pattern forces lane extracts (or a
+ // vector GEP with unknown stride) to drive the address computation, which is
+ // typically more expensive than keeping the indices scalar in a unit-stride
+ // address chain. Add the delta once rather than per external use.
if (AllUsersGEPSWithStoresLoads && !Pointers.empty()) {
const TreeEntry &RootEntry = *VectorizableTree.front();
- const bool AnyRootKeptAsScalar = any_of(RootEntry.Scalars, [&](Value *V) {
- return ExternalUsesAsOriginalScalar.contains(V);
- });
- const Value *CommonBase = nullptr;
- bool HaveCommonBase = true;
- for (const Value *P : Pointers) {
- const Value *Op = getUnderlyingObject(P);
- if (!CommonBase)
- CommonBase = Op;
- else if (CommonBase != Op) {
- HaveCommonBase = false;
- break;
- }
- }
- if (!AnyRootKeptAsScalar && HaveCommonBase) {
+ const Value *CommonBase = getUnderlyingObject(Pointers.front());
+ if (all_of(Pointers, [CommonBase](const Value *P) {
+ return getUnderlyingObject(P) == CommonBase;
+ })) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
auto *VecTy = getWidenedType(UserScalarTy, RootEntry.Scalars.size());
InstructionCost ScalarGEPCost = TTI->getPointersChainCost(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
new file mode 100644
index 0000000000000..c39be0ed960ce
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-vals-used-as-load-indices.ll
@@ -0,0 +1,219 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mcpu=znver2 -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i32 @test(ptr %this, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @test(
+; CHECK-SAME: ptr [[THIS:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ARRAY_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[THIS]], i64 24
+; CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAY_PTR]], align 8
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq ptr [[TMP0]], null
+; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+; CHECK: [[IF_THEN]]:
+; CHECK-NEXT: tail call void @deopt()
+; CHECK-NEXT: unreachable
+; CHECK: [[IF_END]]:
+; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[B]], [[A]]
+; CHECK-NEXT: [[LENGTH:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 8
+; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[LENGTH]], align 8
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[ADD]], 15
+; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD]], 4
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[SHR]], 15
+; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i32 [[AND1]], 16
+; CHECK-NEXT: [[SHR2:%.*]] = lshr i32 [[ADD]], 8
+; CHECK-NEXT: [[AND3:%.*]] = and i32 [[SHR2]], 15
+; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i32 [[AND3]], 32
+; CHECK-NEXT: [[SHR5:%.*]] = lshr i32 [[ADD]], 12
+; CHECK-NEXT: [[AND6:%.*]] = and i32 [[SHR5]], 15
+; CHECK-NEXT: [[TMP18:%.*]] = or disjoint i32 [[AND6]], 48
+; CHECK-NEXT: [[SHR8:%.*]] = lshr i32 [[ADD]], 16
+; CHECK-NEXT: [[AND9:%.*]] = and i32 [[SHR8]], 15
+; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i32 [[AND9]], 64
+; CHECK-NEXT: [[SHR11:%.*]] = lshr i32 [[ADD]], 20
+; CHECK-NEXT: [[AND12:%.*]] = and i32 [[SHR11]], 15
+; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i32 [[AND12]], 80
+; CHECK-NEXT: [[SHR14:%.*]] = lshr i32 [[ADD]], 24
+; CHECK-NEXT: [[AND15:%.*]] = and i32 [[SHR14]], 15
+; CHECK-NEXT: [[TMP24:%.*]] = or disjoint i32 [[AND15]], 96
+; CHECK-NEXT: [[SHR17:%.*]] = lshr i32 [[ADD]], 28
+; CHECK-NEXT: [[TMP26:%.*]] = or disjoint i32 [[SHR17]], 112
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[AND]], [[TMP1]]
+; CHECK-NEXT: [[CMP19:%.*]] = icmp ult i32 [[TMP14]], [[TMP1]]
+; CHECK-NEXT: [[AND23167:%.*]] = and i1 [[CMP]], [[CMP19]]
+; CHECK-NEXT: [[CMP26:%.*]] = icmp ult i32 [[TMP16]], [[TMP1]]
+; CHECK-NEXT: [[AND33168:%.*]] = and i1 [[CMP26]], [[AND23167]]
+; CHECK-NEXT: [[CMP36:%.*]] = icmp ult i32 [[TMP18]], [[TMP1]]
+; CHECK-NEXT: [[AND43169:%.*]] = and i1 [[CMP36]], [[AND33168]]
+; CHECK-NEXT: [[CMP46:%.*]] = icmp ult i32 [[TMP20]], [[TMP1]]
+; CHECK-NEXT: [[AND53170:%.*]] = and i1 [[CMP46]], [[AND43169]]
+; CHECK-NEXT: [[CMP56:%.*]] = icmp ult i32 [[TMP22]], [[TMP1]]
+; CHECK-NEXT: [[AND63171:%.*]] = and i1 [[CMP56]], [[AND53170]]
+; CHECK-NEXT: [[CMP66:%.*]] = icmp ult i32 [[TMP24]], [[TMP1]]
+; CHECK-NEXT: [[AND73172:%.*]] = and i1 [[CMP66]], [[AND63171]]
+; CHECK-NEXT: [[CMP76:%.*]] = icmp ult i32 [[TMP26]], [[TMP1]]
+; CHECK-NEXT: [[TMP12:%.*]] = and i1 [[CMP76]], [[AND73172]]
+; CHECK-NEXT: br i1 [[TMP12]], label %[[IF_END88:.*]], label %[[IF_THEN87:.*]]
+; CHECK: [[IF_THEN87]]:
+; CHECK-NEXT: tail call void @deopt()
+; CHECK-NEXT: unreachable
+; CHECK: [[IF_END88]]:
+; CHECK-NEXT: [[DATA:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i64 16
+; CHECK-NEXT: [[IDXPROM:%.*]] = zext nneg i32 [[AND]] to i64
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i8, ptr [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[CONV89:%.*]] = sext i8 [[TMP13]] to i32
+; CHECK-NEXT: [[IDXPROM90:%.*]] = zext nneg i32 [[TMP14]] to i64
+; CHECK-NEXT: [[ARRAYIDX91:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM90]]
+; CHECK-NEXT: [[TMP15:%.*]] = load i8, ptr [[ARRAYIDX91]], align 1
+; CHECK-NEXT: [[CONV92:%.*]] = sext i8 [[TMP15]] to i32
+; CHECK-NEXT: [[SHL93:%.*]] = shl nsw i32 [[CONV92]], 4
+; CHECK-NEXT: [[ADD94:%.*]] = add nsw i32 [[SHL93]], [[CONV89]]
+; CHECK-NEXT: [[IDXPROM95:%.*]] = zext nneg i32 [[TMP16]] to i64
+; CHECK-NEXT: [[ARRAYIDX96:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM95]]
+; CHECK-NEXT: [[TMP17:%.*]] = load i8, ptr [[ARRAYIDX96]], align 1
+; CHECK-NEXT: [[CONV97:%.*]] = sext i8 [[TMP17]] to i32
+; CHECK-NEXT: [[SHL98:%.*]] = shl nsw i32 [[CONV97]], 8
+; CHECK-NEXT: [[ADD99:%.*]] = add nsw i32 [[ADD94]], [[SHL98]]
+; CHECK-NEXT: [[IDXPROM100:%.*]] = zext nneg i32 [[TMP18]] to i64
+; CHECK-NEXT: [[ARRAYIDX101:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM100]]
+; CHECK-NEXT: [[TMP19:%.*]] = load i8, ptr [[ARRAYIDX101]], align 1
+; CHECK-NEXT: [[CONV102:%.*]] = sext i8 [[TMP19]] to i32
+; CHECK-NEXT: [[SHL103:%.*]] = shl nsw i32 [[CONV102]], 12
+; CHECK-NEXT: [[ADD104:%.*]] = add nsw i32 [[ADD99]], [[SHL103]]
+; CHECK-NEXT: [[IDXPROM105:%.*]] = zext nneg i32 [[TMP20]] to i64
+; CHECK-NEXT: [[ARRAYIDX106:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM105]]
+; CHECK-NEXT: [[TMP21:%.*]] = load i8, ptr [[ARRAYIDX106]], align 1
+; CHECK-NEXT: [[CONV107:%.*]] = sext i8 [[TMP21]] to i32
+; CHECK-NEXT: [[SHL108:%.*]] = shl nsw i32 [[CONV107]], 16
+; CHECK-NEXT: [[ADD109:%.*]] = add nsw i32 [[ADD104]], [[SHL108]]
+; CHECK-NEXT: [[IDXPROM110:%.*]] = zext nneg i32 [[TMP22]] to i64
+; CHECK-NEXT: [[ARRAYIDX111:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM110]]
+; CHECK-NEXT: [[TMP23:%.*]] = load i8, ptr [[ARRAYIDX111]], align 1
+; CHECK-NEXT: [[CONV112:%.*]] = sext i8 [[TMP23]] to i32
+; CHECK-NEXT: [[SHL113:%.*]] = shl nsw i32 [[CONV112]], 20
+; CHECK-NEXT: [[ADD114:%.*]] = add nsw i32 [[ADD109]], [[SHL113]]
+; CHECK-NEXT: [[IDXPROM115:%.*]] = zext nneg i32 [[TMP24]] to i64
+; CHECK-NEXT: [[ARRAYIDX116:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM115]]
+; CHECK-NEXT: [[TMP25:%.*]] = load i8, ptr [[ARRAYIDX116]], align 1
+; CHECK-NEXT: [[CONV117:%.*]] = sext i8 [[TMP25]] to i32
+; CHECK-NEXT: [[SHL118:%.*]] = shl nsw i32 [[CONV117]], 24
+; CHECK-NEXT: [[ADD119:%.*]] = add nsw i32 [[ADD114]], [[SHL118]]
+; CHECK-NEXT: [[IDXPROM120:%.*]] = zext nneg i32 [[TMP26]] to i64
+; CHECK-NEXT: [[ARRAYIDX121:%.*]] = getelementptr inbounds nuw i8, ptr [[DATA]], i64 [[IDXPROM120]]
+; CHECK-NEXT: [[TMP27:%.*]] = load i8, ptr [[ARRAYIDX121]], align 1
+; CHECK-NEXT: [[CONV122:%.*]] = zext i8 [[TMP27]] to i32
+; CHECK-NEXT: [[SHL123:%.*]] = shl i32 [[CONV122]], 28
+; CHECK-NEXT: [[ADD124:%.*]] = add nsw i32 [[ADD119]], [[SHL123]]
+; CHECK-NEXT: [[OR127:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[ADD124]], i32 [[ADD124]], i32 11)
+; CHECK-NEXT: ret i32 [[OR127]]
+;
+entry:
+ %array_ptr = getelementptr inbounds nuw i8, ptr %this, i64 24
+ %0 = load ptr, ptr %array_ptr, align 8
+ %tobool.not = icmp eq ptr %0, null
+ br i1 %tobool.not, label %if.then, label %if.end
+
+if.then:
+ tail call void @deopt()
+ unreachable
+
+if.end:
+ %add = add nsw i32 %b, %a
+ %length = getelementptr inbounds nuw i8, ptr %0, i64 8
+ %1 = load i32, ptr %length, align 8
+ %and = and i32 %add, 15
+ %shr = lshr i32 %add, 4
+ %and1 = and i32 %shr, 15
+ %or = or disjoint i32 %and1, 16
+ %shr2 = lshr i32 %add, 8
+ %and3 = and i32 %shr2, 15
+ %or4 = or disjoint i32 %and3, 32
+ %shr5 = lshr i32 %add, 12
+ %and6 = and i32 %shr5, 15
+ %or7 = or disjoint i32 %and6, 48
+ %shr8 = lshr i32 %add, 16
+ %and9 = and i32 %shr8, 15
+ %or10 = or disjoint i32 %and9, 64
+ %shr11 = lshr i32 %add, 20
+ %and12 = and i32 %shr11, 15
+ %or13 = or disjoint i32 %and12, 80
+ %shr14 = lshr i32 %add, 24
+ %and15 = and i32 %shr14, 15
+ %or16 = or disjoint i32 %and15, 96
+ %shr17 = lshr i32 %add, 28
+ %or18 = or disjoint i32 %shr17, 112
+ %cmp = icmp ult i32 %and, %1
+ %cmp19 = icmp ult i32 %or, %1
+ %and23167 = and i1 %cmp, %cmp19
+ %cmp26 = icmp ult i32 %or4, %1
+ %and33168 = and i1 %cmp26, %and23167
+ %cmp36 = icmp ult i32 %or7, %1
+ %and43169 = and i1 %cmp36, %and33168
+ %cmp46 = icmp ult i32 %or10, %1
+ %and53170 = and i1 %cmp46, %and43169
+ %cmp56 = icmp ult i32 %or13, %1
+ %and63171 = and i1 %cmp56, %and53170
+ %cmp66 = icmp ult i32 %or16, %1
+ %and73172 = and i1 %cmp66, %and63171
+ %cmp76 = icmp ult i32 %or18, %1
+ %and83173 = and i1 %cmp76, %and73172
+ br i1 %and83173, label %if.end88, label %if.then87
+
+if.then87:
+ tail call void @deopt()
+ unreachable
+
+if.end88:
+ %data = getelementptr inbounds nuw i8, ptr %0, i64 16
+ %idxprom = zext nneg i32 %and to i64
+ %arrayidx = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom
+ %2 = load i8, ptr %arrayidx, align 1
+ %conv89 = sext i8 %2 to i32
+ %idxprom90 = zext nneg i32 %or to i64
+ %arrayidx91 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom90
+ %3 = load i8, ptr %arrayidx91, align 1
+ %conv92 = sext i8 %3 to i32
+ %shl93 = shl nsw i32 %conv92, 4
+ %add94 = add nsw i32 %shl93, %conv89
+ %idxprom95 = zext nneg i32 %or4 to i64
+ %arrayidx96 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom95
+ %4 = load i8, ptr %arrayidx96, align 1
+ %conv97 = sext i8 %4 to i32
+ %shl98 = shl nsw i32 %conv97, 8
+ %add99 = add nsw i32 %add94, %shl98
+ %idxprom100 = zext nneg i32 %or7 to i64
+ %arrayidx101 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom100
+ %5 = load i8, ptr %arrayidx101, align 1
+ %conv102 = sext i8 %5 to i32
+ %shl103 = shl nsw i32 %conv102, 12
+ %add104 = add nsw i32 %add99, %shl103
+ %idxprom105 = zext nneg i32 %or10 to i64
+ %arrayidx106 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom105
+ %6 = load i8, ptr %arrayidx106, align 1
+ %conv107 = sext i8 %6 to i32
+ %shl108 = shl nsw i32 %conv107, 16
+ %add109 = add nsw i32 %add104, %shl108
+ %idxprom110 = zext nneg i32 %or13 to i64
+ %arrayidx111 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom110
+ %7 = load i8, ptr %arrayidx111, align 1
+ %conv112 = sext i8 %7 to i32
+ %shl113 = shl nsw i32 %conv112, 20
+ %add114 = add nsw i32 %add109, %shl113
+ %idxprom115 = zext nneg i32 %or16 to i64
+ %arrayidx116 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom115
+ %8 = load i8, ptr %arrayidx116, align 1
+ %conv117 = sext i8 %8 to i32
+ %shl118 = shl nsw i32 %conv117, 24
+ %add119 = add nsw i32 %add114, %shl118
+ %idxprom120 = zext nneg i32 %or18 to i64
+ %arrayidx121 = getelementptr inbounds nuw i8, ptr %data, i64 %idxprom120
+ %9 = load i8, ptr %arrayidx121, align 1
+ %conv122 = zext i8 %9 to i32
+ %shl123 = shl i32 %conv122, 28
+ %add124 = add nsw i32 %add119, %shl123
+ %or127 = tail call i32 @llvm.fshl.i32(i32 %add124, i32 %add124, i32 11)
+ ret i32 %or127
+}
+
+declare void @deopt()
+declare i32 @llvm.fshl.i32(i32, i32, i32)
More information about the llvm-branch-commits
mailing list