[llvm] [SLP]Fix masked gather GEPs cost, skip GEP index seeds (PR #228918)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sun Oct 4 09:52:22 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/228918
Pass the loaded type and the real base sharing to the chain cost of the
masked gather GEPs. Do not use the single-use GEP index chains as seeds:
they were vectorized with all lanes extracted for the scalar addresses.
Part of #227393.
Assisted-by: Cursor
>From 547c18d8885f7e1956c2daf9e71467353567e941 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sun, 4 Oct 2026 09:52:06 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../SLPVectorizer/SLPCostAnalysis.cpp | 14 +++++-
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 19 ++------
.../X86/gather-gep-addressing-cost.ll | 45 +++++++++++++++----
.../X86/once-used-gep-index-seeds.ll | 34 +++++++++-----
4 files changed, 76 insertions(+), 36 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
index a28198c7044ad..03d54345d4705 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPCostAnalysis.cpp
@@ -123,8 +123,20 @@ getGEPCosts(const TargetTransformInfo &TTI, ArrayRef<Value *> Ptrs,
? TTI::PointersChainInfo::getUnknownStride()
: TTI::PointersChainInfo::getKnownStride();
+ // The GEPs of the masked gather loads are accessed with the loaded type
+ // and form a chain only if the lanes share the base.
+ Type *AccessTy = ScalarTy;
+ if (all_of(Ptrs, [](const Value *V) {
+ auto *Ptr = dyn_cast<GetElementPtrInst>(V);
+ return Ptr && Ptr->hasOneUse() && isa<LoadInst>(Ptr->user_back());
+ })) {
+ PtrsInfo.IsSameBaseAddress = all_equal(map_range(Ptrs, [](Value *V) {
+ return cast<GetElementPtrInst>(V)->getPointerOperand();
+ }));
+ AccessTy = Ptrs.front()->user_back()->getType();
+ }
ScalarCost =
- TTI.getPointersChainCost(Ptrs, BasePtr, PtrsInfo, ScalarTy, CostKind);
+ TTI.getPointersChainCost(Ptrs, BasePtr, PtrsInfo, AccessTy, CostKind);
auto *BaseGEP = dyn_cast<GEPOperator>(BasePtr);
if (!BaseGEP) {
auto *It = find_if(Ptrs, IsaPred<GEPOperator>);
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 563f0067b05e9..e03afe2790f3c 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -1035,25 +1035,14 @@ Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
}
/// Returns true if \p I is a part of a single-use chain, computing an address,
-/// which does not pay off the vectorization: a constant table is accessed by a
-/// gather, while the indices, unrelated between the lanes, require a full
-/// buildvector, unlike the ones, shifted by a constant from a common base.
+/// which does not pay off the vectorization: all the lanes are extracted for
+/// the scalar addresses, the extracts delay the memory accesses.
static bool isNonProfitableIndex(const Instruction *I) {
constexpr unsigned MaxIndexChainLength = 3;
- // A constant shift of a common base is a cheap buildvector, while the loads
- // are vectorized together with the indices, computed from them.
- auto IsProfitableOperand = [](const Value *V) {
- if (isa<Constant>(V))
- return true;
- if (const auto *Cast = dyn_cast<CastInst>(V); Cast && Cast->hasOneUse())
- V = Cast->getOperand(0);
- return isa<LoadInst>(V);
- };
const User *U = I->user_back();
for ([[maybe_unused]] unsigned _ : seq<unsigned>(MaxIndexChainLength)) {
- if (const auto *GEP = dyn_cast<GetElementPtrInst>(U))
- return isa<Constant>(GEP->getPointerOperand()) ||
- none_of(I->operand_values(), IsProfitableOperand);
+ if (isa<GetElementPtrInst>(U))
+ return true;
if (!isa<Instruction>(U) || !U->hasOneUse())
return false;
U = U->user_back();
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-gep-addressing-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-gep-addressing-cost.ll
index 8e0f3bb8d4a20..1f2f714a06f30 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-gep-addressing-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-gep-addressing-cost.ll
@@ -14,14 +14,43 @@ define i32 @crc_step(ptr %words, i32 %crc) {
; CHECK-NEXT: [[CONV:%.*]] = zext i32 [[CRC]] to i64
; CHECK-NEXT: [[W:%.*]] = load i64, ptr [[WORDS]], align 8
; CHECK-NEXT: [[XOR:%.*]] = xor i64 [[W]], [[CONV]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i64> poison, i64 [[XOR]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i64> [[TMP0]], i64 [[W]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP3:%.*]] = lshr <8 x i64> [[TMP2]], <i64 8, i64 0, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
-; CHECK-NEXT: [[TMP4:%.*]] = and <8 x i64> [[TMP3]], <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1>
-; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds nuw [4 x i8], <8 x ptr> <ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 1024), ptr @tbl, ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 2048), ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 3072), ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 4096), ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 5120), ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 6144), ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 7168)>, <8 x i64> [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[TMP5]], <8 x i1> splat (i1 true), <8 x i32> poison)
-; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.xor.v8i32(<8 x i32> [[TMP6]])
+; CHECK-NEXT: [[AND:%.*]] = and i64 [[XOR]], 255
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds nuw [4 x i8], ptr @tbl, i64 [[AND]]
+; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[P0]], align 4
+; CHECK-NEXT: [[SHR:%.*]] = lshr i64 [[XOR]], 8
+; CHECK-NEXT: [[AND1:%.*]] = and i64 [[SHR]], 255
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 1024), i64 [[AND1]]
+; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[P1]], align 4
+; CHECK-NEXT: [[X1:%.*]] = xor i32 [[L1]], [[L0]]
+; CHECK-NEXT: [[SHR_1:%.*]] = lshr i64 [[XOR]], 16
+; CHECK-NEXT: [[AND_1:%.*]] = and i64 [[SHR_1]], 255
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 2048), i64 [[AND_1]]
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[P2]], align 4
+; CHECK-NEXT: [[X2:%.*]] = xor i32 [[L2]], [[X1]]
+; CHECK-NEXT: [[SHR_2:%.*]] = lshr i64 [[XOR]], 24
+; CHECK-NEXT: [[AND_2:%.*]] = and i64 [[SHR_2]], 255
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 3072), i64 [[AND_2]]
+; CHECK-NEXT: [[L3:%.*]] = load i32, ptr [[P3]], align 4
+; CHECK-NEXT: [[X3:%.*]] = xor i32 [[L3]], [[X2]]
+; CHECK-NEXT: [[SHR_3:%.*]] = lshr i64 [[W]], 32
+; CHECK-NEXT: [[AND_3:%.*]] = and i64 [[SHR_3]], 255
+; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 4096), i64 [[AND_3]]
+; CHECK-NEXT: [[L4:%.*]] = load i32, ptr [[P4]], align 4
+; CHECK-NEXT: [[X4:%.*]] = xor i32 [[L4]], [[X3]]
+; CHECK-NEXT: [[SHR_4:%.*]] = lshr i64 [[W]], 40
+; CHECK-NEXT: [[AND_4:%.*]] = and i64 [[SHR_4]], 255
+; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 5120), i64 [[AND_4]]
+; CHECK-NEXT: [[L5:%.*]] = load i32, ptr [[P5]], align 4
+; CHECK-NEXT: [[X5:%.*]] = xor i32 [[L5]], [[X4]]
+; CHECK-NEXT: [[SHR_5:%.*]] = lshr i64 [[W]], 48
+; CHECK-NEXT: [[AND_5:%.*]] = and i64 [[SHR_5]], 255
+; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 6144), i64 [[AND_5]]
+; CHECK-NEXT: [[L6:%.*]] = load i32, ptr [[P6]], align 4
+; CHECK-NEXT: [[X6:%.*]] = xor i32 [[L6]], [[X5]]
+; CHECK-NEXT: [[SHR_6:%.*]] = lshr i64 [[W]], 56
+; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds nuw [4 x i8], ptr getelementptr inbounds nuw (i8, ptr @tbl, i64 7168), i64 [[SHR_6]]
+; CHECK-NEXT: [[L7:%.*]] = load i32, ptr [[P7]], align 4
+; CHECK-NEXT: [[TMP7:%.*]] = xor i32 [[L7]], [[X6]]
; CHECK-NEXT: ret i32 [[TMP7]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/once-used-gep-index-seeds.ll b/llvm/test/Transforms/SLPVectorizer/X86/once-used-gep-index-seeds.ll
index dd7b07a7c1b8b..9a394278cef5f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/once-used-gep-index-seeds.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/once-used-gep-index-seeds.ll
@@ -8,12 +8,14 @@
define void @index_pair(ptr %idx, ptr %t, ptr %u, ptr %o) {
; CHECK-LABEL: define void @index_pair(
; CHECK-SAME: ptr [[IDX:%.*]], ptr [[T:%.*]], ptr [[U:%.*]], ptr [[O:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[IDX]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i64> [[TMP1]], splat (i64 1234)
-; CHECK-NEXT: [[TMP3:%.*]] = and <2 x i64> [[TMP2]], splat (i64 1023)
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i64 0
+; CHECK-NEXT: [[A:%.*]] = load i64, ptr [[IDX]], align 8
+; CHECK-NEXT: [[IDX1:%.*]] = getelementptr inbounds i8, ptr [[IDX]], i64 8
+; CHECK-NEXT: [[B:%.*]] = load i64, ptr [[IDX1]], align 8
+; CHECK-NEXT: [[X0:%.*]] = xor i64 [[A]], 1234
+; CHECK-NEXT: [[X1:%.*]] = xor i64 [[B]], 1234
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[X0]], 1023
+; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[X1]], 1023
; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[T]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i64 1
; CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds i32, ptr [[U]], i64 [[TMP5]]
; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[G0]], align 4
; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[G1]], align 4
@@ -40,16 +42,24 @@ define void @index_pair(ptr %idx, ptr %t, ptr %u, ptr %o) {
define void @index_quad(ptr %idx, ptr %t, ptr %u, ptr %v, ptr %w, ptr %o) {
; CHECK-LABEL: define void @index_quad(
; CHECK-SAME: ptr [[IDX:%.*]], ptr [[T:%.*]], ptr [[U:%.*]], ptr [[V:%.*]], ptr [[W:%.*]], ptr [[O:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[IDX]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = xor <4 x i64> [[TMP1]], splat (i64 1234)
-; CHECK-NEXT: [[TMP3:%.*]] = and <4 x i64> [[TMP2]], splat (i64 1023)
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i64 0
+; CHECK-NEXT: [[A:%.*]] = load i64, ptr [[IDX]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i8, ptr [[IDX]], i64 8
+; CHECK-NEXT: [[B:%.*]] = load i64, ptr [[P1]], align 8
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[IDX]], i64 16
+; CHECK-NEXT: [[C:%.*]] = load i64, ptr [[P2]], align 8
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i8, ptr [[IDX]], i64 24
+; CHECK-NEXT: [[D:%.*]] = load i64, ptr [[P3]], align 8
+; CHECK-NEXT: [[X0:%.*]] = xor i64 [[A]], 1234
+; CHECK-NEXT: [[X1:%.*]] = xor i64 [[B]], 1234
+; CHECK-NEXT: [[X2:%.*]] = xor i64 [[C]], 1234
+; CHECK-NEXT: [[X3:%.*]] = xor i64 [[D]], 1234
+; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[X0]], 1023
+; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[X1]], 1023
+; CHECK-NEXT: [[TMP6:%.*]] = and i64 [[X2]], 1023
+; CHECK-NEXT: [[TMP7:%.*]] = and i64 [[X3]], 1023
; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[T]], i64 [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i64 1
; CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds i32, ptr [[U]], i64 [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i64 2
; CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds i32, ptr [[V]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i64 3
; CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds i32, ptr [[W]], i64 [[TMP7]]
; CHECK-NEXT: [[L0:%.*]] = load i32, ptr [[G0]], align 4
; CHECK-NEXT: [[L1:%.*]] = load i32, ptr [[G1]], align 4
More information about the llvm-commits
mailing list