[llvm] [SLP] Fix CmpInst type handling in cost model (PR #190618)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 6 08:37:48 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/190618
>From 66f9001f24b453aa9df40a8d75d5eb31db0df421 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 6 Apr 2026 08:23:52 -0700
Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 74 +++++++-----
.../extracts-from-scalarizable-vector.ll | 27 ++---
.../AArch64/vectorizable-selects-min-max.ll | 19 ++-
.../RISCV/remarks-insert-into-small-vector.ll | 2 +-
.../Transforms/SLPVectorizer/X86/bool-mask.ll | 109 +++---------------
.../SLPVectorizer/X86/cmp-as-alternate-ops.ll | 12 +-
.../X86/identity-match-splat-less-defined.ll | 43 ++++++-
.../X86/inversed-icmp-to-gather.ll | 21 +---
.../X86/non-load-reduced-as-part-of-bv.ll | 20 ++--
.../SLPVectorizer/X86/reduced-value-stored.ll | 35 +++---
10 files changed, 161 insertions(+), 201 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f2ccf198c4c81..1ca868a111bc0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -258,15 +258,18 @@ static bool isValidElementType(Type *Ty) {
!Ty->isPPC_FP128Ty();
}
-/// Returns the type of the given value/instruction \p V. If it is store,
-/// returns the type of its value operand, for Cmp - the types of the compare
-/// operands and for insertelement - the type os the inserted operand.
-/// Otherwise, just the type of the value is returned.
-static Type *getValueType(Value *V) {
+/// Returns the "element type" of the given value/instruction \p V.
+/// For stores, returns the stored value type; for insertelement (when ReVec is
+/// off), the inserted operand type. For compares, the default is to return the
+/// result type (i1); when \p LookThroughCmp is true, returns the type of the
+/// compared operands instead, which is needed for vector width calculations
+/// (the width is determined by the operand type, not the i1 result).
+static Type *getValueType(Value *V, bool LookThroughCmp = false) {
if (auto *SI = dyn_cast<StoreInst>(V))
return SI->getValueOperand()->getType();
- if (auto *CI = dyn_cast<CmpInst>(V))
- return CI->getOperand(0)->getType();
+ if (LookThroughCmp)
+ if (auto *CI = dyn_cast<CmpInst>(V))
+ return CI->getOperand(0)->getType();
if (!SLPReVec)
if (auto *IE = dyn_cast<InsertElementInst>(V))
return IE->getOperand(1)->getType();
@@ -4329,7 +4332,8 @@ class slpvectorizer::BoUpSLP {
bool
hasNonWholeRegisterOrNonPowerOf2Vec(const TargetTransformInfo &TTI) const {
bool IsNonPowerOf2 = !hasFullVectorsOrPowerOf2(
- TTI, getValueType(Scalars.front()), Scalars.size());
+ TTI, getValueType(Scalars.front(), /*LookThroughCmp=*/true),
+ Scalars.size());
assert((!IsNonPowerOf2 || ReuseShuffleIndices.empty()) &&
"Reshuffling not supported with non-power-of-2 vectors yet.");
return IsNonPowerOf2;
@@ -4492,10 +4496,11 @@ class slpvectorizer::BoUpSLP {
std::make_pair(UserTreeIdx.UserTE, UserTreeIdx.EdgeIdx), Last);
// FIXME: Remove once support for ReuseShuffleIndices has been implemented
// for non-power-of-two vectors.
- assert(
- (hasFullVectorsOrPowerOf2(*TTI, getValueType(VL.front()), VL.size()) ||
- ReuseShuffleIndices.empty()) &&
- "Reshuffling scalars not yet supported for nodes with padding");
+ assert((hasFullVectorsOrPowerOf2(
+ *TTI, getValueType(VL.front(), /*LookThroughCmp=*/true),
+ VL.size()) ||
+ ReuseShuffleIndices.empty()) &&
+ "Reshuffling scalars not yet supported for nodes with padding");
Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
ReuseShuffleIndices.end());
if (ReorderIndices.empty()) {
@@ -10964,7 +10969,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
// Easy case: VL has unique values and a "natural" size
size_t NumUniqueScalarValues = UniqueValues.size();
bool IsFullVectors = hasFullVectorsOrPowerOf2(
- TTI, getValueType(UniqueValues.front()), NumUniqueScalarValues);
+ TTI, getValueType(UniqueValues.front(), /*LookThroughCmp=*/true),
+ NumUniqueScalarValues);
if (NumUniqueScalarValues == VL.size() &&
(VectorizeNonPowerOf2 || IsFullVectors)) {
ReuseShuffleIndices.clear();
@@ -10974,7 +10980,8 @@ static bool tryToFindDuplicates(SmallVectorImpl<Value *> &VL,
// FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops.
if ((UserTreeIdx.UserTE &&
UserTreeIdx.UserTE->hasNonWholeRegisterOrNonPowerOf2Vec(TTI)) ||
- !hasFullVectorsOrPowerOf2(TTI, getValueType(VL.front()), VL.size())) {
+ !hasFullVectorsOrPowerOf2(
+ TTI, getValueType(VL.front(), /*LookThroughCmp=*/true), VL.size())) {
LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported "
"for nodes with padding.\n");
ReuseShuffleIndices.clear();
@@ -15687,8 +15694,6 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return 0;
if (isa<InsertElementInst>(VL[0]))
return InstructionCost::getInvalid();
- if (isa<CmpInst>(VL.front()))
- ScalarTy = VL.front()->getType();
return SpillsReloads +
processBuildVector<ShuffleCostEstimator, InstructionCost>(
E, ScalarTy, *TTI, VectorizedVals, *this, CheckedExtracts);
@@ -16152,6 +16157,12 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
}
case Instruction::FCmp:
case Instruction::ICmp:
+ // Override ScalarTy/VecTy with the compared operand type (not i1). The
+ // cost of a compare instruction is determined by the operand width, and
+ // getCmpSelInstrCost expects the compared type as its first type arg.
+ OrigScalarTy = ScalarTy = getValueType(VL0, /*LookThroughCmp=*/true);
+ VecTy = getWidenedType(ScalarTy, VL.size());
+ [[fallthrough]];
case Instruction::Select: {
CmpPredicate VecPred, SwappedVecPred;
auto MatchCmp = m_Cmp(VecPred, m_Value(), m_Value());
@@ -16183,9 +16194,13 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
? CmpInst::BAD_FCMP_PREDICATE
: CmpInst::BAD_ICMP_PREDICATE;
+ // For selects, the "condition type" arg is the condition operand's
+ // type; for standalone compares, it is the result type (i1).
InstructionCost ScalarCost = TTI->getCmpSelInstrCost(
- E->getOpcode(), OrigScalarTy, Builder.getInt1Ty(), CurrentPred,
- CostKind,
+ E->getOpcode(), OrigScalarTy,
+ ShuffleOrOp == Instruction::Select ? VL0->getOperand(0)->getType()
+ : VL0->getType(),
+ CurrentPred, CostKind,
getOperandInfo(
VI->getOperand(ShuffleOrOp == Instruction::Select ? 1 : 0)),
getOperandInfo(
@@ -16198,7 +16213,14 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return ScalarCost;
};
auto GetVectorCost = [&](InstructionCost CommonCost) {
- auto *MaskTy = getWidenedType(Builder.getInt1Ty(), VL.size());
+ // For selects, the condition type may differ from the result type
+ // (e.g. condition is <N x i1> while result is <N x i32>). For
+ // compares, the result type IS the mask (i1/vNi1). Construct the
+ // right type so getCmpSelInstrCost sees the actual mask/result width.
+ auto *MaskTy = getWidenedType(ShuffleOrOp == Instruction::Select
+ ? VL0->getOperand(0)->getType()
+ : VL0->getType(),
+ VL.size());
InstructionCost VecCost = TTI->getCmpSelInstrCost(
E->getOpcode(), VecTy, MaskTy, VecPred, CostKind,
@@ -16207,10 +16229,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
getOperandInfo(
E->getOperand(ShuffleOrOp == Instruction::Select ? 2 : 1)),
VL0);
- if (auto *SI = dyn_cast<SelectInst>(VL0)) {
- auto *CondType =
- getWidenedType(SI->getCondition()->getType(), VL.size());
- unsigned CondNumElements = CondType->getNumElements();
+ if (isa<SelectInst>(VL0)) {
+ unsigned CondNumElements = getNumElements(MaskTy);
unsigned VecTyNumElements = getNumElements(VecTy);
assert(VecTyNumElements >= CondNumElements &&
VecTyNumElements % CondNumElements == 0 &&
@@ -16219,7 +16239,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
// When the return type is i1 but the source is fixed vector type, we
// need to duplicate the condition value.
VecCost += ::getShuffleCost(
- *TTI, TTI::SK_PermuteSingleSrc, CondType,
+ *TTI, TTI::SK_PermuteSingleSrc, MaskTy,
createReplicatedMask(VecTyNumElements / CondNumElements,
CondNumElements));
}
@@ -17855,8 +17875,6 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
auto It = MinBWs.find(TE);
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
- if (isa<CmpInst>(TE->Scalars.front()))
- ScalarTy = TE->Scalars.front()->getType();
auto *VecTy = getWidenedType(ScalarTy, Sz);
const unsigned EntryVF = TE->getVectorFactor();
auto *FinalVecTy = getWidenedType(ScalarTy, EntryVF);
@@ -21215,9 +21233,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
IRBuilderBase::InsertPointGuard Guard(Builder);
Value *V = E->Scalars.front();
- Type *ScalarTy = V->getType();
- if (!isa<CmpInst>(V))
- ScalarTy = getValueType(V);
+ Type *ScalarTy = getValueType(V);
auto It = MinBWs.find(E);
if (It != MinBWs.end()) {
auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy);
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-from-scalarizable-vector.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-from-scalarizable-vector.ll
index c99dd53117e5f..54c31285c194e 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-from-scalarizable-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-from-scalarizable-vector.ll
@@ -4,15 +4,7 @@
define i1 @degenerate() {
; CHECK-LABEL: define i1 @degenerate() {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x fp128> zeroinitializer, i32 0
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt fp128 [[TMP0]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[CMP3:%.*]] = fcmp olt fp128 [[TMP0]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP3]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x fp128> zeroinitializer, i32 0
-; CHECK-NEXT: [[CMP6:%.*]] = fcmp ogt fp128 [[TMP1]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND29:%.*]] = select i1 [[OR_COND]], i1 [[CMP6]], i1 false
-; CHECK-NEXT: [[CMP10:%.*]] = fcmp olt fp128 [[TMP1]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND30:%.*]] = select i1 [[OR_COND29]], i1 [[CMP10]], i1 false
+; CHECK-NEXT: [[OR_COND30:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> zeroinitializer)
; CHECK-NEXT: ret i1 [[OR_COND30]]
;
entry:
@@ -32,16 +24,13 @@ define i1 @with_inputs(<4 x fp128> %a) {
; CHECK-LABEL: define i1 @with_inputs
; CHECK-SAME: (<4 x fp128> [[A:%.*]]) {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x fp128> [[A]], i32 0
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt fp128 [[TMP0]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[CMP3:%.*]] = fcmp olt fp128 [[TMP0]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP3]]
-; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x fp128> [[A]], i32 1
-; CHECK-NEXT: [[CMP6:%.*]] = fcmp ogt fp128 [[TMP1]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND29:%.*]] = select i1 [[OR_COND]], i1 [[CMP6]], i1 false
-; CHECK-NEXT: [[CMP10:%.*]] = fcmp olt fp128 [[TMP1]], 0xL00000000000000000000000000000000
-; CHECK-NEXT: [[OR_COND30:%.*]] = select i1 [[OR_COND29]], i1 [[CMP10]], i1 false
-; CHECK-NEXT: ret i1 [[OR_COND30]]
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x fp128> [[A]], <4 x fp128> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp olt <4 x fp128> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp ogt <4 x fp128> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i1> [[TMP1]], <4 x i1> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> [[TMP4]])
+; CHECK-NEXT: ret i1 [[TMP5]]
;
entry:
%0 = extractelement <4 x fp128> %a, i32 0
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorizable-selects-min-max.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorizable-selects-min-max.ll
index c0b882b98d15b..413a09bbdce49 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vectorizable-selects-min-max.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vectorizable-selects-min-max.ll
@@ -103,12 +103,19 @@ entry:
define void @select_ule_ugt_mix_4xi32(ptr %ptr, i32 %x) {
; CHECK-LABEL: @select_ule_ugt_mix_4xi32(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ult <4 x i32> [[TMP1]], splat (i32 16383)
-; CHECK-NEXT: [[TMP3:%.*]] = icmp ugt <4 x i32> [[TMP1]], splat (i32 16383)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
-; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x i32> [[TMP1]], <4 x i32> splat (i32 16383)
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 poison, i32 16383>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> <i32 16383, i32 poison>, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[TMP0]], <2 x i32> splat (i32 16383)
+; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[PTR]], align 4
+; CHECK-NEXT: [[GEP_2:%.*]] = getelementptr inbounds i32, ptr [[PTR]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x i32>, ptr [[GEP_2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> <i32 poison, i32 16383>, <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> <i32 16383, i32 poison>, <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ult <2 x i32> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = select <2 x i1> [[TMP8]], <2 x i32> [[TMP5]], <2 x i32> splat (i32 16383)
+; CHECK-NEXT: store <2 x i32> [[TMP9]], ptr [[GEP_2]], align 4
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll
index 7fd88e54d4440..e39d887ec75d4 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll
@@ -8,7 +8,7 @@
; YAML-NEXT: Function: test
; YAML-NEXT: Args:
; YAML-NEXT: - String: 'Stores SLP vectorized with cost '
-; YAML-NEXT: - Cost: '-2'
+; YAML-NEXT: - Cost: '0'
; YAML-NEXT: - String: ' and with tree size '
; YAML-NEXT: - TreeSize: '9'
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
index 2780dd363035f..426043033da90 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
@@ -258,44 +258,12 @@ entry:
define i64 @bitmask_8xi64(ptr nocapture noundef readonly %src) {
; SSE2-LABEL: @bitmask_8xi64(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0
-; SSE2-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64
-; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX_1]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_1:%.*]] = icmp eq i64 [[TMP1]], 0
-; SSE2-NEXT: [[OR_1:%.*]] = select i1 [[TOBOOL_NOT_1]], i64 0, i64 2
-; SSE2-NEXT: [[MASK_1_1:%.*]] = or i64 [[OR_1]], [[OR]]
-; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 2
-; SSE2-NEXT: [[TMP2:%.*]] = load i64, ptr [[ARRAYIDX_2]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_2:%.*]] = icmp eq i64 [[TMP2]], 0
-; SSE2-NEXT: [[OR_2:%.*]] = select i1 [[TOBOOL_NOT_2]], i64 0, i64 4
-; SSE2-NEXT: [[MASK_1_2:%.*]] = or i64 [[OR_2]], [[MASK_1_1]]
-; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 3
-; SSE2-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX_3]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_3:%.*]] = icmp eq i64 [[TMP3]], 0
-; SSE2-NEXT: [[OR_3:%.*]] = select i1 [[TOBOOL_NOT_3]], i64 0, i64 8
-; SSE2-NEXT: [[MASK_1_3:%.*]] = or i64 [[OR_3]], [[MASK_1_2]]
-; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 4
-; SSE2-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_4]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_4:%.*]] = icmp eq i64 [[TMP4]], 0
-; SSE2-NEXT: [[OR_4:%.*]] = select i1 [[TOBOOL_NOT_4]], i64 0, i64 16
-; SSE2-NEXT: [[MASK_1_4:%.*]] = or i64 [[OR_4]], [[MASK_1_3]]
-; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5
-; SSE2-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP5]], 0
-; SSE2-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32
-; SSE2-NEXT: [[MASK_1_5:%.*]] = or i64 [[OR_5]], [[MASK_1_4]]
-; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6
-; SSE2-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP6]], 0
-; SSE2-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64
-; SSE2-NEXT: [[MASK_1_6:%.*]] = or i64 [[OR_6]], [[MASK_1_5]]
-; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7
-; SSE2-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 8
-; SSE2-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP7]], 0
-; SSE2-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128
-; SSE2-NEXT: [[MASK_1_7:%.*]] = or i64 [[OR_7]], [[MASK_1_6]]
+; SSE2-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 8
+; SSE2-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
+; SSE2-NEXT: [[TMP2:%.*]] = icmp eq <8 x i64> [[TMP0]], zeroinitializer
+; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP3]], <8 x i64> <i64 1, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, <8 x i64> <i64 0, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>
+; SSE2-NEXT: [[MASK_1_7:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
; SSE2-NEXT: ret i64 [[MASK_1_7]]
;
; SSE4-LABEL: @bitmask_8xi64(
@@ -367,64 +335,13 @@ entry:
define i64 @combined(ptr nocapture noundef readonly %src) {
; SSE2-LABEL: @combined(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = load i64, ptr [[SRC:%.*]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT:%.*]] = icmp ne i64 [[TMP0]], 0
-; SSE2-NEXT: [[OR:%.*]] = zext i1 [[TOBOOL_NOT]] to i64
-; SSE2-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = load i64, ptr [[ARRAYIDX_1]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_1:%.*]] = icmp eq i64 [[TMP1]], 0
-; SSE2-NEXT: [[OR_1:%.*]] = select i1 [[TOBOOL_NOT_1]], i64 0, i64 2
-; SSE2-NEXT: [[MASK_1_1:%.*]] = or i64 [[OR_1]], [[OR]]
-; SSE2-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 2
-; SSE2-NEXT: [[TMP2:%.*]] = load i64, ptr [[ARRAYIDX_2]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_2:%.*]] = icmp eq i64 [[TMP2]], 0
-; SSE2-NEXT: [[OR_2:%.*]] = select i1 [[TOBOOL_NOT_2]], i64 0, i64 4
-; SSE2-NEXT: [[MASK_1_2:%.*]] = or i64 [[OR_2]], [[MASK_1_1]]
-; SSE2-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 3
-; SSE2-NEXT: [[TMP3:%.*]] = load i64, ptr [[ARRAYIDX_3]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_3:%.*]] = icmp eq i64 [[TMP3]], 0
-; SSE2-NEXT: [[OR_3:%.*]] = select i1 [[TOBOOL_NOT_3]], i64 0, i64 8
-; SSE2-NEXT: [[MASK_1_3:%.*]] = or i64 [[OR_3]], [[MASK_1_2]]
-; SSE2-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 4
-; SSE2-NEXT: [[TMP4:%.*]] = load i64, ptr [[ARRAYIDX_4]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_4:%.*]] = icmp eq i64 [[TMP4]], 0
-; SSE2-NEXT: [[OR_4:%.*]] = select i1 [[TOBOOL_NOT_4]], i64 0, i64 16
-; SSE2-NEXT: [[MASK_1_4:%.*]] = or i64 [[OR_4]], [[MASK_1_3]]
-; SSE2-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 5
-; SSE2-NEXT: [[TMP5:%.*]] = load i64, ptr [[ARRAYIDX_5]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_5:%.*]] = icmp eq i64 [[TMP5]], 0
-; SSE2-NEXT: [[OR_5:%.*]] = select i1 [[TOBOOL_NOT_5]], i64 0, i64 32
-; SSE2-NEXT: [[MASK_1_5:%.*]] = or i64 [[OR_5]], [[MASK_1_4]]
-; SSE2-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 6
-; SSE2-NEXT: [[TMP6:%.*]] = load i64, ptr [[ARRAYIDX_6]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_6:%.*]] = icmp eq i64 [[TMP6]], 0
-; SSE2-NEXT: [[OR_6:%.*]] = select i1 [[TOBOOL_NOT_6]], i64 0, i64 64
-; SSE2-NEXT: [[MASK_1_6:%.*]] = or i64 [[OR_6]], [[MASK_1_5]]
-; SSE2-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 7
-; SSE2-NEXT: [[TMP7:%.*]] = load i64, ptr [[ARRAYIDX_7]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_7:%.*]] = icmp eq i64 [[TMP7]], 0
-; SSE2-NEXT: [[OR_7:%.*]] = select i1 [[TOBOOL_NOT_7]], i64 0, i64 128
-; SSE2-NEXT: [[MASK_1_7:%.*]] = or i64 [[OR_7]], [[MASK_1_6]]
-; SSE2-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 8
-; SSE2-NEXT: [[TMP8:%.*]] = load i64, ptr [[ARRAYIDX_8]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_8:%.*]] = icmp eq i64 [[TMP8]], 0
-; SSE2-NEXT: [[OR_8:%.*]] = select i1 [[TOBOOL_NOT_8]], i64 0, i64 [[TMP8]]
-; SSE2-NEXT: [[MASK_1_8:%.*]] = or i64 [[OR_8]], [[MASK_1_7]]
-; SSE2-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 9
-; SSE2-NEXT: [[TMP9:%.*]] = load i64, ptr [[ARRAYIDX_9]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_9:%.*]] = icmp eq i64 [[TMP9]], 0
-; SSE2-NEXT: [[OR_9:%.*]] = select i1 [[TOBOOL_NOT_9]], i64 0, i64 [[TMP9]]
-; SSE2-NEXT: [[MASK_1_9:%.*]] = or i64 [[OR_9]], [[MASK_1_8]]
-; SSE2-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 10
-; SSE2-NEXT: [[TMP10:%.*]] = load i64, ptr [[ARRAYIDX_10]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_10:%.*]] = icmp eq i64 [[TMP10]], 0
-; SSE2-NEXT: [[OR_10:%.*]] = select i1 [[TOBOOL_NOT_10]], i64 0, i64 [[TMP10]]
-; SSE2-NEXT: [[MASK_1_10:%.*]] = or i64 [[OR_10]], [[MASK_1_9]]
-; SSE2-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 11
-; SSE2-NEXT: [[TMP11:%.*]] = load i64, ptr [[ARRAYIDX_11]], align 2
-; SSE2-NEXT: [[TOBOOL_NOT_11:%.*]] = icmp eq i64 [[TMP11]], 0
-; SSE2-NEXT: [[OR_11:%.*]] = select i1 [[TOBOOL_NOT_11]], i64 0, i64 [[TMP10]]
-; SSE2-NEXT: [[MASK_1_11:%.*]] = or i64 [[OR_11]], [[MASK_1_10]]
+; SSE2-NEXT: [[TMP0:%.*]] = load <12 x i64>, ptr [[SRC:%.*]], align 2
+; SSE2-NEXT: [[TMP1:%.*]] = icmp ne <12 x i64> [[TMP0]], zeroinitializer
+; SSE2-NEXT: [[TMP2:%.*]] = icmp eq <12 x i64> [[TMP0]], zeroinitializer
+; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <12 x i1> [[TMP1]], <12 x i1> [[TMP2]], <12 x i32> <i32 0, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; SSE2-NEXT: [[TMP4:%.*]] = shufflevector <12 x i64> [[TMP0]], <12 x i64> <i64 0, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 poison, i64 poison, i64 poison, i64 poison>, <12 x i32> <i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 8, i32 9, i32 10, i32 10>
+; SSE2-NEXT: [[TMP5:%.*]] = select <12 x i1> [[TMP3]], <12 x i64> <i64 1, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0>, <12 x i64> [[TMP4]]
+; SSE2-NEXT: [[MASK_1_11:%.*]] = call i64 @llvm.vector.reduce.or.v12i64(<12 x i64> [[TMP5]])
; SSE2-NEXT: ret i64 [[MASK_1_11]]
;
; SSE4-LABEL: @combined(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp-as-alternate-ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp-as-alternate-ops.ll
index 8b265ed24bebb..d275931c77eab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/cmp-as-alternate-ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp-as-alternate-ops.ll
@@ -44,11 +44,13 @@ define { <2 x float>, <2 x float> } @test1(i32 %conv.i32.i.i.i) {
; CHECK-LABEL: @test1(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[CONV_I32_I_I_I1:%.*]] = fptosi float 0.000000e+00 to i32
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 poison, i32 0>, i32 [[CONV_I32_I_I_I:%.*]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[CONV_I32_I_I_I1]], i32 2
-; CHECK-NEXT: [[TMP2:%.*]] = icmp sgt <4 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[CONV_I32_I_I_I1]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[CONV_I32_I_I_I:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i1> [[TMP3]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP9]], <4 x i1> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP5:%.*]] = select <4 x i1> [[TMP4]], <4 x float> zeroinitializer, <4 x float> zeroinitializer
; CHECK-NEXT: [[TMP6:%.*]] = fadd <4 x float> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> [[TMP6]], <4 x float> poison, <2 x i32> <i32 0, i32 1>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/identity-match-splat-less-defined.ll b/llvm/test/Transforms/SLPVectorizer/X86/identity-match-splat-less-defined.ll
index 43302f28d479e..f911eb0433c3d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/identity-match-splat-less-defined.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/identity-match-splat-less-defined.ll
@@ -3,7 +3,48 @@
define i32 @test() {
; CHECK-LABEL: define i32 @test() {
-; CHECK-NEXT: [[TMP23:%.*]] = call i8 @llvm.vector.reduce.xor.v64i8(<64 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+; CHECK-NEXT: [[TMP1:%.*]] = or i32 0, 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <12 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>, i32 [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <12 x i32> [[TMP2]], <12 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 6, i32 6, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = or <12 x i32> [[TMP3]], <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <12 x i32> [[TMP4]], <12 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 8>
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <12 x i32> [[TMP4]], <12 x i32> poison, <16 x i32> <i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <16 x i32> [[TMP7]], zeroinitializer
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <16 x i1> [[TMP6]], <16 x i1> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i1> [[TMP8]], <16 x i1> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <32 x i1> [[TMP9]], <32 x i1> [[TMP10]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT: [[TMP12:%.*]] = zext <32 x i1> [[TMP11]] to <32 x i8>
+; CHECK-NEXT: [[TMP13:%.*]] = or i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x i32> [[TMP14]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 poison>
+; CHECK-NEXT: [[TMP16:%.*]] = or <4 x i32> [[TMP15]], <i32 0, i32 0, i32 0, i32 poison>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[TMP18:%.*]] = icmp ne <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = icmp eq <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x i1> [[TMP18]], <16 x i1> [[TMP19]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP21:%.*]] = zext <16 x i1> [[TMP20]] to <16 x i8>
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 0>, i32 [[TMP13]], i32 0
+; CHECK-NEXT: [[TMP39:%.*]] = shufflevector <8 x i32> [[TMP22]], <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP40:%.*]] = icmp ne <8 x i32> zeroinitializer, [[TMP39]]
+; CHECK-NEXT: [[TMP25:%.*]] = icmp eq <8 x i32> zeroinitializer, [[TMP39]]
+; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <8 x i1> [[TMP40]], <8 x i1> [[TMP25]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; CHECK-NEXT: [[TMP27:%.*]] = zext <8 x i1> [[TMP26]] to <8 x i8>
+; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP:%.*]] = xor <16 x i8> [[TMP28]], [[TMP21]]
+; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <16 x i8> [[RDX_OP]], <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <32 x i8> [[TMP12]], <32 x i8> [[TMP29]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <32 x i8> [[TMP30]], <32 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = xor <8 x i8> [[TMP31]], [[TMP27]]
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <8 x i8> [[RDX_OP1]], <8 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <32 x i8> [[TMP30]], <32 x i8> [[TMP32]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <32 x i8> [[TMP33]], <32 x i8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = xor <4 x i8> [[TMP34]], splat (i8 1)
+; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <4 x i8> [[RDX_OP2]], <4 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <32 x i8> [[TMP33]], <32 x i8> [[TMP35]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[TMP37:%.*]] = call i8 @llvm.vector.reduce.xor.v32i8(<32 x i8> [[TMP36]])
+; CHECK-NEXT: [[TMP38:%.*]] = call i8 @llvm.vector.reduce.xor.v4i8(<4 x i8> splat (i8 1))
+; CHECK-NEXT: [[TMP23:%.*]] = xor i8 [[TMP38]], [[TMP37]]
; CHECK-NEXT: [[TMP24:%.*]] = sext i8 [[TMP23]] to i32
; CHECK-NEXT: ret i32 [[TMP24]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/inversed-icmp-to-gather.ll b/llvm/test/Transforms/SLPVectorizer/X86/inversed-icmp-to-gather.ll
index b148a5b0e7325..452770182debe 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/inversed-icmp-to-gather.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/inversed-icmp-to-gather.ll
@@ -5,22 +5,11 @@ define i16 @test(i64 %0) {
; CHECK-LABEL: define i16 @test(
; CHECK-SAME: i64 [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[CMP12_NOT_NOT:%.*]] = icmp eq i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP80:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP96:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP112:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP128:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP144:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP160:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[CMP176:%.*]] = icmp ne i64 [[TMP0]], 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i1> poison, i1 [[CMP80]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i1> [[TMP1]], i1 [[CMP96]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i1> [[TMP2]], i1 [[CMP112]], i32 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[CMP128]], i32 3
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[CMP144]], i32 4
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[CMP160]], i32 5
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[CMP176]], i32 6
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[CMP12_NOT_NOT]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <8 x i64> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <8 x i1> [[TMP3]], <8 x i1> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
; CHECK-NEXT: [[TMP9:%.*]] = select <8 x i1> [[TMP8]], <8 x i16> splat (i16 1), <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 2>
; CHECK-NEXT: [[TMP10:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP9]])
; CHECK-NEXT: ret i16 [[TMP10]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
index b965d27fe3ead..ddcd038c2b220 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-load-reduced-as-part-of-bv.ll
@@ -5,16 +5,16 @@ define i1 @foo() {
; CHECK-LABEL: define i1 @foo() {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TOBOOL_NOT_NOT509_I_2329_I_I:%.*]] = icmp ne i32 0, 0
-; CHECK-NEXT: [[STOREMERGE_2333_I_I:%.*]] = select i1 [[TOBOOL_NOT_NOT509_I_2329_I_I]], i32 0, i32 0
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[STOREMERGE_2333_I_I]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x i32> zeroinitializer, [[TMP0]]
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 0>
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> <i1 false, i1 false, i1 undef, i1 undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 3>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 undef, i1 undef, i1 undef, i1 undef>, <8 x i1> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP4:%.*]] = freeze <8 x i1> [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP4]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = select i1 [[TMP5]], i1 false, i1 false
+; CHECK-NEXT: [[TOBOOL_NOT_NOT509_I_1_1_I_I:%.*]] = icmp ne i32 0, 0
+; CHECK-NEXT: [[TOBOOL_NOT_NOT509_I_2329_I_I1:%.*]] = icmp ne i32 0, 0
+; CHECK-NEXT: [[STOREMERGE_2333_I_I:%.*]] = select i1 [[TOBOOL_NOT_NOT509_I_2329_I_I1]], i32 0, i32 0
+; CHECK-NEXT: [[TOBOOL_NOT_NOT509_I_1_2_I_I:%.*]] = icmp ne i32 [[STOREMERGE_2333_I_I]], 0
+; CHECK-NEXT: [[TMP0:%.*]] = call i1 @llvm.vector.reduce.and.v4i1(<4 x i1> zeroinitializer)
+; CHECK-NEXT: [[OP_RDX4:%.*]] = select i1 [[TMP0]], i1 [[TOBOOL_NOT_NOT509_I_1_2_I_I]], i1 false
+; CHECK-NEXT: [[OP_RDX1:%.*]] = select i1 [[TOBOOL_NOT_NOT509_I_2329_I_I1]], i1 [[TOBOOL_NOT_NOT509_I_1_1_I_I]], i1 false
+; CHECK-NEXT: [[OP_RDX2:%.*]] = select i1 [[TOBOOL_NOT_NOT509_I_2329_I_I]], i1 false, i1 false
+; CHECK-NEXT: [[OP_RDX3:%.*]] = select i1 [[OP_RDX4]], i1 [[OP_RDX1]], i1 false
+; CHECK-NEXT: [[OP_RDX:%.*]] = select i1 [[OP_RDX3]], i1 [[OP_RDX2]], i1 false
; CHECK-NEXT: ret i1 [[OP_RDX]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-value-stored.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-value-stored.ll
index 19aa5806dd858..527e6103b2e63 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-value-stored.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-value-stored.ll
@@ -5,26 +5,25 @@ define i16 @test(i32 %call5, i32 %add36) {
; CHECK-LABEL: define i16 @test(
; CHECK-SAME: i32 [[CALL5:%.*]], i32 [[ADD36:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 poison>, i32 [[CALL5]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[ADD36]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i32> <i32 0, i32 1, i32 0, i32 0>, [[TMP1]]
-; CHECK-NEXT: [[TOBOOL6_NOT:%.*]] = icmp eq i32 0, 0
-; CHECK-NEXT: [[CMP17:%.*]] = icmp ne i32 [[CALL5]], 0
-; CHECK-NEXT: [[TOBOOL32_NOT:%.*]] = icmp eq i32 0, 0
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <8 x i32> <i32 poison, i32 poison, i32 0, i32 0, i32 0, i32 poison, i32 0, i32 poison>, i32 [[CALL5]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> [[TMP0]], i32 [[ADD36]], i32 5
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <8 x i32> <i32 0, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 0>
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <8 x i32> <i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>, [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP3]], i32 6
+; CHECK-NEXT: [[COND3:%.*]] = zext i1 [[TMP4]] to i16
; CHECK-NEXT: [[TOBOOL48_NOT:%.*]] = icmp eq i32 0, 0
-; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i16> zeroinitializer, <4 x i16> zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i1> [[TMP2]], <4 x i1> poison, <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 poison, i32 poison, i32 3>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[TOBOOL6_NOT]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[CMP17]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[TOBOOL32_NOT]], i32 5
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TOBOOL48_NOT]], i32 6
-; CHECK-NEXT: [[TMP9:%.*]] = zext <8 x i1> [[TMP8]] to <8 x i16>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i16> [[TMP10]], [[TMP3]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x i16> [[RDX_OP]], <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i16> [[TMP9]], <8 x i16> [[TMP11]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[COND7:%.*]] = zext i1 [[TOBOOL48_NOT]] to i16
+; CHECK-NEXT: [[CMP21:%.*]] = icmp ne i32 0, 0
+; CHECK-NEXT: [[COND26:%.*]] = select i1 [[CMP21]], i16 0, i16 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP3]], i32 5
+; CHECK-NEXT: [[COND42:%.*]] = select i1 [[TMP5]], i16 0, i16 0
+; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP3]], <8 x i16> <i16 1, i16 1, i16 1, i16 0, i16 0, i16 1, i16 0, i16 0>, <8 x i16> <i16 0, i16 0, i16 0, i16 1, i16 1, i16 0, i16 0, i16 0>
; CHECK-NEXT: [[TMP13:%.*]] = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> [[TMP12]])
-; CHECK-NEXT: ret i16 [[TMP13]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = add i16 [[TMP13]], [[COND3]]
+; CHECK-NEXT: [[OP_RDX1:%.*]] = add i16 [[COND7]], [[COND26]]
+; CHECK-NEXT: [[OP_RDX2:%.*]] = add i16 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT: [[OP_RDX3:%.*]] = add i16 [[OP_RDX2]], [[COND42]]
+; CHECK-NEXT: ret i16 [[OP_RDX3]]
;
entry:
%cmp = icmp ne i32 0, 0
>From 7ceeb36e75479dcc81fdf13e1037453b9261e78b Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 6 Apr 2026 08:37:36 -0700
Subject: [PATCH 2/2] Address comments
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 1ca868a111bc0..80ef9105a30cb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -26162,7 +26162,7 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
}
}
- Type *ScalarTy = getValueType(VL[0]);
+ Type *ScalarTy = getValueType(VL[0], /*LookThroughCmp=*/true);
unsigned Sz = R.getVectorElementSize(I0);
unsigned MinVF = R.getMinVF(Sz);
unsigned MaxVF = std::max<unsigned>(
@@ -29064,7 +29064,8 @@ bool SLPVectorizerPass::vectorizeCmpInsts(iterator_range<ItT> CmpInsts,
SmallVector<Value *> Vals;
for (Instruction *V : CmpInsts)
- if (!R.isDeleted(V) && isValidElementType(getValueType(V)))
+ if (!R.isDeleted(V) &&
+ isValidElementType(getValueType(V, /*LookThroughCmp=*/true)))
Vals.push_back(V);
if (Vals.size() <= 1)
return Changed;
More information about the llvm-commits
mailing list