[llvm] [SLP]Support reduced or selects of bitmask as cmp bitcast (PR #181940)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Wed Feb 18 14:58:12 PST 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/181940
>From cf74bff23fcf7781cbe07c4e91ab602ccc3a396b Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 17 Feb 2026 15:49:57 -0800
Subject: [PATCH 1/6] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?=
=?UTF-8?q?itial=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 145 +++++++++++++++++-
.../Transforms/SLPVectorizer/X86/bool-mask.ll | 32 ++--
2 files changed, 156 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7ac5dea2b3d5f..f5a01eeb938a7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2117,6 +2117,14 @@ class slpvectorizer::BoUpSLP {
VectorizableTree.front()->State == TreeEntry::Vectorize;
}
+ /// Returns true if the tree results in the reduced cmp bitcast root.
+ bool isReducedCmpBitcastRoot() const {
+ return VectorizableTree.front()->hasState() &&
+ (VectorizableTree.front()->CombinedOp ==
+ TreeEntry::ReducedCmpBitcast) &&
+ VectorizableTree.front()->State == TreeEntry::Vectorize;
+ }
+
/// Builds external uses of the vectorized scalars, i.e. the list of
/// vectorized scalars to be extracted, their lanes and their scalar users. \p
/// ExternallyUsedValues contains additional list of external uses to handle
@@ -3938,6 +3946,11 @@ class slpvectorizer::BoUpSLP {
const TreeEntry &SelectTE,
SmallVectorImpl<unsigned> &InversedCmpsIndices) const;
+ /// Checks if the tree is reduction or of bit selects, like select %cmp, <1,
+ /// 2, 4, 8, ..>, zeroinitializer, which can be reduced just to a bitcast %cmp
+ /// to in.
+ bool matchesSelectOfBits(const TreeEntry &SelectTE) const;
+
class TreeEntry {
public:
using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
@@ -4077,6 +4090,7 @@ class slpvectorizer::BoUpSLP {
ReducedBitcastBSwap,
ReducedBitcastLoads,
ReducedBitcastBSwapLoads,
+ ReducedCmpBitcast,
};
CombinedOpcode CombinedOp = NotCombinedOp;
@@ -13557,6 +13571,64 @@ bool BoUpSLP::matchesInversedZExtSelect(
return !InversedCmpsIndices.empty();
}
+bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
+ assert(SelectTE.hasState() && SelectTE.getOpcode() == Instruction::Select &&
+ "Expected select node.");
+ if (!SelectTE.ReorderIndices.empty() || !SelectTE.ReuseShuffleIndices.empty())
+ return false;
+ if (!UserIgnoreList)
+ return false;
+ // Check that all reduction operands are or instructions.
+ if (any_of(*UserIgnoreList,
+ [](Value *V) { return !match(V, m_Or(m_Value(), m_Value())); }))
+ return false;
+ const TreeEntry *Op1TE = getOperandEntry(&SelectTE, 1);
+ const TreeEntry *Op2TE = getOperandEntry(&SelectTE, 2);
+ if (!Op1TE->isGather() || !Op2TE->isGather())
+ return false;
+ // No need to check for zeroes reordering.
+ if (!Op1TE->ReorderIndices.empty() || !Op1TE->ReuseShuffleIndices.empty() ||
+ !Op2TE->ReuseShuffleIndices.empty())
+ return false;
+ Type *ScalarTy = Op1TE->Scalars.front()->getType();
+ if (!ScalarTy->isIntegerTy())
+ return false;
+ // Check that second operand is all zeroes.
+ if (any_of(Op2TE->Scalars, [](Value *V) {
+ return !isa<UndefValue>(V) && !match(V, m_ZeroInt());
+ }))
+ return false;
+ // Check that first operand is 1,2,4,...
+ if (any_of(enumerate(Op1TE->Scalars), [](const auto &P) {
+ uint64_t V;
+ return !isa<UndefValue>(P.value()) &&
+ !(match(P.value(), m_ConstantInt(V)) && isPowerOf2_64(V) &&
+ Log2_64(V) == P.index());
+ }))
+ return false;
+ // Check if bitcast is cheaper than select.
+ auto *DstTy = IntegerType::getIntNTy(ScalarTy->getContext(),
+ SelectTE.getVectorFactor());
+ VectorType *OpTy = getWidenedType(DstTy, SelectTE.getVectorFactor());
+ Type *CmpTy = CmpInst::makeCmpResultType(OpTy);
+ VectorType *VecTy = getWidenedType(ScalarTy, SelectTE.getVectorFactor());
+ TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ InstructionCost BitcastCost = TTI->getCastInstrCost(
+ Instruction::BitCast, DstTy, CmpTy, TTI::CastContextHint::None, CostKind);
+ if (DstTy != ScalarTy) {
+ BitcastCost += TTI->getCastInstrCost(Instruction::ZExt, ScalarTy, DstTy,
+ TTI::CastContextHint::None, CostKind);
+ }
+ FastMathFlags FMF;
+ InstructionCost SelectCost =
+ TTI->getCmpSelInstrCost(Instruction::Select, OpTy, CmpTy,
+ CmpInst::BAD_ICMP_PREDICATE, CostKind,
+ getOperandInfo(Op1TE->Scalars),
+ getOperandInfo(Op2TE->Scalars)) +
+ TTI->getArithmeticReductionCost(Instruction::Or, VecTy, FMF, CostKind);
+ return BitcastCost <= SelectCost;
+}
+
void BoUpSLP::transformNodes() {
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
BaseGraphSize = VectorizableTree.size();
@@ -13976,6 +14048,18 @@ void BoUpSLP::transformNodes() {
}
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 1), Op1TE);
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 2), Op2TE);
+ //NB: Fallback to check if select can be converted to cmp bitcast.
+ }
+ if (matchesSelectOfBits(E)) {
+ // This node is a (reduced or) cmp bitcast node.
+ const TreeEntry::CombinedOpcode Code = TreeEntry::ReducedCmpBitcast;
+ E.CombinedOp = Code;
+ auto *Op1TE = getOperandEntry(&E, 1);
+ auto *Op2TE = getOperandEntry(&E, 2);
+ Op1TE->State = TreeEntry::CombinedVectorize;
+ Op1TE->CombinedOp = Code;
+ Op2TE->State = TreeEntry::CombinedVectorize;
+ Op2TE->CombinedOp = Code;
break;
}
break;
@@ -15741,6 +15825,32 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
};
return GetCostDiff(GetScalarCost, GetVectorCost);
}
+ case TreeEntry::ReducedCmpBitcast: {
+ auto GetScalarCost = [&, &TTI = *TTI](unsigned Idx) {
+ if (isa<PoisonValue>(UniqueValues[Idx]))
+ return InstructionCost(TTI::TCC_Free);
+ auto *Sel = dyn_cast<Instruction>(UniqueValues[Idx]);
+ if (!Sel)
+ return InstructionCost(TTI::TCC_Free);
+ InstructionCost ScalarCost = TTI.getInstructionCost(Sel, CostKind);
+ return ScalarCost;
+ };
+ auto GetVectorCost = [&, &TTI = *TTI](InstructionCost CommonCost) {
+ Type *CmpTy = CmpInst::makeCmpResultType(VecTy);
+ auto *DstTy =
+ IntegerType::getIntNTy(ScalarTy->getContext(), E->getVectorFactor());
+ InstructionCost BitcastCost =
+ TTI.getCastInstrCost(Instruction::BitCast, DstTy, CmpTy,
+ TTI::CastContextHint::None, CostKind);
+ if (DstTy != ScalarTy) {
+ BitcastCost +=
+ TTI.getCastInstrCost(Instruction::ZExt, ScalarTy, DstTy,
+ TTI::CastContextHint::None, CostKind);
+ }
+ return BitcastCost + CommonCost;
+ };
+ return GetCostDiff(GetScalarCost, GetVectorCost);
+ }
case Instruction::FNeg:
case Instruction::Add:
case Instruction::FAdd:
@@ -16455,7 +16565,8 @@ InstructionCost BoUpSLP::getSpillCost() {
if (TEPtr->CombinedOp == TreeEntry::ReducedBitcast ||
TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
TEPtr->CombinedOp == TreeEntry::ReducedBitcastLoads ||
- TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads) {
+ TEPtr->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
+ TEPtr->CombinedOp == TreeEntry::ReducedCmpBitcast) {
ScalarOrPseudoEntries.insert(TEPtr.get());
continue;
}
@@ -20499,6 +20610,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
case TreeEntry::ReducedBitcastBSwap:
case TreeEntry::ReducedBitcastLoads:
case TreeEntry::ReducedBitcastBSwapLoads:
+ case TreeEntry::ReducedCmpBitcast:
ShuffleOrOp = E->CombinedOp;
break;
default:
@@ -21492,6 +21604,26 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
E->VectorizedValue = V;
return V;
}
+ case TreeEntry::ReducedCmpBitcast: {
+ assert(UserIgnoreList && "Expected reduction operations only.");
+ setInsertPointAfterBundle(E);
+ TreeEntry *Op1TE = getOperandEntry(E, /*Idx=*/1);
+ TreeEntry *Op2TE = getOperandEntry(E, /*Idx=*/2);
+ Op1TE->VectorizedValue =
+ PoisonValue::get(getWidenedType(ScalarTy, Op1TE->getVectorFactor()));
+ Op2TE->VectorizedValue =
+ PoisonValue::get(getWidenedType(ScalarTy, Op2TE->getVectorFactor()));
+ Value *Cmp = vectorizeOperand(E, /*NodeIdx=*/0);
+ // Set the scalar type properly to avoid casting to the extending type.
+ auto *DstTy =
+ IntegerType::getIntNTy(ScalarTy->getContext(), E->getVectorFactor());
+ auto *V = Builder.CreateBitCast(Cmp, DstTy);
+ ++NumVectorInstructions;
+ V = Builder.CreateIntCast(V, ScalarTy, /*isSigned=*/false);
+ E->VectorizedValue = V;
+ ++NumVectorInstructions;
+ return V;
+ }
default:
llvm_unreachable("unknown inst");
}
@@ -21523,7 +21655,8 @@ Value *BoUpSLP::vectorizeTree(
(TE->CombinedOp == TreeEntry::ReducedBitcast ||
TE->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
((TE->CombinedOp == TreeEntry::ReducedBitcastLoads ||
- TE->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads) &&
+ TE->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
+ TE->CombinedOp == TreeEntry::ReducedCmpBitcast) &&
(!TE->hasState() || TE->getOpcode() != Instruction::Load)))))
continue;
(void)getLastInstructionInBundle(TE.get());
@@ -22085,7 +22218,8 @@ Value *BoUpSLP::vectorizeTree(
if (Entry->CombinedOp == TreeEntry::ReducedBitcast ||
Entry->CombinedOp == TreeEntry::ReducedBitcastBSwap ||
Entry->CombinedOp == TreeEntry::ReducedBitcastLoads ||
- Entry->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads) {
+ Entry->CombinedOp == TreeEntry::ReducedBitcastBSwapLoads ||
+ Entry->CombinedOp == TreeEntry::ReducedCmpBitcast) {
// Skip constant node
if (!Entry->hasState()) {
assert(allConstant(Entry->Scalars) && "Expected constants only.");
@@ -26105,7 +26239,7 @@ class HorizontalReduction {
// Estimate cost.
InstructionCost ReductionCost;
- if (V.isReducedBitcastRoot())
+ if (V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot())
ReductionCost = 0;
else
ReductionCost =
@@ -26233,7 +26367,8 @@ class HorizontalReduction {
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(),
- V.isReducedBitcastRoot()));
+ V.isReducedBitcastRoot() ||
+ V.isReducedCmpBitcastRoot()));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
index b7f79df782429..1eb94d40079a2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
@@ -19,16 +19,16 @@ define i64 @bitmask_16xi8(ptr nocapture noundef readonly %src) {
; SSE-NEXT: entry:
; SSE-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[SRC:%.*]], align 1
; SSE-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
-; SSE-NEXT: [[TMP4:%.*]] = select <16 x i1> [[TMP1]], <16 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768>, <16 x i64> zeroinitializer
-; SSE-NEXT: [[OP_RDX7:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP4]])
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <16 x i1> [[TMP1]] to i16
+; SSE-NEXT: [[OP_RDX7:%.*]] = zext i16 [[TMP2]] to i64
; SSE-NEXT: ret i64 [[OP_RDX7]]
;
; AVX-LABEL: @bitmask_16xi8(
; AVX-NEXT: entry:
; AVX-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[SRC:%.*]], align 1
; AVX-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
-; AVX-NEXT: [[TMP4:%.*]] = select <16 x i1> [[TMP1]], <16 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768>, <16 x i64> zeroinitializer
-; AVX-NEXT: [[OP_RDX4:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP4]])
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <16 x i1> [[TMP1]] to i16
+; AVX-NEXT: [[OP_RDX4:%.*]] = zext i16 [[TMP2]] to i64
; AVX-NEXT: ret i64 [[OP_RDX4]]
;
; AVX512-LABEL: @bitmask_16xi8(
@@ -126,16 +126,16 @@ define i64 @bitmask_4xi16(ptr nocapture noundef readonly %src) {
; SSE-NEXT: entry:
; SSE-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[SRC:%.*]], align 2
; SSE-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[TMP0]], zeroinitializer
-; SSE-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; SSE-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; SSE-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; SSE-NEXT: ret i64 [[OP_RDX3]]
;
; AVX-LABEL: @bitmask_4xi16(
; AVX-NEXT: entry:
; AVX-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[SRC:%.*]], align 2
; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[TMP0]], zeroinitializer
-; AVX-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX-NEXT: ret i64 [[OP_RDX3]]
;
; AVX512-LABEL: @bitmask_4xi16(
@@ -193,16 +193,16 @@ define i64 @bitmask_8xi32(ptr nocapture noundef readonly %src) {
; SSE-NEXT: entry:
; SSE-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[SRC:%.*]], align 4
; SSE-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[TMP0]], zeroinitializer
-; SSE-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; SSE-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; SSE-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; SSE-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; SSE-NEXT: ret i64 [[OP_RDX3]]
;
; AVX-LABEL: @bitmask_8xi32(
; AVX-NEXT: entry:
; AVX-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[SRC:%.*]], align 4
; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[TMP0]], zeroinitializer
-; AVX-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX-NEXT: ret i64 [[OP_RDX3]]
;
; AVX512-LABEL: @bitmask_8xi32(
@@ -302,16 +302,16 @@ define i64 @bitmask_8xi64(ptr nocapture noundef readonly %src) {
; SSE4-NEXT: entry:
; SSE4-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 8
; SSE4-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
-; SSE4-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; SSE4-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; SSE4-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; SSE4-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; SSE4-NEXT: ret i64 [[OP_RDX3]]
;
; AVX-LABEL: @bitmask_8xi64(
; AVX-NEXT: entry:
; AVX-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 8
; AVX-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
-; AVX-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX-NEXT: ret i64 [[OP_RDX3]]
;
; AVX512-LABEL: @bitmask_8xi64(
>From 9a31d7bea57bb94b8d2edd1e4b0a83fb65dac8d1 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 17 Feb 2026 15:51:50 -0800
Subject: [PATCH 2/6] Remove extra parens
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f5a01eeb938a7..049c8b1cac0a3 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2120,8 +2120,8 @@ class slpvectorizer::BoUpSLP {
/// Returns true if the tree results in the reduced cmp bitcast root.
bool isReducedCmpBitcastRoot() const {
return VectorizableTree.front()->hasState() &&
- (VectorizableTree.front()->CombinedOp ==
- TreeEntry::ReducedCmpBitcast) &&
+ VectorizableTree.front()->CombinedOp ==
+ TreeEntry::ReducedCmpBitcast &&
VectorizableTree.front()->State == TreeEntry::Vectorize;
}
>From d62076311740b47bbd71cb70c1ee0e6b99339986 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 17 Feb 2026 15:54:01 -0800
Subject: [PATCH 3/6] Format a comment
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 049c8b1cac0a3..988551bf0652a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -14048,7 +14048,7 @@ void BoUpSLP::transformNodes() {
}
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 1), Op1TE);
OperandsToTreeEntry.emplace_or_assign(std::make_pair(&E, 2), Op2TE);
- //NB: Fallback to check if select can be converted to cmp bitcast.
+ // NB: Fallback to check if select can be converted to cmp bitcast.
}
if (matchesSelectOfBits(E)) {
// This node is a (reduced or) cmp bitcast node.
>From 0300a5b411711edd3cbfa77a1ef7a8a27e7beaf2 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 17 Feb 2026 16:18:26 -0800
Subject: [PATCH 4/6] Address comments
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 15 +++++++++++----
.../Transforms/SLPVectorizer/X86/bool-mask.ll | 16 ++++++++--------
2 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 988551bf0652a..6838361b1c709 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13574,6 +13574,8 @@ bool BoUpSLP::matchesInversedZExtSelect(
bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
assert(SelectTE.hasState() && SelectTE.getOpcode() == Instruction::Select &&
"Expected select node.");
+ if (DL->isBigEndian())
+ return false;
if (!SelectTE.ReorderIndices.empty() || !SelectTE.ReuseShuffleIndices.empty())
return false;
if (!UserIgnoreList)
@@ -13595,14 +13597,13 @@ bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
return false;
// Check that second operand is all zeroes.
if (any_of(Op2TE->Scalars, [](Value *V) {
- return !isa<UndefValue>(V) && !match(V, m_ZeroInt());
+ return !match(V, m_ZeroInt());
}))
return false;
// Check that first operand is 1,2,4,...
if (any_of(enumerate(Op1TE->Scalars), [](const auto &P) {
uint64_t V;
- return !isa<UndefValue>(P.value()) &&
- !(match(P.value(), m_ConstantInt(V)) && isPowerOf2_64(V) &&
+ return !(match(P.value(), m_ConstantInt(V)) && isPowerOf2_64(V) &&
Log2_64(V) == P.index());
}))
return false;
@@ -13612,6 +13613,12 @@ bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
VectorType *OpTy = getWidenedType(DstTy, SelectTE.getVectorFactor());
Type *CmpTy = CmpInst::makeCmpResultType(OpTy);
VectorType *VecTy = getWidenedType(ScalarTy, SelectTE.getVectorFactor());
+ auto It = MinBWs.find(&SelectTE);
+ if (It != MinBWs.end()) {
+ auto *EffectiveScalarTy =
+ IntegerType::get(F->getContext(), It->second.first);
+ VecTy = getWidenedType(EffectiveScalarTy, SelectTE.getVectorFactor());
+ }
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
InstructionCost BitcastCost = TTI->getCastInstrCost(
Instruction::BitCast, DstTy, CmpTy, TTI::CastContextHint::None, CostKind);
@@ -13621,7 +13628,7 @@ bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
}
FastMathFlags FMF;
InstructionCost SelectCost =
- TTI->getCmpSelInstrCost(Instruction::Select, OpTy, CmpTy,
+ TTI->getCmpSelInstrCost(Instruction::Select, VecTy, CmpTy,
CmpInst::BAD_ICMP_PREDICATE, CostKind,
getOperandInfo(Op1TE->Scalars),
getOperandInfo(Op2TE->Scalars)) +
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
index 1eb94d40079a2..9073600eb3808 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bool-mask.ll
@@ -35,8 +35,8 @@ define i64 @bitmask_16xi8(ptr nocapture noundef readonly %src) {
; AVX512-NEXT: entry:
; AVX512-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[SRC:%.*]], align 1
; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
-; AVX512-NEXT: [[TMP4:%.*]] = select <16 x i1> [[TMP1]], <16 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256, i64 512, i64 1024, i64 2048, i64 4096, i64 8192, i64 16384, i64 32768>, <16 x i64> zeroinitializer
-; AVX512-NEXT: [[OP_RDX4:%.*]] = call i64 @llvm.vector.reduce.or.v16i64(<16 x i64> [[TMP4]])
+; AVX512-NEXT: [[TMP2:%.*]] = bitcast <16 x i1> [[TMP1]] to i16
+; AVX512-NEXT: [[OP_RDX4:%.*]] = zext i16 [[TMP2]] to i64
; AVX512-NEXT: ret i64 [[OP_RDX4]]
;
entry:
@@ -142,8 +142,8 @@ define i64 @bitmask_4xi16(ptr nocapture noundef readonly %src) {
; AVX512-NEXT: entry:
; AVX512-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[SRC:%.*]], align 2
; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[TMP0]], zeroinitializer
-; AVX512-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX512-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX512-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX512-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX512-NEXT: ret i64 [[OP_RDX3]]
;
entry:
@@ -209,8 +209,8 @@ define i64 @bitmask_8xi32(ptr nocapture noundef readonly %src) {
; AVX512-NEXT: entry:
; AVX512-NEXT: [[TMP0:%.*]] = load <8 x i32>, ptr [[SRC:%.*]], align 4
; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[TMP0]], zeroinitializer
-; AVX512-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX512-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX512-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX512-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX512-NEXT: ret i64 [[OP_RDX3]]
;
entry:
@@ -318,8 +318,8 @@ define i64 @bitmask_8xi64(ptr nocapture noundef readonly %src) {
; AVX512-NEXT: entry:
; AVX512-NEXT: [[TMP0:%.*]] = load <8 x i64>, ptr [[SRC:%.*]], align 8
; AVX512-NEXT: [[TMP1:%.*]] = icmp ne <8 x i64> [[TMP0]], zeroinitializer
-; AVX512-NEXT: [[TMP4:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> <i64 1, i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128>, <8 x i64> zeroinitializer
-; AVX512-NEXT: [[OP_RDX3:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP4]])
+; AVX512-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; AVX512-NEXT: [[OP_RDX3:%.*]] = zext i8 [[TMP2]] to i64
; AVX512-NEXT: ret i64 [[OP_RDX3]]
;
entry:
>From 601c784fff334abcd36ae5ded7a43e5863816d0d Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 17 Feb 2026 16:23:11 -0800
Subject: [PATCH 5/6] Fix formatting
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6838361b1c709..9f7f27ba4d0b2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -13596,9 +13596,7 @@ bool BoUpSLP::matchesSelectOfBits(const TreeEntry &SelectTE) const {
if (!ScalarTy->isIntegerTy())
return false;
// Check that second operand is all zeroes.
- if (any_of(Op2TE->Scalars, [](Value *V) {
- return !match(V, m_ZeroInt());
- }))
+ if (any_of(Op2TE->Scalars, [](Value *V) { return !match(V, m_ZeroInt()); }))
return false;
// Check that first operand is 1,2,4,...
if (any_of(enumerate(Op1TE->Scalars), [](const auto &P) {
>From 38f5c7e2546b34081432cfd53c652aec660738e0 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Wed, 18 Feb 2026 13:51:05 -0800
Subject: [PATCH 6/6] Address comments
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9f7f27ba4d0b2..2e875bd4806e2 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -21624,9 +21624,11 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
IntegerType::getIntNTy(ScalarTy->getContext(), E->getVectorFactor());
auto *V = Builder.CreateBitCast(Cmp, DstTy);
++NumVectorInstructions;
- V = Builder.CreateIntCast(V, ScalarTy, /*isSigned=*/false);
+ if (DstTy != ScalarTy) {
+ V = Builder.CreateIntCast(V, ScalarTy, /*isSigned=*/false);
+ ++NumVectorInstructions;
+ }
E->VectorizedValue = V;
- ++NumVectorInstructions;
return V;
}
default:
More information about the llvm-commits
mailing list