[llvm] [SLP]Booleanize logical and/or nodes in the wide leaf type (PR #223250)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 13 07:42:34 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
i1 and/or tree nodes with booleanized wide leaves (one-use truncs of
wide values, zero-tests of values from [0, 1]) are registered in MinBWs
and emitted in the wide type; bit 0 of the result is the final value.
---
Full diff: https://github.com/llvm/llvm-project/pull/223250.diff
2 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+161-2)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/logical-binops-booleanized-leaves.ll (+24-22)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4feedf5febff3e..8ac2bba4bca05f 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -460,6 +460,11 @@ class slpvectorizer::BoUpSLP {
calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals = {},
Instruction *RdxRoot = nullptr);
+ /// Finds i1 and/or nodes whose operand nodes are booleanized wide leaves
+ /// (one-use truncs of wide values or zero-tests of values from [0, 1]) and
+ /// registers them in MinBWs to be emitted in the wide leaf type.
+ void detectBooleanizedNodes();
+
/// \returns the vectorization cost of the subtree that starts at \p VL.
/// A negative number means that this is profitable.
InstructionCost getTreeCost(InstructionCost TreeCost,
@@ -5353,6 +5358,14 @@ class slpvectorizer::BoUpSLP {
/// original width.
DenseMap<const TreeEntry *, std::pair<uint64_t, bool>> MinBWs;
+ /// A booleanized node is an i1 node registered in MinBWs to be emitted in
+ /// the wide leaf type. Genuine demoted nodes never have i1 scalars (i1
+ /// cannot be demoted), so an i1 node in MinBWs is always a booleanized one.
+ bool isBooleanizedNode(const TreeEntry *E) const {
+ auto It = MinBWs.find(E);
+ return It != MinBWs.end() && E->Scalars.front()->getType()->isIntegerTy(1);
+ }
+
/// Final size of the reduced vector, if the current graph represents the
/// input for the reduction and it was possible to narrow the size of the
/// reduction.
@@ -16379,6 +16392,10 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
return ScalarCost;
};
auto GetVectorCost = [&](InstructionCost CommonCost) {
+ // An absorbed boolean zero-test leaf emits no conversion; only the
+ // reorder shuffle (in CommonCost) may remain.
+ if (isBooleanizedNode(E))
+ return CommonCost;
// For selects, the condition type may differ from the result type
// (e.g. condition is <N x i1> while result is <N x i32>). For
// compares, the result type IS the mask (i1/vNi1). Construct the
@@ -18222,6 +18239,133 @@ static T *performExtractsShuffleAction(
return Prev;
}
+void BoUpSLP::detectBooleanizedNodes() {
+ // An absorbable boolean leaf is a one-use trunc of a wide value or a
+ // one-use zero-test of a value from [0, 1]; bit 0 of the wide value is the
+ // i1 lane value in both cases.
+ auto GetBoolLeafWideTy = [&](const TreeEntry *OpE) -> Type * {
+ if (OpE->State != TreeEntry::Vectorize || OpE->isAltShuffle() ||
+ OpE->hasCopyableElements() || !OpE->ReuseShuffleIndices.empty() ||
+ OpE->CombinedOp != TreeEntry::NotCombinedOp)
+ return nullptr;
+ unsigned WideOpIdx;
+ bool IsZeroTest = OpE->getOpcode() == Instruction::ICmp;
+ if (OpE->getOpcode() == Instruction::Trunc) {
+ if (!all_of(OpE->Scalars, [](Value *V) {
+ return match(V, m_OneUse(m_Trunc(m_Value())));
+ }))
+ return nullptr;
+ WideOpIdx = 0;
+ } else if (IsZeroTest) {
+ if (!all_of(OpE->Scalars, [](Value *V) {
+ auto *CI = dyn_cast<ICmpInst>(V);
+ return CI && CI->hasOneUse() &&
+ CI->getPredicate() == ICmpInst::ICMP_NE;
+ }))
+ return nullptr;
+ if (all_of(OpE->getOperand(0),
+ [](Value *V) { return match(V, m_Zero()); }))
+ WideOpIdx = 1;
+ else if (all_of(OpE->getOperand(1),
+ [](Value *V) { return match(V, m_Zero()); }))
+ WideOpIdx = 0;
+ else
+ return nullptr;
+ } else {
+ return nullptr;
+ }
+ Type *WideTy = nullptr;
+ for (Value *W : OpE->getOperand(WideOpIdx)) {
+ if (!W->getType()->isIntegerTy() || (WideTy && W->getType() != WideTy) ||
+ !isGuaranteedNotToBePoison(W) ||
+ (IsZeroTest && computeKnownBits(W, *DL).getMaxValue() != 1))
+ return nullptr;
+ WideTy = W->getType();
+ }
+ return WideTy;
+ };
+ // The wide value of a booleanized node is valid only in bit 0, so every
+ // tree user must consume it as an i1 vector: either a booleanized and/or
+ // node (which uses the wide value directly) or a node whose emission
+ // truncates the operand back to the i1 vector type.
+ auto IsSafeBoolUser = [&](const TreeEntry *UserTE, unsigned EdgeIdx) {
+ if (!UserTE->hasState() || UserTE->isAltShuffle())
+ return false;
+ if (Instruction::isBinaryOp(UserTE->getOpcode()))
+ return true;
+ switch (UserTE->getOpcode()) {
+ case Instruction::Store:
+ return EdgeIdx == 0;
+ case Instruction::InsertElement:
+ return EdgeIdx == 1;
+ case Instruction::Select:
+ return EdgeIdx != 0;
+ case Instruction::Freeze:
+ case Instruction::PHI:
+ return true;
+ default:
+ return false;
+ }
+ };
+ // Visit operand nodes before their users so that booleanized and/or nodes
+ // chain: an and/or node whose operand is itself a booleanized and/or node
+ // consumes the wide value directly.
+ for (const std::unique_ptr<TreeEntry> &TE : reverse(VectorizableTree)) {
+ // Logical and/or on i1 with booleanized wide leaves is emitted in the
+ // wide type: bit 0 of a bitwise and/or is the and/or of the operands'
+ // bit 0. The leaf conversions are absorbed and a single trunc of the
+ // wide vector result restores the i1 lanes. Wide leaves must be
+ // non-poison: and/or may mask poison in the i1 type, which the wide ops
+ // would propagate instead.
+ if (TE->State != TreeEntry::Vectorize || TE->isAltShuffle() ||
+ (TE->getOpcode() != Instruction::And &&
+ TE->getOpcode() != Instruction::Or) ||
+ !TE->getMainOp()->getType()->isIntegerTy(1) ||
+ !TE->ReuseShuffleIndices.empty() || TE->hasCopyableElements() ||
+ TE->CombinedOp != TreeEntry::NotCombinedOp ||
+ // The i1 flags (e.g. disjoint) do not apply to the wide op.
+ any_of(TE->Scalars,
+ [](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return !I || I->hasPoisonGeneratingFlags();
+ }) ||
+ any_of(TE->getReassocScalars(),
+ [](Value *V) {
+ return cast<Instruction>(V)->hasPoisonGeneratingFlags();
+ }) ||
+ // Reduction trees are handled by the reduction's own booleanization.
+ UserIgnoreList ||
+ (TE->UserTreeIndex.UserTE &&
+ !IsSafeBoolUser(TE->UserTreeIndex.UserTE,
+ TE->UserTreeIndex.EdgeIdx)) ||
+ !all_of(TE->Scalars,
+ [&](Value *V) { return getTreeEntries(V).size() == 1; }))
+ continue;
+ Type *WideTy = nullptr;
+ bool Bail = false;
+ for (unsigned I : seq<unsigned>(TE->getNumOperands())) {
+ const TreeEntry *OpE = getOperandEntry(TE.get(), I);
+ Type *ColTy = GetBoolLeafWideTy(OpE);
+ if (!ColTy && isBooleanizedNode(OpE) &&
+ (OpE->getOpcode() == Instruction::And ||
+ OpE->getOpcode() == Instruction::Or))
+ ColTy = IntegerType::get(F->getContext(), MinBWs.lookup(OpE).first);
+ if (!ColTy || (WideTy && ColTy != WideTy)) {
+ Bail = true;
+ break;
+ }
+ WideTy = ColTy;
+ }
+ if (Bail || !WideTy)
+ continue;
+ unsigned WideBW = WideTy->getIntegerBitWidth();
+ MinBWs.try_emplace(TE.get(), WideBW, /*IsSigned=*/false);
+ for (unsigned I : seq<unsigned>(TE->getNumOperands()))
+ MinBWs.try_emplace(getOperandEntry(TE.get(), I), WideBW,
+ /*IsSigned=*/false);
+ }
+}
+
InstructionCost
BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
Instruction *RdxRoot) {
@@ -23141,8 +23285,19 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
- Value *V = Builder.CreateCmp(P0, L, R);
- V = PropagateIRFlags(V);
+ // A booleanized zero-test leaf emits no comparison: its value is the
+ // wide operand vector, and the i1 lanes are restored by the trunc at
+ // the booleanized user node. (Trunc leaves are no-ops after the generic
+ // MinBWs cast processing.)
+ bool IsBooleanized = isBooleanizedNode(E);
+ Value *V = IsBooleanized
+ ? (all_of(E->getOperand(0),
+ [](Value *V) { return match(V, m_Zero()); })
+ ? R
+ : L)
+ : Builder.CreateCmp(P0, L, R);
+ if (!IsBooleanized)
+ V = PropagateIRFlags(V);
// Do not cast for cmps.
VecTy = cast<FixedVectorType>(V->getType());
V = FinalShuffle(V, E);
@@ -27499,6 +27654,10 @@ bool BoUpSLP::collectValuesToDemote(
static RecurKind getRdxKind(Value *V);
void BoUpSLP::computeMinimumValueSizes() {
+ // Detect booleanized i1 and/or nodes first: they are registered in MinBWs
+ // like the demoted nodes and processed the same way.
+ detectBooleanizedNodes();
+
// We only attempt to truncate integer expressions.
bool IsStoreOrInsertElt =
getRootNode().hasState() &&
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/logical-binops-booleanized-leaves.ll b/llvm/test/Transforms/SLPVectorizer/X86/logical-binops-booleanized-leaves.ll
index 3a211ca08b70f6..af56b05764a61e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/logical-binops-booleanized-leaves.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/logical-binops-booleanized-leaves.ll
@@ -7,31 +7,37 @@ define void @and_store_8(ptr %p, ptr %q, ptr %out) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[P]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[Q]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = trunc <8 x i8> [[TMP0]] to <8 x i1>
-; CHECK-NEXT: [[TMP3:%.*]] = trunc <8 x i8> [[TMP1]] to <8 x i1>
-; CHECK-NEXT: [[TMP5:%.*]] = and <8 x i1> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i8> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i8> [[TMP2]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = trunc i8 [[TMP3]] to i1
; CHECK-NEXT: store i1 [[TMP4]], ptr [[OUT]], align 1
; CHECK-NEXT: [[O1:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i8> [[TMP2]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i8 [[TMP5]] to i1
; CHECK-NEXT: store i1 [[TMP6]], ptr [[O1]], align 1
; CHECK-NEXT: [[O2:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP5]], i64 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i8> [[TMP2]], i64 2
+; CHECK-NEXT: [[TMP8:%.*]] = trunc i8 [[TMP7]] to i1
; CHECK-NEXT: store i1 [[TMP8]], ptr [[O2]], align 1
; CHECK-NEXT: [[O3:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 3
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i1> [[TMP5]], i64 3
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i8> [[TMP2]], i64 3
+; CHECK-NEXT: [[TMP10:%.*]] = trunc i8 [[TMP9]] to i1
; CHECK-NEXT: store i1 [[TMP10]], ptr [[O3]], align 1
; CHECK-NEXT: [[O4:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 4
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[TMP5]], i64 4
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i8> [[TMP2]], i64 4
+; CHECK-NEXT: [[TMP12:%.*]] = trunc i8 [[TMP11]] to i1
; CHECK-NEXT: store i1 [[TMP12]], ptr [[O4]], align 1
; CHECK-NEXT: [[O5:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 5
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x i1> [[TMP5]], i64 5
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i8> [[TMP2]], i64 5
+; CHECK-NEXT: [[TMP14:%.*]] = trunc i8 [[TMP13]] to i1
; CHECK-NEXT: store i1 [[TMP14]], ptr [[O5]], align 1
; CHECK-NEXT: [[O6:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 6
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <8 x i1> [[TMP5]], i64 6
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i8> [[TMP2]], i64 6
+; CHECK-NEXT: [[TMP16:%.*]] = trunc i8 [[TMP15]] to i1
; CHECK-NEXT: store i1 [[TMP16]], ptr [[O6]], align 1
; CHECK-NEXT: [[O7:%.*]] = getelementptr inbounds nuw i1, ptr [[OUT]], i64 7
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i1> [[TMP5]], i64 7
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i8> [[TMP2]], i64 7
+; CHECK-NEXT: [[TMP18:%.*]] = trunc i8 [[TMP17]] to i1
; CHECK-NEXT: store i1 [[TMP18]], ptr [[O7]], align 1
; CHECK-NEXT: ret void
;
@@ -114,9 +120,8 @@ define <8 x i1> @or_icmp_bv(ptr %p, ptr %q) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[P]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[Q]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <8 x i8> [[TMP0]], zeroinitializer
-; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <8 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = or <8 x i1> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP2:%.*]] = or <8 x i8> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = trunc <8 x i8> [[TMP2]] to <8 x i1>
; CHECK-NEXT: ret <8 x i1> [[TMP3]]
;
entry:
@@ -192,11 +197,9 @@ define <8 x i1> @and_chain_bv(ptr %p, ptr %q, ptr %r) {
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[P]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[Q]], align 1
; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr [[R]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = trunc <8 x i8> [[TMP0]] to <8 x i1>
-; CHECK-NEXT: [[TMP4:%.*]] = trunc <8 x i8> [[TMP1]] to <8 x i1>
-; CHECK-NEXT: [[TMP7:%.*]] = trunc <8 x i8> [[TMP2]] to <8 x i1>
-; CHECK-NEXT: [[TMP6:%.*]] = and <8 x i1> [[TMP3]], [[TMP4]]
-; CHECK-NEXT: [[TMP5:%.*]] = and <8 x i1> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP3:%.*]] = and <8 x i8> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = and <8 x i8> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = trunc <8 x i8> [[TMP4]] to <8 x i1>
; CHECK-NEXT: ret <8 x i1> [[TMP5]]
;
entry:
@@ -478,9 +481,7 @@ define <8 x i1> @and_select_val(ptr %p, ptr %q, ptr %c) {
; CHECK-NEXT: [[C7:%.*]] = load i1, ptr [[C7P]], align 1
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[P]], align 1
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[Q]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = trunc <8 x i8> [[TMP0]] to <8 x i1>
-; CHECK-NEXT: [[TMP13:%.*]] = trunc <8 x i8> [[TMP1]] to <8 x i1>
-; CHECK-NEXT: [[TMP11:%.*]] = and <8 x i1> [[TMP2]], [[TMP13]]
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i8> [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i1> poison, i1 [[C0]], i64 0
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[C1]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[C2]], i64 2
@@ -489,6 +490,7 @@ define <8 x i1> @and_select_val(ptr %p, ptr %q, ptr %c) {
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[C5]], i64 5
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i1> [[TMP8]], i1 [[C6]], i64 6
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[C7]], i64 7
+; CHECK-NEXT: [[TMP11:%.*]] = trunc <8 x i8> [[TMP2]] to <8 x i1>
; CHECK-NEXT: [[TMP12:%.*]] = select <8 x i1> [[TMP10]], <8 x i1> [[TMP11]], <8 x i1> zeroinitializer
; CHECK-NEXT: ret <8 x i1> [[TMP12]]
;
``````````
</details>
https://github.com/llvm/llvm-project/pull/223250
More information about the llvm-commits
mailing list