[llvm] [SLP] Vectorize zero-tested OR/UMax reductions (PR #205473)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jun 28 11:01:21 PDT 2026
https://github.com/ParkHanbum updated https://github.com/llvm/llvm-project/pull/205473
>From 2b40ec7dac871fe11c0614b55486f8b8fad20024 Mon Sep 17 00:00:00 2001
From: Hanbum Park <kese111 at gmail.com>
Date: Tue, 23 Jun 2026 23:36:38 +0900
Subject: [PATCH 1/6] [SLP] Vectorize zero-tested OR/UMax reductions
When a scalar OR or UMax reduction has a single eq/ne-zero use, form
an equivalent lane-wise comparison followed by an i1 OR/AND reduction.
This avoids generating an integer vector reduction only to test its
scalar result, and exposes a boolean reduction that targets can lower
directly. Model the vectorization cost as a vector compare plus the
corresponding boolean reduction.
Add regression tests for both predicates, operand orders, supported
vector widths, and non-transformable uses.
Fixes #195118
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 195 +++++++++++++++++-
.../WebAssembly/or-reduction-zero-test.ll | 42 +---
.../X86/reduction-zero-test-ctpop.ll | 24 ++-
3 files changed, 213 insertions(+), 48 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d1fb8e2beea50..b4f2189983043 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28713,6 +28713,19 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
+ enum class ZeroTestLoweringKind {
+ BoolReduction,
+ Ctpop,
+ };
+ struct ZeroTestLoweringInfo {
+ ZeroTestLoweringKind Kind;
+ InstructionCost Cost;
+ };
+ struct ReductionZeroTestInfo {
+ ICmpInst *Cmp;
+ CmpPredicate Pred;
+ };
+
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
@@ -28744,6 +28757,107 @@ class HorizontalReduction {
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
+ std::optional<ReductionZeroTestInfo> matchReductionZeroTestUse() const {
+ auto *Root = dyn_cast<Instruction>(ReductionRoot);
+ if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
+ (RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
+ return std::nullopt;
+
+ auto *Cmp = dyn_cast<ICmpInst>(*Root->user_begin());
+ CmpPredicate Pred;
+ if (!Cmp || !match(Cmp, m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
+ !ICmpInst::isEquality(Pred))
+ return std::nullopt;
+
+ return ReductionZeroTestInfo{Cmp, Pred};
+ }
+
+ static InstructionCost
+ getBoolReductionZeroTestCost(const TargetTransformInfo &TTI,
+ FixedVectorType *VecTy, CmpPredicate Pred,
+ TTI::TargetCostKind CostKind) {
+ auto *CmpTy = cast<VectorType>(CmpInst::makeCmpResultType(VecTy));
+ unsigned ReductionOpcode =
+ Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
+ return TTI.getCmpSelInstrCost(Instruction::ICmp, VecTy, CmpTy, Pred,
+ CostKind) +
+ TTI.getArithmeticReductionCost(ReductionOpcode, CmpTy, {},
+ CostKind);
+ }
+
+ static InstructionCost
+ getCtpopZeroTestCost(const TargetTransformInfo &TTI, FixedVectorType *VecTy,
+ CmpPredicate Pred, TTI::TargetCostKind CostKind) {
+ LLVMContext &Ctx = VecTy->getContext();
+ ElementCount EC = VecTy->getElementCount();
+ if (EC.isScalable())
+ return InstructionCost::getInvalid();
+
+ auto *CmpTy = cast<VectorType>(CmpInst::makeCmpResultType(VecTy));
+ auto *MaskIntTy = IntegerType::get(Ctx, EC.getFixedValue());
+ InstructionCost CmpCost = TTI.getCmpSelInstrCost(
+ Instruction::ICmp, VecTy, CmpTy, ICmpInst::ICMP_NE, CostKind);
+ InstructionCost BitcastCost = TTI.getCastInstrCost(
+ Instruction::BitCast, MaskIntTy, CmpTy, TTI::CastContextHint::None,
+ CostKind);
+ IntrinsicCostAttributes ICA(Intrinsic::ctpop, MaskIntTy, {MaskIntTy});
+ InstructionCost CtpopCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
+ InstructionCost ScalarCmpCost = TTI.getCmpSelInstrCost(
+ Instruction::ICmp, MaskIntTy, Type::getInt1Ty(Ctx), Pred, CostKind);
+ return CmpCost + BitcastCost + CtpopCost + ScalarCmpCost;
+ }
+
+ static std::optional<ZeroTestLoweringInfo>
+ getBestReductionZeroTestLowering(const TargetTransformInfo &TTI,
+ FixedVectorType *VecTy, CmpPredicate Pred,
+ TTI::TargetCostKind CostKind) {
+ if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
+ return std::nullopt;
+
+ InstructionCost BoolCost =
+ getBoolReductionZeroTestCost(TTI, VecTy, Pred, CostKind);
+ InstructionCost CtpopCost =
+ getCtpopZeroTestCost(TTI, VecTy, Pred, CostKind);
+ if (!BoolCost.isValid() && !CtpopCost.isValid())
+ return std::nullopt;
+ if (!CtpopCost.isValid())
+ return ZeroTestLoweringInfo{ZeroTestLoweringKind::BoolReduction,
+ BoolCost};
+ if (!BoolCost.isValid())
+ return ZeroTestLoweringInfo{ZeroTestLoweringKind::Ctpop, CtpopCost};
+ if (CtpopCost < BoolCost)
+ return ZeroTestLoweringInfo{ZeroTestLoweringKind::Ctpop, CtpopCost};
+ return ZeroTestLoweringInfo{ZeroTestLoweringKind::BoolReduction, BoolCost};
+ }
+
+ static Value *createBoolReductionZeroTest(IRBuilderBase &Builder, Value *Vec,
+ CmpPredicate Pred) {
+ auto *VecTy = dyn_cast<VectorType>(Vec->getType());
+ if (!VecTy)
+ return nullptr;
+
+ Value *Cmp =
+ Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
+ RecurKind ReductionKind =
+ Pred == ICmpInst::ICMP_EQ ? RecurKind::And : RecurKind::Or;
+ return createSimpleReduction(Builder, Cmp, ReductionKind);
+ }
+
+ static Value *createCtpopZeroTest(IRBuilderBase &Builder, Value *Vec,
+ CmpPredicate Pred) {
+ auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+ if (!VecTy)
+ return nullptr;
+
+ Type *MaskIntTy =
+ Builder.getIntNTy(VecTy->getElementCount().getFixedValue());
+ Value *LaneNonZero =
+ Builder.CreateICmpNE(Vec, Constant::getNullValue(VecTy));
+ Value *MaskBits = Builder.CreateBitCast(LaneNonZero, MaskIntTy);
+ Value *Pop = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, MaskBits);
+ return Builder.CreateICmp(Pred, Pop, ConstantInt::get(MaskIntTy, 0));
+ }
+
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
@@ -30224,7 +30338,59 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ // Fold an eq/ne-zero test of a single integer reduction into a lane-wise
+ // comparison and an i1 reduction. For example, transform:
+ // %r = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
+ // %cmp = icmp ne i8 %r, 0
+ // into:
+ // %lanes = icmp ne <16 x i8> %v, zeroinitializer
+ // %cmp = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %lanes)
+ bool ReplacedZeroTest = false;
+ if (auto ZeroTest = matchReductionZeroTestUse()) {
+ auto *Reduction = dyn_cast<IntrinsicInst>(VectorizedTree);
+ Intrinsic::ID ReductionID = RdxKind == RecurKind::Or
+ ? Intrinsic::vector_reduce_or
+ : Intrinsic::vector_reduce_umax;
+ if (Reduction && Reduction->getIntrinsicID() == ReductionID) {
+ Value *Vec = Reduction->getArgOperand(0);
+ auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+ if (VecTy && VecTy->getElementType()->isIntegerTy()) {
+ if (auto Lowering = getBestReductionZeroTestLowering(
+ *TTI, VecTy, ZeroTest->Pred, TTI::TCK_RecipThroughput)) {
+ Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
+ Builder.SetCurrentDebugLocation(ZeroTest->Cmp->getDebugLoc());
+ Value *NewReduction = nullptr;
+ switch (Lowering->Kind) {
+ case ZeroTestLoweringKind::BoolReduction:
+ NewReduction =
+ createBoolReductionZeroTest(Builder, Vec, ZeroTest->Pred);
+ break;
+ case ZeroTestLoweringKind::Ctpop:
+ NewReduction = createCtpopZeroTest(Builder, Vec, ZeroTest->Pred);
+ break;
+ }
+ assert(NewReduction && "Expected valid zero-test lowering.");
+ NewReduction->takeName(ZeroTest->Cmp);
+ ZeroTest->Cmp->replaceAllUsesWith(NewReduction);
+ VectorizedTree = NewReduction;
+ salvageDebugInfo(*ZeroTest->Cmp);
+ ZeroTest->Cmp->dropAllReferences();
+ ZeroTest->Cmp->removeFromParent();
+ V.eraseInstruction(ZeroTest->Cmp);
+ V.eraseInstruction(Reduction);
+ // Reuse the count for the replaced reduction and account for the
+ // new vector compare.
+ ++NumVectorInstructions;
+ if (Lowering->Kind == ZeroTestLoweringKind::Ctpop)
+ NumVectorInstructions += 2;
+ ReplacedZeroTest = true;
+ }
+ }
+ }
+ }
+
+ if (!ReplacedZeroTest)
+ ReductionRoot->replaceAllUsesWith(VectorizedTree);
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
@@ -30648,6 +30814,23 @@ class HorizontalReduction {
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
+ auto getReductionZeroTestCost = [&]() -> std::optional<InstructionCost> {
+ auto ZeroTest = matchReductionZeroTestUse();
+ // Direct generation below is limited to one vectorized reduction.
+ if (!ZeroTest || !DoesRequireReductionOp ||
+ this->ReducedVals.size() != 1 || isa<VectorType>(ScalarTy))
+ return std::nullopt;
+
+ auto *FixedVecTy = dyn_cast<FixedVectorType>(VectorTy);
+ if (!FixedVecTy || !FixedVecTy->getElementType()->isIntegerTy())
+ return std::nullopt;
+
+ auto Lowering = getBestReductionZeroTestLowering(
+ *TTI, FixedVecTy, ZeroTest->Pred, CostKind);
+ if (!Lowering)
+ return std::nullopt;
+ return Lowering->Cost;
+ };
switch (RdxKind) {
case RecurKind::Add:
case RecurKind::Mul:
@@ -30659,7 +30842,9 @@ class HorizontalReduction {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ if (auto ZeroTestCost = getReductionZeroTestCost()) {
+ VectorCost = *ZeroTestCost;
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
@@ -30764,7 +30949,11 @@ class HorizontalReduction {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
+ if (auto ZeroTestCost = getReductionZeroTestCost())
+ VectorCost = *ZeroTestCost;
+ else
+ VectorCost =
+ TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
} else {
// Check if the previous reduction already exists and account it as
// series of operations + single reduction.
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
index b7d0c2694af7f..1ca4943ace277 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
@@ -6,25 +6,8 @@ define i1 @or_reduction_nonzero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
@@ -83,25 +66,8 @@ define i1 @or_reduction_zero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i32 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i32 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
index 67fd3f4acacaa..e53d48d31fb5e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
@@ -5,7 +5,9 @@ define i32 @or_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP2]])
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
@@ -35,8 +37,10 @@ define i32 @or_nonzero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP2]])
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -65,7 +69,9 @@ define i32 @or_zero(ptr %p) {
; CHECK-LABEL: define i32 @or_zero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP2]])
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
@@ -95,7 +101,9 @@ define i32 @umax_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @umax_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP2]])
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
@@ -132,8 +140,10 @@ define i32 @umax_zero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @umax_zero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x i1> [[TMP1]] to i8
+; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.ctpop.i8(i8 [[TMP2]])
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
>From c829c462be9a845f078f4fcb2da325f1cda9ea6b Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 27 Jun 2026 18:06:33 +0900
Subject: [PATCH 2/6] formmating
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b4f2189983043..2b40b189a05ff 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28781,13 +28781,13 @@ class HorizontalReduction {
Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
return TTI.getCmpSelInstrCost(Instruction::ICmp, VecTy, CmpTy, Pred,
CostKind) +
- TTI.getArithmeticReductionCost(ReductionOpcode, CmpTy, {},
- CostKind);
+ TTI.getArithmeticReductionCost(ReductionOpcode, CmpTy, {}, CostKind);
}
- static InstructionCost
- getCtpopZeroTestCost(const TargetTransformInfo &TTI, FixedVectorType *VecTy,
- CmpPredicate Pred, TTI::TargetCostKind CostKind) {
+ static InstructionCost getCtpopZeroTestCost(const TargetTransformInfo &TTI,
+ FixedVectorType *VecTy,
+ CmpPredicate Pred,
+ TTI::TargetCostKind CostKind) {
LLVMContext &Ctx = VecTy->getContext();
ElementCount EC = VecTy->getElementCount();
if (EC.isScalable())
@@ -28797,9 +28797,9 @@ class HorizontalReduction {
auto *MaskIntTy = IntegerType::get(Ctx, EC.getFixedValue());
InstructionCost CmpCost = TTI.getCmpSelInstrCost(
Instruction::ICmp, VecTy, CmpTy, ICmpInst::ICMP_NE, CostKind);
- InstructionCost BitcastCost = TTI.getCastInstrCost(
- Instruction::BitCast, MaskIntTy, CmpTy, TTI::CastContextHint::None,
- CostKind);
+ InstructionCost BitcastCost =
+ TTI.getCastInstrCost(Instruction::BitCast, MaskIntTy, CmpTy,
+ TTI::CastContextHint::None, CostKind);
IntrinsicCostAttributes ICA(Intrinsic::ctpop, MaskIntTy, {MaskIntTy});
InstructionCost CtpopCost = TTI.getIntrinsicInstrCost(ICA, CostKind);
InstructionCost ScalarCmpCost = TTI.getCmpSelInstrCost(
@@ -28836,8 +28836,7 @@ class HorizontalReduction {
if (!VecTy)
return nullptr;
- Value *Cmp =
- Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
+ Value *Cmp = Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
RecurKind ReductionKind =
Pred == ICmpInst::ICMP_EQ ? RecurKind::And : RecurKind::Or;
return createSimpleReduction(Builder, Cmp, ReductionKind);
>From 75281e2a3fa0811ef7f070e39215d21cb1b0e94a Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 27 Jun 2026 18:37:27 +0900
Subject: [PATCH 3/6] add comments for decribe each adding function
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 53 +++++++++++++++++++
1 file changed, 53 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 2b40b189a05ff..8637741cce99e 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28757,6 +28757,12 @@ class HorizontalReduction {
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
+ /// Match an OR/UMax reduction whose only use is an equality compare against
+ /// zero, for example:
+ /// %r = call iN @llvm.vector.reduce.or/umax(...)
+ /// %c = icmp eq/ne iN %r, 0
+ ///
+ /// this is an all-zero / any-nonzero test of the reduced input lanes.
std::optional<ReductionZeroTestInfo> matchReductionZeroTestUse() const {
auto *Root = dyn_cast<Instruction>(ReductionRoot);
if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
@@ -28772,6 +28778,12 @@ class HorizontalReduction {
return ReductionZeroTestInfo{Cmp, Pred};
}
+ /// Cost the boolean-vector form of a reduction zero-test:
+ /// %cmp = icmp eq/ne <N x Ty> %v, zeroinitializer
+ /// %r = call i1 @llvm.vector.reduce.and/or.vNi1(%cmp)
+ ///
+ /// `eq` uses reduce.and for an all-zero test, while `ne` uses reduce.or for
+ /// an any-nonzero test.
static InstructionCost
getBoolReductionZeroTestCost(const TargetTransformInfo &TTI,
FixedVectorType *VecTy, CmpPredicate Pred,
@@ -28784,6 +28796,13 @@ class HorizontalReduction {
TTI.getArithmeticReductionCost(ReductionOpcode, CmpTy, {}, CostKind);
}
+ /// Cost the fixed-width mask+ctpop form of a reduction zero-test:
+ /// %cmp = icmp ne <N x Ty> %v, zeroinitializer
+ /// %mask = bitcast <N x i1> %cmp to iN
+ /// %cnt = call iN @llvm.ctpop.iN(iN %mask)
+ /// %r = icmp eq/ne iN %cnt, 0
+ ///
+ /// `eq` tests all-zero lanes; `ne` tests any-nonzero lane.
static InstructionCost getCtpopZeroTestCost(const TargetTransformInfo &TTI,
FixedVectorType *VecTy,
CmpPredicate Pred,
@@ -28807,6 +28826,16 @@ class HorizontalReduction {
return CmpCost + BitcastCost + CtpopCost + ScalarCmpCost;
}
+ /// Choose the cheaper lowering for an OR/UMax reduction zero-test:
+ /// %r = call iN @llvm.vector.reduce.or/umax(...)
+ /// %c = icmp eq/ne iN %r, 0
+ ///
+ /// Return Ctpop when the fixed-width mask+ctpop form is valid and cheaper
+ /// than the boolean reduction form, or when boolean reduction is invalid:
+ /// %cmp = icmp ne <N x Ty> %v, zeroinitializer
+ /// %mask = bitcast <N x i1> %cmp to iN
+ /// %cnt = call iN @llvm.ctpop.iN(iN %mask)
+ /// %c = icmp eq/ne iN %cnt, 0
static std::optional<ZeroTestLoweringInfo>
getBestReductionZeroTestLowering(const TargetTransformInfo &TTI,
FixedVectorType *VecTy, CmpPredicate Pred,
@@ -28830,6 +28859,19 @@ class HorizontalReduction {
return ZeroTestLoweringInfo{ZeroTestLoweringKind::BoolReduction, BoolCost};
}
+ /// Create the boolean-reduction form of a vector zero-test.
+ /// For a vector input:
+ /// %vec = <N x Ty> ...
+ /// return:
+ /// %cmp = icmp eq/ne <N x Ty> %vec, zeroinitializer
+ /// %r = call i1 @llvm.vector.reduce.and/or.vNi1(%cmp)
+ ///
+ /// `eq` creates an all-zero test:
+ /// reduce.and(icmp eq %vec, 0)
+ /// `ne` creates an any-nonzero test:
+ /// reduce.or(icmp ne %vec, 0)
+ ///
+ /// Return nullptr if the input is not a vector.
static Value *createBoolReductionZeroTest(IRBuilderBase &Builder, Value *Vec,
CmpPredicate Pred) {
auto *VecTy = dyn_cast<VectorType>(Vec->getType());
@@ -28842,6 +28884,17 @@ class HorizontalReduction {
return createSimpleReduction(Builder, Cmp, ReductionKind);
}
+ /// Create the mask+ctpop form of a fixed-vector zero-test.
+ /// For a fixed vector input:
+ /// %vec = <N x Ty> ...
+ /// return:
+ /// %nz = icmp ne <N x Ty> %vec, zeroinitializer
+ /// %mask = bitcast <N x i1> %nz to iN
+ /// %pop = call iN @llvm.ctpop.iN(iN %mask)
+ /// %r = icmp eq/ne iN %pop, 0
+ ///
+ /// `eq` tests whether all lanes are zero; `ne` tests whether any lane is
+ /// non-zero. Return nullptr if the input is not a fixed vector.
static Value *createCtpopZeroTest(IRBuilderBase &Builder, Value *Vec,
CmpPredicate Pred) {
auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
>From e295e4ec45872c7188a9458141418c6f92bd248e Mon Sep 17 00:00:00 2001
From: Hanbum Park <kese111 at gmail.com>
Date: Mon, 29 Jun 2026 02:11:52 +0900
Subject: [PATCH 4/6] Extract some logic into helper functions
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 117 ++++++++++--------
1 file changed, 65 insertions(+), 52 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 8637741cce99e..b564c49e9537d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28910,6 +28910,65 @@ class HorizontalReduction {
return Builder.CreateICmp(Pred, Pop, ConstantInt::get(MaskIntTy, 0));
}
+ /// Fold an eq/ne-zero test of a single integer reduction into a lane-wise
+ /// comparison and an i1 reduction. For example, transform:
+ /// %r = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
+ /// %cmp = icmp ne i8 %r, 0
+ /// into:
+ /// %lanes = icmp ne <16 x i8> %v, zeroinitializer
+ /// %cmp = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %lanes)
+ Value *tryReplaceReductionZeroTest(BoUpSLP &V, IRBuilderBase &Builder,
+ const TargetTransformInfo &TTI,
+ Value *VectorizedTree) {
+ auto ZeroTest = matchReductionZeroTestUse();
+ if (!ZeroTest)
+ return nullptr;
+
+ auto *Reduction = dyn_cast<IntrinsicInst>(VectorizedTree);
+ Intrinsic::ID ReductionID = RdxKind == RecurKind::Or
+ ? Intrinsic::vector_reduce_or
+ : Intrinsic::vector_reduce_umax;
+ if (!Reduction || Reduction->getIntrinsicID() != ReductionID)
+ return nullptr;
+
+ Value *Vec = Reduction->getArgOperand(0);
+ auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+ if (!VecTy || !VecTy->getElementType()->isIntegerTy())
+ return nullptr;
+
+ auto Lowering = getBestReductionZeroTestLowering(
+ TTI, VecTy, ZeroTest->Pred, TTI::TCK_RecipThroughput);
+ if (!Lowering)
+ return nullptr;
+
+ Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
+ Builder.SetCurrentDebugLocation(ZeroTest->Cmp->getDebugLoc());
+ Value *NewReduction = nullptr;
+ switch (Lowering->Kind) {
+ case ZeroTestLoweringKind::BoolReduction:
+ NewReduction = createBoolReductionZeroTest(Builder, Vec, ZeroTest->Pred);
+ break;
+ case ZeroTestLoweringKind::Ctpop:
+ NewReduction = createCtpopZeroTest(Builder, Vec, ZeroTest->Pred);
+ break;
+ }
+ assert(NewReduction && "Expected valid zero-test lowering.");
+
+ NewReduction->takeName(ZeroTest->Cmp);
+ ZeroTest->Cmp->replaceAllUsesWith(NewReduction);
+ salvageDebugInfo(*ZeroTest->Cmp);
+ ZeroTest->Cmp->dropAllReferences();
+ ZeroTest->Cmp->removeFromParent();
+ V.eraseInstruction(ZeroTest->Cmp);
+ V.eraseInstruction(Reduction);
+ // Reuse the count for the replaced reduction and account for the new
+ // vector compare.
+ ++NumVectorInstructions;
+ if (Lowering->Kind == ZeroTestLoweringKind::Ctpop)
+ NumVectorInstructions += 2;
+ return NewReduction;
+ }
+
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
@@ -30390,58 +30449,12 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- // Fold an eq/ne-zero test of a single integer reduction into a lane-wise
- // comparison and an i1 reduction. For example, transform:
- // %r = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
- // %cmp = icmp ne i8 %r, 0
- // into:
- // %lanes = icmp ne <16 x i8> %v, zeroinitializer
- // %cmp = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %lanes)
- bool ReplacedZeroTest = false;
- if (auto ZeroTest = matchReductionZeroTestUse()) {
- auto *Reduction = dyn_cast<IntrinsicInst>(VectorizedTree);
- Intrinsic::ID ReductionID = RdxKind == RecurKind::Or
- ? Intrinsic::vector_reduce_or
- : Intrinsic::vector_reduce_umax;
- if (Reduction && Reduction->getIntrinsicID() == ReductionID) {
- Value *Vec = Reduction->getArgOperand(0);
- auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
- if (VecTy && VecTy->getElementType()->isIntegerTy()) {
- if (auto Lowering = getBestReductionZeroTestLowering(
- *TTI, VecTy, ZeroTest->Pred, TTI::TCK_RecipThroughput)) {
- Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
- Builder.SetCurrentDebugLocation(ZeroTest->Cmp->getDebugLoc());
- Value *NewReduction = nullptr;
- switch (Lowering->Kind) {
- case ZeroTestLoweringKind::BoolReduction:
- NewReduction =
- createBoolReductionZeroTest(Builder, Vec, ZeroTest->Pred);
- break;
- case ZeroTestLoweringKind::Ctpop:
- NewReduction = createCtpopZeroTest(Builder, Vec, ZeroTest->Pred);
- break;
- }
- assert(NewReduction && "Expected valid zero-test lowering.");
- NewReduction->takeName(ZeroTest->Cmp);
- ZeroTest->Cmp->replaceAllUsesWith(NewReduction);
- VectorizedTree = NewReduction;
- salvageDebugInfo(*ZeroTest->Cmp);
- ZeroTest->Cmp->dropAllReferences();
- ZeroTest->Cmp->removeFromParent();
- V.eraseInstruction(ZeroTest->Cmp);
- V.eraseInstruction(Reduction);
- // Reuse the count for the replaced reduction and account for the
- // new vector compare.
- ++NumVectorInstructions;
- if (Lowering->Kind == ZeroTestLoweringKind::Ctpop)
- NumVectorInstructions += 2;
- ReplacedZeroTest = true;
- }
- }
- }
- }
-
- if (!ReplacedZeroTest)
+ // NewReduction is the profitable zero-test lowering, either a lane-wise
+ // compare followed by vector.reduce.{or,and}, or ctpop.
+ if (Value *NewReduction =
+ tryReplaceReductionZeroTest(V, Builder, *TTI, VectorizedTree))
+ VectorizedTree = NewReduction;
+ else
ReductionRoot->replaceAllUsesWith(VectorizedTree);
// The original scalar reduction is expected to have no remaining
>From 1ebb053c13c15d392ade11c3533760a2d19d9313 Mon Sep 17 00:00:00 2001
From: Hanbum Park <kese111 at gmail.com>
Date: Mon, 29 Jun 2026 02:17:57 +0900
Subject: [PATCH 5/6] Limit the new helper functions to ICMP_EQ and ICMP_NE
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b564c49e9537d..e1013ce677763 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28788,6 +28788,9 @@ class HorizontalReduction {
getBoolReductionZeroTestCost(const TargetTransformInfo &TTI,
FixedVectorType *VecTy, CmpPredicate Pred,
TTI::TargetCostKind CostKind) {
+ if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
+ return InstructionCost::getInvalid();
+
auto *CmpTy = cast<VectorType>(CmpInst::makeCmpResultType(VecTy));
unsigned ReductionOpcode =
Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
@@ -28807,6 +28810,9 @@ class HorizontalReduction {
FixedVectorType *VecTy,
CmpPredicate Pred,
TTI::TargetCostKind CostKind) {
+ if (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE)
+ return InstructionCost::getInvalid();
+
LLVMContext &Ctx = VecTy->getContext();
ElementCount EC = VecTy->getElementCount();
if (EC.isScalable())
@@ -28875,7 +28881,7 @@ class HorizontalReduction {
static Value *createBoolReductionZeroTest(IRBuilderBase &Builder, Value *Vec,
CmpPredicate Pred) {
auto *VecTy = dyn_cast<VectorType>(Vec->getType());
- if (!VecTy)
+ if (!VecTy || (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE))
return nullptr;
Value *Cmp = Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
@@ -28898,7 +28904,7 @@ class HorizontalReduction {
static Value *createCtpopZeroTest(IRBuilderBase &Builder, Value *Vec,
CmpPredicate Pred) {
auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
- if (!VecTy)
+ if (!VecTy || (Pred != ICmpInst::ICMP_EQ && Pred != ICmpInst::ICMP_NE))
return nullptr;
Type *MaskIntTy =
>From b992dc73a936684dee7fc1cf0d3e8b5baf694e07 Mon Sep 17 00:00:00 2001
From: Hanbum Park <kese111 at gmail.com>
Date: Mon, 29 Jun 2026 03:01:05 +0900
Subject: [PATCH 6/6] formatting
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index e1013ce677763..2348b66e0538a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28942,8 +28942,8 @@ class HorizontalReduction {
if (!VecTy || !VecTy->getElementType()->isIntegerTy())
return nullptr;
- auto Lowering = getBestReductionZeroTestLowering(
- TTI, VecTy, ZeroTest->Pred, TTI::TCK_RecipThroughput);
+ auto Lowering = getBestReductionZeroTestLowering(TTI, VecTy, ZeroTest->Pred,
+ TTI::TCK_RecipThroughput);
if (!Lowering)
return nullptr;
More information about the llvm-commits
mailing list