[llvm] [SLP] Vectorize zero-tested OR/UMax reductions (PR #205473)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 09:12:40 PDT 2026
https://github.com/ParkHanbum updated https://github.com/llvm/llvm-project/pull/205473
>From a9a41a155151b6d5b375cae3b428efe9d8c2e94c Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 25 Jul 2026 03:32:30 +0900
Subject: [PATCH 1/3] [SLP] Vectorize zero-tested OR/UMax reductions
When a scalar OR or UMax reduction has a single eq/ne-zero use, form
an equivalent lane-wise vector comparison followed by an i1 AND/OR
reduction.
Model the vectorized form as a vector comparison plus the corresponding
boolean reduction, avoiding an integer reduction whose scalar result is
used only by the zero test.
Fixes #195118
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 130 ++++++++++++++++--
.../WebAssembly/or-reduction-zero-test.ll | 42 +-----
.../X86/reduction-zero-test-ctpop.ll | 20 +--
3 files changed, 132 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9da7f2b089119..81c82b581ab55 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29114,6 +29114,11 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
+ enum class ReductionContext {
+ None,
+ CmpZero,
+ };
+
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
@@ -29145,6 +29150,26 @@ class HorizontalReduction {
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
+ /// Return CmpZero for a scalar OR/UMax reduction whose only use is an eq/ne
+ /// comparison against zero.
+ ReductionContext getReductionContext() const {
+ auto *Root = dyn_cast<Instruction>(ReductionRoot);
+ if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
+ (RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
+ return ReductionContext::None;
+
+ CmpPredicate Pred;
+ if (!match(*Root->user_begin(),
+ m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
+ !ICmpInst::isEquality(Pred))
+ return ReductionContext::None;
+ return ReductionContext::CmpZero;
+ }
+
+ static bool isZeroCmpContext(ReductionContext Context) {
+ return Context == ReductionContext::CmpZero;
+ }
+
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
@@ -29947,6 +29972,7 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
IgnoreList.clear();
bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
+ ReductionContext RdxContext = getReductionContext();
// Need to track reduced vals, they may be changed during vectorization of
// subvectors.
@@ -29966,6 +29992,7 @@ class HorizontalReduction {
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
+ ReductionContext VectorizedReductionContext = ReductionContext::None;
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
@@ -30344,6 +30371,17 @@ class HorizontalReduction {
LocalExternallyUsedValues.insert(RdxVal);
V.buildExternalUses(LocalExternallyUsedValues);
+ // Use the zero-test cost only for a complete, standalone scalar
+ // reduction that can become one vector comparison and i1 reduction.
+ ReductionContext CostContext = ReductionContext::None;
+ if (isZeroCmpContext(RdxContext) &&
+ this->ReducedVals.size() == 1 &&
+ VL.size() == this->ReducedVals.front().size() &&
+ VectorValuesAndScales.empty() && !VectorizedTree &&
+ !isa<VectorType>(VL.front()->getType()) && !allConstant(VL) &&
+ !V.isReducedBitcastRoot() && !V.isReducedCmpBitcastRoot())
+ CostContext = RdxContext;
+
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
@@ -30352,7 +30390,7 @@ class HorizontalReduction {
else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
- RdxFMF, V, DT, DL, TLI);
+ CostContext, RdxFMF, V, DT, DL, TLI);
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
@@ -30456,6 +30494,8 @@ class HorizontalReduction {
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
+ if (isZeroCmpContext(CostContext))
+ VectorizedReductionContext = CostContext;
VectorValuesAndScales.emplace_back(
VectorizedRoot,
OptReusedScalars && SameScaleFactor
@@ -30501,8 +30541,8 @@ class HorizontalReduction {
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
- VectorizedTree,
- emitReduction(Builder, *TTI, ReductionRoot->getType()));
+ VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(),
+ VectorizedReductionContext));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30620,7 +30660,18 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ if (isZeroCmpContext(VectorizedReductionContext)) {
+ auto *Cmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ VectorizedTree->takeName(Cmp);
+ Cmp->replaceAllUsesWith(VectorizedTree);
+ salvageDebugInfo(*Cmp);
+ Cmp->dropAllReferences();
+ Cmp->removeFromParent();
+ V.eraseInstruction(Cmp);
+ } else {
+ ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ }
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
@@ -30775,9 +30826,10 @@ class HorizontalReduction {
V.calculateTreeCostAndTrimNonProfitable(VL, RdxRootInst);
V.buildExternalUses(LocalExternallyUsedValues);
- InstructionCost ReductionCost =
- getReductionCost(TTI, VL, EmptySameValuesCounter,
- /*IsCmpSelMinMax=*/false, RdxFMF, V, DT, DL, TLI);
+ InstructionCost ReductionCost = getReductionCost(
+ TTI, VL, EmptySameValuesCounter,
+ /*IsCmpSelMinMax=*/false, ReductionContext::None, RdxFMF, V, DT, DL,
+ TLI);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, RdxRootInst);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
@@ -30960,8 +31012,9 @@ class HorizontalReduction {
InstructionCost getReductionCost(
TargetTransformInfo *TTI, ArrayRef<Value *> ReducedVals,
const SmallMapVector<Value *, unsigned, 16> SameValuesCounter,
- bool IsCmpSelMinMax, FastMathFlags FMF, const BoUpSLP &R,
- DominatorTree &DT, const DataLayout &DL, const TargetLibraryInfo &TLI) {
+ bool IsCmpSelMinMax, ReductionContext Context, FastMathFlags FMF,
+ const BoUpSLP &R, DominatorTree &DT, const DataLayout &DL,
+ const TargetLibraryInfo &TLI) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *ScalarTy = ReducedVals.front()->getType();
unsigned ReduxWidth = ReducedVals.size();
@@ -31044,6 +31097,36 @@ class HorizontalReduction {
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
+ InstructionCost CmpReductionCost = InstructionCost::getInvalid();
+ if (isZeroCmpContext(Context)) {
+ // For a complete OR/UMax reduction used only by an eq/ne zero test,
+ // account for moving the comparison before the reduction:
+ //
+ // %rdx = call iN @llvm.vector.reduce.or/umax(<VF x iN> %vec)
+ // %cmp = icmp eq/ne iN %rdx, 0
+ //
+ // becomes:
+ //
+ // %lane.cmp = icmp eq/ne <VF x iN> %vec, zeroinitializer
+ // %cmp = call i1 @llvm.vector.reduce.and/or(<VF x i1> %lane.cmp)
+ //
+ // Equality requires every lane to be zero, so it uses an AND reduction;
+ // inequality requires any lane to be nonzero, so it uses OR. Add the
+ // vector comparison and boolean reduction costs.
+ assert(DoesRequireReductionOp && !isa<VectorType>(ScalarTy) &&
+ (RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
+ "Unexpected zero comparison reduction");
+ auto *ScalarCmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ CmpPredicate Pred = ScalarCmp->getPredicate();
+ auto *CmpTy = cast<VectorType>(CmpInst::makeCmpResultType(VectorTy));
+ unsigned ReductionOpcode =
+ Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
+ CmpReductionCost = TTI->getCmpSelInstrCost(Instruction::ICmp, VectorTy,
+ CmpTy, Pred, CostKind) +
+ TTI->getArithmeticReductionCost(ReductionOpcode, CmpTy,
+ {}, CostKind);
+ }
switch (RdxKind) {
case RecurKind::Add:
case RecurKind::Mul:
@@ -31055,7 +31138,9 @@ class HorizontalReduction {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ if (isZeroCmpContext(Context)) {
+ VectorCost = CmpReductionCost;
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
@@ -31160,7 +31245,11 @@ class HorizontalReduction {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
+ if (isZeroCmpContext(Context))
+ VectorCost = CmpReductionCost;
+ else
+ VectorCost =
+ TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
} else {
// Check if the previous reduction already exists and account it as
// series of operations + single reduction.
@@ -31200,7 +31289,24 @@ class HorizontalReduction {
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy) {
+ Type *DestTy, ReductionContext Context) {
+ if (isZeroCmpContext(Context)) {
+ assert(VectorValuesAndScales.size() == 1 &&
+ !std::get<3>(VectorValuesAndScales.front()) &&
+ "Expected one complete vector reduction");
+ Value *Vec = std::get<0>(VectorValuesAndScales.front());
+ auto *VecTy = cast<VectorType>(Vec->getType());
+ auto *ScalarCmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ CmpPredicate Pred = ScalarCmp->getPredicate();
+ Builder.SetCurrentDebugLocation(ScalarCmp->getDebugLoc());
+ Value *Cmp = Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
+ RecurKind BoolRdxKind =
+ Pred == ICmpInst::ICMP_EQ ? RecurKind::And : RecurKind::Or;
+ NumVectorInstructions += 2;
+ return createSimpleReduction(Builder, Cmp, BoolRdxKind);
+ }
+
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
index 34ab34d8a2538..94d4e090f2b8a 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
@@ -6,25 +6,8 @@ define i1 @or_reduction_nonzero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
@@ -83,25 +66,8 @@ define i1 @or_reduction_zero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
index 67fd3f4acacaa..4f74d64460e90 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
@@ -5,8 +5,8 @@ define i32 @or_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -35,8 +35,8 @@ define i32 @or_nonzero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -65,8 +65,8 @@ define i32 @or_zero(ptr %p) {
; CHECK-LABEL: define i32 @or_zero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -95,8 +95,8 @@ define i32 @umax_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @umax_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -132,8 +132,8 @@ define i32 @umax_zero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @umax_zero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
>From 105b7ca255f7295690a1624b9f1e639687769bb2 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sun, 26 Jul 2026 00:45:56 +0900
Subject: [PATCH 2/3] Add cost coverage for complete and partial zero-tested
reductions
---
.../X86/reduction-zero-test-partial-cost.ll | 143 ++++++++++++++++++
1 file changed, 143 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
new file mode 100644
index 0000000000000..73fe770795444
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4 -passes=slp-vectorizer -slp-threshold=19 -pass-remarks-output=%t -S < %s | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=COST %s
+
+; COST-LABEL: Function: or_reduction_nonzero_scalar
+; COST: Cost: '-20'
+define i1 @or_reduction_nonzero_scalar(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP2]])
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v0 = load i8, ptr %p, align 1
+ %p1 = getelementptr inbounds i8, ptr %p, i32 1
+ %v1 = load i8, ptr %p1, align 1
+ %p2 = getelementptr inbounds i8, ptr %p, i32 2
+ %v2 = load i8, ptr %p2, align 1
+ %p3 = getelementptr inbounds i8, ptr %p, i32 3
+ %v3 = load i8, ptr %p3, align 1
+ %p4 = getelementptr inbounds i8, ptr %p, i32 4
+ %v4 = load i8, ptr %p4, align 1
+ %p5 = getelementptr inbounds i8, ptr %p, i32 5
+ %v5 = load i8, ptr %p5, align 1
+ %p6 = getelementptr inbounds i8, ptr %p, i32 6
+ %v6 = load i8, ptr %p6, align 1
+ %p7 = getelementptr inbounds i8, ptr %p, i32 7
+ %v7 = load i8, ptr %p7, align 1
+ %p8 = getelementptr inbounds i8, ptr %p, i32 8
+ %v8 = load i8, ptr %p8, align 1
+ %p9 = getelementptr inbounds i8, ptr %p, i32 9
+ %v9 = load i8, ptr %p9, align 1
+ %p10 = getelementptr inbounds i8, ptr %p, i32 10
+ %v10 = load i8, ptr %p10, align 1
+ %p11 = getelementptr inbounds i8, ptr %p, i32 11
+ %v11 = load i8, ptr %p11, align 1
+ %p12 = getelementptr inbounds i8, ptr %p, i32 12
+ %v12 = load i8, ptr %p12, align 1
+ %p13 = getelementptr inbounds i8, ptr %p, i32 13
+ %v13 = load i8, ptr %p13, align 1
+ %p14 = getelementptr inbounds i8, ptr %p, i32 14
+ %v14 = load i8, ptr %p14, align 1
+ %p15 = getelementptr inbounds i8, ptr %p, i32 15
+ %v15 = load i8, ptr %p15, align 1
+
+ %o01 = or i8 %v0, %v1
+ %o23 = or i8 %v2, %v3
+ %o45 = or i8 %v4, %v5
+ %o67 = or i8 %v6, %v7
+ %o89 = or i8 %v8, %v9
+ %o1011 = or i8 %v10, %v11
+ %o1213 = or i8 %v12, %v13
+ %o1415 = or i8 %v14, %v15
+
+ %o0123 = or i8 %o01, %o23
+ %o4567 = or i8 %o45, %o67
+ %o891011 = or i8 %o89, %o1011
+ %o12131415 = or i8 %o1213, %o1415
+
+ %o07 = or i8 %o0123, %o4567
+ %o815 = or i8 %o891011, %o12131415
+ %o015 = or i8 %o07, %o815
+
+ %cmp = icmp ne i8 %o015, 0
+ ret i1 %cmp
+}
+
+; COST-LABEL: Function: or_reduction_nonzero_scalar_remainder
+; COST: Cost: '-21'
+define i1 @or_reduction_nonzero_scalar_remainder(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar_remainder(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[P16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 16
+; CHECK-NEXT: [[V16:%.*]] = load i8, ptr [[P16]], align 1
+; CHECK-NEXT: [[P17:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 17
+; CHECK-NEXT: [[V17:%.*]] = load i8, ptr [[P17]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = or i8 [[TMP2]], [[V16]]
+; CHECK-NEXT: [[OP_RDX1:%.*]] = or i8 [[OP_RDX]], [[V17]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[OP_RDX1]], 0
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v0 = load i8, ptr %p, align 1
+ %p1 = getelementptr inbounds i8, ptr %p, i32 1
+ %v1 = load i8, ptr %p1, align 1
+ %p2 = getelementptr inbounds i8, ptr %p, i32 2
+ %v2 = load i8, ptr %p2, align 1
+ %p3 = getelementptr inbounds i8, ptr %p, i32 3
+ %v3 = load i8, ptr %p3, align 1
+ %p4 = getelementptr inbounds i8, ptr %p, i32 4
+ %v4 = load i8, ptr %p4, align 1
+ %p5 = getelementptr inbounds i8, ptr %p, i32 5
+ %v5 = load i8, ptr %p5, align 1
+ %p6 = getelementptr inbounds i8, ptr %p, i32 6
+ %v6 = load i8, ptr %p6, align 1
+ %p7 = getelementptr inbounds i8, ptr %p, i32 7
+ %v7 = load i8, ptr %p7, align 1
+ %p8 = getelementptr inbounds i8, ptr %p, i32 8
+ %v8 = load i8, ptr %p8, align 1
+ %p9 = getelementptr inbounds i8, ptr %p, i32 9
+ %v9 = load i8, ptr %p9, align 1
+ %p10 = getelementptr inbounds i8, ptr %p, i32 10
+ %v10 = load i8, ptr %p10, align 1
+ %p11 = getelementptr inbounds i8, ptr %p, i32 11
+ %v11 = load i8, ptr %p11, align 1
+ %p12 = getelementptr inbounds i8, ptr %p, i32 12
+ %v12 = load i8, ptr %p12, align 1
+ %p13 = getelementptr inbounds i8, ptr %p, i32 13
+ %v13 = load i8, ptr %p13, align 1
+ %p14 = getelementptr inbounds i8, ptr %p, i32 14
+ %v14 = load i8, ptr %p14, align 1
+ %p15 = getelementptr inbounds i8, ptr %p, i32 15
+ %v15 = load i8, ptr %p15, align 1
+ %p16 = getelementptr inbounds i8, ptr %p, i32 16
+ %v16 = load i8, ptr %p16, align 1
+ %p17 = getelementptr inbounds i8, ptr %p, i32 17
+ %v17 = load i8, ptr %p17, align 1
+
+ %o01 = or i8 %v0, %v1
+ %o23 = or i8 %v2, %v3
+ %o45 = or i8 %v4, %v5
+ %o67 = or i8 %v6, %v7
+ %o89 = or i8 %v8, %v9
+ %o1011 = or i8 %v10, %v11
+ %o1213 = or i8 %v12, %v13
+ %o1415 = or i8 %v14, %v15
+ %o1617 = or i8 %v16, %v17
+
+ %o0123 = or i8 %o01, %o23
+ %o4567 = or i8 %o45, %o67
+ %o891011 = or i8 %o89, %o1011
+ %o12131415 = or i8 %o1213, %o1415
+
+ %o07 = or i8 %o0123, %o4567
+ %o815 = or i8 %o891011, %o12131415
+ %o015 = or i8 %o07, %o815
+ %o017 = or i8 %o015, %o1617
+
+ %cmp = icmp ne i8 %o017, 0
+ ret i1 %cmp
+}
>From 3a4e7a3090ca67ec83763cbcd47a4f41a2bc53fa Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Fri, 31 Jul 2026 19:14:41 +0900
Subject: [PATCH 3/3] Account for removed scalar compare in zero-test costs
Subtract the scalar comparison eliminated when a complete
zero-tested reduction is lowered to a vector comparison
and i1 reduction.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++++--
.../SLPVectorizer/X86/reduction-zero-test-partial-cost.ll | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81c82b581ab55..3f10440635e09 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -31112,7 +31112,8 @@ class HorizontalReduction {
//
// Equality requires every lane to be zero, so it uses an AND reduction;
// inequality requires any lane to be nonzero, so it uses OR. Add the
- // vector comparison and boolean reduction costs.
+ // vector comparison and boolean reduction costs, then remove the scalar
+ // comparison cost eliminated by this replacement.
assert(DoesRequireReductionOp && !isa<VectorType>(ScalarTy) &&
(RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
"Unexpected zero comparison reduction");
@@ -31125,7 +31126,8 @@ class HorizontalReduction {
CmpReductionCost = TTI->getCmpSelInstrCost(Instruction::ICmp, VectorTy,
CmpTy, Pred, CostKind) +
TTI->getArithmeticReductionCost(ReductionOpcode, CmpTy,
- {}, CostKind);
+ {}, CostKind) -
+ TTI->getInstructionCost(ScalarCmp, CostKind);
}
switch (RdxKind) {
case RecurKind::Add:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
index 73fe770795444..139cc786bde67 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
@@ -3,7 +3,7 @@
; RUN: cat %t | FileCheck -check-prefix=COST %s
; COST-LABEL: Function: or_reduction_nonzero_scalar
-; COST: Cost: '-20'
+; COST: Cost: '-21'
define i1 @or_reduction_nonzero_scalar(ptr %p) {
; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
More information about the llvm-commits
mailing list