[llvm] [SLP] Vectorize zero-tested OR/UMax reductions (PR #205473)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 01:54:39 PDT 2026
https://github.com/ParkHanbum updated https://github.com/llvm/llvm-project/pull/205473
>From a9a41a155151b6d5b375cae3b428efe9d8c2e94c Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 25 Jul 2026 03:32:30 +0900
Subject: [PATCH 01/11] [SLP] Vectorize zero-tested OR/UMax reductions
When a scalar OR or UMax reduction has a single eq/ne-zero use, form
an equivalent lane-wise vector comparison followed by an i1 AND/OR
reduction.
Model the vectorized form as a vector comparison plus the corresponding
boolean reduction, avoiding an integer reduction whose scalar result is
used only by the zero test.
Fixes #195118
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 130 ++++++++++++++++--
.../WebAssembly/or-reduction-zero-test.ll | 42 +-----
.../X86/reduction-zero-test-ctpop.ll | 20 +--
3 files changed, 132 insertions(+), 60 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 9da7f2b089119..81c82b581ab55 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29114,6 +29114,11 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
+ enum class ReductionContext {
+ None,
+ CmpZero,
+ };
+
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
@@ -29145,6 +29150,26 @@ class HorizontalReduction {
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
+ /// Return CmpZero for a scalar OR/UMax reduction whose only use is an eq/ne
+ /// comparison against zero.
+ ReductionContext getReductionContext() const {
+ auto *Root = dyn_cast<Instruction>(ReductionRoot);
+ if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
+ (RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
+ return ReductionContext::None;
+
+ CmpPredicate Pred;
+ if (!match(*Root->user_begin(),
+ m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
+ !ICmpInst::isEquality(Pred))
+ return ReductionContext::None;
+ return ReductionContext::CmpZero;
+ }
+
+ static bool isZeroCmpContext(ReductionContext Context) {
+ return Context == ReductionContext::CmpZero;
+ }
+
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
@@ -29947,6 +29972,7 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
IgnoreList.clear();
bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
+ ReductionContext RdxContext = getReductionContext();
// Need to track reduced vals, they may be changed during vectorization of
// subvectors.
@@ -29966,6 +29992,7 @@ class HorizontalReduction {
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
+ ReductionContext VectorizedReductionContext = ReductionContext::None;
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
@@ -30344,6 +30371,17 @@ class HorizontalReduction {
LocalExternallyUsedValues.insert(RdxVal);
V.buildExternalUses(LocalExternallyUsedValues);
+ // Use the zero-test cost only for a complete, standalone scalar
+ // reduction that can become one vector comparison and i1 reduction.
+ ReductionContext CostContext = ReductionContext::None;
+ if (isZeroCmpContext(RdxContext) &&
+ this->ReducedVals.size() == 1 &&
+ VL.size() == this->ReducedVals.front().size() &&
+ VectorValuesAndScales.empty() && !VectorizedTree &&
+ !isa<VectorType>(VL.front()->getType()) && !allConstant(VL) &&
+ !V.isReducedBitcastRoot() && !V.isReducedCmpBitcastRoot())
+ CostContext = RdxContext;
+
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
@@ -30352,7 +30390,7 @@ class HorizontalReduction {
else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
- RdxFMF, V, DT, DL, TLI);
+ CostContext, RdxFMF, V, DT, DL, TLI);
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
@@ -30456,6 +30494,8 @@ class HorizontalReduction {
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
+ if (isZeroCmpContext(CostContext))
+ VectorizedReductionContext = CostContext;
VectorValuesAndScales.emplace_back(
VectorizedRoot,
OptReusedScalars && SameScaleFactor
@@ -30501,8 +30541,8 @@ class HorizontalReduction {
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
- VectorizedTree,
- emitReduction(Builder, *TTI, ReductionRoot->getType()));
+ VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(),
+ VectorizedReductionContext));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30620,7 +30660,18 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ if (isZeroCmpContext(VectorizedReductionContext)) {
+ auto *Cmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ VectorizedTree->takeName(Cmp);
+ Cmp->replaceAllUsesWith(VectorizedTree);
+ salvageDebugInfo(*Cmp);
+ Cmp->dropAllReferences();
+ Cmp->removeFromParent();
+ V.eraseInstruction(Cmp);
+ } else {
+ ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ }
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
@@ -30775,9 +30826,10 @@ class HorizontalReduction {
V.calculateTreeCostAndTrimNonProfitable(VL, RdxRootInst);
V.buildExternalUses(LocalExternallyUsedValues);
- InstructionCost ReductionCost =
- getReductionCost(TTI, VL, EmptySameValuesCounter,
- /*IsCmpSelMinMax=*/false, RdxFMF, V, DT, DL, TLI);
+ InstructionCost ReductionCost = getReductionCost(
+ TTI, VL, EmptySameValuesCounter,
+ /*IsCmpSelMinMax=*/false, ReductionContext::None, RdxFMF, V, DT, DL,
+ TLI);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, RdxRootInst);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
@@ -30960,8 +31012,9 @@ class HorizontalReduction {
InstructionCost getReductionCost(
TargetTransformInfo *TTI, ArrayRef<Value *> ReducedVals,
const SmallMapVector<Value *, unsigned, 16> SameValuesCounter,
- bool IsCmpSelMinMax, FastMathFlags FMF, const BoUpSLP &R,
- DominatorTree &DT, const DataLayout &DL, const TargetLibraryInfo &TLI) {
+ bool IsCmpSelMinMax, ReductionContext Context, FastMathFlags FMF,
+ const BoUpSLP &R, DominatorTree &DT, const DataLayout &DL,
+ const TargetLibraryInfo &TLI) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *ScalarTy = ReducedVals.front()->getType();
unsigned ReduxWidth = ReducedVals.size();
@@ -31044,6 +31097,36 @@ class HorizontalReduction {
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
+ InstructionCost CmpReductionCost = InstructionCost::getInvalid();
+ if (isZeroCmpContext(Context)) {
+ // For a complete OR/UMax reduction used only by an eq/ne zero test,
+ // account for moving the comparison before the reduction:
+ //
+ // %rdx = call iN @llvm.vector.reduce.or/umax(<VF x iN> %vec)
+ // %cmp = icmp eq/ne iN %rdx, 0
+ //
+ // becomes:
+ //
+ // %lane.cmp = icmp eq/ne <VF x iN> %vec, zeroinitializer
+ // %cmp = call i1 @llvm.vector.reduce.and/or(<VF x i1> %lane.cmp)
+ //
+ // Equality requires every lane to be zero, so it uses an AND reduction;
+ // inequality requires any lane to be nonzero, so it uses OR. Add the
+ // vector comparison and boolean reduction costs.
+ assert(DoesRequireReductionOp && !isa<VectorType>(ScalarTy) &&
+ (RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
+ "Unexpected zero comparison reduction");
+ auto *ScalarCmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ CmpPredicate Pred = ScalarCmp->getPredicate();
+ auto *CmpTy = cast<VectorType>(CmpInst::makeCmpResultType(VectorTy));
+ unsigned ReductionOpcode =
+ Pred == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
+ CmpReductionCost = TTI->getCmpSelInstrCost(Instruction::ICmp, VectorTy,
+ CmpTy, Pred, CostKind) +
+ TTI->getArithmeticReductionCost(ReductionOpcode, CmpTy,
+ {}, CostKind);
+ }
switch (RdxKind) {
case RecurKind::Add:
case RecurKind::Mul:
@@ -31055,7 +31138,9 @@ class HorizontalReduction {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ if (isZeroCmpContext(Context)) {
+ VectorCost = CmpReductionCost;
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
@@ -31160,7 +31245,11 @@ class HorizontalReduction {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
+ if (isZeroCmpContext(Context))
+ VectorCost = CmpReductionCost;
+ else
+ VectorCost =
+ TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
} else {
// Check if the previous reduction already exists and account it as
// series of operations + single reduction.
@@ -31200,7 +31289,24 @@ class HorizontalReduction {
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy) {
+ Type *DestTy, ReductionContext Context) {
+ if (isZeroCmpContext(Context)) {
+ assert(VectorValuesAndScales.size() == 1 &&
+ !std::get<3>(VectorValuesAndScales.front()) &&
+ "Expected one complete vector reduction");
+ Value *Vec = std::get<0>(VectorValuesAndScales.front());
+ auto *VecTy = cast<VectorType>(Vec->getType());
+ auto *ScalarCmp =
+ cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
+ CmpPredicate Pred = ScalarCmp->getPredicate();
+ Builder.SetCurrentDebugLocation(ScalarCmp->getDebugLoc());
+ Value *Cmp = Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
+ RecurKind BoolRdxKind =
+ Pred == ICmpInst::ICMP_EQ ? RecurKind::And : RecurKind::Or;
+ NumVectorInstructions += 2;
+ return createSimpleReduction(Builder, Cmp, BoolRdxKind);
+ }
+
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
index 34ab34d8a2538..94d4e090f2b8a 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
@@ -6,25 +6,8 @@ define i1 @or_reduction_nonzero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
@@ -83,25 +66,8 @@ define i1 @or_reduction_zero(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP18]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[TMP0]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
; CHECK-NEXT: ret i1 [[CMP]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
index 67fd3f4acacaa..4f74d64460e90 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
@@ -5,8 +5,8 @@ define i32 @or_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -35,8 +35,8 @@ define i32 @or_nonzero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @or_nonzero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -65,8 +65,8 @@ define i32 @or_zero(ptr %p) {
; CHECK-LABEL: define i32 @or_zero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -95,8 +95,8 @@ define i32 @umax_nonzero(ptr %p) {
; CHECK-LABEL: define i32 @umax_nonzero(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP3:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP3]], 0
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
@@ -132,8 +132,8 @@ define i32 @umax_zero_commuted(ptr %p) {
; CHECK-LABEL: define i32 @umax_zero_commuted(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
-; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
; CHECK-NEXT: ret i32 [[RESULT]]
;
>From 105b7ca255f7295690a1624b9f1e639687769bb2 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sun, 26 Jul 2026 00:45:56 +0900
Subject: [PATCH 02/11] Add cost coverage for complete and partial zero-tested
reductions
---
.../X86/reduction-zero-test-partial-cost.ll | 143 ++++++++++++++++++
1 file changed, 143 insertions(+)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
new file mode 100644
index 0000000000000..73fe770795444
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4 -passes=slp-vectorizer -slp-threshold=19 -pass-remarks-output=%t -S < %s | FileCheck %s
+; RUN: cat %t | FileCheck -check-prefix=COST %s
+
+; COST-LABEL: Function: or_reduction_nonzero_scalar
+; COST: Cost: '-20'
+define i1 @or_reduction_nonzero_scalar(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP2]])
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v0 = load i8, ptr %p, align 1
+ %p1 = getelementptr inbounds i8, ptr %p, i32 1
+ %v1 = load i8, ptr %p1, align 1
+ %p2 = getelementptr inbounds i8, ptr %p, i32 2
+ %v2 = load i8, ptr %p2, align 1
+ %p3 = getelementptr inbounds i8, ptr %p, i32 3
+ %v3 = load i8, ptr %p3, align 1
+ %p4 = getelementptr inbounds i8, ptr %p, i32 4
+ %v4 = load i8, ptr %p4, align 1
+ %p5 = getelementptr inbounds i8, ptr %p, i32 5
+ %v5 = load i8, ptr %p5, align 1
+ %p6 = getelementptr inbounds i8, ptr %p, i32 6
+ %v6 = load i8, ptr %p6, align 1
+ %p7 = getelementptr inbounds i8, ptr %p, i32 7
+ %v7 = load i8, ptr %p7, align 1
+ %p8 = getelementptr inbounds i8, ptr %p, i32 8
+ %v8 = load i8, ptr %p8, align 1
+ %p9 = getelementptr inbounds i8, ptr %p, i32 9
+ %v9 = load i8, ptr %p9, align 1
+ %p10 = getelementptr inbounds i8, ptr %p, i32 10
+ %v10 = load i8, ptr %p10, align 1
+ %p11 = getelementptr inbounds i8, ptr %p, i32 11
+ %v11 = load i8, ptr %p11, align 1
+ %p12 = getelementptr inbounds i8, ptr %p, i32 12
+ %v12 = load i8, ptr %p12, align 1
+ %p13 = getelementptr inbounds i8, ptr %p, i32 13
+ %v13 = load i8, ptr %p13, align 1
+ %p14 = getelementptr inbounds i8, ptr %p, i32 14
+ %v14 = load i8, ptr %p14, align 1
+ %p15 = getelementptr inbounds i8, ptr %p, i32 15
+ %v15 = load i8, ptr %p15, align 1
+
+ %o01 = or i8 %v0, %v1
+ %o23 = or i8 %v2, %v3
+ %o45 = or i8 %v4, %v5
+ %o67 = or i8 %v6, %v7
+ %o89 = or i8 %v8, %v9
+ %o1011 = or i8 %v10, %v11
+ %o1213 = or i8 %v12, %v13
+ %o1415 = or i8 %v14, %v15
+
+ %o0123 = or i8 %o01, %o23
+ %o4567 = or i8 %o45, %o67
+ %o891011 = or i8 %o89, %o1011
+ %o12131415 = or i8 %o1213, %o1415
+
+ %o07 = or i8 %o0123, %o4567
+ %o815 = or i8 %o891011, %o12131415
+ %o015 = or i8 %o07, %o815
+
+ %cmp = icmp ne i8 %o015, 0
+ ret i1 %cmp
+}
+
+; COST-LABEL: Function: or_reduction_nonzero_scalar_remainder
+; COST: Cost: '-21'
+define i1 @or_reduction_nonzero_scalar_remainder(ptr %p) {
+; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar_remainder(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[P16:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 16
+; CHECK-NEXT: [[V16:%.*]] = load i8, ptr [[P16]], align 1
+; CHECK-NEXT: [[P17:%.*]] = getelementptr inbounds i8, ptr [[P]], i32 17
+; CHECK-NEXT: [[V17:%.*]] = load i8, ptr [[P17]], align 1
+; CHECK-NEXT: [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT: [[OP_RDX:%.*]] = or i8 [[TMP2]], [[V16]]
+; CHECK-NEXT: [[OP_RDX1:%.*]] = or i8 [[OP_RDX]], [[V17]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[OP_RDX1]], 0
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v0 = load i8, ptr %p, align 1
+ %p1 = getelementptr inbounds i8, ptr %p, i32 1
+ %v1 = load i8, ptr %p1, align 1
+ %p2 = getelementptr inbounds i8, ptr %p, i32 2
+ %v2 = load i8, ptr %p2, align 1
+ %p3 = getelementptr inbounds i8, ptr %p, i32 3
+ %v3 = load i8, ptr %p3, align 1
+ %p4 = getelementptr inbounds i8, ptr %p, i32 4
+ %v4 = load i8, ptr %p4, align 1
+ %p5 = getelementptr inbounds i8, ptr %p, i32 5
+ %v5 = load i8, ptr %p5, align 1
+ %p6 = getelementptr inbounds i8, ptr %p, i32 6
+ %v6 = load i8, ptr %p6, align 1
+ %p7 = getelementptr inbounds i8, ptr %p, i32 7
+ %v7 = load i8, ptr %p7, align 1
+ %p8 = getelementptr inbounds i8, ptr %p, i32 8
+ %v8 = load i8, ptr %p8, align 1
+ %p9 = getelementptr inbounds i8, ptr %p, i32 9
+ %v9 = load i8, ptr %p9, align 1
+ %p10 = getelementptr inbounds i8, ptr %p, i32 10
+ %v10 = load i8, ptr %p10, align 1
+ %p11 = getelementptr inbounds i8, ptr %p, i32 11
+ %v11 = load i8, ptr %p11, align 1
+ %p12 = getelementptr inbounds i8, ptr %p, i32 12
+ %v12 = load i8, ptr %p12, align 1
+ %p13 = getelementptr inbounds i8, ptr %p, i32 13
+ %v13 = load i8, ptr %p13, align 1
+ %p14 = getelementptr inbounds i8, ptr %p, i32 14
+ %v14 = load i8, ptr %p14, align 1
+ %p15 = getelementptr inbounds i8, ptr %p, i32 15
+ %v15 = load i8, ptr %p15, align 1
+ %p16 = getelementptr inbounds i8, ptr %p, i32 16
+ %v16 = load i8, ptr %p16, align 1
+ %p17 = getelementptr inbounds i8, ptr %p, i32 17
+ %v17 = load i8, ptr %p17, align 1
+
+ %o01 = or i8 %v0, %v1
+ %o23 = or i8 %v2, %v3
+ %o45 = or i8 %v4, %v5
+ %o67 = or i8 %v6, %v7
+ %o89 = or i8 %v8, %v9
+ %o1011 = or i8 %v10, %v11
+ %o1213 = or i8 %v12, %v13
+ %o1415 = or i8 %v14, %v15
+ %o1617 = or i8 %v16, %v17
+
+ %o0123 = or i8 %o01, %o23
+ %o4567 = or i8 %o45, %o67
+ %o891011 = or i8 %o89, %o1011
+ %o12131415 = or i8 %o1213, %o1415
+
+ %o07 = or i8 %o0123, %o4567
+ %o815 = or i8 %o891011, %o12131415
+ %o015 = or i8 %o07, %o815
+ %o017 = or i8 %o015, %o1617
+
+ %cmp = icmp ne i8 %o017, 0
+ ret i1 %cmp
+}
>From 3a4e7a3090ca67ec83763cbcd47a4f41a2bc53fa Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Fri, 31 Jul 2026 19:14:41 +0900
Subject: [PATCH 03/11] Account for removed scalar compare in zero-test costs
Subtract the scalar comparison eliminated when a complete
zero-tested reduction is lowered to a vector comparison
and i1 reduction.
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++++--
.../SLPVectorizer/X86/reduction-zero-test-partial-cost.ll | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 81c82b581ab55..3f10440635e09 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -31112,7 +31112,8 @@ class HorizontalReduction {
//
// Equality requires every lane to be zero, so it uses an AND reduction;
// inequality requires any lane to be nonzero, so it uses OR. Add the
- // vector comparison and boolean reduction costs.
+ // vector comparison and boolean reduction costs, then remove the scalar
+ // comparison cost eliminated by this replacement.
assert(DoesRequireReductionOp && !isa<VectorType>(ScalarTy) &&
(RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
"Unexpected zero comparison reduction");
@@ -31125,7 +31126,8 @@ class HorizontalReduction {
CmpReductionCost = TTI->getCmpSelInstrCost(Instruction::ICmp, VectorTy,
CmpTy, Pred, CostKind) +
TTI->getArithmeticReductionCost(ReductionOpcode, CmpTy,
- {}, CostKind);
+ {}, CostKind) -
+ TTI->getInstructionCost(ScalarCmp, CostKind);
}
switch (RdxKind) {
case RecurKind::Add:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
index 73fe770795444..139cc786bde67 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
@@ -3,7 +3,7 @@
; RUN: cat %t | FileCheck -check-prefix=COST %s
; COST-LABEL: Function: or_reduction_nonzero_scalar
-; COST: Cost: '-20'
+; COST: Cost: '-21'
define i1 @or_reduction_nonzero_scalar(ptr %p) {
; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
>From 3cd9e6d7bff969a39e26a19d3c9abf42c8d7d8fd Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 1 Aug 2026 10:46:52 +0900
Subject: [PATCH 04/11] [TTI][SLP] Use VectorInstrContext for zero-tested
reductions
Encode the zero-comparison reduction context in TTI::VectorInstrContext and use it for SLP reduction costing and lowering instead of an SLP-local enum.
---
.../llvm/Analysis/TargetTransformInfo.h | 1 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 37 +++++++++----------
2 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 107ae4dba5075..2cf60d3bedb59 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,6 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
+ CmpZero, ///< The reduction result is compared against zero.
};
class IntrinsicCostAttributes {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 3f10440635e09..585ef5f63edc6 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29114,10 +29114,6 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
- enum class ReductionContext {
- None,
- CmpZero,
- };
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
@@ -29152,22 +29148,22 @@ class HorizontalReduction {
/// Return CmpZero for a scalar OR/UMax reduction whose only use is an eq/ne
/// comparison against zero.
- ReductionContext getReductionContext() const {
+ TTI::VectorInstrContext getReductionContext() const {
auto *Root = dyn_cast<Instruction>(ReductionRoot);
if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
(RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
- return ReductionContext::None;
+ return TTI::VectorInstrContext::None;
CmpPredicate Pred;
if (!match(*Root->user_begin(),
m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
!ICmpInst::isEquality(Pred))
- return ReductionContext::None;
- return ReductionContext::CmpZero;
+ return TTI::VectorInstrContext::None;
+ return TTI::VectorInstrContext::CmpZero;
}
- static bool isZeroCmpContext(ReductionContext Context) {
- return Context == ReductionContext::CmpZero;
+ static bool isZeroCmpContext(TTI::VectorInstrContext Context) {
+ return Context == TTI::VectorInstrContext::CmpZero;
}
/// Checks if instruction is associative and can be vectorized.
@@ -29972,7 +29968,7 @@ class HorizontalReduction {
if (RK == ReductionOrdering::Ordered)
IgnoreList.clear();
bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
- ReductionContext RdxContext = getReductionContext();
+ TTI::VectorInstrContext RdxContext = getReductionContext();
// Need to track reduced vals, they may be changed during vectorization of
// subvectors.
@@ -29992,7 +29988,8 @@ class HorizontalReduction {
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
- ReductionContext VectorizedReductionContext = ReductionContext::None;
+ TTI::VectorInstrContext VectorizedReductionContext =
+ TTI::VectorInstrContext::None;
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
@@ -30373,7 +30370,8 @@ class HorizontalReduction {
// Use the zero-test cost only for a complete, standalone scalar
// reduction that can become one vector comparison and i1 reduction.
- ReductionContext CostContext = ReductionContext::None;
+ TTI::VectorInstrContext CostContext =
+ TTI::VectorInstrContext::None;
if (isZeroCmpContext(RdxContext) &&
this->ReducedVals.size() == 1 &&
VL.size() == this->ReducedVals.front().size() &&
@@ -30828,8 +30826,8 @@ class HorizontalReduction {
InstructionCost ReductionCost = getReductionCost(
TTI, VL, EmptySameValuesCounter,
- /*IsCmpSelMinMax=*/false, ReductionContext::None, RdxFMF, V, DT, DL,
- TLI);
+ /*IsCmpSelMinMax=*/false, TTI::VectorInstrContext::None, RdxFMF, V,
+ DT, DL, TLI);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, RdxRootInst);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
@@ -31012,9 +31010,9 @@ class HorizontalReduction {
InstructionCost getReductionCost(
TargetTransformInfo *TTI, ArrayRef<Value *> ReducedVals,
const SmallMapVector<Value *, unsigned, 16> SameValuesCounter,
- bool IsCmpSelMinMax, ReductionContext Context, FastMathFlags FMF,
- const BoUpSLP &R, DominatorTree &DT, const DataLayout &DL,
- const TargetLibraryInfo &TLI) {
+ bool IsCmpSelMinMax, TargetTransformInfo::VectorInstrContext Context,
+ FastMathFlags FMF, const BoUpSLP &R, DominatorTree &DT,
+ const DataLayout &DL, const TargetLibraryInfo &TLI) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *ScalarTy = ReducedVals.front()->getType();
unsigned ReduxWidth = ReducedVals.size();
@@ -31291,7 +31289,8 @@ class HorizontalReduction {
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy, ReductionContext Context) {
+ Type *DestTy,
+ TargetTransformInfo::VectorInstrContext Context) {
if (isZeroCmpContext(Context)) {
assert(VectorValuesAndScales.size() == 1 &&
!std::get<3>(VectorValuesAndScales.front()) &&
>From 2739a84f2b6d12390cd76401a404a7b521b67281 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Sat, 1 Aug 2026 11:33:17 +0900
Subject: [PATCH 05/11] formatting
---
llvm/include/llvm/Analysis/TargetTransformInfo.h | 2 +-
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 6 ++----
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 2cf60d3bedb59..10cc047771967 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -191,7 +191,7 @@ enum class VectorInstrContext : uint8_t {
Load, ///< The value being inserted comes from a load (InsertElement only).
Store, ///< The extracted value is stored (ExtractElement only).
BinaryOp, ///< One of the operands is a binary op.
- CmpZero, ///< The reduction result is compared against zero.
+ CmpZero, ///< The reduction result is compared against zero.
};
class IntrinsicCostAttributes {
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 585ef5f63edc6..82664401ad4e4 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30370,10 +30370,8 @@ class HorizontalReduction {
// Use the zero-test cost only for a complete, standalone scalar
// reduction that can become one vector comparison and i1 reduction.
- TTI::VectorInstrContext CostContext =
- TTI::VectorInstrContext::None;
- if (isZeroCmpContext(RdxContext) &&
- this->ReducedVals.size() == 1 &&
+ TTI::VectorInstrContext CostContext = TTI::VectorInstrContext::None;
+ if (isZeroCmpContext(RdxContext) && this->ReducedVals.size() == 1 &&
VL.size() == this->ReducedVals.front().size() &&
VectorValuesAndScales.empty() && !VectorizedTree &&
!isa<VectorType>(VL.front()->getType()) && !allConstant(VL) &&
>From 4600cd597496fab98989eca72760a4cc8977abad Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 13:47:50 +0900
Subject: [PATCH 06/11] restore ReductionContext
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 82664401ad4e4..43ca18af9d8a9 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29114,6 +29114,10 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
+ enum class ReductionContext {
+ None,
+ CmpZero,
+ };
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
>From 86cfe940d68c1749be0bdabbc1fb86306917ce64 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 14:49:36 +0900
Subject: [PATCH 07/11] change order of CostContext for getReductionCost
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 43ca18af9d8a9..630e3e72851e1 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30390,7 +30390,7 @@ class HorizontalReduction {
else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
- CostContext, RdxFMF, V, DT, DL, TLI);
+ RdxFMF, V, DT, DL, TLI, CostContext);
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
@@ -30826,10 +30826,10 @@ class HorizontalReduction {
V.calculateTreeCostAndTrimNonProfitable(VL, RdxRootInst);
V.buildExternalUses(LocalExternallyUsedValues);
- InstructionCost ReductionCost = getReductionCost(
- TTI, VL, EmptySameValuesCounter,
- /*IsCmpSelMinMax=*/false, TTI::VectorInstrContext::None, RdxFMF, V,
- DT, DL, TLI);
+ InstructionCost ReductionCost =
+ getReductionCost(TTI, VL, EmptySameValuesCounter,
+ /*IsCmpSelMinMax=*/false, RdxFMF, V, DT, DL, TLI,
+ TTI::VectorInstrContext::None);
InstructionCost Cost =
V.getTreeCost(TreeCost, VL, ReductionCost, RdxRootInst);
LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost
@@ -31012,9 +31012,9 @@ class HorizontalReduction {
InstructionCost getReductionCost(
TargetTransformInfo *TTI, ArrayRef<Value *> ReducedVals,
const SmallMapVector<Value *, unsigned, 16> SameValuesCounter,
- bool IsCmpSelMinMax, TargetTransformInfo::VectorInstrContext Context,
- FastMathFlags FMF, const BoUpSLP &R, DominatorTree &DT,
- const DataLayout &DL, const TargetLibraryInfo &TLI) {
+ bool IsCmpSelMinMax, FastMathFlags FMF, const BoUpSLP &R,
+ DominatorTree &DT, const DataLayout &DL, const TargetLibraryInfo &TLI,
+ TargetTransformInfo::VectorInstrContext Context) {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
Type *ScalarTy = ReducedVals.front()->getType();
unsigned ReduxWidth = ReducedVals.size();
>From 5003777178b05bceab02d9a9be80b8bf0a8ad965 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 14:34:54 +0900
Subject: [PATCH 08/11] modify condition check for reduction context
---
.../lib/Transforms/Vectorize/SLPVectorizer.cpp | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 630e3e72851e1..f3dd1470e8d38 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29153,17 +29153,25 @@ class HorizontalReduction {
/// Return CmpZero for a scalar OR/UMax reduction whose only use is an eq/ne
/// comparison against zero.
TTI::VectorInstrContext getReductionContext() const {
+ TTI::VectorInstrContext Context = TTI::VectorInstrContext::None;
+
auto *Root = dyn_cast<Instruction>(ReductionRoot);
- if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
- (RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
- return TTI::VectorInstrContext::None;
+ if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse())
+ return Context;
CmpPredicate Pred;
if (!match(*Root->user_begin(),
m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
!ICmpInst::isEquality(Pred))
- return TTI::VectorInstrContext::None;
- return TTI::VectorInstrContext::CmpZero;
+ return Context;
+
+ switch (RdxKind) {
+ case RecurKind::Or:
+ case RecurKind::UMax:
+ return TTI::VectorInstrContext::CmpZero;
+ default:
+ return Context;
+ }
}
static bool isZeroCmpContext(TTI::VectorInstrContext Context) {
>From 4f91cf1c485c120d1cbd6f673fa7373c31a8835b Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 16:43:02 +0900
Subject: [PATCH 09/11] Keep zero-test context out of reduction emission
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 41 ++-----------------
1 file changed, 4 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index f3dd1470e8d38..384c96d6e4ae5 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30000,8 +30000,6 @@ class HorizontalReduction {
// nodes and thus requiring extract if fully vectorized in other trees.
SmallPtrSet<Value *, 4> RequiredExtract;
WeakTrackingVH VectorizedTree = nullptr;
- TTI::VectorInstrContext VectorizedReductionContext =
- TTI::VectorInstrContext::None;
bool CheckForReusedReductionOps = false;
// Try to vectorize elements based on their type.
SmallVector<InstructionsState> States;
@@ -30502,8 +30500,6 @@ class HorizontalReduction {
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
- if (isZeroCmpContext(CostContext))
- VectorizedReductionContext = CostContext;
VectorValuesAndScales.emplace_back(
VectorizedRoot,
OptReusedScalars && SameScaleFactor
@@ -30549,8 +30545,8 @@ class HorizontalReduction {
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
- VectorizedTree, emitReduction(Builder, *TTI, ReductionRoot->getType(),
- VectorizedReductionContext));
+ VectorizedTree,
+ emitReduction(Builder, *TTI, ReductionRoot->getType()));
if (!VectorizedTree) {
if (!CheckForReusedReductionOps) {
@@ -30668,18 +30664,7 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- if (isZeroCmpContext(VectorizedReductionContext)) {
- auto *Cmp =
- cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
- VectorizedTree->takeName(Cmp);
- Cmp->replaceAllUsesWith(VectorizedTree);
- salvageDebugInfo(*Cmp);
- Cmp->dropAllReferences();
- Cmp->removeFromParent();
- V.eraseInstruction(Cmp);
- } else {
- ReductionRoot->replaceAllUsesWith(VectorizedTree);
- }
+ ReductionRoot->replaceAllUsesWith(VectorizedTree);
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
@@ -31299,25 +31284,7 @@ class HorizontalReduction {
/// sub-registers, combines them with the given reduction operation as a
/// vector operation and then performs single (small enough) reduction.
Value *emitReduction(IRBuilderBase &Builder, const TargetTransformInfo &TTI,
- Type *DestTy,
- TargetTransformInfo::VectorInstrContext Context) {
- if (isZeroCmpContext(Context)) {
- assert(VectorValuesAndScales.size() == 1 &&
- !std::get<3>(VectorValuesAndScales.front()) &&
- "Expected one complete vector reduction");
- Value *Vec = std::get<0>(VectorValuesAndScales.front());
- auto *VecTy = cast<VectorType>(Vec->getType());
- auto *ScalarCmp =
- cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
- CmpPredicate Pred = ScalarCmp->getPredicate();
- Builder.SetCurrentDebugLocation(ScalarCmp->getDebugLoc());
- Value *Cmp = Builder.CreateICmp(Pred, Vec, Constant::getNullValue(VecTy));
- RecurKind BoolRdxKind =
- Pred == ICmpInst::ICMP_EQ ? RecurKind::And : RecurKind::Or;
- NumVectorInstructions += 2;
- return createSimpleReduction(Builder, Cmp, BoolRdxKind);
- }
-
+ Type *DestTy) {
Value *ReducedSubTree = nullptr;
// Creates reduction and combines with the previous reduction.
auto CreateSingleOp = [&](Value *Vec, unsigned Scale, bool IsSigned,
>From 4ab2272894c088986179be563a6d8a16798a29d4 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 16:44:59 +0900
Subject: [PATCH 10/11] Restrict zero-test cost to complete reductions
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 31 ++++++-------------
1 file changed, 10 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 384c96d6e4ae5..35c7154df3fda 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29114,11 +29114,6 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
- enum class ReductionContext {
- None,
- CmpZero,
- };
-
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
@@ -30378,16 +30373,6 @@ class HorizontalReduction {
LocalExternallyUsedValues.insert(RdxVal);
V.buildExternalUses(LocalExternallyUsedValues);
- // Use the zero-test cost only for a complete, standalone scalar
- // reduction that can become one vector comparison and i1 reduction.
- TTI::VectorInstrContext CostContext = TTI::VectorInstrContext::None;
- if (isZeroCmpContext(RdxContext) && this->ReducedVals.size() == 1 &&
- VL.size() == this->ReducedVals.front().size() &&
- VectorValuesAndScales.empty() && !VectorizedTree &&
- !isa<VectorType>(VL.front()->getType()) && !allConstant(VL) &&
- !V.isReducedBitcastRoot() && !V.isReducedCmpBitcastRoot())
- CostContext = RdxContext;
-
// Estimate cost.
InstructionCost ReductionCost;
if (RK == ReductionOrdering::Ordered || V.isReducedBitcastRoot() ||
@@ -30396,7 +30381,7 @@ class HorizontalReduction {
else
ReductionCost =
getReductionCost(TTI, VL, SameValuesCounter, IsCmpSelMinMax,
- RdxFMF, V, DT, DL, TLI, CostContext);
+ RdxFMF, V, DT, DL, TLI, RdxContext);
// If the root is a select (min/max idiom), the insert point is the
// compare condition of that select.
Instruction *RdxRootInst = cast<Instruction>(ReductionRoot);
@@ -31090,8 +31075,13 @@ class HorizontalReduction {
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
+ bool IsCompleteReduction =
+ this->ReducedVals.size() == 1 &&
+ ReducedVals.size() == this->ReducedVals.front().size();
+ bool UseCmpZeroCost = isZeroCmpContext(Context) && IsCompleteReduction &&
+ DoesRequireReductionOp && !isa<VectorType>(ScalarTy);
InstructionCost CmpReductionCost = InstructionCost::getInvalid();
- if (isZeroCmpContext(Context)) {
+ if (UseCmpZeroCost) {
// For a complete OR/UMax reduction used only by an eq/ne zero test,
// account for moving the comparison before the reduction:
//
@@ -31107,8 +31097,7 @@ class HorizontalReduction {
// inequality requires any lane to be nonzero, so it uses OR. Add the
// vector comparison and boolean reduction costs, then remove the scalar
// comparison cost eliminated by this replacement.
- assert(DoesRequireReductionOp && !isa<VectorType>(ScalarTy) &&
- (RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
+ assert((RdxKind == RecurKind::Or || RdxKind == RecurKind::UMax) &&
"Unexpected zero comparison reduction");
auto *ScalarCmp =
cast<ICmpInst>(*cast<Instruction>(ReductionRoot)->user_begin());
@@ -31133,7 +31122,7 @@ class HorizontalReduction {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (isZeroCmpContext(Context)) {
+ if (UseCmpZeroCost) {
VectorCost = CmpReductionCost;
} else if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
@@ -31240,7 +31229,7 @@ class HorizontalReduction {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (isZeroCmpContext(Context))
+ if (UseCmpZeroCost)
VectorCost = CmpReductionCost;
else
VectorCost =
>From 18d532d6067247d86073c084abdea723d0afb273 Mon Sep 17 00:00:00 2001
From: hanbeom <kese111 at gmail.com>
Date: Mon, 10 Aug 2026 17:01:06 +0900
Subject: [PATCH 11/11] updated tests
---
.../WebAssembly/or-reduction-zero-test.ll | 127 ++++++++++++------
.../X86/reduction-zero-test-ctpop.ll | 118 +++++++++++-----
.../X86/reduction-zero-test-partial-cost.ll | 6 +-
3 files changed, 173 insertions(+), 78 deletions(-)
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
index 94d4e090f2b8a..63019dae1d9b0 100644
--- a/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/or-reduction-zero-test.ll
@@ -1,14 +1,24 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=slp-vectorizer -S < %s | FileCheck %s
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes=slp-vectorizer -S < %s | FileCheck %s --check-prefix=SLP
+; RUN: opt -mtriple=wasm32 -mattr=+simd128 -passes='slp-vectorizer,vector-combine' -S < %s | FileCheck %s --check-prefix=VEC
define i1 @or_reduction_nonzero(ptr %p) {
-; CHECK-LABEL: define i1 @or_reduction_nonzero(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
-; CHECK-NEXT: ret i1 [[CMP]]
+;
+; SLP-LABEL: define i1 @or_reduction_nonzero(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; SLP-NEXT: [[ENTRY:.*:]]
+; SLP-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; SLP-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; SLP-NEXT: ret i1 [[CMP]]
+;
+; VEC-LABEL: define i1 @or_reduction_nonzero(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; VEC-NEXT: [[ENTRY:.*:]]
+; VEC-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[TMP0]], zeroinitializer
+; VEC-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; VEC-NEXT: ret i1 [[CMP]]
;
entry:
%v0 = load i8, ptr %p, align 1
@@ -62,13 +72,22 @@ entry:
}
define i1 @or_reduction_zero(ptr %p) {
-; CHECK-LABEL: define i1 @or_reduction_zero(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[TMP0]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
-; CHECK-NEXT: ret i1 [[CMP]]
+;
+; SLP-LABEL: define i1 @or_reduction_zero(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[ENTRY:.*:]]
+; SLP-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP0]])
+; SLP-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; SLP-NEXT: ret i1 [[CMP]]
+;
+; VEC-LABEL: define i1 @or_reduction_zero(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[ENTRY:.*:]]
+; VEC-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[TMP0]], zeroinitializer
+; VEC-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v16i1(<16 x i1> [[TMP1]])
+; VEC-NEXT: ret i1 [[CMP]]
;
entry:
%v0 = load i8, ptr %p, align 1
@@ -122,31 +141,59 @@ entry:
}
define i1 @or_reduction_nonzero_multiuse(ptr %p, ptr %out) {
-; CHECK-LABEL: define i1 @or_reduction_nonzero_multiuse(
-; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
-; CHECK-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
-; CHECK-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
-; CHECK-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
-; CHECK-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
-; CHECK-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
-; CHECK-NEXT: [[O:%.*]] = or i8 [[TMP16]], [[TMP17]]
-; CHECK-NEXT: store i8 [[O]], ptr [[OUT]], align 1
-; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[O]], 0
-; CHECK-NEXT: ret i1 [[CMP]]
+;
+; SLP-LABEL: define i1 @or_reduction_nonzero_multiuse(
+; SLP-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[ENTRY:.*:]]
+; SLP-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+; SLP-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
+; SLP-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP1]], [[TMP2]]
+; SLP-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
+; SLP-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
+; SLP-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
+; SLP-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
+; SLP-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
+; SLP-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
+; SLP-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
+; SLP-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
+; SLP-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
+; SLP-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
+; SLP-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
+; SLP-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
+; SLP-NEXT: [[TMP16:%.*]] = extractelement <2 x i8> [[TMP15]], i64 0
+; SLP-NEXT: [[TMP17:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
+; SLP-NEXT: [[O:%.*]] = or i8 [[TMP16]], [[TMP17]]
+; SLP-NEXT: store i8 [[O]], ptr [[OUT]], align 1
+; SLP-NEXT: [[CMP:%.*]] = icmp ne i8 [[O]], 0
+; SLP-NEXT: ret i1 [[CMP]]
+;
+; VEC-LABEL: define i1 @or_reduction_nonzero_multiuse(
+; VEC-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[ENTRY:.*:]]
+; VEC-NEXT: [[TMP0:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 0, i32 8>
+; VEC-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 1, i32 9>
+; VEC-NEXT: [[TMP3:%.*]] = or <2 x i8> [[TMP1]], [[TMP2]]
+; VEC-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 2, i32 10>
+; VEC-NEXT: [[TMP5:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 3, i32 11>
+; VEC-NEXT: [[TMP6:%.*]] = or <2 x i8> [[TMP4]], [[TMP5]]
+; VEC-NEXT: [[TMP7:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 4, i32 12>
+; VEC-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 5, i32 13>
+; VEC-NEXT: [[TMP9:%.*]] = or <2 x i8> [[TMP7]], [[TMP8]]
+; VEC-NEXT: [[TMP10:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 6, i32 14>
+; VEC-NEXT: [[TMP11:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <2 x i32> <i32 7, i32 15>
+; VEC-NEXT: [[TMP12:%.*]] = or <2 x i8> [[TMP10]], [[TMP11]]
+; VEC-NEXT: [[TMP13:%.*]] = or <2 x i8> [[TMP3]], [[TMP6]]
+; VEC-NEXT: [[TMP14:%.*]] = or <2 x i8> [[TMP9]], [[TMP12]]
+; VEC-NEXT: [[TMP15:%.*]] = or <2 x i8> [[TMP13]], [[TMP14]]
+; VEC-NEXT: [[TMP16:%.*]] = shufflevector <16 x i8> [[TMP0]], <16 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; VEC-NEXT: [[TMP17:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[TMP16]])
+; VEC-NEXT: [[TMP18:%.*]] = extractelement <2 x i8> [[TMP15]], i64 1
+; VEC-NEXT: [[O:%.*]] = or i8 [[TMP17]], [[TMP18]]
+; VEC-NEXT: store i8 [[O]], ptr [[OUT]], align 1
+; VEC-NEXT: [[CMP:%.*]] = icmp ne i8 [[O]], 0
+; VEC-NEXT: ret i1 [[CMP]]
;
entry:
%v0 = load i8, ptr %p, align 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
index 4f74d64460e90..53717a1d90de0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-ctpop.ll
@@ -1,14 +1,24 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mcpu=skylake-avx512 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mcpu=skylake-avx512 < %s | FileCheck %s --check-prefix=SLP
+; RUN: opt -S --passes='slp-vectorizer,vector-combine' -mtriple=x86_64-unknown-linux -mcpu=skylake-avx512 < %s | FileCheck %s --check-prefix=VEC
define i32 @or_nonzero(ptr %p) {
-; CHECK-LABEL: define i32 @or_nonzero(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
-; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; SLP-LABEL: define i32 @or_nonzero(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; SLP-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; SLP-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; SLP-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; SLP-NEXT: ret i32 [[RESULT]]
+;
+; VEC-LABEL: define i32 @or_nonzero(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; VEC-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; VEC-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
+; VEC-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; VEC-NEXT: ret i32 [[RESULT]]
;
%input = load <8 x i8>, ptr %p, align 1
%a0 = extractelement <8 x i8> %input, i64 0
@@ -32,13 +42,22 @@ define i32 @or_nonzero(ptr %p) {
}
define i32 @or_nonzero_commuted(ptr %p) {
-; CHECK-LABEL: define i32 @or_nonzero_commuted(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
-; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; SLP-LABEL: define i32 @or_nonzero_commuted(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; SLP-NEXT: [[CMP:%.*]] = icmp ne i8 0, [[TMP1]]
+; SLP-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; SLP-NEXT: ret i32 [[RESULT]]
+;
+; VEC-LABEL: define i32 @or_nonzero_commuted(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; VEC-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
+; VEC-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; VEC-NEXT: ret i32 [[RESULT]]
;
%input = load <8 x i8>, ptr %p, align 1
%a0 = extractelement <8 x i8> %input, i64 0
@@ -62,13 +81,22 @@ define i32 @or_nonzero_commuted(ptr %p) {
}
define i32 @or_zero(ptr %p) {
-; CHECK-LABEL: define i32 @or_zero(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
-; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; SLP-LABEL: define i32 @or_zero(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; SLP-NEXT: [[CMP:%.*]] = icmp eq i8 [[TMP1]], 0
+; SLP-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; SLP-NEXT: ret i32 [[RESULT]]
+;
+; VEC-LABEL: define i32 @or_zero(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; VEC-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
+; VEC-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; VEC-NEXT: ret i32 [[RESULT]]
;
%input = load <8 x i8>, ptr %p, align 1
%a0 = extractelement <8 x i8> %input, i64 0
@@ -92,13 +120,22 @@ define i32 @or_zero(ptr %p) {
}
define i32 @umax_nonzero(ptr %p) {
-; CHECK-LABEL: define i32 @umax_nonzero(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
-; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; SLP-LABEL: define i32 @umax_nonzero(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; SLP-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; SLP-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; SLP-NEXT: ret i32 [[RESULT]]
+;
+; VEC-LABEL: define i32 @umax_nonzero(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; VEC-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; VEC-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; VEC-NEXT: ret i32 [[RESULT]]
;
%input = load <8 x i8>, ptr %p, align 1
%a0 = extractelement <8 x i8> %input, i64 0
@@ -129,13 +166,22 @@ define i32 @umax_nonzero(ptr %p) {
}
define i32 @umax_zero_commuted(ptr %p) {
-; CHECK-LABEL: define i32 @umax_zero_commuted(
-; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
-; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
-; CHECK-NEXT: ret i32 [[RESULT]]
+;
+; SLP-LABEL: define i32 @umax_zero_commuted(
+; SLP-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; SLP-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; SLP-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; SLP-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; SLP-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; SLP-NEXT: ret i32 [[RESULT]]
+;
+; VEC-LABEL: define i32 @umax_zero_commuted(
+; VEC-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; VEC-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; VEC-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> [[INPUT]])
+; VEC-NEXT: [[CMP:%.*]] = icmp eq i8 0, [[TMP1]]
+; VEC-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; VEC-NEXT: ret i32 [[RESULT]]
;
%input = load <8 x i8>, ptr %p, align 1
%a0 = extractelement <8 x i8> %input, i64 0
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
index 139cc786bde67..746c176f4c594 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-zero-test-partial-cost.ll
@@ -5,11 +5,12 @@
; COST-LABEL: Function: or_reduction_nonzero_scalar
; COST: Cost: '-21'
define i1 @or_reduction_nonzero_scalar(ptr %p) {
+;
; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <16 x i8> [[TMP1]], zeroinitializer
-; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP2]])
+; CHECK-NEXT: [[TMP2:%.*]] = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> [[TMP1]])
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP2]], 0
; CHECK-NEXT: ret i1 [[CMP]]
;
%v0 = load i8, ptr %p, align 1
@@ -69,6 +70,7 @@ define i1 @or_reduction_nonzero_scalar(ptr %p) {
; COST-LABEL: Function: or_reduction_nonzero_scalar_remainder
; COST: Cost: '-21'
define i1 @or_reduction_nonzero_scalar_remainder(ptr %p) {
+;
; CHECK-LABEL: define i1 @or_reduction_nonzero_scalar_remainder(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[P]], align 1
More information about the llvm-commits
mailing list