[llvm] [SLP] Vectorize zero-tested OR/UMax reductions (PR #205473)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 23 20:00:04 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
@llvm/pr-subscribers-llvm-transforms
Author: hanbeom (ParkHanbum)
<details>
<summary>Changes</summary>
When a scalar OR or UMax reduction has a single eq/ne-zero use, form
an equivalent lane-wise comparison followed by an i1 OR/AND reduction.
This avoids generating an integer vector reduction only to test its
scalar result, and exposes a boolean reduction that targets can lower
directly. Model the vectorization cost as a vector compare plus the
corresponding boolean reduction.
Add regression tests for both predicates, operand orders, supported
vector widths, and non-transformable uses.
Fixes https://github.com/llvm/llvm-project/issues/195118
---
Full diff: https://github.com/llvm/llvm-project/pull/205473.diff
3 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+86-3)
- (added) llvm/test/Transforms/SLPVectorizer/WebAssembly/reduction-zero.ll (+169)
- (added) llvm/test/Transforms/SLPVectorizer/reduction-zero-test.ll (+169)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4dcf3243a20ad..666526dd580ce 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -28717,6 +28717,12 @@ namespace {
class HorizontalReduction {
using ReductionOpsType = SmallVector<Value *, 16>;
using ReductionOpsListType = SmallVector<ReductionOpsType, 2>;
+ struct ReductionZeroTestInfo {
+ ICmpInst *Cmp;
+ CmpPredicate LanePred;
+ RecurKind BoolReductionKind;
+ };
+
ReductionOpsListType ReductionOps;
/// List of possibly reduced values.
SmallVector<SmallVector<Value *>> ReducedVals;
@@ -28748,6 +28754,23 @@ class HorizontalReduction {
(match(I, m_LogicalAnd()) || match(I, m_LogicalOr()));
}
+ std::optional<ReductionZeroTestInfo> matchReductionZeroTestUse() const {
+ auto *Root = dyn_cast<Instruction>(ReductionRoot);
+ if (!Root || !Root->getType()->isIntegerTy() || !Root->hasOneUse() ||
+ (RdxKind != RecurKind::Or && RdxKind != RecurKind::UMax))
+ return std::nullopt;
+
+ auto *Cmp = dyn_cast<ICmpInst>(*Root->user_begin());
+ CmpPredicate Pred;
+ if (!Cmp || !match(Cmp, m_c_ICmp(Pred, m_Specific(Root), m_ZeroInt())) ||
+ !ICmpInst::isEquality(Pred))
+ return std::nullopt;
+
+ if (Pred == ICmpInst::ICMP_NE)
+ return ReductionZeroTestInfo{Cmp, ICmpInst::ICMP_NE, RecurKind::Or};
+ return ReductionZeroTestInfo{Cmp, ICmpInst::ICMP_EQ, RecurKind::And};
+ }
+
/// Checks if instruction is associative and can be vectorized.
enum class ReductionOrdering { Unordered, Ordered, None };
ReductionOrdering RK = ReductionOrdering::None;
@@ -30207,7 +30230,46 @@ class HorizontalReduction {
}
VectorizedTree = ExtraReductions.front().second;
- ReductionRoot->replaceAllUsesWith(VectorizedTree);
+ // Fold an eq/ne-zero test of a single integer reduction into a lane-wise
+ // comparison and an i1 reduction. For example, transform:
+ // %r = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
+ // %cmp = icmp ne i8 %r, 0
+ // into:
+ // %lanes = icmp ne <16 x i8> %v, zeroinitializer
+ // %cmp = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> %lanes)
+ bool ReplacedZeroTest = false;
+ if (auto ZeroTest = matchReductionZeroTestUse()) {
+ auto *Reduction = dyn_cast<IntrinsicInst>(VectorizedTree);
+ Intrinsic::ID ReductionID = RdxKind == RecurKind::Or
+ ? Intrinsic::vector_reduce_or
+ : Intrinsic::vector_reduce_umax;
+ if (Reduction && Reduction->getIntrinsicID() == ReductionID) {
+ Value *Vec = Reduction->getArgOperand(0);
+ auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
+ if (VecTy && VecTy->getElementType()->isIntegerTy()) {
+ Builder.SetInsertPoint(cast<Instruction>(ReductionRoot));
+ Builder.SetCurrentDebugLocation(ZeroTest->Cmp->getDebugLoc());
+ Value *Cmp = Builder.CreateICmp(ZeroTest->LanePred, Vec,
+ Constant::getNullValue(VecTy));
+ Value *NewReduction =
+ createSimpleReduction(Builder, Cmp, ZeroTest->BoolReductionKind);
+ NewReduction->takeName(ZeroTest->Cmp);
+ ZeroTest->Cmp->replaceAllUsesWith(NewReduction);
+ salvageDebugInfo(*ZeroTest->Cmp);
+ ZeroTest->Cmp->dropAllReferences();
+ ZeroTest->Cmp->removeFromParent();
+ V.eraseInstruction(ZeroTest->Cmp);
+ V.eraseInstruction(Reduction);
+ // Reuse the count for the replaced reduction and account for the
+ // new vector compare.
+ ++NumVectorInstructions;
+ ReplacedZeroTest = true;
+ }
+ }
+ }
+
+ if (!ReplacedZeroTest)
+ ReductionRoot->replaceAllUsesWith(VectorizedTree);
// The original scalar reduction is expected to have no remaining
// uses outside the reduction tree itself. Assert that we got this
@@ -30631,6 +30693,21 @@ class HorizontalReduction {
// 2. The storage does not have any vector with full vector use (first
// vector with full register use).
bool DoesRequireReductionOp = !AllConsts && VectorValuesAndScales.empty();
+ auto getReductionZeroTestCost = [&]() -> std::optional<InstructionCost> {
+ auto ZeroTest = matchReductionZeroTestUse();
+ // Direct generation below is limited to one vectorized reduction.
+ if (!ZeroTest || !DoesRequireReductionOp ||
+ this->ReducedVals.size() != 1 || isa<VectorType>(ScalarTy))
+ return std::nullopt;
+
+ auto *CmpTy =
+ cast<VectorType>(CmpInst::makeCmpResultType(VectorTy));
+ return TTI->getCmpSelInstrCost(Instruction::ICmp, VectorTy, CmpTy,
+ ZeroTest->LanePred, CostKind) +
+ TTI->getArithmeticReductionCost(
+ RecurrenceDescriptor::getOpcode(ZeroTest->BoolReductionKind),
+ CmpTy, {}, CostKind);
+ };
switch (RdxKind) {
case RecurKind::Add:
case RecurKind::Mul:
@@ -30642,7 +30719,9 @@ class HorizontalReduction {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
+ if (auto ZeroTestCost = getReductionZeroTestCost()) {
+ VectorCost = *ZeroTestCost;
+ } else if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
@@ -30747,7 +30826,11 @@ class HorizontalReduction {
Intrinsic::ID Id = getMinMaxReductionIntrinsicOp(RdxKind);
if (!AllConsts) {
if (DoesRequireReductionOp) {
- VectorCost = TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
+ if (auto ZeroTestCost = getReductionZeroTestCost())
+ VectorCost = *ZeroTestCost;
+ else
+ VectorCost =
+ TTI->getMinMaxReductionCost(Id, VectorTy, FMF, CostKind);
} else {
// Check if the previous reduction already exists and account it as
// series of operations + single reduction.
diff --git a/llvm/test/Transforms/SLPVectorizer/WebAssembly/reduction-zero.ll b/llvm/test/Transforms/SLPVectorizer/WebAssembly/reduction-zero.ll
new file mode 100644
index 0000000000000..391d43f8db2ce
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/WebAssembly/reduction-zero.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: %if wasm-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=wasm32-unknown-unknown -mattr=+simd128 < %s | FileCheck %s %}
+
+define i1 @or_i8x16_ne(ptr %p) {
+; CHECK-LABEL: define i1 @or_i8x16_ne(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[V:%.*]] = load <16 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <16 x i8> [[V]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]])
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <16 x i8>, ptr %p, align 1
+ %a0 = extractelement <16 x i8> %v, i64 0
+ %a1 = extractelement <16 x i8> %v, i64 1
+ %a2 = extractelement <16 x i8> %v, i64 2
+ %a3 = extractelement <16 x i8> %v, i64 3
+ %a4 = extractelement <16 x i8> %v, i64 4
+ %a5 = extractelement <16 x i8> %v, i64 5
+ %a6 = extractelement <16 x i8> %v, i64 6
+ %a7 = extractelement <16 x i8> %v, i64 7
+ %a8 = extractelement <16 x i8> %v, i64 8
+ %a9 = extractelement <16 x i8> %v, i64 9
+ %a10 = extractelement <16 x i8> %v, i64 10
+ %a11 = extractelement <16 x i8> %v, i64 11
+ %a12 = extractelement <16 x i8> %v, i64 12
+ %a13 = extractelement <16 x i8> %v, i64 13
+ %a14 = extractelement <16 x i8> %v, i64 14
+ %a15 = extractelement <16 x i8> %v, i64 15
+ %o1 = or i8 %a0, %a1
+ %o2 = or i8 %o1, %a2
+ %o3 = or i8 %o2, %a3
+ %o4 = or i8 %o3, %a4
+ %o5 = or i8 %o4, %a5
+ %o6 = or i8 %o5, %a6
+ %o7 = or i8 %o6, %a7
+ %o8 = or i8 %o7, %a8
+ %o9 = or i8 %o8, %a9
+ %o10 = or i8 %o9, %a10
+ %o11 = or i8 %o10, %a11
+ %o12 = or i8 %o11, %a12
+ %o13 = or i8 %o12, %a13
+ %o14 = or i8 %o13, %a14
+ %o15 = or i8 %o14, %a15
+ %cmp = icmp ne i8 %o15, 0
+ ret i1 %cmp
+}
+
+define i1 @or_i16x8_ne(ptr %p) {
+; CHECK-LABEL: define i1 @or_i16x8_ne(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <8 x i16>, ptr [[P]], align 2
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[V]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <8 x i16>, ptr %p, align 2
+ %a0 = extractelement <8 x i16> %v, i64 0
+ %a1 = extractelement <8 x i16> %v, i64 1
+ %a2 = extractelement <8 x i16> %v, i64 2
+ %a3 = extractelement <8 x i16> %v, i64 3
+ %a4 = extractelement <8 x i16> %v, i64 4
+ %a5 = extractelement <8 x i16> %v, i64 5
+ %a6 = extractelement <8 x i16> %v, i64 6
+ %a7 = extractelement <8 x i16> %v, i64 7
+ %o1 = or i16 %a0, %a1
+ %o2 = or i16 %o1, %a2
+ %o3 = or i16 %o2, %a3
+ %o4 = or i16 %o3, %a4
+ %o5 = or i16 %o4, %a5
+ %o6 = or i16 %o5, %a6
+ %o7 = or i16 %o6, %a7
+ %cmp = icmp ne i16 %o7, 0
+ ret i1 %cmp
+}
+
+define i1 @or_i32x4_ne(ptr %p) {
+; CHECK-LABEL: define i1 @or_i32x4_ne(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[V]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]])
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <4 x i32>, ptr %p, align 4
+ %a0 = extractelement <4 x i32> %v, i64 0
+ %a1 = extractelement <4 x i32> %v, i64 1
+ %a2 = extractelement <4 x i32> %v, i64 2
+ %a3 = extractelement <4 x i32> %v, i64 3
+ %o1 = or i32 %a0, %a1
+ %o2 = or i32 %o1, %a2
+ %o3 = or i32 %o2, %a3
+ %cmp = icmp ne i32 %o3, 0
+ ret i1 %cmp
+}
+
+; The two-lane i64 reduction is below SLP's horizontal reduction limit.
+define i1 @or_i64x2_ne(ptr %p) {
+; CHECK-LABEL: define i1 @or_i64x2_ne(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <2 x i64>, ptr [[P]], align 8
+; CHECK-NEXT: [[A0:%.*]] = extractelement <2 x i64> [[V]], i64 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <2 x i64> [[V]], i64 1
+; CHECK-NEXT: [[O:%.*]] = or i64 [[A0]], [[A1]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i64 [[O]], 0
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <2 x i64>, ptr %p, align 8
+ %a0 = extractelement <2 x i64> %v, i64 0
+ %a1 = extractelement <2 x i64> %v, i64 1
+ %o = or i64 %a0, %a1
+ %cmp = icmp ne i64 %o, 0
+ ret i1 %cmp
+}
+
+; Do not replace a zero-test when the scalar reduction has another use.
+define i1 @or_i32x4_multiuse(ptr %p, ptr %out) {
+; CHECK-LABEL: define i1 @or_i32x4_multiuse(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 4
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT: [[O1:%.*]] = or i32 [[A0]], [[A1]]
+; CHECK-NEXT: [[O2:%.*]] = or i32 [[O1]], [[A2]]
+; CHECK-NEXT: [[O3:%.*]] = or i32 [[O2]], [[A3]]
+; CHECK-NEXT: store i32 [[O3]], ptr [[OUT]], align 4
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[O3]], 0
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <4 x i32>, ptr %p, align 4
+ %a0 = extractelement <4 x i32> %v, i64 0
+ %a1 = extractelement <4 x i32> %v, i64 1
+ %a2 = extractelement <4 x i32> %v, i64 2
+ %a3 = extractelement <4 x i32> %v, i64 3
+ %o1 = or i32 %a0, %a1
+ %o2 = or i32 %o1, %a2
+ %o3 = or i32 %o2, %a3
+ store i32 %o3, ptr %out, align 4
+ %cmp = icmp ne i32 %o3, 0
+ ret i1 %cmp
+}
+
+; Do not replace comparisons against non-zero constants.
+define i1 @or_i32x4_nonzero_constant(ptr %p) {
+; CHECK-LABEL: define i1 @or_i32x4_nonzero_constant(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr [[P]], align 4
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x i32> [[V]], i64 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x i32> [[V]], i64 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x i32> [[V]], i64 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x i32> [[V]], i64 3
+; CHECK-NEXT: [[O1:%.*]] = or i32 [[A0]], [[A1]]
+; CHECK-NEXT: [[O2:%.*]] = or i32 [[O1]], [[A2]]
+; CHECK-NEXT: [[O3:%.*]] = or i32 [[O2]], [[A3]]
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[O3]], 1
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %v = load <4 x i32>, ptr %p, align 4
+ %a0 = extractelement <4 x i32> %v, i64 0
+ %a1 = extractelement <4 x i32> %v, i64 1
+ %a2 = extractelement <4 x i32> %v, i64 2
+ %a3 = extractelement <4 x i32> %v, i64 3
+ %o1 = or i32 %a0, %a1
+ %o2 = or i32 %o1, %a2
+ %o3 = or i32 %o2, %a3
+ %cmp = icmp ne i32 %o3, 1
+ ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/reduction-zero-test.ll b/llvm/test/Transforms/SLPVectorizer/reduction-zero-test.ll
new file mode 100644
index 0000000000000..ecc779b162c41
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/reduction-zero-test.ll
@@ -0,0 +1,169 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: %if x86-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux < %s | FileCheck %s %}
+; RUN: %if aarch64-registered-target %{ opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux < %s | FileCheck %s %}
+
+define i32 @or_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @or_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
+; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %input = load <8 x i8>, ptr %p, align 1
+ %a0 = extractelement <8 x i8> %input, i64 0
+ %a1 = extractelement <8 x i8> %input, i64 1
+ %a2 = extractelement <8 x i8> %input, i64 2
+ %a3 = extractelement <8 x i8> %input, i64 3
+ %a4 = extractelement <8 x i8> %input, i64 4
+ %a5 = extractelement <8 x i8> %input, i64 5
+ %a6 = extractelement <8 x i8> %input, i64 6
+ %a7 = extractelement <8 x i8> %input, i64 7
+ %or1 = or i8 %a0, %a1
+ %or2 = or i8 %or1, %a2
+ %or3 = or i8 %or2, %a3
+ %or4 = or i8 %or3, %a4
+ %or5 = or i8 %or4, %a5
+ %or6 = or i8 %or5, %a6
+ %or7 = or i8 %or6, %a7
+ %cmp = icmp ne i8 %or7, 0
+ %result = zext i1 %cmp to i32
+ ret i32 %result
+}
+
+define i32 @or_zero(ptr %p) {
+; CHECK-LABEL: define i32 @or_zero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
+; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %input = load <8 x i8>, ptr %p, align 1
+ %a0 = extractelement <8 x i8> %input, i64 0
+ %a1 = extractelement <8 x i8> %input, i64 1
+ %a2 = extractelement <8 x i8> %input, i64 2
+ %a3 = extractelement <8 x i8> %input, i64 3
+ %a4 = extractelement <8 x i8> %input, i64 4
+ %a5 = extractelement <8 x i8> %input, i64 5
+ %a6 = extractelement <8 x i8> %input, i64 6
+ %a7 = extractelement <8 x i8> %input, i64 7
+ %or1 = or i8 %a0, %a1
+ %or2 = or i8 %or1, %a2
+ %or3 = or i8 %or2, %a3
+ %or4 = or i8 %or3, %a4
+ %or5 = or i8 %or4, %a5
+ %or6 = or i8 %or5, %a6
+ %or7 = or i8 %or6, %a7
+ %cmp = icmp eq i8 %or7, 0
+ %result = zext i1 %cmp to i32
+ ret i32 %result
+}
+
+define i32 @umax_nonzero(ptr %p) {
+; CHECK-LABEL: define i32 @umax_nonzero(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP1]])
+; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %input = load <8 x i8>, ptr %p, align 1
+ %a0 = extractelement <8 x i8> %input, i64 0
+ %a1 = extractelement <8 x i8> %input, i64 1
+ %a2 = extractelement <8 x i8> %input, i64 2
+ %a3 = extractelement <8 x i8> %input, i64 3
+ %a4 = extractelement <8 x i8> %input, i64 4
+ %a5 = extractelement <8 x i8> %input, i64 5
+ %a6 = extractelement <8 x i8> %input, i64 6
+ %a7 = extractelement <8 x i8> %input, i64 7
+ %c1 = icmp ugt i8 %a0, %a1
+ %m1 = select i1 %c1, i8 %a0, i8 %a1
+ %c2 = icmp ugt i8 %m1, %a2
+ %m2 = select i1 %c2, i8 %m1, i8 %a2
+ %c3 = icmp ugt i8 %m2, %a3
+ %m3 = select i1 %c3, i8 %m2, i8 %a3
+ %c4 = icmp ugt i8 %m3, %a4
+ %m4 = select i1 %c4, i8 %m3, i8 %a4
+ %c5 = icmp ugt i8 %m4, %a5
+ %m5 = select i1 %c5, i8 %m4, i8 %a5
+ %c6 = icmp ugt i8 %m5, %a6
+ %m6 = select i1 %c6, i8 %m5, i8 %a6
+ %c7 = icmp ugt i8 %m6, %a7
+ %m7 = select i1 %c7, i8 %m6, i8 %a7
+ %cmp = icmp ne i8 %m7, 0
+ %result = zext i1 %cmp to i32
+ ret i32 %result
+}
+
+define i32 @umax_zero_commuted(ptr %p) {
+; CHECK-LABEL: define i32 @umax_zero_commuted(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[INPUT]], zeroinitializer
+; CHECK-NEXT: [[CMP:%.*]] = call i1 @llvm.vector.reduce.and.v8i1(<8 x i1> [[TMP1]])
+; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %input = load <8 x i8>, ptr %p, align 1
+ %a0 = extractelement <8 x i8> %input, i64 0
+ %a1 = extractelement <8 x i8> %input, i64 1
+ %a2 = extractelement <8 x i8> %input, i64 2
+ %a3 = extractelement <8 x i8> %input, i64 3
+ %a4 = extractelement <8 x i8> %input, i64 4
+ %a5 = extractelement <8 x i8> %input, i64 5
+ %a6 = extractelement <8 x i8> %input, i64 6
+ %a7 = extractelement <8 x i8> %input, i64 7
+ %c1 = icmp ugt i8 %a0, %a1
+ %m1 = select i1 %c1, i8 %a0, i8 %a1
+ %c2 = icmp ugt i8 %m1, %a2
+ %m2 = select i1 %c2, i8 %m1, i8 %a2
+ %c3 = icmp ugt i8 %m2, %a3
+ %m3 = select i1 %c3, i8 %m2, i8 %a3
+ %c4 = icmp ugt i8 %m3, %a4
+ %m4 = select i1 %c4, i8 %m3, i8 %a4
+ %c5 = icmp ugt i8 %m4, %a5
+ %m5 = select i1 %c5, i8 %m4, i8 %a5
+ %c6 = icmp ugt i8 %m5, %a6
+ %m6 = select i1 %c6, i8 %m5, i8 %a6
+ %c7 = icmp ugt i8 %m6, %a7
+ %m7 = select i1 %c7, i8 %m6, i8 %a7
+ %cmp = icmp eq i8 0, %m7
+ %result = zext i1 %cmp to i32
+ ret i32 %result
+}
+
+define i32 @or_nonzero_multiuse(ptr %p, ptr %out) {
+; CHECK-LABEL: define i32 @or_nonzero_multiuse(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]]) {
+; CHECK-NEXT: [[INPUT:%.*]] = load <8 x i8>, ptr [[P]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> [[INPUT]])
+; CHECK-NEXT: store i8 [[TMP1]], ptr [[OUT]], align 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[TMP1]], 0
+; CHECK-NEXT: [[RESULT:%.*]] = zext i1 [[CMP]] to i32
+; CHECK-NEXT: ret i32 [[RESULT]]
+;
+ %input = load <8 x i8>, ptr %p, align 1
+ %a0 = extractelement <8 x i8> %input, i64 0
+ %a1 = extractelement <8 x i8> %input, i64 1
+ %a2 = extractelement <8 x i8> %input, i64 2
+ %a3 = extractelement <8 x i8> %input, i64 3
+ %a4 = extractelement <8 x i8> %input, i64 4
+ %a5 = extractelement <8 x i8> %input, i64 5
+ %a6 = extractelement <8 x i8> %input, i64 6
+ %a7 = extractelement <8 x i8> %input, i64 7
+ %or1 = or i8 %a0, %a1
+ %or2 = or i8 %or1, %a2
+ %or3 = or i8 %or2, %a3
+ %or4 = or i8 %or3, %a4
+ %or5 = or i8 %or4, %a5
+ %or6 = or i8 %or5, %a6
+ %or7 = or i8 %or6, %a7
+ store i8 %or7, ptr %out, align 1
+ %cmp = icmp ne i8 %or7, 0
+ %result = zext i1 %cmp to i32
+ ret i32 %result
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/205473
More information about the llvm-commits
mailing list