[llvm] [SLP]Analyze widened reduction leaves in the narrow type (PR #216062)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 20 15:58:07 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/216062
>From 34065592405d0e10cbd2fac3dbd3d5a7ae30e831 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 13 Aug 2026 07:23:46 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 136 +++++++++++++++++-
.../Vectorize/SLPVectorizer/SLPUtils.cpp | 54 +++++++
.../Vectorize/SLPVectorizer/SLPUtils.h | 11 ++
.../RISCV/zext-or-nibble-reduction.ll | 67 ++-------
.../X86/zext-or-nibble-reduction.ll | 38 +----
5 files changed, 219 insertions(+), 87 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 85daf23009400..0c1d60d6fcbf0 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29850,6 +29850,11 @@ class HorizontalReduction {
SmallVector<std::tuple<WeakTrackingVH, unsigned, bool, bool>>
VectorValuesAndScales;
+ /// Narrow reduced values mapped to the shift applied after widening.
+ SmallDenseMap<Value *, unsigned> NarrowedLeafShifts;
+ /// zext/shl/packed-binop chain dropped with the reduction.
+ SmallVector<Value *> NarrowedChainInsts;
+
static bool isCmpSelMinMax(Instruction *I) {
return match(I, m_Select(m_Cmp(), m_Value(), m_Value())) &&
RecurrenceDescriptor::isMinMaxRecurrenceKind(getRdxKind(I));
@@ -30260,6 +30265,8 @@ class HorizontalReduction {
ReducedValsToOps.clear();
ReductionOps.clear();
RoundedLinks.clear();
+ NarrowedLeafShifts.clear();
+ NarrowedChainInsts.clear();
RdxKind = getRdxKind(Root);
// Currently, only ordered fadd reductions are supported.
if (RdxKind != RecurKind::FAdd)
@@ -30532,6 +30539,71 @@ class HorizontalReduction {
PossibleOrderedReductionOps.clear();
}
}
+ // InstCombine may pack same-kind binops under a zext, hiding the leaves.
+ // Analyze them in the narrow type and widen/shift the vectorized result.
+ if ((RdxKind == RecurKind::And || RdxKind == RecurKind::Or ||
+ RdxKind == RecurKind::Xor) &&
+ RK == ReductionOrdering::Unordered && !IsCmpSelMinMax &&
+ Ty->isIntegerTy() && !Ty->isIntegerTy(1) &&
+ all_of(ReductionOps,
+ [](ArrayRef<Value *> RdxOps) {
+ return all_of(RdxOps, IsaPred<BinaryOperator>);
+ }) &&
+ any_of(ReducedValsCandidates, IsaPred<ZExtInst>)) {
+ unsigned WideBW = Ty->getIntegerBitWidth();
+ SmallVector<SmallVector<std::pair<Value *, unsigned>>> AllLeaves;
+ SmallVector<Value *> ChainInsts;
+ Type *NarrowTy = nullptr;
+ bool Bail = false;
+ unsigned NumLeaves = 0;
+ SmallPtrSet<Value *, 8> UniqueLeaves;
+ for (Value *Cand : ReducedValsCandidates) {
+ SmallVector<std::pair<Value *, unsigned>> Leaves;
+ collectNarrowedLeaves(Cand, RecurrenceDescriptor::getOpcode(RdxKind),
+ WideBW, RecursionMaxDepth, Leaves, ChainInsts);
+ for (const auto &[N, S] : Leaves) {
+ if (S >= WideBW || isa<Constant>(N) || !N->getType()->isIntegerTy() ||
+ (NarrowTy && N->getType() != NarrowTy) ||
+ !UniqueLeaves.insert(N).second) {
+ Bail = true;
+ break;
+ }
+ if (!NarrowTy)
+ NarrowTy = N->getType();
+ }
+ if (Bail)
+ break;
+ NumLeaves += Leaves.size();
+ AllLeaves.push_back(std::move(Leaves));
+ }
+ if (!Bail && NumLeaves > ReducedValsCandidates.size() && NarrowTy &&
+ NarrowTy != Ty) {
+ // Collected user-first; dies with the reduction iff all users die.
+ SmallPtrSet<Value *, 8> Ignorable;
+ for (ArrayRef<Value *> RdxOps : ReductionOps)
+ Ignorable.insert_range(RdxOps);
+ for (Value *I : ChainInsts)
+ if (all_of(I->users(),
+ [&](User *U) { return Ignorable.contains(U); })) {
+ Ignorable.insert(I);
+ NarrowedChainInsts.push_back(I);
+ }
+ SmallVector<Value *> NewCandidates;
+ for (const auto &[Cand, Leaves] :
+ zip(ReducedValsCandidates, AllLeaves)) {
+ SmallVector<Instruction *> RdxOps;
+ if (auto It = ReducedValsToOps.find(Cand);
+ It != ReducedValsToOps.end())
+ RdxOps = It->second;
+ for (const auto &[N, S] : Leaves) {
+ NarrowedLeafShifts.try_emplace(N, S);
+ NewCandidates.push_back(N);
+ ReducedValsToOps[N].append(RdxOps);
+ }
+ }
+ ReducedValsCandidates = std::move(NewCandidates);
+ }
+ }
// Too many integer reduced values candidates for the ordered reductions
// after adjustements - try to switch to unordered reductions instead.
constexpr unsigned ReducedValsLimit = 1024;
@@ -30696,6 +30768,8 @@ class HorizontalReduction {
continue;
IgnoreList.insert(RdxOp);
}
+ // Dropped with the reduction; do not treat as external uses.
+ IgnoreList.insert_range(NarrowedChainInsts);
// Intersect the fast-math-flags from all reduction operations.
FastMathFlags RdxFMF;
RdxFMF.set();
@@ -31213,6 +31287,26 @@ class HorizontalReduction {
SameValuesCounter, RootTrackedToOrig);
}
+ // Widen and shift per lane. Map back from extractelements to the
+ // original reduced values for the shift amounts.
+ if (!NarrowedLeafShifts.empty()) {
+ Type *WideTy = ReductionRoot->getType();
+ unsigned VF = getNumElements(VectorizedRoot->getType());
+ VectorizedRoot =
+ Builder.CreateZExt(VectorizedRoot, getWidenedType(WideTy, VF));
+ SmallVector<Constant *> ShiftConsts(VF, ConstantInt::get(WideTy, 0));
+ bool AnyShift = false;
+ for (auto [Idx, Val] : enumerate(VL)) {
+ unsigned Shift = NarrowedLeafShifts.at(TrackedToOrig[Pos + Idx]);
+ ShiftConsts[V.findRootLaneForValue(Val)] =
+ ConstantInt::get(WideTy, Shift);
+ AnyShift |= Shift != 0;
+ }
+ if (AnyShift)
+ VectorizedRoot = Builder.CreateShl(
+ VectorizedRoot, ConstantVector::get(ShiftConsts));
+ }
+
Type *ScalarTy = VL.front()->getType();
Type *VecTy = VectorizedRoot->getType();
Type *RedScalarTy = VecTy->getScalarType();
@@ -31222,7 +31316,7 @@ class HorizontalReduction {
? SameValuesCounter.front().second
: 1,
RedScalarTy != ScalarTy->getScalarType()
- ? V.isSignedMinBitwidthRootNode()
+ ? NarrowedLeafShifts.empty() && V.isSignedMinBitwidthRootNode()
: true,
V.isReducedBitcastRoot() || V.isReducedCmpBitcastRoot());
@@ -31323,6 +31417,15 @@ class HorizontalReduction {
if (NeedFreeze)
LHS = Builder.CreateFreeze(LHS);
};
+ // Leftover narrow values need the same zext/shift as the vectorized ones.
+ auto WidenNarrowedVal = [&](Value *OrigV, Value *&V) {
+ auto It = NarrowedLeafShifts.find(OrigV);
+ if (It == NarrowedLeafShifts.end())
+ return;
+ V = Builder.CreateZExt(V, ReductionRoot->getType());
+ if (It->second != 0)
+ V = Builder.CreateShl(V, It->second);
+ };
// Finish the reduction.
// Need to add extra arguments and not vectorized possible reduction values.
// Try to avoid dependencies between the scalar remainders after reductions.
@@ -31343,6 +31446,8 @@ class HorizontalReduction {
auto It2 = TrackedVals.find(RdxVal2);
if (It2 != TrackedVals.end())
StableRdxVal2 = It2->second;
+ WidenNarrowedVal(RdxVal1, StableRdxVal1);
+ WidenNarrowedVal(RdxVal2, StableRdxVal2);
// To prevent poison from leaking across what used to be sequential,
// safe, scalar boolean logic operations, the reduction operand must be
// frozen.
@@ -31850,7 +31955,30 @@ class HorizontalReduction {
case RecurKind::FMul: {
unsigned RdxOpcode = RecurrenceDescriptor::getOpcode(RdxKind);
if (!AllConsts) {
- if (DoesRequireReductionOp) {
+ if (!NarrowedLeafShifts.empty()) {
+ Type *WideTy = ReductionRoot->getType();
+ auto *NarrowVecTy =
+ cast<VectorType>(getWidenedType(ScalarTy, ReduxWidth));
+ auto *WideVecTy =
+ cast<VectorType>(getWidenedType(WideTy, ReduxWidth));
+ if (DoesRequireReductionOp) {
+ VectorCost = TTI->getExtendedReductionCost(
+ RdxOpcode, /*IsUnsigned=*/true, WideTy, NarrowVecTy, FMF,
+ CostKind);
+ for (Value *I : NarrowedChainInsts)
+ VectorCost -=
+ TTI->getInstructionCost(cast<Instruction>(I), CostKind);
+ } else {
+ VectorCost =
+ TTI->getCastInstrCost(Instruction::ZExt, WideVecTy, NarrowVecTy,
+ TTI::CastContextHint::None, CostKind) +
+ TTI->getArithmeticInstrCost(RdxOpcode, WideVecTy, CostKind);
+ }
+ if (any_of(NarrowedLeafShifts,
+ [](const auto &P) { return P.second != 0; }))
+ VectorCost += TTI->getArithmeticInstrCost(Instruction::Shl,
+ WideVecTy, CostKind);
+ } else if (DoesRequireReductionOp) {
if (auto *VecTy = dyn_cast<FixedVectorType>(ScalarTy)) {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
@@ -31937,9 +32065,11 @@ class HorizontalReduction {
}
}
}
+ Type *ScalarCostTy =
+ !NarrowedLeafShifts.empty() ? ReductionRoot->getType() : ScalarTy;
ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
return TTI->getArithmeticInstrCost(
- RdxOpcode, ScalarTy, CostKind,
+ RdxOpcode, ScalarCostTy, CostKind,
TTI::getOperandInfo(RdxOp->getOperand(0)),
TTI::getOperandInfo(RdxOp->getOperand(1)), {}, RdxOp);
});
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
index 84e8cc2b66943..6f17de595b000 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.cpp
@@ -8,6 +8,7 @@
#include "SLPUtils.h"
+#include "llvm/ADT/APInt.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/Analysis/ValueTracking.h"
@@ -770,4 +771,57 @@ Instruction *lookThroughCastRoundTrip(Value *V, bool MustBeElidable) {
return Narrow;
}
+static void
+collectNarrowedLeavesImpl(Value *V, unsigned RdxOpcode, unsigned WideBW,
+ unsigned Shift, unsigned Depth, unsigned MaxDepth,
+ SmallVectorImpl<std::pair<Value *, unsigned>> &Leaves,
+ SmallVectorImpl<Value *> &ChainInsts) {
+ if (Depth >= MaxDepth) {
+ Leaves.emplace_back(V, Shift);
+ return;
+ }
+ if (auto *Z = dyn_cast<ZExtInst>(V);
+ Z && Z->getSrcTy()->isIntegerTy() && !Z->getSrcTy()->isIntegerTy(1)) {
+ ChainInsts.push_back(Z);
+ return collectNarrowedLeavesImpl(Z->getOperand(0), RdxOpcode, WideBW, Shift,
+ Depth + 1, MaxDepth, Leaves, ChainInsts);
+ }
+ auto *BO = dyn_cast<BinaryOperator>(V);
+ if (!BO) {
+ Leaves.emplace_back(V, Shift);
+ return;
+ }
+ if (BO->getOpcode() == RdxOpcode) {
+ ChainInsts.push_back(BO);
+ collectNarrowedLeavesImpl(BO->getOperand(0), RdxOpcode, WideBW, Shift,
+ Depth + 1, MaxDepth, Leaves, ChainInsts);
+ collectNarrowedLeavesImpl(BO->getOperand(1), RdxOpcode, WideBW, Shift,
+ Depth + 1, MaxDepth, Leaves, ChainInsts);
+ return;
+ }
+ const APInt *Amt;
+ unsigned BW = V->getType()->getScalarSizeInBits();
+ auto *Z = dyn_cast<ZExtInst>(BO->getOperand(0));
+ if (BO->getOpcode() == Instruction::Shl && Z &&
+ match(BO->getOperand(1), m_APInt(Amt)) && Amt->ult(BW) &&
+ Z->getSrcTy()->isIntegerTy() && !Z->getSrcTy()->isIntegerTy(1) &&
+ (BW == WideBW ||
+ Z->getSrcTy()->getIntegerBitWidth() + Amt->getZExtValue() <= BW)) {
+ ChainInsts.push_back(BO);
+ ChainInsts.push_back(Z);
+ return collectNarrowedLeavesImpl(Z->getOperand(0), RdxOpcode, WideBW,
+ Shift + Amt->getZExtValue(), Depth + 1,
+ MaxDepth, Leaves, ChainInsts);
+ }
+ Leaves.emplace_back(V, Shift);
+}
+
+void collectNarrowedLeaves(
+ Value *V, unsigned RdxOpcode, unsigned WideBW, unsigned MaxDepth,
+ SmallVectorImpl<std::pair<Value *, unsigned>> &Leaves,
+ SmallVectorImpl<Value *> &ChainInsts) {
+ collectNarrowedLeavesImpl(V, RdxOpcode, WideBW, /*Shift=*/0, /*Depth=*/0,
+ MaxDepth, Leaves, ChainInsts);
+}
+
} // namespace llvm::slpvectorizer
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
index bdc8881e14090..afd944ff57e71 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer/SLPUtils.h
@@ -24,6 +24,7 @@
#include <optional>
#include <string>
+#include <utility>
namespace llvm {
class Constant;
@@ -336,6 +337,16 @@ bool isOnceUsedSeed(const Instruction *I);
/// produce nan/inf.
Instruction *lookThroughCastRoundTrip(Value *V, bool MustBeElidable);
+/// Recursively collects the narrow leaves of the widened reduction value
+/// \p V with the accumulated shift amount \p Shift. zext is looked through
+/// directly, same-kind binops per operand, shl - only if no bits are
+/// shifted out in the current type. Also collects the looked-through
+/// instructions into \p ChainInsts.
+void collectNarrowedLeaves(
+ Value *V, unsigned RdxOpcode, unsigned WideBW, unsigned MaxDepth,
+ SmallVectorImpl<std::pair<Value *, unsigned>> &Leaves,
+ SmallVectorImpl<Value *> &ChainInsts);
+
} // namespace llvm::slpvectorizer
#endif // LLVM_LIB_TRANSFORMS_VECTORIZE_SLPVECTORIZER_SLPUTILS_H
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/zext-or-nibble-reduction.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/zext-or-nibble-reduction.ll
index da87e9f043008..9f158b357c2ad 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/zext-or-nibble-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/zext-or-nibble-reduction.ll
@@ -9,32 +9,13 @@
define i64 @test(ptr %src) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[L0:%.*]] = load i8, ptr [[SRC:%.*]], align 1
-; CHECK-NEXT: [[A0:%.*]] = and i8 [[L0]], 15
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[SH1:%.*]] = shl i8 [[L1]], 4
-; CHECK-NEXT: [[OR8:%.*]] = or disjoint i8 [[SH1]], [[A0]]
-; CHECK-NEXT: [[OR0:%.*]] = zext i8 [[OR8]] to i64
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 2
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[P2]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i8> [[TMP0]], splat (i8 15)
-; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i64> [[TMP2]], <i64 8, i64 12, i64 16, i64 20>
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 6
-; CHECK-NEXT: [[L6:%.*]] = load i8, ptr [[P6]], align 1
-; CHECK-NEXT: [[A6:%.*]] = and i8 [[L6]], 15
-; CHECK-NEXT: [[Z6:%.*]] = zext nneg i8 [[A6]] to i64
-; CHECK-NEXT: [[SH6:%.*]] = shl nuw nsw i64 [[Z6]], 24
-; CHECK-NEXT: [[P7:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 7
-; CHECK-NEXT: [[L7:%.*]] = load i8, ptr [[P7]], align 1
-; CHECK-NEXT: [[A7:%.*]] = and i8 [[L7]], 15
-; CHECK-NEXT: [[Z7:%.*]] = zext nneg i8 [[A7]] to i64
-; CHECK-NEXT: [[SH7:%.*]] = shl nuw nsw i64 [[Z7]], 28
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = or i64 [[TMP4]], [[OR0]]
-; CHECK-NEXT: [[OP_RDX1:%.*]] = or i64 [[SH6]], [[SH7]]
-; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[OP_RDX]], [[OP_RDX1]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[SRC:%.*]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = shl <8 x i64> [[TMP4]], <i64 0, i64 0, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP5]])
; CHECK-NEXT: ret i64 [[TMP6]]
;
entry:
@@ -89,33 +70,13 @@ entry:
define i64 @test_i32_pack(ptr %src) {
; CHECK-LABEL: @test_i32_pack(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[L0:%.*]] = load i8, ptr [[SRC:%.*]], align 1
-; CHECK-NEXT: [[A0:%.*]] = and i8 [[L0]], 15
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[SH1:%.*]] = shl i8 [[L1]], 4
-; CHECK-NEXT: [[OR8:%.*]] = or disjoint i8 [[SH1]], [[A0]]
-; CHECK-NEXT: [[Z01:%.*]] = zext i8 [[OR8]] to i32
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 2
-; CHECK-NEXT: [[L2:%.*]] = load i8, ptr [[P2]], align 1
-; CHECK-NEXT: [[A2:%.*]] = and i8 [[L2]], 15
-; CHECK-NEXT: [[Z2:%.*]] = zext nneg i8 [[A2]] to i32
-; CHECK-NEXT: [[SH2:%.*]] = shl nuw i32 [[Z2]], 8
-; CHECK-NEXT: [[OR16:%.*]] = or disjoint i32 [[SH2]], [[Z01]]
-; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 3
-; CHECK-NEXT: [[L3:%.*]] = load i8, ptr [[P3]], align 1
-; CHECK-NEXT: [[A3:%.*]] = and i8 [[L3]], 15
-; CHECK-NEXT: [[Z3:%.*]] = zext nneg i8 [[A3]] to i32
-; CHECK-NEXT: [[SH3:%.*]] = shl nuw i32 [[Z3]], 12
-; CHECK-NEXT: [[OR32:%.*]] = or disjoint i32 [[SH3]], [[OR16]]
-; CHECK-NEXT: [[OR0:%.*]] = zext i32 [[OR32]] to i64
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[P4]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i8> [[TMP0]], splat (i8 15)
-; CHECK-NEXT: [[TMP2:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i64>
-; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw <4 x i64> [[TMP2]], <i64 16, i64 20, i64 24, i64 28>
-; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP3]])
-; CHECK-NEXT: [[TMP6:%.*]] = or i64 [[TMP4]], [[OR0]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[SRC:%.*]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = shl <8 x i64> [[TMP4]], <i64 0, i64 0, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP5]])
; CHECK-NEXT: ret i64 [[TMP6]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext-or-nibble-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext-or-nibble-reduction.ll
index 85ca961a7970a..656a847ffe8cd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/zext-or-nibble-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/zext-or-nibble-reduction.ll
@@ -9,37 +9,13 @@
define i64 @test(ptr %src) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[L0:%.*]] = load i8, ptr [[SRC:%.*]], align 1
-; CHECK-NEXT: [[A0:%.*]] = and i8 [[L0]], 15
-; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load i8, ptr [[P1]], align 1
-; CHECK-NEXT: [[SH1:%.*]] = shl i8 [[L1]], 4
-; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 2
-; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 4
-; CHECK-NEXT: [[L4:%.*]] = load i8, ptr [[P4]], align 1
-; CHECK-NEXT: [[OR8:%.*]] = or disjoint i8 [[SH1]], [[A0]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr [[P2]], align 1
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i8> [[TMP0]], <2 x i8> poison, <4 x i32> <i32 poison, i32 0, i32 1, i32 poison>
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i8> [[TMP1]], i8 [[OR8]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i8> [[TMP2]], i8 [[L4]], i64 3
-; CHECK-NEXT: [[TMP4:%.*]] = and <4 x i8> [[TMP3]], <i8 -1, i8 15, i8 15, i8 15>
-; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i64>
-; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw <4 x i64> [[TMP5]], <i64 0, i64 8, i64 12, i64 16>
-; CHECK-NEXT: [[P5:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 5
-; CHECK-NEXT: [[L5:%.*]] = load i8, ptr [[P5]], align 1
-; CHECK-NEXT: [[A5:%.*]] = and i8 [[L5]], 15
-; CHECK-NEXT: [[Z5:%.*]] = zext nneg i8 [[A5]] to i64
-; CHECK-NEXT: [[P6:%.*]] = getelementptr inbounds nuw i8, ptr [[SRC]], i64 6
-; CHECK-NEXT: [[TMP7:%.*]] = load <2 x i8>, ptr [[P6]], align 1
-; CHECK-NEXT: [[TMP8:%.*]] = and <2 x i8> [[TMP7]], splat (i8 15)
-; CHECK-NEXT: [[TMP9:%.*]] = zext <2 x i8> [[TMP8]] to <2 x i64>
-; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP16]])
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i64 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i64> [[TMP11]], i64 [[Z5]], i64 1
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i64> [[TMP9]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i64> [[TMP12]], <4 x i64> [[TMP13]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw <4 x i64> [[TMP14]], <i64 0, i64 20, i64 24, i64 28>
-; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.or.v4i64(<4 x i64> [[TMP15]])
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[SRC:%.*]], align 1
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <8 x i8> [[TMP0]], <i8 15, i8 4, i8 15, i8 15, i8 15, i8 15, i8 15, i8 15>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i64>
+; CHECK-NEXT: [[TMP5:%.*]] = shl <8 x i64> [[TMP4]], <i64 0, i64 0, i64 8, i64 12, i64 16, i64 20, i64 24, i64 28>
+; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP5]])
; CHECK-NEXT: ret i64 [[TMP6]]
;
entry:
More information about the llvm-commits
mailing list