[llvm] f15666d - [SLP] Inefficient cost-modelling and codegen for reductions with slp-… (#197875)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jun 10 22:12:03 PDT 2026
Author: Sushant Gokhale
Date: 2026-06-10T22:11:58-07:00
New Revision: f15666db52ffafe798d9247442cb941c9dffd5b5
URL: https://github.com/llvm/llvm-project/commit/f15666db52ffafe798d9247442cb941c9dffd5b5
DIFF: https://github.com/llvm/llvm-project/commit/f15666db52ffafe798d9247442cb941c9dffd5b5.diff
LOG: [SLP] Inefficient cost-modelling and codegen for reductions with slp-… (#197875)
…revec
When revectorizing, starting with reduction, SLP generates slightly
inefficient code for reduction. e.g. In the godbolt link
[here](https://godbolt.org/z/ez7KPnxM5),
`hor_reduction --> original code`
`hor_reduction_revec_as_imagined_in_SLP --> revectorized code would look
like`
Rather than extracting per lane, we can extract original leaf nodes of
the reduction, which are sub-vectors, and then perform usual reduction
as in non-revectorized code. In the above link,
`hor_reduction_ideal_revec --> how the revec code should look like`
Extracting subvectors and achieving the reduction result would be better
than extracting per lane and achieving the same result.
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
llvm/test/Transforms/SLPVectorizer/revec.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5210088ded294..8a2e5a97f7573 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -18045,6 +18045,21 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
if (!DebugCounter::shouldExecute(VectorizedGraphs))
return true;
+ // If we are revectorizing reduction, it may result in same reduction pattern
+ // with shuffles as leaves of the reduction. Prevent SLP from revectorizing
+ // that shuffle pattern.
+ if (SLPReVec && ForReduction && VectorizableTree.size() == 3 &&
+ VectorizableTree[0]->State == TreeEntry::Vectorize &&
+ VectorizableTree[0]->getOpcode() == Instruction::ShuffleVector &&
+ VectorizableTree[1]->isGather() &&
+ isSplat(VectorizableTree[1]->Scalars) &&
+ VectorizableTree[2]->isGather() &&
+ allConstant(VectorizableTree[2]->Scalars)) {
+ LLVM_DEBUG(dbgs() << "SLP: Rejecting reduction tree with 3 nodes(shuffle "
+ "as root and remaining are gather nodes).\n");
+ return true;
+ }
+
// Graph is empty - do nothing.
if (VectorizableTree.empty()) {
assert(ExternalUses.empty() && "We shouldn't have any external users");
@@ -30230,25 +30245,33 @@ class HorizontalReduction {
} else if (auto *VecTy = dyn_cast<FixedVectorType>(DestTy)) {
unsigned DestTyNumElements = getNumElements(VecTy);
unsigned VF = getNumElements(Vec->getType()) / DestTyNumElements;
- Rdx = PoisonValue::get(
- getWidenedType(Vec->getType()->getScalarType(), DestTyNumElements));
- for (unsigned I : seq<unsigned>(DestTyNumElements)) {
- // Do reduction for each lane.
- // e.g., do reduce add for
- // VL[0] = <4 x Ty> <a, b, c, d>
- // VL[1] = <4 x Ty> <e, f, g, h>
- // Lane[0] = <2 x Ty> <a, e>
- // Lane[1] = <2 x Ty> <b, f>
- // Lane[2] = <2 x Ty> <c, g>
- // Lane[3] = <2 x Ty> <d, h>
- // result[0] = reduce add Lane[0]
- // result[1] = reduce add Lane[1]
- // result[2] = reduce add Lane[2]
- // result[3] = reduce add Lane[3]
- SmallVector<int, 16> Mask = createStrideMask(I, DestTyNumElements, VF);
- Value *Lane = Builder.CreateShuffleVector(Vec, Mask);
- Rdx = Builder.CreateInsertElement(
- Rdx, emitReduction(Lane, Builder, &TTI, DestTy), I);
+ // e.g. Consider vector reduce add.
+ // Initial reduction is
+ // clang-format off
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ //
+ // where,
+ // %A = <a, b, c, d>
+ // %B = <e, f, g, h>
+ // %add1 = A + B = <a + e, b + f, c + g, d + h>
+ //
+ // After revectorization with VF=2 and Vec = <a, b, c, d, e, f, g, h>,
+ // the reduction can be expressed as:
+ // %A = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ // %B = shufflevector <8 x i32> %Vec, <8 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ // %add0 = add <4 x i32> zeroinitializer, %A
+ // %add1 = add <4 x i32> %add0, %B
+ // clang-format on
+ Rdx = nullptr;
+ for (auto I : seq<unsigned>(VF)) {
+ auto Position = I * DestTyNumElements;
+ Value *SubVec =
+ createExtractVector(Builder, Vec, DestTyNumElements, Position);
+ if (!Rdx)
+ Rdx = SubVec;
+ else
+ Rdx = createOp(Builder, RdxKind, Rdx, SubVec, "rdx.op", ReductionOps);
}
} else {
Rdx = emitReduction(Vec, Builder, &TTI, DestTy);
@@ -30277,57 +30300,61 @@ class HorizontalReduction {
// If all of the reduced values are constant, the vector cost is 0, since
// the reduction value can be calculated at the compile time.
bool AllConsts = allConstant(ReducedVals);
- auto EvaluateScalarCost = [&](function_ref<InstructionCost()> GenCostFn) {
- InstructionCost Cost = 0;
- // Scalar cost is repeated for N-1 elements.
- int Cnt = ReducedVals.size();
- for (Value *RdxVal : ReducedVals) {
- if (!isa<Instruction>(RdxVal))
- continue;
- if (Cnt == 1) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn();
- break;
- }
- --Cnt;
- if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
- unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
- Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn();
- continue;
- }
- InstructionCost ScalarCost = 0;
- for (User *U : RdxVal->users()) {
- auto *RdxOp = cast<Instruction>(U);
- if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
- if (RdxKind == RecurKind::FAdd) {
- InstructionCost FMACost = canConvertToFMA(
- RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
- if (FMACost.isValid()) {
- LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
- if (auto *I = dyn_cast<Instruction>(RdxVal)) {
- // Also, exclude scalar fmul cost.
- InstructionCost FMulCost =
- TTI->getInstructionCost(I, CostKind);
- LLVM_DEBUG(dbgs() << "Minus FMul cost: " << FMulCost << "\n");
- FMACost -= FMulCost;
+ auto EvaluateScalarCost =
+ [&](function_ref<InstructionCost(Instruction *)> GenCostFn) {
+ InstructionCost Cost = 0;
+ // Scalar cost is repeated for N-1 elements.
+ int Cnt = ReducedVals.size();
+ for (auto [Idx, RdxVal] : enumerate(ReducedVals)) {
+ if (!isa<Instruction>(RdxVal))
+ continue;
+ Instruction *RdxOp = ReducedValsToOps[ReducedVals[Idx]].front();
+ if (Cnt == 1) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost +=
+ (SameValueCount ? SameValueCount - 1 : 0) * GenCostFn(RdxOp);
+ break;
+ }
+ --Cnt;
+ if (RdxVal->hasNUsesOrMore(IsCmpSelMinMax ? 3 : 2)) {
+ unsigned SameValueCount = SameValuesCounter.lookup(RdxVal);
+ Cost += (SameValueCount ? SameValueCount : 1) * GenCostFn(RdxOp);
+ continue;
+ }
+ InstructionCost ScalarCost = 0;
+ for (User *U : RdxVal->users()) {
+ auto *RdxOp = cast<Instruction>(U);
+ if (hasRequiredNumberOfUses(IsCmpSelMinMax, RdxOp)) {
+ if (RdxKind == RecurKind::FAdd) {
+ InstructionCost FMACost = canConvertToFMA(
+ RdxOp, getSameOpcode(RdxOp, TLI), DT, DL, *TTI, TLI);
+ if (FMACost.isValid()) {
+ LLVM_DEBUG(dbgs() << "FMA cost: " << FMACost << "\n");
+ if (auto *I = dyn_cast<Instruction>(RdxVal)) {
+ // Also, exclude scalar fmul cost.
+ InstructionCost FMulCost =
+ TTI->getInstructionCost(I, CostKind);
+ LLVM_DEBUG(dbgs()
+ << "Minus FMul cost: " << FMulCost << "\n");
+ FMACost -= FMulCost;
+ }
+ ScalarCost += FMACost;
+ continue;
+ }
}
- ScalarCost += FMACost;
+ ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
continue;
}
+ ScalarCost = InstructionCost::getInvalid();
+ break;
}
- ScalarCost += TTI->getInstructionCost(RdxOp, CostKind);
- continue;
+ if (ScalarCost.isValid())
+ Cost += ScalarCost;
+ else
+ Cost += GenCostFn(RdxOp);
}
- ScalarCost = InstructionCost::getInvalid();
- break;
- }
- if (ScalarCost.isValid())
- Cost += ScalarCost;
- else
- Cost += GenCostFn();
- }
- return Cost;
- };
+ return Cost;
+ };
// Require reduction cost if:
// 1. This type is not a full register type and no other vectors with the
// same type in the storage (first vector with small type).
@@ -30349,18 +30376,20 @@ class HorizontalReduction {
assert(SLPReVec && "FixedVectorType is not expected.");
unsigned ScalarTyNumElements = VecTy->getNumElements();
for (unsigned I : seq<unsigned>(ReducedVals.size())) {
- VectorCost += TTI->getShuffleCost(
- TTI::SK_PermuteSingleSrc,
- FixedVectorType::get(VecTy->getScalarType(),
- ReducedVals.size()),
- VectorTy,
- createStrideMask(I, ScalarTyNumElements, ReducedVals.size()));
- VectorCost += TTI->getArithmeticReductionCost(RdxOpcode, VecTy,
- FMF, CostKind);
+ VectorCost +=
+ ::getShuffleCost(*TTI, TTI::SK_ExtractSubvector, VectorTy, {},
+ CostKind, I * ScalarTyNumElements, VecTy);
+ }
+ // We get one less arithmetic instruction compared to number of
+ // reduced values. We are also passing CtxI so that the backend can
+ // synthesize FMF from the respective reduction instruction.
+ for (unsigned I : seq<unsigned>(ReducedVals.size() - 1)) {
+ VectorCost += TTI->getArithmeticInstrCost(
+ RdxOpcode, VecTy, CostKind,
+ TTI::getOperandInfo(ReducedVals[I]),
+ TTI::getOperandInfo(ReducedVals[I + 1]), {},
+ ReducedValsToOps[ReducedVals[I]].front());
}
- VectorCost += TTI->getScalarizationOverhead(
- VecTy, APInt::getAllOnes(ScalarTyNumElements), /*Insert*/ true,
- /*Extract*/ false, TTI::TCK_RecipThroughput);
} else {
Type *RedTy = VectorTy->getElementType();
auto [RType, IsSigned] = R.getRootNodeTypeWithNoCast().value_or(
@@ -30429,8 +30458,11 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
- return TTI->getArithmeticInstrCost(RdxOpcode, ScalarTy, CostKind);
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
+ return TTI->getArithmeticInstrCost(
+ RdxOpcode, ScalarTy, CostKind,
+ TTI::getOperandInfo(RdxOp->getOperand(0)),
+ TTI::getOperandInfo(RdxOp->getOperand(1)), {}, RdxOp);
});
break;
}
@@ -30465,7 +30497,7 @@ class HorizontalReduction {
}
}
}
- ScalarCost = EvaluateScalarCost([&]() {
+ ScalarCost = EvaluateScalarCost([&](Instruction *RdxOp) {
IntrinsicCostAttributes ICA(Id, ScalarTy, {ScalarTy, ScalarTy}, FMF);
return TTI->getIntrinsicInstrCost(ICA, CostKind);
});
diff --git a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
index 3f4436f33fad6..274d9a6e950a5 100644
--- a/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
+++ b/llvm/test/Transforms/SLPVectorizer/SystemZ/revec-fix-128169.ll
@@ -38,18 +38,13 @@ define void @e(<4 x i16> %0) {
; THRESH-NEXT: [[TMP7:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; THRESH-NEXT: [[TMP9:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> [[TMP7]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
; THRESH-NEXT: [[TMP13:%.*]] = icmp sgt <16 x i16> [[TMP9]], zeroinitializer
-; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; THRESH-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]])
-; THRESH-NEXT: [[TMP23:%.*]] = insertelement <4 x i1> poison, i1 [[TMP15]], i64 0
-; THRESH-NEXT: [[TMP16:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; THRESH-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP16]])
-; THRESH-NEXT: [[TMP24:%.*]] = insertelement <4 x i1> [[TMP23]], i1 [[TMP17]], i64 1
-; THRESH-NEXT: [[TMP18:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; THRESH-NEXT: [[TMP19:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]])
-; THRESH-NEXT: [[TMP22:%.*]] = insertelement <4 x i1> [[TMP24]], i1 [[TMP19]], i64 2
-; THRESH-NEXT: [[TMP20:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; THRESH-NEXT: [[TMP21:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP20]])
-; THRESH-NEXT: [[TMP25:%.*]] = insertelement <4 x i1> [[TMP22]], i1 [[TMP21]], i64 3
+; THRESH-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; THRESH-NEXT: [[TMP11:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; THRESH-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP14]], [[TMP11]]
+; THRESH-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; THRESH-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP12]]
+; THRESH-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP13]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; THRESH-NEXT: [[TMP25:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP15]]
; THRESH-NEXT: [[TMP26]] = zext <4 x i1> [[TMP25]] to <4 x i32>
; THRESH-NEXT: br label [[VECTOR_BODY]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
index 0e50b135afe7f..c69e07b77affa 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/revec-reduced-value-vectorized-later.ll
@@ -5,42 +5,37 @@ define <4 x i16> @test(<4 x i16> %a) {
; CHECK-LABEL: define <4 x i16> @test(
; CHECK-SAME: <4 x i16> [[A:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[A]], <4 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>, <16 x i16> [[TMP0]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 12, i32 13, i32 14, i32 15>
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP3:%.*]] = add <16 x i16> [[TMP2]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[TMP3]], <16 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i16> [[TMP4]], <4 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[TMP6:%.*]] = add <16 x i16> [[TMP5]], zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = add <16 x i16> [[TMP3]], zeroinitializer
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP9:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP8]])
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i16> poison, i16 [[TMP9]], i64 0
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP12:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP11]])
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i16> [[TMP10]], i16 [[TMP12]], i64 1
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP15:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP14]])
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> [[TMP13]], i16 [[TMP15]], i64 2
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <16 x i16> [[TMP6]], <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP18:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP17]])
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[TMP18]], i64 3
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP21:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP20]])
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> poison, i16 [[TMP21]], i64 0
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP24:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP23]])
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[TMP24]], i64 1
-; CHECK-NEXT: [[TMP26:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP27:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP26]])
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x i16> [[TMP25]], i16 [[TMP27]], i64 2
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <16 x i16> [[TMP7]], <16 x i16> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP30:%.*]] = call i16 @llvm.vector.reduce.or.v4i16(<4 x i16> [[TMP29]])
-; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i16> [[TMP28]], i16 [[TMP30]], i64 3
-; CHECK-NEXT: [[OP_RDX:%.*]] = or <4 x i16> [[TMP31]], [[TMP4]]
-; CHECK-NEXT: [[OP_RDX15:%.*]] = or <4 x i16> zeroinitializer, [[TMP19]]
-; CHECK-NEXT: [[OP_RDX16:%.*]] = or <4 x i16> [[OP_RDX]], [[OP_RDX15]]
-; CHECK-NEXT: ret <4 x i16> [[OP_RDX16]]
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[A]], <4 x i16> poison, <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <24 x i16> <i16 0, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, <24 x i16> [[TMP0]], <24 x i32> <i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[TMP2:%.*]] = add <24 x i16> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <24 x i16> [[TMP2]], <24 x i16> poison, <48 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <48 x i16> [[TMP3]], <48 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP5:%.*]] = add <48 x i16> [[TMP3]], zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i16> [[TMP6]], [[TMP7]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = or <4 x i16> [[RDX_OP]], [[TMP8]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = or <4 x i16> [[RDX_OP1]], [[TMP9]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 16, i32 17, i32 18, i32 19>
+; CHECK-NEXT: [[RDX_OP3:%.*]] = or <4 x i16> [[RDX_OP2]], [[TMP10]]
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[RDX_OP4:%.*]] = or <4 x i16> [[RDX_OP3]], [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 24, i32 25, i32 26, i32 27>
+; CHECK-NEXT: [[RDX_OP5:%.*]] = or <4 x i16> [[RDX_OP4]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: [[RDX_OP6:%.*]] = or <4 x i16> [[RDX_OP5]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 32, i32 33, i32 34, i32 35>
+; CHECK-NEXT: [[RDX_OP7:%.*]] = or <4 x i16> [[RDX_OP6]], [[TMP14]]
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 36, i32 37, i32 38, i32 39>
+; CHECK-NEXT: [[RDX_OP8:%.*]] = or <4 x i16> [[RDX_OP7]], [[TMP15]]
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 40, i32 41, i32 42, i32 43>
+; CHECK-NEXT: [[RDX_OP9:%.*]] = or <4 x i16> [[RDX_OP8]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <48 x i16> [[TMP5]], <48 x i16> poison, <4 x i32> <i32 44, i32 45, i32 46, i32 47>
+; CHECK-NEXT: [[RDX_OP10:%.*]] = or <4 x i16> [[RDX_OP9]], [[TMP17]]
+; CHECK-NEXT: [[OP_RDX9:%.*]] = or <4 x i16> [[RDX_OP10]], [[TMP4]]
+; CHECK-NEXT: ret <4 x i16> [[OP_RDX9]]
;
entry:
%subi = add <4 x i16> %a, %a
diff --git a/llvm/test/Transforms/SLPVectorizer/revec.ll b/llvm/test/Transforms/SLPVectorizer/revec.ll
index 66507f647a828..346669c53824f 100644
--- a/llvm/test/Transforms/SLPVectorizer/revec.ll
+++ b/llvm/test/Transforms/SLPVectorizer/revec.ll
@@ -156,18 +156,13 @@ define <4 x i1> @test6(ptr %in1, ptr %in2) {
; CHECK-NEXT: [[TMP22:%.*]] = and <16 x i1> [[TMP11]], [[TMP21]]
; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <32 x i1> [[TMP6]], <32 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
; CHECK-NEXT: [[TMP24:%.*]] = and <16 x i1> [[TMP22]], [[TMP23]]
-; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP26:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP25]])
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x i1> poison, i1 [[TMP26]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
-; CHECK-NEXT: [[TMP29:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP28]])
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i1> [[TMP27]], i1 [[TMP29]], i64 1
-; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
-; CHECK-NEXT: [[TMP32:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP31]])
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i1> [[TMP30]], i1 [[TMP32]], i64 2
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
-; CHECK-NEXT: [[TMP35:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP34]])
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x i1> [[TMP33]], i1 [[TMP35]], i64 3
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP15]], [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = or <4 x i1> [[RDX_OP]], [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <16 x i1> [[TMP24]], <16 x i1> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP36:%.*]] = or <4 x i1> [[RDX_OP1]], [[TMP14]]
; CHECK-NEXT: [[VBSL:%.*]] = select <4 x i1> [[TMP36]], <4 x i32> <i32 1, i32 2, i32 3, i32 4>, <4 x i32> <i32 5, i32 6, i32 7, i32 8>
; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <4 x i32> [[VBSL]], <i32 2, i32 3, i32 4, i32 5>
; CHECK-NEXT: ret <4 x i1> [[CMP]]
@@ -474,3 +469,32 @@ entry:
store <4 x float> %11, ptr %1, align 16
ret i32 0
}
+
+define <4 x i32> @hor_reduction_four_points(ptr %a) {
+; CHECK-LABEL: @hor_reduction_four_points(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <16 x i32>, ptr [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[RDX_OP:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[RDX_OP1:%.*]] = add <4 x i32> [[RDX_OP]], [[TMP3]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[RDX_OP2:%.*]] = add <4 x i32> [[RDX_OP1]], [[TMP4]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = add <4 x i32> [[RDX_OP2]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <4 x i32> [[OP_RDX]]
+;
+entry:
+ %gep1 = getelementptr <4 x i32>, ptr %a, i64 1
+ %gep2 = getelementptr <4 x i32>, ptr %a, i64 2
+ %gep3 = getelementptr <4 x i32>, ptr %a, i64 3
+ %a0 = load <4 x i32>, ptr %a
+ %a1 = load <4 x i32>, ptr %gep1
+ %a2 = load <4 x i32>, ptr %gep2
+ %a3 = load <4 x i32>, ptr %gep3
+ %add0 = add <4 x i32> <i32 0, i32 1, i32 2, i32 3>, %a0
+ %add1 = add <4 x i32> %add0, %a1
+ %add2 = add <4 x i32> %add1, %a2
+ %add3 = add <4 x i32> %add2, %a3
+ ret <4 x i32> %add3
+}
More information about the llvm-commits
mailing list