[llvm] [SLP]Initial support for ordreded reductions (PR #182644)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Feb 20 17:47:33 PST 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-vectorizers
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Patches models ordered reductions as a serie of extracelements for the
cases, which cannot be modeled as unordered reductions.
Fixes #<!-- -->50590
---
Patch is 73.79 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/182644.diff
8 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+60-23)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll (+7-13)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll (+63-29)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll (+241-41)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/fminnum.ll (+221-41)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll (+5-7)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/phi.ll (+26-24)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reorder-non-empty.ll (+10-7)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4caa1707f0f27..b1df83e021cb7 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -25349,31 +25349,37 @@ class HorizontalReduction {
}
/// Checks if instruction is associative and can be vectorized.
- static bool isVectorizable(RecurKind Kind, Instruction *I,
- bool TwoElementReduction = false) {
+ enum class ReductionKind {Unordered, Ordered, None};
+ ReductionKind RK = ReductionKind::None;
+ static ReductionKind isVectorizable(RecurKind Kind, Instruction *I,
+ bool TwoElementReduction = false) {
if (Kind == RecurKind::None)
- return false;
+ return ReductionKind::None;
// Integer ops that map to select instructions or intrinsics are fine.
if (RecurrenceDescriptor::isIntMinMaxRecurrenceKind(Kind) ||
isBoolLogicOp(I))
- return true;
+ return ReductionKind::Unordered;
// No need to check for associativity, if 2 reduced values.
if (TwoElementReduction)
- return true;
+ return ReductionKind::Unordered;
if (Kind == RecurKind::FMax || Kind == RecurKind::FMin) {
// FP min/max are associative except for NaN and -0.0. We do not
// have to rule out -0.0 here because the intrinsic semantics do not
// specify a fixed result for it.
- return I->getFastMathFlags().noNaNs();
+ return I->getFastMathFlags().noNaNs() ? ReductionKind::Unordered
+ : ReductionKind::Ordered;
}
if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
- return true;
+ return ReductionKind::Unordered;
+
+ if (I->isAssociative())
+ return ReductionKind::Unordered;
- return I->isAssociative();
+ return ::isCommutative(I) ? ReductionKind::Ordered : ReductionKind::None;
}
static Value *getRdxOperand(Instruction *I, unsigned Index) {
@@ -25675,13 +25681,10 @@ class HorizontalReduction {
// Analyze "regular" integer/FP types for reductions - no target-specific
// types or pointers.
assert(ReductionRoot && "Reduction root is not set!");
- if (!isVectorizable(RdxKind, cast<Instruction>(ReductionRoot),
- all_of(ReducedVals, [](ArrayRef<Value *> Ops) {
- return Ops.size() == 2;
- })))
- return false;
-
- return true;
+ return isVectorizable(RdxKind, cast<Instruction>(ReductionRoot),
+ all_of(ReducedVals, [](ArrayRef<Value *> Ops) {
+ return Ops.size() == 2;
+ })) != ReductionKind::None;
}
/// Try to find a reduction tree.
@@ -25689,7 +25692,8 @@ class HorizontalReduction {
ScalarEvolution &SE, const DataLayout &DL,
const TargetLibraryInfo &TLI) {
RdxKind = HorizontalReduction::getRdxKind(Root);
- if (!isVectorizable(RdxKind, Root))
+ RK = isVectorizable(RdxKind, Root);
+ if (RK == ReductionKind::None)
return false;
// Analyze "regular" integer/FP types for reductions - no target-specific
@@ -25728,16 +25732,21 @@ class HorizontalReduction {
// reduction opcode or has too many uses - possible reduced value.
// Also, do not try to reduce const values, if the operation is not
// foldable.
- if (!EdgeInst || Level > RecursionMaxDepth ||
+ bool IsReducedVal = !EdgeInst || Level > RecursionMaxDepth ||
getRdxKind(EdgeInst) != RdxKind ||
IsCmpSelMinMax != isCmpSelMinMax(EdgeInst) ||
- !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst) ||
- !isVectorizable(RdxKind, EdgeInst) ||
+ !hasRequiredNumberOfUses(IsCmpSelMinMax, EdgeInst);
+ ReductionKind CurrentRK = IsReducedVal
+ ? ReductionKind::None
+ : isVectorizable(RdxKind, EdgeInst);
+ if (CurrentRK == ReductionKind::None ||
(R.isAnalyzedReductionRoot(EdgeInst) &&
all_of(EdgeInst->operands(), IsaPred<Constant>))) {
PossibleReducedVals.push_back(EdgeVal);
continue;
}
+ if (CurrentRK == ReductionKind::Ordered)
+ RK = ReductionKind::Ordered;
ReductionOps.push_back(EdgeInst);
}
};
@@ -25943,6 +25952,10 @@ class HorizontalReduction {
for (Value *U : IgnoreList)
if (auto *FPMO = dyn_cast<FPMathOperator>(U))
RdxFMF &= FPMO->getFastMathFlags();
+ // For ordered reductions here we need to generate extractelement
+ // instructions, so clear IgnoreList.
+ if (RK == ReductionKind::Ordered)
+ IgnoreList.clear();
bool IsCmpSelMinMax = isCmpSelMinMax(cast<Instruction>(ReductionRoot));
// Need to track reduced vals, they may be changed during vectorization of
@@ -26054,6 +26067,8 @@ class HorizontalReduction {
// Emit code for constant values.
if (Candidates.size() > 1 && allConstant(Candidates)) {
+ if (RK == ReductionKind::Ordered)
+ continue;
Value *Res = Candidates.front();
Value *OrigV = TrackedToOrig.at(Candidates.front());
++VectorizedVals.try_emplace(OrigV).first->getSecond();
@@ -26075,9 +26090,9 @@ class HorizontalReduction {
// Check if we support repeated scalar values processing (optimization of
// original scalar identity operations on matched horizontal reductions).
- IsSupportedHorRdxIdentityOp = RdxKind != RecurKind::Mul &&
- RdxKind != RecurKind::FMul &&
- RdxKind != RecurKind::FMulAdd;
+ IsSupportedHorRdxIdentityOp =
+ RK == ReductionKind::Unordered && RdxKind != RecurKind::Mul &&
+ RdxKind != RecurKind::FMul && RdxKind != RecurKind::FMulAdd;
// Gather same values.
SmallMapVector<Value *, unsigned, 16> SameValuesCounter;
if (IsSupportedHorRdxIdentityOp)
@@ -26346,6 +26361,23 @@ class HorizontalReduction {
// Vectorize a tree.
Value *VectorizedRoot = V.vectorizeTree(
LocalExternallyUsedValues, InsertPt, VectorValuesAndScales);
+ if (RK == ReductionKind::Ordered) {
+ // No need to generate reduction here, eit extractelements instead in
+ // the tree vectorizer.
+ assert(VectorizedRoot && "Expected vectorized tree");
+ // Count vectorized reduced values to exclude them from final
+ // reduction.
+ for (Value *RdxVal : VL)
+ ++VectorizedVals.try_emplace(RdxVal).first->getSecond();
+ Pos += ReduxWidth;
+ Start = Pos;
+ ReduxWidth = NumReducedVals - Pos;
+ if (ReduxWidth > 1)
+ ReduxWidth = GetVectorFactor(NumReducedVals - Pos);
+ AnyVectorized = true;
+ VectorizedTree = ReductionRoot;
+ continue;
+ }
// Update TrackedToOrig mapping, since the tracked values might be
// updated.
for (Value *RdxVal : Candidates) {
@@ -26410,6 +26442,11 @@ class HorizontalReduction {
continue;
}
}
+ // Early exit for the ordered reductions.
+ // No need to do anything else here, so we can just exit.
+ if (RK == ReductionKind::Ordered)
+ return VectorizedTree;
+
if (!VectorValuesAndScales.empty())
VectorizedTree = GetNewVectorizedTree(
VectorizedTree,
@@ -27458,7 +27495,7 @@ bool SLPVectorizerPass::vectorizeHorReduction(
continue;
if (Value *VectorizedV = TryToReduce(Inst)) {
Res = true;
- if (auto *I = dyn_cast<Instruction>(VectorizedV)) {
+ if (auto *I = dyn_cast<Instruction>(VectorizedV); I && I != Inst) {
// Try to find another reduction.
Stack.emplace(I, Level);
continue;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
index 70b7adfd6456e..c9a2219b12a8a 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
@@ -31,22 +31,16 @@ define i32 @s352() {
; CHECK-NEXT: [[DOT_115:%.*]] = phi float [ 0.000000e+00, [[PREHEADER]] ], [ [[ADD39:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA:%.*]], ptr @global_data, i64 0, i32 0, i64 [[INDVARS_IV]]
; CHECK-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], ptr @global_data, i64 0, i32 3, i64 [[INDVARS_IV]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[ARRAYIDX6]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP1]], [[TMP3]]
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[ARRAYIDX6]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <4 x float> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[DOT_115]], [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x float> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
; CHECK-NEXT: [[ADD15:%.*]] = fadd float [[ADD]], [[TMP6]]
-; CHECK-NEXT: [[TMP7:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2
-; CHECK-NEXT: [[ARRAYIDX18:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], ptr @global_data, i64 0, i32 0, i64 [[TMP7]]
-; CHECK-NEXT: [[ARRAYIDX21:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], ptr @global_data, i64 0, i32 3, i64 [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = load <2 x float>, ptr [[ARRAYIDX18]], align 4
-; CHECK-NEXT: [[TMP11:%.*]] = load <2 x float>, ptr [[ARRAYIDX21]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = fmul <2 x float> [[TMP9]], [[TMP11]]
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x float> [[TMP12]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
; CHECK-NEXT: [[ADD23:%.*]] = fadd float [[ADD15]], [[TMP13]]
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x float> [[TMP12]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
; CHECK-NEXT: [[ADD31:%.*]] = fadd float [[ADD23]], [[TMP14]]
; CHECK-NEXT: [[TMP15:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4
; CHECK-NEXT: [[ARRAYIDX34:%.*]] = getelementptr inbounds [[STRUCT_GLOBALDATA]], ptr @global_data, i64 0, i32 0, i64 [[TMP15]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
index a333d162297bc..1b9dbaca0a34d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/dot-product.ll
@@ -9,23 +9,62 @@
;
define double @dot4f64(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %ptry) {
-; CHECK-LABEL: @dot4f64(
-; CHECK-NEXT: [[PTRX2:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 2
-; CHECK-NEXT: [[PTRY2:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[PTRX2]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = load <2 x double>, ptr [[PTRY2]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x double> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
-; CHECK-NEXT: [[DOT01:%.*]] = fadd double [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP6]], i32 0
-; CHECK-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
-; CHECK-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP10]]
-; CHECK-NEXT: ret double [[DOT0123]]
+; SSE2-LABEL: @dot4f64(
+; SSE2-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[PTRX:%.*]], align 4
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[PTRY:%.*]], align 4
+; SSE2-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP1]], [[TMP2]]
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; SSE2-NEXT: [[DOT01:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP3]], i32 2
+; SSE2-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP6]]
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; SSE2-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP7]]
+; SSE2-NEXT: ret double [[DOT0123]]
+;
+; SSE4-LABEL: @dot4f64(
+; SSE4-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[PTRX:%.*]], align 4
+; SSE4-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[PTRY:%.*]], align 4
+; SSE4-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP1]], [[TMP2]]
+; SSE4-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 0
+; SSE4-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; SSE4-NEXT: [[DOT01:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; SSE4-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP3]], i32 2
+; SSE4-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP6]]
+; SSE4-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; SSE4-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP7]]
+; SSE4-NEXT: ret double [[DOT0123]]
+;
+; AVX-LABEL: @dot4f64(
+; AVX-NEXT: [[PTRX2:%.*]] = getelementptr inbounds double, ptr [[PTRX:%.*]], i64 2
+; AVX-NEXT: [[PTRY2:%.*]] = getelementptr inbounds double, ptr [[PTRY:%.*]], i64 2
+; AVX-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[PTRX]], align 4
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[PTRY]], align 4
+; AVX-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT: [[TMP4:%.*]] = load <2 x double>, ptr [[PTRX2]], align 4
+; AVX-NEXT: [[TMP5:%.*]] = load <2 x double>, ptr [[PTRY2]], align 4
+; AVX-NEXT: [[TMP6:%.*]] = fmul <2 x double> [[TMP4]], [[TMP5]]
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
+; AVX-NEXT: [[DOT01:%.*]] = fadd double [[TMP7]], [[TMP8]]
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP6]], i32 0
+; AVX-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP9]]
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
+; AVX-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP10]]
+; AVX-NEXT: ret double [[DOT0123]]
+;
+; AVX2-LABEL: @dot4f64(
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr [[PTRX:%.*]], align 4
+; AVX2-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr [[PTRY:%.*]], align 4
+; AVX2-NEXT: [[TMP3:%.*]] = fmul <4 x double> [[TMP1]], [[TMP2]]
+; AVX2-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 0
+; AVX2-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; AVX2-NEXT: [[DOT01:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; AVX2-NEXT: [[TMP6:%.*]] = extractelement <4 x double> [[TMP3]], i32 2
+; AVX2-NEXT: [[DOT012:%.*]] = fadd double [[DOT01]], [[TMP6]]
+; AVX2-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; AVX2-NEXT: [[DOT0123:%.*]] = fadd double [[DOT012]], [[TMP7]]
+; AVX2-NEXT: ret double [[DOT0123]]
;
%ptrx1 = getelementptr inbounds double, ptr %ptrx, i64 1
%ptry1 = getelementptr inbounds double, ptr %ptry, i64 1
@@ -53,20 +92,15 @@ define double @dot4f64(ptr dereferenceable(32) %ptrx, ptr dereferenceable(32) %p
define float @dot4f32(ptr dereferenceable(16) %ptrx, ptr dereferenceable(16) %ptry) {
; CHECK-LABEL: @dot4f32(
-; CHECK-NEXT: [[PTRX2:%.*]] = getelementptr inbounds float, ptr [[PTRX:%.*]], i64 2
-; CHECK-NEXT: [[PTRY2:%.*]] = getelementptr inbounds float, ptr [[PTRY:%.*]], i64 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[PTRX]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[PTRY]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP1]], [[TMP2]]
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr [[PTRX2]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = load <2 x float>, ptr [[PTRY2]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = fmul <2 x float> [[TMP4]], [[TMP5]]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[PTRX:%.*]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[PTRY:%.*]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
; CHECK-NEXT: [[DOT01:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
; CHECK-NEXT: [[DOT012:%.*]] = fadd float [[DOT01]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
; CHECK-NEXT: [[DOT0123:%.*]] = fadd float [[DOT012]], [[TMP10]]
; CHECK-NEXT: ret float [[DOT0123]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
index a42567c5e2e46..822f1051d45d6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmaxnum.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-unknown -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,COREI7
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,BDVER1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -passes=slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256
@@ -99,6 +99,46 @@ define void @fmaxnum_8f64() #0 {
; SSE-NEXT: store <2 x double> [[TMP12]], ptr getelementptr inbounds ([8 x double], ptr @dst64, i32 0, i64 6), align 4
; SSE-NEXT: ret void
;
+; COREI7-LABEL: @fmaxnum_8f64(
+; COREI7-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr @srcA64, align 4
+; COREI7-NEXT: [[TMP2:%.*]] = load <4 x double>, ptr @srcB64, align 4
+; COREI7-NEXT: [[TMP3:%.*]] = call <4 x double> @llvm.maxnum.v4f64(<4 x double> [[TMP1]], <4 x double> [[TMP2]])
+; COREI7-NEXT: store <4 x double> [[TMP3]], ptr @dst64, align 4
+; COREI7-NEXT: [[TMP4:%.*]] = load <4 x double>, ptr getelementptr inbounds ([8 x double], ptr @srcA64, i32 0, i64 4), align 4
+; COREI7-NEXT: [[TMP5:%.*]] = load <4 x double>, ptr getelementptr inbounds ([8 x double], ptr @srcB64, i32 0, i64 4), align 4
+; COREI7-NEXT: [[TMP6:%.*]] = call <4 x double> @llvm.maxnum.v4f64(<...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/182644
More information about the llvm-commits
mailing list