[llvm] [SLP]Use poor-throughput instructions as vectorization seeds (PR #206518)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Jun 30 15:00:51 PDT 2026
https://github.com/alexey-bataev updated https://github.com/llvm/llvm-project/pull/206518
>From c61f7c3ce3ecc0664831916fa468df46375949de Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Mon, 29 Jun 2026 09:21:46 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../llvm/Transforms/Vectorize/SLPVectorizer.h | 3 +-
.../Transforms/Vectorize/SLPVectorizer.cpp | 213 +++++++++++++++++-
.../PhaseOrdering/X86/vector-reductions.ll | 23 +-
.../Transforms/SLPVectorizer/X86/c-ray.ll | 51 +++--
.../X86/delayed-gather-emission.ll | 9 +-
...ulable-node-with-non-schedulable-parent.ll | 15 +-
.../X86/poor-throughput-seeds.ll | 108 +++++++--
.../SLPVectorizer/X86/reduction2.ll | 30 ++-
8 files changed, 375 insertions(+), 77 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index af010994107a5..b0b709d5dc817 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -96,7 +96,8 @@ struct SLPVectorizerPass : public OptionalPassInfoMixin<SLPVectorizerPass> {
/// \param MaxVFOnly Vectorize only using maximal allowed register size.
/// \returns true if a value was vectorized.
bool tryToVectorizeList(ArrayRef<Value *> VL, slpvectorizer::BoUpSLP &R,
- bool MaxVFOnly = false);
+ bool MaxVFOnly = false,
+ bool LimitToRegisterVF = false);
/// Try to vectorize a chain that may start at the operands of \p I.
bool tryToVectorize(Instruction *I, slpvectorizer::BoUpSLP &R,
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index d1fb8e2beea50..acdea1c337669 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -248,6 +248,11 @@ static cl::opt<bool> NonVectReductions(
cl::desc(
"Use non-vectorizable instructions as potential reduction roots."));
+static cl::opt<bool> VectorizePoorThroughput(
+ "slp-vectorize-poor-throughput", cl::init(true), cl::Hidden,
+ cl::desc("Use poor-throughput instructions (e.g. fdiv, frem, fsqrt) as "
+ "standalone vectorization seeds."));
+
/// True when \p slp-vectorize-non-power-of-2 is enabled and \p NumElts is a
/// supported non-power-of-2 width: \p NumElts + 1 must be a power of two
/// (e.g. 3 or 7 lanes, i.e. almost a full power-of-2 register).
@@ -10676,8 +10681,8 @@ buildIntrinsicArgTypes(const CallInst *CI, const Intrinsic::ID ID,
/// function (if possible) calls. Returns invalid cost for the corresponding
/// calls, if they cannot be vectorized/will be scalarized.
static std::pair<InstructionCost, InstructionCost>
-getVectorCallCosts(CallInst *CI, Type *VecTy, TargetTransformInfo *TTI,
- TargetLibraryInfo *TLI, ArrayRef<Type *> ArgTys) {
+getVectorCallCosts(CallInst *CI, Type *VecTy, const TargetTransformInfo *TTI,
+ const TargetLibraryInfo *TLI, ArrayRef<Type *> ArgTys) {
auto Shape = VFShape::get(CI->getFunctionType(),
ElementCount::getFixed(getNumElements(VecTy)),
false /*HasGlobalPred*/);
@@ -10720,6 +10725,101 @@ getVectorCallCosts(CallInst *CI, Type *VecTy, TargetTransformInfo *TTI,
return {IntrinsicCost, LibCost};
}
+/// \returns the reciprocal-throughput cost of \p I widened to \p VF lanes (an
+/// arithmetic op or a vectorizable call).
+static InstructionCost getVectorOpCost(Instruction *I, unsigned VF,
+ const TargetTransformInfo &TTI,
+ const TargetLibraryInfo &TLI) {
+ assert((isa<BinaryOperator, CallInst>(I)) &&
+ "getVectorOpCost expects an arithmetic op or a vectorizable call.");
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ Type *VecTy = getWidenedType(I->getType(), VF);
+ if (auto *CI = dyn_cast<CallInst>(I)) {
+ Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &TLI);
+ SmallVector<Type *> ArgTys = buildIntrinsicArgTypes(CI, ID, VF, 0, &TTI);
+ auto [IntrCost, LibCost] =
+ getVectorCallCosts(CI, VecTy, &TTI, &TLI, ArgTys);
+ return LibCost < IntrCost ? LibCost : IntrCost;
+ }
+ return TTI.getArithmeticInstrCost(I->getOpcode(), VecTy, CostKind);
+}
+
+namespace {
+/// Caches the instruction kinds that isPoorThroughputOp already proved cheap
+/// (not poor-throughput) - the opcode for binary operators, or the intrinsic id
+/// / callee for calls - so the expensive cost-model query is skipped for the
+/// common case. Poor-throughput kinds are rare, so they are not cached and are
+/// re-checked per instruction, which also keeps that check element-type exact.
+/// ponytail: a cheap verdict is shared across element widths of one opcode /
+/// intrinsic id / callee. This only changes which standalone seeds are tried;
+/// the cost model still gates every transform.
+struct PoorThroughputOpCache {
+ // Binary-operator opcodes are a small dense range, so one bit per opcode
+ // marks it cheap, giving O(1) lookups with no hashing.
+ SmallBitVector CheapOpcodes = SmallBitVector(Instruction::OtherOpsEnd);
+ // Calls are sparse, so small hashed sets of the cheap intrinsic ids /
+ // callees.
+ SmallDenseSet<Intrinsic::ID> CheapIntrinsics;
+ SmallDenseSet<const Function *> CheapCallees;
+};
+} // namespace
+
+/// Returns true if \p I is an expensive, poor-throughput scalar operation
+/// (typically fdiv, frem or fsqrt) whose vector form is cheaper, so combining
+/// several of them into one vector op improves the block throughput they
+/// dominate. \p Cache remembers the cheap (not poor-throughput) opcodes /
+/// intrinsic ids / callees so the cost-model query is skipped on repeats.
+static bool isPoorThroughputOp(Instruction *I, const TargetTransformInfo &TTI,
+ const TargetLibraryInfo &TLI,
+ PoorThroughputOpCache &Cache) {
+ if (!isa<BinaryOperator, CallInst>(I))
+ return false;
+ Type *Ty = I->getType();
+ if ((Ty->isVectorTy() && !SLPReVec) || Ty->isAggregateType() ||
+ !isValidElementType(Ty))
+ return false;
+ // The checks above are cheap and type-specific; the cost-model query below is
+ // the expensive part, skipped for kinds already known to be cheap.
+ auto Analyze = [&]() {
+ constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
+ InstructionCost ScalarCost = TTI.getInstructionCost(I, CostKind);
+ if (ScalarCost < TTI::TCC_Expensive)
+ return false;
+ // Only worth seeding when widening to the smallest vector saves throughput.
+ constexpr unsigned MinVF = 2;
+ InstructionCost VecCost = getVectorOpCost(I, MinVF, TTI, TLI);
+ return VecCost < ScalarCost * MinVF;
+ };
+ if (auto *CI = dyn_cast<CallInst>(I)) {
+ // Calls map to a vectorizable intrinsic (keyed by id) or another direct
+ // call (keyed by callee); indirect calls have no cheap stable key.
+ if (Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, &TLI)) {
+ if (Cache.CheapIntrinsics.contains(ID))
+ return false;
+ if (Analyze())
+ return true;
+ Cache.CheapIntrinsics.insert(ID);
+ return false;
+ }
+ const Function *Callee = CI->getCalledFunction();
+ if (!Callee)
+ return Analyze();
+ if (Cache.CheapCallees.contains(Callee))
+ return false;
+ if (Analyze())
+ return true;
+ Cache.CheapCallees.insert(Callee);
+ return false;
+ }
+ unsigned Opcode = I->getOpcode();
+ if (Cache.CheapOpcodes.test(Opcode))
+ return false;
+ if (Analyze())
+ return true;
+ Cache.CheapOpcodes.set(Opcode);
+ return false;
+}
+
/// Find the innermost loop starting from \p L, for which at least a single
/// value in \p VL is not invariant.
static const Loop *findInnermostNonInvariantLoop(const Loop *L,
@@ -19700,6 +19800,30 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
ArrayRef<Value *> VectorizedVals,
InstructionCost ReductionCost,
Instruction *RdxRoot) {
+ // Bypass the instruction-count veto for poor-throughput ops (fdiv/frem/fsqrt)
+ // when the tree saves at least the throughput of widening one of them (the
+ // RootVF scalar instances removed, but 1 vector added) per iteration.
+ // TreeCost is already trip-count-scaled, so the condition is TreeCost <=
+ // -(RootVF - 1) * TCC_Expensive * TripCount.
+ auto BypassesInstCountCheck = [&]() {
+ if (!VectorizePoorThroughput)
+ return false;
+ uint64_t TripCount = 0;
+ PoorThroughputOpCache PoorThroughputCache;
+ for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
+ const TreeEntry &TE = *Ptr;
+ if (DeletedNodes.contains(&TE) || TE.isGather() ||
+ TransformedToGatherNodes.contains(&TE) ||
+ TE.State == TreeEntry::CombinedVectorize || !TE.hasState())
+ continue;
+ if (isPoorThroughputOp(TE.getMainOp(), *TTI, *TLI, PoorThroughputCache))
+ TripCount = std::max(TripCount, getScaleToLoopIterations(TE));
+ }
+ if (TripCount == 0)
+ return false;
+ int64_t RootVF = VectorizableTree.front()->getVectorFactor();
+ return TreeCost + (RootVF - 1) * TTI::TCC_Expensive * TripCount <= 0;
+ };
// Reject vectorization if the vector code would produce more instructions
// than the scalar code. The cost model may underestimate overhead from
// shuffles, inserts, and extracts.
@@ -19715,7 +19839,7 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
unsigned NumVector = getNumVectorInsts();
LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector
<< " scalar=" << NumScalar << "\n");
- if (NumVector > NumScalar) {
+ if (NumVector > NumScalar && !BypassesInstCountCheck()) {
LLVM_DEBUG(dbgs() << "SLP: Rejecting tree: vector inst count "
<< NumVector << " > scalar inst count " << NumScalar
<< ".\n");
@@ -28536,7 +28660,8 @@ void SLPVectorizerPass::collectSeedInstructions(BasicBlock *BB) {
}
bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
- bool MaxVFOnly) {
+ bool MaxVFOnly,
+ bool LimitToRegisterVF) {
if (VL.size() < 2)
return false;
@@ -28576,6 +28701,13 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
unsigned MaxVF = std::max<unsigned>(
getFloorFullVectorNumberOfElements(*TTI, ScalarTy, VL.size()), MinVF);
MaxVF = std::min(R.getMaximumVF(Sz, S.getOpcode()), MaxVF);
+ // For independent scalar seeds (e.g. the poor-throughput fdiv/frem/fsqrt
+ // fallback), widening past a single vector register requires gathering the
+ // scattered operands across multiple registers and is never profitable, yet
+ // each such factor still builds and costs a full tree. Cap the factor at one
+ // register to skip those wide, never-taken attempts and bound compile time.
+ if (LimitToRegisterVF && Sz != 0)
+ MaxVF = std::min(MaxVF, std::max(MinVF, R.getMaxVecRegSize() / Sz));
if (MaxVF < 2) {
R.getORE()->emit([&]() {
return OptimizationRemarkMissed(SV_NAME, "SmallVF", I0)
@@ -32491,6 +32623,13 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
// Stores are processed after all other instructions/roots.
SmallSetVector<StoreInst *, 8> PostProcessStores;
SmallSetVector<Instruction *, 8> FMACandidates;
+ // Poor-throughput instructions (fdiv/frem/fsqrt) that were not captured by
+ // any other root. They are tried as standalone seeds after everything else.
+ SmallSetVector<Instruction *, 8> PoorThroughputSeeds;
+ // Memoizes the poor-throughput classification so the cost-model query runs at
+ // most once per opcode / intrinsic id / callee instead of once per
+ // instruction in this block.
+ PoorThroughputOpCache PoorThroughputCache;
auto VectorizeInsertsAndCmps = [&](bool AtTerminator) {
bool Changed = vectorizeInserts(PostProcessInserts, BB, R, FMACandidates);
if (AtTerminator) {
@@ -32638,6 +32777,9 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
PostProcessStores.insert(SI);
else if (isNonVectorizableInst(&*It, TLI))
PostProcessInsts.insert(&*It);
+ else if (VectorizePoorThroughput &&
+ isPoorThroughputOp(&*It, *TTI, *TLI, PoorThroughputCache))
+ PoorThroughputSeeds.insert(&*It);
}
// Late post-process: run operand-chain vectorization for stores.
@@ -32690,6 +32832,69 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
assert(Empty.empty() &&
"No new FMA candidates expected during AllowFMACandidates retry.");
+ // Final fallback: combine leftover poor-throughput instructions
+ // (fdiv/frem/fsqrt) into vector operations when no other seed captured them.
+ // The cost model still gates the actual transformation.
+ if (PoorThroughputSeeds.size() >= 2) {
+ SmallVector<Value *> Seeds;
+ for (Instruction *I : PoorThroughputSeeds)
+ if (!R.isDeleted(I) && isValidElementType(getValueType(I)))
+ Seeds.push_back(I);
+ // Group seeds that can actually be vectorized together - same type and
+ // opcode and, for calls, the same intrinsic id and callee - so each attempt
+ // sees a homogeneous bundle and incompatible seeds are never mixed.
+ auto SeedIntrinsicID = [this](const Value *V) {
+ const auto *CI = dyn_cast<CallInst>(V);
+ return CI ? getVectorIntrinsicIDForCall(CI, TLI)
+ : Intrinsic::not_intrinsic;
+ };
+ auto SeedCallee = [](const Value *V) -> StringRef {
+ if (const auto *CI = dyn_cast<CallInst>(V))
+ if (const Function *F = CI->getCalledFunction())
+ return F->getName();
+ return {};
+ };
+ auto SeedSorter = [&](Value *V1, Value *V2) {
+ if (V1 == V2)
+ return false;
+ auto *I1 = cast<Instruction>(V1);
+ auto *I2 = cast<Instruction>(V2);
+ Type *T1 = I1->getType();
+ Type *T2 = I2->getType();
+ if (T1->getTypeID() != T2->getTypeID())
+ return T1->getTypeID() < T2->getTypeID();
+ if (T1->getScalarSizeInBits() != T2->getScalarSizeInBits())
+ return T1->getScalarSizeInBits() < T2->getScalarSizeInBits();
+ if (I1->getOpcode() != I2->getOpcode())
+ return I1->getOpcode() < I2->getOpcode();
+ Intrinsic::ID ID1 = SeedIntrinsicID(V1);
+ Intrinsic::ID ID2 = SeedIntrinsicID(V2);
+ if (ID1 != ID2)
+ return ID1 < ID2;
+ if (int C = SeedCallee(V1).compare(SeedCallee(V2)))
+ return C < 0;
+ return I1->comesBefore(I2);
+ };
+ auto AreCompatibleSeeds = [&](ArrayRef<Value *> VL, Value *V) {
+ if (VL.empty() || VL.back() == V)
+ return true;
+ auto *I1 = cast<Instruction>(VL.back());
+ auto *I2 = cast<Instruction>(V);
+ return I1->getType() == I2->getType() &&
+ I1->getOpcode() == I2->getOpcode() &&
+ SeedIntrinsicID(VL.back()) == SeedIntrinsicID(V) &&
+ SeedCallee(VL.back()) == SeedCallee(V);
+ };
+ if (Seeds.size() >= 2)
+ Changed |= tryToVectorizeSequence<Value>(
+ Seeds, SeedSorter, AreCompatibleSeeds,
+ [this, &R](ArrayRef<Value *> Candidates, bool MaxVFOnly) {
+ return tryToVectorizeList(Candidates, R, MaxVFOnly,
+ /*LimitToRegisterVF=*/true);
+ },
+ /*MaxVFOnly=*/false, R);
+ }
+
return Changed;
}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
index c3464a21466de..78f68c420f6ab 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions.ll
@@ -269,24 +269,21 @@ define i1 @cmp_lt_gt(double %a, double %b, double %c) {
; CHECK-NEXT: entry:
; CHECK-NEXT: [[FNEG:%.*]] = fneg double [[B:%.*]]
; CHECK-NEXT: [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
-; CHECK-NEXT: [[C:%.*]] = fsub double [[C1:%.*]], [[B]]
-; CHECK-NEXT: [[ADD:%.*]] = fsub double [[FNEG]], [[C1]]
+; CHECK-NEXT: [[C:%.*]] = fsub double [[FNEG]], [[C1:%.*]]
+; CHECK-NEXT: [[ADD:%.*]] = fsub double [[C1]], [[B]]
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[ADD]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[C]], i64 1
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> poison, double [[MUL]], i64 0
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP5]], <2 x double> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = fdiv <2 x double> [[TMP3]], [[TMP6]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x double> [[TMP7]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp ule <4 x double> [[TMP9]], <double 1.000000e+00, double 1.000000e+00, double f0x3EB0C6F7A0B5ED8D, double f0x3EB0C6F7A0B5ED8D>
-; CHECK-NEXT: [[TMP11:%.*]] = fcmp uge <4 x double> [[TMP9]], <double 1.000000e+00, double 1.000000e+00, double f0x3EB0C6F7A0B5ED8D, double f0x3EB0C6F7A0B5ED8D>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP10]], <4 x i1> [[TMP11]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i1> [[TMP11]], <4 x i1> poison, <4 x i32> <i32 poison, i32 poison, i32 3, i32 poison>
-; CHECK-NEXT: [[FOLDEXTEXTBINOP:%.*]] = or <4 x i1> [[SHIFT]], [[TMP8]]
-; CHECK-NEXT: [[SHIFT3:%.*]] = shufflevector <4 x i1> [[TMP10]], <4 x i1> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[FOLDEXTEXTBINOP4:%.*]] = or <4 x i1> [[SHIFT3]], [[TMP8]]
-; CHECK-NEXT: [[SHIFT6:%.*]] = shufflevector <4 x i1> [[FOLDEXTEXTBINOP]], <4 x i1> poison, <4 x i32> <i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[FOLDEXTEXTBINOP7:%.*]] = and <4 x i1> [[SHIFT6]], [[FOLDEXTEXTBINOP4]]
-; CHECK-NEXT: [[RETVAL_0:%.*]] = extractelement <4 x i1> [[FOLDEXTEXTBINOP7]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = fcmp uge <2 x double> [[TMP7]], splat (double f0x3EB0C6F7A0B5ED8D)
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <2 x i1> [[TMP8]], <2 x i1> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[FOLDEXTEXTBINOP:%.*]] = or <2 x i1> [[TMP8]], [[SHIFT]]
+; CHECK-NEXT: [[TMP9:%.*]] = fcmp ule <2 x double> [[TMP7]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[SHIFT3:%.*]] = shufflevector <2 x i1> [[TMP9]], <2 x i1> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = or <2 x i1> [[TMP9]], [[SHIFT3]]
+; CHECK-NEXT: [[FOLDEXTEXTBINOP4:%.*]] = and <2 x i1> [[FOLDEXTEXTBINOP]], [[TMP10]]
+; CHECK-NEXT: [[RETVAL_0:%.*]] = extractelement <2 x i1> [[FOLDEXTEXTBINOP4]], i64 0
; CHECK-NEXT: ret i1 [[RETVAL_0]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
index bbd4cfd057a0f..a052f85ca3963 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/c-ray.ll
@@ -64,18 +64,25 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; SSE2: if.end:
; SSE2-NEXT: [[CALL:%.*]] = tail call double @sqrt(double noundef [[TMP25]])
; SSE2-NEXT: [[FNEG87:%.*]] = fneg double [[TMP12]]
-; SSE2-NEXT: [[ADD:%.*]] = fsub double [[CALL]], [[TMP12]]
; SSE2-NEXT: [[MUL88:%.*]] = fmul double [[TMP4]], 2.000000e+00
-; SSE2-NEXT: [[TMP34:%.*]] = fdiv double [[ADD]], [[MUL88]]
-; SSE2-NEXT: [[SUB90:%.*]] = fsub double [[FNEG87]], [[CALL]]
-; SSE2-NEXT: [[TMP35:%.*]] = fdiv double [[SUB90]], [[MUL88]]
+; SSE2-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[FNEG87]], i32 0
+; SSE2-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[CALL]], i32 1
+; SSE2-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP27]], <2 x double> poison, <2 x i32> <i32 1, i32 poison>
+; SSE2-NEXT: [[TMP29:%.*]] = insertelement <2 x double> [[TMP28]], double [[TMP12]], i32 1
+; SSE2-NEXT: [[TMP30:%.*]] = fsub <2 x double> [[TMP27]], [[TMP29]]
+; SSE2-NEXT: [[TMP31:%.*]] = insertelement <2 x double> poison, double [[MUL88]], i32 0
+; SSE2-NEXT: [[TMP32:%.*]] = shufflevector <2 x double> [[TMP31]], <2 x double> poison, <2 x i32> zeroinitializer
+; SSE2-NEXT: [[TMP33:%.*]] = fdiv <2 x double> [[TMP30]], [[TMP32]]
+; SSE2-NEXT: [[TMP34:%.*]] = extractelement <2 x double> [[TMP33]], i32 1
; SSE2-NEXT: [[CMP93:%.*]] = fcmp olt double [[TMP34]], f0x3EB0C6F7A0B5ED8D
+; SSE2-NEXT: [[TMP35:%.*]] = extractelement <2 x double> [[TMP33]], i32 0
; SSE2-NEXT: [[CMP94:%.*]] = fcmp olt double [[TMP35]], f0x3EB0C6F7A0B5ED8D
; SSE2-NEXT: [[OR_COND:%.*]] = select i1 [[CMP93]], i1 [[CMP94]], i1 false
; SSE2-NEXT: br i1 [[OR_COND]], label [[CLEANUP]], label [[LOR_LHS_FALSE:%.*]]
; SSE2: lor.lhs.false:
-; SSE2-NEXT: [[TMP38:%.*]] = fcmp ule double [[TMP34]], 1.000000e+00
-; SSE2-NEXT: [[TMP37:%.*]] = fcmp ule double [[TMP35]], 1.000000e+00
+; SSE2-NEXT: [[TMP36:%.*]] = fcmp ule <2 x double> [[TMP33]], splat (double 1.000000e+00)
+; SSE2-NEXT: [[TMP37:%.*]] = extractelement <2 x i1> [[TMP36]], i32 0
+; SSE2-NEXT: [[TMP38:%.*]] = extractelement <2 x i1> [[TMP36]], i32 1
; SSE2-NEXT: [[OR_COND106:%.*]] = select i1 [[TMP38]], i1 true, i1 [[TMP37]]
; SSE2-NEXT: [[SPEC_SELECT:%.*]] = zext i1 [[OR_COND106]] to i32
; SSE2-NEXT: br label [[CLEANUP]]
@@ -138,18 +145,24 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; AVX: if.end:
; AVX-NEXT: [[CALL:%.*]] = tail call double @sqrt(double noundef [[TMP25]])
; AVX-NEXT: [[FNEG87:%.*]] = fneg double [[TMP12]]
-; AVX-NEXT: [[ADD:%.*]] = fsub double [[CALL]], [[TMP12]]
; AVX-NEXT: [[MUL88:%.*]] = fmul double [[TMP4]], 2.000000e+00
-; AVX-NEXT: [[TMP31:%.*]] = fdiv double [[ADD]], [[MUL88]]
+; AVX-NEXT: [[ADD:%.*]] = fsub double [[CALL]], [[TMP12]]
; AVX-NEXT: [[SUB90:%.*]] = fsub double [[FNEG87]], [[CALL]]
-; AVX-NEXT: [[TMP32:%.*]] = fdiv double [[SUB90]], [[MUL88]]
+; AVX-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[SUB90]], i32 0
+; AVX-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[ADD]], i32 1
+; AVX-NEXT: [[TMP28:%.*]] = insertelement <2 x double> poison, double [[MUL88]], i32 0
+; AVX-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> poison, <2 x i32> zeroinitializer
+; AVX-NEXT: [[TMP30:%.*]] = fdiv <2 x double> [[TMP27]], [[TMP29]]
+; AVX-NEXT: [[TMP31:%.*]] = extractelement <2 x double> [[TMP30]], i32 1
; AVX-NEXT: [[CMP93:%.*]] = fcmp olt double [[TMP31]], f0x3EB0C6F7A0B5ED8D
+; AVX-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[TMP30]], i32 0
; AVX-NEXT: [[CMP94:%.*]] = fcmp olt double [[TMP32]], f0x3EB0C6F7A0B5ED8D
; AVX-NEXT: [[OR_COND:%.*]] = select i1 [[CMP93]], i1 [[CMP94]], i1 false
; AVX-NEXT: br i1 [[OR_COND]], label [[CLEANUP]], label [[LOR_LHS_FALSE:%.*]]
; AVX: lor.lhs.false:
-; AVX-NEXT: [[TMP35:%.*]] = fcmp ule double [[TMP31]], 1.000000e+00
-; AVX-NEXT: [[TMP34:%.*]] = fcmp ule double [[TMP32]], 1.000000e+00
+; AVX-NEXT: [[TMP33:%.*]] = fcmp ule <2 x double> [[TMP30]], splat (double 1.000000e+00)
+; AVX-NEXT: [[TMP34:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0
+; AVX-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1
; AVX-NEXT: [[OR_COND106:%.*]] = select i1 [[TMP35]], i1 true, i1 [[TMP34]]
; AVX-NEXT: [[SPEC_SELECT:%.*]] = zext i1 [[OR_COND106]] to i32
; AVX-NEXT: br label [[CLEANUP]]
@@ -212,18 +225,24 @@ define i32 @ray_sphere(ptr nocapture noundef readonly %sph, ptr nocapture nounde
; AVX2: if.end:
; AVX2-NEXT: [[CALL:%.*]] = tail call double @sqrt(double noundef [[TMP25]])
; AVX2-NEXT: [[FNEG87:%.*]] = fneg double [[TMP12]]
-; AVX2-NEXT: [[ADD:%.*]] = fsub double [[CALL]], [[TMP12]]
; AVX2-NEXT: [[MUL88:%.*]] = fmul double [[TMP4]], 2.000000e+00
-; AVX2-NEXT: [[TMP31:%.*]] = fdiv double [[ADD]], [[MUL88]]
+; AVX2-NEXT: [[ADD:%.*]] = fsub double [[CALL]], [[TMP12]]
; AVX2-NEXT: [[SUB90:%.*]] = fsub double [[FNEG87]], [[CALL]]
-; AVX2-NEXT: [[TMP32:%.*]] = fdiv double [[SUB90]], [[MUL88]]
+; AVX2-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[SUB90]], i32 0
+; AVX2-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[ADD]], i32 1
+; AVX2-NEXT: [[TMP28:%.*]] = insertelement <2 x double> poison, double [[MUL88]], i32 0
+; AVX2-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP28]], <2 x double> poison, <2 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP30:%.*]] = fdiv <2 x double> [[TMP27]], [[TMP29]]
+; AVX2-NEXT: [[TMP31:%.*]] = extractelement <2 x double> [[TMP30]], i32 1
; AVX2-NEXT: [[CMP93:%.*]] = fcmp olt double [[TMP31]], f0x3EB0C6F7A0B5ED8D
+; AVX2-NEXT: [[TMP32:%.*]] = extractelement <2 x double> [[TMP30]], i32 0
; AVX2-NEXT: [[CMP94:%.*]] = fcmp olt double [[TMP32]], f0x3EB0C6F7A0B5ED8D
; AVX2-NEXT: [[OR_COND:%.*]] = select i1 [[CMP93]], i1 [[CMP94]], i1 false
; AVX2-NEXT: br i1 [[OR_COND]], label [[CLEANUP]], label [[LOR_LHS_FALSE:%.*]]
; AVX2: lor.lhs.false:
-; AVX2-NEXT: [[TMP35:%.*]] = fcmp ule double [[TMP31]], 1.000000e+00
-; AVX2-NEXT: [[TMP34:%.*]] = fcmp ule double [[TMP32]], 1.000000e+00
+; AVX2-NEXT: [[TMP33:%.*]] = fcmp ule <2 x double> [[TMP30]], splat (double 1.000000e+00)
+; AVX2-NEXT: [[TMP34:%.*]] = extractelement <2 x i1> [[TMP33]], i32 0
+; AVX2-NEXT: [[TMP35:%.*]] = extractelement <2 x i1> [[TMP33]], i32 1
; AVX2-NEXT: [[OR_COND106:%.*]] = select i1 [[TMP35]], i1 true, i1 [[TMP34]]
; AVX2-NEXT: [[SPEC_SELECT:%.*]] = zext i1 [[OR_COND106]] to i32
; AVX2-NEXT: br label [[CLEANUP]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/delayed-gather-emission.ll b/llvm/test/Transforms/SLPVectorizer/X86/delayed-gather-emission.ll
index e276a5831b8f7..421257d611062 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/delayed-gather-emission.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/delayed-gather-emission.ll
@@ -20,9 +20,14 @@ define void @test() {
; CHECK: bb2:
; CHECK-NEXT: [[I:%.*]] = phi float [ [[DOTPRE]], [[BB1]] ], [ [[I2]], [[BB2]] ]
; CHECK-NEXT: [[GULF_0]] = phi float [ [[FOXTROT_0]], [[BB1]] ], [ [[TMP6:%.*]], [[BB2]] ]
-; CHECK-NEXT: [[TMP6]] = fdiv float [[I]], [[GULF_0]]
; CHECK-NEXT: [[I1:%.*]] = load float, ptr poison, align 4
-; CHECK-NEXT: [[TMP7:%.*]] = fdiv float [[GULF_0]], [[I1]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> poison, float [[I]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float [[GULF_0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[GULF_0]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[I1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = fdiv <2 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP6]] = extractelement <2 x float> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x float> [[TMP4]], i32 1
; CHECK-NEXT: [[MUL:%.*]] = fmul float [[TMP6]], [[TMP7]]
; CHECK-NEXT: tail call void @foo(float [[MUL]])
; CHECK-NEXT: [[I2]] = load float, ptr poison, align 4
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-node-with-non-schedulable-parent.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-node-with-non-schedulable-parent.ll
index 4ba9930af2a75..dc9a7b6196b80 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-node-with-non-schedulable-parent.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-node-with-non-schedulable-parent.ll
@@ -4,25 +4,26 @@
define float @test() {
; CHECK-LABEL: define float @test() {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[FADD:%.*]] = fadd float 0.000000e+00, 0.000000e+00
; CHECK-NEXT: [[LOAD:%.*]] = load float, ptr null, align 4
-; CHECK-NEXT: [[FADD1:%.*]] = fadd float [[FADD]], [[LOAD]]
-; CHECK-NEXT: [[FADD2:%.*]] = fadd float 0.000000e+00, 0.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> <float poison, float 0.000000e+00>, float [[LOAD]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> zeroinitializer, [[TMP0]]
; CHECK-NEXT: [[FMUL:%.*]] = fmul float 0.000000e+00, 0.000000e+00
; CHECK-NEXT: [[FMUL3:%.*]] = fmul float 0.000000e+00, 0.000000e+00
; CHECK-NEXT: [[FADD4:%.*]] = fadd float [[FMUL3]], 0.000000e+00
; CHECK-NEXT: br i1 false, label %[[BB5:.*]], label %[[BB6:.*]]
; CHECK: [[BB5]]:
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
; CHECK-NEXT: br label %[[BB8:.*]]
; CHECK: [[BB6]]:
-; CHECK-NEXT: [[FDIV:%.*]] = fdiv float [[FADD1]], 0.000000e+00
-; CHECK-NEXT: [[FDIV7:%.*]] = fdiv float [[FADD2]], 0.000000e+00
+; CHECK-NEXT: [[TMP3:%.*]] = fdiv <2 x float> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x float> [[TMP3]], i32 1
; CHECK-NEXT: br label %[[BB8]]
; CHECK: [[BB8]]:
-; CHECK-NEXT: [[PHI:%.*]] = phi float [ 0.000000e+00, %[[BB5]] ], [ [[FDIV7]], %[[BB6]] ]
+; CHECK-NEXT: [[PHI:%.*]] = phi float [ 0.000000e+00, %[[BB5]] ], [ [[TMP5]], %[[BB6]] ]
; CHECK-NEXT: [[PHI9:%.*]] = phi float [ [[FMUL]], %[[BB5]] ], [ 0.000000e+00, %[[BB6]] ]
; CHECK-NEXT: [[PHI10:%.*]] = phi float [ [[FADD4]], %[[BB5]] ], [ [[LOAD]], %[[BB6]] ]
-; CHECK-NEXT: [[TMP10:%.*]] = phi float [ [[FADD1]], %[[BB5]] ], [ [[FDIV]], %[[BB6]] ]
+; CHECK-NEXT: [[TMP10:%.*]] = phi float [ [[TMP2]], %[[BB5]] ], [ [[TMP4]], %[[BB6]] ]
; CHECK-NEXT: ret float [[TMP10]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll b/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
index c7e91a4412191..6e31f137de616 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=icelake-server -passes=slp-vectorizer -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=icelake-server -passes=slp-vectorizer -S -slp-vectorize-poor-throughput=false | FileCheck %s --check-prefix=DISABLED
declare double @llvm.sqrt.f64(double)
@@ -17,6 +18,19 @@ define void @prim(double %x, double %y, double %z, double %w, ptr %p0, ptr %p1)
; CHECK-NEXT: store double [[Y3]], ptr [[P1]], align 8
; CHECK-NEXT: ret void
;
+; DISABLED-LABEL: define void @prim(
+; DISABLED-SAME: double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]], double [[W:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR1:[0-9]+]] {
+; DISABLED-NEXT: [[ENTRY:.*:]]
+; DISABLED-NEXT: [[X1:%.*]] = fsub double [[X]], [[Z]]
+; DISABLED-NEXT: [[Y1:%.*]] = fadd double [[Y]], [[W]]
+; DISABLED-NEXT: [[X2:%.*]] = fdiv double [[X1]], [[Z]]
+; DISABLED-NEXT: [[Y2:%.*]] = fdiv double [[Y1]], [[W]]
+; DISABLED-NEXT: [[X3:%.*]] = fsub double [[X2]], [[Z]]
+; DISABLED-NEXT: [[Y3:%.*]] = fadd double [[Y2]], [[W]]
+; DISABLED-NEXT: store double [[X3]], ptr [[P0]], align 8
+; DISABLED-NEXT: store double [[Y3]], ptr [[P1]], align 8
+; DISABLED-NEXT: ret void
+;
entry:
%x1 = fsub double %x, %z
%y1 = fadd double %y, %w
@@ -33,16 +47,35 @@ define void @prim_sqrt(double %x, double %y, double %z, double %w, ptr %p0, ptr
; CHECK-LABEL: define void @prim_sqrt(
; CHECK-SAME: double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]], double [[W:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[X1:%.*]] = fsub double [[X]], [[Z]]
-; CHECK-NEXT: [[Y1:%.*]] = fadd double [[Y]], [[W]]
-; CHECK-NEXT: [[TMP8:%.*]] = call double @llvm.sqrt.f64(double [[X1]])
-; CHECK-NEXT: [[TMP9:%.*]] = call double @llvm.sqrt.f64(double [[Y1]])
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[X]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[Y]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[Z]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[W]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP5]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP7:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP6]])
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
; CHECK-NEXT: [[X3:%.*]] = fsub double [[TMP8]], [[Z]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
; CHECK-NEXT: [[Y3:%.*]] = fadd double [[TMP9]], [[W]]
; CHECK-NEXT: store double [[X3]], ptr [[P0]], align 8
; CHECK-NEXT: store double [[Y3]], ptr [[P1]], align 8
; CHECK-NEXT: ret void
;
+; DISABLED-LABEL: define void @prim_sqrt(
+; DISABLED-SAME: double [[X:%.*]], double [[Y:%.*]], double [[Z:%.*]], double [[W:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]]) #[[ATTR1]] {
+; DISABLED-NEXT: [[ENTRY:.*:]]
+; DISABLED-NEXT: [[X1:%.*]] = fsub double [[X]], [[Z]]
+; DISABLED-NEXT: [[Y1:%.*]] = fadd double [[Y]], [[W]]
+; DISABLED-NEXT: [[X2:%.*]] = call double @llvm.sqrt.f64(double [[X1]])
+; DISABLED-NEXT: [[Y2:%.*]] = call double @llvm.sqrt.f64(double [[Y1]])
+; DISABLED-NEXT: [[X3:%.*]] = fsub double [[X2]], [[Z]]
+; DISABLED-NEXT: [[Y3:%.*]] = fadd double [[Y2]], [[W]]
+; DISABLED-NEXT: store double [[X3]], ptr [[P0]], align 8
+; DISABLED-NEXT: store double [[Y3]], ptr [[P1]], align 8
+; DISABLED-NEXT: ret void
+;
entry:
%x1 = fsub double %x, %z
%y1 = fadd double %y, %w
@@ -59,32 +92,59 @@ define void @prim_v4(double %x0, double %x1, double %x2, double %x3, double %d0,
; CHECK-LABEL: define void @prim_v4(
; CHECK-SAME: double [[X0:%.*]], double [[X1:%.*]], double [[X2:%.*]], double [[X3:%.*]], double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]]) #[[ATTR1]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A0:%.*]] = fsub double [[X0]], [[D0]]
-; CHECK-NEXT: [[A1:%.*]] = fadd double [[X1]], [[D1]]
-; CHECK-NEXT: [[A2:%.*]] = fsub double [[X2]], [[D2]]
-; CHECK-NEXT: [[A3:%.*]] = fadd double [[X3]], [[D3]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> poison, double [[A1]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[A3]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[A0]], i32 2
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[A2]], i32 3
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> poison, double [[D1]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[D3]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[D0]], i32 2
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[D2]], i32 3
-; CHECK-NEXT: [[TMP8:%.*]] = fdiv <4 x double> [[TMP3]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[TMP8]], [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = fsub <4 x double> [[TMP8]], [[TMP7]]
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x double> [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[X0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[X2]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[D0]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[D2]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> poison, double [[X1]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> [[TMP5]], double [[X3]], i32 1
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x double> poison, double [[D1]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[TMP7]], double [[D3]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = fadd <2 x double> [[TMP6]], [[TMP8]]
+; CHECK-NEXT: [[TMP10:%.*]] = fdiv <2 x double> [[TMP4]], [[TMP3]]
+; CHECK-NEXT: [[TMP11:%.*]] = fsub <2 x double> [[TMP10]], [[TMP3]]
+; CHECK-NEXT: [[TMP12:%.*]] = fdiv <2 x double> [[TMP9]], [[TMP8]]
+; CHECK-NEXT: [[TMP13:%.*]] = fadd <2 x double> [[TMP12]], [[TMP8]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[TMP11]], i32 0
; CHECK-NEXT: store double [[TMP14]], ptr [[P0]], align 8
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x double> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x double> [[TMP13]], i32 0
; CHECK-NEXT: store double [[TMP15]], ptr [[P1]], align 8
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <4 x double> [[TMP11]], i32 3
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x double> [[TMP11]], i32 1
; CHECK-NEXT: store double [[TMP16]], ptr [[P2]], align 8
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x double> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x double> [[TMP13]], i32 1
; CHECK-NEXT: store double [[TMP17]], ptr [[P3]], align 8
; CHECK-NEXT: ret void
;
+; DISABLED-LABEL: define void @prim_v4(
+; DISABLED-SAME: double [[X0:%.*]], double [[X1:%.*]], double [[X2:%.*]], double [[X3:%.*]], double [[D0:%.*]], double [[D1:%.*]], double [[D2:%.*]], double [[D3:%.*]], ptr [[P0:%.*]], ptr [[P1:%.*]], ptr [[P2:%.*]], ptr [[P3:%.*]]) #[[ATTR1]] {
+; DISABLED-NEXT: [[ENTRY:.*:]]
+; DISABLED-NEXT: [[A0:%.*]] = fsub double [[X0]], [[D0]]
+; DISABLED-NEXT: [[A1:%.*]] = fadd double [[X1]], [[D1]]
+; DISABLED-NEXT: [[A2:%.*]] = fsub double [[X2]], [[D2]]
+; DISABLED-NEXT: [[A3:%.*]] = fadd double [[X3]], [[D3]]
+; DISABLED-NEXT: [[TMP0:%.*]] = insertelement <4 x double> poison, double [[A1]], i32 0
+; DISABLED-NEXT: [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[A3]], i32 1
+; DISABLED-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[A0]], i32 2
+; DISABLED-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[A2]], i32 3
+; DISABLED-NEXT: [[TMP4:%.*]] = insertelement <4 x double> poison, double [[D1]], i32 0
+; DISABLED-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[D3]], i32 1
+; DISABLED-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[D0]], i32 2
+; DISABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[D2]], i32 3
+; DISABLED-NEXT: [[TMP8:%.*]] = fdiv <4 x double> [[TMP3]], [[TMP7]]
+; DISABLED-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[TMP8]], [[TMP7]]
+; DISABLED-NEXT: [[TMP10:%.*]] = fsub <4 x double> [[TMP8]], [[TMP7]]
+; DISABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; DISABLED-NEXT: [[TMP12:%.*]] = extractelement <4 x double> [[TMP11]], i32 2
+; DISABLED-NEXT: store double [[TMP12]], ptr [[P0]], align 8
+; DISABLED-NEXT: [[TMP13:%.*]] = extractelement <4 x double> [[TMP11]], i32 0
+; DISABLED-NEXT: store double [[TMP13]], ptr [[P1]], align 8
+; DISABLED-NEXT: [[TMP14:%.*]] = extractelement <4 x double> [[TMP11]], i32 3
+; DISABLED-NEXT: store double [[TMP14]], ptr [[P2]], align 8
+; DISABLED-NEXT: [[TMP15:%.*]] = extractelement <4 x double> [[TMP11]], i32 1
+; DISABLED-NEXT: store double [[TMP15]], ptr [[P3]], align 8
+; DISABLED-NEXT: ret void
+;
entry:
%a0 = fsub double %x0, %d0
%a1 = fadd double %x1, %d1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
index 1d75d84f2f22b..a9599eae5882a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -86,18 +86,24 @@ define i1 @fcmp_lt_gt(double %a, double %b, double %c) {
; CHECK-LABEL: @fcmp_lt_gt(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[FNEG:%.*]] = fneg double [[B:%.*]]
-; CHECK-NEXT: [[ADD:%.*]] = fsub double [[C:%.*]], [[B]]
; CHECK-NEXT: [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
-; CHECK-NEXT: [[TMP8:%.*]] = fdiv double [[ADD]], [[MUL]]
+; CHECK-NEXT: [[ADD:%.*]] = fsub double [[C:%.*]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub double [[FNEG]], [[C]]
-; CHECK-NEXT: [[TMP9:%.*]] = fdiv double [[SUB]], [[MUL]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[SUB]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[ADD]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[MUL]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = fdiv <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
; CHECK-NEXT: [[CMP:%.*]] = fcmp olt double [[TMP8]], f0x3EB0C6F7A0B5ED8D
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
; CHECK-NEXT: [[CMP4:%.*]] = fcmp olt double [[TMP9]], f0x3EB0C6F7A0B5ED8D
; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[CMP]], [[CMP4]]
; CHECK-NEXT: br i1 [[OR_COND]], label [[CLEANUP:%.*]], label [[LOR_LHS_FALSE:%.*]]
; CHECK: lor.lhs.false:
-; CHECK-NEXT: [[TMP12:%.*]] = fcmp ule double [[TMP8]], 1.000000e+00
-; CHECK-NEXT: [[TMP11:%.*]] = fcmp ule double [[TMP9]], 1.000000e+00
+; CHECK-NEXT: [[TMP7:%.*]] = fcmp ule <2 x double> [[TMP4]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP7]], i32 1
; CHECK-NEXT: [[NOT_OR_COND9:%.*]] = or i1 [[TMP11]], [[TMP12]]
; CHECK-NEXT: ret i1 [[NOT_OR_COND9]]
; CHECK: cleanup:
@@ -128,13 +134,17 @@ cleanup:
define i1 @fcmp_lt(double %a, double %b, double %c) {
; CHECK-LABEL: @fcmp_lt(
; CHECK-NEXT: [[FNEG:%.*]] = fneg double [[B:%.*]]
-; CHECK-NEXT: [[ADD:%.*]] = fsub double [[C:%.*]], [[B]]
; CHECK-NEXT: [[MUL:%.*]] = fmul double [[A:%.*]], 2.000000e+00
-; CHECK-NEXT: [[DIV:%.*]] = fdiv double [[ADD]], [[MUL]]
+; CHECK-NEXT: [[ADD:%.*]] = fsub double [[C:%.*]], [[B]]
; CHECK-NEXT: [[SUB:%.*]] = fsub double [[FNEG]], [[C]]
-; CHECK-NEXT: [[DIV3:%.*]] = fdiv double [[SUB]], [[MUL]]
-; CHECK-NEXT: [[TMP11:%.*]] = fcmp uge double [[DIV]], f0x3EB0C6F7A0B5ED8D
-; CHECK-NEXT: [[TMP10:%.*]] = fcmp uge double [[DIV3]], f0x3EB0C6F7A0B5ED8D
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[SUB]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[ADD]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[MUL]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP5:%.*]] = fdiv <2 x double> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = fcmp uge <2 x double> [[TMP5]], splat (double f0x3EB0C6F7A0B5ED8D)
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1
; CHECK-NEXT: [[NOT_OR_COND:%.*]] = or i1 [[TMP10]], [[TMP11]]
; CHECK-NEXT: ret i1 [[NOT_OR_COND]]
;
More information about the llvm-commits
mailing list