[llvm] [SLP]Make the instruction-count check loop-aware (PR #210074)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 07:41:25 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/210074
Raw getNum{Scalar,Vector}Insts() counted one-time, LICM-hoisted
broadcasts/buildvectors against the loop body, rejecting profitable loop
trees (508.namd_r). Weight each entry by its loop-nest trip
count and drop nest-invariant ones; flat code is unchanged (scale 1).
Fixes #207572
>From 12cac22fdc9ea2d42d40dfedcf5144ef400cb024 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Thu, 16 Jul 2026 07:41:11 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 87 +++++++-----
.../X86/deleted-instructions-clear.ll | 10 +-
.../X86/loop-invariant-gather-inst-count.ll | 128 +++++++++---------
.../phi-removed-on-operand-vectorization.ll | 15 +-
.../Transforms/SLPVectorizer/X86/rgb_phi.ll | 18 +--
5 files changed, 140 insertions(+), 118 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index b76cb80426456..22dfe1cf4f933 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -3747,12 +3747,15 @@ class slpvectorizer::BoUpSLP {
Instruction *I,
const SmallDenseSet<Value *> *VectorizedVals = nullptr) const;
- /// Estimates the number of scalar instructions in the tree.
- unsigned getNumScalarInsts() const;
+ /// Estimates the number of scalar instructions in the tree, each weighted by
+ /// its loop-nest trip count (nest-invariant entries are dropped when
+ /// \p TreeLoop is non-null).
+ uint64_t getNumScalarInsts(const Loop *TreeLoop);
/// Estimates the number of vector instructions (including buildvectors,
- /// shuffles, and extracts) that the tree will produce.
- unsigned getNumVectorInsts() const;
+ /// shuffles, and extracts) the tree produces, weighted like
+ /// getNumScalarInsts().
+ uint64_t getNumVectorInsts(const Loop *TreeLoop);
/// Return information about the vector formed for the specified index
/// of a vector of (the same) instruction.
@@ -3802,6 +3805,10 @@ class slpvectorizer::BoUpSLP {
uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE,
Instruction *U = nullptr);
+ /// \returns the loop-nest execution scale of \p TE.
+ uint64_t getEntryEffectiveScale(const TreeEntry &TE,
+ Instruction *U = nullptr);
+
/// Get the loop nest for the given loop \p L.
ArrayRef<const Loop *> getLoopNest(const Loop *L);
@@ -13824,12 +13831,16 @@ static InstructionCost canConvertToFMA(ArrayRef<Value *> VL,
TargetTransformInfo &TTI,
const TargetLibraryInfo &TLI);
-unsigned BoUpSLP::getNumScalarInsts() const {
- unsigned Count = 0;
+uint64_t BoUpSLP::getNumScalarInsts(const Loop *TreeLoop) {
+ uint64_t Total = 0;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
const TreeEntry &TE = *Ptr;
if (DeletedNodes.contains(&TE))
continue;
+ uint64_t Scale = getEntryEffectiveScale(TE);
+ if (TreeLoop && Scale <= 1)
+ continue;
+ unsigned Count = 0;
if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) {
// Count extractelement scalars in gathers - they exist in the scalar
// code regardless of vectorization. ExtractElement instructions
@@ -13837,12 +13848,13 @@ unsigned BoUpSLP::getNumScalarInsts() const {
for (Value *V : TE.Scalars)
if (isa<ExtractElementInst>(V))
++Count;
+ Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
// CombinedVectorize entries (e.g. the fmul child of an FMulAdd, or the
// cmp child of a MinMax select) are absorbed into the parent on both
- // scalar and vector sides. The backend fuses fadd+fmul → fma and
- // select+cmp → smin/smax even for scalar code, so skip to avoid
+ // scalar and vector sides. The backend fuses fadd+fmul -> fma and
+ // select+cmp -> smin/smax even for scalar code, so skip to avoid
// double-counting.
if (TE.State == TreeEntry::CombinedVectorize)
continue;
@@ -13867,7 +13879,7 @@ unsigned BoUpSLP::getNumScalarInsts() const {
}
// Even when the whole node is not combined, individual scalar
// instructions may be fused by the backend. Each fused pair (e.g.
- // fadd+fmul → fma, select+cmp → smin/smax) becomes a single scalar
+ // fadd+fmul -> fma, select+cmp -> smin/smax) becomes a single scalar
// instruction, absorbing the operand instruction. Subtract 1 for each
// such match to avoid over-counting the scalar side.
if (TE.CombinedOp == TreeEntry::NotCombinedOp && TE.hasState()) {
@@ -13901,12 +13913,13 @@ unsigned BoUpSLP::getNumScalarInsts() const {
}
}
}
+ Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
}
- return Count;
+ return Total;
}
-unsigned BoUpSLP::getNumVectorInsts() const {
- unsigned Count = 0;
+uint64_t BoUpSLP::getNumVectorInsts(const Loop *TreeLoop) {
+ uint64_t Total = 0;
SmallPtrSet<Value *, 4> GatherExtractSourceVecs;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
const TreeEntry &TE = *Ptr;
@@ -13914,9 +13927,11 @@ unsigned BoUpSLP::getNumVectorInsts() const {
continue;
if (TE.State == TreeEntry::CombinedVectorize)
continue;
- bool IsGatherOrTransformed =
- TE.isGather() || TransformedToGatherNodes.contains(&TE);
- if (IsGatherOrTransformed) {
+ uint64_t Scale = getEntryEffectiveScale(TE);
+ if (TreeLoop && Scale <= 1)
+ continue;
+ unsigned Count = 0;
+ if (TE.isGather() || TransformedToGatherNodes.contains(&TE)) {
if (TE.hasState()) {
if (const TreeEntry *E =
getSameValuesTreeEntry(TE.getMainOp(), TE.Scalars);
@@ -13926,7 +13941,7 @@ unsigned BoUpSLP::getNumVectorInsts() const {
if (const TreeEntry *E =
getSameValuesTreeEntry(TE.getMainOp(), RevScalars);
E && E->getVectorFactor() == TE.getVectorFactor()) {
- ++Count;
+ Total = SaturatingAdd(Total, Scale);
continue;
}
}
@@ -13944,6 +13959,7 @@ unsigned BoUpSLP::getNumVectorInsts() const {
++Count;
}
}
+ Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
// InsertElement/ExtractElement vectorize entries don't produce real
@@ -13959,6 +13975,7 @@ unsigned BoUpSLP::getNumVectorInsts() const {
Count += 2;
if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
++Count;
+ Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
continue;
}
if (TE.State == TreeEntry::SplitVectorize)
@@ -13967,8 +13984,9 @@ unsigned BoUpSLP::getNumVectorInsts() const {
++Count;
if (!TE.ReorderIndices.empty() || !TE.ReuseShuffleIndices.empty())
++Count;
+ Total = SaturatingMultiplyAdd<uint64_t>(Count, Scale, Total);
}
- Count += GatherExtractSourceVecs.size();
+ Total += GatherExtractSourceVecs.size();
// Count extract instructions from ExternalUses, skipping insertelements
// (those get folded into shuffles, not real extracts).
SmallPtrSet<Value *, 8> CountedExtracts;
@@ -13981,9 +13999,9 @@ unsigned BoUpSLP::getNumVectorInsts() const {
continue;
if (!CountedExtracts.insert(EU.Scalar).second)
continue;
- ++Count;
+ ++Total;
}
- return Count;
+ return Total;
}
void BoUpSLP::TreeEntry::buildAltOpShuffleMask(
@@ -16844,6 +16862,12 @@ uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE,
return std::clamp<uint64_t>(Avg, 1, BaseScale);
}
+uint64_t BoUpSLP::getEntryEffectiveScale(const TreeEntry &TE, Instruction *U) {
+ if (TE.isGather() || TE.State == TreeEntry::SplitVectorize)
+ return getGatherNodeEffectiveScale(TE, U);
+ return getScaleToLoopIterations(TE);
+}
+
InstructionCost
BoUpSLP::getVectorSpillReloadCost(const TreeEntry *E, Type *ScalarTy,
Type *VecTy, Type *FinalVecTy,
@@ -19376,8 +19400,6 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
// per-lane refined scale that accounts for LICM-hoistable insertelements
// when an operand is invariant in the current loop nest but defined in
// an outer loop. This prevents over-costing cross-loop-nest buildvectors.
- const bool IsGatherLike =
- TE.isGather() || TE.State == TreeEntry::SplitVectorize;
if (!CostIsFree && !TE.isGather() && TE.hasState()) {
if (PrevVecParent == TE.getMainOp()->getParent()) {
Scale = PrevScale;
@@ -19386,10 +19408,7 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
}
}
if (!CostIsFree && !Scale) {
- Scale =
- IsGatherLike
- ? getGatherNodeEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr)
- : getScaleToLoopIterations(TE);
+ Scale = getEntryEffectiveScale(TE, TE.Idx == 0 ? RdxRoot : nullptr);
C *= Scale;
EntryToScale.try_emplace(&TE, Scale);
if (!TE.isGather() && TE.hasState()) {
@@ -19768,12 +19787,8 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
continue;
}
uint64_t Scale = EntryToScale.lookup(TE.get());
- if (!Scale) {
- const bool IsGatherLike =
- TE->isGather() || TE->State == TreeEntry::SplitVectorize;
- Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get())
- : getScaleToLoopIterations(*TE.get());
- }
+ if (!Scale)
+ Scale = getEntryEffectiveScale(*TE);
C *= Scale;
NodesCosts.try_emplace(TE.get(), C);
}
@@ -19857,8 +19872,14 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
(!SLPReVec ||
!isa<VectorType>(
VectorizableTree.front()->Scalars.front()->getType()))) {
- unsigned NumScalar = getNumScalarInsts();
- unsigned NumVector = getNumVectorInsts();
+ // Loop containing the tree root; null for flat code or disabled
+ // loop-aware modeling. Shared by both calls below.
+ const Loop *TreeLoop = nullptr;
+ if (LoopAwareTripCount != 0 && VectorizableTree.front()->hasState())
+ TreeLoop =
+ LI->getLoopFor(VectorizableTree.front()->getMainOp()->getParent());
+ uint64_t NumScalar = getNumScalarInsts(TreeLoop);
+ uint64_t NumVector = getNumVectorInsts(TreeLoop);
LLVM_DEBUG(dbgs() << "SLP: Inst count check: vector=" << NumVector
<< " scalar=" << NumScalar << "\n");
if (NumVector > NumScalar && !BypassesInstCountCheck()) {
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
index ae6da0913cc24..94631869b15fe 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll
@@ -7,7 +7,7 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 0, i32 poison>, i32 [[ARG]], i64 3
; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x i32> poison, i32 [[ARG1]], i64 0
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i32> [[TMP27]], <2 x i32> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[ARG]], i64 1
; CHECK-NEXT: br label %[[BB3:.*]]
; CHECK: [[BB3]]:
; CHECK-NEXT: [[TMP3:%.*]] = phi i64 [ 0, %[[BB3]] ], [ 0, %[[BB]] ]
@@ -28,9 +28,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[OR11:%.*]] = or i32 [[TRUNC27]], 0
; CHECK-NEXT: [[TMP8:%.*]] = or <4 x i32> zeroinitializer, [[TMP2]]
; CHECK-NEXT: [[TMP9:%.*]] = mul <4 x i32> [[TMP5]], [[TMP8]]
-; CHECK-NEXT: [[XOR38:%.*]] = xor i32 [[ARG]], [[TRUNC28]]
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[TRUNC19]], i64 0
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <2 x i32> [[TMP27]], i32 [[TRUNC28]], i64 1
; CHECK-NEXT: [[TMP14:%.*]] = xor <2 x i32> [[TMP28]], [[TMP29]]
+; CHECK-NEXT: [[XOR31:%.*]] = xor i32 [[ARG1]], [[TRUNC19]]
; CHECK-NEXT: [[SHL:%.*]] = shl i32 0, 1
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> poison, i32 [[SHL]], i64 0
; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x i32> [[TMP23]], i32 [[TRUNC10]], i64 1
@@ -38,9 +38,9 @@ define void @test(i32 %arg, i32 %arg1, i64 %arg2) {
; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 3>
; CHECK-NEXT: [[TMP26:%.*]] = xor <4 x i32> [[TMP11]], [[TMP25]]
; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <4 x i32> [[TMP26]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[XOR31]], i64 5
; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <8 x i32> [[TMP24]], <8 x i32> [[TMP30]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 7>
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[XOR38]], i64 7
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP32]], <8 x i32> [[TMP30]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP16]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 5, i32 6, i32 7>
; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll
index 923af5efa5d2e..9c7849d0de123 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/loop-invariant-gather-inst-count.ll
@@ -25,70 +25,70 @@ define void @test(ptr noalias readonly %0, ptr noalias readonly %1, i32 %2, doub
; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP19]]
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 16
; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP21]], align 8
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 8
-; CHECK-NEXT: [[TMP24:%.*]] = load double, ptr [[TMP23]], align 8
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP26:%.*]] = load double, ptr [[TMP25]], align 8
-; CHECK-NEXT: [[TMP27:%.*]] = load double, ptr [[TMP15]], align 8
-; CHECK-NEXT: [[TMP28:%.*]] = load double, ptr [[TMP20]], align 8
-; CHECK-NEXT: [[TMP29:%.*]] = zext nneg i32 [[TMP11]] to i64
-; CHECK-NEXT: br label %[[BB30:.*]]
-; CHECK: [[BB30]]:
-; CHECK-NEXT: [[TMP31:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP81:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP32:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP80:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP33:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP54:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP34:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP56:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP35:%.*]] = phi double [ [[TMP28]], %[[BB10]] ], [ [[TMP59:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP36:%.*]] = phi double [ [[TMP27]], %[[BB10]] ], [ [[TMP62:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP37:%.*]] = phi double [ [[TMP17]], %[[BB10]] ], [ [[TMP70:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP38:%.*]] = phi double [ [[TMP22]], %[[BB10]] ], [ [[TMP68:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP39:%.*]] = phi double [ [[TMP24]], %[[BB10]] ], [ [[TMP66:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP40:%.*]] = phi double [ [[TMP26]], %[[BB10]] ], [ [[TMP64:%.*]], %[[BB30]] ]
-; CHECK-NEXT: [[TMP41:%.*]] = fsub double [[TMP3]], [[TMP35]]
-; CHECK-NEXT: [[TMP42:%.*]] = fsub double [[TMP3]], [[TMP36]]
-; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[TMP41]], [[TMP41]]
-; CHECK-NEXT: [[TMP44:%.*]] = fmul double [[TMP42]], [[TMP42]]
-; CHECK-NEXT: [[TMP45:%.*]] = fsub double [[TMP4]], [[TMP40]]
-; CHECK-NEXT: [[TMP46:%.*]] = fsub double [[TMP4]], [[TMP39]]
-; CHECK-NEXT: [[TMP47:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP45]], double [[TMP45]], double [[TMP43]])
-; CHECK-NEXT: [[TMP48:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP46]], double [[TMP46]], double [[TMP44]])
-; CHECK-NEXT: [[TMP49:%.*]] = fsub double [[TMP5]], [[TMP38]]
-; CHECK-NEXT: [[TMP50:%.*]] = fsub double [[TMP5]], [[TMP37]]
-; CHECK-NEXT: [[TMP51:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP49]], double [[TMP49]], double [[TMP47]])
-; CHECK-NEXT: [[TMP52:%.*]] = tail call double @llvm.fmuladd.f64(double [[TMP50]], double [[TMP50]], double [[TMP48]])
-; CHECK-NEXT: [[TMP53:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP31]]
-; CHECK-NEXT: [[TMP54]] = load i32, ptr [[TMP53]], align 4
-; CHECK-NEXT: [[TMP55:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP53]], i64 4
-; CHECK-NEXT: [[TMP56]] = load i32, ptr [[TMP55]], align 4
-; CHECK-NEXT: [[TMP57:%.*]] = sext i32 [[TMP54]] to i64
-; CHECK-NEXT: [[TMP58:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP57]]
-; CHECK-NEXT: [[TMP59]] = load double, ptr [[TMP58]], align 8
-; CHECK-NEXT: [[TMP60:%.*]] = sext i32 [[TMP56]] to i64
-; CHECK-NEXT: [[TMP61:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP60]]
-; CHECK-NEXT: [[TMP62]] = load double, ptr [[TMP61]], align 8
-; CHECK-NEXT: [[TMP63:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 8
-; CHECK-NEXT: [[TMP64]] = load double, ptr [[TMP63]], align 8
-; CHECK-NEXT: [[TMP65:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 8
-; CHECK-NEXT: [[TMP66]] = load double, ptr [[TMP65]], align 8
-; CHECK-NEXT: [[TMP67:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP58]], i64 16
-; CHECK-NEXT: [[TMP68]] = load double, ptr [[TMP67]], align 8
-; CHECK-NEXT: [[TMP69:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP61]], i64 16
-; CHECK-NEXT: [[TMP70]] = load double, ptr [[TMP69]], align 8
-; CHECK-NEXT: [[TMP71:%.*]] = fcmp olt double [[TMP51]], [[TMP6]]
-; CHECK-NEXT: [[TMP72:%.*]] = fcmp olt double [[TMP52]], [[TMP6]]
-; CHECK-NEXT: [[TMP73:%.*]] = zext nneg i32 [[TMP32]] to i64
-; CHECK-NEXT: [[TMP74:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP73]]
-; CHECK-NEXT: store i32 [[TMP33]], ptr [[TMP74]], align 4
-; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP71]] to i32
-; CHECK-NEXT: [[TMP76:%.*]] = add nuw nsw i32 [[TMP32]], [[TMP75]]
-; CHECK-NEXT: [[TMP77:%.*]] = zext nneg i32 [[TMP76]] to i64
-; CHECK-NEXT: [[TMP78:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP77]]
-; CHECK-NEXT: store i32 [[TMP34]], ptr [[TMP78]], align 4
-; CHECK-NEXT: [[TMP79:%.*]] = zext i1 [[TMP72]] to i32
-; CHECK-NEXT: [[TMP80]] = add nuw nsw i32 [[TMP76]], [[TMP79]]
-; CHECK-NEXT: [[TMP81]] = add nuw nsw i64 [[TMP31]], 2
-; CHECK-NEXT: [[TMP82:%.*]] = icmp samesign ult i64 [[TMP81]], [[TMP29]]
-; CHECK-NEXT: br i1 [[TMP82]], label %[[BB30]], label %[[BB83]]
+; CHECK-NEXT: [[TMP23:%.*]] = load <2 x double>, ptr [[TMP15]], align 8
+; CHECK-NEXT: [[TMP24:%.*]] = load <2 x double>, ptr [[TMP20]], align 8
+; CHECK-NEXT: [[TMP25:%.*]] = zext nneg i32 [[TMP11]] to i64
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <2 x double> poison, double [[TMP17]], i64 0
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <2 x double> [[TMP26]], double [[TMP22]], i64 1
+; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <2 x double> [[TMP23]], <2 x double> [[TMP24]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <2 x double> poison, double [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP31:%.*]] = shufflevector <2 x double> [[TMP30]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <2 x double> [[TMP32]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <2 x double> poison, double [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <2 x double> [[TMP34]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP37:%.*]] = shufflevector <2 x double> [[TMP36]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[BB38:.*]]
+; CHECK: [[BB38]]:
+; CHECK-NEXT: [[TMP39:%.*]] = phi i64 [ 2, %[[BB10]] ], [ [[TMP77:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP40:%.*]] = phi i32 [ 0, %[[BB10]] ], [ [[TMP76:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP41:%.*]] = phi i32 [ [[TMP18]], %[[BB10]] ], [ [[TMP50:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP42:%.*]] = phi i32 [ [[TMP13]], %[[BB10]] ], [ [[TMP52:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP43:%.*]] = phi <2 x double> [ [[TMP27]], %[[BB10]] ], [ [[TMP80:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP44:%.*]] = phi <2 x double> [ [[TMP28]], %[[BB10]] ], [ [[TMP81:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP45:%.*]] = phi <2 x double> [ [[TMP29]], %[[BB10]] ], [ [[TMP82:%.*]], %[[BB38]] ]
+; CHECK-NEXT: [[TMP46:%.*]] = fsub <2 x double> [[TMP35]], [[TMP45]]
+; CHECK-NEXT: [[TMP47:%.*]] = fsub <2 x double> [[TMP33]], [[TMP44]]
+; CHECK-NEXT: [[TMP48:%.*]] = fsub <2 x double> [[TMP31]], [[TMP43]]
+; CHECK-NEXT: [[TMP49:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP1]], i64 [[TMP39]]
+; CHECK-NEXT: [[TMP50]] = load i32, ptr [[TMP49]], align 4
+; CHECK-NEXT: [[TMP51:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP49]], i64 4
+; CHECK-NEXT: [[TMP52]] = load i32, ptr [[TMP51]], align 4
+; CHECK-NEXT: [[TMP53:%.*]] = sext i32 [[TMP50]] to i64
+; CHECK-NEXT: [[TMP54:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP53]]
+; CHECK-NEXT: [[TMP55:%.*]] = sext i32 [[TMP52]] to i64
+; CHECK-NEXT: [[TMP56:%.*]] = getelementptr inbounds [32 x i8], ptr [[TMP0]], i64 [[TMP55]]
+; CHECK-NEXT: [[TMP57:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP54]], i64 16
+; CHECK-NEXT: [[TMP58:%.*]] = load double, ptr [[TMP57]], align 8
+; CHECK-NEXT: [[TMP59:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP56]], i64 16
+; CHECK-NEXT: [[TMP60:%.*]] = load double, ptr [[TMP59]], align 8
+; CHECK-NEXT: [[TMP61:%.*]] = fmul <2 x double> [[TMP46]], [[TMP46]]
+; CHECK-NEXT: [[TMP62:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP47]], <2 x double> [[TMP47]], <2 x double> [[TMP61]])
+; CHECK-NEXT: [[TMP63:%.*]] = call <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[TMP48]], <2 x double> [[TMP48]], <2 x double> [[TMP62]])
+; CHECK-NEXT: [[TMP64:%.*]] = load <2 x double>, ptr [[TMP54]], align 8
+; CHECK-NEXT: [[TMP65:%.*]] = load <2 x double>, ptr [[TMP56]], align 8
+; CHECK-NEXT: [[TMP66:%.*]] = fcmp olt <2 x double> [[TMP63]], [[TMP37]]
+; CHECK-NEXT: [[TMP67:%.*]] = zext nneg i32 [[TMP40]] to i64
+; CHECK-NEXT: [[TMP68:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP67]]
+; CHECK-NEXT: store i32 [[TMP41]], ptr [[TMP68]], align 4
+; CHECK-NEXT: [[TMP69:%.*]] = extractelement <2 x i1> [[TMP66]], i64 1
+; CHECK-NEXT: [[TMP70:%.*]] = zext i1 [[TMP69]] to i32
+; CHECK-NEXT: [[TMP71:%.*]] = add nuw nsw i32 [[TMP40]], [[TMP70]]
+; CHECK-NEXT: [[TMP72:%.*]] = zext nneg i32 [[TMP71]] to i64
+; CHECK-NEXT: [[TMP73:%.*]] = getelementptr inbounds nuw [4 x i8], ptr [[TMP7]], i64 [[TMP72]]
+; CHECK-NEXT: store i32 [[TMP42]], ptr [[TMP73]], align 4
+; CHECK-NEXT: [[TMP74:%.*]] = extractelement <2 x i1> [[TMP66]], i64 0
+; CHECK-NEXT: [[TMP75:%.*]] = zext i1 [[TMP74]] to i32
+; CHECK-NEXT: [[TMP76]] = add nuw nsw i32 [[TMP71]], [[TMP75]]
+; CHECK-NEXT: [[TMP77]] = add nuw nsw i64 [[TMP39]], 2
+; CHECK-NEXT: [[TMP78:%.*]] = icmp samesign ult i64 [[TMP77]], [[TMP25]]
+; CHECK-NEXT: [[TMP79:%.*]] = insertelement <2 x double> poison, double [[TMP60]], i64 0
+; CHECK-NEXT: [[TMP80]] = insertelement <2 x double> [[TMP79]], double [[TMP58]], i64 1
+; CHECK-NEXT: [[TMP81]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP82]] = shufflevector <2 x double> [[TMP65]], <2 x double> [[TMP64]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: br i1 [[TMP78]], label %[[BB38]], label %[[BB83]]
; CHECK: [[BB83]]:
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll
index b47242eceafc2..aefa17e9c6a6e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/phi-removed-on-operand-vectorization.ll
@@ -5,17 +5,18 @@ define i32 @test(double %mul321.i) {
; CHECK-LABEL: define i32 @test(
; CHECK-SAME: double [[MUL321_I:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[MUL321_I]], i64 0
; CHECK-NEXT: br label %[[DO_BODY220_I:.*]]
; CHECK: [[DO_BODY220_I]]:
-; CHECK-NEXT: [[C1_2_I:%.*]] = phi double [ [[MUL321_I]], %[[DO_BODY221_I:.*]] ], [ 0.000000e+00, %[[ENTRY]] ]
-; CHECK-NEXT: [[S1_1_I:%.*]] = phi double [ [[ADD318_I:%.*]], %[[DO_BODY221_I]] ], [ 0.000000e+00, %[[ENTRY]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[TMP6:%.*]], %[[DO_BODY221_I:.*]] ], [ zeroinitializer, %[[ENTRY]] ]
; CHECK-NEXT: br label %[[DO_BODY221_I]]
; CHECK: [[DO_BODY221_I]]:
-; CHECK-NEXT: [[SUB311_I1:%.*]] = fadd double [[C1_2_I]], 0.000000e+00
-; CHECK-NEXT: [[ADD315_I:%.*]] = fadd double [[S1_1_I]], 0.000000e+00
-; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[SUB311_I1]], 0.000000e+00
-; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[ADD315_I]], 0.000000e+00
-; CHECK-NEXT: [[ADD318_I]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x double> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x double> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP3]], i64 1
+; CHECK-NEXT: [[ADD318_I:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT: [[TMP6]] = insertelement <2 x double> [[TMP0]], double [[ADD318_I]], i64 1
; CHECK-NEXT: br label %[[DO_BODY220_I]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
index eb649f700bda6..8d3a93bb8d49c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
@@ -23,25 +23,23 @@ target triple = "i386-apple-macosx10.9.0"
define float @foo(ptr nocapture readonly %A) {
; CHECK-LABEL: @foo(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[A:%.*]], align 4
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4
; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 2
; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP0]], i64 0
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[TMP4:%.*]] = phi float [ [[TMP3]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ]
; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
; CHECK-NEXT: [[B_032:%.*]] = phi float [ [[TMP2]], [[ENTRY]] ], [ [[ADD14:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
-; CHECK-NEXT: [[G_031:%.*]] = phi float [ [[TMP1]], [[ENTRY]] ], [ [[TMP16:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
-; CHECK-NEXT: [[R_030:%.*]] = phi float [ [[TMP3]], [[ENTRY]] ], [ [[TMP15:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
-; CHECK-NEXT: [[MUL:%.*]] = fmul float [[TMP4]], 7.000000e+00
-; CHECK-NEXT: [[TMP15]] = fadd float [[R_030]], [[MUL]]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <2 x float> [ [[TMP0]], [[ENTRY]] ], [ [[TMP10:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
; CHECK-NEXT: [[TMP6:%.*]] = add nsw i64 [[INDVARS_IV]], 1
; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP6]]
; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[ARRAYIDX7]], align 4
-; CHECK-NEXT: [[MUL8:%.*]] = fmul float [[TMP7]], 8.000000e+00
-; CHECK-NEXT: [[TMP16]] = fadd float [[G_031]], [[MUL8]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = fmul <2 x float> [[TMP8]], <float 7.000000e+00, float 8.000000e+00>
+; CHECK-NEXT: [[TMP10]] = fadd <2 x float> [[TMP5]], [[TMP9]]
; CHECK-NEXT: [[TMP12:%.*]] = add nsw i64 [[INDVARS_IV]], 2
; CHECK-NEXT: [[ARRAYIDX12:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP12]]
; CHECK-NEXT: [[TMP13:%.*]] = load float, ptr [[ARRAYIDX12]], align 4
@@ -56,6 +54,8 @@ define float @foo(ptr nocapture readonly %A) {
; CHECK-NEXT: [[DOTPRE]] = load float, ptr [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4
; CHECK-NEXT: br label [[FOR_BODY]]
; CHECK: for.end:
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <2 x float> [[TMP10]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x float> [[TMP10]], i64 1
; CHECK-NEXT: [[ADD16:%.*]] = fadd float [[TMP15]], [[TMP16]]
; CHECK-NEXT: [[ADD17:%.*]] = fadd float [[ADD16]], [[ADD14]]
; CHECK-NEXT: ret float [[ADD17]]
More information about the llvm-commits
mailing list