[llvm] 4b1b51a - [SLP]Initial non-power-of-2 support (but still whole register) for reductions
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 21 12:39:48 PDT 2024
Author: Alexey Bataev
Date: 2024-10-21T12:25:39-07:00
New Revision: 4b1b51ac52445f2308174287c721ad7f60a8053b
URL: https://github.com/llvm/llvm-project/commit/4b1b51ac52445f2308174287c721ad7f60a8053b
DIFF: https://github.com/llvm/llvm-project/commit/4b1b51ac52445f2308174287c721ad7f60a8053b.diff
LOG: [SLP]Initial non-power-of-2 support (but still whole register) for reductions
Enables initial non-power-of-2 support (but still requires number of
elements, forming whole registers) for reductions.
Enables extra vectorization for
MultiSource/Benchmarks/7zip/7zip-benchmark, CINT2006/464.h264ref and
CFP2017rate/526.blender_r (checked for SSE2)
Reviewers: RKSimon
Reviewed By: RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/112361
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
llvm/test/Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll
llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index a11e3f3815cbf7..756b25ac985612 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -291,6 +291,8 @@ getFloorFullVectorNumberOfElements(const TargetTransformInfo &TTI, Type *Ty,
if (NumParts == 0 || NumParts >= Sz)
return bit_floor(Sz);
unsigned RegVF = bit_ceil(divideCeil(Sz, NumParts));
+ if (RegVF > Sz)
+ return bit_floor(Sz);
return (Sz / RegVF) * RegVF;
}
@@ -1505,6 +1507,12 @@ class BoUpSLP {
/// vectorizable. We do not vectorize such trees.
bool isTreeTinyAndNotFullyVectorizable(bool ForReduction = false) const;
+ /// Checks if the graph and all its subgraphs cannot be better vectorized.
+ /// It may happen, if all gather nodes are loads and they cannot be
+ /// "clusterized". In this case even subgraphs cannot be vectorized more
+ /// effectively than the base graph.
+ bool isTreeNotExtendable() const;
+
/// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values
/// can be load combined in the backend. Load combining may not be allowed in
/// the IR optimizer, so we do not want to alter the pattern. For example,
@@ -3047,7 +3055,9 @@ class BoUpSLP {
/// vector loads/masked gathers instead of regular gathers. Later these loads
/// are reshufled to build final gathered nodes.
void tryToVectorizeGatheredLoads(
- ArrayRef<SmallVector<std::pair<LoadInst *, int>>> GatheredLoads);
+ const SmallMapVector<std::tuple<BasicBlock *, Value *, Type *>,
+ SmallVector<SmallVector<std::pair<LoadInst *, int>>>,
+ 8> &GatheredLoads);
/// Reorder commutative or alt operands to get better probability of
/// generating vectorized code.
@@ -3059,7 +3069,7 @@ class BoUpSLP {
/// Helper for `findExternalStoreUsersReorderIndices()`. It iterates over the
/// users of \p TE and collects the stores. It returns the map from the store
/// pointers to the collected stores.
- DenseMap<Value *, SmallVector<StoreInst *>>
+ SmallVector<SmallVector<StoreInst *>>
collectUserStores(const BoUpSLP::TreeEntry *TE) const;
/// Helper for `findExternalStoreUsersReorderIndices()`. It checks if the
@@ -4657,7 +4667,8 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) {
static bool arePointersCompatible(Value *Ptr1, Value *Ptr2,
const TargetLibraryInfo &TLI,
bool CompareOpcodes = true) {
- if (getUnderlyingObject(Ptr1) != getUnderlyingObject(Ptr2))
+ if (getUnderlyingObject(Ptr1, RecursionMaxDepth) !=
+ getUnderlyingObject(Ptr2, RecursionMaxDepth))
return false;
auto *GEP1 = dyn_cast<GetElementPtrInst>(Ptr1);
auto *GEP2 = dyn_cast<GetElementPtrInst>(Ptr2);
@@ -5177,30 +5188,40 @@ BoUpSLP::canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
return LoadsState::Gather;
}
-static bool clusterSortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy,
+static bool clusterSortPtrAccesses(ArrayRef<Value *> VL,
+ ArrayRef<BasicBlock *> BBs, Type *ElemTy,
const DataLayout &DL, ScalarEvolution &SE,
SmallVectorImpl<unsigned> &SortedIndices) {
- assert(llvm::all_of(
- VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
- "Expected list of pointer operands.");
+ assert(
+ all_of(VL, [](const Value *V) { return V->getType()->isPointerTy(); }) &&
+ "Expected list of pointer operands.");
// Map from bases to a vector of (Ptr, Offset, OrigIdx), which we insert each
// Ptr into, sort and return the sorted indices with values next to one
// another.
- MapVector<Value *, SmallVector<std::tuple<Value *, int, unsigned>>> Bases;
- Bases[VL[0]].push_back(std::make_tuple(VL[0], 0U, 0U));
-
- unsigned Cnt = 1;
- for (Value *Ptr : VL.drop_front()) {
- bool Found = any_of(Bases, [&](auto &Base) {
- std::optional<int> Diff =
- getPointersDiff(ElemTy, Base.first, ElemTy, Ptr, DL, SE,
- /*StrictCheck=*/true);
- if (!Diff)
- return false;
+ SmallMapVector<std::pair<BasicBlock *, Value *>,
+ SmallVector<SmallVector<std::tuple<Value *, int, unsigned>>>, 8>
+ Bases;
+ Bases
+ .try_emplace(std::make_pair(
+ BBs.front(), getUnderlyingObject(VL.front(), RecursionMaxDepth)))
+ .first->second.emplace_back().emplace_back(VL.front(), 0U, 0U);
- Base.second.emplace_back(Ptr, *Diff, Cnt++);
- return true;
- });
+ SortedIndices.clear();
+ for (auto [Cnt, Ptr] : enumerate(VL.drop_front())) {
+ auto Key = std::make_pair(BBs[Cnt + 1],
+ getUnderlyingObject(Ptr, RecursionMaxDepth));
+ bool Found = any_of(Bases.try_emplace(Key).first->second,
+ [&, &Cnt = Cnt, &Ptr = Ptr](auto &Base) {
+ std::optional<int> Diff = getPointersDiff(
+ ElemTy, std::get<0>(Base.front()), ElemTy,
+ Ptr, DL, SE,
+ /*StrictCheck=*/true);
+ if (!Diff)
+ return false;
+
+ Base.emplace_back(Ptr, *Diff, Cnt + 1);
+ return true;
+ });
if (!Found) {
// If we haven't found enough to usefully cluster, return early.
@@ -5208,71 +5229,66 @@ static bool clusterSortPtrAccesses(ArrayRef<Value *> VL, Type *ElemTy,
return false;
// Not found already - add a new Base
- Bases[Ptr].emplace_back(Ptr, 0, Cnt++);
+ Bases.find(Key)->second.emplace_back().emplace_back(Ptr, 0, Cnt + 1);
}
}
- // For each of the bases sort the pointers by Offset and check if any of the
- // base become consecutively allocated.
- bool AnyConsecutive = false;
- for (auto &Base : Bases) {
- auto &Vec = Base.second;
- if (Vec.size() > 1) {
- llvm::stable_sort(Vec, [](const std::tuple<Value *, int, unsigned> &X,
- const std::tuple<Value *, int, unsigned> &Y) {
- return std::get<1>(X) < std::get<1>(Y);
- });
- int InitialOffset = std::get<1>(Vec[0]);
- AnyConsecutive |= all_of(enumerate(Vec), [InitialOffset](const auto &P) {
- return std::get<1>(P.value()) == int(P.index()) + InitialOffset;
- });
- }
- }
+ if (Bases.size() == VL.size())
+ return false;
- // Fill SortedIndices array only if it looks worth-while to sort the ptrs.
- SortedIndices.clear();
- if (!AnyConsecutive)
+ if (Bases.size() == 1 && (Bases.front().second.size() == 1 ||
+ Bases.front().second.size() == VL.size()))
return false;
- // If we have a better order, also sort the base pointers by increasing
- // (variable) values if possible, to try and keep the order more regular. In
- // order to create a valid strict-weak order we cluster by the Root of gep
- // chains and sort within each.
- SmallVector<std::tuple<Value *, Value *, Value *>> SortedBases;
+ // For each of the bases sort the pointers by Offset and check if any of the
+ // base become consecutively allocated.
+ auto ComparePointers = [](Value *Ptr1, Value *Ptr2) {
+ SmallPtrSet<Value *, 13> FirstPointers;
+ SmallPtrSet<Value *, 13> SecondPointers;
+ Value *P1 = Ptr1;
+ Value *P2 = Ptr2;
+ if (P1 == P2)
+ return false;
+ unsigned Depth = 0;
+ while (!FirstPointers.contains(P2) && !SecondPointers.contains(P1) &&
+ Depth <= RecursionMaxDepth) {
+ FirstPointers.insert(P1);
+ SecondPointers.insert(P2);
+ P1 = getUnderlyingObject(P1, /*MaxLookup=*/1);
+ P2 = getUnderlyingObject(P2, /*MaxLookup=*/1);
+ ++Depth;
+ }
+ assert((FirstPointers.contains(P2) || SecondPointers.contains(P1)) &&
+ "Unable to find matching root.");
+ return FirstPointers.contains(P2) && !SecondPointers.contains(P1);
+ };
for (auto &Base : Bases) {
- Value *Strip = Base.first->stripInBoundsConstantOffsets();
- Value *Root = Strip;
- while (auto *Gep = dyn_cast<GetElementPtrInst>(Root))
- Root = Gep->getOperand(0);
- SortedBases.emplace_back(Base.first, Strip, Root);
- }
- auto *Begin = SortedBases.begin();
- auto *End = SortedBases.end();
- while (Begin != End) {
- Value *Root = std::get<2>(*Begin);
- auto *Mid = std::stable_partition(
- Begin, End, [&Root](auto V) { return std::get<2>(V) == Root; });
- DenseMap<Value *, DenseMap<Value *, bool>> LessThan;
- for (auto *I = Begin; I < Mid; ++I)
- LessThan.try_emplace(std::get<1>(*I));
- for (auto *I = Begin; I < Mid; ++I) {
- Value *V = std::get<1>(*I);
- while (auto *Gep = dyn_cast<GetElementPtrInst>(V)) {
- V = Gep->getOperand(0);
- if (LessThan.contains(V))
- LessThan[V][std::get<1>(*I)] = true;
- }
- }
- std::stable_sort(Begin, Mid, [&LessThan](auto &V1, auto &V2) {
- return LessThan[std::get<1>(V1)][std::get<1>(V2)];
+ for (auto &Vec : Base.second) {
+ if (Vec.size() > 1) {
+ stable_sort(Vec, [](const std::tuple<Value *, int, unsigned> &X,
+ const std::tuple<Value *, int, unsigned> &Y) {
+ return std::get<1>(X) < std::get<1>(Y);
+ });
+ int InitialOffset = std::get<1>(Vec[0]);
+ bool AnyConsecutive =
+ all_of(enumerate(Vec), [InitialOffset](const auto &P) {
+ return std::get<1>(P.value()) == int(P.index()) + InitialOffset;
+ });
+ // Fill SortedIndices array only if it looks worth-while to sort the
+ // ptrs.
+ if (!AnyConsecutive)
+ return false;
+ }
+ }
+ stable_sort(Base.second, [&](const auto &V1, const auto &V2) {
+ return ComparePointers(std::get<0>(V1.front()), std::get<0>(V2.front()));
});
- Begin = Mid;
}
- // Collect the final order of sorted indices
- for (auto Base : SortedBases)
- for (auto &T : Bases[std::get<0>(Base)])
- SortedIndices.push_back(std::get<2>(T));
+ for (auto &T : Bases)
+ for (const auto &Vec : T.second)
+ for (const auto &P : Vec)
+ SortedIndices.push_back(std::get<2>(P));
assert(SortedIndices.size() == VL.size() &&
"Expected SortedIndices to be the size of VL");
@@ -5286,15 +5302,19 @@ BoUpSLP::findPartiallyOrderedLoads(const BoUpSLP::TreeEntry &TE) {
SmallVector<Value *> Ptrs;
Ptrs.reserve(TE.Scalars.size());
+ SmallVector<BasicBlock *> BBs;
+ BBs.reserve(TE.Scalars.size());
for (Value *V : TE.Scalars) {
auto *L = dyn_cast<LoadInst>(V);
if (!L || !L->isSimple())
return std::nullopt;
Ptrs.push_back(L->getPointerOperand());
+ BBs.push_back(L->getParent());
}
BoUpSLP::OrdersType Order;
- if (clusterSortPtrAccesses(Ptrs, ScalarTy, *DL, *SE, Order))
+ if (!LoadEntriesToVectorize.contains(TE.Idx) &&
+ clusterSortPtrAccesses(Ptrs, BBs, ScalarTy, *DL, *SE, Order))
return std::move(Order);
return std::nullopt;
}
@@ -5662,7 +5682,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) {
}
// FIXME: Remove the non-power-of-two check once findReusedOrderedScalars
// has been auditted for correctness with non-power-of-two vectors.
- if (!TE.hasNonWholeRegisterOrNonPowerOf2Vec(*TTI))
+ if (!VectorizeNonPowerOf2 || !TE.hasNonWholeRegisterOrNonPowerOf2Vec(*TTI))
if (std::optional<OrdersType> CurrentOrder = findReusedOrderedScalars(TE))
return CurrentOrder;
}
@@ -6393,13 +6413,15 @@ void BoUpSLP::buildExternalUses(
}
}
-DenseMap<Value *, SmallVector<StoreInst *>>
+SmallVector<SmallVector<StoreInst *>>
BoUpSLP::collectUserStores(const BoUpSLP::TreeEntry *TE) const {
- DenseMap<Value *, SmallVector<StoreInst *>> PtrToStoresMap;
+ SmallDenseMap<std::tuple<BasicBlock *, Type *, Value *>,
+ SmallVector<StoreInst *>, 8>
+ PtrToStoresMap;
for (unsigned Lane : seq<unsigned>(0, TE->Scalars.size())) {
Value *V = TE->Scalars[Lane];
// Don't iterate over the users of constant data.
- if (isa<ConstantData>(V))
+ if (!isa<Instruction>(V))
continue;
// To save compilation time we don't visit if we have too many users.
if (V->hasNUsesOrMore(UsesLimit))
@@ -6417,25 +6439,34 @@ BoUpSLP::collectUserStores(const BoUpSLP::TreeEntry *TE) const {
if (getTreeEntry(U))
continue;
- Value *Ptr = getUnderlyingObject(SI->getPointerOperand());
- auto &StoresVec = PtrToStoresMap[Ptr];
+ Value *Ptr =
+ getUnderlyingObject(SI->getPointerOperand(), RecursionMaxDepth);
+ auto &StoresVec = PtrToStoresMap[{SI->getParent(),
+ SI->getValueOperand()->getType(), Ptr}];
// For now just keep one store per pointer object per lane.
// TODO: Extend this to support multiple stores per pointer per lane
if (StoresVec.size() > Lane)
continue;
- // Skip if in
diff erent BBs.
- if (!StoresVec.empty() &&
- SI->getParent() != StoresVec.back()->getParent())
- continue;
- // Make sure that the stores are of the same type.
- if (!StoresVec.empty() &&
- SI->getValueOperand()->getType() !=
- StoresVec.back()->getValueOperand()->getType())
- continue;
+ if (!StoresVec.empty()) {
+ std::optional<int> Diff = getPointersDiff(
+ SI->getValueOperand()->getType(), SI->getPointerOperand(),
+ SI->getValueOperand()->getType(),
+ StoresVec.front()->getPointerOperand(), *DL, *SE,
+ /*StrictCheck=*/true);
+ // We failed to compare the pointers so just abandon this store.
+ if (!Diff)
+ continue;
+ }
StoresVec.push_back(SI);
}
}
- return PtrToStoresMap;
+ SmallVector<SmallVector<StoreInst *>> Res(PtrToStoresMap.size());
+ unsigned I = 0;
+ for (auto &P : PtrToStoresMap) {
+ Res[I].swap(P.second);
+ ++I;
+ }
+ return Res;
}
bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
@@ -6445,9 +6476,9 @@ bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
// To avoid calling getPointersDiff() while sorting we create a vector of
// pairs {store, offset from first} and sort this instead.
- SmallVector<std::pair<StoreInst *, int>> StoreOffsetVec(StoresVec.size());
+ SmallVector<std::pair<int, unsigned>> StoreOffsetVec;
StoreInst *S0 = StoresVec[0];
- StoreOffsetVec[0] = {S0, 0};
+ StoreOffsetVec.emplace_back(0, 0);
Type *S0Ty = S0->getValueOperand()->getType();
Value *S0Ptr = S0->getPointerOperand();
for (unsigned Idx : seq<unsigned>(1, StoresVec.size())) {
@@ -6456,41 +6487,36 @@ bool BoUpSLP::canFormVector(ArrayRef<StoreInst *> StoresVec,
getPointersDiff(S0Ty, S0Ptr, SI->getValueOperand()->getType(),
SI->getPointerOperand(), *DL, *SE,
/*StrictCheck=*/true);
- // We failed to compare the pointers so just abandon this StoresVec.
- if (!Diff)
- return false;
- StoreOffsetVec[Idx] = {StoresVec[Idx], *Diff};
+ StoreOffsetVec.emplace_back(*Diff, Idx);
}
- // Sort the vector based on the pointers. We create a copy because we may
- // need the original later for calculating the reorder (shuffle) indices.
- stable_sort(StoreOffsetVec, [](const std::pair<StoreInst *, int> &Pair1,
- const std::pair<StoreInst *, int> &Pair2) {
- int Offset1 = Pair1.second;
- int Offset2 = Pair2.second;
- return Offset1 < Offset2;
- });
-
// Check if the stores are consecutive by checking if their
diff erence is 1.
- for (unsigned Idx : seq<unsigned>(1, StoreOffsetVec.size()))
- if (StoreOffsetVec[Idx].second != StoreOffsetVec[Idx - 1].second + 1)
+ if (StoreOffsetVec.size() != StoresVec.size())
+ return false;
+ sort(StoreOffsetVec,
+ [](const std::pair<int, unsigned> &L,
+ const std::pair<int, unsigned> &R) { return L.first < R.first; });
+ unsigned Idx = 0;
+ int PrevDist = 0;
+ for (const auto &P : StoreOffsetVec) {
+ if (Idx > 0 && P.first != PrevDist + 1)
return false;
+ PrevDist = P.first;
+ ++Idx;
+ }
// Calculate the shuffle indices according to their offset against the sorted
// StoreOffsetVec.
- ReorderIndices.reserve(StoresVec.size());
- for (StoreInst *SI : StoresVec) {
- unsigned Idx = find_if(StoreOffsetVec,
- [SI](const std::pair<StoreInst *, int> &Pair) {
- return Pair.first == SI;
- }) -
- StoreOffsetVec.begin();
- ReorderIndices.push_back(Idx);
+ ReorderIndices.assign(StoresVec.size(), 0);
+ bool IsIdentity = true;
+ for (auto [I, P] : enumerate(StoreOffsetVec)) {
+ ReorderIndices[P.second] = I;
+ IsIdentity &= P.second == I;
}
// Identity order (e.g., {0,1,2,3}) is modeled as an empty OrdersType in
// reorderTopToBottom() and reorderBottomToTop(), so we are following the
// same convention here.
- if (isIdentityOrder(ReorderIndices))
+ if (IsIdentity)
ReorderIndices.clear();
return true;
@@ -6508,8 +6534,7 @@ SmallVector<BoUpSLP::OrdersType, 1>
BoUpSLP::findExternalStoreUsersReorderIndices(TreeEntry *TE) const {
unsigned NumLanes = TE->Scalars.size();
- DenseMap<Value *, SmallVector<StoreInst *>> PtrToStoresMap =
- collectUserStores(TE);
+ SmallVector<SmallVector<StoreInst *>> Stores = collectUserStores(TE);
// Holds the reorder indices for each candidate store vector that is a user of
// the current TreeEntry.
@@ -6518,8 +6543,7 @@ BoUpSLP::findExternalStoreUsersReorderIndices(TreeEntry *TE) const {
// Now inspect the stores collected per pointer and look for vectorization
// candidates. For each candidate calculate the reorder index vector and push
// it into `ExternalReorderIndices`
- for (const auto &Pair : PtrToStoresMap) {
- auto &StoresVec = Pair.second;
+ for (ArrayRef<StoreInst *> StoresVec : Stores) {
// If we have fewer than NumLanes stores, then we can't form a vector.
if (StoresVec.size() != NumLanes)
continue;
@@ -6574,9 +6598,13 @@ static void gatherPossiblyVectorizableLoads(
continue;
bool IsFound = false;
for (auto [Map, Data] : zip(ClusteredDistToLoad, ClusteredLoads)) {
- if (LI->getParent() != Data.front().first->getParent() ||
- LI->getType() != Data.front().first->getType())
- continue;
+ assert(LI->getParent() == Data.front().first->getParent() &&
+ LI->getType() == Data.front().first->getType() &&
+ getUnderlyingObject(LI->getPointerOperand(), RecursionMaxDepth) ==
+ getUnderlyingObject(Data.front().first->getPointerOperand(),
+ RecursionMaxDepth) &&
+ "Expected loads with the same type, same parent and same "
+ "underlying pointer.");
std::optional<int> Dist = getPointersDiff(
LI->getType(), LI->getPointerOperand(), Data.front().first->getType(),
Data.front().first->getPointerOperand(), DL, SE,
@@ -6704,7 +6732,9 @@ static void gatherPossiblyVectorizableLoads(
}
void BoUpSLP::tryToVectorizeGatheredLoads(
- ArrayRef<SmallVector<std::pair<LoadInst *, int>>> GatheredLoads) {
+ const SmallMapVector<std::tuple<BasicBlock *, Value *, Type *>,
+ SmallVector<SmallVector<std::pair<LoadInst *, int>>>,
+ 8> &GatheredLoads) {
GatheredLoadsEntriesFirst = VectorizableTree.size();
SmallVector<SmallPtrSet<const Value *, 4>> LoadSetsToVectorize(
@@ -6737,7 +6767,10 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
SmallVector<int> CandidateVFs;
if (VectorizeNonPowerOf2 && has_single_bit(MaxVF + 1))
CandidateVFs.push_back(MaxVF);
- for (int NumElts = bit_floor(MaxVF); NumElts > 1; NumElts /= 2) {
+ for (int NumElts = getFloorFullVectorNumberOfElements(
+ *TTI, Loads.front()->getType(), MaxVF);
+ NumElts > 1; NumElts = getFloorFullVectorNumberOfElements(
+ *TTI, Loads.front()->getType(), NumElts - 1)) {
CandidateVFs.push_back(NumElts);
if (VectorizeNonPowerOf2 && NumElts > 2)
CandidateVFs.push_back(NumElts - 1);
@@ -6751,9 +6784,10 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
if (Final && NumElts > BestVF)
continue;
SmallVector<unsigned> MaskedGatherVectorized;
- for (unsigned Cnt = StartIdx, E = Loads.size(); Cnt + NumElts <= E;
+ for (unsigned Cnt = StartIdx, E = Loads.size(); Cnt < E;
++Cnt) {
- ArrayRef<LoadInst *> Slice = ArrayRef(Loads).slice(Cnt, NumElts);
+ ArrayRef<LoadInst *> Slice =
+ ArrayRef(Loads).slice(Cnt, std::min(NumElts, E - Cnt));
if (VectorizedLoads.count(Slice.front()) ||
VectorizedLoads.count(Slice.back()) ||
areKnownNonVectorizableLoads(Slice))
@@ -7099,24 +7133,27 @@ void BoUpSLP::tryToVectorizeGatheredLoads(
}
return NonVectorized;
};
- SmallVector<LoadInst *> NonVectorized = ProcessGatheredLoads(GatheredLoads);
- if (!GatheredLoads.empty() && !NonVectorized.empty() &&
- std::accumulate(
- GatheredLoads.begin(), GatheredLoads.end(), 0u,
- [](unsigned S, ArrayRef<std::pair<LoadInst *, int>> LoadsDists) {
- return S + LoadsDists.size();
- }) != NonVectorized.size() &&
- IsMaskedGatherSupported(NonVectorized)) {
- SmallVector<SmallVector<std::pair<LoadInst *, int>>> FinalGatheredLoads;
- for (LoadInst *LI : NonVectorized) {
- // Reinsert non-vectorized loads to other list of loads with the same
- // base pointers.
- gatherPossiblyVectorizableLoads(*this, LI, *DL, *SE, *TTI,
- FinalGatheredLoads,
- /*AddNew=*/false);
- }
- // Final attempt to vectorize non-vectorized loads.
- (void)ProcessGatheredLoads(FinalGatheredLoads, /*Final=*/true);
+ for (const auto &GLs : GatheredLoads) {
+ const auto &Ref = GLs.second;
+ SmallVector<LoadInst *> NonVectorized = ProcessGatheredLoads(Ref);
+ if (!Ref.empty() && !NonVectorized.empty() &&
+ std::accumulate(
+ Ref.begin(), Ref.end(), 0u,
+ [](unsigned S, ArrayRef<std::pair<LoadInst *, int>> LoadsDists) {
+ return S + LoadsDists.size();
+ }) != NonVectorized.size() &&
+ IsMaskedGatherSupported(NonVectorized)) {
+ SmallVector<SmallVector<std::pair<LoadInst *, int>>> FinalGatheredLoads;
+ for (LoadInst *LI : NonVectorized) {
+ // Reinsert non-vectorized loads to other list of loads with the same
+ // base pointers.
+ gatherPossiblyVectorizableLoads(*this, LI, *DL, *SE, *TTI,
+ FinalGatheredLoads,
+ /*AddNew=*/false);
+ }
+ // Final attempt to vectorize non-vectorized loads.
+ (void)ProcessGatheredLoads(FinalGatheredLoads, /*Final=*/true);
+ }
}
// Try to vectorize postponed load entries, previously marked as gathered.
for (unsigned Idx : LoadEntriesToVectorize) {
@@ -7363,13 +7400,6 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
OrdersType &CurrentOrder, SmallVectorImpl<Value *> &PointerOps) {
assert(S.MainOp && "Expected instructions with same/alternate opcodes only.");
- if (S.MainOp->getType()->isFloatingPointTy() &&
- TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
- auto *I = dyn_cast<Instruction>(V);
- return I && (I->isBinaryOp() || isa<CallInst>(I)) && !I->isFast();
- }))
- return TreeEntry::NeedToGather;
-
unsigned ShuffleOrOp =
S.isAltShuffle() ? (unsigned)Instruction::ShuffleVector : S.getOpcode();
auto *VL0 = cast<Instruction>(S.OpValue);
@@ -7534,6 +7564,12 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
case Instruction::Or:
case Instruction::Xor:
case Instruction::Freeze:
+ if (S.MainOp->getType()->isFloatingPointTy() &&
+ TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && I->isBinaryOp() && !I->isFast();
+ }))
+ return TreeEntry::NeedToGather;
return TreeEntry::Vectorize;
case Instruction::GetElementPtr: {
// We don't combine GEPs with complicated (nested) indexing.
@@ -7625,6 +7661,12 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState(
return TreeEntry::NeedToGather;
}
case Instruction::Call: {
+ if (S.MainOp->getType()->isFloatingPointTy() &&
+ TTI->isFPVectorizationPotentiallyUnsafe() && any_of(VL, [](Value *V) {
+ auto *I = dyn_cast<Instruction>(V);
+ return I && !I->isFast();
+ }))
+ return TreeEntry::NeedToGather;
// Check if the calls are all to the same vectorizable intrinsic or
// library function.
CallInst *CI = cast<CallInst>(VL0);
@@ -9344,8 +9386,13 @@ void BoUpSLP::transformNodes() {
// insertvector instructions.
unsigned StartIdx = 0;
unsigned End = VL.size();
- for (unsigned VF = VL.size() / 2; VF >= MinVF; VF = bit_ceil(VF) / 2) {
- SmallVector<unsigned> Slices;
+ for (unsigned VF = getFloorFullVectorNumberOfElements(
+ *TTI, VL.front()->getType(), VL.size() - 1);
+ VF >= MinVF; VF = getFloorFullVectorNumberOfElements(
+ *TTI, VL.front()->getType(), VF - 1)) {
+ if (StartIdx + VF > End)
+ continue;
+ SmallVector<std::pair<unsigned, unsigned>> Slices;
for (unsigned Cnt = StartIdx; Cnt + VF <= End; Cnt += VF) {
ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
// If any instruction is vectorized already - do not try again.
@@ -9375,7 +9422,10 @@ void BoUpSLP::transformNodes() {
if (IsSplat)
continue;
InstructionsState S = getSameOpcode(Slice, *TLI);
- if (!S.getOpcode() || S.isAltShuffle() || !allSameBlock(Slice))
+ if (!S.getOpcode() || S.isAltShuffle() || !allSameBlock(Slice) ||
+ (S.getOpcode() == Instruction::Load &&
+ areKnownNonVectorizableLoads(Slice)) ||
+ (S.getOpcode() != Instruction::Load && !has_single_bit(VF)))
continue;
if (VF == 2) {
// Try to vectorize reduced values or if all users are vectorized.
@@ -9395,8 +9445,16 @@ void BoUpSLP::transformNodes() {
canVectorizeLoads(Slice, Slice.front(), Order, PointerOps);
// Do not vectorize gathers.
if (Res == LoadsState::ScatterVectorize ||
- Res == LoadsState::Gather)
+ Res == LoadsState::Gather) {
+ if (Res == LoadsState::Gather) {
+ registerNonVectorizableLoads(Slice);
+ // If reductions and the scalars from the root node are
+ // analyzed - mark as non-vectorizable reduction.
+ if (UserIgnoreList && E.Idx == 0)
+ analyzedReductionVals(Slice);
+ }
continue;
+ }
} else if (S.getOpcode() == Instruction::ExtractElement ||
(TTI->getInstructionCost(
cast<Instruction>(Slice.front()), CostKind) <
@@ -9411,17 +9469,17 @@ void BoUpSLP::transformNodes() {
}
}
}
- Slices.emplace_back(Cnt);
+ Slices.emplace_back(Cnt, Slice.size());
}
- auto AddCombinedNode = [&](unsigned Idx, unsigned Cnt) {
+ auto AddCombinedNode = [&](unsigned Idx, unsigned Cnt, unsigned Sz) {
E.CombinedEntriesWithIndices.emplace_back(Idx, Cnt);
if (StartIdx == Cnt)
- StartIdx = Cnt + VF;
- if (End == Cnt + VF)
+ StartIdx = Cnt + Sz;
+ if (End == Cnt + Sz)
End = Cnt;
};
- for (unsigned Cnt : Slices) {
- ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
+ for (auto [Cnt, Sz] : Slices) {
+ ArrayRef<Value *> Slice = VL.slice(Cnt, Sz);
// If any instruction is vectorized already - do not try again.
if (TreeEntry *SE = getTreeEntry(Slice.front());
SE || getTreeEntry(Slice.back())) {
@@ -9430,7 +9488,7 @@ void BoUpSLP::transformNodes() {
if (VF != SE->getVectorFactor() || !SE->isSame(Slice))
continue;
SE->UserTreeIndices.emplace_back(&E, UINT_MAX);
- AddCombinedNode(SE->Idx, Cnt);
+ AddCombinedNode(SE->Idx, Cnt, Sz);
continue;
}
unsigned PrevSize = VectorizableTree.size();
@@ -9442,12 +9500,14 @@ void BoUpSLP::transformNodes() {
VectorizableTree[PrevSize]->getOpcode() !=
Instruction::ExtractElement &&
!isSplat(Slice)) {
+ if (UserIgnoreList && E.Idx == 0 && VF == 2)
+ analyzedReductionVals(Slice);
VectorizableTree.pop_back();
assert(PrevEntriesSize == LoadEntriesToVectorize.size() &&
"LoadEntriesToVectorize expected to remain the same");
continue;
}
- AddCombinedNode(PrevSize, Cnt);
+ AddCombinedNode(PrevSize, Cnt, Sz);
}
}
}
@@ -9542,11 +9602,24 @@ void BoUpSLP::transformNodes() {
VectorizableTree.front()->Scalars.size() == SmallVF) ||
(VectorizableTree.size() <= 2 && UserIgnoreList))
return;
+
+ if (VectorizableTree.front()->isNonPowOf2Vec() &&
+ getCanonicalGraphSize() != getTreeSize() && UserIgnoreList &&
+ getCanonicalGraphSize() <= SmallTree &&
+ count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
+ [](const std::unique_ptr<TreeEntry> &TE) {
+ return TE->isGather() &&
+ TE->getOpcode() == Instruction::Load &&
+ !allSameBlock(TE->Scalars);
+ }) == 1)
+ return;
}
// A list of loads to be gathered during the vectorization process. We can
// try to vectorize them at the end, if profitable.
- SmallVector<SmallVector<std::pair<LoadInst *, int>>> GatheredLoads;
+ SmallMapVector<std::tuple<BasicBlock *, Value *, Type *>,
+ SmallVector<SmallVector<std::pair<LoadInst *, int>>>, 8>
+ GatheredLoads;
for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
TreeEntry &E = *TE;
@@ -9558,9 +9631,21 @@ void BoUpSLP::transformNodes() {
!isVectorized(V) &&
!isDeleted(cast<Instruction>(V));
}))) &&
- !isSplat(E.Scalars))
- gatherPossiblyVectorizableLoads(*this, E.Scalars, *DL, *SE, *TTI,
- GatheredLoads);
+ !isSplat(E.Scalars)) {
+ for (Value *V : E.Scalars) {
+ auto *LI = dyn_cast<LoadInst>(V);
+ if (!LI)
+ continue;
+ if (isDeleted(LI) || isVectorized(LI) || !LI->isSimple())
+ continue;
+ gatherPossiblyVectorizableLoads(
+ *this, V, *DL, *SE, *TTI,
+ GatheredLoads[std::make_tuple(
+ LI->getParent(),
+ getUnderlyingObject(LI->getPointerOperand(), RecursionMaxDepth),
+ LI->getType())]);
+ }
+ }
}
// Try to vectorize gathered loads if this is not just a gather of loads.
if (!GatheredLoads.empty())
@@ -11515,6 +11600,34 @@ bool BoUpSLP::isTreeTinyAndNotFullyVectorizable(bool ForReduction) const {
return true;
}
+bool BoUpSLP::isTreeNotExtendable() const {
+ if (getCanonicalGraphSize() != getTreeSize()) {
+ constexpr unsigned SmallTree = 3;
+ if (VectorizableTree.front()->isNonPowOf2Vec() &&
+ getCanonicalGraphSize() <= SmallTree &&
+ count_if(ArrayRef(VectorizableTree).drop_front(getCanonicalGraphSize()),
+ [](const std::unique_ptr<TreeEntry> &TE) {
+ return TE->isGather() &&
+ TE->getOpcode() == Instruction::Load &&
+ !allSameBlock(TE->Scalars);
+ }) == 1)
+ return true;
+ return false;
+ }
+ bool Res = false;
+ for (unsigned Idx : seq<unsigned>(getTreeSize())) {
+ TreeEntry &E = *VectorizableTree[Idx];
+ if (!E.isGather())
+ continue;
+ if (E.getOpcode() && E.getOpcode() != Instruction::Load)
+ return false;
+ if (isSplat(E.Scalars) || allConstant(E.Scalars))
+ continue;
+ Res = true;
+ }
+ return Res;
+}
+
InstructionCost BoUpSLP::getSpillCost() const {
// Walk from the bottom of the tree to the top, tracking which values are
// live. When we see a call instruction that is not part of our tree,
@@ -18771,7 +18884,8 @@ class HorizontalReduction {
auto GenerateLoadsSubkey = [&](size_t Key, LoadInst *LI) {
Key = hash_combine(hash_value(LI->getParent()), Key);
- Value *Ptr = getUnderlyingObject(LI->getPointerOperand());
+ Value *Ptr =
+ getUnderlyingObject(LI->getPointerOperand(), RecursionMaxDepth);
if (!LoadKeyUsed.insert(Key).second) {
auto LIt = LoadsMap.find(std::make_pair(Key, Ptr));
if (LIt != LoadsMap.end()) {
@@ -19094,8 +19208,28 @@ class HorizontalReduction {
RegMaxNumber * RedValsMaxNumber);
unsigned ReduxWidth = NumReducedVals;
+ auto GetVectorFactor = [&, &TTI = *TTI](unsigned ReduxWidth) {
+ unsigned NumParts, NumRegs;
+ Type *ScalarTy = Candidates.front()->getType();
+ ReduxWidth =
+ getFloorFullVectorNumberOfElements(TTI, ScalarTy, ReduxWidth);
+ VectorType *Tp = getWidenedType(ScalarTy, ReduxWidth);
+ NumParts = TTI.getNumberOfParts(Tp);
+ NumRegs =
+ TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
+ while (NumParts > NumRegs) {
+ ReduxWidth = bit_floor(ReduxWidth - 1);
+ VectorType *Tp = getWidenedType(ScalarTy, ReduxWidth);
+ NumParts = TTI.getNumberOfParts(Tp);
+ NumRegs =
+ TTI.getNumberOfRegisters(TTI.getRegisterClassForType(true, Tp));
+ }
+ if (NumParts > NumRegs / 2)
+ ReduxWidth = bit_floor(ReduxWidth);
+ return ReduxWidth;
+ };
if (!VectorizeNonPowerOf2 || !has_single_bit(ReduxWidth + 1))
- ReduxWidth = bit_floor(ReduxWidth);
+ ReduxWidth = GetVectorFactor(ReduxWidth);
ReduxWidth = std::min(ReduxWidth, MaxElts);
unsigned Start = 0;
@@ -19103,10 +19237,7 @@ class HorizontalReduction {
// Restarts vectorization attempt with lower vector factor.
unsigned PrevReduxWidth = ReduxWidth;
bool CheckForReusedReductionOpsLocal = false;
- auto &&AdjustReducedVals = [&Pos, &Start, &ReduxWidth, NumReducedVals,
- &CheckForReusedReductionOpsLocal,
- &PrevReduxWidth, &V,
- &IgnoreList](bool IgnoreVL = false) {
+ auto AdjustReducedVals = [&](bool IgnoreVL = false) {
bool IsAnyRedOpGathered = !IgnoreVL && V.isAnyGathered(IgnoreList);
if (!CheckForReusedReductionOpsLocal && PrevReduxWidth == ReduxWidth) {
// Check if any of the reduction ops are gathered. If so, worth
@@ -19117,10 +19248,13 @@ class HorizontalReduction {
if (Pos < NumReducedVals - ReduxWidth + 1)
return IsAnyRedOpGathered;
Pos = Start;
- ReduxWidth = bit_ceil(ReduxWidth) / 2;
+ --ReduxWidth;
+ if (ReduxWidth > 1)
+ ReduxWidth = GetVectorFactor(ReduxWidth);
return IsAnyRedOpGathered;
};
bool AnyVectorized = false;
+ SmallDenseSet<std::pair<unsigned, unsigned>, 8> IgnoredCandidates;
while (Pos < NumReducedVals - ReduxWidth + 1 &&
ReduxWidth >= ReductionLimit) {
// Dependency in tree of the reduction ops - drop this attempt, try
@@ -19132,8 +19266,15 @@ class HorizontalReduction {
}
PrevReduxWidth = ReduxWidth;
ArrayRef<Value *> VL(std::next(Candidates.begin(), Pos), ReduxWidth);
- // Beeing analyzed already - skip.
- if (V.areAnalyzedReductionVals(VL)) {
+ // Been analyzed already - skip.
+ if (IgnoredCandidates.contains(std::make_pair(Pos, ReduxWidth)) ||
+ (!has_single_bit(ReduxWidth) &&
+ (IgnoredCandidates.contains(
+ std::make_pair(Pos, bit_floor(ReduxWidth))) ||
+ IgnoredCandidates.contains(
+ std::make_pair(Pos + (ReduxWidth - bit_floor(ReduxWidth)),
+ bit_floor(ReduxWidth))))) ||
+ V.areAnalyzedReductionVals(VL)) {
(void)AdjustReducedVals(/*IgnoreVL=*/true);
continue;
}
@@ -19239,8 +19380,24 @@ class HorizontalReduction {
<< " and threshold "
<< ore::NV("Threshold", -SLPCostThreshold);
});
- if (!AdjustReducedVals())
+ if (!AdjustReducedVals()) {
V.analyzedReductionVals(VL);
+ unsigned Offset = Pos == Start ? Pos : Pos - 1;
+ if (ReduxWidth > ReductionLimit && V.isTreeNotExtendable()) {
+ // Add subvectors of VL to the list of the analyzed values.
+ for (unsigned VF = getFloorFullVectorNumberOfElements(
+ *TTI, VL.front()->getType(), ReduxWidth - 1);
+ VF >= ReductionLimit;
+ VF = getFloorFullVectorNumberOfElements(
+ *TTI, VL.front()->getType(), VF - 1)) {
+ if (has_single_bit(VF) &&
+ V.getCanonicalGraphSize() != V.getTreeSize())
+ continue;
+ for (unsigned Idx : seq<unsigned>(ReduxWidth - VF))
+ IgnoredCandidates.insert(std::make_pair(Offset + Idx, VF));
+ }
+ }
+ }
continue;
}
@@ -19349,7 +19506,9 @@ class HorizontalReduction {
}
Pos += ReduxWidth;
Start = Pos;
- ReduxWidth = llvm::bit_floor(NumReducedVals - Pos);
+ ReduxWidth = NumReducedVals - Pos;
+ if (ReduxWidth > 1)
+ ReduxWidth = GetVectorFactor(NumReducedVals - Pos);
AnyVectorized = true;
}
if (OptReusedScalars && !AnyVectorized) {
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
index fffa626cae0dda..c431b058f0d2d5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/tsc-s116.ll
@@ -17,18 +17,17 @@
define void @s116_modified(ptr %a) {
; CHECK-LABEL: @s116_modified(
-; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 1
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 3
+; CHECK-NEXT: [[A:%.*]] = getelementptr inbounds float, ptr [[GEP1:%.*]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr [[GEP1]], i64 3
; CHECK-NEXT: [[LD0:%.*]] = load float, ptr [[A]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[GEP1]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[GEP3]], align 4
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[LD0]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[LD0]], i32 1
; CHECK-NEXT: [[TMP6:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP5]], <2 x float> [[TMP2]], i64 2)
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[TMP6]], <4 x i32> <i32 1, i32 1, i32 5, i32 6>
; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <4 x float> [[TMP6]], [[TMP7]]
-; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[A]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP8]], ptr [[GEP1]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds float, ptr %a, i64 1
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
index 833bc56c4ec6b6..2191d04cd797d4 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll
@@ -7,8 +7,7 @@ define void @vec3_vectorize_call(ptr %Colour, float %0) {
; NON-POWER-OF-2-NEXT: entry:
; NON-POWER-OF-2-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4
; NON-POWER-OF-2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0:%.*]], i32 2
-; NON-POWER-OF-2-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
-; NON-POWER-OF-2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> [[TMP3]], <3 x i32> <i32 3, i32 4, i32 2>
+; NON-POWER-OF-2-NEXT: [[TMP4:%.*]] = call <3 x float> @llvm.vector.insert.v3f32.v2f32(<3 x float> [[TMP2]], <2 x float> [[TMP1]], i64 0)
; NON-POWER-OF-2-NEXT: [[TMP5:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP4]], <3 x float> zeroinitializer, <3 x float> zeroinitializer)
; NON-POWER-OF-2-NEXT: store <3 x float> [[TMP5]], ptr [[COLOUR]], align 4
; NON-POWER-OF-2-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll b/llvm/test/Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll
index 757d0b1708b6fb..234b658032383e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gather-node-same-as-vect-but-order.ll
@@ -11,19 +11,21 @@ define void @foo(ptr %i7, i32 %0, i1 %tobool62.not) {
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = sitofp <2 x i32> [[TMP3]] to <2 x float>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
; CHECK-NEXT: [[Y0:%.*]] = getelementptr i8, ptr [[RC21]], i64 8
; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[Y0]], align 4
; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[I7]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = load <2 x float>, ptr [[RC21]], align 4
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> poison, float [[TMP6]], i32 2
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP10]], float [[TMP7]], i32 3
-; CHECK-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP11]], <2 x float> [[TMP8]], i64 0)
+; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> poison, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i32 2
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP10]], <2 x float> [[TMP11]], i64 0)
; CHECK-NEXT: [[TMP12:%.*]] = fcmp olt <4 x float> [[TMP13]], zeroinitializer
; CHECK-NEXT: [[TMP14:%.*]] = fcmp olt <4 x float> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = select <4 x i1> [[TMP14]], <4 x float> [[TMP5]], <4 x float> zeroinitializer
; CHECK-NEXT: [[TMP16:%.*]] = select <4 x i1> [[TMP12]], <4 x float> zeroinitializer, <4 x float> [[TMP15]]
-; CHECK-NEXT: store <4 x float> [[TMP16]], ptr [[RC21]], align 4
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x float> [[TMP16]], <4 x float> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: store <4 x float> [[TMP27]], ptr [[RC21]], align 4
; CHECK-NEXT: br label [[IF_END:%.*]]
; CHECK: entry.if.end72_crit_edge:
; CHECK-NEXT: br label [[IF_END72:%.*]]
@@ -46,8 +48,7 @@ define void @foo(ptr %i7, i32 %0, i1 %tobool62.not) {
; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP25:%.*]] = mul <4 x i32> [[TMP23]], [[TMP24]]
; CHECK-NEXT: [[TMP26:%.*]] = sitofp <4 x i32> [[TMP25]] to <4 x float>
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x float> [[TMP26]], <4 x float> poison, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-; CHECK-NEXT: store <4 x float> [[TMP27]], ptr [[RC21]], align 4
+; CHECK-NEXT: store <4 x float> [[TMP26]], ptr [[RC21]], align 4
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 72e29839230e81..c9ff2d6426d2b6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -318,22 +318,14 @@ entry:
define float @f(ptr nocapture readonly %x) {
; CHECK-LABEL: @f(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = load <32 x float>, ptr [[X:%.*]], align 4
-; CHECK-NEXT: [[ARRAYIDX_32:%.*]] = getelementptr inbounds float, ptr [[X]], i64 32
-; CHECK-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr [[ARRAYIDX_32]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> [[TMP0]])
-; CHECK-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP1]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <48 x float>, ptr [[X:%.*]], align 4
+; CHECK-NEXT: [[OP_RDX:%.*]] = call fast float @llvm.vector.reduce.fadd.v48f32(float 0.000000e+00, <48 x float> [[TMP0]])
; CHECK-NEXT: ret float [[OP_RDX]]
;
; THRESHOLD-LABEL: @f(
; THRESHOLD-NEXT: entry:
-; THRESHOLD-NEXT: [[TMP0:%.*]] = load <32 x float>, ptr [[X:%.*]], align 4
-; THRESHOLD-NEXT: [[ARRAYIDX_32:%.*]] = getelementptr inbounds float, ptr [[X]], i64 32
-; THRESHOLD-NEXT: [[TMP1:%.*]] = load <16 x float>, ptr [[ARRAYIDX_32]], align 4
-; THRESHOLD-NEXT: [[TMP2:%.*]] = call fast float @llvm.vector.reduce.fadd.v32f32(float 0.000000e+00, <32 x float> [[TMP0]])
-; THRESHOLD-NEXT: [[TMP3:%.*]] = call fast float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP1]])
-; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP2]], [[TMP3]]
+; THRESHOLD-NEXT: [[TMP0:%.*]] = load <48 x float>, ptr [[X:%.*]], align 4
+; THRESHOLD-NEXT: [[OP_RDX:%.*]] = call fast float @llvm.vector.reduce.fadd.v48f32(float 0.000000e+00, <48 x float> [[TMP0]])
; THRESHOLD-NEXT: ret float [[OP_RDX]]
;
entry:
@@ -606,18 +598,14 @@ define float @loadadd31(ptr nocapture readonly %x) {
; CHECK-LABEL: @loadadd31(
; CHECK-NEXT: entry:
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X:%.*]], i64 1
-; CHECK-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[ARRAYIDX]], align 4
-; CHECK-NEXT: [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, ptr [[X]], i64 17
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr [[ARRAYIDX_16]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <24 x float>, ptr [[ARRAYIDX]], align 4
; CHECK-NEXT: [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, ptr [[X]], i64 25
; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[ARRAYIDX_24]], align 4
; CHECK-NEXT: [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, ptr [[X]], i64 29
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[ARRAYIDX_28]], align 4
; CHECK-NEXT: [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, ptr [[X]], i64 30
; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[ARRAYIDX_29]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = call fast float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP0]])
-; CHECK-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP1]])
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[OP_RDX:%.*]] = call fast float @llvm.vector.reduce.fadd.v24f32(float 0.000000e+00, <24 x float> [[TMP0]])
; CHECK-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP2]])
; CHECK-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], [[TMP7]]
; CHECK-NEXT: [[OP_RDX2:%.*]] = fadd fast float [[OP_RDX1]], [[TMP3]]
@@ -627,18 +615,14 @@ define float @loadadd31(ptr nocapture readonly %x) {
; THRESHOLD-LABEL: @loadadd31(
; THRESHOLD-NEXT: entry:
; THRESHOLD-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[X:%.*]], i64 1
-; THRESHOLD-NEXT: [[TMP0:%.*]] = load <16 x float>, ptr [[ARRAYIDX]], align 4
-; THRESHOLD-NEXT: [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, ptr [[X]], i64 17
-; THRESHOLD-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr [[ARRAYIDX_16]], align 4
+; THRESHOLD-NEXT: [[TMP0:%.*]] = load <24 x float>, ptr [[ARRAYIDX]], align 4
; THRESHOLD-NEXT: [[ARRAYIDX_24:%.*]] = getelementptr inbounds float, ptr [[X]], i64 25
; THRESHOLD-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[ARRAYIDX_24]], align 4
; THRESHOLD-NEXT: [[ARRAYIDX_28:%.*]] = getelementptr inbounds float, ptr [[X]], i64 29
; THRESHOLD-NEXT: [[TMP3:%.*]] = load float, ptr [[ARRAYIDX_28]], align 4
; THRESHOLD-NEXT: [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, ptr [[X]], i64 30
; THRESHOLD-NEXT: [[TMP4:%.*]] = load float, ptr [[ARRAYIDX_29]], align 4
-; THRESHOLD-NEXT: [[TMP5:%.*]] = call fast float @llvm.vector.reduce.fadd.v16f32(float 0.000000e+00, <16 x float> [[TMP0]])
-; THRESHOLD-NEXT: [[TMP6:%.*]] = call fast float @llvm.vector.reduce.fadd.v8f32(float 0.000000e+00, <8 x float> [[TMP1]])
-; THRESHOLD-NEXT: [[OP_RDX:%.*]] = fadd fast float [[TMP5]], [[TMP6]]
+; THRESHOLD-NEXT: [[OP_RDX:%.*]] = call fast float @llvm.vector.reduce.fadd.v24f32(float 0.000000e+00, <24 x float> [[TMP0]])
; THRESHOLD-NEXT: [[TMP7:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float 0.000000e+00, <4 x float> [[TMP2]])
; THRESHOLD-NEXT: [[OP_RDX1:%.*]] = fadd fast float [[OP_RDX]], [[TMP7]]
; THRESHOLD-NEXT: [[OP_RDX2:%.*]] = fadd fast float [[OP_RDX1]], [[TMP3]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index a7201e776fb4cd..0bc91d42b0f132 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -1013,11 +1013,11 @@ define i32 @maxi8_wrong_parent(i32) {
; THRESH-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr @arr, align 16
; THRESH-NEXT: br label [[PP:%.*]]
; THRESH: pp:
-; THRESH-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
-; THRESH-NEXT: [[TMP4:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 4), align 16
-; THRESH-NEXT: [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> poison, <4 x i32> [[TMP4]], i64 4)
-; THRESH-NEXT: [[TMP6:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP5]], <2 x i32> [[TMP2]], i64 0)
-; THRESH-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP6]], <2 x i32> [[TMP3]], i64 2)
+; THRESH-NEXT: [[TMP3:%.*]] = load <4 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 2), align 8
+; THRESH-NEXT: [[TMP4:%.*]] = load <2 x i32>, ptr getelementptr inbounds ([32 x i32], ptr @arr, i64 0, i64 6), align 8
+; THRESH-NEXT: [[TMP5:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v4i32(<8 x i32> poison, <4 x i32> [[TMP3]], i64 0)
+; THRESH-NEXT: [[TMP6:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP5]], <2 x i32> [[TMP4]], i64 4)
+; THRESH-NEXT: [[TMP7:%.*]] = call <8 x i32> @llvm.vector.insert.v8i32.v2i32(<8 x i32> [[TMP6]], <2 x i32> [[TMP2]], i64 6)
; THRESH-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.smax.v8i32(<8 x i32> [[TMP7]])
; THRESH-NEXT: ret i32 [[TMP8]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
index 47dd84c7f6e9cc..4898111960c0fd 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-order-detection.ll
@@ -7,10 +7,11 @@ define void @e(ptr %c, i64 %0) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP1:%.*]] = load ptr, ptr [[C]], align 8
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr i8, ptr [[TMP1]], i64 96
-; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr i8, ptr [[TMP1]], i64 112
-; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX1]], align 8
+; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr i8, ptr [[TMP1]], i64 104
+; CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = load ptr, ptr [[C]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x ptr>, ptr [[ARRAYIDX]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = load <2 x ptr>, ptr [[ARRAYIDX5]], align 8
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x ptr> [[TMP18]], <2 x ptr> poison, <2 x i32> <i32 1, i32 0>
; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> poison, ptr [[TMP3]], i32 0
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x ptr> [[TMP5]], <2 x ptr> poison, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <6 x ptr> poison, ptr [[TMP2]], i32 2
@@ -18,7 +19,7 @@ define void @e(ptr %c, i64 %0) {
; CHECK-NEXT: [[TMP9:%.*]] = call <6 x ptr> @llvm.vector.insert.v6p0.v2p0(<6 x ptr> [[TMP8]], <2 x ptr> [[TMP4]], i64 0)
; CHECK-NEXT: [[TMP10:%.*]] = call <6 x ptr> @llvm.vector.insert.v6p0.v2p0(<6 x ptr> [[TMP9]], <2 x ptr> [[TMP6]], i64 4)
; CHECK-NEXT: [[TMP11:%.*]] = ptrtoint <6 x ptr> [[TMP10]] to <6 x i64>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <6 x i64> [[TMP11]], <6 x i64> poison, <32 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <6 x i64> [[TMP11]], <6 x i64> poison, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3, i32 3, i32 4, i32 4, i32 4, i32 4, i32 5, i32 5, i32 5>
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <32 x i64> poison, i64 [[TMP0]], i32 0
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <32 x i64> [[TMP13]], <32 x i64> poison, <32 x i32> zeroinitializer
; CHECK-NEXT: [[TMP15:%.*]] = or <32 x i64> [[TMP14]], [[TMP12]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
index fd3c1a57aff344..a821362a883a11 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll
@@ -7,8 +7,7 @@ define void @vec3_vectorize_call(ptr %Colour, float %0) {
; NON-POW2-NEXT: entry:
; NON-POW2-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4
; NON-POW2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0:%.*]], i32 2
-; NON-POW2-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <3 x i32> <i32 0, i32 1, i32 poison>
-; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> [[TMP3]], <3 x i32> <i32 3, i32 4, i32 2>
+; NON-POW2-NEXT: [[TMP4:%.*]] = call <3 x float> @llvm.vector.insert.v3f32.v2f32(<3 x float> [[TMP2]], <2 x float> [[TMP1]], i64 0)
; NON-POW2-NEXT: [[TMP5:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP4]], <3 x float> zeroinitializer, <3 x float> zeroinitializer)
; NON-POW2-NEXT: store <3 x float> [[TMP5]], ptr [[COLOUR]], align 4
; NON-POW2-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
index e1b091cc6fcda7..9719e60a6a695d 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vect-gather-same-nodes.ll
@@ -8,18 +8,18 @@ define void @test(ptr %a, ptr %b) {
; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr null, align 4
; CHECK-NEXT: [[ARRAYIDX120:%.*]] = getelementptr [4 x float], ptr [[B:%.*]], i64 0, i64 3
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[ARRAYIDX120]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP2]], float [[TMP0]], i32 2
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body:
; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr null, align 4
; CHECK-NEXT: [[TMP4:%.*]] = load <2 x float>, ptr [[A:%.*]], align 4
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP2]], float [[TMP3]], i32 2
-; CHECK-NEXT: [[TMP7:%.*]] = call <4 x float> @llvm.vector.insert.v4f32.v2f32(<4 x float> [[TMP6]], <2 x float> [[TMP1]], i64 0)
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP3]], i32 1
; CHECK-NEXT: [[TMP8:%.*]] = fmul <4 x float> [[TMP5]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x float> [[TMP8]], <4 x float> poison, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
-; CHECK-NEXT: [[TMP10:%.*]] = fmul <4 x float> [[TMP5]], zeroinitializer
-; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x float> [[TMP9]], [[TMP10]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <4 x float> [[TMP9]], zeroinitializer
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <4 x float> [[TMP8]], [[TMP10]]
; CHECK-NEXT: [[TMP12:%.*]] = fadd <4 x float> [[TMP11]], zeroinitializer
; CHECK-NEXT: store <4 x float> [[TMP12]], ptr [[RESULT]], align 4
; CHECK-NEXT: br label [[FOR_BODY]]
More information about the llvm-commits
mailing list