[llvm] [SLP] Refine loop-aware gather cost and admit sibling-loop subtrees (PR #192801)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Apr 18 10:36:21 PDT 2026
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-systemz
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Factor the loop-nest walk out of getScaleToLoopIterations() into a pure
getLoopNestScale(). Reused by a new getGatherNodeEffectiveScale()
(-slp-per-lane-gather-scale, on by default) that averages per-lane
scales so LICM-hoistable gather operands no longer pay the inner loop's
trip count. Also, relax the tree-builder's loop-nest guard to admit
sibling inner loops sharing a common outer loop only when SCEV proves
equal backedge-taken counts.
---
Patch is 77.11 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/192801.diff
12 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+174-47)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll (+98-126)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll (+7-4)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/slp-fma-loss.ll (+9-10)
- (modified) llvm/test/Transforms/SLPVectorizer/SystemZ/vec-elt-insertion.ll (+1-1)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/buildvectors-parent-phi-nodes.ll (+6-6)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/cse.ll (+2-5)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/deleted-instructions-clear.ll (+9-8)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/deleted-node-with-copyable-operands.ll (+12-14)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/parent-node-schedulable-with-multi-copyables.ll (+27-27)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll (+20-23)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/reduced-val-extracted-and-externally-used.ll (+13-22)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 6625384616c26..41b1464f07f94 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -243,6 +243,18 @@ static cl::opt<unsigned> LoopAwareTripCount(
cl::desc("Loop trip count, considered by the cost model during "
"modeling (0=loops are ignored and considered flat code)"));
+/// Refine the loop-aware cost scaling of gather/buildvector tree entries by
+/// using the per-lane execution scale of the operand that feeds each lane,
+/// instead of a single whole-entry scale. This matches the LICM hoisting
+/// performed by optimizeGatherSequence() at codegen time: lanes whose
+/// operands are loop-invariant in an inner loop contribute the outer loop's
+/// execution scale rather than the inner loop's, which avoids over-costing
+/// buildvectors that bridge values from outer loop nests into an inner loop.
+static cl::opt<bool> PerLaneGatherScale(
+ "slp-per-lane-gather-scale", cl::init(true), cl::Hidden,
+ cl::desc("Use per-lane execution scale for gather/buildvector tree "
+ "entries to model LICM-hoistable buildvector sequences."));
+
// Limit the number of alias checks. The limit is chosen so that
// it has no negative effect on the llvm benchmarks.
static const unsigned AliasedCheckLimit = 10;
@@ -3890,10 +3902,27 @@ class slpvectorizer::BoUpSLP {
/// external use.
/// \p U is the user of the vectorized value from the entry, if using the
/// parent for the external use.
- unsigned getScaleToLoopIterations(const TreeEntry &TE,
+ uint64_t getScaleToLoopIterations(const TreeEntry &TE,
Value *Scalar = nullptr,
Instruction *U = nullptr);
+ /// \returns the product of trip counts of the loop \p L and all of its
+ /// enclosing loops. Unlike the state kept by getScaleToLoopIterations(),
+ /// this helper depends only on the loop structure and is independent of
+ /// per-entry operand invariance. Returns 1 when loop-aware cost modeling
+ /// is disabled or \p L is null.
+ uint64_t getLoopNestScale(const Loop *L);
+
+ /// \returns a refined execution scale for a gather/buildvector tree entry
+ /// \p TE. The scale is computed as the average of per-lane execution
+ /// scales: each lane's scale is the loop-nest scale of the loop that
+ /// contains the lane's defining instruction (or 1 if the lane is a
+ /// constant / loop-invariant non-instruction value). This models the
+ /// LICM hoisting that optimizeGatherSequence() performs after vectorization
+ /// for inserts with loop-invariant operands. Falls back to the whole-entry
+ /// scale when per-lane information is unavailable or the feature is off.
+ uint64_t getGatherNodeEffectiveScale(const TreeEntry &TE);
+
/// Get the loop nest for the given loop \p L.
ArrayRef<const Loop *> getLoopNest(const Loop *L);
@@ -4867,9 +4896,10 @@ class slpvectorizer::BoUpSLP {
/// Maps the loops to their loop nests.
SmallDenseMap<const Loop *, SmallVector<const Loop *>> LoopToLoopNest;
- /// Maps the loops to their scale factor, which is built as a multiplication
- /// of the tripcounts of the loops in the loop nest.
- SmallDenseMap<const Loop *, unsigned> LoopToScaleFactor;
+ /// Per-loop cache of nest scale factors: the product of trip counts of the
+ /// loop and all of its ancestors. Shared by getLoopNestScale() and (via it)
+ /// by getScaleToLoopIterations() and getGatherNodeEffectiveScale().
+ SmallDenseMap<const Loop *, uint64_t> LoopNestScaleCache;
/// This POD struct describes one external user in the vectorized tree.
struct ExternalUser {
@@ -12431,21 +12461,61 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
S.getMainOp()->getParent()) {
BasicBlock *Parent = S.getMainOp()->getParent();
if (const Loop *L = LI->getLoopFor(Parent)) {
- // Check that the new loop nest is not involved.
- // Otherwise, mark it as a gather node.
+ // Check that the new loop nest shares the same outer structure as the
+ // tree's current loop nest. Completely disjoint nests (different
+ // outermost loops) are forced to gather because their scales cannot be
+ // meaningfully combined. Sibling inner loops inside a common outer
+ // loop are allowed: the cost model scales each entry by its own loop
+ // via getScaleToLoopIterations(), so a tree that spans sibling inner
+ // loops (e.g. a PHI at their merge block) can still be costed
+ // correctly. Contract CurrentLoopNest to the longest common prefix
+ // with the new entry's nest so subsequent entries in yet another
+ // sibling can also be admitted.
L = findInnermostNonInvariantLoop(L, VL);
if (L) {
SmallVector<const Loop *> NewLoopNest(getLoopNest(L));
+ unsigned CommonLen = 0;
for (const auto [L1, L2] : zip(CurrentLoopNest, NewLoopNest)) {
- if (L1 != L2) {
- LLVM_DEBUG(dbgs() << "SLP: Different loop nest.\n");
+ if (L1 != L2)
+ break;
+ ++CommonLen;
+ }
+ if (!CurrentLoopNest.empty() && CommonLen == 0) {
+ // No shared outer loop at all.
+ LLVM_DEBUG(dbgs() << "SLP: Different loop nest.\n");
+ newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
+ return;
+ }
+ if (CurrentLoopNest.empty()) {
+ CurrentLoopNest.assign(NewLoopNest);
+ } else if (CommonLen < CurrentLoopNest.size() &&
+ CommonLen < NewLoopNest.size()) {
+ // Divergence below the common prefix: the tree now spans sibling
+ // loops at depth CommonLen. Admitting them into one tree makes
+ // the profitability decision JOINT across both siblings, so a
+ // very hot sibling could otherwise let an unprofitable cold
+ // sibling ride along "for free" (per-entry scaling of the cold
+ // sibling's entries would be dwarfed by the hot one). Require
+ // SCEV-proven equal backedge-taken counts for the diverging
+ // siblings before joining; otherwise force gather.
+ const Loop *SibA = CurrentLoopNest[CommonLen];
+ const Loop *SibB = NewLoopNest[CommonLen];
+ const SCEV *BecA = SE->getBackedgeTakenCount(SibA);
+ const SCEV *BecB = SE->getBackedgeTakenCount(SibB);
+ if (isa<SCEVCouldNotCompute>(BecA) || BecA != BecB) {
+ LLVM_DEBUG(dbgs()
+ << "SLP: Sibling loops have different trip counts.\n");
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
return;
}
+ CurrentLoopNest.truncate(CommonLen);
+ } else if (NewLoopNest.size() > CurrentLoopNest.size()) {
+ // New entry lives deeper in the same nest chain; extend.
+ CurrentLoopNest.append(
+ std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
+ NewLoopNest.end());
}
- if (NewLoopNest.size() > CurrentLoopNest.size())
- CurrentLoopNest.append(std::next(NewLoopNest.begin(), CurrentLoopNest.size()),
- NewLoopNest.end());
+ // Otherwise NewLoopNest is a prefix of CurrentLoopNest: keep as-is.
}
}
}
@@ -15841,7 +15911,7 @@ static unsigned getLoopTripCount(const Loop *L, ScalarEvolution &SE) {
return LoopAwareTripCount;
}
-unsigned BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
+uint64_t BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
Instruction *U) {
BasicBlock *Parent = nullptr;
if (U) {
@@ -15868,31 +15938,77 @@ unsigned BoUpSLP::getScaleToLoopIterations(const TreeEntry &TE, Value *Scalar,
} else {
Parent = TE.getMainOp()->getParent();
}
- if (const Loop *L = LI->getLoopFor(Parent)) {
- const auto It = LoopToScaleFactor.find(L);
- if (It != LoopToScaleFactor.end())
- return It->second;
- unsigned Scale = 1;
- if (const Loop *NonInvL = findInnermostNonInvariantLoop(
- L, Scalar ? ArrayRef(Scalar) : ArrayRef(TE.Scalars))) {
- Scale = getLoopTripCount(NonInvL, *SE);
- for (const Loop *LN : getLoopNest(NonInvL)) {
- if (LN == L)
- break;
- auto LNRes = LoopToScaleFactor.try_emplace(LN, 0);
- auto &LoopScale = LNRes.first->getSecond();
- if (!LNRes.second) {
- Scale *= LoopScale;
- break;
- }
- Scale *= getLoopTripCount(LN, *SE);
- LoopScale = Scale;
- }
- }
- LoopToScaleFactor.try_emplace(L, Scale);
- return Scale;
- }
- return 1;
+ const Loop *L = LI->getLoopFor(Parent);
+ if (!L)
+ return 1;
+ // The entry's cost is paid once per execution of the innermost loop in
+ // which some of its operands are variant. Operands that are invariant in
+ // all enclosing loops are executed once (LICM will hoist them out).
+ return getLoopNestScale(findInnermostNonInvariantLoop(
+ L, Scalar ? ArrayRef(Scalar) : ArrayRef(TE.Scalars)));
+}
+
+uint64_t BoUpSLP::getLoopNestScale(const Loop *L) {
+ if (!L || LoopAwareTripCount == 0)
+ return 1;
+ if (auto It = LoopNestScaleCache.find(L); It != LoopNestScaleCache.end())
+ return It->second;
+ // Collect loops from L outward up to (but not including) the first cached
+ // ancestor or the function top, then walk back inward multiplying trip
+ // counts. Use uint64_t to avoid silent overflow on deep/large nests.
+ SmallVector<const Loop *> Chain;
+ for (const Loop *Cur = L; Cur; Cur = Cur->getParentLoop()) {
+ if (LoopNestScaleCache.contains(Cur))
+ break;
+ Chain.push_back(Cur);
+ }
+ assert(!Chain.empty() && "Early-return above should have handled cache hit.");
+ uint64_t Scale = 1;
+ if (const Loop *Parent = Chain.back()->getParentLoop())
+ Scale = LoopNestScaleCache.lookup(Parent);
+ // Walk from the outermost uncached loop inward, accumulating trip counts.
+ // Use SaturatingMultiply to clamp at uint64_t max on deep/large nests
+ // rather than wrapping around.
+ for (const Loop *Cur : reverse(Chain)) {
+ uint64_t TC = std::max<uint64_t>(1, getLoopTripCount(Cur, *SE));
+ Scale = SaturatingMultiply(Scale, TC);
+ LoopNestScaleCache.try_emplace(Cur, std::max<uint64_t>(1, Scale));
+ }
+ return std::max<uint64_t>(1, Scale);
+}
+
+uint64_t BoUpSLP::getGatherNodeEffectiveScale(const TreeEntry &TE) {
+ // Only meaningful for gather/buildvector-like entries; the per-lane
+ // insertelements that make up such an entry are LICM-hoistable by
+ // optimizeGatherSequence() when their operand is loop-invariant.
+ assert((TE.isGather() || TE.State == TreeEntry::SplitVectorize) &&
+ "Expected gather/split tree entry.");
+
+ uint64_t BaseScale = getScaleToLoopIterations(TE);
+ if (!PerLaneGatherScale || LoopAwareTripCount == 0 || BaseScale <= 1)
+ return BaseScale;
+
+ // Average the per-lane execution scales: for each lane, reuse the same
+ // scale helper the rest of the cost model uses, but ask it about that
+ // one lane's value. Lanes that are loop-invariant in the current nest
+ // collapse to their outer-loop scale (or 1 for fully invariant/constant
+ // lanes), which matches the LICM hoisting performed by
+ // optimizeGatherSequence(). Cap per-lane contributions by BaseScale so a
+ // refinement can never raise the cost above the whole-entry scale.
+ uint64_t Sum = 0;
+ unsigned N = 0;
+ for (Value *V : TE.Scalars) {
+ if (isConstant(V))
+ continue;
+ ++N;
+ uint64_t LaneScale = getScaleToLoopIterations(TE, V);
+ Sum += std::min(LaneScale, BaseScale);
+ }
+ if (N == 0)
+ return BaseScale;
+ // Ceil-divide so we never round the effective scale down below 1.
+ uint64_t Avg = (Sum + N - 1) / N;
+ return std::clamp<uint64_t>(Avg, 1, BaseScale);
}
InstructionCost
@@ -17641,7 +17757,7 @@ InstructionCost BoUpSLP::getSpillCost() {
if (It != MinBWs.end())
ScalarTy = IntegerType::get(ScalarTy->getContext(), It->second.first);
auto *VecTy = getWidenedType(ScalarTy, Op->getVectorFactor());
- unsigned Scale = getScaleToLoopIterations(*Op);
+ uint64_t Scale = getScaleToLoopIterations(*Op);
InstructionCost KeepLiveCost = TTI->getCostOfKeepingLiveOverCall(VecTy);
KeepLiveCost *= Scale;
Cost += KeepLiveCost;
@@ -18008,8 +18124,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
};
constexpr TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
InstructionCost Cost = 0;
- SmallDenseMap<const TreeEntry *, unsigned> EntryToScale;
- unsigned PrevScale = 0;
+ SmallDenseMap<const TreeEntry *, uint64_t> EntryToScale;
+ uint64_t PrevScale = 0;
BasicBlock *PrevVecParent = nullptr;
for (const std::unique_ptr<TreeEntry> &Ptr : VectorizableTree) {
TreeEntry &TE = *Ptr;
@@ -18044,8 +18160,14 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
"Expected gather nodes with users only.");
InstructionCost C = getEntryCost(&TE, VectorizedVals, CheckedExtracts);
- unsigned Scale = 0;
+ uint64_t Scale = 0;
bool CostIsFree = C == 0;
+ // For gather/buildvector (and split-vectorize) entries, prefer the
+ // per-lane refined scale that accounts for LICM-hoistable insertelements
+ // when an operand is invariant in the current loop nest but defined in
+ // an outer loop. This prevents over-costing cross-loop-nest buildvectors.
+ const bool IsGatherLike =
+ TE.isGather() || TE.State == TreeEntry::SplitVectorize;
if (!CostIsFree && !TE.isGather() && TE.hasState()) {
if (PrevVecParent == TE.getMainOp()->getParent()) {
Scale = PrevScale;
@@ -18054,7 +18176,8 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
}
}
if (!CostIsFree && !Scale) {
- Scale = getScaleToLoopIterations(TE);
+ Scale = IsGatherLike ? getGatherNodeEffectiveScale(TE)
+ : getScaleToLoopIterations(TE);
C *= Scale;
EntryToScale.try_emplace(&TE, Scale);
if (!TE.isGather() && TE.hasState()) {
@@ -18411,9 +18534,13 @@ InstructionCost BoUpSLP::calculateTreeCostAndTrimNonProfitable(
NodesCosts.try_emplace(TE.get(), C);
continue;
}
- unsigned Scale = EntryToScale.lookup(TE.get());
- if (!Scale)
- Scale = getScaleToLoopIterations(*TE.get());
+ uint64_t Scale = EntryToScale.lookup(TE.get());
+ if (!Scale) {
+ const bool IsGatherLike =
+ TE->isGather() || TE->State == TreeEntry::SplitVectorize;
+ Scale = IsGatherLike ? getGatherNodeEffectiveScale(*TE.get())
+ : getScaleToLoopIterations(*TE.get());
+ }
C *= Scale;
NodesCosts.try_emplace(TE.get(), C);
}
@@ -18491,13 +18618,13 @@ InstructionCost BoUpSLP::getTreeCost(InstructionCost TreeCost,
}
InstructionCost Cost = TreeCost;
- SmallDenseMap<std::tuple<const TreeEntry *, Value *, Instruction *>, unsigned>
+ SmallDenseMap<std::tuple<const TreeEntry *, Value *, Instruction *>, uint64_t>
EntryToScale;
auto ScaleCost = [&](InstructionCost C, const TreeEntry &TE,
Value *Scalar = nullptr, Instruction *U = nullptr) {
if (!C.isValid() || C == 0)
return C;
- unsigned &Scale =
+ uint64_t &Scale =
EntryToScale.try_emplace(std::make_tuple(&TE, Scalar, U), 0)
.first->getSecond();
if (!Scale)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
index 0eeb72570635c..e11805feef2b8 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/externally-used-copyables.ll
@@ -5,171 +5,143 @@ define void @test(i64 %0, i64 %1, i64 %2, i64 %3, i64 %.sroa.3341.0.copyload, i6
; CHECK-LABEL: define void @test(
; CHECK-SAME: i64 [[TMP0:%.*]], i64 [[TMP1:%.*]], i64 [[TMP2:%.*]], i64 [[TMP3:%.*]], i64 [[DOTSROA_3341_0_COPYLOAD:%.*]], i64 [[DOTSROA_3308_0_COPYLOAD:%.*]], i64 [[DOTNEG1:%.*]], i64 [[INDVAR3788:%.*]], i64 [[TMP4:%.*]], i64 [[TMP5:%.*]], i64 [[TMP6:%.*]], i64 [[TMP7:%.*]], i64 [[TMP8:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[_LR_PH_PREHEADER:.*:]]
-; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP0]], 24
; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP0]], 40
; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP0]], 48
-; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP0]], 1
-; CHECK-NEXT: [[TMP13:%.*]] = sub i64 1, [[TMP0]]
-; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[TMP0]], 11
-; CHECK-NEXT: [[TMP15:%.*]] = sub i64 1, [[TMP14]]
-; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[TMP14]], 8
-; CHECK-NEXT: [[TMP17:%.*]] = or i64 [[TMP0]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP35]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = add <2 x i64> [[TMP12]], <i64 0, i64 1>
+; CHECK-NEXT: [[TMP14:%.*]] = shl <2 x i64> [[TMP12]], <i64 11, i64 0>
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <2 x i64> [[TMP35]], <2 x i64> [[TMP14]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP16:%.*]] = sub <2 x i64> splat (i64 1), [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = add <2 x i64> [[TMP14]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP37:%.*]] = or <2 x i64> [[TMP14]], <i64 8, i64 1>
+; CHECK-NEXT: [[TMP40:%.*]] = shufflevector <2 x i64> [[TMP17]], <2 x i64> [[TMP37]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP0]], 1
; CHECK-NEXT: [[TMP19:%.*]] = or i64 [[TMP18]], [[TMP0]]
; CHECK-NEXT: [[TMP20:%.*]] = or i64 [[TMP19]], 1
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i64> [[TMP12]], <2 x i64> <i64 24, i64 poison>, <2 x i32> <i32 2, i32 0>
+; CHECK-NEXT: [[TMP41:%.*]] = mul <2 x i64> [[TMP12]], [[TMP23]]
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i64> poison, i64 [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> poison, i64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x i64> [[TMP45]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP46:%.*]] = shufflevector <2 x i64> [[TMP41]], <2 x i64> poison, <32 x i32> <i32 0, i32 poison, i32 poison, i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP74:%.*]] = shufflevector <32 x i64> [[TMP44]], <32 x i64> [[TMP46]], <32 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 5, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 32, i32 poison, i32 poison, i32 poison, i32 36, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP0]], [[TMP0]]
+; CHECK-NEXT: [[TMP85:%.*]] = shufflevector <2 x i64> [[TMP15]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <2 x i64> [[TMP40]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP33:%.*]] = shufflevector <4 x i64> <i64 poison, i64 poison, i64 2, i64 2>, <4 x i64> [[TMP32]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
; CHECK-NEXT: br label %[[DOTLR_PH1977_US:.*]]
; CHECK: [[_LR_PH1977_US:.*:]]
; CHECK-NEXT: [[INDVAR37888:%.*]] = phi i64 [ 0, [[DOTLR_PH_PREHEADER:%.*]] ], [ 1, %[[DOTLR_PH1977_US]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP16]], [[TMP0]]
-; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP20]], [[TMP0]]
-; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP15]], [[TMP0]]
-; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP17]], [[TMP0]]
+; CHECK-NEXT: [[TMP34:%.*]] = mul <4 x i64> [[TMP27]], [[TMP33]]
+; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP20]...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/192801
More information about the llvm-commits
mailing list