[llvm] [SLP] Retune look-ahead scores for constants (PR #207548)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 4 16:05:26 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/207548
Rescale LookAheadHeuristics scores and add ScoreSameConstants /
ScoreConstantScaleFactor so identical constants no longer score the
same as same-opcode instruction matches, fixing suboptimal operand
reordering around repeated constants (shift amounts, splat values).
Also retry a SplitVectorize alt-shuffle build for copyable-element
bundles when scheduling fails, instead of always falling back to a
gather, to fix regressions.
>From 1f07a60720d53631e2782591a12375073f246b6c Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 4 Jul 2026 16:05:13 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 53 ++-
llvm/test/Transforms/PhaseOrdering/X86/avg.ll | 392 ++++--------------
.../AArch64/long-non-power-of-2.ll | 77 ++--
.../RISCV/partial-vec-invalid-cost.ll | 2 +-
.../SLPVectorizer/X86/bad-reduction.ll | 8 +-
.../buildvector-postpone-for-dependency.ll | 7 +-
.../buildvector-reused-with-bv-subvector.ll | 8 +-
.../commutable-node-with-non-sched-parent.ll | 12 +-
.../SLPVectorizer/X86/debug-info-salvage.ll | 2 +-
.../X86/non-power-of-2-subvectors-insert.ll | 11 +-
.../non-schedulable-parent-multi-copyables.ll | 8 +-
...ecalc-copyable-operand-deps-shared-inst.ll | 33 +-
.../X86/reduced-ordered-values-update.ll | 5 +-
13 files changed, 196 insertions(+), 422 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 92befc3c2ff9b..ccef34850670a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2649,34 +2649,36 @@ class slpvectorizer::BoUpSLP {
// tree entry node in the different lane.
/// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
- static const int ScoreConsecutiveLoads = 4;
+ static constexpr int ScoreConsecutiveLoads = 40;
/// The same load multiple times. This should have a better score than
/// `ScoreSplat` because it in x86 for a 2-lane vector we can represent it
/// with `movddup (%reg), xmm0` which has a throughput of 0.5 versus 0.5 for
/// a vector load and 1.0 for a broadcast.
- static const int ScoreSplatLoads = 3;
+ static constexpr int ScoreSplatLoads = 30;
/// Loads from reversed memory addresses, e.g. load(A[i+1]), load(A[i]).
- static const int ScoreReversedLoads = 3;
+ static constexpr int ScoreReversedLoads = 30;
/// A load candidate for masked gather.
- static const int ScoreMaskedGatherCandidate = 1;
+ static constexpr int ScoreMaskedGatherCandidate = 10;
/// ExtractElementInst from same vector and consecutive indexes.
- static const int ScoreConsecutiveExtracts = 4;
+ static constexpr int ScoreConsecutiveExtracts = 40;
/// ExtractElementInst from same vector and reversed indices.
- static const int ScoreReversedExtracts = 3;
+ static constexpr int ScoreReversedExtracts = 30;
/// Constants.
- static const int ScoreConstants = 2;
+ static constexpr int ScoreConstants = 15;
+ /// Same constants.
+ static constexpr int ScoreSameConstants = 17;
/// Instructions with the same opcode.
- static const int ScoreSameOpcode = 2;
+ static constexpr int ScoreSameOpcode = 20;
/// Instructions with alt opcodes (e.g, add + sub).
- static const int ScoreAltOpcodes = 1;
+ static constexpr int ScoreAltOpcodes = 10;
/// Identical instructions (a.k.a. splat or broadcast).
- static const int ScoreSplat = 1;
+ static constexpr int ScoreSplat = 10;
/// Matching with an undef is preferable to failing.
- static const int ScoreUndef = 1;
+ static constexpr int ScoreUndef = 10;
/// Score for failing to find a decent match.
- static const int ScoreFail = 0;
+ static constexpr int ScoreFail = 0;
/// Score if all users are vectorized.
- static const int ScoreAllUserVectorized = 1;
+ static constexpr int ScoreAllUserVectorized = 10;
/// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
/// \p U1 and \p U2 are the users of \p V1 and \p V2.
@@ -2710,6 +2712,10 @@ class slpvectorizer::BoUpSLP {
AllUsersAreInternal(V1, V2)))
return LookAheadHeuristics::ScoreSplatLoads;
}
+ if (isa<UndefValue>(V1))
+ return LookAheadHeuristics::ScoreUndef;
+ if (isConstant(V1))
+ return LookAheadHeuristics::ScoreSameConstants;
return LookAheadHeuristics::ScoreSplat;
}
@@ -2761,7 +2767,7 @@ class slpvectorizer::BoUpSLP {
// Consider constants and buildvector compatible.
if ((C1 && isa<InsertElementInst>(V2)) ||
(C2 && isa<InsertElementInst>(V1)))
- return LookAheadHeuristics::ScoreConstants;
+ return LookAheadHeuristics::ScoreSameOpcode;
// Extracts from consecutive indexes of the same vector better score as
// the extracts could be optimized away.
@@ -3112,7 +3118,9 @@ class slpvectorizer::BoUpSLP {
/// Score scaling factor for fully compatible instructions but with
/// different number of external uses. Allows better selection of the
/// instructions with less external uses.
- static const int ScoreScaleFactor = 10;
+ static constexpr int ScoreScaleFactor = 10;
+ /// Scale factor for constants only.
+ static constexpr int ScoreConstantScaleFactor = 6;
/// \Returns the look-ahead score, which tells us how much the sub-trees
/// rooted at \p LHS and \p RHS match, the more they match the higher the
@@ -3130,7 +3138,8 @@ class slpvectorizer::BoUpSLP {
LookAhead.getScoreAtLevelRec(LHS, RHS, /*U1=*/nullptr, /*U2=*/nullptr,
/*CurrLevel=*/1, MainAltOps);
if (Score) {
- int SplatScore = getSplatScore(Lane, OpIdx, Idx, UsedLanes);
+ int SplatScore =
+ getSplatScore(Lane, OpIdx, Idx, UsedLanes) * ScoreScaleFactor;
if (Score <= -SplatScore) {
// Failed score.
Score = 0;
@@ -3141,7 +3150,10 @@ class slpvectorizer::BoUpSLP {
// uses. It does not affect actual selection of the best
// compatible operand in general, just allows to select the
// operand with all vectorized uses.
- Score *= ScoreScaleFactor;
+ const int SF = (LHS == RHS && isConstant(LHS))
+ ? ScoreConstantScaleFactor
+ : ScoreScaleFactor;
+ Score *= SF;
Score += getExternalUseScore(Lane, OpIdx, Idx);
IsUsed = true;
}
@@ -13188,6 +13200,13 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
+ // Last chance to try to vectorize copyable node.
+ if (S.areInstructionsWithCopyableElements() &&
+ ReuseShuffleIndices.empty()) {
+ InstructionsState AltS = getSameOpcode(VL, *TLI);
+ if (AltS && AltS.isAltShuffle() && TrySplitNode(AltS))
+ return;
+ }
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 270b17cd18085..9084407f008ae 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -26,9 +26,8 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE2-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
-; SSE2-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE2-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
-; SSE2-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
+; SSE2-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
+; SSE2-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
; SSE2-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; SSE2-NEXT: [[TMP21:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
@@ -39,16 +38,11 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE2-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE2-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
-; SSE2-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE2-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
+; SSE2-NEXT: [[TMP47:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 48)
+; SSE2-NEXT: [[TMP48:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 56)
; SSE2-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE2-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
-; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
-; SSE2-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56
-; SSE2-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE2-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
+; SSE2-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
; SSE2-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE2-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE2-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
@@ -72,29 +66,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE2-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE2-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE2-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
-; SSE2-NEXT: [[ADD5_14:%.*]] = add nuw nsw i64 [[ADD_14]], [[CONV4_14]]
-; SSE2-NEXT: [[ADD5_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_8_0_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD_15:%.*]] = add nuw nsw i64 [[A_SROA_17_8_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV1_6]], 1
-; SSE2-NEXT: [[ADD5_15:%.*]] = add nuw nsw i64 [[ADD_15]], [[B_SROA_17_8_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD5_6:%.*]] = add nuw nsw i64 [[ADD_6]], [[CONV4_6]]
-; SSE2-NEXT: [[TMP45:%.*]] = shl nuw i64 [[ADD5_15]], 55
-; SSE2-NEXT: [[TMP46:%.*]] = shl nuw nsw i64 [[ADD5_6]], 47
-; SSE2-NEXT: [[RETVAL_SROA_17_8_INSERT_EXT:%.*]] = and i64 [[TMP45]], -72057594037927936
-; SSE2-NEXT: [[RETVAL_SROA_7_0_INSERT_SHIFT:%.*]] = and i64 [[TMP46]], 71776119061217280
-; SSE2-NEXT: [[TMP47:%.*]] = shl nuw nsw i64 [[ADD5_14]], 47
-; SSE2-NEXT: [[TMP48:%.*]] = shl nuw i64 [[ADD5_7]], 55
-; SSE2-NEXT: [[RETVAL_SROA_16_8_INSERT_SHIFT:%.*]] = and i64 [[TMP47]], 71776119061217280
-; SSE2-NEXT: [[RETVAL_SROA_8_0_INSERT_EXT:%.*]] = and i64 [[TMP48]], -72057594037927936
-; SSE2-NEXT: [[RETVAL_SROA_16_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_17_8_INSERT_EXT]], [[RETVAL_SROA_16_8_INSERT_SHIFT]]
-; SSE2-NEXT: [[RETVAL_SROA_7_0_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_8_0_INSERT_EXT]], [[RETVAL_SROA_7_0_INSERT_SHIFT]]
+; SSE2-NEXT: [[TMP71:%.*]] = and <2 x i64> [[TMP47]], splat (i64 255)
+; SSE2-NEXT: [[TMP51:%.*]] = add nuw nsw <2 x i64> [[TMP70]], splat (i64 1)
+; SSE2-NEXT: [[TMP72:%.*]] = add nuw nsw <2 x i64> [[TMP51]], [[TMP71]]
+; SSE2-NEXT: [[TMP73:%.*]] = add nuw nsw <2 x i64> [[TMP46]], splat (i64 1)
+; SSE2-NEXT: [[TMP74:%.*]] = add nuw nsw <2 x i64> [[TMP73]], [[TMP48]]
+; SSE2-NEXT: [[TMP75:%.*]] = shl nuw <2 x i64> [[TMP74]], splat (i64 55)
+; SSE2-NEXT: [[TMP76:%.*]] = and <2 x i64> [[TMP75]], splat (i64 -72057594037927936)
+; SSE2-NEXT: [[TMP77:%.*]] = shl nuw nsw <2 x i64> [[TMP72]], splat (i64 47)
+; SSE2-NEXT: [[TMP78:%.*]] = and <2 x i64> [[TMP77]], splat (i64 71776119061217280)
+; SSE2-NEXT: [[TMP52:%.*]] = or disjoint <2 x i64> [[TMP76]], [[TMP78]]
; SSE2-NEXT: [[TMP49:%.*]] = shl nuw nsw <2 x i64> [[TMP44]], splat (i64 39)
; SSE2-NEXT: [[TMP50:%.*]] = and <2 x i64> [[TMP49]], splat (i64 280375465082880)
-; SSE2-NEXT: [[TMP51:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_7_0_INSERT_INSERT]], i64 0
-; SSE2-NEXT: [[TMP52:%.*]] = insertelement <2 x i64> [[TMP51]], i64 [[RETVAL_SROA_16_8_INSERT_INSERT]], i64 1
; SSE2-NEXT: [[TMP53:%.*]] = or disjoint <2 x i64> [[TMP52]], [[TMP50]]
; SSE2-NEXT: [[TMP54:%.*]] = shl nuw nsw <2 x i64> [[TMP40]], splat (i64 31)
; SSE2-NEXT: [[TMP55:%.*]] = and <2 x i64> [[TMP54]], splat (i64 1095216660480)
@@ -125,10 +108,9 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE4-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE4-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
-; SSE4-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE4-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
+; SSE4-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
+; SSE4-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
; SSE4-NEXT: [[TMP7:%.*]] = trunc i64 [[A_COERCE1]] to i16
-; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
; SSE4-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
; SSE4-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1:%.*]], i64 1
@@ -136,23 +118,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE4-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE4-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
-; SSE4-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
+; SSE4-NEXT: [[TMP47:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 48)
+; SSE4-NEXT: [[TMP48:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 56)
; SSE4-NEXT: [[TMP15:%.*]] = trunc i64 [[B_COERCE1]] to i16
-; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
; SSE4-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE4-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
-; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
-; SSE4-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
+; SSE4-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
; SSE4-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; SSE4-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1
; SSE4-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8)
-; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
-; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56
; SSE4-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; SSE4-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1
; SSE4-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE4-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE4-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE4-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
@@ -174,29 +151,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE4-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE4-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE4-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE4-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
-; SSE4-NEXT: [[ADD5_14:%.*]] = add nuw nsw i64 [[ADD_14]], [[CONV4_14]]
-; SSE4-NEXT: [[ADD5_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_8_0_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD_15:%.*]] = add nuw nsw i64 [[A_SROA_17_8_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV1_6]], 1
-; SSE4-NEXT: [[ADD5_15:%.*]] = add nuw nsw i64 [[ADD_15]], [[B_SROA_17_8_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD5_6:%.*]] = add nuw nsw i64 [[ADD_6]], [[CONV4_6]]
-; SSE4-NEXT: [[TMP45:%.*]] = shl nuw i64 [[ADD5_15]], 55
-; SSE4-NEXT: [[TMP46:%.*]] = shl nuw nsw i64 [[ADD5_6]], 47
-; SSE4-NEXT: [[RETVAL_SROA_17_8_INSERT_EXT:%.*]] = and i64 [[TMP45]], -72057594037927936
-; SSE4-NEXT: [[RETVAL_SROA_7_0_INSERT_SHIFT:%.*]] = and i64 [[TMP46]], 71776119061217280
-; SSE4-NEXT: [[TMP47:%.*]] = shl nuw nsw i64 [[ADD5_14]], 47
-; SSE4-NEXT: [[TMP48:%.*]] = shl nuw i64 [[ADD5_7]], 55
-; SSE4-NEXT: [[RETVAL_SROA_16_8_INSERT_SHIFT:%.*]] = and i64 [[TMP47]], 71776119061217280
-; SSE4-NEXT: [[RETVAL_SROA_8_0_INSERT_EXT:%.*]] = and i64 [[TMP48]], -72057594037927936
-; SSE4-NEXT: [[RETVAL_SROA_16_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_17_8_INSERT_EXT]], [[RETVAL_SROA_16_8_INSERT_SHIFT]]
-; SSE4-NEXT: [[RETVAL_SROA_7_0_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_8_0_INSERT_EXT]], [[RETVAL_SROA_7_0_INSERT_SHIFT]]
+; SSE4-NEXT: [[TMP71:%.*]] = and <2 x i64> [[TMP47]], splat (i64 255)
+; SSE4-NEXT: [[TMP51:%.*]] = add nuw nsw <2 x i64> [[TMP70]], splat (i64 1)
+; SSE4-NEXT: [[TMP72:%.*]] = add nuw nsw <2 x i64> [[TMP51]], [[TMP71]]
+; SSE4-NEXT: [[TMP73:%.*]] = add nuw nsw <2 x i64> [[TMP46]], splat (i64 1)
+; SSE4-NEXT: [[TMP74:%.*]] = add nuw nsw <2 x i64> [[TMP73]], [[TMP48]]
+; SSE4-NEXT: [[TMP75:%.*]] = shl nuw <2 x i64> [[TMP74]], splat (i64 55)
+; SSE4-NEXT: [[TMP76:%.*]] = and <2 x i64> [[TMP75]], splat (i64 -72057594037927936)
+; SSE4-NEXT: [[TMP77:%.*]] = shl nuw nsw <2 x i64> [[TMP72]], splat (i64 47)
+; SSE4-NEXT: [[TMP78:%.*]] = and <2 x i64> [[TMP77]], splat (i64 71776119061217280)
+; SSE4-NEXT: [[TMP52:%.*]] = or disjoint <2 x i64> [[TMP76]], [[TMP78]]
; SSE4-NEXT: [[TMP49:%.*]] = shl nuw nsw <2 x i64> [[TMP44]], splat (i64 39)
; SSE4-NEXT: [[TMP50:%.*]] = and <2 x i64> [[TMP49]], splat (i64 280375465082880)
-; SSE4-NEXT: [[TMP51:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_7_0_INSERT_INSERT]], i64 0
-; SSE4-NEXT: [[TMP52:%.*]] = insertelement <2 x i64> [[TMP51]], i64 [[RETVAL_SROA_16_8_INSERT_INSERT]], i64 1
; SSE4-NEXT: [[TMP53:%.*]] = or disjoint <2 x i64> [[TMP52]], [[TMP50]]
; SSE4-NEXT: [[TMP54:%.*]] = shl nuw nsw <2 x i64> [[TMP40]], splat (i64 31)
; SSE4-NEXT: [[TMP55:%.*]] = and <2 x i64> [[TMP54]], splat (i64 1095216660480)
@@ -242,38 +208,38 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; AVX-NEXT: [[ADD5_9:%.*]] = add nuw nsw i16 [[ADD_9]], [[TMP7]]
; AVX-NEXT: [[TMP8:%.*]] = shl nuw i16 [[ADD5_1]], 7
; AVX-NEXT: [[TMP9:%.*]] = and i16 [[TMP8]], -256
-; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[A_COERCE0]], i64 0
+; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE0]], i64 0
; AVX-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
; AVX-NEXT: [[TMP12:%.*]] = lshr <8 x i64> [[TMP11]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
-; AVX-NEXT: [[TMP13:%.*]] = and <8 x i64> [[TMP12]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
; AVX-NEXT: [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED:%.*]] = zext i16 [[TMP9]] to i64
-; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> poison, i64 [[B_COERCE0]], i64 0
+; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0]], i64 0
; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x i64> [[TMP14]], i64 [[ADD5]], i64 1
; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x i64> [[TMP15]], i64 [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED]], i64 2
; AVX-NEXT: [[TMP17:%.*]] = shufflevector <8 x i64> [[TMP16]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 2>
; AVX-NEXT: [[TMP18:%.*]] = lshr <8 x i64> [[TMP17]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
; AVX-NEXT: [[TMP19:%.*]] = and <8 x i64> [[TMP18]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP20:%.*]] = add nuw nsw <8 x i64> [[TMP13]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
-; AVX-NEXT: [[TMP21:%.*]] = add nuw nsw <8 x i64> [[TMP19]], [[TMP20]]
+; AVX-NEXT: [[TMP30:%.*]] = and <8 x i64> [[TMP12]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX-NEXT: [[TMP20:%.*]] = add nuw nsw <8 x i64> [[TMP19]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX-NEXT: [[TMP21:%.*]] = add nuw nsw <8 x i64> [[TMP20]], [[TMP30]]
; AVX-NEXT: [[TMP22:%.*]] = shl nuw <8 x i64> [[TMP21]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
; AVX-NEXT: [[TMP23:%.*]] = and <8 x i64> [[TMP22]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
; AVX-NEXT: [[TMP24:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP23]])
; AVX-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP24]], 0
; AVX-NEXT: [[TMP25:%.*]] = shl nuw i16 [[ADD5_9]], 7
; AVX-NEXT: [[TMP26:%.*]] = and i16 [[TMP25]], -256
-; AVX-NEXT: [[TMP27:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[A_COERCE1]], i64 0
+; AVX-NEXT: [[TMP27:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE1]], i64 0
; AVX-NEXT: [[TMP28:%.*]] = shufflevector <8 x i64> [[TMP27]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
; AVX-NEXT: [[TMP29:%.*]] = lshr <8 x i64> [[TMP28]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
-; AVX-NEXT: [[TMP30:%.*]] = and <8 x i64> [[TMP29]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
; AVX-NEXT: [[RETVAL_SROA_11_8_INSERT_SHIFT_MASKED:%.*]] = zext i16 [[TMP26]] to i64
-; AVX-NEXT: [[TMP31:%.*]] = insertelement <8 x i64> poison, i64 [[B_COERCE1]], i64 0
+; AVX-NEXT: [[TMP31:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE1]], i64 0
; AVX-NEXT: [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[ADD5_8]], i64 1
; AVX-NEXT: [[TMP33:%.*]] = insertelement <8 x i64> [[TMP32]], i64 [[RETVAL_SROA_11_8_INSERT_SHIFT_MASKED]], i64 2
; AVX-NEXT: [[TMP34:%.*]] = shufflevector <8 x i64> [[TMP33]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 2>
; AVX-NEXT: [[TMP35:%.*]] = lshr <8 x i64> [[TMP34]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
; AVX-NEXT: [[TMP36:%.*]] = and <8 x i64> [[TMP35]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP37:%.*]] = add nuw nsw <8 x i64> [[TMP30]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
-; AVX-NEXT: [[TMP38:%.*]] = add nuw nsw <8 x i64> [[TMP36]], [[TMP37]]
+; AVX-NEXT: [[TMP42:%.*]] = and <8 x i64> [[TMP29]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
+; AVX-NEXT: [[TMP37:%.*]] = add nuw nsw <8 x i64> [[TMP36]], <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 0, i64 0>
+; AVX-NEXT: [[TMP38:%.*]] = add nuw nsw <8 x i64> [[TMP37]], [[TMP42]]
; AVX-NEXT: [[TMP39:%.*]] = shl nuw <8 x i64> [[TMP38]], <i64 55, i64 47, i64 39, i64 31, i64 23, i64 15, i64 0, i64 0>
; AVX-NEXT: [[TMP40:%.*]] = and <8 x i64> [[TMP39]], <i64 -72057594037927936, i64 71776119061217280, i64 280375465082880, i64 1095216660480, i64 4278190080, i64 16711680, i64 -1, i64 -1>
; AVX-NEXT: [[TMP41:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP40]])
@@ -728,234 +694,54 @@ for.body: ; preds = %for.cond
}
define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, i64 %b.coerce1) {
-; SSE2-LABEL: @avgr_8_u16(
-; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32
-; SSE2-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32
-; SSE2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE2-NEXT: [[TMP2:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i32
-; SSE2-NEXT: [[A_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 32
-; SSE2-NEXT: [[TMP18:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
-; SSE2-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[TMP18]], i64 [[B_COERCE1:%.*]], i64 1
-; SSE2-NEXT: [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32>
-; SSE2-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32
-; SSE2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; SSE2-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
-; SSE2-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[A_COERCE1]], i64 1
-; SSE2-NEXT: [[TMP7:%.*]] = and <2 x i64> [[TMP6]], splat (i64 65535)
-; SSE2-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP3]], splat (i64 65535)
-; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP2]], i64 1
-; SSE2-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16)
-; SSE2-NEXT: [[CONV_6:%.*]] = lshr i64 [[A_COERCE1]], 48
-; SSE2-NEXT: [[CONV2_4:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16)
-; SSE2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
-; SSE2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535
-; SSE2-NEXT: [[CONV2_2:%.*]] = and i64 [[B_SROA_3_0_EXTRACT_SHIFT]], 65535
-; SSE2-NEXT: [[TMP31:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1)
-; SSE2-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP31]], [[TMP8]]
-; SSE2-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP14]], splat (i64 1)
-; SSE2-NEXT: [[TMP16:%.*]] = add nuw nsw <2 x i32> [[TMP11]], splat (i32 1)
-; SSE2-NEXT: [[TMP17:%.*]] = add nuw nsw <2 x i32> [[TMP16]], [[TMP20]]
-; SSE2-NEXT: [[CONV_7:%.*]] = and i64 [[A_SROA_8_8_EXTRACT_SHIFT]], 65535
-; SSE2-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = and i64 [[B_SROA_8_8_EXTRACT_SHIFT]], 65535
-; SSE2-NEXT: [[ADD_4:%.*]] = add nuw nsw i64 [[A_SROA_4_0_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[CONV_7]], 1
-; SSE2-NEXT: [[ADD3_3:%.*]] = add nuw nsw i64 [[ADD_3]], [[B_SROA_4_0_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD3_4:%.*]] = add nuw nsw i64 [[ADD_4]], [[CONV2_4]]
-; SSE2-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV_6]], 1
-; SSE2-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_9_8_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD3_7:%.*]] = add nuw nsw i64 [[ADD_6]], [[B_SROA_9_8_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD3_2:%.*]] = add nuw nsw i64 [[ADD_7]], [[CONV2_2]]
-; SSE2-NEXT: [[TMP12:%.*]] = shl nuw i64 [[ADD3_7]], 47
-; SSE2-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[ADD3_2]], 31
-; SSE2-NEXT: [[RETVAL_SROA_9_8_INSERT_EXT:%.*]] = and i64 [[TMP12]], -281474976710656
-; SSE2-NEXT: [[SHR_4:%.*]] = and i64 [[TMP9]], 281470681743360
-; SSE2-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[ADD3_3]], 31
-; SSE2-NEXT: [[TMP21:%.*]] = shl nuw i64 [[ADD3_4]], 47
-; SSE2-NEXT: [[RETVAL_SROA_8_8_INSERT_SHIFT:%.*]] = and i64 [[TMP13]], 281470681743360
-; SSE2-NEXT: [[RETVAL_SROA_7_8_INSERT_INSERT:%.*]] = and i64 [[TMP21]], -281474976710656
-; SSE2-NEXT: [[RETVAL_SROA_8_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_9_8_INSERT_EXT]], [[RETVAL_SROA_8_8_INSERT_SHIFT]]
-; SSE2-NEXT: [[RETVAL_SROA_5_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_7_8_INSERT_INSERT]], [[SHR_4]]
-; SSE2-NEXT: [[TMP22:%.*]] = shl nuw <2 x i32> [[TMP17]], splat (i32 15)
-; SSE2-NEXT: [[TMP23:%.*]] = and <2 x i32> [[TMP22]], splat (i32 -65536)
-; SSE2-NEXT: [[TMP24:%.*]] = zext <2 x i32> [[TMP23]] to <2 x i64>
-; SSE2-NEXT: [[TMP25:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_5_8_INSERT_INSERT]], i64 0
-; SSE2-NEXT: [[TMP26:%.*]] = insertelement <2 x i64> [[TMP25]], i64 [[RETVAL_SROA_8_8_INSERT_INSERT]], i64 1
-; SSE2-NEXT: [[TMP27:%.*]] = or disjoint <2 x i64> [[TMP26]], [[TMP24]]
-; SSE2-NEXT: [[TMP28:%.*]] = or disjoint <2 x i64> [[TMP27]], [[TMP15]]
-; SSE2-NEXT: [[TMP29:%.*]] = extractelement <2 x i64> [[TMP28]], i64 0
-; SSE2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP29]], 0
-; SSE2-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[TMP28]], i64 1
-; SSE2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP30]], 1
-; SSE2-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
-;
-; SSE4-LABEL: @avgr_8_u16(
-; SSE4-NEXT: entry:
-; SSE4-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32
-; SSE4-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32
-; SSE4-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE4-NEXT: [[TMP1:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i32
-; SSE4-NEXT: [[A_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 32
-; SSE4-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
-; SSE4-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE1:%.*]], i64 1
-; SSE4-NEXT: [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32>
-; SSE4-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32
-; SSE4-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; SSE4-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
-; SSE4-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[A_COERCE1]], i64 1
-; SSE4-NEXT: [[TMP7:%.*]] = and <2 x i64> [[TMP6]], splat (i64 65535)
-; SSE4-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP3]], splat (i64 65535)
-; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP1]], i64 1
-; SSE4-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16)
-; SSE4-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
-; SSE4-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16)
-; SSE4-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
-; SSE4-NEXT: [[CONV_2:%.*]] = and i64 [[A_SROA_3_0_EXTRACT_SHIFT]], 65535
-; SSE4-NEXT: [[CONV2_2:%.*]] = and i64 [[B_SROA_3_0_EXTRACT_SHIFT]], 65535
-; SSE4-NEXT: [[TMP13:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1)
-; SSE4-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP13]], [[TMP8]]
-; SSE4-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP14]], splat (i64 1)
-; SSE4-NEXT: [[TMP16:%.*]] = add nuw nsw <2 x i32> [[TMP11]], splat (i32 1)
-; SSE4-NEXT: [[TMP17:%.*]] = add nuw nsw <2 x i32> [[TMP16]], [[TMP12]]
-; SSE4-NEXT: [[CONV_6:%.*]] = and i64 [[A_SROA_8_8_EXTRACT_SHIFT]], 65535
-; SSE4-NEXT: [[CONV2_6:%.*]] = and i64 [[B_SROA_8_8_EXTRACT_SHIFT]], 65535
-; SSE4-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[A_SROA_4_0_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV_6]], 1
-; SSE4-NEXT: [[ADD3_6:%.*]] = add nuw nsw i64 [[ADD_6]], [[CONV2_6]]
-; SSE4-NEXT: [[ADD3_3:%.*]] = add nuw nsw i64 [[ADD_3]], [[B_SROA_4_0_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_9_8_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_2:%.*]] = add nuw nsw i64 [[CONV_2]], 1
-; SSE4-NEXT: [[ADD3_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_9_8_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD3_2:%.*]] = add nuw nsw i64 [[ADD_2]], [[CONV2_2]]
-; SSE4-NEXT: [[TMP18:%.*]] = shl nuw i64 [[ADD3_7]], 47
-; SSE4-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[ADD3_2]], 31
-; SSE4-NEXT: [[RETVAL_SROA_9_8_INSERT_EXT:%.*]] = and i64 [[TMP18]], -281474976710656
-; SSE4-NEXT: [[RETVAL_SROA_3_0_INSERT_SHIFT:%.*]] = and i64 [[TMP19]], 281470681743360
-; SSE4-NEXT: [[TMP20:%.*]] = shl nuw nsw i64 [[ADD3_6]], 31
-; SSE4-NEXT: [[TMP21:%.*]] = shl nuw i64 [[ADD3_3]], 47
-; SSE4-NEXT: [[RETVAL_SROA_8_8_INSERT_SHIFT:%.*]] = and i64 [[TMP20]], 281470681743360
-; SSE4-NEXT: [[RETVAL_SROA_4_0_INSERT_EXT:%.*]] = and i64 [[TMP21]], -281474976710656
-; SSE4-NEXT: [[RETVAL_SROA_8_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_9_8_INSERT_EXT]], [[RETVAL_SROA_8_8_INSERT_SHIFT]]
-; SSE4-NEXT: [[RETVAL_SROA_3_0_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_4_0_INSERT_EXT]], [[RETVAL_SROA_3_0_INSERT_SHIFT]]
-; SSE4-NEXT: [[TMP22:%.*]] = shl nuw <2 x i32> [[TMP17]], splat (i32 15)
-; SSE4-NEXT: [[TMP23:%.*]] = and <2 x i32> [[TMP22]], splat (i32 -65536)
-; SSE4-NEXT: [[TMP24:%.*]] = zext <2 x i32> [[TMP23]] to <2 x i64>
-; SSE4-NEXT: [[TMP25:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_3_0_INSERT_INSERT]], i64 0
-; SSE4-NEXT: [[TMP26:%.*]] = insertelement <2 x i64> [[TMP25]], i64 [[RETVAL_SROA_8_8_INSERT_INSERT]], i64 1
-; SSE4-NEXT: [[TMP27:%.*]] = or disjoint <2 x i64> [[TMP26]], [[TMP24]]
-; SSE4-NEXT: [[TMP28:%.*]] = or disjoint <2 x i64> [[TMP27]], [[TMP15]]
-; SSE4-NEXT: [[TMP29:%.*]] = extractelement <2 x i64> [[TMP28]], i64 0
-; SSE4-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP29]], 0
-; SSE4-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[TMP28]], i64 1
-; SSE4-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP30]], 1
-; SSE4-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
-;
-; AVX2-LABEL: @avgr_8_u16(
-; AVX2-NEXT: entry:
-; AVX2-NEXT: [[TMP1:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i32
-; AVX2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE1]], i64 0
-; AVX2-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[A_COERCE2:%.*]], i64 1
-; AVX2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP6]], <i64 32, i64 48>
-; AVX2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
-; AVX2-NEXT: [[TMP18:%.*]] = trunc i64 [[A_COERCE2]] to i32
-; AVX2-NEXT: [[TMP20:%.*]] = insertelement <2 x i64> [[TMP6]], i64 [[B_COERCE0:%.*]], i64 0
-; AVX2-NEXT: [[TMP21:%.*]] = lshr <2 x i64> [[TMP20]], <i64 48, i64 32>
-; AVX2-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> [[TMP20]], i64 [[B_COERCE1:%.*]], i64 1
-; AVX2-NEXT: [[TMP4:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i32>
-; AVX2-NEXT: [[TMP9:%.*]] = lshr <2 x i64> [[TMP3]], <i64 32, i64 48>
-; AVX2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; AVX2-NEXT: [[TMP7:%.*]] = and <2 x i64> [[TMP6]], splat (i64 65535)
-; AVX2-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP3]], splat (i64 65535)
-; AVX2-NEXT: [[TMP19:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i64 0
-; AVX2-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[TMP18]], i64 1
-; AVX2-NEXT: [[TMP11:%.*]] = lshr <2 x i32> [[TMP10]], splat (i32 16)
-; AVX2-NEXT: [[TMP12:%.*]] = lshr <2 x i32> [[TMP4]], splat (i32 16)
-; AVX2-NEXT: [[TMP36:%.*]] = and <2 x i64> [[TMP5]], <i64 65535, i64 -1>
-; AVX2-NEXT: [[TMP39:%.*]] = and <2 x i64> [[TMP9]], <i64 65535, i64 -1>
-; AVX2-NEXT: [[TMP13:%.*]] = add nuw nsw <2 x i64> [[TMP7]], splat (i64 1)
-; AVX2-NEXT: [[TMP14:%.*]] = add nuw nsw <2 x i64> [[TMP13]], [[TMP8]]
-; AVX2-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP14]], splat (i64 1)
-; AVX2-NEXT: [[TMP16:%.*]] = add nuw nsw <2 x i32> [[TMP11]], splat (i32 1)
-; AVX2-NEXT: [[TMP17:%.*]] = add nuw nsw <2 x i32> [[TMP16]], [[TMP12]]
-; AVX2-NEXT: [[TMP34:%.*]] = and <2 x i64> [[TMP21]], <i64 -1, i64 65535>
-; AVX2-NEXT: [[CONV2_6:%.*]] = and i64 [[B_SROA_8_8_EXTRACT_SHIFT]], 65535
-; AVX2-NEXT: [[ADD_3:%.*]] = add nuw nsw i64 [[A_SROA_4_0_EXTRACT_SHIFT]], 1
-; AVX2-NEXT: [[TMP37:%.*]] = add nuw nsw <2 x i64> [[TMP34]], <i64 0, i64 1>
-; AVX2-NEXT: [[TMP35:%.*]] = insertelement <2 x i64> poison, i64 [[ADD_3]], i64 0
-; AVX2-NEXT: [[TMP25:%.*]] = insertelement <2 x i64> [[TMP35]], i64 [[CONV2_6]], i64 1
-; AVX2-NEXT: [[TMP38:%.*]] = add nuw nsw <2 x i64> [[TMP25]], [[TMP37]]
-; AVX2-NEXT: [[TMP42:%.*]] = add nuw nsw <2 x i64> [[TMP36]], splat (i64 1)
-; AVX2-NEXT: [[TMP40:%.*]] = add nuw nsw <2 x i64> [[TMP42]], [[TMP39]]
-; AVX2-NEXT: [[TMP41:%.*]] = shl nuw <2 x i64> [[TMP40]], <i64 31, i64 47>
-; AVX2-NEXT: [[TMP31:%.*]] = and <2 x i64> [[TMP41]], <i64 281470681743360, i64 -281474976710656>
-; AVX2-NEXT: [[TMP32:%.*]] = shl nuw <2 x i64> [[TMP38]], <i64 47, i64 31>
-; AVX2-NEXT: [[TMP33:%.*]] = and <2 x i64> [[TMP32]], <i64 -281474976710656, i64 281470681743360>
-; AVX2-NEXT: [[TMP26:%.*]] = or disjoint <2 x i64> [[TMP31]], [[TMP33]]
-; AVX2-NEXT: [[TMP22:%.*]] = shl nuw <2 x i32> [[TMP17]], splat (i32 15)
-; AVX2-NEXT: [[TMP23:%.*]] = and <2 x i32> [[TMP22]], splat (i32 -65536)
-; AVX2-NEXT: [[TMP24:%.*]] = zext <2 x i32> [[TMP23]] to <2 x i64>
-; AVX2-NEXT: [[TMP27:%.*]] = or disjoint <2 x i64> [[TMP26]], [[TMP24]]
-; AVX2-NEXT: [[TMP28:%.*]] = or disjoint <2 x i64> [[TMP27]], [[TMP15]]
-; AVX2-NEXT: [[TMP29:%.*]] = extractelement <2 x i64> [[TMP28]], i64 0
-; AVX2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP29]], 0
-; AVX2-NEXT: [[TMP30:%.*]] = extractelement <2 x i64> [[TMP28]], i64 1
-; AVX2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP30]], 1
-; AVX2-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
-;
-; AVX512-LABEL: @avgr_8_u16(
-; AVX512-NEXT: entry:
-; AVX512-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32
-; AVX512-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
-; AVX512-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[B_COERCE1:%.*]], i64 1
-; AVX512-NEXT: [[TMP7:%.*]] = lshr <2 x i64> [[TMP2]], <i64 32, i64 48>
-; AVX512-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE2:%.*]], i64 1
-; AVX512-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP10]], <i64 48, i64 32>
-; AVX512-NEXT: [[TMP4:%.*]] = trunc i64 [[B_COERCE1]] to i32
-; AVX512-NEXT: [[TMP32:%.*]] = insertelement <2 x i64> [[TMP2]], i64 [[B_COERCE0:%.*]], i64 0
-; AVX512-NEXT: [[TMP49:%.*]] = lshr <2 x i64> [[TMP32]], <i64 48, i64 32>
-; AVX512-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[B_COERCE0]], i64 0
-; AVX512-NEXT: [[TMP6:%.*]] = trunc <2 x i64> [[TMP5]] to <2 x i32>
-; AVX512-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP5]], <i64 32, i64 48>
-; AVX512-NEXT: [[TMP8:%.*]] = and <2 x i64> [[TMP2]], splat (i64 65535)
-; AVX512-NEXT: [[TMP9:%.*]] = and <2 x i64> [[TMP5]], splat (i64 65535)
-; AVX512-NEXT: [[TMP11:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; AVX512-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP11]], i32 [[TMP4]], i64 1
-; AVX512-NEXT: [[TMP13:%.*]] = lshr <2 x i32> [[TMP12]], splat (i32 16)
-; AVX512-NEXT: [[TMP15:%.*]] = lshr <2 x i32> [[TMP6]], splat (i32 16)
-; AVX512-NEXT: [[TMP18:%.*]] = and <2 x i64> [[TMP7]], <i64 65535, i64 -1>
-; AVX512-NEXT: [[TMP21:%.*]] = and <2 x i64> [[TMP14]], <i64 65535, i64 -1>
-; AVX512-NEXT: [[TMP22:%.*]] = add nuw nsw <2 x i64> [[TMP8]], splat (i64 1)
-; AVX512-NEXT: [[TMP23:%.*]] = add nuw nsw <2 x i64> [[TMP22]], [[TMP9]]
-; AVX512-NEXT: [[TMP24:%.*]] = lshr <2 x i64> [[TMP23]], splat (i64 1)
-; AVX512-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i32> [[TMP13]], splat (i32 1)
-; AVX512-NEXT: [[TMP26:%.*]] = add nuw nsw <2 x i32> [[TMP25]], [[TMP15]]
-; AVX512-NEXT: [[TMP30:%.*]] = and <2 x i64> [[TMP49]], <i64 -1, i64 65535>
-; AVX512-NEXT: [[TMP27:%.*]] = add nuw nsw <2 x i64> [[TMP3]], <i64 1, i64 poison>
-; AVX512-NEXT: [[TMP28:%.*]] = and <2 x i64> [[TMP3]], <i64 poison, i64 65535>
-; AVX512-NEXT: [[TMP29:%.*]] = shufflevector <2 x i64> [[TMP27]], <2 x i64> [[TMP28]], <2 x i32> <i32 0, i32 3>
-; AVX512-NEXT: [[TMP33:%.*]] = add nuw nsw <2 x i64> [[TMP30]], <i64 0, i64 1>
-; AVX512-NEXT: [[TMP34:%.*]] = add nuw nsw <2 x i64> [[TMP29]], [[TMP33]]
-; AVX512-NEXT: [[TMP35:%.*]] = add nuw nsw <2 x i64> [[TMP18]], splat (i64 1)
-; AVX512-NEXT: [[TMP36:%.*]] = add nuw nsw <2 x i64> [[TMP35]], [[TMP21]]
-; AVX512-NEXT: [[TMP37:%.*]] = shl nuw <2 x i64> [[TMP36]], <i64 31, i64 47>
-; AVX512-NEXT: [[TMP38:%.*]] = and <2 x i64> [[TMP37]], <i64 281470681743360, i64 -281474976710656>
-; AVX512-NEXT: [[TMP39:%.*]] = shl nuw <2 x i64> [[TMP34]], <i64 47, i64 31>
-; AVX512-NEXT: [[TMP40:%.*]] = and <2 x i64> [[TMP39]], <i64 -281474976710656, i64 281470681743360>
-; AVX512-NEXT: [[TMP41:%.*]] = or disjoint <2 x i64> [[TMP38]], [[TMP40]]
-; AVX512-NEXT: [[TMP42:%.*]] = shl nuw <2 x i32> [[TMP26]], splat (i32 15)
-; AVX512-NEXT: [[TMP43:%.*]] = and <2 x i32> [[TMP42]], splat (i32 -65536)
-; AVX512-NEXT: [[TMP44:%.*]] = zext <2 x i32> [[TMP43]] to <2 x i64>
-; AVX512-NEXT: [[TMP45:%.*]] = or disjoint <2 x i64> [[TMP41]], [[TMP44]]
-; AVX512-NEXT: [[TMP46:%.*]] = or disjoint <2 x i64> [[TMP45]], [[TMP24]]
-; AVX512-NEXT: [[TMP47:%.*]] = extractelement <2 x i64> [[TMP46]], i64 0
-; AVX512-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP47]], 0
-; AVX512-NEXT: [[TMP48:%.*]] = extractelement <2 x i64> [[TMP46]], i64 1
-; AVX512-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP48]], 1
-; AVX512-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
+; CHECK-LABEL: @avgr_8_u16(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1:%.*]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
+; CHECK-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[A_COERCE1]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[B_COERCE1:%.*]], i64 1
+; CHECK-NEXT: [[TMP9:%.*]] = lshr <2 x i64> [[TMP8]], splat (i64 32)
+; CHECK-NEXT: [[TMP10:%.*]] = lshr <2 x i64> [[TMP8]], splat (i64 48)
+; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[B_COERCE1]] to i32
+; CHECK-NEXT: [[TMP12:%.*]] = and <2 x i64> [[TMP2]], splat (i64 65535)
+; CHECK-NEXT: [[TMP13:%.*]] = and <2 x i64> [[TMP8]], splat (i64 65535)
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> [[TMP14]], i32 [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = lshr <2 x i32> [[TMP15]], splat (i32 16)
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP11]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = lshr <2 x i32> [[TMP18]], splat (i32 16)
+; CHECK-NEXT: [[TMP20:%.*]] = add nuw nsw <2 x i64> [[TMP12]], splat (i64 1)
+; CHECK-NEXT: [[TMP21:%.*]] = add nuw nsw <2 x i64> [[TMP20]], [[TMP13]]
+; CHECK-NEXT: [[TMP22:%.*]] = lshr <2 x i64> [[TMP21]], splat (i64 1)
+; CHECK-NEXT: [[TMP23:%.*]] = add nuw nsw <2 x i32> [[TMP16]], splat (i32 1)
+; CHECK-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i32> [[TMP23]], [[TMP19]]
+; CHECK-NEXT: [[TMP25:%.*]] = and <2 x i64> [[TMP3]], splat (i64 65535)
+; CHECK-NEXT: [[TMP26:%.*]] = and <2 x i64> [[TMP9]], splat (i64 65535)
+; CHECK-NEXT: [[TMP27:%.*]] = add nuw nsw <2 x i64> [[TMP25]], splat (i64 1)
+; CHECK-NEXT: [[TMP28:%.*]] = add nuw nsw <2 x i64> [[TMP27]], [[TMP26]]
+; CHECK-NEXT: [[TMP29:%.*]] = add nuw nsw <2 x i64> [[TMP4]], splat (i64 1)
+; CHECK-NEXT: [[TMP30:%.*]] = add nuw nsw <2 x i64> [[TMP29]], [[TMP10]]
+; CHECK-NEXT: [[TMP31:%.*]] = shl nuw <2 x i64> [[TMP30]], splat (i64 47)
+; CHECK-NEXT: [[TMP32:%.*]] = and <2 x i64> [[TMP31]], splat (i64 -281474976710656)
+; CHECK-NEXT: [[TMP33:%.*]] = shl nuw nsw <2 x i64> [[TMP28]], splat (i64 31)
+; CHECK-NEXT: [[TMP34:%.*]] = and <2 x i64> [[TMP33]], splat (i64 281470681743360)
+; CHECK-NEXT: [[TMP35:%.*]] = or disjoint <2 x i64> [[TMP32]], [[TMP34]]
+; CHECK-NEXT: [[TMP36:%.*]] = shl nuw <2 x i32> [[TMP24]], splat (i32 15)
+; CHECK-NEXT: [[TMP37:%.*]] = and <2 x i32> [[TMP36]], splat (i32 -65536)
+; CHECK-NEXT: [[TMP38:%.*]] = zext <2 x i32> [[TMP37]] to <2 x i64>
+; CHECK-NEXT: [[TMP39:%.*]] = or disjoint <2 x i64> [[TMP35]], [[TMP38]]
+; CHECK-NEXT: [[TMP40:%.*]] = or disjoint <2 x i64> [[TMP39]], [[TMP22]]
+; CHECK-NEXT: [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP40]], i64 0
+; CHECK-NEXT: [[TMP41:%.*]] = insertvalue { i64, i64 } poison, i64 [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT]], 0
+; CHECK-NEXT: [[VEC2STRUCT_SLOT_SROA_0_8_VEC_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP40]], i64 1
+; CHECK-NEXT: [[VEC2STRUCT4:%.*]] = insertvalue { i64, i64 } [[TMP41]], i64 [[VEC2STRUCT_SLOT_SROA_0_8_VEC_EXTRACT]], 1
+; CHECK-NEXT: ret { i64, i64 } [[VEC2STRUCT4]]
;
entry:
%retval = alloca %"struct.std::array8", align 2
@@ -1313,5 +1099,3 @@ for.body: ; preds = %for.cond
%inc = add nuw nsw i64 %i.0, 1
br label %for.cond
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll
index 8466d04f3f87d..9a66b2ca84810 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll
@@ -6,62 +6,59 @@ define i1 @test(ptr %arg, ptr %arg1, i64 %arg2, ptr %arg3) {
; CHECK-SAME: ptr [[ARG:%.*]], ptr [[ARG1:%.*]], i64 [[ARG2:%.*]], ptr [[ARG3:%.*]]) {
; CHECK-NEXT: [[BB:.*:]]
; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i8, ptr [[ARG1]], i64 [[ARG2]]
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x ptr> <ptr poison, ptr null>, ptr [[ARG1]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, <2 x ptr> [[TMP0]], <2 x i64> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x ptr> <ptr poison, ptr null, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison>, ptr [[ARG3]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x ptr> [[TMP2]], <8 x ptr> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0>
-; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, <8 x ptr> [[TMP3]], <8 x i64> <i64 -440, i64 -440, i64 -440, i64 -440, i64 -32, i64 -440, i64 -440, i64 -440>
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x ptr> <ptr null, ptr poison>, ptr [[ARG3]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, <2 x ptr> [[TMP5]], <2 x i64> <i64 -32, i64 -432>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x ptr> [[TMP6]], <2 x ptr> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i8, ptr null, i64 0
+; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr i8, ptr null, i64 -32
+; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[ARG3]], i64 -440
; CHECK-NEXT: [[GETELEMENTPTR7:%.*]] = getelementptr i8, ptr [[ARG1]], i64 0
; CHECK-NEXT: [[GETELEMENTPTR8:%.*]] = getelementptr i8, ptr [[ARG3]], i64 -432
; CHECK-NEXT: [[ICMP:%.*]] = icmp ult ptr null, [[GETELEMENTPTR8]]
; CHECK-NEXT: [[AND:%.*]] = and i1 false, [[ICMP]]
; CHECK-NEXT: [[ICMP9:%.*]] = icmp ult ptr [[GETELEMENTPTR7]], null
; CHECK-NEXT: [[AND10:%.*]] = and i1 [[ICMP9]], false
+; CHECK-NEXT: [[ICMP11:%.*]] = icmp ult ptr [[GETELEMENTPTR7]], null
+; CHECK-NEXT: [[AND37:%.*]] = and i1 [[ICMP11]], false
+; CHECK-NEXT: [[ICMP14:%.*]] = icmp ult ptr [[GETELEMENTPTR4]], [[GETELEMENTPTR8]]
+; CHECK-NEXT: [[AND85:%.*]] = and i1 false, [[ICMP14]]
; CHECK-NEXT: [[ICMP17:%.*]] = icmp ult ptr null, null
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <8 x ptr> <ptr poison, ptr poison, ptr null, ptr null, ptr null, ptr null, ptr null, ptr null>, <8 x ptr> [[TMP8]], <8 x i32> <i32 8, i32 9, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x ptr> [[TMP7]], <4 x ptr> poison, <8 x i32> <i32 poison, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x ptr> [[TMP6]], <2 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x ptr> <ptr null, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison>, <8 x ptr> [[TMP11]], <8 x i32> <i32 0, i32 9, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <8 x ptr> [[TMP12]], <8 x ptr> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP14:%.*]] = icmp ult <8 x ptr> [[TMP9]], [[TMP13]]
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i1> <i1 false, i1 false, i1 poison, i1 false, i1 false, i1 false, i1 false, i1 false>, i1 [[ICMP17]], i32 2
-; CHECK-NEXT: [[TMP16:%.*]] = and <8 x i1> [[TMP14]], [[TMP15]]
; CHECK-NEXT: [[ICMP36:%.*]] = icmp ult ptr null, [[GETELEMENTPTR8]]
-; CHECK-NEXT: [[AND37:%.*]] = and i1 false, [[ICMP36]]
+; CHECK-NEXT: [[TMP33:%.*]] = and i1 [[ICMP17]], [[ICMP36]]
+; CHECK-NEXT: [[ICMP21:%.*]] = icmp ult ptr null, [[GETELEMENTPTR8]]
+; CHECK-NEXT: [[AND22:%.*]] = and i1 false, [[ICMP21]]
+; CHECK-NEXT: [[ICMP24:%.*]] = icmp ult ptr null, [[GETELEMENTPTR8]]
+; CHECK-NEXT: [[AND25:%.*]] = and i1 false, [[ICMP24]]
; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x ptr> <ptr poison, ptr poison, ptr null, ptr null>, ptr [[ARG]], i32 1
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x ptr> [[TMP1]], <2 x ptr> poison, <4 x i32> <i32 0, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <4 x ptr> [[TMP17]], <4 x ptr> [[TMP18]], <4 x i32> <i32 4, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x ptr> [[TMP17]], ptr [[GETELEMENTPTR7]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x ptr> <ptr poison, ptr poison, ptr null, ptr null>, ptr [[GETELEMENTPTR5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x ptr> [[TMP2]], ptr [[GETELEMENTPTR8]], i32 1
; CHECK-NEXT: [[TMP20:%.*]] = icmp ult <4 x ptr> [[TMP19]], [[TMP7]]
-; CHECK-NEXT: [[TMP21:%.*]] = and <4 x i1> zeroinitializer, [[TMP20]]
; CHECK-NEXT: [[ICMP58:%.*]] = icmp ult ptr [[GETELEMENTPTR]], null
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x ptr> <ptr poison, ptr null>, ptr [[GETELEMENTPTR]], i32 0
-; CHECK-NEXT: [[TMP23:%.*]] = icmp ult <2 x ptr> [[TMP22]], splat (ptr null)
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x ptr> <ptr null, ptr null, ptr null, ptr null, ptr poison, ptr poison, ptr null, ptr null>, ptr [[ARG]], i32 4
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <8 x ptr> [[TMP24]], ptr [[ARG1]], i32 5
-; CHECK-NEXT: [[TMP26:%.*]] = icmp ult <8 x ptr> [[TMP25]], [[TMP4]]
-; CHECK-NEXT: [[TMP27:%.*]] = insertelement <8 x i1> <i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 false, i1 false>, i1 [[ICMP58]], i32 1
-; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x i1> [[TMP23]], <2 x i1> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP29:%.*]] = shufflevector <8 x i1> [[TMP27]], <8 x i1> [[TMP28]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 6, i32 7>
-; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <8 x i1> [[TMP29]], <8 x i1> <i1 false, i1 false, i1 undef, i1 undef, i1 undef, i1 undef, i1 undef, i1 undef>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 6, i32 7>
-; CHECK-NEXT: [[TMP31:%.*]] = and <8 x i1> [[TMP30]], [[TMP26]]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <8 x ptr> [[TMP4]], i32 0
+; CHECK-NEXT: [[ICMP62:%.*]] = icmp ult ptr [[GETELEMENTPTR]], null
; CHECK-NEXT: [[ICMP84:%.*]] = icmp ult ptr null, [[TMP32]]
-; CHECK-NEXT: [[AND85:%.*]] = and i1 false, [[ICMP84]]
-; CHECK-NEXT: [[TMP33:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP31]])
-; CHECK-NEXT: [[TMP34:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-; CHECK-NEXT: [[RDX_OP:%.*]] = or <4 x i1> [[TMP34]], [[TMP21]]
-; CHECK-NEXT: [[TMP35:%.*]] = shufflevector <4 x i1> [[RDX_OP]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP36:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> [[TMP35]], <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP37:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP36]])
+; CHECK-NEXT: [[ICMP70:%.*]] = icmp ult ptr [[ARG]], [[GETELEMENTPTR5]]
+; CHECK-NEXT: [[ICMP75:%.*]] = icmp ult ptr [[ARG1]], [[TMP32]]
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <16 x ptr> <ptr poison, ptr poison, ptr poison, ptr poison, ptr null, ptr null, ptr poison, ptr poison, ptr poison, ptr poison, ptr poison, ptr null, ptr null, ptr null, ptr poison, ptr poison>, ptr [[GETELEMENTPTR8]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <16 x ptr> [[TMP5]], ptr [[TMP32]], i32 8
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <16 x ptr> [[TMP6]], <16 x ptr> poison, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 5, i32 0, i32 0, i32 8, i32 8, i32 8, i32 11, i32 12, i32 13, i32 8, i32 8>
+; CHECK-NEXT: [[TMP8:%.*]] = icmp ult <16 x ptr> splat (ptr null), [[TMP18]]
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i1> [[TMP20]], <4 x i1> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <16 x i1> <i1 false, i1 false, i1 false, i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 false, i1 poison, i1 poison, i1 poison, i1 poison, i1 poison, i1 false, i1 false>, <16 x i1> [[TMP9]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 16, i32 17, i32 18, i32 19, i32 8, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 14, i32 15>
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <16 x i1> [[TMP10]], i1 [[ICMP58]], i32 9
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <16 x i1> [[TMP11]], i1 [[ICMP62]], i32 10
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <16 x i1> [[TMP12]], i1 [[ICMP84]], i32 11
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <16 x i1> [[TMP13]], i1 [[ICMP70]], i32 12
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <16 x i1> [[TMP14]], i1 [[ICMP75]], i32 13
+; CHECK-NEXT: [[TMP16:%.*]] = and <16 x i1> [[TMP15]], [[TMP8]]
+; CHECK-NEXT: [[ICMP85:%.*]] = icmp ult ptr null, [[TMP32]]
+; CHECK-NEXT: [[AND86:%.*]] = and i1 false, [[ICMP85]]
+; CHECK-NEXT: [[TMP37:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP16]])
; CHECK-NEXT: [[OP_RDX16:%.*]] = or i1 [[TMP37]], [[AND]]
; CHECK-NEXT: [[OP_RDX17:%.*]] = or i1 [[AND10]], [[AND37]]
; CHECK-NEXT: [[OP_RDX18:%.*]] = or i1 [[AND85]], [[TMP33]]
+; CHECK-NEXT: [[OP_RDX3:%.*]] = or i1 [[AND22]], [[AND25]]
; CHECK-NEXT: [[OP_RDX19:%.*]] = or i1 [[OP_RDX16]], [[OP_RDX17]]
-; CHECK-NEXT: [[OP_RDX20:%.*]] = or i1 [[OP_RDX19]], [[OP_RDX18]]
+; CHECK-NEXT: [[OP_RDX5:%.*]] = or i1 [[OP_RDX18]], [[OP_RDX3]]
+; CHECK-NEXT: [[OP_RDX6:%.*]] = or i1 [[OP_RDX19]], [[OP_RDX5]]
+; CHECK-NEXT: [[OP_RDX20:%.*]] = or i1 [[OP_RDX6]], [[AND86]]
; CHECK-NEXT: ret i1 [[OP_RDX20]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
index d91a600eeaf16..4caa9949d3319 100644
--- a/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll
@@ -7,8 +7,8 @@ define void @partial_vec_invalid_cost() #0 {
; CHECK-LABEL: define void @partial_vec_invalid_cost(
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 1
; CHECK-NEXT: [[LSHR_2:%.*]] = lshr i96 0, 0
+; CHECK-NEXT: [[LSHR_1:%.*]] = lshr i96 0, 1
; CHECK-NEXT: [[ADD_1:%.*]] = add i96 -1, 1
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x i96> poison, i96 [[LSHR_1]], i32 0
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i96> [[TMP0]], i96 [[LSHR_2]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
index ecf9c0bf44c6c..8978d4273f305 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
@@ -424,16 +424,16 @@ define i64 @load64le_nop_shift_disjoint(ptr %arg) {
define void @PR39538(ptr %t0, ptr %t1) {
; CHECK-LABEL: @PR39538(
; CHECK-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr [[T0:%.*]], align 1
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 1, i32 4, i32 9, i32 12>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
; CHECK-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 0, i32 5, i32 8, i32 13>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 1, i32 5, i32 9, i32 13>
; CHECK-NEXT: [[TMP5:%.*]] = zext <4 x i8> [[TMP4]] to <4 x i32>
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 2, i32 6, i32 10, i32 14>
; CHECK-NEXT: [[TMP7:%.*]] = zext <4 x i8> [[TMP6]] to <4 x i32>
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <16 x i8> [[TMP1]], <16 x i8> poison, <4 x i32> <i32 3, i32 7, i32 11, i32 15>
; CHECK-NEXT: [[TMP9:%.*]] = zext <4 x i8> [[TMP8]] to <4 x i32>
-; CHECK-NEXT: [[TMP10:%.*]] = shl nuw <4 x i32> [[TMP3]], <i32 16, i32 24, i32 16, i32 24>
-; CHECK-NEXT: [[TMP11:%.*]] = shl nuw <4 x i32> [[TMP5]], <i32 24, i32 16, i32 24, i32 16>
+; CHECK-NEXT: [[TMP10:%.*]] = shl nuw <4 x i32> [[TMP3]], splat (i32 24)
+; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw <4 x i32> [[TMP5]], splat (i32 16)
; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw <4 x i32> [[TMP7]], splat (i32 8)
; CHECK-NEXT: [[TMP13:%.*]] = or <4 x i32> [[TMP11]], [[TMP10]]
; CHECK-NEXT: [[TMP14:%.*]] = or <4 x i32> [[TMP13]], [[TMP12]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
index aa424b9031e77..2be154b99169a 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll
@@ -11,12 +11,11 @@ define void @test() {
; CHECK-NEXT: [[TMP0:%.*]] = phi <4 x i32> [ poison, %[[BB1]] ], [ [[TMP4:%.*]], %[[BB6]] ]
; CHECK-NEXT: ret void
; CHECK: [[BB6]]:
-; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP8:%.*]], %[[BB6]] ]
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x i32> [ zeroinitializer, %[[BB]] ], [ [[TMP5:%.*]], %[[BB6]] ]
; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 5, i32 4>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> <i32 1, i32 0, i32 poison, i32 poison>, <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP4]] = mul <4 x i32> [[TMP3]], zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> <i32 0, i32 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP8]] = mul <2 x i32> zeroinitializer, [[TMP7]]
+; CHECK-NEXT: [[TMP5]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: br i1 false, label %[[BB2]], label %[[BB6]]
;
bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-reused-with-bv-subvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-reused-with-bv-subvector.ll
index fbf63230b6edf..da6a399370613 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/buildvector-reused-with-bv-subvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/buildvector-reused-with-bv-subvector.ll
@@ -6,9 +6,9 @@ define void @test(ptr %0, i64 %1, i64 %2) {
; CHECK-SAME: ptr [[TMP0:%.*]], i64 [[TMP1:%.*]], i64 [[TMP2:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x ptr> poison, ptr [[TMP0]], i32 0
; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x ptr> [[TMP4]], <4 x ptr> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint <4 x ptr> [[TMP5]] to <4 x i64>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP7]], <4 x i64> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 2, i32 2, i32 1, i32 3, i32 1>
; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint <4 x ptr> [[TMP5]] to <4 x i64>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <8 x i32> <i32 0, i32 0, i32 1, i32 2, i32 2, i32 1, i32 3, i32 1>
+; CHECK-NEXT: [[TMP16:%.*]] = ptrtoint <4 x ptr> [[TMP5]] to <4 x i64>
; CHECK-NEXT: br [[DOTPREHEADER_LR_PH:label %.*]]
; CHECK: [[_PREHEADER_LR_PH:.*:]]
; CHECK-NEXT: br [[DOTPREHEADER_US_US_PREHEADER:label %.*]]
@@ -16,8 +16,8 @@ define void @test(ptr %0, i64 %1, i64 %2) {
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i64> poison, i64 [[TMP1]], i32 0
; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> [[TMP9]], i64 [[TMP2]], i32 1
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 0, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP16]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <4 x i64> [[TMP16]], <4 x i64> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <8 x i64> [[TMP11]], <8 x i64> [[TMP17]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: [[TMP13:%.*]] = or <8 x i64> [[TMP12]], [[TMP8]]
; CHECK-NEXT: br [[DOTPREHEADER_US_US:label %.*]]
; CHECK: [[_PREHEADER_US_US:.*:]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/commutable-node-with-non-sched-parent.ll b/llvm/test/Transforms/SLPVectorizer/X86/commutable-node-with-non-sched-parent.ll
index 8807ca26767f0..c5857bef1caff 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/commutable-node-with-non-sched-parent.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/commutable-node-with-non-sched-parent.ll
@@ -6,21 +6,15 @@ define void @test() {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: br i1 false, label %[[BB1:.*]], label %[[BB9:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[SHL4:%.*]] = shl i32 0, 0
-; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 1, i32 poison>, i32 [[SHL4]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = ashr <2 x i32> <i32 0, i32 -1>, [[TMP0]]
-; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> <i32 0, i32 -1>, [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> [[TMP6]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 0, i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 -1, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; CHECK-NEXT: br label %[[BB6:.*]]
; CHECK: [[BB6]]:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <4 x i32> [ [[TMP4]], %[[BB1]] ]
+; CHECK-NEXT: [[TMP5:%.*]] = phi <4 x i32> [ [[TMP4]], %[[BB1]] ]
; CHECK-NEXT: br label %[[BB9]]
; CHECK: [[BB9]]:
-; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 poison, i32 0>, %[[BB]] ], [ [[TMP10]], %[[BB6]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ <i32 0, i32 0, i32 poison, i32 0>, %[[BB]] ], [ [[TMP5]], %[[BB6]] ]
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP7]], i32 3
; CHECK-NEXT: [[OR:%.*]] = or i32 [[TMP8]], 0
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll b/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
index 8b39816480c1e..0a573de35edeb 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr=+avx2 < %s -slp-threshold=-7| FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux -mattr=+avx2 < %s -slp-threshold=-5| FileCheck %s
define void @test(i8 %a, i8 %b, ptr %p) {
; CHECK-LABEL: define void @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
index 91ccf019d602e..0f8b48f62fb3e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll
@@ -3,15 +3,8 @@
define void @test() {
; CHECK-LABEL: define void @test() {
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i64> poison, i64 1, i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i64> [[TMP2]], <4 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <16 x i64> <i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 0, i64 1, i64 0, i64 undef, i64 undef, i64 undef, i64 undef>, <16 x i64> [[TMP3]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19>
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <16 x i64> [[TMP4]], <16 x i64> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 12, i32 3, i32 4, i32 5, i32 13, i32 6, i32 7, i32 8, i32 14, i32 9, i32 10, i32 11, i32 15>
-; CHECK-NEXT: [[TMP8:%.*]] = trunc <16 x i64> [[TMP7]] to <16 x i1>
-; CHECK-NEXT: [[TMP9:%.*]] = or <16 x i1> [[TMP8]], zeroinitializer
-; CHECK-NEXT: [[TMP10:%.*]] = freeze <16 x i1> [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = icmp eq <16 x i1> [[TMP10]], zeroinitializer
+; CHECK-NEXT: [[TMP1:%.*]] = freeze <16 x i16> <i16 0, i16 0, i16 0, i16 1, i16 0, i16 0, i16 0, i16 1, i16 0, i16 0, i16 0, i16 1, i16 0, i16 1, i16 0, i16 1>
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i16> [[TMP1]], zeroinitializer
; CHECK-NEXT: ret void
;
%xor108.i.i.i = xor i64 0, 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-parent-multi-copyables.ll b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-parent-multi-copyables.ll
index c4ade9aad1b42..9aa0b13096299 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-parent-multi-copyables.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-parent-multi-copyables.ll
@@ -6,14 +6,8 @@ define void @test() {
; CHECK-NEXT: [[BB:.*]]:
; CHECK-NEXT: br i1 false, label %[[BB1:.*]], label %[[BB6:.*]]
; CHECK: [[BB1]]:
-; CHECK-NEXT: [[SHL4:%.*]] = shl i32 0, 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 1, i32 poison>, i32 [[SHL4]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = ashr <2 x i32> <i32 0, i32 -1>, [[TMP5]]
-; CHECK-NEXT: [[TMP8:%.*]] = add <2 x i32> <i32 0, i32 -1>, [[TMP5]]
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> [[TMP8]], <2 x i32> <i32 0, i32 3>
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> poison, i32 0, i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> <i32 0, i32 -1, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 4, i32 5, i32 poison>
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; CHECK-NEXT: br label %[[BB6]]
; CHECK: [[BB6]]:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/recalc-copyable-operand-deps-shared-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/recalc-copyable-operand-deps-shared-inst.ll
index d730dc82ed7bb..32ab8d6356ed4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/recalc-copyable-operand-deps-shared-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/recalc-copyable-operand-deps-shared-inst.ll
@@ -6,29 +6,26 @@ define i32 @test(ptr %r, i32 %0) {
; CHECK-SAME: ptr [[R:%.*]], i32 [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[SHR_I:%.*]] = lshr i32 6, [[TMP0]]
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = add <2 x i32> [[TMP13]], splat (i32 1)
-; CHECK-NEXT: [[TMP16:%.*]] = and <2 x i32> [[TMP13]], splat (i32 1)
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP15]], <2 x i32> [[TMP16]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[TMP0]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> <i32 -1, i32 poison>, i32 [[SHR_I]], i32 1
-; CHECK-NEXT: [[TMP8:%.*]] = sub <2 x i32> [[TMP20]], [[TMP7]]
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP13]], <2 x i32> [[TMP7]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP10:%.*]] = lshr <2 x i32> <i32 8, i32 0>, [[TMP9]]
-; CHECK-NEXT: [[TMP11:%.*]] = sub <2 x i32> <i32 8, i32 0>, [[TMP9]]
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP10]], <2 x i32> [[TMP11]], <2 x i32> <i32 0, i32 3>
-; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[TMP12]], <i32 0, i32 1>
+; CHECK-NEXT: [[SUB8_I:%.*]] = sub i32 0, [[SHR_I]]
+; CHECK-NEXT: [[XOR_I:%.*]] = xor i32 [[SUB8_I]], 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> <i32 -1, i32 poison>, i32 [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = and <2 x i32> <i32 8, i32 1>, [[TMP1]]
+; CHECK-NEXT: [[MUL47_NEG_I:%.*]] = shl i32 [[SUB8_I]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[SUB8_I]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> <i32 1, i32 poison>, <2 x i32> <i32 2, i32 1>
; CHECK-NEXT: [[TMP14:%.*]] = add <2 x i32> [[TMP8]], [[TMP12]]
-; CHECK-NEXT: [[MUL47_NEG_I:%.*]] = extractelement <2 x i32> [[TMP14]], i32 1
; CHECK-NEXT: store i32 [[MUL47_NEG_I]], ptr [[R]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[XOR_I]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = lshr <2 x i32> [[TMP2]], [[TMP3]]
; CHECK-NEXT: [[TMP4:%.*]] = add <2 x i32> [[TMP2]], [[TMP3]]
-; CHECK-NEXT: [[TMP17:%.*]] = add <2 x i32> [[TMP4]], [[TMP14]]
-; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <2 x i32> [[TMP17]], <2 x i32> <i32 -805306497, i32 poison>, <2 x i32> <i32 2, i32 1>
-; CHECK-NEXT: [[TMP19:%.*]] = add <2 x i32> [[TMP17]], [[TMP18]]
+; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP11:%.*]] = add <2 x i32> [[TMP14]], [[TMP10]]
+; CHECK-NEXT: [[TMP19:%.*]] = shl <2 x i32> [[TMP11]], splat (i32 1)
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
+; CHECK-NEXT: [[NOT263_I:%.*]] = add i32 [[TMP5]], -805306497
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP19]], i32 1
-; CHECK-NEXT: [[SUB265_I:%.*]] = or i32 [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[SUB265_I:%.*]] = or i32 [[NOT263_I]], [[TMP6]]
; CHECK-NEXT: ret i32 [[SUB265_I]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduced-ordered-values-update.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduced-ordered-values-update.ll
index 936007313d32d..b83826fba56ac 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduced-ordered-values-update.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduced-ordered-values-update.ll
@@ -6,10 +6,7 @@ define double @test(double %0) {
; CHECK-SAME: double [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP4:%.*]] = fmul double [[TMP0]], 0.000000e+00
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> poison, double [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP2]], <double 0.000000e+00, double -0.000000e+00>
-; CHECK-NEXT: [[ADD35_I_31721:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT: [[ADD35_I_31721:%.*]] = fadd double [[TMP4]], 0.000000e+00
; CHECK-NEXT: [[ADD35_I_1_3:%.*]] = fadd double [[ADD35_I_31721]], [[TMP4]]
; CHECK-NEXT: [[ADD35_I_2_2:%.*]] = fadd double [[ADD35_I_31721]], [[ADD35_I_1_3]]
; CHECK-NEXT: [[ADD35_I_4_1:%.*]] = fadd double [[ADD35_I_2_2]], [[ADD35_I_2_2]]
More information about the llvm-commits
mailing list