[llvm] [SLP] Retune look-ahead scores for constants (PR #207548)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Jul 4 16:06:00 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-transforms
Author: Alexey Bataev (alexey-bataev)
<details>
<summary>Changes</summary>
Rescale LookAheadHeuristics scores and add ScoreSameConstants /
ScoreConstantScaleFactor so identical constants no longer score the
same as same-opcode instruction matches, fixing suboptimal operand
reordering around repeated constants (shift amounts, splat values).
Also retry a SplitVectorize alt-shuffle build for copyable-element
bundles when scheduling fails, instead of always falling back to a
gather, to fix regressions.
---
Patch is 74.15 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/207548.diff
13 Files Affected:
- (modified) llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp (+36-17)
- (modified) llvm/test/Transforms/PhaseOrdering/X86/avg.ll (+88-304)
- (modified) llvm/test/Transforms/SLPVectorizer/AArch64/long-non-power-of-2.ll (+37-40)
- (modified) llvm/test/Transforms/SLPVectorizer/RISCV/partial-vec-invalid-cost.ll (+1-1)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll (+4-4)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/buildvector-postpone-for-dependency.ll (+3-4)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/buildvector-reused-with-bv-subvector.ll (+4-4)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/commutable-node-with-non-sched-parent.ll (+3-9)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/debug-info-salvage.ll (+1-1)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/non-power-of-2-subvectors-insert.ll (+2-9)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/non-schedulable-parent-multi-copyables.ll (+1-7)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/recalc-copyable-operand-deps-shared-inst.ll (+15-18)
- (modified) llvm/test/Transforms/SLPVectorizer/X86/reduced-ordered-values-update.ll (+1-4)
``````````diff
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 92befc3c2ff9b..ccef34850670a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -2649,34 +2649,36 @@ class slpvectorizer::BoUpSLP {
// tree entry node in the different lane.
/// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
- static const int ScoreConsecutiveLoads = 4;
+ static constexpr int ScoreConsecutiveLoads = 40;
/// The same load multiple times. This should have a better score than
/// `ScoreSplat` because it in x86 for a 2-lane vector we can represent it
/// with `movddup (%reg), xmm0` which has a throughput of 0.5 versus 0.5 for
/// a vector load and 1.0 for a broadcast.
- static const int ScoreSplatLoads = 3;
+ static constexpr int ScoreSplatLoads = 30;
/// Loads from reversed memory addresses, e.g. load(A[i+1]), load(A[i]).
- static const int ScoreReversedLoads = 3;
+ static constexpr int ScoreReversedLoads = 30;
/// A load candidate for masked gather.
- static const int ScoreMaskedGatherCandidate = 1;
+ static constexpr int ScoreMaskedGatherCandidate = 10;
/// ExtractElementInst from same vector and consecutive indexes.
- static const int ScoreConsecutiveExtracts = 4;
+ static constexpr int ScoreConsecutiveExtracts = 40;
/// ExtractElementInst from same vector and reversed indices.
- static const int ScoreReversedExtracts = 3;
+ static constexpr int ScoreReversedExtracts = 30;
/// Constants.
- static const int ScoreConstants = 2;
+ static constexpr int ScoreConstants = 15;
+ /// Same constants.
+ static constexpr int ScoreSameConstants = 17;
/// Instructions with the same opcode.
- static const int ScoreSameOpcode = 2;
+ static constexpr int ScoreSameOpcode = 20;
/// Instructions with alt opcodes (e.g, add + sub).
- static const int ScoreAltOpcodes = 1;
+ static constexpr int ScoreAltOpcodes = 10;
/// Identical instructions (a.k.a. splat or broadcast).
- static const int ScoreSplat = 1;
+ static constexpr int ScoreSplat = 10;
/// Matching with an undef is preferable to failing.
- static const int ScoreUndef = 1;
+ static constexpr int ScoreUndef = 10;
/// Score for failing to find a decent match.
- static const int ScoreFail = 0;
+ static constexpr int ScoreFail = 0;
/// Score if all users are vectorized.
- static const int ScoreAllUserVectorized = 1;
+ static constexpr int ScoreAllUserVectorized = 10;
/// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
/// \p U1 and \p U2 are the users of \p V1 and \p V2.
@@ -2710,6 +2712,10 @@ class slpvectorizer::BoUpSLP {
AllUsersAreInternal(V1, V2)))
return LookAheadHeuristics::ScoreSplatLoads;
}
+ if (isa<UndefValue>(V1))
+ return LookAheadHeuristics::ScoreUndef;
+ if (isConstant(V1))
+ return LookAheadHeuristics::ScoreSameConstants;
return LookAheadHeuristics::ScoreSplat;
}
@@ -2761,7 +2767,7 @@ class slpvectorizer::BoUpSLP {
// Consider constants and buildvector compatible.
if ((C1 && isa<InsertElementInst>(V2)) ||
(C2 && isa<InsertElementInst>(V1)))
- return LookAheadHeuristics::ScoreConstants;
+ return LookAheadHeuristics::ScoreSameOpcode;
// Extracts from consecutive indexes of the same vector better score as
// the extracts could be optimized away.
@@ -3112,7 +3118,9 @@ class slpvectorizer::BoUpSLP {
/// Score scaling factor for fully compatible instructions but with
/// different number of external uses. Allows better selection of the
/// instructions with less external uses.
- static const int ScoreScaleFactor = 10;
+ static constexpr int ScoreScaleFactor = 10;
+ /// Scale factor for constants only.
+ static constexpr int ScoreConstantScaleFactor = 6;
/// \Returns the look-ahead score, which tells us how much the sub-trees
/// rooted at \p LHS and \p RHS match, the more they match the higher the
@@ -3130,7 +3138,8 @@ class slpvectorizer::BoUpSLP {
LookAhead.getScoreAtLevelRec(LHS, RHS, /*U1=*/nullptr, /*U2=*/nullptr,
/*CurrLevel=*/1, MainAltOps);
if (Score) {
- int SplatScore = getSplatScore(Lane, OpIdx, Idx, UsedLanes);
+ int SplatScore =
+ getSplatScore(Lane, OpIdx, Idx, UsedLanes) * ScoreScaleFactor;
if (Score <= -SplatScore) {
// Failed score.
Score = 0;
@@ -3141,7 +3150,10 @@ class slpvectorizer::BoUpSLP {
// uses. It does not affect actual selection of the best
// compatible operand in general, just allows to select the
// operand with all vectorized uses.
- Score *= ScoreScaleFactor;
+ const int SF = (LHS == RHS && isConstant(LHS))
+ ? ScoreConstantScaleFactor
+ : ScoreScaleFactor;
+ Score *= SF;
Score += getExternalUseScore(Lane, OpIdx, Idx);
IsUsed = true;
}
@@ -13188,6 +13200,13 @@ void BoUpSLP::buildTreeRec(ArrayRef<Value *> VLRef, unsigned Depth,
// Last chance to try to vectorize alternate node.
if (S.isAltShuffle() && ReuseShuffleIndices.empty() && TrySplitNode(S))
return;
+ // Last chance to try to vectorize copyable node.
+ if (S.areInstructionsWithCopyableElements() &&
+ ReuseShuffleIndices.empty()) {
+ InstructionsState AltS = getSameOpcode(VL, *TLI);
+ if (AltS && AltS.isAltShuffle() && TrySplitNode(AltS))
+ return;
+ }
newGatherTreeEntry(VL, S, UserTreeIdx, ReuseShuffleIndices);
NonScheduledFirst.insert(VL.front());
if (S.getOpcode() == Instruction::Load &&
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 270b17cd18085..9084407f008ae 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -26,9 +26,8 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE2-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
-; SSE2-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE2-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
-; SSE2-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
+; SSE2-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
+; SSE2-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
; SSE2-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; SSE2-NEXT: [[TMP21:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
@@ -39,16 +38,11 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE2-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE2-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
-; SSE2-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE2-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
+; SSE2-NEXT: [[TMP47:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 48)
+; SSE2-NEXT: [[TMP48:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 56)
; SSE2-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE2-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
-; SSE2-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
-; SSE2-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56
-; SSE2-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE2-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
+; SSE2-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
; SSE2-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE2-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE2-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
@@ -72,29 +66,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE2-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE2-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE2-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE2-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
-; SSE2-NEXT: [[ADD5_14:%.*]] = add nuw nsw i64 [[ADD_14]], [[CONV4_14]]
-; SSE2-NEXT: [[ADD5_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_8_0_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD_15:%.*]] = add nuw nsw i64 [[A_SROA_17_8_EXTRACT_SHIFT]], 1
-; SSE2-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV1_6]], 1
-; SSE2-NEXT: [[ADD5_15:%.*]] = add nuw nsw i64 [[ADD_15]], [[B_SROA_17_8_EXTRACT_SHIFT]]
-; SSE2-NEXT: [[ADD5_6:%.*]] = add nuw nsw i64 [[ADD_6]], [[CONV4_6]]
-; SSE2-NEXT: [[TMP45:%.*]] = shl nuw i64 [[ADD5_15]], 55
-; SSE2-NEXT: [[TMP46:%.*]] = shl nuw nsw i64 [[ADD5_6]], 47
-; SSE2-NEXT: [[RETVAL_SROA_17_8_INSERT_EXT:%.*]] = and i64 [[TMP45]], -72057594037927936
-; SSE2-NEXT: [[RETVAL_SROA_7_0_INSERT_SHIFT:%.*]] = and i64 [[TMP46]], 71776119061217280
-; SSE2-NEXT: [[TMP47:%.*]] = shl nuw nsw i64 [[ADD5_14]], 47
-; SSE2-NEXT: [[TMP48:%.*]] = shl nuw i64 [[ADD5_7]], 55
-; SSE2-NEXT: [[RETVAL_SROA_16_8_INSERT_SHIFT:%.*]] = and i64 [[TMP47]], 71776119061217280
-; SSE2-NEXT: [[RETVAL_SROA_8_0_INSERT_EXT:%.*]] = and i64 [[TMP48]], -72057594037927936
-; SSE2-NEXT: [[RETVAL_SROA_16_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_17_8_INSERT_EXT]], [[RETVAL_SROA_16_8_INSERT_SHIFT]]
-; SSE2-NEXT: [[RETVAL_SROA_7_0_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_8_0_INSERT_EXT]], [[RETVAL_SROA_7_0_INSERT_SHIFT]]
+; SSE2-NEXT: [[TMP71:%.*]] = and <2 x i64> [[TMP47]], splat (i64 255)
+; SSE2-NEXT: [[TMP51:%.*]] = add nuw nsw <2 x i64> [[TMP70]], splat (i64 1)
+; SSE2-NEXT: [[TMP72:%.*]] = add nuw nsw <2 x i64> [[TMP51]], [[TMP71]]
+; SSE2-NEXT: [[TMP73:%.*]] = add nuw nsw <2 x i64> [[TMP46]], splat (i64 1)
+; SSE2-NEXT: [[TMP74:%.*]] = add nuw nsw <2 x i64> [[TMP73]], [[TMP48]]
+; SSE2-NEXT: [[TMP75:%.*]] = shl nuw <2 x i64> [[TMP74]], splat (i64 55)
+; SSE2-NEXT: [[TMP76:%.*]] = and <2 x i64> [[TMP75]], splat (i64 -72057594037927936)
+; SSE2-NEXT: [[TMP77:%.*]] = shl nuw nsw <2 x i64> [[TMP72]], splat (i64 47)
+; SSE2-NEXT: [[TMP78:%.*]] = and <2 x i64> [[TMP77]], splat (i64 71776119061217280)
+; SSE2-NEXT: [[TMP52:%.*]] = or disjoint <2 x i64> [[TMP76]], [[TMP78]]
; SSE2-NEXT: [[TMP49:%.*]] = shl nuw nsw <2 x i64> [[TMP44]], splat (i64 39)
; SSE2-NEXT: [[TMP50:%.*]] = and <2 x i64> [[TMP49]], splat (i64 280375465082880)
-; SSE2-NEXT: [[TMP51:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_7_0_INSERT_INSERT]], i64 0
-; SSE2-NEXT: [[TMP52:%.*]] = insertelement <2 x i64> [[TMP51]], i64 [[RETVAL_SROA_16_8_INSERT_INSERT]], i64 1
; SSE2-NEXT: [[TMP53:%.*]] = or disjoint <2 x i64> [[TMP52]], [[TMP50]]
; SSE2-NEXT: [[TMP54:%.*]] = shl nuw nsw <2 x i64> [[TMP40]], splat (i64 31)
; SSE2-NEXT: [[TMP55:%.*]] = and <2 x i64> [[TMP54]], splat (i64 1095216660480)
@@ -125,10 +108,9 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE4-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE4-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
-; SSE4-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE4-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
+; SSE4-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
+; SSE4-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
; SSE4-NEXT: [[TMP7:%.*]] = trunc i64 [[A_COERCE1]] to i16
-; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
; SSE4-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
; SSE4-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1:%.*]], i64 1
@@ -136,23 +118,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE4-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE4-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
-; SSE4-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
+; SSE4-NEXT: [[TMP47:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 48)
+; SSE4-NEXT: [[TMP48:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 56)
; SSE4-NEXT: [[TMP15:%.*]] = trunc i64 [[B_COERCE1]] to i16
-; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
; SSE4-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE4-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
-; SSE4-NEXT: [[CONV1_6:%.*]] = and i64 [[A_SROA_7_0_EXTRACT_SHIFT]], 255
-; SSE4-NEXT: [[CONV1_14:%.*]] = and i64 [[A_SROA_16_8_EXTRACT_SHIFT]], 255
+; SSE4-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
; SSE4-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
; SSE4-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1
; SSE4-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8)
-; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
-; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 56
; SSE4-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
; SSE4-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1
; SSE4-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
-; SSE4-NEXT: [[CONV4_6:%.*]] = and i64 [[B_SROA_7_0_EXTRACT_SHIFT]], 255
; SSE4-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE4-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE4-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
@@ -174,29 +151,18 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE4-NEXT: [[TMP42:%.*]] = and <2 x i64> [[TMP14]], splat (i64 255)
; SSE4-NEXT: [[TMP43:%.*]] = add nuw nsw <2 x i64> [[TMP41]], splat (i64 1)
; SSE4-NEXT: [[TMP44:%.*]] = add nuw nsw <2 x i64> [[TMP43]], [[TMP42]]
-; SSE4-NEXT: [[CONV4_14:%.*]] = and i64 [[B_SROA_16_8_EXTRACT_SHIFT]], 255
-; SSE4-NEXT: [[ADD_7:%.*]] = add nuw nsw i64 [[A_SROA_8_0_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_14:%.*]] = add nuw nsw i64 [[CONV1_14]], 1
-; SSE4-NEXT: [[ADD5_14:%.*]] = add nuw nsw i64 [[ADD_14]], [[CONV4_14]]
-; SSE4-NEXT: [[ADD5_7:%.*]] = add nuw nsw i64 [[ADD_7]], [[B_SROA_8_0_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD_15:%.*]] = add nuw nsw i64 [[A_SROA_17_8_EXTRACT_SHIFT]], 1
-; SSE4-NEXT: [[ADD_6:%.*]] = add nuw nsw i64 [[CONV1_6]], 1
-; SSE4-NEXT: [[ADD5_15:%.*]] = add nuw nsw i64 [[ADD_15]], [[B_SROA_17_8_EXTRACT_SHIFT]]
-; SSE4-NEXT: [[ADD5_6:%.*]] = add nuw nsw i64 [[ADD_6]], [[CONV4_6]]
-; SSE4-NEXT: [[TMP45:%.*]] = shl nuw i64 [[ADD5_15]], 55
-; SSE4-NEXT: [[TMP46:%.*]] = shl nuw nsw i64 [[ADD5_6]], 47
-; SSE4-NEXT: [[RETVAL_SROA_17_8_INSERT_EXT:%.*]] = and i64 [[TMP45]], -72057594037927936
-; SSE4-NEXT: [[RETVAL_SROA_7_0_INSERT_SHIFT:%.*]] = and i64 [[TMP46]], 71776119061217280
-; SSE4-NEXT: [[TMP47:%.*]] = shl nuw nsw i64 [[ADD5_14]], 47
-; SSE4-NEXT: [[TMP48:%.*]] = shl nuw i64 [[ADD5_7]], 55
-; SSE4-NEXT: [[RETVAL_SROA_16_8_INSERT_SHIFT:%.*]] = and i64 [[TMP47]], 71776119061217280
-; SSE4-NEXT: [[RETVAL_SROA_8_0_INSERT_EXT:%.*]] = and i64 [[TMP48]], -72057594037927936
-; SSE4-NEXT: [[RETVAL_SROA_16_8_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_17_8_INSERT_EXT]], [[RETVAL_SROA_16_8_INSERT_SHIFT]]
-; SSE4-NEXT: [[RETVAL_SROA_7_0_INSERT_INSERT:%.*]] = or disjoint i64 [[RETVAL_SROA_8_0_INSERT_EXT]], [[RETVAL_SROA_7_0_INSERT_SHIFT]]
+; SSE4-NEXT: [[TMP71:%.*]] = and <2 x i64> [[TMP47]], splat (i64 255)
+; SSE4-NEXT: [[TMP51:%.*]] = add nuw nsw <2 x i64> [[TMP70]], splat (i64 1)
+; SSE4-NEXT: [[TMP72:%.*]] = add nuw nsw <2 x i64> [[TMP51]], [[TMP71]]
+; SSE4-NEXT: [[TMP73:%.*]] = add nuw nsw <2 x i64> [[TMP46]], splat (i64 1)
+; SSE4-NEXT: [[TMP74:%.*]] = add nuw nsw <2 x i64> [[TMP73]], [[TMP48]]
+; SSE4-NEXT: [[TMP75:%.*]] = shl nuw <2 x i64> [[TMP74]], splat (i64 55)
+; SSE4-NEXT: [[TMP76:%.*]] = and <2 x i64> [[TMP75]], splat (i64 -72057594037927936)
+; SSE4-NEXT: [[TMP77:%.*]] = shl nuw nsw <2 x i64> [[TMP72]], splat (i64 47)
+; SSE4-NEXT: [[TMP78:%.*]] = and <2 x i64> [[TMP77]], splat (i64 71776119061217280)
+; SSE4-NEXT: [[TMP52:%.*]] = or disjoint <2 x i64> [[TMP76]], [[TMP78]]
; SSE4-NEXT: [[TMP49:%.*]] = shl nuw nsw <2 x i64> [[TMP44]], splat (i64 39)
; SSE4-NEXT: [[TMP50:%.*]] = and <2 x i64> [[TMP49]], splat (i64 280375465082880)
-; SSE4-NEXT: [[TMP51:%.*]] = insertelement <2 x i64> poison, i64 [[RETVAL_SROA_7_0_INSERT_INSERT]], i64 0
-; SSE4-NEXT: [[TMP52:%.*]] = insertelement <2 x i64> [[TMP51]], i64 [[RETVAL_SROA_16_8_INSERT_INSERT]], i64 1
; SSE4-NEXT: [[TMP53:%.*]] = or disjoint <2 x i64> [[TMP52]], [[TMP50]]
; SSE4-NEXT: [[TMP54:%.*]] = shl nuw nsw <2 x i64> [[TMP40]], splat (i64 31)
; SSE4-NEXT: [[TMP55:%.*]] = and <2 x i64> [[TMP54]], splat (i64 1095216660480)
@@ -242,38 +208,38 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; AVX-NEXT: [[ADD5_9:%.*]] = add nuw nsw i16 [[ADD_9]], [[TMP7]]
; AVX-NEXT: [[TMP8:%.*]] = shl nuw i16 [[ADD5_1]], 7
; AVX-NEXT: [[TMP9:%.*]] = and i16 [[TMP8]], -256
-; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[A_COERCE0]], i64 0
+; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x i64> <i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 poison, i64 -1, i64 -1>, i64 [[B_COERCE0]], i64 0
; AVX-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 7>
; AVX-NEXT: [[TMP12:%.*]] = lshr <8 x i64> [[TMP11]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 0, i64 0>
-; AVX-NEXT: [[TMP13:%.*]] = and <8 x i64> [[TMP12]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 0, i64 0>
; AVX-NEXT: [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED:%.*]] = zext i16 [[TMP9]] to i64
-; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> poison, i64 [[B_COERCE0]], i64 0
+; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x i64> poison, i64 [[A_COERCE0]], i64 0
; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x i64> [[TMP14]], i64 [[ADD5]], i64 1
; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x i64> [[TMP15]], i64 [[RETVAL_SROA_2_0_INSERT_SHIFT_MASKED]], i64 2
; AVX-NEXT: [[TMP17:%.*]] = shufflevector <8 x i64> [[TMP16]], <8 x i64> poison, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 2>
; AVX-NEXT: [[TMP18:%.*]] = lshr <8 x i64> [[TMP17]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 1, i64 0>
; AVX-NEXT: [[TMP19:%.*]] = and <8 x i64> [[TMP18]], <i64 -1, i64 255, i64 255, i64 255, i64 255, i64 255, i64 -1, i64 -1>
-; AVX-NEXT: [[TMP20:%.*]] = add nuw nsw <8 x i64> [[T...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/207548
More information about the llvm-commits
mailing list