[llvm] d7c1e60 - [SLP]Recalculate gather costs after tree trimming
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 30 07:30:07 PDT 2026
Author: Alexey Bataev
Date: 2026-07-30T10:30:02-04:00
New Revision: d7c1e6054fb66a5954a6403d3803f7bc5f1eb449
URL: https://github.com/llvm/llvm-project/commit/d7c1e6054fb66a5954a6403d3803f7bc5f1eb449
DIFF: https://github.com/llvm/llvm-project/commit/d7c1e6054fb66a5954a6403d3803f7bc5f1eb449.diff
LOG: [SLP]Recalculate gather costs after tree trimming
Gather node costs are computed against the set of vectorized nodes
available for reuse. Tree trimming changes that set, but the costs were
not recalculated: a gather that reused a trimmed-away vectorized value
kept its 0 cost, and the node transformed to a gather matched the same
sibling gather for a free reuse, so the buildvector cost was lost and
unprofitable trees were vectorized.
Recalculate costs of all gather nodes after trimming.
Fixes #212983
Reviewers:
Pull Request: https://github.com/llvm/llvm-project/pull/213034
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/PhaseOrdering/X86/avg.ll
llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-node-gather-cost.ll
llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll
llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index c2b8d81ff5d41..7feb31a2dc019 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -19411,18 +19411,21 @@ BoUpSLP::calculateTreeCostAndTrimNonProfitable(ArrayRef<Value *> VectorizedVals,
}
if (DeletedNodes.contains(TE.get()))
continue;
- if (!NodesCosts.contains(TE.get())) {
+ // Gather costs depend on the set of vectorized nodes available for
+ // reuse, which changes during trimming, so recalculate them for all
+ // gather nodes, not just for the transformed ones.
+ if (TE->isGather() || !NodesCosts.contains(TE.get())) {
InstructionCost C =
getEntryCost(TE.get(), VectorizedVals, CheckedExtracts);
if (!C.isValid() || C == 0) {
- NodesCosts.try_emplace(TE.get(), C);
+ NodesCosts[TE.get()] = C;
continue;
}
uint64_t Scale = EntryToScale.lookup(TE.get());
if (!Scale)
Scale = getEntryEffectiveScale(*TE);
C *= Scale;
- NodesCosts.try_emplace(TE.get(), C);
+ NodesCosts[TE.get()] = C;
}
}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
index 9084407f008ae..9ab29f23ab197 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/avg.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE2
; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE4
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,SSE2
; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE4
-; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX
; PR128424
@@ -16,24 +16,18 @@
define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, i64 %b.coerce1) {
; SSE2-LABEL: @avgr_16_u8(
; SSE2-NEXT: entry:
-; SSE2-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
-; SSE2-NEXT: [[TMP7:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; SSE2-NEXT: [[TMP15:%.*]] = trunc i64 [[A_COERCE1:%.*]] to i16
-; SSE2-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> [[TMP7]], i16 [[TMP15]], i64 1
-; SSE2-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
-; SSE2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1]], i64 1
+; SSE2-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
+; SSE2-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1:%.*]], i64 1
+; SSE2-NEXT: [[TMP7:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i16>
; SSE2-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 16)
; SSE2-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE2-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE2-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
; SSE2-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
; SSE2-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
-; SSE2-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
-; SSE2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
-; SSE2-NEXT: [[TMP21:%.*]] = trunc i64 [[B_COERCE1:%.*]] to i16
-; SSE2-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP19]], i16 [[TMP21]], i64 1
-; SSE2-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
-; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1]], i64 1
+; SSE2-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
+; SSE2-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1:%.*]], i64 1
+; SSE2-NEXT: [[TMP15:%.*]] = trunc <2 x i64> [[TMP10]] to <2 x i16>
; SSE2-NEXT: [[TMP11:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 16)
; SSE2-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE2-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
@@ -43,11 +37,11 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
; SSE2-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE2-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
; SSE2-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
+; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP7]], splat (i16 8)
+; SSE2-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP15]], splat (i16 8)
; SSE2-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE2-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
; SSE2-NEXT: [[TMP26:%.*]] = lshr <2 x i64> [[TMP25]], splat (i64 1)
-; SSE2-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP18]], splat (i16 8)
-; SSE2-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
; SSE2-NEXT: [[TMP27:%.*]] = add nuw nsw <2 x i16> [[TMP20]], splat (i16 1)
; SSE2-NEXT: [[TMP28:%.*]] = add nuw nsw <2 x i16> [[TMP27]], [[TMP23]]
; SSE2-NEXT: [[TMP29:%.*]] = and <2 x i64> [[TMP3]], splat (i64 255)
@@ -101,34 +95,28 @@ define { i64, i64 } @avgr_16_u8(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0,
;
; SSE4-LABEL: @avgr_16_u8(
; SSE4-NEXT: entry:
-; SSE4-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i16
-; SSE4-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
+; SSE4-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
; SSE4-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1:%.*]], i64 1
+; SSE4-NEXT: [[TMP19:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i16>
; SSE4-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 16)
; SSE4-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 24)
; SSE4-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; SSE4-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 40)
; SSE4-NEXT: [[TMP45:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
; SSE4-NEXT: [[TMP46:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 56)
-; SSE4-NEXT: [[TMP7:%.*]] = trunc i64 [[A_COERCE1]] to i16
-; SSE4-NEXT: [[TMP8:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i16
-; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
+; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
; SSE4-NEXT: [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[B_COERCE1:%.*]], i64 1
+; SSE4-NEXT: [[TMP22:%.*]] = trunc <2 x i64> [[TMP10]] to <2 x i16>
; SSE4-NEXT: [[TMP11:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 16)
; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 24)
; SSE4-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 32)
; SSE4-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 40)
; SSE4-NEXT: [[TMP47:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 48)
; SSE4-NEXT: [[TMP48:%.*]] = lshr <2 x i64> [[TMP10]], splat (i64 56)
-; SSE4-NEXT: [[TMP15:%.*]] = trunc i64 [[B_COERCE1]] to i16
; SSE4-NEXT: [[TMP16:%.*]] = and <2 x i64> [[TMP2]], splat (i64 255)
; SSE4-NEXT: [[TMP17:%.*]] = and <2 x i64> [[TMP10]], splat (i64 255)
; SSE4-NEXT: [[TMP70:%.*]] = and <2 x i64> [[TMP45]], splat (i64 255)
-; SSE4-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
-; SSE4-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> [[TMP18]], i16 [[TMP7]], i64 1
; SSE4-NEXT: [[TMP20:%.*]] = lshr <2 x i16> [[TMP19]], splat (i16 8)
-; SSE4-NEXT: [[TMP21:%.*]] = insertelement <2 x i16> poison, i16 [[TMP8]], i64 0
-; SSE4-NEXT: [[TMP22:%.*]] = insertelement <2 x i16> [[TMP21]], i16 [[TMP15]], i64 1
; SSE4-NEXT: [[TMP23:%.*]] = lshr <2 x i16> [[TMP22]], splat (i16 8)
; SSE4-NEXT: [[TMP24:%.*]] = add nuw nsw <2 x i64> [[TMP16]], splat (i64 1)
; SSE4-NEXT: [[TMP25:%.*]] = add nuw nsw <2 x i64> [[TMP24]], [[TMP17]]
@@ -497,117 +485,129 @@ define { i64, i64 } @avgr_16_u8_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
;
; SSE4-LABEL: @avgr_16_u8_alt(
; SSE4-NEXT: entry:
-; SSE4-NEXT: [[A_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0:%.*]], 56
+; SSE4-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
+; SSE4-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> [[TMP0]], i64 [[A_COERCE1:%.*]], i64 1
+; SSE4-NEXT: [[TMP2:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i8>
+; SSE4-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 8)
+; SSE4-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 16)
+; SSE4-NEXT: [[TMP5:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 24)
+; SSE4-NEXT: [[TMP6:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 32)
+; SSE4-NEXT: [[TMP7:%.*]] = lshr <2 x i64> [[TMP1]], splat (i64 40)
; SSE4-NEXT: [[A_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 48
-; SSE4-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 40
-; SSE4-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32
-; SSE4-NEXT: [[A_SROA_5_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 24
-; SSE4-NEXT: [[A_SROA_6_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 16
-; SSE4-NEXT: [[A_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 8
-; SSE4-NEXT: [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_8_0_EXTRACT_SHIFT]] to i8
; SSE4-NEXT: [[A_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_2_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_3_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_4_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_5_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_5_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_6_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_6_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_7_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_7_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_8_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE0]] to i8
-; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0:%.*]], 56
-; SSE4-NEXT: [[B_SROA_7_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 48
-; SSE4-NEXT: [[B_SROA_6_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 40
-; SSE4-NEXT: [[B_SROA_5_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32
-; SSE4-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 24
-; SSE4-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 16
-; SSE4-NEXT: [[B_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 8
+; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 56
; SSE4-NEXT: [[B_SROA_8_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_8_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_7_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_6_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_5_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_5_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_4_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_6_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_3_0_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_7_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_2_0_EXTRACT_SHIFT]] to i8
+; SSE4-NEXT: [[B_COERCE0:%.*]] = lshr i64 [[A_COERCE1]], 48
; SSE4-NEXT: [[B_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_COERCE0]] to i8
-; SSE4-NEXT: [[TMP0:%.*]] = insertelement <8 x i8> poison, i8 [[A_SROA_8_0_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP1:%.*]] = insertelement <8 x i8> [[TMP0]], i8 [[A_SROA_7_0_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP2:%.*]] = insertelement <8 x i8> [[TMP1]], i8 [[A_SROA_6_0_EXTRACT_TRUNC]], i64 2
-; SSE4-NEXT: [[TMP3:%.*]] = insertelement <8 x i8> [[TMP2]], i8 [[A_SROA_5_0_EXTRACT_TRUNC]], i64 3
-; SSE4-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> [[TMP3]], i8 [[A_SROA_4_0_EXTRACT_TRUNC]], i64 4
-; SSE4-NEXT: [[TMP5:%.*]] = insertelement <8 x i8> [[TMP4]], i8 [[A_SROA_3_0_EXTRACT_TRUNC]], i64 5
-; SSE4-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> [[TMP5]], i8 [[A_SROA_2_0_EXTRACT_TRUNC]], i64 6
-; SSE4-NEXT: [[TMP7:%.*]] = insertelement <8 x i8> [[TMP6]], i8 [[A_SROA_0_0_EXTRACT_TRUNC]], i64 7
-; SSE4-NEXT: [[TMP8:%.*]] = lshr <8 x i8> [[TMP7]], splat (i8 1)
-; SSE4-NEXT: [[TMP9:%.*]] = insertelement <8 x i8> poison, i8 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP10:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[B_SROA_7_0_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP11:%.*]] = insertelement <8 x i8> [[TMP10]], i8 [[B_SROA_6_0_EXTRACT_TRUNC]], i64 2
-; SSE4-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP11]], i8 [[B_SROA_5_0_EXTRACT_TRUNC]], i64 3
-; SSE4-NEXT: [[TMP13:%.*]] = insertelement <8 x i8> [[TMP12]], i8 [[B_SROA_4_0_EXTRACT_TRUNC]], i64 4
-; SSE4-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[B_SROA_3_0_EXTRACT_TRUNC]], i64 5
-; SSE4-NEXT: [[TMP15:%.*]] = insertelement <8 x i8> [[TMP14]], i8 [[B_SROA_2_0_EXTRACT_TRUNC]], i64 6
-; SSE4-NEXT: [[TMP16:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[B_SROA_8_0_EXTRACT_TRUNC]], i64 7
-; SSE4-NEXT: [[TMP17:%.*]] = lshr <8 x i8> [[TMP16]], splat (i8 1)
-; SSE4-NEXT: [[TMP18:%.*]] = add nuw <8 x i8> [[TMP17]], [[TMP8]]
-; SSE4-NEXT: [[TMP19:%.*]] = or <8 x i8> [[TMP16]], [[TMP7]]
-; SSE4-NEXT: [[TMP20:%.*]] = and <8 x i8> [[TMP19]], splat (i8 1)
-; SSE4-NEXT: [[TMP21:%.*]] = add nuw <8 x i8> [[TMP18]], [[TMP20]]
-; SSE4-NEXT: [[TMP22:%.*]] = zext <8 x i8> [[TMP21]] to <8 x i64>
-; SSE4-NEXT: [[TMP23:%.*]] = shl nuw <8 x i64> [[TMP22]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
-; SSE4-NEXT: [[TMP24:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP23]])
-; SSE4-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP24]], 0
-; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1:%.*]], 56
-; SSE4-NEXT: [[A_SROA_11_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 48
-; SSE4-NEXT: [[A_SROA_12_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 40
-; SSE4-NEXT: [[A_SROA_13_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 32
-; SSE4-NEXT: [[A_SROA_14_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 24
-; SSE4-NEXT: [[A_SROA_15_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 16
-; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 8
+; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 56
; SSE4-NEXT: [[A_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_17_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_11_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_11_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_12_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_12_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_13_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_13_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_14_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_14_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_15_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_15_8_EXTRACT_SHIFT]] to i8
+; SSE4-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE1:%.*]], i64 0
+; SSE4-NEXT: [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[B_COERCE2:%.*]], i64 1
+; SSE4-NEXT: [[TMP10:%.*]] = trunc <2 x i64> [[TMP9]] to <2 x i8>
+; SSE4-NEXT: [[TMP11:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 8)
+; SSE4-NEXT: [[TMP12:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 16)
+; SSE4-NEXT: [[TMP13:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 24)
+; SSE4-NEXT: [[TMP14:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 32)
+; SSE4-NEXT: [[TMP15:%.*]] = lshr <2 x i64> [[TMP9]], splat (i64 40)
+; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
; SSE4-NEXT: [[A_SROA_16_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_16_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[A_SROA_17_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE1]] to i8
-; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1:%.*]], 56
-; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 48
-; SSE4-NEXT: [[B_SROA_15_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 40
-; SSE4-NEXT: [[B_SROA_14_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; SSE4-NEXT: [[B_SROA_13_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 24
-; SSE4-NEXT: [[B_SROA_12_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 16
-; SSE4-NEXT: [[B_SROA_11_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 8
+; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 56
; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_17_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_11_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_16_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_12_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_15_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_13_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_14_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_14_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_13_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_15_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_12_8_EXTRACT_SHIFT]] to i8
+; SSE4-NEXT: [[B_SROA_11_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE2]], 48
; SSE4-NEXT: [[B_SROA_16_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_11_8_EXTRACT_SHIFT]] to i8
-; SSE4-NEXT: [[B_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_COERCE1]] to i8
-; SSE4-NEXT: [[TMP25:%.*]] = insertelement <8 x i8> poison, i8 [[A_SROA_17_8_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP26:%.*]] = insertelement <8 x i8> [[TMP25]], i8 [[A_SROA_16_8_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP27:%.*]] = insertelement <8 x i8> [[TMP26]], i8 [[A_SROA_15_8_EXTRACT_TRUNC]], i64 2
-; SSE4-NEXT: [[TMP28:%.*]] = insertelement <8 x i8> [[TMP27]], i8 [[A_SROA_14_8_EXTRACT_TRUNC]], i64 3
-; SSE4-NEXT: [[TMP29:%.*]] = insertelement <8 x i8> [[TMP28]], i8 [[A_SROA_13_8_EXTRACT_TRUNC]], i64 4
-; SSE4-NEXT: [[TMP30:%.*]] = insertelement <8 x i8> [[TMP29]], i8 [[A_SROA_12_8_EXTRACT_TRUNC]], i64 5
-; SSE4-NEXT: [[TMP31:%.*]] = insertelement <8 x i8> [[TMP30]], i8 [[A_SROA_11_8_EXTRACT_TRUNC]], i64 6
-; SSE4-NEXT: [[TMP32:%.*]] = insertelement <8 x i8> [[TMP31]], i8 [[A_SROA_9_8_EXTRACT_TRUNC]], i64 7
-; SSE4-NEXT: [[TMP33:%.*]] = lshr <8 x i8> [[TMP32]], splat (i8 1)
-; SSE4-NEXT: [[TMP34:%.*]] = insertelement <8 x i8> poison, i8 [[B_SROA_9_8_EXTRACT_TRUNC]], i64 0
-; SSE4-NEXT: [[TMP35:%.*]] = insertelement <8 x i8> [[TMP34]], i8 [[B_SROA_16_8_EXTRACT_TRUNC]], i64 1
-; SSE4-NEXT: [[TMP36:%.*]] = insertelement <8 x i8> [[TMP35]], i8 [[B_SROA_15_8_EXTRACT_TRUNC]], i64 2
-; SSE4-NEXT: [[TMP37:%.*]] = insertelement <8 x i8> [[TMP36]], i8 [[B_SROA_14_8_EXTRACT_TRUNC]], i64 3
-; SSE4-NEXT: [[TMP38:%.*]] = insertelement <8 x i8> [[TMP37]], i8 [[B_SROA_13_8_EXTRACT_TRUNC]], i64 4
-; SSE4-NEXT: [[TMP39:%.*]] = insertelement <8 x i8> [[TMP38]], i8 [[B_SROA_12_8_EXTRACT_TRUNC]], i64 5
-; SSE4-NEXT: [[TMP40:%.*]] = insertelement <8 x i8> [[TMP39]], i8 [[B_SROA_11_8_EXTRACT_TRUNC]], i64 6
-; SSE4-NEXT: [[TMP41:%.*]] = insertelement <8 x i8> [[TMP40]], i8 [[B_SROA_17_8_EXTRACT_TRUNC]], i64 7
-; SSE4-NEXT: [[TMP42:%.*]] = lshr <8 x i8> [[TMP41]], splat (i8 1)
-; SSE4-NEXT: [[TMP43:%.*]] = add nuw <8 x i8> [[TMP42]], [[TMP33]]
-; SSE4-NEXT: [[TMP44:%.*]] = or <8 x i8> [[TMP41]], [[TMP32]]
-; SSE4-NEXT: [[TMP45:%.*]] = and <8 x i8> [[TMP44]], splat (i8 1)
-; SSE4-NEXT: [[TMP46:%.*]] = add nuw <8 x i8> [[TMP43]], [[TMP45]]
-; SSE4-NEXT: [[TMP47:%.*]] = zext <8 x i8> [[TMP46]] to <8 x i64>
-; SSE4-NEXT: [[TMP48:%.*]] = shl nuw <8 x i64> [[TMP47]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
-; SSE4-NEXT: [[TMP49:%.*]] = tail call i64 @llvm.vector.reduce.or.v8i64(<8 x i64> [[TMP48]])
+; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_SHIFT1:%.*]] = lshr i64 [[B_COERCE2]], 56
+; SSE4-NEXT: [[B_SROA_17_8_EXTRACT_TRUNC1:%.*]] = trunc nuw i64 [[B_SROA_17_8_EXTRACT_SHIFT1]] to i8
+; SSE4-NEXT: [[TMP16:%.*]] = trunc <2 x i64> [[TMP3]] to <2 x i8>
+; SSE4-NEXT: [[TMP17:%.*]] = trunc <2 x i64> [[TMP4]] to <2 x i8>
+; SSE4-NEXT: [[TMP18:%.*]] = trunc <2 x i64> [[TMP5]] to <2 x i8>
+; SSE4-NEXT: [[TMP19:%.*]] = trunc <2 x i64> [[TMP6]] to <2 x i8>
+; SSE4-NEXT: [[TMP20:%.*]] = trunc <2 x i64> [[TMP7]] to <2 x i8>
+; SSE4-NEXT: [[TMP21:%.*]] = trunc <2 x i64> [[TMP11]] to <2 x i8>
+; SSE4-NEXT: [[TMP22:%.*]] = trunc <2 x i64> [[TMP12]] to <2 x i8>
+; SSE4-NEXT: [[TMP23:%.*]] = trunc <2 x i64> [[TMP13]] to <2 x i8>
+; SSE4-NEXT: [[TMP24:%.*]] = trunc <2 x i64> [[TMP14]] to <2 x i8>
+; SSE4-NEXT: [[TMP25:%.*]] = trunc <2 x i64> [[TMP15]] to <2 x i8>
+; SSE4-NEXT: [[TMP26:%.*]] = lshr <2 x i8> [[TMP2]], splat (i8 1)
+; SSE4-NEXT: [[TMP27:%.*]] = lshr <2 x i8> [[TMP10]], splat (i8 1)
+; SSE4-NEXT: [[TMP28:%.*]] = add nuw <2 x i8> [[TMP27]], [[TMP26]]
+; SSE4-NEXT: [[TMP29:%.*]] = or <2 x i8> [[TMP10]], [[TMP2]]
+; SSE4-NEXT: [[TMP30:%.*]] = and <2 x i8> [[TMP29]], splat (i8 1)
+; SSE4-NEXT: [[TMP31:%.*]] = add nuw <2 x i8> [[TMP28]], [[TMP30]]
+; SSE4-NEXT: [[TMP32:%.*]] = lshr <2 x i8> [[TMP16]], splat (i8 1)
+; SSE4-NEXT: [[TMP33:%.*]] = lshr <2 x i8> [[TMP21]], splat (i8 1)
+; SSE4-NEXT: [[TMP34:%.*]] = add nuw <2 x i8> [[TMP33]], [[TMP32]]
+; SSE4-NEXT: [[TMP35:%.*]] = or <2 x i8> [[TMP21]], [[TMP16]]
+; SSE4-NEXT: [[TMP36:%.*]] = and <2 x i8> [[TMP35]], splat (i8 1)
+; SSE4-NEXT: [[TMP37:%.*]] = add nuw <2 x i8> [[TMP34]], [[TMP36]]
+; SSE4-NEXT: [[TMP38:%.*]] = lshr <2 x i8> [[TMP17]], splat (i8 1)
+; SSE4-NEXT: [[TMP39:%.*]] = lshr <2 x i8> [[TMP22]], splat (i8 1)
+; SSE4-NEXT: [[TMP40:%.*]] = add nuw <2 x i8> [[TMP39]], [[TMP38]]
+; SSE4-NEXT: [[TMP41:%.*]] = or <2 x i8> [[TMP22]], [[TMP17]]
+; SSE4-NEXT: [[TMP42:%.*]] = and <2 x i8> [[TMP41]], splat (i8 1)
+; SSE4-NEXT: [[TMP43:%.*]] = add nuw <2 x i8> [[TMP40]], [[TMP42]]
+; SSE4-NEXT: [[TMP44:%.*]] = lshr <2 x i8> [[TMP18]], splat (i8 1)
+; SSE4-NEXT: [[TMP45:%.*]] = lshr <2 x i8> [[TMP23]], splat (i8 1)
+; SSE4-NEXT: [[TMP46:%.*]] = add nuw <2 x i8> [[TMP45]], [[TMP44]]
+; SSE4-NEXT: [[TMP47:%.*]] = or <2 x i8> [[TMP23]], [[TMP18]]
+; SSE4-NEXT: [[TMP48:%.*]] = and <2 x i8> [[TMP47]], splat (i8 1)
+; SSE4-NEXT: [[TMP104:%.*]] = add nuw <2 x i8> [[TMP46]], [[TMP48]]
+; SSE4-NEXT: [[TMP50:%.*]] = lshr <2 x i8> [[TMP19]], splat (i8 1)
+; SSE4-NEXT: [[TMP51:%.*]] = lshr <2 x i8> [[TMP24]], splat (i8 1)
+; SSE4-NEXT: [[TMP52:%.*]] = add nuw <2 x i8> [[TMP51]], [[TMP50]]
+; SSE4-NEXT: [[TMP53:%.*]] = or <2 x i8> [[TMP24]], [[TMP19]]
+; SSE4-NEXT: [[TMP54:%.*]] = and <2 x i8> [[TMP53]], splat (i8 1)
+; SSE4-NEXT: [[TMP55:%.*]] = add nuw <2 x i8> [[TMP52]], [[TMP54]]
+; SSE4-NEXT: [[TMP56:%.*]] = lshr <2 x i8> [[TMP20]], splat (i8 1)
+; SSE4-NEXT: [[TMP57:%.*]] = lshr <2 x i8> [[TMP25]], splat (i8 1)
+; SSE4-NEXT: [[TMP58:%.*]] = add nuw <2 x i8> [[TMP57]], [[TMP56]]
+; SSE4-NEXT: [[TMP59:%.*]] = or <2 x i8> [[TMP25]], [[TMP20]]
+; SSE4-NEXT: [[TMP60:%.*]] = and <2 x i8> [[TMP59]], splat (i8 1)
+; SSE4-NEXT: [[TMP61:%.*]] = add nuw <2 x i8> [[TMP58]], [[TMP60]]
+; SSE4-NEXT: [[TMP62:%.*]] = insertelement <2 x i8> poison, i8 [[A_SROA_2_0_EXTRACT_TRUNC]], i64 0
+; SSE4-NEXT: [[TMP63:%.*]] = insertelement <2 x i8> [[TMP62]], i8 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 1
+; SSE4-NEXT: [[TMP64:%.*]] = lshr <2 x i8> [[TMP63]], splat (i8 1)
+; SSE4-NEXT: [[TMP65:%.*]] = insertelement <2 x i8> poison, i8 [[A_SROA_16_8_EXTRACT_TRUNC]], i64 0
+; SSE4-NEXT: [[TMP66:%.*]] = insertelement <2 x i8> [[TMP65]], i8 [[B_SROA_16_8_EXTRACT_TRUNC]], i64 1
+; SSE4-NEXT: [[TMP67:%.*]] = lshr <2 x i8> [[TMP66]], splat (i8 1)
+; SSE4-NEXT: [[TMP68:%.*]] = add nuw <2 x i8> [[TMP67]], [[TMP64]]
+; SSE4-NEXT: [[TMP69:%.*]] = or <2 x i8> [[TMP66]], [[TMP63]]
+; SSE4-NEXT: [[TMP70:%.*]] = and <2 x i8> [[TMP69]], splat (i8 1)
+; SSE4-NEXT: [[TMP71:%.*]] = add nuw <2 x i8> [[TMP68]], [[TMP70]]
+; SSE4-NEXT: [[TMP72:%.*]] = insertelement <2 x i8> poison, i8 [[B_SROA_8_0_EXTRACT_TRUNC]], i64 0
+; SSE4-NEXT: [[TMP73:%.*]] = insertelement <2 x i8> [[TMP72]], i8 [[A_SROA_9_8_EXTRACT_TRUNC]], i64 1
+; SSE4-NEXT: [[TMP74:%.*]] = lshr <2 x i8> [[TMP73]], splat (i8 1)
+; SSE4-NEXT: [[TMP75:%.*]] = insertelement <2 x i8> poison, i8 [[B_SROA_17_8_EXTRACT_TRUNC]], i64 0
+; SSE4-NEXT: [[TMP76:%.*]] = insertelement <2 x i8> [[TMP75]], i8 [[B_SROA_17_8_EXTRACT_TRUNC1]], i64 1
+; SSE4-NEXT: [[TMP77:%.*]] = lshr <2 x i8> [[TMP76]], splat (i8 1)
+; SSE4-NEXT: [[TMP78:%.*]] = add nuw <2 x i8> [[TMP77]], [[TMP74]]
+; SSE4-NEXT: [[TMP79:%.*]] = or <2 x i8> [[TMP76]], [[TMP73]]
+; SSE4-NEXT: [[TMP80:%.*]] = and <2 x i8> [[TMP79]], splat (i8 1)
+; SSE4-NEXT: [[TMP81:%.*]] = add nuw <2 x i8> [[TMP78]], [[TMP80]]
+; SSE4-NEXT: [[TMP82:%.*]] = zext <2 x i8> [[TMP81]] to <2 x i64>
+; SSE4-NEXT: [[TMP83:%.*]] = shl nuw <2 x i64> [[TMP82]], splat (i64 56)
+; SSE4-NEXT: [[TMP84:%.*]] = zext <2 x i8> [[TMP71]] to <2 x i64>
+; SSE4-NEXT: [[TMP85:%.*]] = shl nuw nsw <2 x i64> [[TMP84]], splat (i64 48)
+; SSE4-NEXT: [[TMP86:%.*]] = or disjoint <2 x i64> [[TMP83]], [[TMP85]]
+; SSE4-NEXT: [[TMP87:%.*]] = zext <2 x i8> [[TMP61]] to <2 x i64>
+; SSE4-NEXT: [[TMP88:%.*]] = shl nuw nsw <2 x i64> [[TMP87]], splat (i64 40)
+; SSE4-NEXT: [[TMP89:%.*]] = or disjoint <2 x i64> [[TMP86]], [[TMP88]]
+; SSE4-NEXT: [[TMP90:%.*]] = zext <2 x i8> [[TMP55]] to <2 x i64>
+; SSE4-NEXT: [[TMP91:%.*]] = shl nuw nsw <2 x i64> [[TMP90]], splat (i64 32)
+; SSE4-NEXT: [[TMP92:%.*]] = or disjoint <2 x i64> [[TMP89]], [[TMP91]]
+; SSE4-NEXT: [[TMP93:%.*]] = zext <2 x i8> [[TMP104]] to <2 x i64>
+; SSE4-NEXT: [[TMP94:%.*]] = shl nuw nsw <2 x i64> [[TMP93]], splat (i64 24)
+; SSE4-NEXT: [[TMP95:%.*]] = or disjoint <2 x i64> [[TMP92]], [[TMP94]]
+; SSE4-NEXT: [[TMP96:%.*]] = zext <2 x i8> [[TMP43]] to <2 x i64>
+; SSE4-NEXT: [[TMP97:%.*]] = shl nuw nsw <2 x i64> [[TMP96]], splat (i64 16)
+; SSE4-NEXT: [[TMP98:%.*]] = zext <2 x i8> [[TMP37]] to <2 x i64>
+; SSE4-NEXT: [[TMP99:%.*]] = shl nuw nsw <2 x i64> [[TMP98]], splat (i64 8)
+; SSE4-NEXT: [[TMP100:%.*]] = or disjoint <2 x i64> [[TMP95]], [[TMP97]]
+; SSE4-NEXT: [[TMP101:%.*]] = zext <2 x i8> [[TMP31]] to <2 x i64>
+; SSE4-NEXT: [[TMP102:%.*]] = or <2 x i64> [[TMP100]], [[TMP99]]
+; SSE4-NEXT: [[TMP103:%.*]] = or <2 x i64> [[TMP102]], [[TMP101]]
+; SSE4-NEXT: [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <2 x i64> [[TMP103]], i64 0
+; SSE4-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[VEC2STRUCT_SLOT_SROA_0_0_VEC_EXTRACT]], 0
+; SSE4-NEXT: [[TMP49:%.*]] = extractelement <2 x i64> [[TMP103]], i64 1
; SSE4-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP49]], 1
; SSE4-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
@@ -696,25 +696,19 @@ for.body: ; preds = %for.cond
define { i64, i64 } @avgr_8_u16(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerce0, i64 %b.coerce1) {
; CHECK-LABEL: @avgr_8_u16(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = trunc i64 [[A_COERCE0:%.*]] to i32
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i64> [[TMP1]], i64 [[A_COERCE1:%.*]], i64 1
+; CHECK-NEXT: [[TMP15:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i32>
; CHECK-NEXT: [[TMP3:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 32)
; CHECK-NEXT: [[TMP4:%.*]] = lshr <2 x i64> [[TMP2]], splat (i64 48)
-; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[A_COERCE1]] to i32
-; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[B_COERCE0:%.*]] to i32
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[B_COERCE1:%.*]], i64 1
+; CHECK-NEXT: [[TMP18:%.*]] = trunc <2 x i64> [[TMP8]] to <2 x i32>
; CHECK-NEXT: [[TMP9:%.*]] = lshr <2 x i64> [[TMP8]], splat (i64 32)
; CHECK-NEXT: [[TMP10:%.*]] = lshr <2 x i64> [[TMP8]], splat (i64 48)
-; CHECK-NEXT: [[TMP11:%.*]] = trunc i64 [[B_COERCE1]] to i32
; CHECK-NEXT: [[TMP12:%.*]] = and <2 x i64> [[TMP2]], splat (i64 65535)
; CHECK-NEXT: [[TMP13:%.*]] = and <2 x i64> [[TMP8]], splat (i64 65535)
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x i32> poison, i32 [[TMP0]], i64 0
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x i32> [[TMP14]], i32 [[TMP5]], i64 1
; CHECK-NEXT: [[TMP16:%.*]] = lshr <2 x i32> [[TMP15]], splat (i32 16)
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <2 x i32> poison, i32 [[TMP6]], i64 0
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP11]], i64 1
; CHECK-NEXT: [[TMP19:%.*]] = lshr <2 x i32> [[TMP18]], splat (i32 16)
; CHECK-NEXT: [[TMP20:%.*]] = add nuw nsw <2 x i64> [[TMP12]], splat (i64 1)
; CHECK-NEXT: [[TMP21:%.*]] = add nuw nsw <2 x i64> [[TMP20]], [[TMP13]]
@@ -958,105 +952,41 @@ define { i64, i64 } @avgr_8_u16_alt(i64 %a.coerce0, i64 %a.coerce1, i64 %b.coerc
; SSE4-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP33]], 1
; SSE4-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
-; AVX2-LABEL: @avgr_8_u16_alt(
-; AVX2-NEXT: entry:
-; AVX2-NEXT: [[A_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0:%.*]], 48
-; AVX2-NEXT: [[A_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 32
-; AVX2-NEXT: [[A_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE0]], 16
-; AVX2-NEXT: [[A_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_4_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_2_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_3_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE0]] to i16
-; AVX2-NEXT: [[B_SROA_4_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0:%.*]], 48
-; AVX2-NEXT: [[B_SROA_3_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 32
-; AVX2-NEXT: [[B_SROA_2_0_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE0]], 16
-; AVX2-NEXT: [[B_SROA_4_0_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_4_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_2_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_3_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_3_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_2_0_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_0_0_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_COERCE0]] to i16
-; AVX2-NEXT: [[TMP0:%.*]] = insertelement <4 x i16> poison, i16 [[A_SROA_4_0_EXTRACT_TRUNC]], i64 0
-; AVX2-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> [[TMP0]], i16 [[A_SROA_3_0_EXTRACT_TRUNC]], i64 1
-; AVX2-NEXT: [[TMP2:%.*]] = insertelement <4 x i16> [[TMP1]], i16 [[A_SROA_2_0_EXTRACT_TRUNC]], i64 2
-; AVX2-NEXT: [[TMP3:%.*]] = insertelement <4 x i16> [[TMP2]], i16 [[A_SROA_0_0_EXTRACT_TRUNC]], i64 3
-; AVX2-NEXT: [[TMP4:%.*]] = lshr <4 x i16> [[TMP3]], splat (i16 1)
-; AVX2-NEXT: [[TMP5:%.*]] = insertelement <4 x i16> poison, i16 [[B_SROA_0_0_EXTRACT_TRUNC]], i64 0
-; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i16> [[TMP5]], i16 [[B_SROA_3_0_EXTRACT_TRUNC]], i64 1
-; AVX2-NEXT: [[TMP7:%.*]] = insertelement <4 x i16> [[TMP6]], i16 [[B_SROA_2_0_EXTRACT_TRUNC]], i64 2
-; AVX2-NEXT: [[TMP8:%.*]] = insertelement <4 x i16> [[TMP7]], i16 [[B_SROA_4_0_EXTRACT_TRUNC]], i64 3
-; AVX2-NEXT: [[TMP9:%.*]] = lshr <4 x i16> [[TMP8]], splat (i16 1)
-; AVX2-NEXT: [[TMP10:%.*]] = add nuw <4 x i16> [[TMP9]], [[TMP4]]
-; AVX2-NEXT: [[TMP11:%.*]] = or <4 x i16> [[TMP8]], [[TMP3]]
-; AVX2-NEXT: [[TMP12:%.*]] = and <4 x i16> [[TMP11]], splat (i16 1)
-; AVX2-NEXT: [[TMP13:%.*]] = add nuw <4 x i16> [[TMP10]], [[TMP12]]
-; AVX2-NEXT: [[TMP15:%.*]] = bitcast <4 x i16> [[TMP13]] to i64
-; AVX2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP15]], 0
-; AVX2-NEXT: [[A_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1:%.*]], 48
-; AVX2-NEXT: [[A_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 32
-; AVX2-NEXT: [[A_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[A_COERCE1]], 16
-; AVX2-NEXT: [[A_SROA_5_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[A_SROA_9_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_7_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_7_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_8_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_SROA_8_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[A_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[A_COERCE1]] to i16
-; AVX2-NEXT: [[B_SROA_9_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1:%.*]], 48
-; AVX2-NEXT: [[B_SROA_8_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 32
-; AVX2-NEXT: [[B_SROA_7_8_EXTRACT_SHIFT:%.*]] = lshr i64 [[B_COERCE1]], 16
-; AVX2-NEXT: [[B_SROA_9_8_EXTRACT_TRUNC:%.*]] = trunc nuw i64 [[B_SROA_9_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_7_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_8_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_8_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_SROA_7_8_EXTRACT_SHIFT]] to i16
-; AVX2-NEXT: [[B_SROA_5_8_EXTRACT_TRUNC:%.*]] = trunc i64 [[B_COERCE1]] to i16
-; AVX2-NEXT: [[TMP16:%.*]] = insertelement <4 x i16> poison, i16 [[A_SROA_9_8_EXTRACT_TRUNC]], i64 0
-; AVX2-NEXT: [[TMP17:%.*]] = insertelement <4 x i16> [[TMP16]], i16 [[A_SROA_8_8_EXTRACT_TRUNC]], i64 1
-; AVX2-NEXT: [[TMP18:%.*]] = insertelement <4 x i16> [[TMP17]], i16 [[A_SROA_7_8_EXTRACT_TRUNC]], i64 2
-; AVX2-NEXT: [[TMP19:%.*]] = insertelement <4 x i16> [[TMP18]], i16 [[A_SROA_5_8_EXTRACT_TRUNC]], i64 3
-; AVX2-NEXT: [[TMP20:%.*]] = lshr <4 x i16> [[TMP19]], splat (i16 1)
-; AVX2-NEXT: [[TMP21:%.*]] = insertelement <4 x i16> poison, i16 [[B_SROA_5_8_EXTRACT_TRUNC]], i64 0
-; AVX2-NEXT: [[TMP22:%.*]] = insertelement <4 x i16> [[TMP21]], i16 [[B_SROA_8_8_EXTRACT_TRUNC]], i64 1
-; AVX2-NEXT: [[TMP23:%.*]] = insertelement <4 x i16> [[TMP22]], i16 [[B_SROA_7_8_EXTRACT_TRUNC]], i64 2
-; AVX2-NEXT: [[TMP24:%.*]] = insertelement <4 x i16> [[TMP23]], i16 [[B_SROA_9_8_EXTRACT_TRUNC]], i64 3
-; AVX2-NEXT: [[TMP25:%.*]] = lshr <4 x i16> [[TMP24]], splat (i16 1)
-; AVX2-NEXT: [[TMP26:%.*]] = add nuw <4 x i16> [[TMP25]], [[TMP20]]
-; AVX2-NEXT: [[TMP27:%.*]] = or <4 x i16> [[TMP24]], [[TMP19]]
-; AVX2-NEXT: [[TMP28:%.*]] = and <4 x i16> [[TMP27]], splat (i16 1)
-; AVX2-NEXT: [[TMP29:%.*]] = add nuw <4 x i16> [[TMP26]], [[TMP28]]
-; AVX2-NEXT: [[TMP31:%.*]] = bitcast <4 x i16> [[TMP29]] to i64
-; AVX2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP31]], 1
-; AVX2-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
-;
-; AVX512-LABEL: @avgr_8_u16_alt(
-; AVX512-NEXT: entry:
-; AVX512-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
-; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: [[TMP2:%.*]] = lshr <4 x i64> [[TMP1]], <i64 0, i64 16, i64 32, i64 48>
-; AVX512-NEXT: [[TMP3:%.*]] = trunc <4 x i64> [[TMP2]] to <4 x i16>
-; AVX512-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
-; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: [[TMP6:%.*]] = lshr <4 x i64> [[TMP5]], <i64 0, i64 16, i64 32, i64 48>
-; AVX512-NEXT: [[TMP7:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i16>
-; AVX512-NEXT: [[TMP8:%.*]] = lshr <4 x i16> [[TMP3]], splat (i16 1)
-; AVX512-NEXT: [[TMP9:%.*]] = lshr <4 x i16> [[TMP7]], splat (i16 1)
-; AVX512-NEXT: [[TMP10:%.*]] = add nuw <4 x i16> [[TMP9]], [[TMP8]]
-; AVX512-NEXT: [[TMP11:%.*]] = or <4 x i16> [[TMP7]], [[TMP3]]
-; AVX512-NEXT: [[TMP12:%.*]] = and <4 x i16> [[TMP11]], splat (i16 1)
-; AVX512-NEXT: [[TMP13:%.*]] = add nuw <4 x i16> [[TMP10]], [[TMP12]]
-; AVX512-NEXT: [[TMP15:%.*]] = bitcast <4 x i16> [[TMP13]] to i64
-; AVX512-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP15]], 0
-; AVX512-NEXT: [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[A_COERCE1:%.*]], i64 0
-; AVX512-NEXT: [[TMP17:%.*]] = shufflevector <4 x i64> [[TMP16]], <4 x i64> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: [[TMP18:%.*]] = lshr <4 x i64> [[TMP17]], <i64 0, i64 16, i64 32, i64 48>
-; AVX512-NEXT: [[TMP19:%.*]] = trunc <4 x i64> [[TMP18]] to <4 x i16>
-; AVX512-NEXT: [[TMP20:%.*]] = insertelement <4 x i64> poison, i64 [[B_COERCE1:%.*]], i64 0
-; AVX512-NEXT: [[TMP21:%.*]] = shufflevector <4 x i64> [[TMP20]], <4 x i64> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: [[TMP22:%.*]] = lshr <4 x i64> [[TMP21]], <i64 0, i64 16, i64 32, i64 48>
-; AVX512-NEXT: [[TMP23:%.*]] = trunc <4 x i64> [[TMP22]] to <4 x i16>
-; AVX512-NEXT: [[TMP24:%.*]] = lshr <4 x i16> [[TMP19]], splat (i16 1)
-; AVX512-NEXT: [[TMP25:%.*]] = lshr <4 x i16> [[TMP23]], splat (i16 1)
-; AVX512-NEXT: [[TMP26:%.*]] = add nuw <4 x i16> [[TMP25]], [[TMP24]]
-; AVX512-NEXT: [[TMP27:%.*]] = or <4 x i16> [[TMP23]], [[TMP19]]
-; AVX512-NEXT: [[TMP28:%.*]] = and <4 x i16> [[TMP27]], splat (i16 1)
-; AVX512-NEXT: [[TMP29:%.*]] = add nuw <4 x i16> [[TMP26]], [[TMP28]]
-; AVX512-NEXT: [[TMP31:%.*]] = bitcast <4 x i16> [[TMP29]] to i64
-; AVX512-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP31]], 1
-; AVX512-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
+; AVX-LABEL: @avgr_8_u16_alt(
+; AVX-NEXT: entry:
+; AVX-NEXT: [[TMP0:%.*]] = insertelement <4 x i64> poison, i64 [[A_COERCE0:%.*]], i64 0
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[TMP0]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX-NEXT: [[TMP2:%.*]] = lshr <4 x i64> [[TMP1]], <i64 0, i64 16, i64 32, i64 48>
+; AVX-NEXT: [[TMP3:%.*]] = trunc <4 x i64> [[TMP2]] to <4 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> poison, i64 [[B_COERCE0:%.*]], i64 0
+; AVX-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX-NEXT: [[TMP6:%.*]] = lshr <4 x i64> [[TMP5]], <i64 0, i64 16, i64 32, i64 48>
+; AVX-NEXT: [[TMP7:%.*]] = trunc <4 x i64> [[TMP6]] to <4 x i16>
+; AVX-NEXT: [[TMP8:%.*]] = lshr <4 x i16> [[TMP3]], splat (i16 1)
+; AVX-NEXT: [[TMP9:%.*]] = lshr <4 x i16> [[TMP7]], splat (i16 1)
+; AVX-NEXT: [[TMP10:%.*]] = add nuw <4 x i16> [[TMP9]], [[TMP8]]
+; AVX-NEXT: [[TMP11:%.*]] = or <4 x i16> [[TMP7]], [[TMP3]]
+; AVX-NEXT: [[TMP12:%.*]] = and <4 x i16> [[TMP11]], splat (i16 1)
+; AVX-NEXT: [[TMP13:%.*]] = add nuw <4 x i16> [[TMP10]], [[TMP12]]
+; AVX-NEXT: [[TMP14:%.*]] = bitcast <4 x i16> [[TMP13]] to i64
+; AVX-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP14]], 0
+; AVX-NEXT: [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[A_COERCE1:%.*]], i64 0
+; AVX-NEXT: [[TMP16:%.*]] = shufflevector <4 x i64> [[TMP15]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX-NEXT: [[TMP17:%.*]] = lshr <4 x i64> [[TMP16]], <i64 0, i64 16, i64 32, i64 48>
+; AVX-NEXT: [[TMP18:%.*]] = trunc <4 x i64> [[TMP17]] to <4 x i16>
+; AVX-NEXT: [[TMP19:%.*]] = insertelement <4 x i64> poison, i64 [[B_COERCE1:%.*]], i64 0
+; AVX-NEXT: [[TMP20:%.*]] = shufflevector <4 x i64> [[TMP19]], <4 x i64> poison, <4 x i32> zeroinitializer
+; AVX-NEXT: [[TMP21:%.*]] = lshr <4 x i64> [[TMP20]], <i64 0, i64 16, i64 32, i64 48>
+; AVX-NEXT: [[TMP22:%.*]] = trunc <4 x i64> [[TMP21]] to <4 x i16>
+; AVX-NEXT: [[TMP23:%.*]] = lshr <4 x i16> [[TMP18]], splat (i16 1)
+; AVX-NEXT: [[TMP24:%.*]] = lshr <4 x i16> [[TMP22]], splat (i16 1)
+; AVX-NEXT: [[TMP25:%.*]] = add nuw <4 x i16> [[TMP24]], [[TMP23]]
+; AVX-NEXT: [[TMP26:%.*]] = or <4 x i16> [[TMP22]], [[TMP18]]
+; AVX-NEXT: [[TMP27:%.*]] = and <4 x i16> [[TMP26]], splat (i16 1)
+; AVX-NEXT: [[TMP28:%.*]] = add nuw <4 x i16> [[TMP25]], [[TMP27]]
+; AVX-NEXT: [[TMP29:%.*]] = bitcast <4 x i16> [[TMP28]] to i64
+; AVX-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP29]], 1
+; AVX-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
entry:
%retval = alloca %"struct.std::array8", align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-node-gather-cost.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-node-gather-cost.ll
index e344fcbebbb79..951611bcbc6e9 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-node-gather-cost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/trimmed-node-gather-cost.ll
@@ -34,24 +34,23 @@ define void @test(ptr %start1, ptr %start2, ptr %start3, ptr %r, i32 %n) {
; CHECK-NEXT: [[M2_0:%.*]] = phi i32 [ [[ADD8]], %[[IF_THEN]] ], [ [[TMP1]], %[[FOR_BODY4]] ]
; CHECK-NEXT: [[M3_0:%.*]] = phi i32 [ [[ADD9]], %[[IF_THEN]] ], [ [[TMP2]], %[[FOR_BODY4]] ]
; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[M1_0]], [[M2_0]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[M1_0]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[M2_0]], i64 1
; CHECK-NEXT: br i1 [[CMP10]], label %[[IF_THEN11:.*]], label %[[IF_END18]]
; CHECK: [[IF_THEN11]]:
-; CHECK-NEXT: [[TMP5:%.*]] = call <2 x i32> @llvm.smax.v2i32(<2 x i32> [[TMP4]], <2 x i32> zeroinitializer)
-; CHECK-NEXT: [[TMP6:%.*]] = call <2 x i32> @llvm.umin.v2i32(<2 x i32> [[TMP5]], <2 x i32> splat (i32 128))
-; CHECK-NEXT: [[TMP7:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M3_0]], i32 0)
+; CHECK-NEXT: [[TMP7:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M1_0]], i32 0)
; CHECK-NEXT: [[TMP8:%.*]] = tail call i32 @llvm.umin.i32(i32 [[TMP7]], i32 128)
+; CHECK-NEXT: [[TMP5:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M2_0]], i32 0)
+; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 @llvm.umin.i32(i32 [[TMP5]], i32 128)
+; CHECK-NEXT: [[TMP9:%.*]] = tail call i32 @llvm.smax.i32(i32 [[M3_0]], i32 0)
+; CHECK-NEXT: [[TMP10:%.*]] = tail call i32 @llvm.umin.i32(i32 [[TMP9]], i32 128)
; CHECK-NEXT: br label %[[IF_END18]]
; CHECK: [[IF_END18]]:
-; CHECK-NEXT: [[M3_1:%.*]] = phi i32 [ [[TMP8]], %[[IF_THEN11]] ], [ [[M3_0]], %[[IF_END]] ]
-; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ [[TMP6]], %[[IF_THEN11]] ], [ [[TMP4]], %[[IF_END]] ]
-; CHECK-NEXT: [[M1_1:%.*]] = extractelement <2 x i32> [[TMP9]], i64 0
+; CHECK-NEXT: [[M1_1:%.*]] = phi i32 [ [[TMP8]], %[[IF_THEN11]] ], [ [[M1_0]], %[[IF_END]] ]
+; CHECK-NEXT: [[M2_1:%.*]] = phi i32 [ [[TMP6]], %[[IF_THEN11]] ], [ [[M2_0]], %[[IF_END]] ]
+; CHECK-NEXT: [[M3_1:%.*]] = phi i32 [ [[TMP10]], %[[IF_THEN11]] ], [ [[M3_0]], %[[IF_END]] ]
; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[M1_1]], 4
; CHECK-NEXT: [[CONV:%.*]] = trunc i32 [[SHR]] to i8
; CHECK-NEXT: [[INCDEC_PTR19:%.*]] = getelementptr inbounds nuw i8, ptr [[R_ADDR_156]], i64 1
; CHECK-NEXT: store i8 [[CONV]], ptr [[R_ADDR_156]], align 1
-; CHECK-NEXT: [[M2_1:%.*]] = extractelement <2 x i32> [[TMP9]], i64 1
; CHECK-NEXT: [[SHR20:%.*]] = lshr i32 [[M2_1]], 4
; CHECK-NEXT: [[CONV21:%.*]] = trunc i32 [[SHR20]] to i8
; CHECK-NEXT: [[INCDEC_PTR22:%.*]] = getelementptr inbounds nuw i8, ptr [[R_ADDR_156]], i64 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll
index d7ba46ff439a6..35562af9ac3d4 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-parent-instr-copyable-regular.ll
@@ -9,22 +9,24 @@ define void @test(ptr %0, ptr %1, ptr %2, ptr %3, double %4, double %5) {
; CHECK-NEXT: [[TMP7:%.*]] = load double, ptr [[TMP1]], align 8
; CHECK-NEXT: [[TMP8:%.*]] = load double, ptr [[TMP0]], align 8
; CHECK-NEXT: [[TMP9:%.*]] = load double, ptr [[TMP2]], align 8
-; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP9]], 0.000000e+00
-; CHECK-NEXT: [[TMP16:%.*]] = fmul double [[TMP4]], 0.000000e+00
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> poison, double [[TMP9]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> [[TMP10]], double [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[TMP8]], i64 2
+; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 0>
+; CHECK-NEXT: [[TMP25:%.*]] = fmul <4 x double> [[TMP20]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 0.000000e+00>
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> poison, double [[TMP4]], i64 0
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP19]], double [[TMP5]], i64 1
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x double> [[TMP11]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP9]], i64 2
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[TMP8]], i64 3
+; CHECK-NEXT: [[TMP26:%.*]] = fmul <4 x double> [[TMP18]], <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double 1.000000e+00>
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x double> [[TMP18]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double 0.000000e+00, double 0.000000e+00, double 1.000000e+00, double poison>, double [[TMP4]], i64 3
; CHECK-NEXT: [[TMP14:%.*]] = fmul <4 x double> [[TMP12]], [[TMP13]]
-; CHECK-NEXT: [[TMP17:%.*]] = fmul double [[TMP15]], [[TMP8]]
-; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP9]], [[TMP16]]
+; CHECK-NEXT: [[TMP27:%.*]] = fmul <4 x double> [[TMP25]], [[TMP26]]
; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> <double 1.000000e+00, double 1.000000e+00, double poison, double poison>, <4 x i32> <i32 4, i32 5, i32 0, i32 0>
; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP12]], [[TMP21]]
-; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x double> <double 0.000000e+00, double poison, double 1.000000e+00, double 0.000000e+00>, double [[TMP7]], i64 1
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <4 x double> [[TMP16]], <4 x double> <double 0.000000e+00, double poison, double 1.000000e+00, double 0.000000e+00>, <4 x i32> <i32 4, i32 1, i32 6, i32 7>
; CHECK-NEXT: [[TMP24:%.*]] = fmul <4 x double> [[TMP23]], [[TMP14]]
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x double> [[TMP24]], double [[TMP18]], i64 0
-; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x double> [[TMP25]], double [[TMP17]], i64 2
-; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <4 x double> [[TMP26]], <4 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
; CHECK-NEXT: [[TMP28:%.*]] = fmul <4 x double> [[TMP27]], zeroinitializer
; CHECK-NEXT: [[TMP29:%.*]] = fmul <4 x double> [[TMP22]], <double 1.000000e+00, double 1.000000e+00, double 0.000000e+00, double 0.000000e+00>
; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <4 x double> [[TMP12]], <4 x double> <double poison, double 0.000000e+00, double 0.000000e+00, double 0.000000e+00>, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll b/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
index d982ce4524438..f39367d4d8b24 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/poor-throughput-seeds.ll
@@ -129,25 +129,17 @@ define void @prim_v4(double %x0, double %x1, double %x2, double %x3, double %d0,
; DISABLED-NEXT: [[A1:%.*]] = fadd double [[X1]], [[D1]]
; DISABLED-NEXT: [[A2:%.*]] = fsub double [[X2]], [[D2]]
; DISABLED-NEXT: [[A3:%.*]] = fadd double [[X3]], [[D3]]
-; DISABLED-NEXT: [[TMP0:%.*]] = insertelement <4 x double> poison, double [[A1]], i64 0
-; DISABLED-NEXT: [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[A3]], i64 1
-; DISABLED-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[A0]], i64 2
-; DISABLED-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[A2]], i64 3
-; DISABLED-NEXT: [[TMP4:%.*]] = insertelement <4 x double> poison, double [[D1]], i64 0
-; DISABLED-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[D3]], i64 1
-; DISABLED-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP5]], double [[D0]], i64 2
-; DISABLED-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[D2]], i64 3
-; DISABLED-NEXT: [[TMP8:%.*]] = fdiv <4 x double> [[TMP3]], [[TMP7]]
-; DISABLED-NEXT: [[TMP9:%.*]] = fadd <4 x double> [[TMP8]], [[TMP7]]
-; DISABLED-NEXT: [[TMP10:%.*]] = fsub <4 x double> [[TMP8]], [[TMP7]]
-; DISABLED-NEXT: [[TMP11:%.*]] = shufflevector <4 x double> [[TMP9]], <4 x double> [[TMP10]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-; DISABLED-NEXT: [[TMP12:%.*]] = extractelement <4 x double> [[TMP11]], i64 2
+; DISABLED-NEXT: [[Q0:%.*]] = fdiv double [[A0]], [[D0]]
+; DISABLED-NEXT: [[Q1:%.*]] = fdiv double [[A1]], [[D1]]
+; DISABLED-NEXT: [[Q2:%.*]] = fdiv double [[A2]], [[D2]]
+; DISABLED-NEXT: [[Q3:%.*]] = fdiv double [[A3]], [[D3]]
+; DISABLED-NEXT: [[TMP12:%.*]] = fsub double [[Q0]], [[D0]]
+; DISABLED-NEXT: [[TMP13:%.*]] = fadd double [[Q1]], [[D1]]
+; DISABLED-NEXT: [[TMP14:%.*]] = fsub double [[Q2]], [[D2]]
+; DISABLED-NEXT: [[TMP15:%.*]] = fadd double [[Q3]], [[D3]]
; DISABLED-NEXT: store double [[TMP12]], ptr [[P0]], align 8
-; DISABLED-NEXT: [[TMP13:%.*]] = extractelement <4 x double> [[TMP11]], i64 0
; DISABLED-NEXT: store double [[TMP13]], ptr [[P1]], align 8
-; DISABLED-NEXT: [[TMP14:%.*]] = extractelement <4 x double> [[TMP11]], i64 3
; DISABLED-NEXT: store double [[TMP14]], ptr [[P2]], align 8
-; DISABLED-NEXT: [[TMP15:%.*]] = extractelement <4 x double> [[TMP11]], i64 1
; DISABLED-NEXT: store double [[TMP15]], ptr [[P3]], align 8
; DISABLED-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
index d0a6f1236fa56..cb3649b03f7a1 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr46983.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 -slp-threshold=-1 | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,CHECK-SSE42
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefixes=CHECK,CHECK-AVX
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,CHECK-AVX2
-; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,CHECK-AVX512
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 -slp-threshold=-1 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s
+; RUN: opt < %s -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s
define void @store_i32(ptr nocapture %0, i32 %1, i32 %2) {
; CHECK-LABEL: define void @store_i32(
@@ -102,99 +102,20 @@ define void @store_i8(ptr nocapture %0, i32 %1, i32 %2) {
}
define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) {
-; CHECK-SSE2-LABEL: define void @store_i64(
-; CHECK-SSE2-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
-; CHECK-SSE2-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-SSE2-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
-; CHECK-SSE2-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-SSE2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-SSE2-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
-; CHECK-SSE2-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
-; CHECK-SSE2-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
-; CHECK-SSE2-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], splat (i32 255)
-; CHECK-SSE2-NEXT: [[TMP12:%.*]] = and <4 x i64> [[TMP9]], splat (i64 4294967295)
-; CHECK-SSE2-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP12]], <4 x i64> splat (i64 255)
-; CHECK-SSE2-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-SSE2-NEXT: ret void
-;
-; CHECK-SSE42-LABEL: define void @store_i64(
-; CHECK-SSE42-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
-; CHECK-SSE42-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-SSE42-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
-; CHECK-SSE42-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-SSE42-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-SSE42-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
-; CHECK-SSE42-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
-; CHECK-SSE42-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
-; CHECK-SSE42-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], splat (i32 255)
-; CHECK-SSE42-NEXT: [[TMP12:%.*]] = and <4 x i64> [[TMP9]], splat (i64 4294967295)
-; CHECK-SSE42-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP12]], <4 x i64> splat (i64 255)
-; CHECK-SSE42-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-SSE42-NEXT: ret void
-;
-; CHECK-AVX-LABEL: define void @store_i64(
-; CHECK-AVX-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
-; CHECK-AVX-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 1
-; CHECK-AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 2
-; CHECK-AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP0]], i64 3
-; CHECK-AVX-NEXT: [[TMP8:%.*]] = load i64, ptr [[TMP7]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
-; CHECK-AVX-NEXT: [[TMP9:%.*]] = load i64, ptr [[TMP6]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX-NEXT: [[TMP10:%.*]] = mul i64 [[TMP8]], [[TMP4]]
-; CHECK-AVX-NEXT: [[TMP11:%.*]] = mul i64 [[TMP9]], [[TMP4]]
-; CHECK-AVX-NEXT: [[TMP12:%.*]] = lshr i64 [[TMP10]], 15
-; CHECK-AVX-NEXT: [[TMP13:%.*]] = lshr i64 [[TMP11]], 15
-; CHECK-AVX-NEXT: [[TMP14:%.*]] = trunc i64 [[TMP12]] to i32
-; CHECK-AVX-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP13]] to i32
-; CHECK-AVX-NEXT: [[TMP16:%.*]] = load i64, ptr [[TMP5]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX-NEXT: [[TMP18:%.*]] = mul i64 [[TMP16]], [[TMP4]]
-; CHECK-AVX-NEXT: [[TMP19:%.*]] = mul i64 [[TMP17]], [[TMP4]]
-; CHECK-AVX-NEXT: [[TMP20:%.*]] = insertelement <2 x i64> poison, i64 [[TMP19]], i64 0
-; CHECK-AVX-NEXT: [[TMP21:%.*]] = insertelement <2 x i64> [[TMP20]], i64 [[TMP18]], i64 1
-; CHECK-AVX-NEXT: [[TMP22:%.*]] = lshr <2 x i64> [[TMP21]], splat (i64 15)
-; CHECK-AVX-NEXT: [[TMP23:%.*]] = trunc <2 x i64> [[TMP22]] to <2 x i32>
-; CHECK-AVX-NEXT: [[TMP24:%.*]] = shufflevector <2 x i32> [[TMP23]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-AVX-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP24]], i32 [[TMP15]], i64 2
-; CHECK-AVX-NEXT: [[TMP26:%.*]] = insertelement <4 x i32> [[TMP25]], i32 [[TMP14]], i64 3
-; CHECK-AVX-NEXT: [[TMP27:%.*]] = icmp ult <4 x i32> [[TMP26]], splat (i32 255)
-; CHECK-AVX-NEXT: [[TMP28:%.*]] = shufflevector <2 x i64> [[TMP22]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
-; CHECK-AVX-NEXT: [[TMP29:%.*]] = insertelement <4 x i64> [[TMP28]], i64 [[TMP13]], i64 2
-; CHECK-AVX-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> [[TMP29]], i64 [[TMP12]], i64 3
-; CHECK-AVX-NEXT: [[TMP31:%.*]] = and <4 x i64> [[TMP30]], splat (i64 4294967295)
-; CHECK-AVX-NEXT: [[TMP32:%.*]] = select <4 x i1> [[TMP27]], <4 x i64> [[TMP31]], <4 x i64> splat (i64 255)
-; CHECK-AVX-NEXT: store <4 x i64> [[TMP32]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX-NEXT: ret void
-;
-; CHECK-AVX2-LABEL: define void @store_i64(
-; CHECK-AVX2-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
-; CHECK-AVX2-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-AVX2-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
-; CHECK-AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-AVX2-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
-; CHECK-AVX2-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
-; CHECK-AVX2-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
-; CHECK-AVX2-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], splat (i32 255)
-; CHECK-AVX2-NEXT: [[TMP12:%.*]] = and <4 x i64> [[TMP9]], splat (i64 4294967295)
-; CHECK-AVX2-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP12]], <4 x i64> splat (i64 255)
-; CHECK-AVX2-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX2-NEXT: ret void
-;
-; CHECK-AVX512-LABEL: define void @store_i64(
-; CHECK-AVX512-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
-; CHECK-AVX512-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
-; CHECK-AVX512-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
-; CHECK-AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
-; CHECK-AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
-; CHECK-AVX512-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
-; CHECK-AVX512-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
-; CHECK-AVX512-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
-; CHECK-AVX512-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], splat (i32 255)
-; CHECK-AVX512-NEXT: [[TMP12:%.*]] = and <4 x i64> [[TMP9]], splat (i64 4294967295)
-; CHECK-AVX512-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP12]], <4 x i64> splat (i64 255)
-; CHECK-AVX512-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
-; CHECK-AVX512-NEXT: ret void
+; CHECK-LABEL: define void @store_i64(
+; CHECK-SAME: ptr captures(none) [[TMP0:%.*]], i32 [[TMP1:%.*]], i32 [[TMP2:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP4:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5:![0-9]+]]
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i64> [[TMP6]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP8:%.*]] = mul <4 x i64> [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = lshr <4 x i64> [[TMP8]], splat (i64 15)
+; CHECK-NEXT: [[TMP10:%.*]] = trunc <4 x i64> [[TMP9]] to <4 x i32>
+; CHECK-NEXT: [[TMP11:%.*]] = icmp ult <4 x i32> [[TMP10]], splat (i32 255)
+; CHECK-NEXT: [[TMP12:%.*]] = and <4 x i64> [[TMP9]], splat (i64 4294967295)
+; CHECK-NEXT: [[TMP13:%.*]] = select <4 x i1> [[TMP11]], <4 x i64> [[TMP12]], <4 x i64> splat (i64 255)
+; CHECK-NEXT: store <4 x i64> [[TMP13]], ptr [[TMP0]], align 8, !tbaa [[LONG_TBAA5]]
+; CHECK-NEXT: ret void
;
%4 = zext i32 %1 to i64
%5 = load i64, ptr %0, align 8, !tbaa !7
@@ -242,44 +163,3 @@ define void @store_i64(ptr nocapture %0, i32 %1, i32 %2) {
!6 = !{!4, !4, i64 0}
!7 = !{!8, !8, i64 0}
!8 = !{!"long", !4, i64 0}
-;.
-; CHECK-SSE2: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK-SSE2: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
-; CHECK-SSE2: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
-; CHECK-SSE2: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK-SSE2: [[CHAR_TBAA4]] = !{[[META2]], [[META2]], i64 0}
-; CHECK-SSE2: [[LONG_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK-SSE2: [[META6]] = !{!"long", [[META2]], i64 0}
-;.
-; CHECK-SSE42: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK-SSE42: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
-; CHECK-SSE42: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
-; CHECK-SSE42: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK-SSE42: [[CHAR_TBAA4]] = !{[[META2]], [[META2]], i64 0}
-; CHECK-SSE42: [[LONG_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK-SSE42: [[META6]] = !{!"long", [[META2]], i64 0}
-;.
-; CHECK-AVX: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK-AVX: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
-; CHECK-AVX: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
-; CHECK-AVX: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK-AVX: [[CHAR_TBAA4]] = !{[[META2]], [[META2]], i64 0}
-; CHECK-AVX: [[LONG_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK-AVX: [[META6]] = !{!"long", [[META2]], i64 0}
-;.
-; CHECK-AVX2: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK-AVX2: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
-; CHECK-AVX2: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
-; CHECK-AVX2: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK-AVX2: [[CHAR_TBAA4]] = !{[[META2]], [[META2]], i64 0}
-; CHECK-AVX2: [[LONG_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK-AVX2: [[META6]] = !{!"long", [[META2]], i64 0}
-;.
-; CHECK-AVX512: [[INT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
-; CHECK-AVX512: [[META1]] = !{!"int", [[META2:![0-9]+]], i64 0}
-; CHECK-AVX512: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
-; CHECK-AVX512: [[META3]] = !{!"Simple C++ TBAA"}
-; CHECK-AVX512: [[CHAR_TBAA4]] = !{[[META2]], [[META2]], i64 0}
-; CHECK-AVX512: [[LONG_TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
-; CHECK-AVX512: [[META6]] = !{!"long", [[META2]], i64 0}
-;.
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
index ff2684bca8aba..d805e95c28079 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/split-node-reused-in-later-vector.ll
@@ -7,22 +7,9 @@ define i1 @test() {
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i1> <i1 poison, i1 false>, i1 [[TMP1]], i64 0
; CHECK-NEXT: [[TMP3:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> <i32 1, i32 0>, <2 x i32> zeroinitializer
; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 0, i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP7]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP8]], <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> [[TMP10]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-; CHECK-NEXT: [[TMP12:%.*]] = mul <8 x i32> [[TMP9]], zeroinitializer
-; CHECK-NEXT: [[TMP13:%.*]] = urem <8 x i32> [[TMP12]], splat (i32 46337)
-; CHECK-NEXT: [[TMP14:%.*]] = mul <8 x i32> [[TMP13]], zeroinitializer
-; CHECK-NEXT: [[TMP15:%.*]] = urem <8 x i32> [[TMP14]], splat (i32 46337)
-; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <8 x i32> [[TMP15]], <8 x i32> poison, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 6, i32 7, i32 2, i32 3>
-; CHECK-NEXT: [[TMP17:%.*]] = add <8 x i32> poison, [[TMP16]]
-; CHECK-NEXT: [[TMP18:%.*]] = or <8 x i32> poison, [[TMP16]]
-; CHECK-NEXT: [[TMP19:%.*]] = shufflevector <8 x i32> [[TMP17]], <8 x i32> [[TMP18]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 12, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP20:%.*]] = freeze <8 x i32> [[TMP19]]
+; CHECK-NEXT: [[TMP20:%.*]] = freeze <8 x i32> poison
; CHECK-NEXT: [[TMP21:%.*]] = select <8 x i1> zeroinitializer, <8 x i32> [[TMP20]], <8 x i32> zeroinitializer
; CHECK-NEXT: [[TMP22:%.*]] = mul <8 x i32> [[TMP11]], zeroinitializer
; CHECK-NEXT: [[TMP23:%.*]] = urem <8 x i32> [[TMP22]], splat (i32 46337)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll b/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
index 6308f01135d55..a8f8ed8c4c397 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/subvector-minbitwidth-unsigned-value.ll
@@ -5,12 +5,10 @@ define i1 @test(i64 %v1, ptr %v2, i32 %v3, i1 %v4) {
; CHECK-LABEL: define i1 @test(
; CHECK-SAME: i64 [[V1:%.*]], ptr [[V2:%.*]], i32 [[V3:%.*]], i1 [[V4:%.*]]) {
; CHECK-NEXT: [[NEWFUNCROOT:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = lshr i64 [[V1]], 40
-; CHECK-NEXT: [[TT3:%.*]] = lshr i64 [[V1]], 32
-; CHECK-NEXT: [[TMP1:%.*]] = trunc i64 [[TMP0]] to i32
-; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[TT3]] to i32
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[TMP2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP1]], i64 1
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[V1]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], <i64 32, i64 40>
+; CHECK-NEXT: [[TMP4:%.*]] = trunc <2 x i64> [[TMP2]] to <2 x i32>
; CHECK-NEXT: [[TMP5:%.*]] = and <2 x i32> [[TMP4]], <i32 1, i32 255>
; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <2 x i32> [[TMP5]], zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[V3]], i64 0
More information about the llvm-commits
mailing list