[llvm-branch-commits] [llvm] db7a2f3 - Precommit transform tests that have poison as insertelement's placeholder
Juneyoung Lee via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Wed Dec 23 18:50:59 PST 2020
Author: Juneyoung Lee
Date: 2020-12-24T11:46:17+09:00
New Revision: db7a2f347f132b3920415013d62d1adfb18d8d58
URL: https://github.com/llvm/llvm-project/commit/db7a2f347f132b3920415013d62d1adfb18d8d58
DIFF: https://github.com/llvm/llvm-project/commit/db7a2f347f132b3920415013d62d1adfb18d8d58.diff
LOG: Precommit transform tests that have poison as insertelement's placeholder
This commit copies existing tests at llvm/Transforms and replaces
'insertelement undef' in those files with 'insertelement poison'.
(see https://reviews.llvm.org/D93586)
Tests listed using this script:
grep -R -E '^[^;]*insertelement <.*> undef,' . | cut -d":" -f1 | uniq |
wc -l
Tests updated:
file_org=llvm/test/Transforms/$1
file=${file_org%.ll}-inseltpoison.ll
cp $file_org $file
sed -i -E 's/^([^;]*)insertelement <(.*)> undef/\1insertelement <\2> poison/g' $file
head -1 $file | grep "Assertions have been autogenerated by utils/update_test_checks.py" -q
if [ "$?" == 1 ]; then
echo "$file : should be manually updated"
# I manually updated the script
exit 1
fi
python3 ./llvm/utils/update_test_checks.py --opt-binary=./build-releaseassert/bin/opt $file
Added:
llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
llvm/test/Transforms/BDCE/vectors-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/AMDGPU/bypass-slow-div-debug-info-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll
llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather-inseltpoison.ll
llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash-inseltpoison.ll
llvm/test/Transforms/GVN/non-integral-pointers-inseltpoison.ll
llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions-inseltpoison.ll
llvm/test/Transforms/InferFunctionAttrs/dereferenceable-inseltpoison.ll
llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll
llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-sse-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-sse2-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-sse41-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-vector-shifts-inseltpoison.ll
llvm/test/Transforms/InstCombine/X86/x86-xop-inseltpoison.ll
llvm/test/Transforms/InstCombine/bitcast-inseltpoison.ll
llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll
llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
llvm/test/Transforms/InstCombine/fold-vector-zero-inseltpoison.ll
llvm/test/Transforms/InstCombine/icmp-bc-vec-inseltpoison.ll
llvm/test/Transforms/InstCombine/inselt-binop-inseltpoison.ll
llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
llvm/test/Transforms/InstCombine/scalarization-inseltpoison.ll
llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
llvm/test/Transforms/InstCombine/shift-add-inseltpoison.ll
llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll
llvm/test/Transforms/InstCombine/udiv-pow2-vscale-inseltpoison.ll
llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
llvm/test/Transforms/InstCombine/vec_extract_var_elt-inseltpoison.ll
llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll
llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll
llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
llvm/test/Transforms/InstCombine/vector_insertelt_shuffle-inseltpoison.ll
llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
llvm/test/Transforms/InstCombine/vscale_insertelement-inseltpoison.ll
llvm/test/Transforms/InstSimplify/ConstProp/InsertElement-inseltpoison.ll
llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll
llvm/test/Transforms/InstSimplify/ConstProp/vscale-shufflevector-inseltpoison.ll
llvm/test/Transforms/InstSimplify/select-inseltpoison.ll
llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll
llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains-inseltpoison.ll
llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll
llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather-xfail-inseltpoison.ll
llvm/test/Transforms/PGOProfile/counter_promo_nest-inseltpoison.ll
llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
llvm/test/Transforms/PhaseOrdering/X86/horiz-math-inseltpoison.ll
llvm/test/Transforms/PhaseOrdering/X86/scalarization-inseltpoison.ll
llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order-inseltpoison.ll
llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AArch64/insertelement-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AMDGPU/bswap-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/AMDGPU/round-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/ARM/extract-insert-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/fptosi-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/pr31599-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/pr44067-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/sign-extend-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll
llvm/test/Transforms/SLPVectorizer/vectorizable-functions-inseltpoison.ll
llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
llvm/test/Transforms/Scalarizer/order-bug-inseltpoison.ll
llvm/test/Transforms/SimplifyCFG/ARM/speculate-vector-ops-inseltpoison.ll
llvm/test/Transforms/SimplifyCFG/speculate-vector-ops-inseltpoison.ll
llvm/test/Transforms/SpeculativeExecution/spec-other-inseltpoison.ll
llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop-inseltpoison.ll
llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
llvm/test/Transforms/VectorCombine/Hexagon/load-inseltpoison.ll
llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
llvm/test/Transforms/VectorCombine/X86/insert-binop-inseltpoison.ll
llvm/test/Transforms/VectorCombine/X86/insert-binop-with-constant-inseltpoison.ll
llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
llvm/test/Transforms/VectorCombine/X86/scalarize-cmp-inseltpoison.ll
Modified:
Removed:
################################################################################
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll b/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
new file mode 100644
index 000000000000..845a00dc4e9d
--- /dev/null
+++ b/llvm/test/Transforms/Attributor/dereferenceable-2-inseltpoison.ll
@@ -0,0 +1,847 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function-signature --check-attributes
+; RUN: opt -attributor -enable-new-pm=0 -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_NPM,NOT_CGSCC_OPM,NOT_TUNIT_NPM,IS__TUNIT____,IS________OPM,IS__TUNIT_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor -attributor-manifest-internal -attributor-max-iterations-verify -attributor-annotate-decl-cs -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_CGSCC_OPM,NOT_CGSCC_NPM,NOT_TUNIT_OPM,IS__TUNIT____,IS________NPM,IS__TUNIT_NPM
+; RUN: opt -attributor-cgscc -enable-new-pm=0 -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_NPM,IS__CGSCC____,IS________OPM,IS__CGSCC_OPM
+; RUN: opt -aa-pipeline=basic-aa -passes=attributor-cgscc -attributor-manifest-internal -attributor-annotate-decl-cs -S < %s | FileCheck %s --check-prefixes=CHECK,NOT_TUNIT_NPM,NOT_TUNIT_OPM,NOT_CGSCC_OPM,IS__CGSCC____,IS________NPM,IS__CGSCC_NPM
+
+; Determine dereference-ability before unused loads get deleted:
+; https://bugs.llvm.org/show_bug.cgi?id=21780
+
+define <4 x double> @PR21780(double* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0:#.*]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 1
+; IS__TUNIT____-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 2
+; IS__TUNIT____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT: [[T0:%.*]] = load double, double* [[PTR]], align 8
+; IS__TUNIT____-NEXT: [[T1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
+; IS__TUNIT____-NEXT: [[T2:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; IS__TUNIT____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT: [[VECINIT0:%.*]] = insertelement <4 x double> poison, double [[T0]], i32 0
+; IS__TUNIT____-NEXT: [[VECINIT1:%.*]] = insertelement <4 x double> [[VECINIT0]], double [[T1]], i32 1
+; IS__TUNIT____-NEXT: [[VECINIT2:%.*]] = insertelement <4 x double> [[VECINIT1]], double [[T2]], i32 2
+; IS__TUNIT____-NEXT: [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT2]], double [[T3]], i32 3
+; IS__TUNIT____-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x double> [[VECINIT3]], <4 x double> [[VECINIT3]], <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; IS__TUNIT____-NEXT: ret <4 x double> [[SHUFFLE]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0:#.*]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 1
+; IS__CGSCC____-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 2
+; IS__CGSCC____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT: [[T0:%.*]] = load double, double* [[PTR]], align 8
+; IS__CGSCC____-NEXT: [[T1:%.*]] = load double, double* [[ARRAYIDX1]], align 8
+; IS__CGSCC____-NEXT: [[T2:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; IS__CGSCC____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT: [[VECINIT0:%.*]] = insertelement <4 x double> poison, double [[T0]], i32 0
+; IS__CGSCC____-NEXT: [[VECINIT1:%.*]] = insertelement <4 x double> [[VECINIT0]], double [[T1]], i32 1
+; IS__CGSCC____-NEXT: [[VECINIT2:%.*]] = insertelement <4 x double> [[VECINIT1]], double [[T2]], i32 2
+; IS__CGSCC____-NEXT: [[VECINIT3:%.*]] = insertelement <4 x double> [[VECINIT2]], double [[T3]], i32 3
+; IS__CGSCC____-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x double> [[VECINIT3]], <4 x double> [[VECINIT3]], <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+; IS__CGSCC____-NEXT: ret <4 x double> [[SHUFFLE]]
+;
+
+ ; GEP of index 0 is simplified away.
+ %arrayidx1 = getelementptr inbounds double, double* %ptr, i64 1
+ %arrayidx2 = getelementptr inbounds double, double* %ptr, i64 2
+ %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+
+ %t0 = load double, double* %ptr, align 8
+ %t1 = load double, double* %arrayidx1, align 8
+ %t2 = load double, double* %arrayidx2, align 8
+ %t3 = load double, double* %arrayidx3, align 8
+
+ %vecinit0 = insertelement <4 x double> poison, double %t0, i32 0
+ %vecinit1 = insertelement <4 x double> %vecinit0, double %t1, i32 1
+ %vecinit2 = insertelement <4 x double> %vecinit1, double %t2, i32 2
+ %vecinit3 = insertelement <4 x double> %vecinit2, double %t3, i32 3
+ %shuffle = shufflevector <4 x double> %vecinit3, <4 x double> %vecinit3, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+ ret <4 x double> %shuffle
+}
+
+
+define double @PR21780_only_access3_with_inbounds(double* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT: ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_only_access3_with_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT: ret double [[T3]]
+;
+
+ %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+ %t3 = load double, double* %arrayidx3, align 8
+ ret double %t3
+}
+
+define double @PR21780_only_access3_without_inbounds(double* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree readonly align 8 [[PTR:%.*]]) [[ATTR0]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT: ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_only_access3_without_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree readonly align 8 [[PTR:%.*]]) [[ATTR0]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT: ret double [[T3]]
+;
+ %arrayidx3 = getelementptr double, double* %ptr, i64 3
+ %t3 = load double, double* %arrayidx3, align 8
+ ret double %t3
+}
+
+define double @PR21780_without_inbounds(double* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind readonly willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@PR21780_without_inbounds
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__TUNIT____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__TUNIT____-NEXT: ret double [[T3]]
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind readonly willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@PR21780_without_inbounds
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readonly align 8 dereferenceable(32) [[PTR:%.*]]) [[ATTR0]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX3:%.*]] = getelementptr double, double* [[PTR]], i64 3
+; IS__CGSCC____-NEXT: [[T3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; IS__CGSCC____-NEXT: ret double [[T3]]
+;
+
+ %arrayidx1 = getelementptr double, double* %ptr, i64 1
+ %arrayidx2 = getelementptr double, double* %ptr, i64 2
+ %arrayidx3 = getelementptr double, double* %ptr, i64 3
+
+ %t0 = load double, double* %ptr, align 8
+ %t1 = load double, double* %arrayidx1, align 8
+ %t2 = load double, double* %arrayidx2, align 8
+ %t3 = load double, double* %arrayidx3, align 8
+
+ ret double %t3
+}
+
+; Unsimplified, but still valid. Also, throw in some bogus arguments.
+
+define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@gep0
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull writeonly dereferenceable(1) [[OTHER:%.*]], i8* nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]]) [[ATTR1:#.*]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX2:%.*]] = getelementptr i8, i8* [[PTR]], i64 2
+; IS__TUNIT____-NEXT: [[T2:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; IS__TUNIT____-NEXT: store i8 [[T2]], i8* [[OTHER]], align 1
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@gep0
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull writeonly dereferenceable(1) [[OTHER:%.*]], i8* nocapture nofree nonnull readonly dereferenceable(3) [[PTR:%.*]]) [[ATTR1:#.*]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX2:%.*]] = getelementptr i8, i8* [[PTR]], i64 2
+; IS__CGSCC____-NEXT: [[T2:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; IS__CGSCC____-NEXT: store i8 [[T2]], i8* [[OTHER]], align 1
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ store i8 %t2, i8* %other
+ ret void
+}
+
+; Order of accesses does not change computation.
+; Multiple arguments may be dereferenceable.
+
+define void @ordering(i8* %ptr1, i32* %ptr2) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@ordering
+; IS__TUNIT____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR1:%.*]], i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]]) [[ATTR2:#.*]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@ordering
+; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR1:%.*]], i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR2:%.*]]) [[ATTR2:#.*]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %a20 = getelementptr i32, i32* %ptr2, i64 0
+ %a12 = getelementptr i8, i8* %ptr1, i64 2
+ %t12 = load i8, i8* %a12
+ %a11 = getelementptr i8, i8* %ptr1, i64 1
+ %t20 = load i32, i32* %a20
+ %a10 = getelementptr i8, i8* %ptr1, i64 0
+ %t10 = load i8, i8* %a10
+ %t11 = load i8, i8* %a11
+ %a21 = getelementptr i32, i32* %ptr2, i64 1
+ %t21 = load i32, i32* %a21
+ ret void
+}
+
+; Not in entry block.
+
+define void @not_entry_but_guaranteed_to_execute(i8* %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
+; IS__TUNIT____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: entry:
+; IS__TUNIT____-NEXT: br label [[EXIT:%.*]]
+; IS__TUNIT____: exit:
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_entry_but_guaranteed_to_execute
+; IS__CGSCC____-SAME: (i8* nocapture nofree nonnull readnone dereferenceable(3) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: entry:
+; IS__CGSCC____-NEXT: br label [[EXIT:%.*]]
+; IS__CGSCC____: exit:
+; IS__CGSCC____-NEXT: ret void
+;
+entry:
+ br label %exit
+exit:
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+}
+
+; Not in entry block and not guaranteed to execute.
+
+define void @not_entry_not_guaranteed_to_execute(i8* %ptr, i1 %cond) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[PTR:%.*]], i1 [[COND:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: entry:
+; IS__TUNIT____-NEXT: br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__TUNIT____: loads:
+; IS__TUNIT____-NEXT: ret void
+; IS__TUNIT____: exit:
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_entry_not_guaranteed_to_execute
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[PTR:%.*]], i1 [[COND:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: entry:
+; IS__CGSCC____-NEXT: br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__CGSCC____: loads:
+; IS__CGSCC____-NEXT: ret void
+; IS__CGSCC____: exit:
+; IS__CGSCC____-NEXT: ret void
+;
+entry:
+ br i1 %cond, label %loads, label %exit
+loads:
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+exit:
+ ret void
+}
+
+; The last load may not execute, so derefenceable bytes only covers the 1st two loads.
+
+define void @partial_in_entry(i16* %ptr, i1 %cond) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@partial_in_entry
+; IS__TUNIT____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 [[COND:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: entry:
+; IS__TUNIT____-NEXT: br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__TUNIT____: loads:
+; IS__TUNIT____-NEXT: ret void
+; IS__TUNIT____: exit:
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@partial_in_entry
+; IS__CGSCC____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(4) [[PTR:%.*]], i1 [[COND:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: entry:
+; IS__CGSCC____-NEXT: br i1 [[COND]], label [[LOADS:%.*]], label [[EXIT:%.*]]
+; IS__CGSCC____: loads:
+; IS__CGSCC____-NEXT: ret void
+; IS__CGSCC____: exit:
+; IS__CGSCC____-NEXT: ret void
+;
+entry:
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load i16, i16* %arrayidx0
+ %t1 = load i16, i16* %arrayidx1
+ br i1 %cond, label %loads, label %exit
+loads:
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+exit:
+ ret void
+}
+
+; The volatile load can't be used to prove a non-volatile access is allowed.
+; The 2nd and 3rd loads may never execute.
+
+define void @volatile_is_not_dereferenceable(i16* %ptr) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nounwind willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@volatile_is_not_dereferenceable
+; IS__TUNIT____-SAME: (i16* nofree align 2 [[PTR:%.*]]) [[ATTR3:#.*]] {
+; IS__TUNIT____-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i16, i16* [[PTR]], i64 0
+; IS__TUNIT____-NEXT: [[T0:%.*]] = load volatile i16, i16* [[ARRAYIDX0]], align 2
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nounwind willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@volatile_is_not_dereferenceable
+; IS__CGSCC____-SAME: (i16* nofree align 2 [[PTR:%.*]]) [[ATTR3:#.*]] {
+; IS__CGSCC____-NEXT: [[ARRAYIDX0:%.*]] = getelementptr i16, i16* [[PTR]], i64 0
+; IS__CGSCC____-NEXT: [[T0:%.*]] = load volatile i16, i16* [[ARRAYIDX0]], align 2
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load volatile i16, i16* %arrayidx0
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; TODO: We should allow inference for atomic (but not volatile) ops.
+
+define void @atomic_is_alright(i16* %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@atomic_is_alright
+; IS__TUNIT____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@atomic_is_alright
+; IS__CGSCC____-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(6) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load atomic i16, i16* %arrayidx0 unordered, align 2
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+declare void @may_not_return()
+
+define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
+; CHECK-LABEL: define {{[^@]+}}@not_guaranteed_to_transfer_execution
+; CHECK-SAME: (i16* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) {
+; CHECK-NEXT: call void @may_not_return()
+; CHECK-NEXT: ret void
+;
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load i16, i16* %arrayidx0
+ call void @may_not_return()
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; We must have consecutive accesses.
+
+define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@variable_gep_index
+; IS__TUNIT____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@variable_gep_index
+; IS__CGSCC____-SAME: (i8* nocapture nofree readnone [[UNUSED:%.*]], i8* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]], i64 [[VARIABLE_INDEX:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 %variable_index
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %ptr
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+}
+
+; Deal with >1 GEP index.
+
+define void @multi_index_gep(<4 x i8>* %ptr) {
+; FIXME: %ptr should be dereferenceable(4)
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@multi_index_gep
+; IS__TUNIT____-SAME: (<4 x i8>* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@multi_index_gep
+; IS__CGSCC____-SAME: (<4 x i8>* nocapture nofree nonnull readnone dereferenceable(1) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx00 = getelementptr <4 x i8>, <4 x i8>* %ptr, i64 0, i64 0
+ %t0 = load i8, i8* %arrayidx00
+ ret void
+}
+
+; Could round weird bitwidths down?
+
+define void @not_byte_multiple(i9* %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@not_byte_multiple
+; IS__TUNIT____-SAME: (i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@not_byte_multiple
+; IS__CGSCC____-SAME: (i9* nocapture nofree nonnull readnone align 2 dereferenceable(2) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx0 = getelementptr i9, i9* %ptr, i64 0
+ %t0 = load i9, i9* %arrayidx0
+ ret void
+}
+
+; Missing direct access from the pointer.
+
+define void @no_pointer_deref(i16* %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@no_pointer_deref
+; IS__TUNIT____-SAME: (i16* nocapture nofree readnone align 2 [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@no_pointer_deref
+; IS__CGSCC____-SAME: (i16* nocapture nofree readnone align 2 [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; Out-of-order is ok, but missing access concludes dereferenceable range.
+
+define void @non_consecutive(i32* %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@non_consecutive
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@non_consecutive
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %t1 = load i32, i32* %arrayidx1
+ %t0 = load i32, i32* %arrayidx0
+ %t3 = load i32, i32* %arrayidx3
+ ret void
+}
+
+; Improve on existing dereferenceable attribute.
+
+define void @more_bytes(i32* dereferenceable(8) %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@more_bytes
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@more_bytes
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+; Improve on existing dereferenceable_or_null attribute.
+
+define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@more_bytes_and_not_null
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@more_bytes_and_not_null
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+; But don't pessimize existing dereferenceable attribute.
+
+define void @better_bytes(i32* dereferenceable(100) %ptr) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@better_bytes
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@better_bytes
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(100) [[PTR:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+define void @bitcast(i32* %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcast
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcast
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ %t0 = load float, float* %arrayidx0
+ %t1 = load float, float* %arrayidx1
+ ret void
+}
+
+define void @bitcast_
diff erent_sizes(double* %arg1, i8* %arg2) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@bitcast_
diff erent_sizes
+; IS__TUNIT____-SAME: (double* nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@bitcast_
diff erent_sizes
+; IS__CGSCC____-SAME: (double* nocapture nofree nonnull readnone align 4 dereferenceable(12) [[ARG1:%.*]], i8* nocapture nofree nonnull readnone align 4 dereferenceable(16) [[ARG2:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %ptr1 = bitcast double* %arg1 to float*
+ %a10 = getelementptr float, float* %ptr1, i64 0
+ %a11 = getelementptr float, float* %ptr1, i64 1
+ %a12 = getelementptr float, float* %ptr1, i64 2
+ %ld10 = load float, float* %a10
+ %ld11 = load float, float* %a11
+ %ld12 = load float, float* %a12
+
+ %ptr2 = bitcast i8* %arg2 to i64*
+ %a20 = getelementptr i64, i64* %ptr2, i64 0
+ %a21 = getelementptr i64, i64* %ptr2, i64 1
+ %ld20 = load i64, i64* %a20
+ %ld21 = load i64, i64* %a21
+ ret void
+}
+
+define void @negative_offset(i32* %arg) {
+; IS__TUNIT____: Function Attrs: nofree nosync nounwind readnone willreturn
+; IS__TUNIT____-LABEL: define {{[^@]+}}@negative_offset
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]]) [[ATTR2]] {
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: nofree norecurse nosync nounwind readnone willreturn
+; IS__CGSCC____-LABEL: define {{[^@]+}}@negative_offset
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull readnone align 4 dereferenceable(4) [[ARG:%.*]]) [[ATTR2]] {
+; IS__CGSCC____-NEXT: ret void
+;
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 -1
+ %t0 = load float, float* %arrayidx0
+ %t1 = load float, float* %arrayidx1
+ ret void
+}
+
+define void @stores(i32* %arg) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@stores
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR4:#.*]] {
+; IS__TUNIT____-NEXT: [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__TUNIT____-NEXT: [[ARRAYIDX0:%.*]] = getelementptr float, float* [[PTR]], i64 0
+; IS__TUNIT____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__TUNIT____-NEXT: store float 1.000000e+00, float* [[ARRAYIDX0]], align 4
+; IS__TUNIT____-NEXT: store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@stores
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR4:#.*]] {
+; IS__CGSCC____-NEXT: [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__CGSCC____-NEXT: [[ARRAYIDX0:%.*]] = getelementptr float, float* [[PTR]], i64 0
+; IS__CGSCC____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__CGSCC____-NEXT: store float 1.000000e+00, float* [[ARRAYIDX0]], align 4
+; IS__CGSCC____-NEXT: store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__CGSCC____-NEXT: ret void
+;
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ store float 1.0, float* %arrayidx0
+ store float 2.0, float* %arrayidx1
+ ret void
+}
+
+define void @load_store(i32* %arg) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@load_store
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__TUNIT____-NEXT: [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__TUNIT____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__TUNIT____-NEXT: store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@load_store
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 4 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__CGSCC____-NEXT: [[PTR:%.*]] = bitcast i32* [[ARG]] to float*
+; IS__CGSCC____-NEXT: [[ARRAYIDX1:%.*]] = getelementptr float, float* [[PTR]], i64 1
+; IS__CGSCC____-NEXT: store float 2.000000e+00, float* [[ARRAYIDX1]], align 4
+; IS__CGSCC____-NEXT: ret void
+;
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ %t1 = load float, float* %arrayidx0
+ store float 2.0, float* %arrayidx1
+ ret void
+}
+
+define void @
diff erent_size1(i32* %arg) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@
diff erent_size1
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__TUNIT____-NEXT: [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__TUNIT____-NEXT: store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__TUNIT____-NEXT: store i32 0, i32* [[ARG]], align 8
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@
diff erent_size1
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__CGSCC____-NEXT: [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__CGSCC____-NEXT: store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__CGSCC____-NEXT: store i32 0, i32* [[ARG]], align 8
+; IS__CGSCC____-NEXT: ret void
+;
+ %arg-cast = bitcast i32* %arg to double*
+ store double 0.000000e+00, double* %arg-cast
+ store i32 0, i32* %arg
+ ret void
+}
+
+define void @
diff erent_size2(i32* %arg) {
+; IS__TUNIT____: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT____-LABEL: define {{[^@]+}}@
diff erent_size2
+; IS__TUNIT____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__TUNIT____-NEXT: store i32 0, i32* [[ARG]], align 8
+; IS__TUNIT____-NEXT: [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__TUNIT____-NEXT: store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__TUNIT____-NEXT: ret void
+;
+; IS__CGSCC____: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC____-LABEL: define {{[^@]+}}@
diff erent_size2
+; IS__CGSCC____-SAME: (i32* nocapture nofree nonnull writeonly align 8 dereferenceable(8) [[ARG:%.*]]) [[ATTR4]] {
+; IS__CGSCC____-NEXT: store i32 0, i32* [[ARG]], align 8
+; IS__CGSCC____-NEXT: [[ARG_CAST:%.*]] = bitcast i32* [[ARG]] to double*
+; IS__CGSCC____-NEXT: store double 0.000000e+00, double* [[ARG_CAST]], align 8
+; IS__CGSCC____-NEXT: ret void
+;
+ store i32 0, i32* %arg
+ %arg-cast = bitcast i32* %arg to double*
+ store double 0.000000e+00, double* %arg-cast
+ ret void
+}
+
+; Make use of MustBeExecuted Explorer
+;
+; [CFG]
+; entry
+; / \
+; l1 l2
+; | X |
+; l3 l4
+; \ /
+; l5
+; / \
+; l6 l7
+; \ /
+; end
+; According to the above CFG, we can see that instructions in l5 Block must be executed.
+; Therefore, %p must be dereferenced.
+;
+; ATTRIBUTOR_CGSCC_NPM-LABEL: define i32 @require_cfg_analysis(i32 %c, i32* {{.*}} dereferenceable(4) %p)
+define i32 @require_cfg_analysis(i32 %c, i32* %p) {
+; IS__TUNIT_OPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_OPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__TUNIT_OPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree writeonly [[P:%.*]]) [[ATTR4:#.*]] {
+; IS__TUNIT_OPM-NEXT: [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__TUNIT_OPM-NEXT: br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__TUNIT_OPM: l1:
+; IS__TUNIT_OPM-NEXT: [[TOBOOL2:%.*]] = icmp eq i32 [[C]], 1
+; IS__TUNIT_OPM-NEXT: br i1 [[TOBOOL2]], label [[L3:%.*]], label [[L4:%.*]]
+; IS__TUNIT_OPM: l2:
+; IS__TUNIT_OPM-NEXT: [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__TUNIT_OPM-NEXT: br i1 [[TOBOOL3]], label [[L3]], label [[L4]]
+; IS__TUNIT_OPM: l3:
+; IS__TUNIT_OPM-NEXT: br label [[L5:%.*]]
+; IS__TUNIT_OPM: l4:
+; IS__TUNIT_OPM-NEXT: br label [[L5]]
+; IS__TUNIT_OPM: l5:
+; IS__TUNIT_OPM-NEXT: [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__TUNIT_OPM-NEXT: br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__TUNIT_OPM: l6:
+; IS__TUNIT_OPM-NEXT: store i32 0, i32* [[P]], align 4
+; IS__TUNIT_OPM-NEXT: br label [[END:%.*]]
+; IS__TUNIT_OPM: l7:
+; IS__TUNIT_OPM-NEXT: store i32 1, i32* [[P]], align 4
+; IS__TUNIT_OPM-NEXT: br label [[END]]
+; IS__TUNIT_OPM: end:
+; IS__TUNIT_OPM-NEXT: ret i32 1
+;
+; IS__TUNIT_NPM: Function Attrs: argmemonly nofree nosync nounwind willreturn writeonly
+; IS__TUNIT_NPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__TUNIT_NPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) [[ATTR4:#.*]] {
+; IS__TUNIT_NPM-NEXT: [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__TUNIT_NPM-NEXT: br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__TUNIT_NPM: l1:
+; IS__TUNIT_NPM-NEXT: br label [[L4:%.*]]
+; IS__TUNIT_NPM: l2:
+; IS__TUNIT_NPM-NEXT: [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__TUNIT_NPM-NEXT: br i1 [[TOBOOL3]], label [[L3:%.*]], label [[L4]]
+; IS__TUNIT_NPM: l3:
+; IS__TUNIT_NPM-NEXT: br label [[L5:%.*]]
+; IS__TUNIT_NPM: l4:
+; IS__TUNIT_NPM-NEXT: br label [[L5]]
+; IS__TUNIT_NPM: l5:
+; IS__TUNIT_NPM-NEXT: [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__TUNIT_NPM-NEXT: br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__TUNIT_NPM: l6:
+; IS__TUNIT_NPM-NEXT: store i32 0, i32* [[P]], align 4
+; IS__TUNIT_NPM-NEXT: br label [[END:%.*]]
+; IS__TUNIT_NPM: l7:
+; IS__TUNIT_NPM-NEXT: store i32 1, i32* [[P]], align 4
+; IS__TUNIT_NPM-NEXT: br label [[END]]
+; IS__TUNIT_NPM: end:
+; IS__TUNIT_NPM-NEXT: ret i32 1
+;
+; IS__CGSCC_OPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_OPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__CGSCC_OPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree writeonly [[P:%.*]]) [[ATTR4:#.*]] {
+; IS__CGSCC_OPM-NEXT: [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC_OPM-NEXT: br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC_OPM: l1:
+; IS__CGSCC_OPM-NEXT: [[TOBOOL2:%.*]] = icmp eq i32 [[C]], 1
+; IS__CGSCC_OPM-NEXT: br i1 [[TOBOOL2]], label [[L3:%.*]], label [[L4:%.*]]
+; IS__CGSCC_OPM: l2:
+; IS__CGSCC_OPM-NEXT: [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__CGSCC_OPM-NEXT: br i1 [[TOBOOL3]], label [[L3]], label [[L4]]
+; IS__CGSCC_OPM: l3:
+; IS__CGSCC_OPM-NEXT: br label [[L5:%.*]]
+; IS__CGSCC_OPM: l4:
+; IS__CGSCC_OPM-NEXT: br label [[L5]]
+; IS__CGSCC_OPM: l5:
+; IS__CGSCC_OPM-NEXT: [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__CGSCC_OPM-NEXT: br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__CGSCC_OPM: l6:
+; IS__CGSCC_OPM-NEXT: store i32 0, i32* [[P]], align 4
+; IS__CGSCC_OPM-NEXT: br label [[END:%.*]]
+; IS__CGSCC_OPM: l7:
+; IS__CGSCC_OPM-NEXT: store i32 1, i32* [[P]], align 4
+; IS__CGSCC_OPM-NEXT: br label [[END]]
+; IS__CGSCC_OPM: end:
+; IS__CGSCC_OPM-NEXT: ret i32 1
+;
+; IS__CGSCC_NPM: Function Attrs: argmemonly nofree norecurse nosync nounwind willreturn writeonly
+; IS__CGSCC_NPM-LABEL: define {{[^@]+}}@require_cfg_analysis
+; IS__CGSCC_NPM-SAME: (i32 [[C:%.*]], i32* nocapture nofree nonnull writeonly align 4 dereferenceable(4) [[P:%.*]]) [[ATTR4:#.*]] {
+; IS__CGSCC_NPM-NEXT: [[TOBOOL1:%.*]] = icmp eq i32 [[C]], 0
+; IS__CGSCC_NPM-NEXT: br i1 [[TOBOOL1]], label [[L1:%.*]], label [[L2:%.*]]
+; IS__CGSCC_NPM: l1:
+; IS__CGSCC_NPM-NEXT: br label [[L4:%.*]]
+; IS__CGSCC_NPM: l2:
+; IS__CGSCC_NPM-NEXT: [[TOBOOL3:%.*]] = icmp eq i32 [[C]], 2
+; IS__CGSCC_NPM-NEXT: br i1 [[TOBOOL3]], label [[L3:%.*]], label [[L4]]
+; IS__CGSCC_NPM: l3:
+; IS__CGSCC_NPM-NEXT: br label [[L5:%.*]]
+; IS__CGSCC_NPM: l4:
+; IS__CGSCC_NPM-NEXT: br label [[L5]]
+; IS__CGSCC_NPM: l5:
+; IS__CGSCC_NPM-NEXT: [[TOBOOL4:%.*]] = icmp eq i32 [[C]], 4
+; IS__CGSCC_NPM-NEXT: br i1 [[TOBOOL4]], label [[L6:%.*]], label [[L7:%.*]]
+; IS__CGSCC_NPM: l6:
+; IS__CGSCC_NPM-NEXT: store i32 0, i32* [[P]], align 4
+; IS__CGSCC_NPM-NEXT: br label [[END:%.*]]
+; IS__CGSCC_NPM: l7:
+; IS__CGSCC_NPM-NEXT: store i32 1, i32* [[P]], align 4
+; IS__CGSCC_NPM-NEXT: br label [[END]]
+; IS__CGSCC_NPM: end:
+; IS__CGSCC_NPM-NEXT: ret i32 1
+;
+ %tobool1 = icmp eq i32 %c, 0
+ br i1 %tobool1, label %l1, label %l2
+l1:
+ %tobool2 = icmp eq i32 %c, 1
+ br i1 %tobool2, label %l3, label %l4
+l2:
+ %tobool3 = icmp eq i32 %c, 2
+ br i1 %tobool3, label %l3, label %l4
+l3:
+ br label %l5
+l4:
+ br label %l5
+l5:
+ %tobool4 = icmp eq i32 %c, 4
+ br i1 %tobool4, label %l6, label %l7
+l6:
+ store i32 0, i32* %p
+ br label %end
+l7:
+ store i32 1, i32* %p
+ br label %end
+end:
+ ret i32 1
+}
diff --git a/llvm/test/Transforms/BDCE/vectors-inseltpoison.ll b/llvm/test/Transforms/BDCE/vectors-inseltpoison.ll
new file mode 100644
index 000000000000..c38562b0096b
--- /dev/null
+++ b/llvm/test/Transforms/BDCE/vectors-inseltpoison.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -bdce < %s | FileCheck %s
+
+; BDCE applied to integer vectors.
+
+define <2 x i32> @test_basic(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test_basic(
+; CHECK-NEXT: [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT: [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT: [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
+; CHECK-NEXT: [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT: [[D:%.*]] = ashr <2 x i32> [[C]], <i32 3, i32 3>
+; CHECK-NEXT: ret <2 x i32> [[D]]
+;
+ %a2 = add <2 x i32> %a, <i32 1, i32 1>
+ %a3 = and <2 x i32> %a2, <i32 4, i32 4>
+ %b2 = add <2 x i32> %b, <i32 1, i32 1>
+ %b3 = and <2 x i32> %b2, <i32 8, i32 8>
+ %c = or <2 x i32> %a3, %b3
+ %d = ashr <2 x i32> %c, <i32 3, i32 3>
+ ret <2 x i32> %d
+}
+
+; Going vector -> scalar
+define i32 @test_extractelement(<2 x i32> %a, <2 x i32> %b) {
+; CHECK-LABEL: @test_extractelement(
+; CHECK-NEXT: [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT: [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
+; CHECK-NEXT: [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
+; CHECK-NEXT: [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT: [[D:%.*]] = extractelement <2 x i32> [[C]], i32 0
+; CHECK-NEXT: [[E:%.*]] = ashr i32 [[D]], 3
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %a2 = add <2 x i32> %a, <i32 1, i32 1>
+ %a3 = and <2 x i32> %a2, <i32 4, i32 4>
+ %b2 = add <2 x i32> %b, <i32 1, i32 1>
+ %b3 = and <2 x i32> %b2, <i32 8, i32 8>
+ %c = or <2 x i32> %a3, %b3
+ %d = extractelement <2 x i32> %c, i32 0
+ %e = ashr i32 %d, 3
+ ret i32 %e
+}
+
+; Going scalar -> vector
+define <2 x i32> @test_insertelement(i32 %a, i32 %b) {
+; CHECK-LABEL: @test_insertelement(
+; CHECK-NEXT: [[X3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
+; CHECK-NEXT: [[Y:%.*]] = insertelement <2 x i32> poison, i32 [[B:%.*]], i32 0
+; CHECK-NEXT: [[Y2:%.*]] = insertelement <2 x i32> [[Y]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT: [[Y3:%.*]] = and <2 x i32> [[Y2]], <i32 8, i32 8>
+; CHECK-NEXT: [[Z:%.*]] = or <2 x i32> [[X3]], [[Y3]]
+; CHECK-NEXT: [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
+; CHECK-NEXT: ret <2 x i32> [[U]]
+;
+ %x = insertelement <2 x i32> poison, i32 %a, i32 0
+ %x2 = insertelement <2 x i32> %x, i32 %b, i32 1
+ %x3 = and <2 x i32> %x2, <i32 4, i32 4>
+ %y = insertelement <2 x i32> poison, i32 %b, i32 0
+ %y2 = insertelement <2 x i32> %y, i32 %a, i32 1
+ %y3 = and <2 x i32> %y2, <i32 8, i32 8>
+ %z = or <2 x i32> %x3, %y3
+ %u = ashr <2 x i32> %z, <i32 3, i32 3>
+ ret <2 x i32> %u
+}
+
+; Some non-int vectors and conversions
+define <2 x i32> @test_conversion(<2 x i32> %a) {
+; CHECK-LABEL: @test_conversion(
+; CHECK-NEXT: [[A2:%.*]] = add <2 x i32> [[A:%.*]], <i32 1, i32 1>
+; CHECK-NEXT: [[A3:%.*]] = and <2 x i32> [[A2]], <i32 2, i32 2>
+; CHECK-NEXT: [[X:%.*]] = uitofp <2 x i32> [[A3]] to <2 x double>
+; CHECK-NEXT: [[Y:%.*]] = fadd <2 x double> [[X]], <double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT: [[Z:%.*]] = fptoui <2 x double> [[Y]] to <2 x i32>
+; CHECK-NEXT: [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
+; CHECK-NEXT: ret <2 x i32> [[U]]
+;
+ %a2 = add <2 x i32> %a, <i32 1, i32 1>
+ %a3 = and <2 x i32> %a2, <i32 2, i32 2>
+ %x = uitofp <2 x i32> %a3 to <2 x double>
+ %y = fadd <2 x double> %x, <double 1.0, double 1.0>
+ %z = fptoui <2 x double> %y to <2 x i32>
+ %u = ashr <2 x i32> %z, <i32 3, i32 3>
+ ret <2 x i32> %u
+}
+
+; Assumption invalidation (adapted from invalidate-assumptions.ll)
+define <2 x i1> @test_assumption_invalidation(<2 x i1> %b, <2 x i8> %x) {
+; CHECK-LABEL: @test_assumption_invalidation(
+; CHECK-NEXT: [[LITTLE_NUMBER:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i8>
+; CHECK-NEXT: [[BIG_NUMBER:%.*]] = shl <2 x i8> zeroinitializer, <i8 1, i8 1>
+; CHECK-NEXT: [[SUB:%.*]] = sub <2 x i8> [[BIG_NUMBER]], [[LITTLE_NUMBER]]
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc <2 x i8> [[SUB]] to <2 x i1>
+; CHECK-NEXT: ret <2 x i1> [[TRUNC]]
+;
+ %setbit = or <2 x i8> %x, <i8 64, i8 64>
+ %little_number = zext <2 x i1> %b to <2 x i8>
+ %big_number = shl <2 x i8> %setbit, <i8 1, i8 1>
+ %sub = sub nuw <2 x i8> %big_number, %little_number
+ %trunc = trunc <2 x i8> %sub to <2 x i1>
+ ret <2 x i1> %trunc
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll
new file mode 100644
index 000000000000..5611ac7068ba
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AArch64/gather-scatter-opt-inseltpoison.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+%struct.a = type { i32, i32 }
+ at c = external dso_local global %struct.a, align 4
+ at glob_array = internal unnamed_addr constant [16 x i32] [i32 1, i32 1, i32 2, i32 3, i32 5, i32 8, i32 13, i32 21, i32 34, i32 55, i32 89, i32 144, i32 233, i32 377, i32 610, i32 987], align 16
+
+define <vscale x 4 x i32> @splat_base(i32* %base, <vscale x 4 x i64> %index, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @splat_base(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <vscale x 4 x i64> [[INDEX:%.*]]
+; CHECK-NEXT: [[RES:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <vscale x 4 x i32*> poison, i32* %base, i32 0
+ %broadcast.splat = shufflevector <vscale x 4 x i32*> %broadcast.splatinsert, <vscale x 4 x i32*> undef, <vscale x 4 x i32> zeroinitializer
+ %gep = getelementptr i32, <vscale x 4 x i32*> %broadcast.splat, <vscale x 4 x i64> %index
+ %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %gep, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @splat_struct(%struct.a* %base, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @splat_struct(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_A:%.*]], %struct.a* [[BASE:%.*]], i64 0, i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <vscale x 4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP2]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[RES]]
+;
+ %gep = getelementptr %struct.a, %struct.a* %base, <vscale x 4 x i64> zeroinitializer, i32 1
+ %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %gep, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @scalar_index(i32* %base, i64 %index, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @scalar_index(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[INDEX:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <vscale x 4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP2]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <vscale x 4 x i32*> poison, i32* %base, i32 0
+ %broadcast.splat = shufflevector <vscale x 4 x i32*> %broadcast.splatinsert, <vscale x 4 x i32*> undef, <vscale x 4 x i32> zeroinitializer
+ %gep = getelementptr i32, <vscale x 4 x i32*> %broadcast.splat, i64 %index
+ %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %gep, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @splat_index(i32* %base, i64 %index, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @splat_index(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[INDEX:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <vscale x 4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP2]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <vscale x 4 x i64> poison, i64 %index, i32 0
+ %broadcast.splat = shufflevector <vscale x 4 x i64> %broadcast.splatinsert, <vscale x 4 x i64> undef, <vscale x 4 x i32> zeroinitializer
+ %gep = getelementptr i32, i32* %base, <vscale x 4 x i64> %broadcast.splat
+ %res = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %gep, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %res
+}
+
+define <vscale x 4 x i32> @test_global_array(<vscale x 4 x i64> %indxs, <vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @test_global_array(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @glob_array, i64 0, i64 0), <vscale x 4 x i64> [[INDXS:%.*]]
+; CHECK-NEXT: [[G:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[G]]
+;
+ %p = getelementptr inbounds [16 x i32], [16 x i32]* @glob_array, i64 0, <vscale x 4 x i64> %indxs
+ %g = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %p, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %g
+}
+
+define <vscale x 4 x i32> @global_struct_splat(<vscale x 4 x i1> %mask) #0 {
+; CHECK-LABEL: @global_struct_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> shufflevector (<vscale x 4 x i32*> insertelement (<vscale x 4 x i32*> undef, i32* getelementptr inbounds (%struct.a, %struct.a* @c, i64 0, i32 1), i32 0), <vscale x 4 x i32*> undef, <vscale x 4 x i32> zeroinitializer), i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> undef)
+; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP1]]
+;
+ %1 = insertelement <vscale x 4 x %struct.a*> poison, %struct.a* @c, i32 0
+ %2 = shufflevector <vscale x 4 x %struct.a*> %1, <vscale x 4 x %struct.a*> undef, <vscale x 4 x i32> zeroinitializer
+ %3 = getelementptr %struct.a, <vscale x 4 x %struct.a*> %2, <vscale x 4 x i64> zeroinitializer, i32 1
+ %4 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %3, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> undef)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @splat_ptr_gather(i32* %ptr, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru) #0 {
+; CHECK-LABEL: @splat_ptr_gather(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], <vscale x 4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK:%.*]], <vscale x 4 x i32> [[PASSTHRU:%.*]])
+; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP2]]
+;
+ %1 = insertelement <vscale x 4 x i32*> poison, i32* %ptr, i32 0
+ %2 = shufflevector <vscale x 4 x i32*> %1, <vscale x 4 x i32*> undef, <vscale x 4 x i32> zeroinitializer
+ %3 = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*> %2, i32 4, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %passthru)
+ ret <vscale x 4 x i32> %3
+}
+
+define void @splat_ptr_scatter(i32* %ptr, <vscale x 4 x i1> %mask, <vscale x 4 x i32> %val) #0 {
+; CHECK-LABEL: @splat_ptr_scatter(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], <vscale x 4 x i64> zeroinitializer
+; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> [[VAL:%.*]], <vscale x 4 x i32*> [[TMP1]], i32 4, <vscale x 4 x i1> [[MASK:%.*]])
+; CHECK-NEXT: ret void
+;
+ %1 = insertelement <vscale x 4 x i32*> poison, i32* %ptr, i32 0
+ %2 = shufflevector <vscale x 4 x i32*> %1, <vscale x 4 x i32*> undef, <vscale x 4 x i32> zeroinitializer
+ call void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32> %val, <vscale x 4 x i32*> %2, i32 4, <vscale x 4 x i1> %mask)
+ ret void
+}
+
+declare <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0i32(<vscale x 4 x i32*>, i32, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare void @llvm.masked.scatter.nxv4i32.nxv4p0i32(<vscale x 4 x i32>, <vscale x 4 x i32*>, i32, <vscale x 4 x i1>)
+
+attributes #0 = { "target-features"="+sve" }
diff --git a/llvm/test/Transforms/CodeGenPrepare/AMDGPU/bypass-slow-div-debug-info-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/bypass-slow-div-debug-info-inseltpoison.ll
new file mode 100644
index 000000000000..8c198a3c7d60
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/AMDGPU/bypass-slow-div-debug-info-inseltpoison.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -codegenprepare %s | FileCheck %s
+; Make sure BypassSlowDivision doesn't drop debug info
+
+define i64 @sdiv64(i64 %a, i64 %b) {
+; CHECK-LABEL: @sdiv64(
+; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[A:%.*]], [[B:%.*]], [[DBG6:!dbg !.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4294967296, [[DBG6]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 0, [[DBG6]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP9:%.*]], [[DBG6]]
+; CHECK: 4:
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[B]] to i32, [[DBG6]]
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[A]] to i32, [[DBG6]]
+; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[TMP5]], [[DBG6]]
+; CHECK-NEXT: [[TMP8:%.*]] = zext i32 [[TMP7]] to i64, [[DBG6]]
+; CHECK-NEXT: br label [[TMP11:%.*]], [[DBG6]]
+; CHECK: 9:
+; CHECK-NEXT: [[TMP10:%.*]] = sdiv i64 [[A]], [[B]], [[DBG6]]
+; CHECK-NEXT: br label [[TMP11]], [[DBG6]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i64 [ [[TMP8]], [[TMP4]] ], [ [[TMP10]], [[TMP9]] ], [[DBG6]]
+; CHECK-NEXT: ret i64 [[TMP12]]
+;
+ %d = sdiv i64 %a, %b, !dbg !6
+ ret i64 %d
+}
+
+; FIXME: The debugloc for the rem parts end up with the dbg of the
+; division.
+define <2 x i64> @sdivrem64(i64 %a, i64 %b) {
+; CHECK-LABEL: @sdivrem64(
+; CHECK-NEXT: [[TMP1:%.*]] = or i64 [[A:%.*]], [[B:%.*]], [[DBG6]]
+; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4294967296, [[DBG6]]
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[TMP2]], 0, [[DBG6]]
+; CHECK-NEXT: br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP11:%.*]], [[DBG6]]
+; CHECK: 4:
+; CHECK-NEXT: [[TMP5:%.*]] = trunc i64 [[B]] to i32, [[DBG6]]
+; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[A]] to i32, [[DBG6]]
+; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[TMP5]], [[DBG6]]
+; CHECK-NEXT: [[TMP8:%.*]] = urem i32 [[TMP6]], [[TMP5]], [[DBG6]]
+; CHECK-NEXT: [[TMP9:%.*]] = zext i32 [[TMP7]] to i64, [[DBG6]]
+; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[TMP8]] to i64, [[DBG6]]
+; CHECK-NEXT: br label [[TMP14:%.*]], [[DBG6]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = sdiv i64 [[A]], [[B]], [[DBG6]]
+; CHECK-NEXT: [[TMP13:%.*]] = srem i64 [[A]], [[B]], [[DBG6]]
+; CHECK-NEXT: br label [[TMP14]], [[DBG6]]
+; CHECK: 14:
+; CHECK-NEXT: [[TMP15:%.*]] = phi i64 [ [[TMP9]], [[TMP4]] ], [ [[TMP12]], [[TMP11]] ], [[DBG6]]
+; CHECK-NEXT: [[TMP16:%.*]] = phi i64 [ [[TMP10]], [[TMP4]] ], [ [[TMP13]], [[TMP11]] ], [[DBG6]]
+; CHECK-NEXT: [[INS0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP15]], i32 0
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x i64> [[INS0]], i64 [[TMP16]], i32 1
+; CHECK-NEXT: ret <2 x i64> [[INS1]]
+;
+ %d = sdiv i64 %a, %b, !dbg !6
+ %r = srem i64 %a, %b, !dbg !10
+ %ins0 = insertelement <2 x i64> poison, i64 %d, i32 0
+ %ins1 = insertelement <2 x i64> %ins0, i64 %r, i32 1
+ ret <2 x i64> %ins1
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.5 ", isOptimized: false, runtimeVersion: 0, emissionKind: NoDebug, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "basic.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 1, !"Debug Info Version", i32 3}
+!5 = !{!"clang version 3.5 "}
+!6 = !DILocation(line: 3, scope: !7)
+!7 = distinct !DILexicalBlock(scope: !8, file: !1, line: 3)
+!8 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !9, scopeLine: 1, virtualIndex: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !2)
+!10 = !DILocation(line: 4, scope: !7)
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector-inseltpoison.ll
new file mode 100644
index 000000000000..f43b8f6d540d
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sink-add-mul-shufflevector-inseltpoison.ll
@@ -0,0 +1,219 @@
+; RUN: opt -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp < %s -codegenprepare -S | FileCheck -check-prefix=CHECK %s
+
+define void @sink_add_mul(i32* %s1, i32 %x, i32* %d, i32 %n) {
+; CHECK-LABEL: @sink_add_mul(
+; CHECK: vector.ph:
+; CHECK-NOT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: vector.body:
+; CHECK: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+entry:
+ %cmp6 = icmp sgt i32 %n, 0
+ br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i32 %n, -4
+ %broadcast.splatinsert8 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+ %1 = bitcast i32* %0 to <4 x i32>*
+ %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+ %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat9
+ %3 = getelementptr inbounds i32, i32* %d, i32 %index
+ %4 = bitcast i32* %3 to <4 x i32>*
+ %wide.load10 = load <4 x i32>, <4 x i32>* %4, align 4
+ %5 = add nsw <4 x i32> %wide.load10, %2
+ %6 = bitcast i32* %3 to <4 x i32>*
+ store <4 x i32> %5, <4 x i32>* %6, align 4
+ %index.next = add i32 %index, 4
+ %7 = icmp eq i32 %index.next, %n.vec
+ br i1 %7, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+}
+
+define void @sink_add_mul_multiple(i32* %s1, i32* %s2, i32 %x, i32* %d, i32* %d2, i32 %n) {
+; CHECK-LABEL: @sink_add_mul_multiple(
+; CHECK: vector.ph:
+; CHECK-NOT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> undef, i32 [[X:%.*]], i32 0
+; CHECK-NOT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: vector.body:
+; CHECK: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 %x, i32 0
+; CHECK: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: mul nsw <4 x i32> %wide.load, [[TMP3]]
+; CHECK: [[TMP2b:%.*]] = insertelement <4 x i32> poison, i32 %x, i32 0
+; CHECK: [[TMP3b:%.*]] = shufflevector <4 x i32> [[TMP2b]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: mul nsw <4 x i32> %wide.load18, [[TMP3b]]
+;
+entry:
+ %cmp13 = icmp sgt i32 %n, 0
+ br i1 %cmp13, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i32 %n, -4
+ %broadcast.splatinsert15 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+ %1 = bitcast i32* %0 to <4 x i32>*
+ %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+ %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat16
+ %3 = getelementptr inbounds i32, i32* %d, i32 %index
+ %4 = bitcast i32* %3 to <4 x i32>*
+ %wide.load17 = load <4 x i32>, <4 x i32>* %4, align 4
+ %5 = add nsw <4 x i32> %wide.load17, %2
+ %6 = bitcast i32* %3 to <4 x i32>*
+ store <4 x i32> %5, <4 x i32>* %6, align 4
+ %7 = getelementptr inbounds i32, i32* %s2, i32 %index
+ %8 = bitcast i32* %7 to <4 x i32>*
+ %wide.load18 = load <4 x i32>, <4 x i32>* %8, align 4
+ %9 = mul nsw <4 x i32> %wide.load18, %broadcast.splat16
+ %10 = getelementptr inbounds i32, i32* %d2, i32 %index
+ %11 = bitcast i32* %10 to <4 x i32>*
+ %wide.load19 = load <4 x i32>, <4 x i32>* %11, align 4
+ %12 = add nsw <4 x i32> %wide.load19, %9
+ %13 = bitcast i32* %10 to <4 x i32>*
+ store <4 x i32> %12, <4 x i32>* %13, align 4
+ %index.next = add i32 %index, 4
+ %14 = icmp eq i32 %index.next, %n.vec
+ br i1 %14, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+}
+
+
+define void @sink_add_sub_unsinkable(i32* %s1, i32* %s2, i32 %x, i32* %d, i32* %d2, i32 %n) {
+; CHECK-LABEL: @sink_add_sub_unsinkable(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP13:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT: br i1 [[CMP13]], label [[VECTOR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT15]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+;
+entry:
+ %cmp13 = icmp sgt i32 %n, 0
+ br i1 %cmp13, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i32 %n, -4
+ %broadcast.splatinsert15 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+ %1 = bitcast i32* %0 to <4 x i32>*
+ %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+ %2 = mul nsw <4 x i32> %wide.load, %broadcast.splat16
+ %3 = getelementptr inbounds i32, i32* %d, i32 %index
+ %4 = bitcast i32* %3 to <4 x i32>*
+ %wide.load17 = load <4 x i32>, <4 x i32>* %4, align 4
+ %5 = add nsw <4 x i32> %wide.load17, %2
+ %6 = bitcast i32* %3 to <4 x i32>*
+ store <4 x i32> %5, <4 x i32>* %6, align 4
+ %7 = getelementptr inbounds i32, i32* %s2, i32 %index
+ %8 = bitcast i32* %7 to <4 x i32>*
+ %wide.load18 = load <4 x i32>, <4 x i32>* %8, align 4
+ %9 = sub nsw <4 x i32> %broadcast.splat16, %wide.load18
+ %10 = getelementptr inbounds i32, i32* %d2, i32 %index
+ %11 = bitcast i32* %10 to <4 x i32>*
+ %wide.load19 = load <4 x i32>, <4 x i32>* %11, align 4
+ %12 = add nsw <4 x i32> %wide.load19, %9
+ %13 = bitcast i32* %10 to <4 x i32>*
+ store <4 x i32> %12, <4 x i32>* %13, align 4
+ %index.next = add i32 %index, 4
+ %14 = icmp eq i32 %index.next, %n.vec
+ br i1 %14, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+}
+
+define void @sink_sub(i32* %s1, i32 %x, i32* %d, i32 %n) {
+; CHECK-LABEL: @sink_sub(
+; CHECK: vector.ph:
+; CHECK-NOT: [[BROADCAST_SPLATINSERT8:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NOT: [[BROADCAST_SPLAT9:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK: vector.body:
+; CHECK: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+entry:
+ %cmp6 = icmp sgt i32 %n, 0
+ br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i32 %n, -4
+ %broadcast.splatinsert8 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+ %1 = bitcast i32* %0 to <4 x i32>*
+ %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+ %2 = sub nsw <4 x i32> %wide.load, %broadcast.splat9
+ %3 = getelementptr inbounds i32, i32* %d, i32 %index
+ %4 = bitcast i32* %3 to <4 x i32>*
+ store <4 x i32> %2, <4 x i32>* %4, align 4
+ %index.next = add i32 %index, 4
+ %5 = icmp eq i32 %index.next, %n.vec
+ br i1 %5, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+}
+
+define void @sink_sub_unsinkable(i32* %s1, i32 %x, i32* %d, i32 %n) {
+entry:
+; CHECK-LABEL: @sink_sub_unsinkable(
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N]], -4
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT15]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NOT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NOT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> zeroinitializer
+;
+ %cmp6 = icmp sgt i32 %n, 0
+ br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
+
+vector.ph: ; preds = %for.body.preheader
+ %n.vec = and i32 %n, -4
+ %broadcast.splatinsert8 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i32, i32* %s1, i32 %index
+ %1 = bitcast i32* %0 to <4 x i32>*
+ %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+ %2 = sub nsw <4 x i32> %broadcast.splat9, %wide.load
+ %3 = getelementptr inbounds i32, i32* %d, i32 %index
+ %4 = bitcast i32* %3 to <4 x i32>*
+ store <4 x i32> %2, <4 x i32>* %4, align 4
+ %index.next = add i32 %index, 4
+ %5 = icmp eq i32 %index.next, %n.vec
+ br i1 %5, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup: ; preds = %for.body, %middle.block, %entry
+ ret void
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain-inseltpoison.ll
new file mode 100644
index 000000000000..7cffedea8a35
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/ARM/sinkchain-inseltpoison.ll
@@ -0,0 +1,107 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp < %s -codegenprepare -S | FileCheck -check-prefix=CHECK %s
+
+; Sink the shufflevector/insertelement pair, followed by the trunc. The sunk instruction end up dead.
+define signext i8 @dead(i16* noalias nocapture readonly %s1, i16 zeroext %x, i8* noalias nocapture %d, i32 %n) {
+; CHECK-LABEL: @dead(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N:%.*]], -8
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i16 [[X:%.*]] to i8
+; CHECK-NEXT: [[L6:%.*]] = getelementptr inbounds i16, i16* [[S1:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[L7:%.*]] = bitcast i16* [[L6]] to <8 x i16>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[L7]], align 2
+; CHECK-NEXT: [[L8:%.*]] = trunc <8 x i16> [[WIDE_LOAD]] to <8 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[L9:%.*]] = mul <8 x i8> [[TMP2]], [[L8]]
+; CHECK-NEXT: [[L13:%.*]] = getelementptr inbounds i8, i8* [[D:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[L14:%.*]] = bitcast i8* [[L13]] to <8 x i8>*
+; CHECK-NEXT: store <8 x i8> [[L9]], <8 x i8>* [[L14]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT: [[L15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[L15]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; CHECK: exit:
+; CHECK-NEXT: ret i8 0
+;
+entry:
+ %n.vec = and i32 %n, -8
+ %l0 = trunc i16 %x to i8
+ %l1 = insertelement <8 x i8> poison, i8 %l0, i32 0
+ %broadcast.splat26 = shufflevector <8 x i8> %l1, <8 x i8> undef, <8 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+ %l6 = getelementptr inbounds i16, i16* %s1, i32 %index
+ %l7 = bitcast i16* %l6 to <8 x i16>*
+ %wide.load = load <8 x i16>, <8 x i16>* %l7, align 2
+ %l8 = trunc <8 x i16> %wide.load to <8 x i8>
+ %l9 = mul <8 x i8> %broadcast.splat26, %l8
+ %l13 = getelementptr inbounds i8, i8* %d, i32 %index
+ %l14 = bitcast i8* %l13 to <8 x i8>*
+ store <8 x i8> %l9, <8 x i8>* %l14, align 1
+ %index.next = add i32 %index, 8
+ %l15 = icmp eq i32 %index.next, %n.vec
+ br i1 %l15, label %exit, label %vector.body
+
+exit: ; preds = %vector.body
+ ret i8 0
+}
+
+; Same as above, but the shuffle has an extra use meaning it shouldnt be deleted
+define signext i8 @alive(i16* noalias nocapture readonly %s1, i16 zeroext %x, i8* noalias nocapture %d, i32 %n) {
+; CHECK-LABEL: @alive(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[N:%.*]], -8
+; CHECK-NEXT: [[L0:%.*]] = trunc i16 [[X:%.*]] to i8
+; CHECK-NEXT: [[L1:%.*]] = insertelement <8 x i8> poison, i8 [[L0]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT26:%.*]] = shufflevector <8 x i8> [[L1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[L2:%.*]] = sub <8 x i8> zeroinitializer, [[BROADCAST_SPLAT26]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = trunc i16 [[X]] to i8
+; CHECK-NEXT: [[L6:%.*]] = getelementptr inbounds i16, i16* [[S1:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[L7:%.*]] = bitcast i16* [[L6]] to <8 x i16>*
+; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, <8 x i16>* [[L7]], align 2
+; CHECK-NEXT: [[L8:%.*]] = trunc <8 x i16> [[WIDE_LOAD]] to <8 x i8>
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i8> poison, i8 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[L9:%.*]] = mul <8 x i8> [[TMP2]], [[L8]]
+; CHECK-NEXT: [[L13:%.*]] = getelementptr inbounds i8, i8* [[D:%.*]], i32 [[INDEX]]
+; CHECK-NEXT: [[L14:%.*]] = bitcast i8* [[L13]] to <8 x i8>*
+; CHECK-NEXT: store <8 x i8> [[L9]], <8 x i8>* [[L14]], align 1
+; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 8
+; CHECK-NEXT: [[L15:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT: br i1 [[L15]], label [[EXIT:%.*]], label [[VECTOR_BODY]]
+; CHECK: exit:
+; CHECK-NEXT: ret i8 0
+;
+entry:
+ %n.vec = and i32 %n, -8
+ %l0 = trunc i16 %x to i8
+ %l1 = insertelement <8 x i8> poison, i8 %l0, i32 0
+ %broadcast.splat26 = shufflevector <8 x i8> %l1, <8 x i8> undef, <8 x i32> zeroinitializer
+ %l2 = sub <8 x i8> zeroinitializer, %broadcast.splat26
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %entry
+ %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
+ %l6 = getelementptr inbounds i16, i16* %s1, i32 %index
+ %l7 = bitcast i16* %l6 to <8 x i16>*
+ %wide.load = load <8 x i16>, <8 x i16>* %l7, align 2
+ %l8 = trunc <8 x i16> %wide.load to <8 x i8>
+ %l9 = mul <8 x i8> %broadcast.splat26, %l8
+ %l13 = getelementptr inbounds i8, i8* %d, i32 %index
+ %l14 = bitcast i8* %l13 to <8 x i8>*
+ store <8 x i8> %l9, <8 x i8>* %l14, align 1
+ %index.next = add i32 %index, 8
+ %l15 = icmp eq i32 %index.next, %n.vec
+ br i1 %l15, label %exit, label %vector.body
+
+exit: ; preds = %vector.body
+ ret i8 0
+}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
new file mode 100644
index 000000000000..88967ac1ef7c
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/gather-scatter-opt-inseltpoison.ll
@@ -0,0 +1,113 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.a = type { i32, i32 }
+ at c = external dso_local global %struct.a, align 4
+ at glob_array = internal unnamed_addr constant [16 x i32] [i32 1, i32 1, i32 2, i32 3, i32 5, i32 8, i32 13, i32 21, i32 34, i32 55, i32 89, i32 144, i32 233, i32 377, i32 610, i32 987], align 16
+
+define <4 x i32> @splat_base(i32* %base, <4 x i64> %index) {
+; CHECK-LABEL: @splat_base(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <4 x i64> [[INDEX:%.*]]
+; CHECK-NEXT: [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP1]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <4 x i32*> poison, i32* %base, i32 0
+ %broadcast.splat = shufflevector <4 x i32*> %broadcast.splatinsert, <4 x i32*> undef, <4 x i32> zeroinitializer
+ %gep = getelementptr i32, <4 x i32*> %broadcast.splat, <4 x i64> %index
+ %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @splat_struct(%struct.a* %base) {
+; CHECK-LABEL: @splat_struct(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [[STRUCT_A:%.*]], %struct.a* [[BASE:%.*]], i64 0, i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %gep = getelementptr %struct.a, %struct.a* %base, <4 x i64> zeroinitializer, i32 1
+ %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @scalar_index(i32* %base, i64 %index) {
+; CHECK-LABEL: @scalar_index(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[INDEX:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <4 x i32*> poison, i32* %base, i32 0
+ %broadcast.splat = shufflevector <4 x i32*> %broadcast.splatinsert, <4 x i32*> undef, <4 x i32> zeroinitializer
+ %gep = getelementptr i32, <4 x i32*> %broadcast.splat, i64 %index
+ %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @splat_index(i32* %base, i64 %index) {
+; CHECK-LABEL: @splat_index(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[INDEX:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, i32* [[TMP1]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP2]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %broadcast.splatinsert = insertelement <4 x i64> poison, i64 %index, i32 0
+ %broadcast.splat = shufflevector <4 x i64> %broadcast.splatinsert, <4 x i64> undef, <4 x i32> zeroinitializer
+ %gep = getelementptr i32, i32* %base, <4 x i64> %broadcast.splat
+ %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %gep, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @test_global_array(<4 x i64> %indxs) {
+; CHECK-LABEL: @test_global_array(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @glob_array, i64 0, i64 0), <4 x i64> [[INDXS:%.*]]
+; CHECK-NEXT: [[G:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP1]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[G]]
+;
+ %p = getelementptr inbounds [16 x i32], [16 x i32]* @glob_array, i64 0, <4 x i64> %indxs
+ %g = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %p, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %g
+}
+
+define <4 x i32> @global_struct_splat() {
+; CHECK-LABEL: @global_struct_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> <i32* getelementptr inbounds (%struct.a, %struct.a* @c, i64 0, i32 1), i32* getelementptr inbounds (%struct.a, %struct.a* @c, i64 0, i32 1), i32* getelementptr inbounds (%struct.a, %struct.a* @c, i64 0, i32 1), i32* getelementptr inbounds (%struct.a, %struct.a* @c, i64 0, i32 1)>, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = insertelement <4 x %struct.a*> poison, %struct.a* @c, i32 0
+ %2 = shufflevector <4 x %struct.a*> %1, <4 x %struct.a*> undef, <4 x i32> zeroinitializer
+ %3 = getelementptr %struct.a, <4 x %struct.a*> %2, <4 x i64> zeroinitializer, i32 1
+ %4 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %3, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef)
+ ret <4 x i32> %4
+}
+
+define <4 x i32> @splat_ptr_gather(i32* %ptr, <4 x i1> %mask, <4 x i32> %passthru) {
+; CHECK-LABEL: @splat_ptr_gather(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], <4 x i64> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP1]], i32 4, <4 x i1> [[MASK:%.*]], <4 x i32> [[PASSTHRU:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = insertelement <4 x i32*> poison, i32* %ptr, i32 0
+ %2 = shufflevector <4 x i32*> %1, <4 x i32*> undef, <4 x i32> zeroinitializer
+ %3 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %2, i32 4, <4 x i1> %mask, <4 x i32> %passthru)
+ ret <4 x i32> %3
+}
+
+define void @splat_ptr_scatter(i32* %ptr, <4 x i1> %mask, <4 x i32> %val) {
+; CHECK-LABEL: @splat_ptr_scatter(
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, i32* [[PTR:%.*]], <4 x i64> zeroinitializer
+; CHECK-NEXT: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> [[VAL:%.*]], <4 x i32*> [[TMP1]], i32 4, <4 x i1> [[MASK:%.*]])
+; CHECK-NEXT: ret void
+;
+ %1 = insertelement <4 x i32*> poison, i32* %ptr, i32 0
+ %2 = shufflevector <4 x i32*> %1, <4 x i32*> undef, <4 x i32> zeroinitializer
+ call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %val, <4 x i32*> %2, i32 4, <4 x i1> %mask)
+ ret void
+}
+
+declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*>, i32, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32>, <4 x i32*>, i32, <4 x i1>)
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll
new file mode 100644
index 000000000000..c208ac6a9fbe
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-inseltpoison.ll
@@ -0,0 +1,321 @@
+; RUN: opt -S -codegenprepare < %s | FileCheck %s
+
+target datalayout =
+"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at x = external global [1 x [2 x <4 x float>]]
+
+; Can we sink single addressing mode computation to use?
+define void @test1(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test1
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+ %v = load i32, i32* %casted, align 4
+ br label %fallthrough
+
+fallthrough:
+ ret void
+}
+
+declare void @foo(i32)
+
+; Make sure sinking two copies of addressing mode into
diff erent blocks works
+define void @test2(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test2
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %next, label %fallthrough
+
+next:
+; CHECK-LABEL: next:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v2 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v2)
+ br label %fallthrough
+
+fallthrough:
+ ret void
+}
+
+; If we have two loads in the same block, only need one copy of addressing mode
+; - instruction selection will duplicate if needed
+define void @test3(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test3
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+ %v2 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v2)
+ br label %fallthrough
+
+fallthrough:
+ ret void
+}
+
+; Can we still sink addressing mode if there's a cold use of the
+; address itself?
+define void @test4(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test4
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+ ret void
+
+rare.1:
+; CHECK-LABEL: rare.1:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ call void @slowpath(i32 %v1, i32* %casted) cold
+ br label %fallthrough
+}
+
+; Negative test - don't want to duplicate addressing into hot path
+define void @test5(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test5
+entry:
+; CHECK: %addr = getelementptr inbounds
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+ ret void
+
+rare.1:
+ call void @slowpath(i32 %v1, i32* %casted) ;; NOT COLD
+ br label %fallthrough
+}
+
+; Negative test - opt for size
+define void @test6(i1 %cond, i64* %base) minsize {
+; CHECK-LABEL: @test6
+entry:
+; CHECK: %addr = getelementptr
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+ ret void
+
+rare.1:
+ call void @slowpath(i32 %v1, i32* %casted) cold
+ br label %fallthrough
+}
+
+; Negative test - opt for size
+define void @test6_pgso(i1 %cond, i64* %base) !prof !14 {
+; CHECK-LABEL: @test6
+entry:
+; CHECK: %addr = getelementptr
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK-NOT: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %rare.1, label %fallthrough
+
+fallthrough:
+ ret void
+
+rare.1:
+ call void @slowpath(i32 %v1, i32* %casted) cold
+ br label %fallthrough
+}
+
+; Make sure sinking two copies of addressing mode into
diff erent blocks works
+; when there are cold paths for each.
+define void @test7(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test7
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br i1 %cond, label %if.then, label %fallthrough
+
+if.then:
+; CHECK-LABEL: if.then:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v1 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v1)
+ %cmp = icmp eq i32 %v1, 0
+ br i1 %cmp, label %rare.1, label %next
+
+next:
+; CHECK-LABEL: next:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %v2 = load i32, i32* %casted, align 4
+ call void @foo(i32 %v2)
+ %cmp2 = icmp eq i32 %v2, 0
+ br i1 %cmp2, label %rare.1, label %fallthrough
+
+fallthrough:
+ ret void
+
+rare.1:
+; CHECK-LABEL: rare.1:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ call void @slowpath(i32 %v1, i32* %casted) cold
+ br label %next
+
+rare.2:
+; CHECK-LABEL: rare.2:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ call void @slowpath(i32 %v2, i32* %casted) cold
+ br label %fallthrough
+}
+
+declare void @slowpath(i32, i32*)
+
+; Make sure we don't end up in an infinite loop after we fail to sink.
+; CHECK-LABEL: define void @test8
+; CHECK: %ptr = getelementptr i8, i8* %aFOO_load_ptr2int_2void, i32 undef
+define void @test8() {
+allocas:
+ %aFOO_load = load float*, float** undef
+ %aFOO_load_ptr2int = ptrtoint float* %aFOO_load to i64
+ %aFOO_load_ptr2int_broadcast_init = insertelement <4 x i64> poison, i64 %aFOO_load_ptr2int, i32 0
+ %aFOO_load_ptr2int_2void = inttoptr i64 %aFOO_load_ptr2int to i8*
+ %ptr = getelementptr i8, i8* %aFOO_load_ptr2int_2void, i32 undef
+ br label %load.i145
+
+load.i145:
+ %ptr.i143 = bitcast i8* %ptr to <4 x float>*
+ %valall.i144 = load <4 x float>, <4 x float>* %ptr.i143, align 4
+ %x_offset = getelementptr [1 x [2 x <4 x float>]], [1 x [2 x <4 x float>]]* @x, i32 0, i64 0
+ br label %pl_loop.i.i122
+
+pl_loop.i.i122:
+ br label %pl_loop.i.i122
+}
+
+; Make sure we can sink address computation even
+; if there is a cycle in phi nodes.
+define void @test9(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test9
+entry:
+ %addr = getelementptr inbounds i64, i64* %base, i64 5
+ %casted = bitcast i64* %addr to i32*
+ br label %header
+
+header:
+ %iv = phi i32 [0, %entry], [%iv.inc, %backedge]
+ %casted.loop = phi i32* [%casted, %entry], [%casted.merged, %backedge]
+ br i1 %cond, label %if.then, label %backedge
+
+if.then:
+ call void @foo(i32 %iv)
+ %addr.1 = getelementptr inbounds i64, i64* %base, i64 5
+ %casted.1 = bitcast i64* %addr.1 to i32*
+ br label %backedge
+
+backedge:
+; CHECK-LABEL: backedge:
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+ %casted.merged = phi i32* [%casted.loop, %header], [%casted.1, %if.then]
+ %v = load i32, i32* %casted.merged, align 4
+ call void @foo(i32 %v)
+ %iv.inc = add i32 %iv, 1
+ %cmp = icmp slt i32 %iv.inc, 1000
+ br i1 %cmp, label %header, label %exit
+
+exit:
+ ret void
+}
+
+; Make sure we can eliminate a select when both arguments perform equivalent
+; address computation.
+define void @test10(i1 %cond, i64* %base) {
+; CHECK-LABEL: @test10
+; CHECK: getelementptr inbounds i8, {{.+}} 40
+; CHECK-NOT: select
+entry:
+ %gep1 = getelementptr inbounds i64, i64* %base, i64 5
+ %gep1.casted = bitcast i64* %gep1 to i32*
+ %base.casted = bitcast i64* %base to i32*
+ %gep2 = getelementptr inbounds i32, i32* %base.casted, i64 10
+ %casted.merged = select i1 %cond, i32* %gep1.casted, i32* %gep2
+ %v = load i32, i32* %casted.merged, align 4
+ call void @foo(i32 %v)
+ ret void
+}
+
+; Found by fuzzer, getSExtValue of > 64 bit constant
+define void @i96_mul(i1* %base, i96 %offset) {
+BB:
+ ;; RHS = 0x7FFFFFFFFFFFFFFFFFFFFFFF
+ %B84 = mul i96 %offset, 39614081257132168796771975167
+ %G23 = getelementptr i1, i1* %base, i96 %B84
+ store i1 false, i1* %G23
+ ret void
+}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 0}
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll
new file mode 100644
index 000000000000..1d26beeb236e
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/vec-shift-inseltpoison.ll
@@ -0,0 +1,406 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S < %s | FileCheck %s --check-prefixes=ALL,AVX1
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2 -S < %s | FileCheck %s --check-prefixes=ALL,AVX2
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx512bw -S < %s | FileCheck %s --check-prefixes=ALL,AVX512BW
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx,+xop -S < %s | FileCheck %s --check-prefixes=ALL,XOP
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx2,+xop -S < %s | FileCheck %s --check-prefixes=ALL,XOP
+; RUN: opt -codegenprepare -mtriple=x86_64-- -mattr=+avx -S -enable-debugify < %s 2>&1 | FileCheck %s -check-prefix=DEBUG
+
+define <4 x i32> @vector_variable_shift_right_v4i32(<4 x i1> %cond, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) {
+; AVX1-LABEL: @vector_variable_shift_right_v4i32(
+; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX1-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SPLAT1]]
+; AVX1-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[Z]], [[SPLAT2]]
+; AVX1-NEXT: [[TMP3:%.*]] = select <4 x i1> [[COND]], <4 x i32> [[TMP1]], <4 x i32> [[TMP2]]
+; AVX1-NEXT: ret <4 x i32> [[TMP3]]
+;
+; AVX2-LABEL: @vector_variable_shift_right_v4i32(
+; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX2-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]
+; AVX2-NEXT: ret <4 x i32> [[SH]]
+;
+; AVX512BW-LABEL: @vector_variable_shift_right_v4i32(
+; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX512BW-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]
+; AVX512BW-NEXT: ret <4 x i32> [[SH]]
+;
+; XOP-LABEL: @vector_variable_shift_right_v4i32(
+; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; XOP-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND:%.*]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; XOP-NEXT: [[SH:%.*]] = lshr <4 x i32> [[Z:%.*]], [[SEL]]
+; XOP-NEXT: ret <4 x i32> [[SH]]
+;
+ %splat1 = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+ %splat2 = shufflevector <4 x i32> %y, <4 x i32> undef, <4 x i32> zeroinitializer
+ %sel = select <4 x i1> %cond, <4 x i32> %splat1, <4 x i32> %splat2
+ %sh = lshr <4 x i32> %z, %sel
+ ret <4 x i32> %sh
+}
+
+define <16 x i16> @vector_variable_shift_right_v16i16(<16 x i1> %cond, <16 x i16> %x, <16 x i16> %y, <16 x i16> %z) {
+; AVX1-LABEL: @vector_variable_shift_right_v16i16(
+; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]
+; AVX1-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SPLAT1]]
+; AVX1-NEXT: [[TMP2:%.*]] = lshr <16 x i16> [[Z]], [[SPLAT2]]
+; AVX1-NEXT: [[TMP3:%.*]] = select <16 x i1> [[COND]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]
+; AVX1-NEXT: ret <16 x i16> [[TMP3]]
+;
+; AVX2-LABEL: @vector_variable_shift_right_v16i16(
+; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX2-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]
+; AVX2-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SPLAT1]]
+; AVX2-NEXT: [[TMP2:%.*]] = lshr <16 x i16> [[Z]], [[SPLAT2]]
+; AVX2-NEXT: [[TMP3:%.*]] = select <16 x i1> [[COND]], <16 x i16> [[TMP1]], <16 x i16> [[TMP2]]
+; AVX2-NEXT: ret <16 x i16> [[TMP3]]
+;
+; AVX512BW-LABEL: @vector_variable_shift_right_v16i16(
+; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]
+; AVX512BW-NEXT: [[SH:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SEL]]
+; AVX512BW-NEXT: ret <16 x i16> [[SH]]
+;
+; XOP-LABEL: @vector_variable_shift_right_v16i16(
+; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <16 x i16> [[X:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <16 x i16> [[Y:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; XOP-NEXT: [[SEL:%.*]] = select <16 x i1> [[COND:%.*]], <16 x i16> [[SPLAT1]], <16 x i16> [[SPLAT2]]
+; XOP-NEXT: [[SH:%.*]] = lshr <16 x i16> [[Z:%.*]], [[SEL]]
+; XOP-NEXT: ret <16 x i16> [[SH]]
+;
+ %splat1 = shufflevector <16 x i16> %x, <16 x i16> undef, <16 x i32> zeroinitializer
+ %splat2 = shufflevector <16 x i16> %y, <16 x i16> undef, <16 x i32> zeroinitializer
+ %sel = select <16 x i1> %cond, <16 x i16> %splat1, <16 x i16> %splat2
+ %sh = lshr <16 x i16> %z, %sel
+ ret <16 x i16> %sh
+}
+
+define <32 x i8> @vector_variable_shift_right_v32i8(<32 x i1> %cond, <32 x i8> %x, <32 x i8> %y, <32 x i8> %z) {
+; ALL-LABEL: @vector_variable_shift_right_v32i8(
+; ALL-NEXT: [[SPLAT1:%.*]] = shufflevector <32 x i8> [[X:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; ALL-NEXT: [[SPLAT2:%.*]] = shufflevector <32 x i8> [[Y:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; ALL-NEXT: [[SEL:%.*]] = select <32 x i1> [[COND:%.*]], <32 x i8> [[SPLAT1]], <32 x i8> [[SPLAT2]]
+; ALL-NEXT: [[SH:%.*]] = lshr <32 x i8> [[Z:%.*]], [[SEL]]
+; ALL-NEXT: ret <32 x i8> [[SH]]
+;
+ %splat1 = shufflevector <32 x i8> %x, <32 x i8> undef, <32 x i32> zeroinitializer
+ %splat2 = shufflevector <32 x i8> %y, <32 x i8> undef, <32 x i32> zeroinitializer
+ %sel = select <32 x i1> %cond, <32 x i8> %splat1, <32 x i8> %splat2
+ %sh = lshr <32 x i8> %z, %sel
+ ret <32 x i8> %sh
+}
+
+; PR37428 - https://bugs.llvm.org/show_bug.cgi?id=37428
+
+define void @vector_variable_shift_left_loop(i32* nocapture %arr, i8* nocapture readonly %control, i32 %count, i32 %amt0, i32 %amt1, i32 %x) {
+; AVX1-LABEL: @vector_variable_shift_left_loop(
+; AVX1-NEXT: entry:
+; AVX1-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0
+; AVX1-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64
+; AVX1-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]
+; AVX1: vector.ph:
+; AVX1-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292
+; AVX1-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> poison, i32 [[AMT0:%.*]], i32 0
+; AVX1-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[AMT1:%.*]], i32 0
+; AVX1-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; AVX1-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: br label [[VECTOR_BODY:%.*]]
+; AVX1: vector.body:
+; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX1-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX1-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX1-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX1-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[TMP5:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP4]]
+; AVX1-NEXT: [[TMP6:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX1-NEXT: [[TMP7:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP6]]
+; AVX1-NEXT: [[TMP8:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP5]], <4 x i32> [[TMP7]]
+; AVX1-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX1-NEXT: [[TMP10:%.*]] = bitcast i32* [[TMP9]] to <4 x i32>*
+; AVX1-NEXT: store <4 x i32> [[TMP8]], <4 x i32>* [[TMP10]], align 4
+; AVX1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; AVX1-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX1-NEXT: br i1 [[TMP11]], label [[EXIT]], label [[VECTOR_BODY]]
+; AVX1: exit:
+; AVX1-NEXT: ret void
+;
+; AVX2-LABEL: @vector_variable_shift_left_loop(
+; AVX2-NEXT: entry:
+; AVX2-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0
+; AVX2-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64
+; AVX2-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]
+; AVX2: vector.ph:
+; AVX2-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292
+; AVX2-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> poison, i32 [[AMT0:%.*]], i32 0
+; AVX2-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[AMT1:%.*]], i32 0
+; AVX2-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; AVX2-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: br label [[VECTOR_BODY:%.*]]
+; AVX2: vector.body:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX2-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX2-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX2-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; AVX2-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* [[TMP6]], align 4
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; AVX2-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX2-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]
+; AVX2: exit:
+; AVX2-NEXT: ret void
+;
+; AVX512BW-LABEL: @vector_variable_shift_left_loop(
+; AVX512BW-NEXT: entry:
+; AVX512BW-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0
+; AVX512BW-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64
+; AVX512BW-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]
+; AVX512BW: vector.ph:
+; AVX512BW-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292
+; AVX512BW-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> poison, i32 [[AMT0:%.*]], i32 0
+; AVX512BW-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[AMT1:%.*]], i32 0
+; AVX512BW-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; AVX512BW-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: br label [[VECTOR_BODY:%.*]]
+; AVX512BW: vector.body:
+; AVX512BW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX512BW-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX512BW-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; AVX512BW-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX512BW-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX512BW-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; AVX512BW-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]
+; AVX512BW-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX512BW-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; AVX512BW-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* [[TMP6]], align 4
+; AVX512BW-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; AVX512BW-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512BW-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]
+; AVX512BW: exit:
+; AVX512BW-NEXT: ret void
+;
+; XOP-LABEL: @vector_variable_shift_left_loop(
+; XOP-NEXT: entry:
+; XOP-NEXT: [[CMP16:%.*]] = icmp sgt i32 [[COUNT:%.*]], 0
+; XOP-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[COUNT]] to i64
+; XOP-NEXT: br i1 [[CMP16]], label [[VECTOR_PH:%.*]], label [[EXIT:%.*]]
+; XOP: vector.ph:
+; XOP-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967292
+; XOP-NEXT: [[SPLATINSERT18:%.*]] = insertelement <4 x i32> poison, i32 [[AMT0:%.*]], i32 0
+; XOP-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[SPLATINSERT18]], <4 x i32> undef, <4 x i32> zeroinitializer
+; XOP-NEXT: [[SPLATINSERT20:%.*]] = insertelement <4 x i32> poison, i32 [[AMT1:%.*]], i32 0
+; XOP-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[SPLATINSERT20]], <4 x i32> undef, <4 x i32> zeroinitializer
+; XOP-NEXT: [[SPLATINSERT22:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; XOP-NEXT: [[SPLAT3:%.*]] = shufflevector <4 x i32> [[SPLATINSERT22]], <4 x i32> undef, <4 x i32> zeroinitializer
+; XOP-NEXT: br label [[VECTOR_BODY:%.*]]
+; XOP: vector.body:
+; XOP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; XOP-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; XOP-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to <4 x i8>*
+; XOP-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; XOP-NEXT: [[TMP2:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer
+; XOP-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[SPLAT1]], <4 x i32> [[SPLAT2]]
+; XOP-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[SPLAT3]], [[TMP3]]
+; XOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; XOP-NEXT: [[TMP6:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; XOP-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* [[TMP6]], align 4
+; XOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; XOP-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; XOP-NEXT: br i1 [[TMP7]], label [[EXIT]], label [[VECTOR_BODY]]
+; XOP: exit:
+; XOP-NEXT: ret void
+;
+entry:
+ %cmp16 = icmp sgt i32 %count, 0
+ %wide.trip.count = zext i32 %count to i64
+ br i1 %cmp16, label %vector.ph, label %exit
+
+vector.ph:
+ %n.vec = and i64 %wide.trip.count, 4294967292
+ %splatinsert18 = insertelement <4 x i32> poison, i32 %amt0, i32 0
+ %splat1 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer
+ %splatinsert20 = insertelement <4 x i32> poison, i32 %amt1, i32 0
+ %splat2 = shufflevector <4 x i32> %splatinsert20, <4 x i32> undef, <4 x i32> zeroinitializer
+ %splatinsert22 = insertelement <4 x i32> poison, i32 %x, i32 0
+ %splat3 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %0 = getelementptr inbounds i8, i8* %control, i64 %index
+ %1 = bitcast i8* %0 to <4 x i8>*
+ %wide.load = load <4 x i8>, <4 x i8>* %1, align 1
+ %2 = icmp eq <4 x i8> %wide.load, zeroinitializer
+ %3 = select <4 x i1> %2, <4 x i32> %splat1, <4 x i32> %splat2
+ %4 = shl <4 x i32> %splat3, %3
+ %5 = getelementptr inbounds i32, i32* %arr, i64 %index
+ %6 = bitcast i32* %5 to <4 x i32>*
+ store <4 x i32> %4, <4 x i32>* %6, align 4
+ %index.next = add i64 %index, 4
+ %7 = icmp eq i64 %index.next, %n.vec
+ br i1 %7, label %exit, label %vector.body
+
+exit:
+ ret void
+}
+
+; PR37426 - https://bugs.llvm.org/show_bug.cgi?id=37426
+; If we don't have real vector shift instructions (AVX1), convert the funnel
+; shift into 2 funnel shifts and sink the splat shuffles into the loop.
+
+define void @fancierRotate2(i32* %arr, i8* %control, i32 %rot0, i32 %rot1) {
+; AVX1-LABEL: @fancierRotate2(
+; AVX1-NEXT: entry:
+; AVX1-NEXT: [[I0:%.*]] = insertelement <8 x i32> poison, i32 [[ROT0:%.*]], i32 0
+; AVX1-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX1-NEXT: [[I1:%.*]] = insertelement <8 x i32> poison, i32 [[ROT1:%.*]], i32 0
+; AVX1-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX1-NEXT: br label [[LOOP:%.*]]
+; AVX1: loop:
+; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; AVX1-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX1-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to <8 x i8>*
+; AVX1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[T1]], align 1
+; AVX1-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX1-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]
+; AVX1-NEXT: [[T4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX1-NEXT: [[T5:%.*]] = bitcast i32* [[T4]] to <8 x i32>*
+; AVX1-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[T5]], align 4
+; AVX1-NEXT: [[TMP0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX1-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[TMP0]])
+; AVX1-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX1-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[TMP2]])
+; AVX1-NEXT: [[TMP4:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[TMP1]], <8 x i32> [[TMP3]]
+; AVX1-NEXT: store <8 x i32> [[TMP4]], <8 x i32>* [[T5]], align 4
+; AVX1-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; AVX1-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX1-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]
+; AVX1: exit:
+; AVX1-NEXT: ret void
+;
+; AVX2-LABEL: @fancierRotate2(
+; AVX2-NEXT: entry:
+; AVX2-NEXT: [[I0:%.*]] = insertelement <8 x i32> poison, i32 [[ROT0:%.*]], i32 0
+; AVX2-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX2-NEXT: [[I1:%.*]] = insertelement <8 x i32> poison, i32 [[ROT1:%.*]], i32 0
+; AVX2-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX2-NEXT: br label [[LOOP:%.*]]
+; AVX2: loop:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; AVX2-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX2-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to <8 x i8>*
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[T1]], align 1
+; AVX2-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX2-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]
+; AVX2-NEXT: [[T4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX2-NEXT: [[T5:%.*]] = bitcast i32* [[T4]] to <8 x i32>*
+; AVX2-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[T5]], align 4
+; AVX2-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])
+; AVX2-NEXT: store <8 x i32> [[ROT]], <8 x i32>* [[T5]], align 4
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; AVX2-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]
+; AVX2: exit:
+; AVX2-NEXT: ret void
+;
+; AVX512BW-LABEL: @fancierRotate2(
+; AVX512BW-NEXT: entry:
+; AVX512BW-NEXT: [[I0:%.*]] = insertelement <8 x i32> poison, i32 [[ROT0:%.*]], i32 0
+; AVX512BW-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX512BW-NEXT: [[I1:%.*]] = insertelement <8 x i32> poison, i32 [[ROT1:%.*]], i32 0
+; AVX512BW-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; AVX512BW-NEXT: br label [[LOOP:%.*]]
+; AVX512BW: loop:
+; AVX512BW-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; AVX512BW-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; AVX512BW-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to <8 x i8>*
+; AVX512BW-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[T1]], align 1
+; AVX512BW-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; AVX512BW-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]
+; AVX512BW-NEXT: [[T4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; AVX512BW-NEXT: [[T5:%.*]] = bitcast i32* [[T4]] to <8 x i32>*
+; AVX512BW-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[T5]], align 4
+; AVX512BW-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])
+; AVX512BW-NEXT: store <8 x i32> [[ROT]], <8 x i32>* [[T5]], align 4
+; AVX512BW-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; AVX512BW-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512BW-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]
+; AVX512BW: exit:
+; AVX512BW-NEXT: ret void
+;
+; XOP-LABEL: @fancierRotate2(
+; XOP-NEXT: entry:
+; XOP-NEXT: [[I0:%.*]] = insertelement <8 x i32> poison, i32 [[ROT0:%.*]], i32 0
+; XOP-NEXT: [[S0:%.*]] = shufflevector <8 x i32> [[I0]], <8 x i32> undef, <8 x i32> zeroinitializer
+; XOP-NEXT: [[I1:%.*]] = insertelement <8 x i32> poison, i32 [[ROT1:%.*]], i32 0
+; XOP-NEXT: [[S1:%.*]] = shufflevector <8 x i32> [[I1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; XOP-NEXT: br label [[LOOP:%.*]]
+; XOP: loop:
+; XOP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[LOOP]] ]
+; XOP-NEXT: [[T0:%.*]] = getelementptr inbounds i8, i8* [[CONTROL:%.*]], i64 [[INDEX]]
+; XOP-NEXT: [[T1:%.*]] = bitcast i8* [[T0]] to <8 x i8>*
+; XOP-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, <8 x i8>* [[T1]], align 1
+; XOP-NEXT: [[T2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer
+; XOP-NEXT: [[SHAMT:%.*]] = select <8 x i1> [[T2]], <8 x i32> [[S0]], <8 x i32> [[S1]]
+; XOP-NEXT: [[T4:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; XOP-NEXT: [[T5:%.*]] = bitcast i32* [[T4]] to <8 x i32>*
+; XOP-NEXT: [[WIDE_LOAD21:%.*]] = load <8 x i32>, <8 x i32>* [[T5]], align 4
+; XOP-NEXT: [[ROT:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD21]], <8 x i32> [[WIDE_LOAD21]], <8 x i32> [[SHAMT]])
+; XOP-NEXT: store <8 x i32> [[ROT]], <8 x i32>* [[T5]], align 4
+; XOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; XOP-NEXT: [[T7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; XOP-NEXT: br i1 [[T7]], label [[EXIT:%.*]], label [[LOOP]]
+; XOP: exit:
+; XOP-NEXT: ret void
+;
+entry:
+ %i0 = insertelement <8 x i32> poison, i32 %rot0, i32 0
+ %s0 = shufflevector <8 x i32> %i0, <8 x i32> undef, <8 x i32> zeroinitializer
+ %i1 = insertelement <8 x i32> poison, i32 %rot1, i32 0
+ %s1 = shufflevector <8 x i32> %i1, <8 x i32> undef, <8 x i32> zeroinitializer
+ br label %loop
+
+loop:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %loop ]
+ %t0 = getelementptr inbounds i8, i8* %control, i64 %index
+ %t1 = bitcast i8* %t0 to <8 x i8>*
+ %wide.load = load <8 x i8>, <8 x i8>* %t1, align 1
+ %t2 = icmp eq <8 x i8> %wide.load, zeroinitializer
+ %shamt = select <8 x i1> %t2, <8 x i32> %s0, <8 x i32> %s1
+ %t4 = getelementptr inbounds i32, i32* %arr, i64 %index
+ %t5 = bitcast i32* %t4 to <8 x i32>*
+ %wide.load21 = load <8 x i32>, <8 x i32>* %t5, align 4
+ %rot = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %wide.load21, <8 x i32> %wide.load21, <8 x i32> %shamt)
+ store <8 x i32> %rot, <8 x i32>* %t5, align 4
+ %index.next = add i64 %index, 8
+ %t7 = icmp eq i64 %index.next, 1024
+ br i1 %t7, label %exit, label %loop
+
+exit:
+ ret void
+}
+
+declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>) #1
+
+; Check that every instruction inserted by -codegenprepare has a debug location.
+; DEBUG: CheckModuleDebugify: PASS
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
new file mode 100644
index 000000000000..4e9f09fa32bc
--- /dev/null
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/x86-shuffle-sink-inseltpoison.ll
@@ -0,0 +1,257 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -codegenprepare -mcpu=corei7 %s | FileCheck %s --check-prefixes=CHECK,CHECK-SSE2
+; RUN: opt -S -codegenprepare -mcpu=bdver2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-XOP
+; RUN: opt -S -codegenprepare -mcpu=core-avx2 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX2
+; RUN: opt -S -codegenprepare -mcpu=skylake-avx512 %s | FileCheck %s --check-prefixes=CHECK,CHECK-AVX,CHECK-AVX512BW
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin10.9.0"
+
+define <16 x i8> @test_8bit(<16 x i8> %lhs, <16 x i8> %tmp, i1 %tst) {
+; CHECK-LABEL: @test_8bit(
+; CHECK-NEXT: [[MASK:%.*]] = shufflevector <16 x i8> [[TMP:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK: if_true:
+; CHECK-NEXT: ret <16 x i8> [[MASK]]
+; CHECK: if_false:
+; CHECK-NEXT: [[RES:%.*]] = shl <16 x i8> [[LHS:%.*]], [[MASK]]
+; CHECK-NEXT: ret <16 x i8> [[RES]]
+;
+ %mask = shufflevector <16 x i8> %tmp, <16 x i8> undef, <16 x i32> zeroinitializer
+ br i1 %tst, label %if_true, label %if_false
+
+if_true:
+ ret <16 x i8> %mask
+
+if_false:
+ %res = shl <16 x i8> %lhs, %mask
+ ret <16 x i8> %res
+}
+
+define <8 x i16> @test_16bit(<8 x i16> %lhs, <8 x i16> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_16bit(
+; CHECK-SSE2-NEXT: [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-SSE2-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2: if_true:
+; CHECK-SSE2-NEXT: ret <8 x i16> [[MASK]]
+; CHECK-SSE2: if_false:
+; CHECK-SSE2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[TMP]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-SSE2-NEXT: [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT: ret <8 x i16> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_16bit(
+; CHECK-XOP-NEXT: [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-XOP-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP: if_true:
+; CHECK-XOP-NEXT: ret <8 x i16> [[MASK]]
+; CHECK-XOP: if_false:
+; CHECK-XOP-NEXT: [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT: ret <8 x i16> [[RES]]
+;
+; CHECK-AVX2-LABEL: @test_16bit(
+; CHECK-AVX2-NEXT: [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX2-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX2: if_true:
+; CHECK-AVX2-NEXT: ret <8 x i16> [[MASK]]
+; CHECK-AVX2: if_false:
+; CHECK-AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[TMP]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX2-NEXT: [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[TMP1]]
+; CHECK-AVX2-NEXT: ret <8 x i16> [[RES]]
+;
+; CHECK-AVX512BW-LABEL: @test_16bit(
+; CHECK-AVX512BW-NEXT: [[MASK:%.*]] = shufflevector <8 x i16> [[TMP:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-AVX512BW-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX512BW: if_true:
+; CHECK-AVX512BW-NEXT: ret <8 x i16> [[MASK]]
+; CHECK-AVX512BW: if_false:
+; CHECK-AVX512BW-NEXT: [[RES:%.*]] = shl <8 x i16> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX512BW-NEXT: ret <8 x i16> [[RES]]
+;
+ %mask = shufflevector <8 x i16> %tmp, <8 x i16> undef, <8 x i32> zeroinitializer
+ br i1 %tst, label %if_true, label %if_false
+
+if_true:
+ ret <8 x i16> %mask
+
+if_false:
+ %res = shl <8 x i16> %lhs, %mask
+ ret <8 x i16> %res
+}
+
+define <4 x i32> @test_notsplat(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
+; CHECK-LABEL: @test_notsplat(
+; CHECK-NEXT: [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+; CHECK-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK: if_true:
+; CHECK-NEXT: ret <4 x i32> [[MASK]]
+; CHECK: if_false:
+; CHECK-NEXT: [[RES:%.*]] = shl <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-NEXT: ret <4 x i32> [[RES]]
+;
+ %mask = shufflevector <4 x i32> %tmp, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 0>
+ br i1 %tst, label %if_true, label %if_false
+
+if_true:
+ ret <4 x i32> %mask
+
+if_false:
+ %res = shl <4 x i32> %lhs, %mask
+ ret <4 x i32> %res
+}
+
+define <4 x i32> @test_32bit(<4 x i32> %lhs, <4 x i32> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_32bit(
+; CHECK-SSE2-NEXT: [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2: if_true:
+; CHECK-SSE2-NEXT: ret <4 x i32> [[MASK]]
+; CHECK-SSE2: if_false:
+; CHECK-SSE2-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-SSE2-NEXT: [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT: ret <4 x i32> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_32bit(
+; CHECK-XOP-NEXT: [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-XOP-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP: if_true:
+; CHECK-XOP-NEXT: ret <4 x i32> [[MASK]]
+; CHECK-XOP: if_false:
+; CHECK-XOP-NEXT: [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT: ret <4 x i32> [[RES]]
+;
+; CHECK-AVX-LABEL: @test_32bit(
+; CHECK-AVX-NEXT: [[MASK:%.*]] = shufflevector <4 x i32> [[TMP:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-AVX-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX: if_true:
+; CHECK-AVX-NEXT: ret <4 x i32> [[MASK]]
+; CHECK-AVX: if_false:
+; CHECK-AVX-NEXT: [[RES:%.*]] = ashr <4 x i32> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX-NEXT: ret <4 x i32> [[RES]]
+;
+ %mask = shufflevector <4 x i32> %tmp, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+ br i1 %tst, label %if_true, label %if_false
+
+if_true:
+ ret <4 x i32> %mask
+
+if_false:
+ %res = ashr <4 x i32> %lhs, %mask
+ ret <4 x i32> %res
+}
+
+define <2 x i64> @test_64bit(<2 x i64> %lhs, <2 x i64> %tmp, i1 %tst) {
+; CHECK-SSE2-LABEL: @test_64bit(
+; CHECK-SSE2-NEXT: [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-SSE2-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-SSE2: if_true:
+; CHECK-SSE2-NEXT: ret <2 x i64> [[MASK]]
+; CHECK-SSE2: if_false:
+; CHECK-SSE2-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-SSE2-NEXT: [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[TMP1]]
+; CHECK-SSE2-NEXT: ret <2 x i64> [[RES]]
+;
+; CHECK-XOP-LABEL: @test_64bit(
+; CHECK-XOP-NEXT: [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-XOP-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-XOP: if_true:
+; CHECK-XOP-NEXT: ret <2 x i64> [[MASK]]
+; CHECK-XOP: if_false:
+; CHECK-XOP-NEXT: [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[MASK]]
+; CHECK-XOP-NEXT: ret <2 x i64> [[RES]]
+;
+; CHECK-AVX-LABEL: @test_64bit(
+; CHECK-AVX-NEXT: [[MASK:%.*]] = shufflevector <2 x i64> [[TMP:%.*]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-AVX-NEXT: br i1 [[TST:%.*]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
+; CHECK-AVX: if_true:
+; CHECK-AVX-NEXT: ret <2 x i64> [[MASK]]
+; CHECK-AVX: if_false:
+; CHECK-AVX-NEXT: [[RES:%.*]] = lshr <2 x i64> [[LHS:%.*]], [[MASK]]
+; CHECK-AVX-NEXT: ret <2 x i64> [[RES]]
+;
+ %mask = shufflevector <2 x i64> %tmp, <2 x i64> undef, <2 x i32> zeroinitializer
+ br i1 %tst, label %if_true, label %if_false
+
+if_true:
+ ret <2 x i64> %mask
+
+if_false:
+ %res = lshr <2 x i64> %lhs, %mask
+ ret <2 x i64> %res
+}
+
+define void @funnel_splatvar(i32* nocapture %arr, i32 %rot) {
+; CHECK-SSE2-LABEL: @funnel_splatvar(
+; CHECK-SSE2-NEXT: entry:
+; CHECK-SSE2-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <8 x i32> poison, i32 [[ROT:%.*]], i32 0
+; CHECK-SSE2-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-SSE2: vector.body:
+; CHECK-SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-SSE2-NEXT: [[T0:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; CHECK-SSE2-NEXT: [[T1:%.*]] = bitcast i32* [[T0]] to <8 x i32>*
+; CHECK-SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[T1]], align 4
+; CHECK-SSE2-NEXT: [[TMP0:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT15]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-SSE2-NEXT: [[T2:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD]], <8 x i32> [[WIDE_LOAD]], <8 x i32> [[TMP0]])
+; CHECK-SSE2-NEXT: store <8 x i32> [[T2]], <8 x i32>* [[T1]], align 4
+; CHECK-SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-SSE2-NEXT: [[T3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 65536
+; CHECK-SSE2-NEXT: br i1 [[T3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK-SSE2: for.cond.cleanup:
+; CHECK-SSE2-NEXT: ret void
+;
+; CHECK-XOP-LABEL: @funnel_splatvar(
+; CHECK-XOP-NEXT: entry:
+; CHECK-XOP-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <8 x i32> poison, i32 [[ROT:%.*]], i32 0
+; CHECK-XOP-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT15]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-XOP-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-XOP: vector.body:
+; CHECK-XOP-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-XOP-NEXT: [[T0:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; CHECK-XOP-NEXT: [[T1:%.*]] = bitcast i32* [[T0]] to <8 x i32>*
+; CHECK-XOP-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[T1]], align 4
+; CHECK-XOP-NEXT: [[T2:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD]], <8 x i32> [[WIDE_LOAD]], <8 x i32> [[BROADCAST_SPLAT16]])
+; CHECK-XOP-NEXT: store <8 x i32> [[T2]], <8 x i32>* [[T1]], align 4
+; CHECK-XOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-XOP-NEXT: [[T3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 65536
+; CHECK-XOP-NEXT: br i1 [[T3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK-XOP: for.cond.cleanup:
+; CHECK-XOP-NEXT: ret void
+;
+; CHECK-AVX-LABEL: @funnel_splatvar(
+; CHECK-AVX-NEXT: entry:
+; CHECK-AVX-NEXT: [[BROADCAST_SPLATINSERT15:%.*]] = insertelement <8 x i32> poison, i32 [[ROT:%.*]], i32 0
+; CHECK-AVX-NEXT: [[BROADCAST_SPLAT16:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT15]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-AVX-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK-AVX: vector.body:
+; CHECK-AVX-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-AVX-NEXT: [[T0:%.*]] = getelementptr inbounds i32, i32* [[ARR:%.*]], i64 [[INDEX]]
+; CHECK-AVX-NEXT: [[T1:%.*]] = bitcast i32* [[T0]] to <8 x i32>*
+; CHECK-AVX-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, <8 x i32>* [[T1]], align 4
+; CHECK-AVX-NEXT: [[T2:%.*]] = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> [[WIDE_LOAD]], <8 x i32> [[WIDE_LOAD]], <8 x i32> [[BROADCAST_SPLAT16]])
+; CHECK-AVX-NEXT: store <8 x i32> [[T2]], <8 x i32>* [[T1]], align 4
+; CHECK-AVX-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-AVX-NEXT: [[T3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 65536
+; CHECK-AVX-NEXT: br i1 [[T3]], label [[FOR_COND_CLEANUP:%.*]], label [[VECTOR_BODY]]
+; CHECK-AVX: for.cond.cleanup:
+; CHECK-AVX-NEXT: ret void
+;
+entry:
+ %broadcast.splatinsert15 = insertelement <8 x i32> poison, i32 %rot, i32 0
+ %broadcast.splat16 = shufflevector <8 x i32> %broadcast.splatinsert15, <8 x i32> undef, <8 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %t0 = getelementptr inbounds i32, i32* %arr, i64 %index
+ %t1 = bitcast i32* %t0 to <8 x i32>*
+ %wide.load = load <8 x i32>, <8 x i32>* %t1, align 4
+ %t2 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %wide.load, <8 x i32> %wide.load, <8 x i32> %broadcast.splat16)
+ store <8 x i32> %t2, <8 x i32>* %t1, align 4
+ %index.next = add i64 %index, 8
+ %t3 = icmp eq i64 %index.next, 65536
+ br i1 %t3, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:
+ ret void
+}
+
+declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
diff --git a/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather-inseltpoison.ll b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather-inseltpoison.ll
new file mode 100644
index 000000000000..1d5428702a88
--- /dev/null
+++ b/llvm/test/Transforms/GVN/2016-08-30-MaskedScatterGather-inseltpoison.ll
@@ -0,0 +1,42 @@
+; RUN: opt < %s -basic-aa -gvn -S | FileCheck %s
+
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+
+; This test ensures that masked scatter and gather operations, which take vectors of pointers,
+; do not have pointer aliasing ignored when being processed.
+; No scatter/gather calls should end up eliminated
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+define spir_kernel void @test(<2 x i32*> %in1, <2 x i32*> %in2, i32* %out) {
+entry:
+ ; Just some temporary storage
+ %tmp.0 = alloca i32
+ %tmp.1 = alloca i32
+ %tmp.i = insertelement <2 x i32*> poison, i32* %tmp.0, i32 0
+ %tmp = insertelement <2 x i32*> %tmp.i, i32* %tmp.1, i32 1
+ ; Read from in1 and in2
+ %in1.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in1, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ %in2.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in2, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in1 to the allocas
+ call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in1.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+ ; Read in1 from the allocas
+ ; This gather should alias the scatter we just saw
+ %tmp.v.0 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in2 to the allocas
+ call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in2.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+ ; Read in2 from the allocas
+ ; This gather should alias the scatter we just saw, and not be eliminated
+ %tmp.v.1 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in2 to out for good measure
+ %tmp.v.1.0 = extractelement <2 x i32> %tmp.v.1, i32 0
+ %tmp.v.1.1 = extractelement <2 x i32> %tmp.v.1, i32 1
+ store i32 %tmp.v.1.0, i32* %out
+ %out.1 = getelementptr i32, i32* %out, i32 1
+ store i32 %tmp.v.1.1, i32* %out.1
+ ret void
+}
diff --git a/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash-inseltpoison.ll b/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash-inseltpoison.ll
new file mode 100644
index 000000000000..6750b858e03f
--- /dev/null
+++ b/llvm/test/Transforms/GVN/constexpr-vector-constainsundef-crash-inseltpoison.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S %s | FileCheck %s
+
+; Reduced test case from
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=24278
+
+; Make sure we do not crash when dealing with a vector constant expression.
+define <4 x i64*> @test(i64* %ptr) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[L3:%.*]] = load i64, i64* [[PTR:%.*]], align 4
+; CHECK-NEXT: [[I6:%.*]] = insertelement <4 x i64*> getelementptr (i64, i64* null, <4 x i64> <i64 poison, i64 poison, i64 poison, i64 -128>), i64* undef, i64 [[L3]]
+; CHECK-NEXT: ret <4 x i64*> [[I6]]
+;
+entry:
+ %B9 = sdiv i16 -32768, 256
+ %L3 = load i64, i64* %ptr, align 4
+ %B3 = sub i16 0, %B9
+ %0 = insertelement <4 x i16> poison, i16 %B3, i32 3
+ %1 = sub <4 x i16> zeroinitializer, %0
+ %2 = sext <4 x i16> %1 to <4 x i32>
+ %3 = getelementptr inbounds i64, i64* null, <4 x i32> %2
+ %I6 = insertelement <4 x i64*> %3, i64* undef, i64 %L3
+ ret <4 x i64*> %I6
+}
diff --git a/llvm/test/Transforms/GVN/non-integral-pointers-inseltpoison.ll b/llvm/test/Transforms/GVN/non-integral-pointers-inseltpoison.ll
new file mode 100644
index 000000000000..2aef7620841b
--- /dev/null
+++ b/llvm/test/Transforms/GVN/non-integral-pointers-inseltpoison.ll
@@ -0,0 +1,456 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -gvn -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:4:5"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @f0(i1 %alwaysFalse, i64 %val, i64* %loc) {
+; CHECK-LABEL: @f0(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store i64 [[VAL:%.*]], i64* [[LOC:%.*]], align 8
+; CHECK-NEXT: br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
+; CHECK: neverTaken:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i64* [[LOC]] to i8 addrspace(4)**
+; CHECK-NEXT: [[PTR:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)** [[LOC_BC]], align 8
+; CHECK-NEXT: store i8 5, i8 addrspace(4)* [[PTR]], align 1
+; CHECK-NEXT: ret void
+; CHECK: alwaysTaken:
+; CHECK-NEXT: ret void
+;
+ entry:
+ store i64 %val, i64* %loc
+ br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+ neverTaken:
+ %loc.bc = bitcast i64* %loc to i8 addrspace(4)**
+ %ptr = load i8 addrspace(4)*, i8 addrspace(4)** %loc.bc
+ store i8 5, i8 addrspace(4)* %ptr
+ ret void
+
+ alwaysTaken:
+ ret void
+}
+
+define i64 @f1(i1 %alwaysFalse, i8 addrspace(4)* %val, i8 addrspace(4)** %loc) {
+; CHECK-LABEL: @f1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store i8 addrspace(4)* [[VAL:%.*]], i8 addrspace(4)** [[LOC:%.*]], align 8
+; CHECK-NEXT: br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
+; CHECK: neverTaken:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)** [[LOC]] to i64*
+; CHECK-NEXT: [[INT:%.*]] = load i64, i64* [[LOC_BC]], align 8
+; CHECK-NEXT: ret i64 [[INT]]
+; CHECK: alwaysTaken:
+; CHECK-NEXT: ret i64 42
+;
+ entry:
+ store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+ br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+ neverTaken:
+ %loc.bc = bitcast i8 addrspace(4)** %loc to i64*
+ %int = load i64, i64* %loc.bc
+ ret i64 %int
+
+ alwaysTaken:
+ ret i64 42
+}
+
+;; Note: For terseness, we stop using the %alwaysfalse trick for the
+;; tests below and just exercise the bits of forwarding logic directly.
+
+declare void @llvm.memset.p4i8.i64(i8 addrspace(4)* nocapture, i8, i64, i1) nounwind
+
+; Can't forward as the load might be dead. (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_memset(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memset(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 7, i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+ entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 7, i64 8, i1 false)
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_memset_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memset_vload(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 7, i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret <1 x i8 addrspace(4)*> [[REF]]
+;
+ entry:
+ %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+ call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 7, i64 8, i1 false)
+ %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+ ret <1 x i8 addrspace(4)*> %ref
+}
+
+
+; Can forward since we can do so w/o breaking types
+define i8 addrspace(4)* @forward_memset_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memset_zero(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8 0, i64 8, i1 false)
+; CHECK-NEXT: ret i8 addrspace(4)* null
+;
+ entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ call void @llvm.memset.p4i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8 0, i64 8, i1 false)
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+; Can't forward as the load might be dead. (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_store(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_store(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT: store i64 5, i64 addrspace(4)* [[LOC_BC]], align 8
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+ entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i64 addrspace(4)*
+ store i64 5, i64 addrspace(4)* %loc.bc
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_store_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_store_vload(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT: store i64 5, i64 addrspace(4)* [[LOC_BC]], align 8
+; CHECK-NEXT: [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret <1 x i8 addrspace(4)*> [[REF]]
+;
+ entry:
+ %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i64 addrspace(4)*
+ store i64 5, i64 addrspace(4)* %loc.bc
+ %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+ ret <1 x i8 addrspace(4)*> %ref
+}
+
+; Nulls have known bit patterns, so we can forward
+define i8 addrspace(4)* @forward_store_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_store_zero(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i64 addrspace(4)*
+; CHECK-NEXT: store i64 0, i64 addrspace(4)* [[LOC_BC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* null
+;
+ entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i64 addrspace(4)*
+ store i64 0, i64 addrspace(4)* %loc.bc
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+; Nulls have known bit patterns, so we can forward
+define i8 addrspace(4)* @forward_store_zero2(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_store_zero2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i32> addrspace(4)*
+; CHECK-NEXT: store <2 x i32> zeroinitializer, <2 x i32> addrspace(4)* [[LOC_BC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* null
+;
+ entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i32> addrspace(4)*
+ store <2 x i32> zeroinitializer, <2 x i32> addrspace(4)* %loc.bc
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+
+
+ at NonZeroConstant = constant <4 x i64> <i64 3, i64 3, i64 3, i64 3>
+ at NonZeroConstant2 = constant <4 x i64 addrspace(4)*> <
+ i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3),
+ i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3),
+ i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3),
+ i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3)>
+ at ZeroConstant = constant <4 x i64> zeroinitializer
+
+
+; Can't forward as the load might be dead. (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_forward_memcopy(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcopy(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+define i64 addrspace(4)* @neg_forward_memcopy2(i64 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcopy2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i64 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret i64 addrspace(4)* [[REF]]
+;
+entry:
+ %loc.bc = bitcast i64 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* %loc
+ ret i64 addrspace(4)* %ref
+}
+
+; TODO: missed optimization
+define i8 addrspace(4)* @forward_memcopy(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memcopy(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+define i64 addrspace(4)* @forward_memcopy2(i64 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memcopy2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i64 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 8, i1 false)
+; CHECK-NEXT: ret i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3)
+;
+entry:
+ %loc.bc = bitcast i64 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load i64 addrspace(4)*, i64 addrspace(4)* addrspace(4)* %loc
+ ret i64 addrspace(4)* %ref
+}
+
+define <1 x i8 addrspace(4)*> @neg_forward_memcpy_vload(<1 x i8 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcpy_vload(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <1 x i8 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* [[LOC]], align 8
+; CHECK-NEXT: ret <1 x i8 addrspace(4)*> [[REF]]
+;
+entry:
+ %loc.bc = bitcast <1 x i8 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load <1 x i8 addrspace(4)*>, <1 x i8 addrspace(4)*> addrspace(4)* %loc
+ ret <1 x i8 addrspace(4)*> %ref
+}
+
+define <4 x i64 addrspace(4)*> @neg_forward_memcpy_vload2(<4 x i64 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcpy_vload2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @NonZeroConstant to i8*), i64 32, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* [[LOC]], align 32
+; CHECK-NEXT: ret <4 x i64 addrspace(4)*> [[REF]]
+;
+entry:
+ %loc.bc = bitcast <4 x i64 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64>* @NonZeroConstant to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false)
+ %ref = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* %loc
+ ret <4 x i64 addrspace(4)*> %ref
+}
+
+define <4 x i64> @neg_forward_memcpy_vload3(<4 x i64> addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_forward_memcpy_vload3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 32, i1 false)
+; CHECK-NEXT: [[REF:%.*]] = load <4 x i64>, <4 x i64> addrspace(4)* [[LOC]], align 32
+; CHECK-NEXT: ret <4 x i64> [[REF]]
+;
+entry:
+ %loc.bc = bitcast <4 x i64> addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false)
+ %ref = load <4 x i64>, <4 x i64> addrspace(4)* %loc
+ ret <4 x i64> %ref
+}
+
+define <1 x i64 addrspace(4)*> @forward_memcpy_vload3(<4 x i64 addrspace(4)*> addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memcpy_vload3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast <4 x i64 addrspace(4)*> addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*), i64 32, i1 false)
+; CHECK-NEXT: ret <1 x i64 addrspace(4)*> <i64 addrspace(4)* getelementptr (i64, i64 addrspace(4)* null, i32 3)>
+;
+entry:
+ %loc.bc = bitcast <4 x i64 addrspace(4)*> addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64 addrspace(4)*>* @NonZeroConstant2 to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 32, i1 false)
+ %ref = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*> addrspace(4)* %loc
+ %val = extractelement <4 x i64 addrspace(4)*> %ref, i32 0
+ %ret = insertelement <1 x i64 addrspace(4)*> poison, i64 addrspace(4)* %val, i32 0
+ ret <1 x i64 addrspace(4)*> %ret
+}
+
+; Can forward since we can do so w/o breaking types
+define i8 addrspace(4)* @forward_memcpy_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @forward_memcpy_zero(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to i8 addrspace(4)*
+; CHECK-NEXT: call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 [[LOC_BC]], i8* bitcast (<4 x i64>* @ZeroConstant to i8*), i64 8, i1 false)
+; CHECK-NEXT: ret i8 addrspace(4)* null
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to i8 addrspace(4)*
+ %src.bc = bitcast <4 x i64>* @ZeroConstant to i8*
+ call void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* align 4 %loc.bc, i8* %src.bc, i64 8, i1 false)
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc
+ ret i8 addrspace(4)* %ref
+}
+
+declare void @llvm.memcpy.p4i8.p0i8.i64(i8 addrspace(4)* nocapture, i8* nocapture, i64, i1) nounwind
+
+
+; Same as the neg_forward_store cases, but for non defs.
+; (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_store_clobber(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_store_clobber(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT: store <2 x i64> <i64 4, i64 4>, <2 x i64> addrspace(4)* [[LOC_BC]], align 16
+; CHECK-NEXT: [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC_OFF]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+ store <2 x i64> <i64 4, i64 4>, <2 x i64> addrspace(4)* %loc.bc
+ %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+ ret i8 addrspace(4)* %ref
+}
+
+declare void @use(<2 x i64>) inaccessiblememonly
+
+; Same as the neg_forward_store cases, but for non defs.
+; (Pretend we wrote out the alwaysfalse idiom above.)
+define i8 addrspace(4)* @neg_load_clobber(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @neg_load_clobber(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT: [[V:%.*]] = load <2 x i64>, <2 x i64> addrspace(4)* [[LOC_BC]], align 16
+; CHECK-NEXT: call void @use(<2 x i64> [[V]])
+; CHECK-NEXT: [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT: [[REF:%.*]] = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC_OFF]], align 8
+; CHECK-NEXT: ret i8 addrspace(4)* [[REF]]
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+ %v = load <2 x i64>, <2 x i64> addrspace(4)* %loc.bc
+ call void @use(<2 x i64> %v)
+ %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+ ret i8 addrspace(4)* %ref
+}
+
+define i8 addrspace(4)* @store_clobber_zero(i8 addrspace(4)* addrspace(4)* %loc) {
+; CHECK-LABEL: @store_clobber_zero(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)* addrspace(4)* [[LOC:%.*]] to <2 x i64> addrspace(4)*
+; CHECK-NEXT: store <2 x i64> zeroinitializer, <2 x i64> addrspace(4)* [[LOC_BC]], align 16
+; CHECK-NEXT: [[LOC_OFF:%.*]] = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* [[LOC]], i64 1
+; CHECK-NEXT: ret i8 addrspace(4)* null
+;
+entry:
+ %loc.bc = bitcast i8 addrspace(4)* addrspace(4)* %loc to <2 x i64> addrspace(4)*
+ store <2 x i64> zeroinitializer, <2 x i64> addrspace(4)* %loc.bc
+ %loc.off = getelementptr i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc, i64 1
+ %ref = load i8 addrspace(4)*, i8 addrspace(4)* addrspace(4)* %loc.off
+ ret i8 addrspace(4)* %ref
+}
+
+
+define void @smaller_vector(i8* %p) {
+; CHECK-LABEL: @smaller_vector(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[A:%.*]] = bitcast i8* [[P:%.*]] to <4 x i64 addrspace(4)*>*
+; CHECK-NEXT: [[B:%.*]] = bitcast i8* [[P]] to <2 x i64 addrspace(4)*>*
+; CHECK-NEXT: [[V4:%.*]] = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*>* [[A]], align 32
+; CHECK-NEXT: [[V2:%.*]] = load <2 x i64 addrspace(4)*>, <2 x i64 addrspace(4)*>* [[B]], align 32
+; CHECK-NEXT: call void @use.v2(<2 x i64 addrspace(4)*> [[V2]])
+; CHECK-NEXT: call void @use.v4(<4 x i64 addrspace(4)*> [[V4]])
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = bitcast i8* %p to <4 x i64 addrspace(4)*>*
+ %b = bitcast i8* %p to <2 x i64 addrspace(4)*>*
+ %v4 = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*>* %a, align 32
+ %v2 = load <2 x i64 addrspace(4)*>, <2 x i64 addrspace(4)*>* %b, align 32
+ call void @use.v2(<2 x i64 addrspace(4)*> %v2)
+ call void @use.v4(<4 x i64 addrspace(4)*> %v4)
+ ret void
+}
+
+define i64 addrspace(4)* @vector_extract(i8* %p) {
+; CHECK-LABEL: @vector_extract(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[A:%.*]] = bitcast i8* [[P:%.*]] to <4 x i64 addrspace(4)*>*
+; CHECK-NEXT: [[B:%.*]] = bitcast i8* [[P]] to i64 addrspace(4)**
+; CHECK-NEXT: [[V4:%.*]] = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*>* [[A]], align 32
+; CHECK-NEXT: [[RES:%.*]] = load i64 addrspace(4)*, i64 addrspace(4)** [[B]], align 32
+; CHECK-NEXT: call void @use.v4(<4 x i64 addrspace(4)*> [[V4]])
+; CHECK-NEXT: ret i64 addrspace(4)* [[RES]]
+;
+entry:
+ %a = bitcast i8* %p to <4 x i64 addrspace(4)*>*
+ %b = bitcast i8* %p to i64 addrspace(4)**
+ %v4 = load <4 x i64 addrspace(4)*>, <4 x i64 addrspace(4)*>* %a, align 32
+ %res = load i64 addrspace(4)*, i64 addrspace(4)** %b, align 32
+ call void @use.v4(<4 x i64 addrspace(4)*> %v4)
+ ret i64 addrspace(4)* %res
+}
+
+declare void @use.v2(<2 x i64 addrspace(4)*>)
+declare void @use.v4(<4 x i64 addrspace(4)*>)
+ define i8 addrspace(5)* @multini(i1 %alwaysFalse, i8 addrspace(4)* %val, i8 addrspace(4)** %loc) {
+; CHECK-LABEL: @multini(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store i8 addrspace(4)* [[VAL:%.*]], i8 addrspace(4)** [[LOC:%.*]], align 8
+; CHECK-NEXT: br i1 [[ALWAYSFALSE:%.*]], label [[NEVERTAKEN:%.*]], label [[ALWAYSTAKEN:%.*]]
+; CHECK: neverTaken:
+; CHECK-NEXT: [[LOC_BC:%.*]] = bitcast i8 addrspace(4)** [[LOC]] to i8 addrspace(5)**
+; CHECK-NEXT: [[DIFFERENTAS:%.*]] = load i8 addrspace(5)*, i8 addrspace(5)** [[LOC_BC]], align 8
+; CHECK-NEXT: ret i8 addrspace(5)* [[DIFFERENTAS]]
+; CHECK: alwaysTaken:
+; CHECK-NEXT: ret i8 addrspace(5)* null
+;
+ entry:
+ store i8 addrspace(4)* %val, i8 addrspace(4)** %loc
+ br i1 %alwaysFalse, label %neverTaken, label %alwaysTaken
+
+ neverTaken:
+ %loc.bc = bitcast i8 addrspace(4)** %loc to i8 addrspace(5)**
+ %
diff erentas = load i8 addrspace(5)*, i8 addrspace(5)** %loc.bc
+ ret i8 addrspace(5)* %
diff erentas
+
+ alwaysTaken:
+ ret i8 addrspace(5)* null
+ }
diff --git a/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions-inseltpoison.ll b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions-inseltpoison.ll
new file mode 100644
index 000000000000..e0bf4f1058f9
--- /dev/null
+++ b/llvm/test/Transforms/InferAddressSpaces/AMDGPU/old-pass-regressions-inseltpoison.ll
@@ -0,0 +1,143 @@
+; RUN: opt -data-layout=A5 -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
+
+; Regression tests from old HSAIL addrspacecast optimization pass
+
+ at data = internal addrspace(1) global [100 x double] [double 0.00, double 1.000000e-01, double 2.000000e-01, double 3.000000e-01, double 4.000000e-01, double 5.000000e-01, double 6.000000e-01, double 7.000000e-01, double 8.000000e-01, double 9.000000e-01, double 1.00, double 1.10, double 1.20, double 1.30, double 1.40, double 1.50, double 1.60, double 1.70, double 1.80, double 1.90, double 2.00, double 2.10, double 2.20, double 2.30, double 2.40, double 2.50, double 2.60, double 2.70, double 2.80, double 2.90, double 3.00, double 3.10, double 3.20, double 3.30, double 3.40, double 3.50, double 3.60, double 3.70, double 3.80, double 3.90, double 4.00, double 4.10, double 4.20, double 4.30, double 4.40, double 4.50, double 4.60, double 4.70, double 4.80, double 4.90, double 5.00, double 5.10, double 5.20, double 5.30, double 5.40, double 5.50, double 5.60, double 5.70, double 5.80, double 5.90, double 6.00, double 6.10, double 6.20, double 6.30, double 6.40, double 6.50, double 6.60, double 6.70, double 6.80, double 6.90, double 7.00, double 7.10, double 7.20, double 7.30, double 7.40, double 7.50, double 7.60, double 7.70, double 7.80, double 7.90, double 8.00, double 8.10, double 8.20, double 8.30, double 8.40, double 8.50, double 8.60, double 8.70, double 8.80, double 8.90, double 9.00, double 9.10, double 9.20, double 9.30, double 9.40, double 9.50, double 9.60, double 9.70, double 9.80, double 9.90], align 8
+
+
+; Should generate flat load
+
+; CHECK-LABEL: @generic_address_bitcast_const(
+; CHECK: %vecload1 = load <2 x double>, <2 x double> addrspace(1)* bitcast (double addrspace(1)* getelementptr inbounds ([100 x double], [100 x double] addrspace(1)* @data, i64 0, i64 4) to <2 x double> addrspace(1)*), align 8
+define amdgpu_kernel void @generic_address_bitcast_const(i64 %arg0, i32 addrspace(1)* nocapture %results) #0 {
+entry:
+ %tmp1 = call i32 @llvm.amdgcn.workitem.id.x()
+ %tmp2 = zext i32 %tmp1 to i64
+ %tmp3 = add i64 %tmp2, %arg0
+ %vecload1 = load <2 x double>, <2 x double>* bitcast (double* getelementptr ([100 x double], [100 x double]* addrspacecast ([100 x double] addrspace(1)* @data to [100 x double]*), i64 0, i64 4) to <2 x double>*), align 8
+ %cmp = fcmp ord <2 x double> %vecload1, zeroinitializer
+ %sext = sext <2 x i1> %cmp to <2 x i64>
+ %tmp4 = extractelement <2 x i64> %sext, i64 0
+ %tmp5 = extractelement <2 x i64> %sext, i64 1
+ %tmp6 = and i64 %tmp4, %tmp5
+ %tmp7 = lshr i64 %tmp6, 63
+ %tmp8 = trunc i64 %tmp7 to i32
+ %idxprom = and i64 %tmp3, 4294967295
+ %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %results, i64 %idxprom
+ store i32 %tmp8, i32 addrspace(1)* %arrayidx, align 4
+ ret void
+}
+
+ at generic_address_bug9749.val = internal addrspace(1) global float 0.0, align 4
+
+declare i32 @_Z9get_fencePv(i8*)
+%opencl.pipe_t = type opaque
+
+; This is a compile time assert bug, but we still want to check optimization
+; is performed to generate ld_global.
+; CHECK-LABEL: @generic_address_pipe_bug9673(
+; CHECK: %tmp1 = bitcast %opencl.pipe_t addrspace(3)* %in_pipe to i32 addrspace(3)*
+; CHECK: %add.ptr = getelementptr inbounds i32, i32 addrspace(3)* %tmp1, i32 2
+; CHECK: %tmp2 = load i32, i32 addrspace(3)* %add.ptr, align 4
+define amdgpu_kernel void @generic_address_pipe_bug9673(%opencl.pipe_t addrspace(3)* nocapture %in_pipe, i32 addrspace(1)* nocapture %dst) #0 {
+entry:
+ %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+ %tmp1 = bitcast %opencl.pipe_t addrspace(3)* %in_pipe to i32 addrspace(3)*
+ %add.ptr = getelementptr inbounds i32, i32 addrspace(3)* %tmp1, i32 2
+ %tmp2 = load i32, i32 addrspace(3)* %add.ptr, align 4
+ %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %dst, i32 %tmp
+ store i32 %tmp2, i32 addrspace(1)* %arrayidx, align 4
+ ret void
+}
+
+; Should generate flat load
+; CHECK-LABEL: @generic_address_bug9749(
+; CHECK: br i1
+; CHECK: load float, float*
+; CHECK: br label
+define amdgpu_kernel void @generic_address_bug9749(i32 addrspace(1)* nocapture %results) #0 {
+entry:
+ %ptr = alloca float*, align 8, addrspace(5)
+ %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+ %tmp1 = zext i32 %tmp to i64
+ store float 0x3FB99999A0000000, float addrspace(1)* @generic_address_bug9749.val, align 4
+ store volatile float* addrspacecast (float addrspace(1)* @generic_address_bug9749.val to float*), float* addrspace(5)* %ptr, align 8
+ %tmp2 = load volatile float*, float* addrspace(5)* %ptr, align 8
+ %tmp3 = load float, float addrspace(1)* @generic_address_bug9749.val, align 4
+ %tmp4 = bitcast float* %tmp2 to i8*
+ %call.i = call i32 @_Z9get_fencePv(i8* %tmp4) #1
+ %switch.i.i = icmp ult i32 %call.i, 4
+ br i1 %switch.i.i, label %if.end.i, label %helperFunction.exit
+
+if.end.i: ; preds = %entry
+ %tmp5 = load float, float* %tmp2, align 4
+ %not.cmp.i = fcmp oeq float %tmp5, %tmp3
+ %phitmp = zext i1 %not.cmp.i to i32
+ br label %helperFunction.exit
+
+helperFunction.exit: ; preds = %if.end.i, %entry
+ %retval.0.i = phi i32 [ 0, %entry ], [ %phitmp, %if.end.i ]
+ %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %results, i64 %tmp1
+ store i32 %retval.0.i, i32 addrspace(1)* %arrayidx, align 4
+ ret void
+}
+
+; CHECK-LABEL: @generic_address_opt_phi_bug9776_simple_phi_kernel(
+; CHECK: phi i32 addrspace(3)*
+; CHECK: store i32 %i.03, i32 addrspace(3)* %
+define amdgpu_kernel void @generic_address_opt_phi_bug9776_simple_phi_kernel(i32 addrspace(3)* nocapture %in, i32 %numElems) #0 {
+entry:
+ %cmp1 = icmp eq i32 %numElems, 0
+ br i1 %cmp1, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph: ; preds = %entry
+ %tmp = addrspacecast i32 addrspace(3)* %in to i32*
+ br label %for.body
+
+for.body: ; preds = %for.body, %for.body.lr.ph
+ %i.03 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+ %ptr.02 = phi i32* [ %tmp, %for.body.lr.ph ], [ %add.ptr, %for.body ]
+ store i32 %i.03, i32* %ptr.02, align 4
+ %add.ptr = getelementptr inbounds i32, i32* %ptr.02, i64 4
+ %inc = add nuw i32 %i.03, 1
+ %exitcond = icmp eq i32 %inc, %numElems
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body, %entry
+ ret void
+}
+
+; CHECK-LABEL: @generic_address_bug9899(
+; CHECK: %vecload = load <2 x i32>, <2 x i32> addrspace(3)*
+; CHECK: store <2 x i32> %tmp16, <2 x i32> addrspace(3)*
+define amdgpu_kernel void @generic_address_bug9899(i64 %arg0, i32 addrspace(3)* nocapture %sourceA, i32 addrspace(3)* nocapture %destValues) #0 {
+entry:
+ %tmp1 = call i32 @llvm.amdgcn.workitem.id.x()
+ %tmp2 = zext i32 %tmp1 to i64
+ %tmp3 = add i64 %tmp2, %arg0
+ %sext = shl i64 %tmp3, 32
+ %tmp4 = addrspacecast i32 addrspace(3)* %destValues to i32*
+ %tmp5 = addrspacecast i32 addrspace(3)* %sourceA to i32*
+ %tmp6 = ashr exact i64 %sext, 31
+ %tmp7 = getelementptr inbounds i32, i32* %tmp5, i64 %tmp6
+ %arrayidx_v4 = bitcast i32* %tmp7 to <2 x i32>*
+ %vecload = load <2 x i32>, <2 x i32>* %arrayidx_v4, align 4
+ %tmp8 = extractelement <2 x i32> %vecload, i32 0
+ %tmp9 = extractelement <2 x i32> %vecload, i32 1
+ %tmp10 = icmp eq i32 %tmp8, 0
+ %tmp11 = select i1 %tmp10, i32 32, i32 %tmp8
+ %tmp12 = icmp eq i32 %tmp9, 0
+ %tmp13 = select i1 %tmp12, i32 32, i32 %tmp9
+ %tmp14 = getelementptr inbounds i32, i32* %tmp4, i64 %tmp6
+ %tmp15 = insertelement <2 x i32> poison, i32 %tmp11, i32 0
+ %tmp16 = insertelement <2 x i32> %tmp15, i32 %tmp13, i32 1
+ %arrayidx_v41 = bitcast i32* %tmp14 to <2 x i32>*
+ store <2 x i32> %tmp16, <2 x i32>* %arrayidx_v41, align 4
+ ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+attributes #2 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InferFunctionAttrs/dereferenceable-inseltpoison.ll b/llvm/test/Transforms/InferFunctionAttrs/dereferenceable-inseltpoison.ll
new file mode 100644
index 000000000000..ac3245e8a86c
--- /dev/null
+++ b/llvm/test/Transforms/InferFunctionAttrs/dereferenceable-inseltpoison.ll
@@ -0,0 +1,357 @@
+; RUN: opt < %s -inferattrs -S | FileCheck %s
+
+
+
+; Determine dereference-ability before unused loads get deleted:
+; https://bugs.llvm.org/show_bug.cgi?id=21780
+
+define <4 x double> @PR21780(double* %ptr) {
+; CHECK-LABEL: @PR21780(double* %ptr)
+
+ ; GEP of index 0 is simplified away.
+ %arrayidx1 = getelementptr inbounds double, double* %ptr, i64 1
+ %arrayidx2 = getelementptr inbounds double, double* %ptr, i64 2
+ %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+
+ %t0 = load double, double* %ptr, align 8
+ %t1 = load double, double* %arrayidx1, align 8
+ %t2 = load double, double* %arrayidx2, align 8
+ %t3 = load double, double* %arrayidx3, align 8
+
+ %vecinit0 = insertelement <4 x double> poison, double %t0, i32 0
+ %vecinit1 = insertelement <4 x double> %vecinit0, double %t1, i32 1
+ %vecinit2 = insertelement <4 x double> %vecinit1, double %t2, i32 2
+ %vecinit3 = insertelement <4 x double> %vecinit2, double %t3, i32 3
+ %shuffle = shufflevector <4 x double> %vecinit3, <4 x double> %vecinit3, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+ ret <4 x double> %shuffle
+}
+
+
+define double @PR21780_only_access3_with_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_only_access3_with_inbounds(double* %ptr)
+
+ %arrayidx3 = getelementptr inbounds double, double* %ptr, i64 3
+ %t3 = load double, double* %arrayidx3, align 8
+ ret double %t3
+}
+
+define double @PR21780_only_access3_without_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_only_access3_without_inbounds(double* %ptr)
+ %arrayidx3 = getelementptr double, double* %ptr, i64 3
+ %t3 = load double, double* %arrayidx3, align 8
+ ret double %t3
+}
+
+define double @PR21780_without_inbounds(double* %ptr) {
+; CHECK-LABEL: @PR21780_without_inbounds(double* %ptr)
+
+ %arrayidx1 = getelementptr double, double* %ptr, i64 1
+ %arrayidx2 = getelementptr double, double* %ptr, i64 2
+ %arrayidx3 = getelementptr double, double* %ptr, i64 3
+
+ %t0 = load double, double* %ptr, align 8
+ %t1 = load double, double* %arrayidx1, align 8
+ %t2 = load double, double* %arrayidx2, align 8
+ %t3 = load double, double* %arrayidx3, align 8
+
+ ret double %t3
+}
+
+; Unsimplified, but still valid. Also, throw in some bogus arguments.
+
+define void @gep0(i8* %unused, i8* %other, i8* %ptr) {
+; CHECK-LABEL: @gep0(i8* %unused, i8* %other, i8* %ptr)
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ store i8 %t2, i8* %other
+ ret void
+}
+
+; Order of accesses does not change computation.
+; Multiple arguments may be dereferenceable.
+
+define void @ordering(i8* %ptr1, i32* %ptr2) {
+; CHECK-LABEL: @ordering(i8* %ptr1, i32* %ptr2)
+ %a20 = getelementptr i32, i32* %ptr2, i64 0
+ %a12 = getelementptr i8, i8* %ptr1, i64 2
+ %t12 = load i8, i8* %a12
+ %a11 = getelementptr i8, i8* %ptr1, i64 1
+ %t20 = load i32, i32* %a20
+ %a10 = getelementptr i8, i8* %ptr1, i64 0
+ %t10 = load i8, i8* %a10
+ %t11 = load i8, i8* %a11
+ %a21 = getelementptr i32, i32* %ptr2, i64 1
+ %t21 = load i32, i32* %a21
+ ret void
+}
+
+; Not in entry block.
+
+define void @not_entry_but_guaranteed_to_execute(i8* %ptr) {
+; CHECK-LABEL: @not_entry_but_guaranteed_to_execute(i8* %ptr)
+entry:
+ br label %exit
+exit:
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+}
+
+; Not in entry block and not guaranteed to execute.
+
+define void @not_entry_not_guaranteed_to_execute(i8* %ptr, i1 %cond) {
+; CHECK-LABEL: @not_entry_not_guaranteed_to_execute(i8* %ptr, i1 %cond)
+entry:
+ br i1 %cond, label %loads, label %exit
+loads:
+ %arrayidx0 = getelementptr i8, i8* %ptr, i64 0
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 1
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %arrayidx0
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+exit:
+ ret void
+}
+
+; The last load may not execute, so derefenceable bytes only covers the 1st two loads.
+
+define void @partial_in_entry(i16* %ptr, i1 %cond) {
+; CHECK-LABEL: @partial_in_entry(i16* %ptr, i1 %cond)
+entry:
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load i16, i16* %arrayidx0
+ %t1 = load i16, i16* %arrayidx1
+ br i1 %cond, label %loads, label %exit
+loads:
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+exit:
+ ret void
+}
+
+; The volatile load can't be used to prove a non-volatile access is allowed.
+; The 2nd and 3rd loads may never execute.
+
+define void @volatile_is_not_dereferenceable(i16* %ptr) {
+; CHECK-LABEL: @volatile_is_not_dereferenceable(i16* %ptr)
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load volatile i16, i16* %arrayidx0
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; TODO: We should allow inference for atomic (but not volatile) ops.
+
+define void @atomic_is_alright(i16* %ptr) {
+; CHECK-LABEL: @atomic_is_alright(i16* %ptr)
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load atomic i16, i16* %arrayidx0 unordered, align 2
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+declare void @may_not_return()
+
+define void @not_guaranteed_to_transfer_execution(i16* %ptr) {
+; CHECK-LABEL: @not_guaranteed_to_transfer_execution(i16* %ptr)
+ %arrayidx0 = getelementptr i16, i16* %ptr, i64 0
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t0 = load i16, i16* %arrayidx0
+ call void @may_not_return()
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; We must have consecutive accesses.
+
+define void @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index) {
+; CHECK-LABEL: @variable_gep_index(i8* %unused, i8* %ptr, i64 %variable_index)
+ %arrayidx1 = getelementptr i8, i8* %ptr, i64 %variable_index
+ %arrayidx2 = getelementptr i8, i8* %ptr, i64 2
+ %t0 = load i8, i8* %ptr
+ %t1 = load i8, i8* %arrayidx1
+ %t2 = load i8, i8* %arrayidx2
+ ret void
+}
+
+; Deal with >1 GEP index.
+
+define void @multi_index_gep(<4 x i8>* %ptr) {
+; CHECK-LABEL: @multi_index_gep(<4 x i8>* %ptr)
+; FIXME: %ptr should be dereferenceable(4)
+ %arrayidx00 = getelementptr <4 x i8>, <4 x i8>* %ptr, i64 0, i64 0
+ %t0 = load i8, i8* %arrayidx00
+ ret void
+}
+
+; Could round weird bitwidths down?
+
+define void @not_byte_multiple(i9* %ptr) {
+; CHECK-LABEL: @not_byte_multiple(i9* %ptr)
+ %arrayidx0 = getelementptr i9, i9* %ptr, i64 0
+ %t0 = load i9, i9* %arrayidx0
+ ret void
+}
+
+; Missing direct access from the pointer.
+
+define void @no_pointer_deref(i16* %ptr) {
+; CHECK-LABEL: @no_pointer_deref(i16* %ptr)
+ %arrayidx1 = getelementptr i16, i16* %ptr, i64 1
+ %arrayidx2 = getelementptr i16, i16* %ptr, i64 2
+ %t1 = load i16, i16* %arrayidx1
+ %t2 = load i16, i16* %arrayidx2
+ ret void
+}
+
+; Out-of-order is ok, but missing access concludes dereferenceable range.
+
+define void @non_consecutive(i32* %ptr) {
+; CHECK-LABEL: @non_consecutive(i32* %ptr)
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %t1 = load i32, i32* %arrayidx1
+ %t0 = load i32, i32* %arrayidx0
+ %t3 = load i32, i32* %arrayidx3
+ ret void
+}
+
+; Improve on existing dereferenceable attribute.
+
+define void @more_bytes(i32* dereferenceable(8) %ptr) {
+; CHECK-LABEL: @more_bytes(i32* dereferenceable(8) %ptr)
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+; Improve on existing dereferenceable_or_null attribute.
+
+define void @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr) {
+; CHECK-LABEL: @more_bytes_and_not_null(i32* dereferenceable_or_null(8) %ptr)
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+; But don't pessimize existing dereferenceable attribute.
+
+define void @better_bytes(i32* dereferenceable(100) %ptr) {
+; CHECK-LABEL: @better_bytes(i32* dereferenceable(100) %ptr)
+ %arrayidx3 = getelementptr i32, i32* %ptr, i64 3
+ %arrayidx1 = getelementptr i32, i32* %ptr, i64 1
+ %arrayidx0 = getelementptr i32, i32* %ptr, i64 0
+ %arrayidx2 = getelementptr i32, i32* %ptr, i64 2
+ %t3 = load i32, i32* %arrayidx3
+ %t1 = load i32, i32* %arrayidx1
+ %t2 = load i32, i32* %arrayidx2
+ %t0 = load i32, i32* %arrayidx0
+ ret void
+}
+
+define void @bitcast(i32* %arg) {
+; CHECK-LABEL: @bitcast(i32* %arg)
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ %t0 = load float, float* %arrayidx0
+ %t1 = load float, float* %arrayidx1
+ ret void
+}
+
+define void @bitcast_
diff erent_sizes(double* %arg1, i8* %arg2) {
+; CHECK-LABEL: @bitcast_
diff erent_sizes(double* %arg1, i8* %arg2)
+ %ptr1 = bitcast double* %arg1 to float*
+ %a10 = getelementptr float, float* %ptr1, i64 0
+ %a11 = getelementptr float, float* %ptr1, i64 1
+ %a12 = getelementptr float, float* %ptr1, i64 2
+ %ld10 = load float, float* %a10
+ %ld11 = load float, float* %a11
+ %ld12 = load float, float* %a12
+
+ %ptr2 = bitcast i8* %arg2 to i64*
+ %a20 = getelementptr i64, i64* %ptr2, i64 0
+ %a21 = getelementptr i64, i64* %ptr2, i64 1
+ %ld20 = load i64, i64* %a20
+ %ld21 = load i64, i64* %a21
+ ret void
+}
+
+define void @negative_offset(i32* %arg) {
+; CHECK-LABEL: @negative_offset(i32* %arg)
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 -1
+ %t0 = load float, float* %arrayidx0
+ %t1 = load float, float* %arrayidx1
+ ret void
+}
+
+define void @stores(i32* %arg) {
+; CHECK-LABEL: @stores(i32* %arg)
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ store float 1.0, float* %arrayidx0
+ store float 2.0, float* %arrayidx1
+ ret void
+}
+
+define void @load_store(i32* %arg) {
+; CHECK-LABEL: @load_store(i32* %arg)
+ %ptr = bitcast i32* %arg to float*
+ %arrayidx0 = getelementptr float, float* %ptr, i64 0
+ %arrayidx1 = getelementptr float, float* %ptr, i64 1
+ %t1 = load float, float* %arrayidx0
+ store float 2.0, float* %arrayidx1
+ ret void
+}
+
+define void @
diff erent_size1(i32* %arg) {
+; CHECK-LABEL: @
diff erent_size1(i32* %arg)
+ %arg-cast = bitcast i32* %arg to double*
+ store double 0.000000e+00, double* %arg-cast
+ store i32 0, i32* %arg
+ ret void
+}
+
+define void @
diff erent_size2(i32* %arg) {
+; CHECK-LABEL: @
diff erent_size2(i32* %arg)
+ store i32 0, i32* %arg
+ %arg-cast = bitcast i32* %arg to double*
+ store double 0.000000e+00, double* %arg-cast
+ ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll
new file mode 100644
index 000000000000..d0cc892b0e69
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-bitcast-inseltpoison.ll
@@ -0,0 +1,13 @@
+; RUN: opt -instcombine -mtriple=aarch64-linux-gnu -mattr=+sve -S < %s | FileCheck %s
+
+; We shouldn't fold bitcast(insert <vscale x 1 x iX> .., iX %val, i32 0)
+; into bitcast(iX %val) for scalable vectors.
+define <vscale x 2 x i8> @bitcast_of_insert_i8_i16(i16 %val) #0 {
+; CHECK-LABEL: @bitcast_of_insert_i8_i16(
+; CHECK-NOT: bitcast i16 %val to <vscale x 2 x i8>
+; CHECK: bitcast <vscale x 1 x i16> %op2 to <vscale x 2 x i8>
+entry:
+ %op2 = insertelement <vscale x 1 x i16> poison, i16 %val, i32 0
+ %0 = bitcast <vscale x 1 x i16> %op2 to <vscale x 2 x i8>
+ ret <vscale x 2 x i8> %0
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
new file mode 100644
index 000000000000..8363ed461f94
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts-inseltpoison.ll
@@ -0,0 +1,3828 @@
+; RUN: opt -S -instcombine -mtriple=amdgcn-amd-amdhsa %s | FileCheck %s
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ ret float %data
+}
+
+; CHECK-LABEL: @buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v4f32_2(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt0 = extractelement <2 x float> %data, i32 0
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: %ins0 = insertvalue { float, float } undef, float %elt0, 0
+; CHECK-NEXT: %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
+; CHECK-NEXT: ret { float, float } %ins1
+define amdgpu_ps { float, float } @extract_elt0_elt1_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ %elt1 = extractelement <4 x float> %data, i32 1
+ %ins0 = insertvalue { float, float } undef, float %elt0, 0
+ %ins1 = insertvalue { float, float } %ins0, float %elt1, 1
+ ret { float, float } %ins1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_2(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt0 = extractelement <3 x float> %data, i32 0
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 1
+; CHECK-NEXT: %elt2 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
+; CHECK-NEXT: %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
+; CHECK-NEXT: %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
+; CHECK-NEXT: ret { float, float, float } %ins2
+define amdgpu_ps { float, float, float } @extract_elt0_elt1_elt2_buffer_load_v4f32_2(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ %elt1 = extractelement <4 x float> %data, i32 1
+ %elt2 = extractelement <4 x float> %data, i32 2
+ %ins0 = insertvalue { float, float, float } undef, float %elt0, 0
+ %ins1 = insertvalue { float, float, float } %ins0, float %elt1, 1
+ %ins2 = insertvalue { float, float, float } %ins1, float %elt2, 2
+ ret { float, float, float } %ins2
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_3(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 undef, i32 1>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_3(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ %elt2 = extractelement <4 x float> %data, i32 2
+ %ins0 = insertelement <2 x float> poison, float %elt0, i32 0
+ %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 4, i32 1>
+ %ret = fadd <2 x float> %ins1, %shuf
+ ret <2 x float> %ret
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_4(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+; CHECK-NEXT: ret <2 x float> %ret
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_4(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ %elt2 = extractelement <4 x float> %data, i32 2
+ %ins0 = insertelement <2 x float> poison, float %elt0, i32 0
+ %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+ %shuf = shufflevector <4 x float> undef, <4 x float> %data, <2 x i32> <i32 5, i32 1>
+ %ret = fadd <2 x float> %ins1, %shuf
+ ret <2 x float> %ret
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_buffer_load_v4f32_5(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %ins1 = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 2, i32 2>
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: %ret = fadd <2 x float> %ins1, %shuf
+define amdgpu_ps <2 x float> @extract_elt0_elt1_elt2_buffer_load_v4f32_5(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt2 = extractelement <4 x float> %data, i32 2
+ %ins0 = insertelement <2 x float> poison, float %elt2, i32 0
+ %ins1 = insertelement <2 x float> %ins0, float %elt2, i32 1
+ %shuf = shufflevector <4 x float> %data, <4 x float> %data, <2 x i32> <i32 0, i32 5>
+ %ret = fadd <2 x float> %ins1, %shuf
+ ret <2 x float> %ret
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.buffer.load.f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.buffer.load.v1f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.buffer.load.v2f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.buffer.load.v3f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.buffer.load.v4f32(<4 x i32>, i32, i32, i1, i1) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 true)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 true, i1 false)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; The initial insertion point is at the extractelement
+; CHECK-LABEL: @extract01_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %1 = shufflevector <2 x float> %tmp, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: %tmp1 = bitcast <4 x float> %1 to <2 x double>
+; CHECK-NEXT: %tmp2 = extractelement <2 x double> %tmp1, i32 0
+; CHECK-NEXT: ret double %tmp2
+define double @extract01_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+ %tmp1 = bitcast <4 x float> %tmp to <2 x double>
+ %tmp2 = extractelement <2 x double> %tmp1, i32 0
+ ret double %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract_lo16_0_bitcast_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.buffer.load.format.f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false)
+; CHECK-NEXT: %1 = bitcast float %tmp to i32
+; CHECK-NEXT: %tmp2 = trunc i32 %1 to i16
+; CHECK-NEXT: ret i16 %tmp2
+define i16 @extract_lo16_0_bitcast_buffer_load_format_v4f32(i32 %arg) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32> undef, i32 %arg, i32 16, i1 false, i1 false) #3
+ %tmp1 = bitcast <4 x float> %tmp to <8 x i16>
+ %tmp2 = extractelement <8 x i16> %tmp1, i32 0
+ ret i16 %tmp2
+}
+
+declare float @llvm.amdgcn.buffer.load.format.f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.buffer.load.format.v1f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v2f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 12
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_raw_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+ %tmp2 = extractelement <4 x float> %tmp1, i32 0
+ ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32>, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v2f16(
+; CHECK: %data = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_raw_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x half> %data, i32 0
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v2f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_raw_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v3f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_raw_buffer_load_v3f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x half> @llvm.amdgcn.raw.buffer.load.v3f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x half> %data, i32 1
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_raw_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 6
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt3_raw_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 3
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v4f16(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x half>
+define amdgpu_ps <2 x half> @extract_elt0_elt1_raw_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x half> %data, <4 x half> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x half> %shuf
+}
+
+declare half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32>, i32, i32, i32) #1
+declare <2 x half> @llvm.amdgcn.raw.buffer.load.v2f16(<4 x i32>, i32, i32, i32) #1
+declare <3 x half> @llvm.amdgcn.raw.buffer.load.v3f16(<4 x i32>, i32, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.raw.buffer.load.v4f16(<4 x i32>, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v2i8(
+; CHECK: %data = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt0_raw_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.raw.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x i8> %data, i32 0
+ ret i8 %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v2i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_raw_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.raw.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v3i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_raw_buffer_load_v3i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x i8> @llvm.amdgcn.raw.buffer.load.v3i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x i8> %data, i32 1
+ ret i8 %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_raw_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 3
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt3_raw_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 3
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v4i8(
+; CHECK-NEXT: %data = call <2 x i8> @llvm.amdgcn.raw.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x i8>
+define amdgpu_ps <2 x i8> @extract_elt0_elt1_raw_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x i8> %data, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x i8> %shuf
+}
+
+declare i8 @llvm.amdgcn.raw.buffer.load.i8(<4 x i32>, i32, i32, i32) #1
+declare <2 x i8> @llvm.amdgcn.raw.buffer.load.v2i8(<4 x i32>, i32, i32, i32) #1
+declare <3 x i8> @llvm.amdgcn.raw.buffer.load.v3i8(<4 x i32>, i32, i32, i32) #1
+declare <4 x i8> @llvm.amdgcn.raw.buffer.load.v4i8(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.s.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @s_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @s_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @s_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_s_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v2f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_s_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_s_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 12
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt3_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_s_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_s_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt2_elt3_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_s_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_s_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_s_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_s_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_s_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_s_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_s_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_s_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_s_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; Do not trim to vec3 s_buffer_load in instcombine, as the load will most likely be widened
+; to vec4 anyway during lowering.
+; CHECK-LABEL: @extract_elt1_elt2_elt3_s_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_s_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_s_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_s_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_s_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %tmp = call <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+ %tmp2 = extractelement <4 x float> %tmp1, i32 0
+ ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_s_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_s_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.s.buffer.load.v2f32(<4 x i32>, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.s.buffer.load.v3f32(<4 x i32>, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.s.buffer.load.v4f32(<4 x i32>, i32, i32) #1
+declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_s_buffer_load_v2f16(
+; CHECK: %data = call half @llvm.amdgcn.s.buffer.load.f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_s_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.s.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt0 = extractelement <2 x half> %data, i32 0
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v2f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.s.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_s_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.s.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <2 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v3f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.s.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_s_buffer_load_v3f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x half> @llvm.amdgcn.s.buffer.load.v3f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <3 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.s.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_s_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.s.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 1
+ ret half %elt1
+}
+
+
+; CHECK-LABEL: @extract_elt3_s_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 6
+; CHECK-NEXT: %data = call half @llvm.amdgcn.s.buffer.load.f16(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt3_s_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.s.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 3
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_s_buffer_load_v4f16(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.s.buffer.load.v2f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <2 x half>
+define amdgpu_ps <2 x half> @extract_elt0_elt1_s_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.s.buffer.load.v4f16(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x half> %data, <4 x half> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x half> %shuf
+}
+
+declare half @llvm.amdgcn.s.buffer.load.f16(<4 x i32>, i32, i32) #1
+declare <2 x half> @llvm.amdgcn.s.buffer.load.v2f16(<4 x i32>, i32, i32) #1
+declare <3 x half> @llvm.amdgcn.s.buffer.load.v3f16(<4 x i32>, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.s.buffer.load.v4f16(<4 x i32>, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_s_buffer_load_v2i8(
+; CHECK: %data = call i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt0_s_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.s.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt0 = extractelement <2 x i8> %data, i32 0
+ ret i8 %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v2i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_s_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.s.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <2 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v3i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_s_buffer_load_v3i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <3 x i8> @llvm.amdgcn.s.buffer.load.v3i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <3 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_s_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_s_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_s_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 3
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32> %rsrc, i32 %1, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt3_s_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 3
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_s_buffer_load_v4i8(
+; CHECK-NEXT: %data = call <2 x i8> @llvm.amdgcn.s.buffer.load.v2i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+; CHECK-NEXT: ret <2 x i8>
+define amdgpu_ps <2 x i8> @extract_elt0_elt1_s_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %ofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32> %rsrc, i32 %ofs, i32 0)
+ %shuf = shufflevector <4 x i8> %data, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x i8> %shuf
+}
+
+declare i8 @llvm.amdgcn.s.buffer.load.i8(<4 x i32>, i32, i32) #1
+declare <2 x i8> @llvm.amdgcn.s.buffer.load.v2i8(<4 x i32>, i32, i32) #1
+declare <3 x i8> @llvm.amdgcn.s.buffer.load.v3i8(<4 x i32>, i32, i32) #1
+declare <4 x i8> @llvm.amdgcn.s.buffer.load.v4i8(<4 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4i32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %tmp
+define float @extract0_bitcast_raw_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = extractelement <4 x float> %tmp, i32 0
+ ret float %tmp1
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v2f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 12
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 8
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %1 = add i32 %ofs, 4
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_struct_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+ %tmp2 = extractelement <4 x float> %tmp1, i32 0
+ ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v2f16(
+; CHECK: %data = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_struct_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x half> %data, i32 0
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v2f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_struct_buffer_load_v2f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v3f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_struct_buffer_load_v3f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x half> @llvm.amdgcn.struct.buffer.load.v3f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 2
+; CHECK-NEXT: %data = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_struct_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_v4f16(
+; CHECK-NEXT: %1 = add i32 %ofs, 6
+; CHECK-NEXT: %data = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt3_struct_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 3
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v4f16(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x half>
+define amdgpu_ps <2 x half> @extract_elt0_elt1_struct_buffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x half> %data, <4 x half> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x half> %shuf
+}
+
+declare half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x half> @llvm.amdgcn.struct.buffer.load.v2f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x half> @llvm.amdgcn.struct.buffer.load.v3f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.struct.buffer.load.v4f16(<4 x i32>, i32, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v2i8(
+; CHECK: %data = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt0_struct_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.struct.buffer.load.v2i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x i8> %data, i32 0
+ ret i8 %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v2i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_struct_buffer_load_v2i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x i8> @llvm.amdgcn.struct.buffer.load.v2i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v3i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_struct_buffer_load_v3i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x i8> @llvm.amdgcn.struct.buffer.load.v3i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 1
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt1_struct_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 1
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_v4i8(
+; CHECK-NEXT: %1 = add i32 %ofs, 3
+; CHECK-NEXT: %data = call i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32> %rsrc, i32 %idx, i32 %1, i32 %sofs, i32 0)
+; CHECK-NEXT: ret i8 %data
+define amdgpu_ps i8 @extract_elt3_struct_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x i8> %data, i32 3
+ ret i8 %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v4i8(
+; CHECK-NEXT: %data = call <2 x i8> @llvm.amdgcn.struct.buffer.load.v2i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x i8>
+define amdgpu_ps <2 x i8> @extract_elt0_elt1_struct_buffer_load_v4i8(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x i8> %data, <4 x i8> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x i8> %shuf
+}
+
+declare i8 @llvm.amdgcn.struct.buffer.load.i8(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x i8> @llvm.amdgcn.struct.buffer.load.v2i8(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x i8> @llvm.amdgcn.struct.buffer.load.v3i8(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x i8> @llvm.amdgcn.struct.buffer.load.v4i8(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; CHECK-LABEL: @extract_elt3_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x half> %data, i32 3
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt3_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 3
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x half> %data, i32 2
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt2_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 2
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt1_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x half> %data, i32 1
+; CHECK-NEXT: ret half %elt1
+define amdgpu_ps half @extract_elt1_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 1
+ ret half %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_tbuffer_load_v4f16(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_raw_tbuffer_load_v4f16(<4 x i32> inreg %rsrc, i32 %arg0, i32 inreg %arg1) #0 {
+ %data = call <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 78, i32 0)
+ %elt1 = extractelement <4 x half> %data, i32 0
+ ret half %elt1
+}
+
+declare half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x half> @llvm.amdgcn.raw.tbuffer.load.v2f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.raw.tbuffer.load.v4f16(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ ret float %data
+}
+
+; CHECK-LABEL: @struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1, i32 inreg %arg2) #0 {
+ %data = call <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 %arg2, i32 78, i32 0), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.tbuffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @tbuffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @tbuffer_load_f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ ret float %data
+}
+
+; CHECK-LABEL: @tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ ret <2 x float> %data
+}
+
+; CHECK-LABEL: @tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <2 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <4 x float> %data, i32 3
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_tbuffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt0 = extractelement <3 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <3 x float> %data, i32 1
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %elt1 = extractelement <3 x float> %data, i32 2
+ ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_tbuffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_tbuffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_tbuffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_tbuffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %tmp = call <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false)
+ %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+ %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+ ret i32 %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_tbuffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.tbuffer.load.f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_tbuffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %arg0, i32 %arg1) #0 {
+ %data = call <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32> %rsrc, i32 %arg0, i32 %arg1, i32 0, i32 0, i32 14, i32 4, i1 false, i1 false), !fpmath !0
+ %elt0 = extractelement <2 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare float @llvm.amdgcn.tbuffer.load.f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <1 x float> @llvm.amdgcn.tbuffer.load.v1f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <2 x float> @llvm.amdgcn.tbuffer.load.v2f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <3 x float> @llvm.amdgcn.tbuffer.load.v3f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+declare <4 x float> @llvm.amdgcn.tbuffer.load.v4f32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+
+declare <4 x i32> @llvm.amdgcn.tbuffer.load.v4i32(<4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+ %data.vec = extractvalue {<4 x float>,i32} %data, 0
+ %elt0 = extractelement <4 x float> %data.vec, i32 0
+ ret float %elt0
+}
+
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+ %data.vec = extractvalue {<4 x float>,i32} %data, 0
+ %elt0 = extractelement <4 x float> %data.vec, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32 15, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(
+; CHECK-NEXT: ret float undef
+define amdgpu_ps float @extract_elt0_dmask_0000_image_sample_3d_v4f32_f32(float %s, float %t, float %r, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32 0, float %s, float %t, float %r, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1darray.f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0001_image_sample_1darray_v4f32_f32(float %s, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32 1, float %s, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0010_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 2, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0100_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 4, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_1000_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 8, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_1001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 9, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %1 = insertelement <2 x float> undef, float %data, i32 0
+; CHECK-NEXT: ret <2 x float> %1
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt0_elt1_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %1 = insertelement <3 x float> undef, float %data, i32 0
+; CHECK-NEXT: ret <3 x float> %1
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0001_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <2 x float> %data, <2 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0011_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 3, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.1d.v2f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %shuf = shufflevector <2 x float> %data, <2 x float> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0101_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 5, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_0111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.1d.v3f32.f32(i32 7, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+ ret <3 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_image_sample_cl_2darray_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cl.2darray.f32.f32(i32 2, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_image_sample_cl_2darray_v4f32_f32(float %s, float %t, float %slice, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32 15, float %s, float %t, float %slice, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 1
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cl.2darray.v4f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt2_image_sample_d_cube_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cube.f32.f32.f32(i32 4, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt2_image_sample_d_cube_v4f32_f32_f32(float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32 15, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %face, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 2
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cube.v4f32.f32.f32(i32, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cl.1d.f32.f32.f32(i32 8, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt3_image_sample_d_cl_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 3
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.l.1d.f32.f32(i32 4, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_dmask_0110_image_sample_l_1d_v2f32_f32(float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32 6, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <2 x float> %data, i32 1
+ ret float %elt0
+}
+
+declare <2 x float> @llvm.amdgcn.image.sample.l.1d.v2f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.1d.f32.f32.f32(i32 8, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt1_dmask_1001_image_sample_b_1d_v4f32_f32_f32(float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32 9, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 1
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.1d.v4f32.f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.b.cl.1d.v2f32.f32.f32(i32 12, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt2_dmask_1101_image_sample_b_cl_1d_v4f32_f32_f32(float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32 13, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+ ret <2 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.image.sample.lz.1d.v2f32.f32(i32 10, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @extract_elt1_elt3_image_sample_lz_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 3>
+ ret <2 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.image.sample.cd.1d.v3f32.f32.f32(i32 14, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret <3 x float> %data
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_image_sample_cd_1d_v4f32_f32_f32(float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+ ret <3 x float> %shuf
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 8, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x half> %data, i32 3
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 4, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x half> %data, i32 2
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 2, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x half> %data, i32 1
+ ret half %elt0
+}
+
+; CHECK-LABEL: @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call <3 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v3f16.f32.f32(i32 7, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = shufflevector <3 x half> %data, <3 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to3_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ ret <4 x half> %res
+}
+
+; CHECK-LABEL: @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call <2 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v2f16.f32.f32(i32 3, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = shufflevector <2 x half> %data, <2 x half> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to2_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+ ret <4 x half> %res
+}
+
+; CHECK-LABEL: @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: %res = insertelement <4 x half> undef, half %data, i64 0
+; CHECK-NEXT: ret <4 x half> %res
+define amdgpu_ps <4 x half> @extract_elt_to1_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %res = shufflevector <4 x half> %data, <4 x half> undef, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+ ret <4 x half> %res
+}
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(
+; CHECK-NEXT: %data = call half @llvm.amdgcn.image.sample.cd.cl.1d.f16.f32.f32(i32 1, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret half %data
+define amdgpu_ps half @extract_elt0_image_sample_cd_cl_1d_v4f16_f32_f32(float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32 15, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x half> %data, i32 0
+ ret half %elt0
+}
+
+declare <4 x half> @llvm.amdgcn.image.sample.cd.cl.1d.v4f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.1d.f32.f32(i32 1, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cl_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cl.1d.f32.f32(i32 1, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cl_1d_v4f32_f32(float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.cl.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_l_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.l.1d.f32.f32(i32 1, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_l_1d_v4f32_f32(float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32 15, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.1d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.1d.f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.1d.v4f32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.cl.1d.f32.f32.f32(i32 1, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_1d_v4f32_f32_f32(float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32 15, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_lz_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.lz.1d.f32.f32(i32 1, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_lz_1d_v4f32_f32(float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32 15, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.1d.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.cl.1d.f32.f32.f32(i32 1, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_1d_v4f32_f32_f32(float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32 15, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.1d.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.o.1d.f32.f32(i32 1, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cl_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cl.o.1d.f32.f32(i32 1, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cl_o_1d_v4f32_f32(i32 %offset, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cl.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_d_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.d.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.d.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_l_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.l.o.1d.f32.f32(i32 1, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_l_o_1d_v4f32_f32(i32 %offset, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.l.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.o.1d.v4f32.f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.b.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_lz_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.lz.o.1d.f32.f32(i32 1, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_lz_o_1d_v4f32_f32(i32 %offset, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.lz.o.1d.v4f32.f32(i32, i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cd.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cd_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.cd.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.cd.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cl.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cl_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cl.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.d.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.d.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_d_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.d.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.l.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_l_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.l.o.1d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.o.1d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.b.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_b_cl_o_1d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %bias, float %zcompare, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.b.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.lz.o.1d.f32.f32(i32 1, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_lz_o_1d_v4f32_f32(i32 %offset, float %zcompare, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32 15, i32 %offset, float %zcompare, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.lz.o.1d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.sample.c.cd.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.c.cd.cl.o.1d.f32.f32.f32(i32 1, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_sample_c_cd_cl_o_1d_v4f32_f32_f32(i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32 15, i32 %offset, float %zcompare, float %dsdh, float %dsdv, float %s, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.c.cd.cl.o.1d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4
+; --------------------------------------------------------------------
+
+; Don't handle gather4*
+
+; CHECK-LABEL: @extract_elt0_image_gather4_2d_v4f32_f32(
+; CHECK: %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_2d_v4f32_f32(float %s, float %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_cl_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_cl_2d_v4f32_f32(float %s, float %t, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32 2, float %s, float %t, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.l
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_l_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_l_2d_v4f32_f32(float %s, float %t, float %lod, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32 4, float %s, float %t, float %lod, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_2darray_v4f32_f32_f32(float %bias, float %s, float %t, float %slice, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32 8, float %bias, float %s, float %t, float %slice, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.2darray.v4f32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.cl
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_cl_cube_v4f32_f32_f32(float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32 1, float %bias, float %s, float %t, float %face, float %clamp, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.cube.v4f32.f32.f32(i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.lz
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_lz_2d_v4f32_f16(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_lz_2d_v4f32_f16(half %s, half %t, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32 1, half %s, half %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.2d.v4f32.f16(i32, half, half, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_cl_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.cl.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_l_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_l_o_2d_v4f32_f32(i32 %offset, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.l.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_lz_o_2d_v4f32_f32(i32 %offset, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.lz.o.2d.v4f32.f32(i32, i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_cl_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.cl.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.l.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_l_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, float %lod, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.l.o.2d.v4f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.b.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_b_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.b.cl.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_b_cl_o_2d_v4f32_f32_f32(i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32 1, i32 %offset, float %bias, float %zcompare, float %s, float %t, float %clamp, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.b.cl.o.2d.v4f32.f32.f32(i32, i32, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.gather4.c.lz.o
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+define amdgpu_ps float @extract_elt0_image_gather4_c_lz_o_2d_v4f32_f32(i32 %offset, float %zcompare, float %s, float %t, <8 x i32> inreg %gather4r, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32 1, i32 %offset, float %zcompare, float %s, float %t, <8 x i32> %gather4r, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.o.2d.v4f32.f32(i32, i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.getlod
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_getlod_1d_v4f32_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.getlod.1d.f32.f32(i32 1, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_getlod_1d_v4f32_f32(float %s, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32 15, float %s, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.getlod.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_load_2dmsaa_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.load.2dmsaa.f32.i32(i32 1, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_load_2dmsaa_v4f32_i32(i32 %s, i32 %t, i32 %sample, <8 x i32> inreg %sampler) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32 15, i32 %s, i32 %t, i32 %sample, <8 x i32> %sampler, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.load.mip
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_load_mip_1d_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.load.mip.1d.f32.i32(i32 1, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_load_mip_1d_v4f32_i32(i32 %s, i32 %mip, <8 x i32> inreg %sampler) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.image.getresinfo
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_image_getresinfo_1d_v4f32_i32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.image.getresinfo.1d.f32.i32(i32 1, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_image_getresinfo_1d_v4f32_i32(i32 %mip, <8 x i32> inreg %sampler) #0 {
+ %data = call <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32 15, i32 %mip, <8 x i32> %sampler, i32 0, i32 0)
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+
+; --------------------------------------------------------------------
+; TFE / LWE
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_tfe_image_load_1d_v4f32i32_i32(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+define amdgpu_ps float @extract_elt0_tfe_image_load_1d_v4f32i32_i32(i32 %s, <8 x i32> inreg %rsrc) #0 {
+ %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+ %rgba = extractvalue { <4 x float>, i32 } %data, 0
+ %elt0 = extractelement <4 x float> %rgba, i32 0
+ ret float %elt0
+}
+
+declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32, i32, <8 x i32>, i32, i32) #1
+
+; CHECK: @tfe_check_assert(
+; CHECK: %data = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1)
+; CHECK-NEXT: ret float %data
+define amdgpu_hs float @tfe_check_assert() #0 {
+ %data = call nsz <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 15, i32 undef, i32 undef, <8 x i32> undef, i32 0, i32 1) #2
+ %elt0 = extractelement <4 x float> %data, i32 0
+ ret float %elt0
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32 immarg, i32, i32, <8 x i32>, i32 immarg, i32 immarg) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
+
+!0 = !{float 2.500000e+00}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
new file mode 100644
index 000000000000..d279995f3768
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-addsub-inseltpoison.ll
@@ -0,0 +1,194 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>)
+declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>)
+declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>)
+declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>)
+declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8 immarg) #0
+
+;
+; Demanded Elts
+;
+
+define double @elts_addsub_v2f64(<2 x double> %0, <2 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v2f64(
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP1:%.*]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT: [[TMP4:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT: ret double [[TMP5]]
+;
+ %3 = shufflevector <2 x double> %0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+ %4 = shufflevector <2 x double> %1, <2 x double> undef, <2 x i32> <i32 1, i32 1>
+ %5 = tail call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %3, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @elts_addsub_v2f64_sub(<2 x double> %0, <2 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v2f64_sub(
+; CHECK-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT: ret double [[TMP4]]
+;
+ %3 = shufflevector <2 x double> %0, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+ %4 = shufflevector <2 x double> %1, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+ %5 = tail call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %3, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define float @elts_addsub_v4f32(<4 x float> %0, <4 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v4f32(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP0:%.*]], <4 x float> [[TMP1:%.*]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret float [[TMP6]]
+;
+ %3 = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %4 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %5 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %3, <4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 0
+ %7 = extractelement <4 x float> %5, i32 1
+ %8 = fadd float %6, %7
+ ret float %8
+}
+
+define float @elts_addsub_v4f32_add(<4 x float> %0, <4 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v4f32_add(
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret float [[TMP6]]
+;
+ %3 = shufflevector <4 x float> %0, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %4 = shufflevector <4 x float> %1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %5 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %3, <4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 1
+ %7 = extractelement <4 x float> %5, i32 3
+ %8 = fadd float %6, %7
+ ret float %8
+}
+
+define double @elts_addsub_v4f64(<4 x double> %0, <4 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v4f64(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> [[TMP0:%.*]], <4 x double> [[TMP1:%.*]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret double [[TMP6]]
+;
+ %3 = shufflevector <4 x double> %0, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+ %4 = shufflevector <4 x double> %1, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+ %5 = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %3, <4 x double> %4)
+ %6 = extractelement <4 x double> %5, i32 0
+ %7 = extractelement <4 x double> %5, i32 1
+ %8 = fadd double %6, %7
+ ret double %8
+}
+
+define double @elts_addsub_v4f64_add(<4 x double> %0, <4 x double> %1) {
+; CHECK-LABEL: @elts_addsub_v4f64_add(
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x double> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP3]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret double [[TMP6]]
+;
+ %3 = shufflevector <4 x double> %0, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+ %4 = shufflevector <4 x double> %1, <4 x double> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
+ %5 = tail call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %3, <4 x double> %4)
+ %6 = extractelement <4 x double> %5, i32 1
+ %7 = extractelement <4 x double> %5, i32 3
+ %8 = fadd double %6, %7
+ ret double %8
+}
+
+define float @elts_addsub_v8f32(<8 x float> %0, <8 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v8f32(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> [[TMP0:%.*]], <8 x float> [[TMP1:%.*]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret float [[TMP6]]
+;
+ %3 = shufflevector <8 x float> %0, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+ %4 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+ %5 = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %3, <8 x float> %4)
+ %6 = extractelement <8 x float> %5, i32 0
+ %7 = extractelement <8 x float> %5, i32 1
+ %8 = fadd float %6, %7
+ ret float %8
+}
+
+define float @elts_addsub_v8f32_sub(<8 x float> %0, <8 x float> %1) {
+; CHECK-LABEL: @elts_addsub_v8f32_sub(
+; CHECK-NEXT: [[TMP3:%.*]] = fsub <8 x float> [[TMP0:%.*]], [[TMP1:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP3]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[TMP4]], [[TMP5]]
+; CHECK-NEXT: ret float [[TMP6]]
+;
+ %3 = shufflevector <8 x float> %0, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+ %4 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 4, i32 4, i32 4>
+ %5 = tail call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %3, <8 x float> %4)
+ %6 = extractelement <8 x float> %5, i32 0
+ %7 = extractelement <8 x float> %5, i32 4
+ %8 = fadd float %6, %7
+ ret float %8
+}
+
+define void @PR46277(float %0, float %1, float %2, float %3, <4 x float> %4, float* %5) {
+; CHECK-LABEL: @PR46277(
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[TMP0:%.*]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP1:%.*]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> [[TMP8]], <4 x float> [[TMP4:%.*]])
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, float* [[TMP5:%.*]], i64 1
+; CHECK-NEXT: store float [[TMP10]], float* [[TMP5]], align 4
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i32 1
+; CHECK-NEXT: store float [[TMP12]], float* [[TMP11]], align 4
+; CHECK-NEXT: ret void
+;
+ %7 = insertelement <4 x float> poison, float %0, i32 0
+ %8 = insertelement <4 x float> %7, float %1, i32 1
+ %9 = insertelement <4 x float> %8, float %2, i32 2
+ %10 = insertelement <4 x float> %9, float %3, i32 3
+ %11 = tail call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %10, <4 x float> %4)
+ %12 = extractelement <4 x float> %11, i32 0
+ %13 = getelementptr inbounds float, float* %5, i64 1
+ store float %12, float* %5, align 4
+ %14 = extractelement <4 x float> %11, i32 1
+ store float %14, float* %13, align 4
+ ret void
+}
+
+define double @PR48476_fsub(<2 x double> %x) {
+; CHECK-LABEL: @PR48476_fsub(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <2 x double> <double 0.000000e+00, double undef>, [[X:%.*]]
+; CHECK-NEXT: [[T2:%.*]] = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[X]], i8 6)
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <2 x double> [[T2]], i32 0
+; CHECK-NEXT: ret double [[VECEXT]]
+;
+ %t1 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> zeroinitializer, <2 x double> %x)
+ %t2 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %t1, <2 x double> %x, i8 6)
+ %vecext = extractelement <2 x double> %t2, i32 0
+ ret double %vecext
+}
+
+define double @PR48476_fadd_fsub(<2 x double> %x) {
+; CHECK-LABEL: @PR48476_fadd_fsub(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], <double undef, double 0.000000e+00>
+; CHECK-NEXT: [[S:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = fsub <2 x double> [[S]], [[X]]
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[VECEXT]]
+;
+ %t1 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> zeroinitializer, <2 x double> %x)
+ %s = shufflevector <2 x double> %t1, <2 x double> undef, <2 x i32> <i32 1, i32 0>
+ %t2 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %s, <2 x double> %x)
+ %vecext = extractelement <2 x double> %t2, i32 0
+ ret double %vecext
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
new file mode 100644
index 000000000000..c7a3e01186b2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-avx512-inseltpoison.ll
@@ -0,0 +1,3407 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_add_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP4]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_add_ss_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_add_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_ss_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+ ret <4 x float> %4
+}
+
+define float @test_add_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_add_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_add_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fadd double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP4]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_add_sd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fadd double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_add_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_sd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+ ret <2 x double> %2
+}
+
+define double @test_add_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_sub_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP4]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_sub_ss_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_sub_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_ss_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+ ret <4 x float> %4
+}
+
+define float @test_sub_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_sub_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_sub_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fsub double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP4]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_sub_sd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fsub double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_sub_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_sd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+ ret <2 x double> %2
+}
+
+define double @test_sub_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.sub.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_mul_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP4]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_mul_ss_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_mul_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_ss_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+ ret <4 x float> %4
+}
+
+define float @test_mul_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_mul_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_mul_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP4]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_mul_sd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_mul_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_sd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+ ret <2 x double> %2
+}
+
+define double @test_mul_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.mul.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_div_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP4]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_round(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_div_ss_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 8)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_div_ss_mask_round(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_ss_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 8)
+ ret <4 x float> %4
+}
+
+define float @test_div_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_div_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_div_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[A]], double [[TMP3]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP4]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_round(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_div_sd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 8)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = fdiv double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP5]], double [[TMP3]], double [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_div_sd_mask_round(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_sd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 8)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 8)
+ ret <2 x double> %2
+}
+
+define double @test_div_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.div.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_max_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_max_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_max_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define float @test_max_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_max_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_max_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_max_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define double @test_max_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32)
+
+define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_min_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> undef, i8 -1, i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %a, <4 x float> %3, <4 x float> undef, i8 -1, i32 4)
+ ret <4 x float> %4
+}
+
+define <4 x float> @test_min_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_min_ss_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %a, <4 x float> %b, <4 x float> %3, i8 %mask, i32 4)
+ ret <4 x float> %4
+}
+
+define float @test_min_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.avx512.mask.min.ss.round(<4 x float> %4, <4 x float> %8, <4 x float> undef, i8 -1, i32 8)
+ %10 = extractelement <4 x float> %9, i32 1
+ ret float %10
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32)
+
+define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_min_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> undef, i8 -1, i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %a, <2 x double> %1, <2 x double> undef, i8 -1, i32 4)
+ ret <2 x double> %2
+}
+
+define <2 x double> @test_min_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_min_sd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x double> [[C:%.*]], i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %a, <2 x double> %b, <2 x double> %1, i8 %mask, i32 4)
+ ret <2 x double> %2
+}
+
+define double @test_min_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.avx512.mask.min.sd.round(<2 x double> %2, <2 x double> %4, <2 x double> undef, i8 -1, i32 8)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float>, <4 x float>, i32, i8, i32)
+
+define i8 @test_cmp_ss(<4 x float> %a, <4 x float> %b, i8 %mask) {
+; CHECK-LABEL: @test_cmp_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i32 3, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = tail call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %3, <4 x float> %6, i32 3, i8 %mask, i32 4)
+ ret i8 %7
+}
+
+declare i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double>, <2 x double>, i32, i8, i32)
+
+define i8 @test_cmp_sd(<2 x double> %a, <2 x double> %b, i8 %mask) {
+; CHECK-LABEL: @test_cmp_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 3, i8 [[MASK:%.*]], i32 4)
+; CHECK-NEXT: ret i8 [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %3 = tail call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %1, <2 x double> %2, i32 3, i8 %mask, i32 4)
+ ret i8 %3
+}
+
+define i64 @test(float %f, double %d) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
+; CHECK-NEXT: [[T0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT: [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT: [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT: [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT: [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
+; CHECK-NEXT: [[T4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT: [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT: [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT: [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT: [[T8:%.*]] = add i32 [[T0]], [[T2]]
+; CHECK-NEXT: [[T9:%.*]] = add i32 [[T4]], [[T6]]
+; CHECK-NEXT: [[T10:%.*]] = add i32 [[T8]], [[T9]]
+; CHECK-NEXT: [[T11:%.*]] = sext i32 [[T10]] to i64
+; CHECK-NEXT: [[T12:%.*]] = add i64 [[T1]], [[T3]]
+; CHECK-NEXT: [[T13:%.*]] = add i64 [[T5]], [[T7]]
+; CHECK-NEXT: [[T14:%.*]] = add i64 [[T12]], [[T13]]
+; CHECK-NEXT: [[T15:%.*]] = add i64 [[T14]], [[T11]]
+; CHECK-NEXT: ret i64 [[T15]]
+;
+ %v00 = insertelement <4 x float> poison, float %f, i32 0
+ %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+ %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+ %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+ %t0 = tail call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %v03, i32 4)
+ %v10 = insertelement <4 x float> poison, float %f, i32 0
+ %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+ %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+ %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+ %t1 = tail call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %v13, i32 4)
+ %v20 = insertelement <4 x float> poison, float %f, i32 0
+ %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+ %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+ %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+ %t2 = tail call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %v23, i32 4)
+ %v30 = insertelement <4 x float> poison, float %f, i32 0
+ %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+ %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+ %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+ %t3 = tail call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %v33, i32 4)
+ %v40 = insertelement <2 x double> poison, double %d, i32 0
+ %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+ %t4 = tail call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %v41, i32 4)
+ %v50 = insertelement <2 x double> poison, double %d, i32 0
+ %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+ %t5 = tail call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %v51, i32 4)
+ %v60 = insertelement <2 x double> poison, double %d, i32 0
+ %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+ %t6 = tail call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %v61, i32 4)
+ %v70 = insertelement <2 x double> poison, double %d, i32 0
+ %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+ %t7 = tail call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %v71, i32 4)
+ %t8 = add i32 %t0, %t2
+ %t9 = add i32 %t4, %t6
+ %t10 = add i32 %t8, %t9
+ %t11 = sext i32 %t10 to i64
+ %t12 = add i64 %t1, %t3
+ %t13 = add i64 %t5, %t7
+ %t14 = add i64 %t12, %t13
+ %t15 = add i64 %t11, %t14
+ ret i64 %t15
+}
+
+declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.vcvtss2si64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double>, i32)
+declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32)
+
+define i64 @test2(float %f, double %d) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[V03:%.*]] = insertelement <4 x float> undef, float [[F:%.*]], i32 0
+; CHECK-NEXT: [[T0:%.*]] = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> [[V03]], i32 4)
+; CHECK-NEXT: [[V13:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> [[V13]], i32 4)
+; CHECK-NEXT: [[V23:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T2:%.*]] = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> [[V23]], i32 4)
+; CHECK-NEXT: [[V33:%.*]] = insertelement <4 x float> undef, float [[F]], i32 0
+; CHECK-NEXT: [[T3:%.*]] = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> [[V33]], i32 4)
+; CHECK-NEXT: [[V41:%.*]] = insertelement <2 x double> undef, double [[D:%.*]], i32 0
+; CHECK-NEXT: [[T4:%.*]] = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> [[V41]], i32 4)
+; CHECK-NEXT: [[V51:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T5:%.*]] = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> [[V51]], i32 4)
+; CHECK-NEXT: [[V61:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> [[V61]], i32 4)
+; CHECK-NEXT: [[V71:%.*]] = insertelement <2 x double> undef, double [[D]], i32 0
+; CHECK-NEXT: [[T7:%.*]] = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> [[V71]], i32 4)
+; CHECK-NEXT: [[T8:%.*]] = add i32 [[T0]], [[T2]]
+; CHECK-NEXT: [[T9:%.*]] = add i32 [[T4]], [[T6]]
+; CHECK-NEXT: [[T10:%.*]] = add i32 [[T8]], [[T9]]
+; CHECK-NEXT: [[T11:%.*]] = sext i32 [[T10]] to i64
+; CHECK-NEXT: [[T12:%.*]] = add i64 [[T1]], [[T3]]
+; CHECK-NEXT: [[T13:%.*]] = add i64 [[T5]], [[T7]]
+; CHECK-NEXT: [[T14:%.*]] = add i64 [[T12]], [[T13]]
+; CHECK-NEXT: [[T15:%.*]] = add i64 [[T14]], [[T11]]
+; CHECK-NEXT: ret i64 [[T15]]
+;
+ %v00 = insertelement <4 x float> poison, float %f, i32 0
+ %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+ %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+ %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+ %t0 = tail call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %v03, i32 4)
+ %v10 = insertelement <4 x float> poison, float %f, i32 0
+ %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+ %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+ %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+ %t1 = tail call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %v13, i32 4)
+ %v20 = insertelement <4 x float> poison, float %f, i32 0
+ %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+ %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+ %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+ %t2 = tail call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %v23, i32 4)
+ %v30 = insertelement <4 x float> poison, float %f, i32 0
+ %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+ %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+ %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+ %t3 = tail call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %v33, i32 4)
+ %v40 = insertelement <2 x double> poison, double %d, i32 0
+ %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+ %t4 = tail call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %v41, i32 4)
+ %v50 = insertelement <2 x double> poison, double %d, i32 0
+ %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+ %t5 = tail call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %v51, i32 4)
+ %v60 = insertelement <2 x double> poison, double %d, i32 0
+ %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+ %t6 = tail call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %v61, i32 4)
+ %v70 = insertelement <2 x double> poison, double %d, i32 0
+ %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+ %t7 = tail call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %v71, i32 4)
+ %t8 = add i32 %t0, %t2
+ %t9 = add i32 %t4, %t6
+ %t10 = add i32 %t8, %t9
+ %t11 = sext i32 %t10 to i64
+ %t12 = add i64 %t1, %t3
+ %t13 = add i64 %t5, %t7
+ %t14 = add i64 %t12, %t13
+ %t15 = add i64 %t11, %t14
+ ret i64 %t15
+}
+
+declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32)
+declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32)
+declare i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32)
+declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32)
+declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32)
+
+declare float @llvm.fma.f32(float, float, float) #1
+
+define <4 x float> @test_mask_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = extractelement <4 x float> %a, i64 0
+ %8 = extractelement <4 x float> %3, i64 0
+ %9 = extractelement <4 x float> %6, i64 0
+ %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %10, float %7
+ %14 = insertelement <4 x float> %a, float %13, i64 0
+ ret <4 x float> %14
+}
+
+define float @test_mask_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP1]]
+; CHECK-NEXT: ret float [[TMP7]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %3, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %c, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float %4
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 0
+ ret float %12
+}
+
+define float @test_mask_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %3, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %c, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float %4
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 1
+ ret float %12
+}
+
+declare double @llvm.fma.f64(double, double, double) #1
+
+define <2 x double> @test_mask_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP1]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+ %3 = extractelement <2 x double> %a, i64 0
+ %4 = extractelement <2 x double> %1, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = bitcast i8 %mask to <8 x i1>
+ %8 = extractelement <8 x i1> %7, i64 0
+ %9 = select i1 %8, double %6, double %3
+ %10 = insertelement <2 x double> %a, double %9, i64 0
+ ret <2 x double> %10
+}
+
+define double @test_mask_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP1]]
+; CHECK-NEXT: ret double [[TMP7]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %1, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %c, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double %2
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 0
+ ret double %10
+}
+
+define double @test_mask_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %1, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %c, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double %2
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 1
+ ret double %10
+}
+
+define <4 x float> @test_maskz_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float 0.000000e+00
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %c, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = extractelement <4 x float> %a, i64 0
+ %8 = extractelement <4 x float> %3, i64 0
+ %9 = extractelement <4 x float> %6, i64 0
+ %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %10, float 0.000000e+00
+ %14 = insertelement <4 x float> %a, float %13, i64 0
+ ret <4 x float> %14
+}
+
+define float @test_maskz_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float 0.000000e+00
+; CHECK-NEXT: ret float [[TMP7]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %3, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %c, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float 0.000000e+00
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 0
+ ret float %12
+}
+
+define float @test_maskz_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %3, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %c, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float 0.000000e+00
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 1
+ ret float %12
+}
+
+define <2 x double> @test_maskz_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double 0.000000e+00
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[A]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %b, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %c, double 2.000000e+00, i32 1
+ %3 = extractelement <2 x double> %a, i64 0
+ %4 = extractelement <2 x double> %1, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = bitcast i8 %mask to <8 x i1>
+ %8 = extractelement <8 x i1> %7, i64 0
+ %9 = select i1 %8, double %6, double 0.000000e+00
+ %10 = insertelement <2 x double> %a, double %9, i64 0
+ ret <2 x double> %10
+}
+
+define double @test_maskz_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double 0.000000e+00
+; CHECK-NEXT: ret double [[TMP7]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %1, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %c, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double 0.000000e+00
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 0
+ ret double %10
+}
+
+define double @test_maskz_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_maskz_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %1, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %c, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double 0.000000e+00
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 1
+ ret double %10
+}
+
+define <4 x float> @test_mask3_vfmadd_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[C]], float [[TMP7]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP8]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = extractelement <4 x float> %3, i64 0
+ %8 = extractelement <4 x float> %6, i64 0
+ %9 = extractelement <4 x float> %c, i64 0
+ %10 = call float @llvm.fma.f32(float %7, float %8, float %9)
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %10, float %9
+ %14 = insertelement <4 x float> %c, float %13, i64 0
+ ret <4 x float> %14
+}
+
+define float @test_mask3_vfmadd_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], float [[TMP4]], float [[TMP3]]
+; CHECK-NEXT: ret float [[TMP7]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %a, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %3, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float %6
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 0
+ ret float %12
+}
+
+define float @test_mask3_vfmadd_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = extractelement <4 x float> %a, i64 0
+ %5 = extractelement <4 x float> %b, i64 0
+ %6 = extractelement <4 x float> %3, i64 0
+ %7 = call float @llvm.fma.f32(float %4, float %5, float %6)
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, float %7, float %6
+ %11 = insertelement <4 x float> %3, float %10, i64 0
+ %12 = extractelement <4 x float> %11, i32 1
+ ret float %12
+}
+
+define <2 x double> @test_mask3_vfmadd_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP3]]
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x double> [[C]], double [[TMP7]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP8]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %3 = extractelement <2 x double> %1, i64 0
+ %4 = extractelement <2 x double> %2, i64 0
+ %5 = extractelement <2 x double> %c, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = bitcast i8 %mask to <8 x i1>
+ %8 = extractelement <8 x i1> %7, i64 0
+ %9 = select i1 %8, double %6, double %5
+ %10 = insertelement <2 x double> %c, double %9, i64 0
+ ret <2 x double> %10
+}
+
+define double @test_mask3_vfmadd_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i1> [[TMP5]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP6]], double [[TMP4]], double [[TMP3]]
+; CHECK-NEXT: ret double [[TMP7]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %a, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %1, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double %4
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 0
+ ret double %10
+}
+
+define double @test_mask3_vfmadd_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmadd_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = extractelement <2 x double> %a, i64 0
+ %3 = extractelement <2 x double> %b, i64 0
+ %4 = extractelement <2 x double> %1, i64 0
+ %5 = call double @llvm.fma.f64(double %2, double %3, double %4)
+ %6 = bitcast i8 %mask to <8 x i1>
+ %7 = extractelement <8 x i1> %6, i64 0
+ %8 = select i1 %7, double %5, double %4
+ %9 = insertelement <2 x double> %1, double %8, i64 0
+ %10 = extractelement <2 x double> %9, i32 1
+ ret double %10
+}
+
+define <4 x float> @test_mask3_vfmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[C]], float [[TMP9]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP10]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+ %8 = extractelement <4 x float> %3, i64 0
+ %9 = extractelement <4 x float> %6, i64 0
+ %10 = extractelement <4 x float> %7, i64 0
+ %11 = call float @llvm.fma.f32(float %8, float %9, float %10)
+ %12 = extractelement <4 x float> %c, i64 0
+ %13 = bitcast i8 %mask to <8 x i1>
+ %14 = extractelement <8 x i1> %13, i64 0
+ %15 = select i1 %14, float %11, float %12
+ %16 = insertelement <4 x float> %c, float %15, i64 0
+ ret <4 x float> %16
+}
+
+define float @test_mask3_vfmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT: ret float [[TMP9]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+ %5 = extractelement <4 x float> %a, i64 0
+ %6 = extractelement <4 x float> %b, i64 0
+ %7 = extractelement <4 x float> %4, i64 0
+ %8 = call float @llvm.fma.f32(float %5, float %6, float %7)
+ %9 = extractelement <4 x float> %3, i64 0
+ %10 = bitcast i8 %mask to <8 x i1>
+ %11 = extractelement <8 x i1> %10, i64 0
+ %12 = select i1 %11, float %8, float %9
+ %13 = insertelement <4 x float> %3, float %12, i64 0
+ %14 = extractelement <4 x float> %13, i32 0
+ ret float %14
+}
+
+define float @test_mask3_vfmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+ %5 = extractelement <4 x float> %a, i64 0
+ %6 = extractelement <4 x float> %b, i64 0
+ %7 = extractelement <4 x float> %4, i64 0
+ %8 = call float @llvm.fma.f32(float %5, float %6, float %7)
+ %9 = extractelement <4 x float> %3, i64 0
+ %10 = bitcast i8 %mask to <8 x i1>
+ %11 = extractelement <8 x i1> %10, i64 0
+ %12 = select i1 %11, float %8, float %9
+ %13 = insertelement <4 x float> %3, float %12, i64 0
+ %14 = extractelement <4 x float> %13, i32 1
+ ret float %14
+}
+
+define float @test_mask3_vfmsub_ss_1_unary_fneg(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_ss_1_unary_fneg(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fneg <4 x float> %3
+ %5 = extractelement <4 x float> %a, i64 0
+ %6 = extractelement <4 x float> %b, i64 0
+ %7 = extractelement <4 x float> %4, i64 0
+ %8 = call float @llvm.fma.f32(float %5, float %6, float %7)
+ %9 = extractelement <4 x float> %3, i64 0
+ %10 = bitcast i8 %mask to <8 x i1>
+ %11 = extractelement <8 x i1> %10, i64 0
+ %12 = select i1 %11, float %8, float %9
+ %13 = insertelement <4 x float> %3, float %12, i64 0
+ %14 = extractelement <4 x float> %13, i32 1
+ ret float %14
+}
+
+define <2 x double> @test_mask3_vfmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = fneg double [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x double> [[C]], double [[TMP9]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP10]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
+ %4 = extractelement <2 x double> %1, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = extractelement <2 x double> %3, i64 0
+ %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+ %8 = extractelement <2 x double> %c, i64 0
+ %9 = bitcast i8 %mask to <8 x i1>
+ %10 = extractelement <8 x i1> %9, i64 0
+ %11 = select i1 %10, double %7, double %8
+ %12 = insertelement <2 x double> %c, double %11, i64 0
+ ret <2 x double> %12
+}
+
+define double @test_mask3_vfmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = fneg double [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT: ret double [[TMP9]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+ %3 = extractelement <2 x double> %a, i64 0
+ %4 = extractelement <2 x double> %b, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = extractelement <2 x double> %1, i64 0
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, double %6, double %7
+ %11 = insertelement <2 x double> %1, double %10, i64 0
+ %12 = extractelement <2 x double> %11, i32 0
+ ret double %12
+}
+
+define double @test_mask3_vfmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+ %3 = extractelement <2 x double> %a, i64 0
+ %4 = extractelement <2 x double> %b, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = extractelement <2 x double> %1, i64 0
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, double %6, double %7
+ %11 = insertelement <2 x double> %1, double %10, i64 0
+ %12 = extractelement <2 x double> %11, i32 1
+ ret double %12
+}
+
+define double @test_mask3_vfmsub_sd_1_unary_fneg(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfmsub_sd_1_unary_fneg(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fneg <2 x double> %1
+ %3 = extractelement <2 x double> %a, i64 0
+ %4 = extractelement <2 x double> %b, i64 0
+ %5 = extractelement <2 x double> %2, i64 0
+ %6 = call double @llvm.fma.f64(double %3, double %4, double %5)
+ %7 = extractelement <2 x double> %1, i64 0
+ %8 = bitcast i8 %mask to <8 x i1>
+ %9 = extractelement <8 x i1> %8, i64 0
+ %10 = select i1 %9, double %6, double %7
+ %11 = insertelement <2 x double> %1, double %10, i64 0
+ %12 = extractelement <2 x double> %11, i32 1
+ ret double %12
+}
+
+define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fneg float [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[C]], float [[TMP10]], i64 0
+; CHECK-NEXT: ret <4 x float> [[TMP11]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 4.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 5.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 6.000000e+00, i32 3
+ %7 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+ %8 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %c
+ %9 = extractelement <4 x float> %7, i64 0
+ %10 = extractelement <4 x float> %6, i64 0
+ %11 = extractelement <4 x float> %8, i64 0
+ %12 = call float @llvm.fma.f32(float %9, float %10, float %11)
+ %13 = extractelement <4 x float> %c, i64 0
+ %14 = bitcast i8 %mask to <8 x i1>
+ %15 = extractelement <8 x i1> %14, i64 0
+ %16 = select i1 %15, float %12, float %13
+ %17 = insertelement <4 x float> %c, float %16, i64 0
+ ret <4 x float> %17
+}
+
+define float @test_mask3_vfnmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = fneg float [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = fneg float [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT: ret float [[TMP10]]
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
+ %5 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+ %6 = extractelement <4 x float> %4, i64 0
+ %7 = extractelement <4 x float> %b, i64 0
+ %8 = extractelement <4 x float> %5, i64 0
+ %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
+ %10 = extractelement <4 x float> %3, i64 0
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %9, float %10
+ %14 = insertelement <4 x float> %3, float %13, i64 0
+ %15 = extractelement <4 x float> %14, i32 0
+ ret float %15
+}
+
+define float @test_mask3_vfnmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
+ %5 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %3
+ %6 = extractelement <4 x float> %4, i64 0
+ %7 = extractelement <4 x float> %b, i64 0
+ %8 = extractelement <4 x float> %5, i64 0
+ %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
+ %10 = extractelement <4 x float> %3, i64 0
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %9, float %10
+ %14 = insertelement <4 x float> %3, float %13, i64 0
+ %15 = extractelement <4 x float> %14, i32 1
+ ret float %15
+}
+
+define float @test_mask3_vfnmsub_ss_1_unary_fneg(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_ss_1_unary_fneg(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = fneg <4 x float> %a
+ %5 = fneg <4 x float> %3
+ %6 = extractelement <4 x float> %4, i64 0
+ %7 = extractelement <4 x float> %b, i64 0
+ %8 = extractelement <4 x float> %5, i64 0
+ %9 = call float @llvm.fma.f32(float %6, float %7, float %8)
+ %10 = extractelement <4 x float> %3, i64 0
+ %11 = bitcast i8 %mask to <8 x i1>
+ %12 = extractelement <8 x i1> %11, i64 0
+ %13 = select i1 %12, float %9, float %10
+ %14 = insertelement <4 x float> %3, float %13, i64 0
+ %15 = extractelement <4 x float> %14, i32 1
+ ret float %15
+}
+
+define <2 x double> @test_mask3_vfnmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = fneg double [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fneg double [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <2 x double> [[C]], double [[TMP10]], i64 0
+; CHECK-NEXT: ret <2 x double> [[TMP11]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+ %4 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %c
+ %5 = extractelement <2 x double> %3, i64 0
+ %6 = extractelement <2 x double> %2, i64 0
+ %7 = extractelement <2 x double> %4, i64 0
+ %8 = call double @llvm.fma.f64(double %5, double %6, double %7)
+ %9 = extractelement <2 x double> %c, i64 0
+ %10 = bitcast i8 %mask to <8 x i1>
+ %11 = extractelement <8 x i1> %10, i64 0
+ %12 = select i1 %11, double %8, double %9
+ %13 = insertelement <2 x double> %c, double %12, i64 0
+ ret <2 x double> %13
+}
+
+define double @test_mask3_vfnmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = fneg double [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = fneg double [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT: ret double [[TMP10]]
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a
+ %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+ %4 = extractelement <2 x double> %2, i64 0
+ %5 = extractelement <2 x double> %b, i64 0
+ %6 = extractelement <2 x double> %3, i64 0
+ %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+ %8 = extractelement <2 x double> %1, i64 0
+ %9 = bitcast i8 %mask to <8 x i1>
+ %10 = extractelement <8 x i1> %9, i64 0
+ %11 = select i1 %10, double %7, double %8
+ %12 = insertelement <2 x double> %1, double %11, i64 0
+ %13 = extractelement <2 x double> %12, i32 0
+ ret double %13
+}
+
+define double @test_mask3_vfnmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a
+ %3 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
+ %4 = extractelement <2 x double> %2, i64 0
+ %5 = extractelement <2 x double> %b, i64 0
+ %6 = extractelement <2 x double> %3, i64 0
+ %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+ %8 = extractelement <2 x double> %1, i64 0
+ %9 = bitcast i8 %mask to <8 x i1>
+ %10 = extractelement <8 x i1> %9, i64 0
+ %11 = select i1 %10, double %7, double %8
+ %12 = insertelement <2 x double> %1, double %11, i64 0
+ %13 = extractelement <2 x double> %12, i32 1
+ ret double %13
+}
+
+define double @test_mask3_vfnmsub_sd_1_unary_fneg(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mask3_vfnmsub_sd_1_unary_fneg(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
+ %2 = fneg <2 x double> %a
+ %3 = fneg <2 x double> %1
+ %4 = extractelement <2 x double> %2, i64 0
+ %5 = extractelement <2 x double> %b, i64 0
+ %6 = extractelement <2 x double> %3, i64 0
+ %7 = call double @llvm.fma.f64(double %4, double %5, double %6)
+ %8 = extractelement <2 x double> %1, i64 0
+ %9 = bitcast i8 %mask to <8 x i1>
+ %10 = extractelement <8 x i1> %9, i64 0
+ %11 = select i1 %10, double %7, double %8
+ %12 = insertelement <2 x double> %1, double %11, i64 0
+ %13 = extractelement <2 x double> %12, i32 1
+ ret double %13
+}
+
+declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>)
+
+define <8 x i32> @identity_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @identity_test_permvar_si_256(
+; CHECK-NEXT: ret <8 x i32> [[A0:%.*]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @identity_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_si_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[A0:%.*]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i32> [[TMP2]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+ ret <8 x i32> %3
+}
+
+define <8 x i32> @zero_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @zero_test_permvar_si_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @zero_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_si_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+ ret <8 x i32> %3
+}
+
+define <8 x i32> @shuffle_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_si_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @shuffle_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_si_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+ ret <8 x i32> %3
+}
+
+define <8 x i32> @undef_test_permvar_si_256(<8 x i32> %a0) {
+; CHECK-LABEL: @undef_test_permvar_si_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @undef_test_permvar_si_256_mask(<8 x i32> %a0, <8 x i32> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_si_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A0:%.*]], <8 x i32> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i32> [[TMP1]], <8 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %passthru
+ ret <8 x i32> %3
+}
+
+declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>)
+
+define <8 x float> @identity_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @identity_test_permvar_sf_256(
+; CHECK-NEXT: ret <8 x float> [[A0:%.*]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+ ret <8 x float> %1
+}
+
+define <8 x float> @identity_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_sf_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x float> [[A0:%.*]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x float> [[TMP2]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+ ret <8 x float> %3
+}
+
+define <8 x float> @zero_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @zero_test_permvar_sf_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: ret <8 x float> [[TMP1]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
+ ret <8 x float> %1
+}
+
+define <8 x float> @zero_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_sf_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x float> [[TMP3]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+ ret <8 x float> %3
+}
+
+define <8 x float> @shuffle_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x float> [[TMP1]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <8 x float> %1
+}
+
+define <8 x float> @shuffle_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x float> [[TMP3]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+ ret <8 x float> %3
+}
+
+define <8 x float> @undef_test_permvar_sf_256(<8 x float> %a0) {
+; CHECK-LABEL: @undef_test_permvar_sf_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x float> [[TMP1]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <8 x float> %1
+}
+
+define <8 x float> @undef_test_permvar_sf_256_mask(<8 x float> %a0, <8 x float> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_sf_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A0:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x float> [[TMP1]], <8 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x float> [[TMP3]]
+;
+ %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x float> %1, <8 x float> %passthru
+ ret <8 x float> %3
+}
+
+declare <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64>, <4 x i64>)
+
+define <4 x i64> @identity_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @identity_test_permvar_di_256(
+; CHECK-NEXT: ret <4 x i64> [[A0:%.*]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @identity_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_di_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[A0:%.*]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x i64> [[TMP2]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+ ret <4 x i64> %3
+}
+
+define <4 x i64> @zero_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @zero_test_permvar_di_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @zero_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_di_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+ ret <4 x i64> %3
+}
+
+define <4 x i64> @shuffle_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_di_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @shuffle_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_di_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+ ret <4 x i64> %3
+}
+
+define <4 x i64> @undef_test_permvar_di_256(<4 x i64> %a0) {
+; CHECK-LABEL: @undef_test_permvar_di_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @undef_test_permvar_di_256_mask(<4 x i64> %a0, <4 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_di_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A0:%.*]], <4 x i64> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x i64> [[TMP1]], <4 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %1 = call <4 x i64> @llvm.x86.avx512.permvar.di.256(<4 x i64> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %passthru
+ ret <4 x i64> %3
+}
+
+declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>)
+
+define <4 x double> @identity_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @identity_test_permvar_df_256(
+; CHECK-NEXT: ret <4 x double> [[A0:%.*]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+ ret <4 x double> %1
+}
+
+define <4 x double> @identity_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_df_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP1]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[A0:%.*]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x double> [[TMP2]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 0, i64 1, i64 2, i64 3>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+ ret <4 x double> %3
+}
+
+define <4 x double> @zero_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @zero_test_permvar_df_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x double> [[TMP1]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> zeroinitializer)
+ ret <4 x double> %1
+}
+
+define <4 x double> @zero_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_df_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x double> [[TMP3]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+ ret <4 x double> %3
+}
+
+define <4 x double> @shuffle_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_df_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x double> [[TMP1]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+ ret <4 x double> %1
+}
+
+define <4 x double> @shuffle_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_df_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x double> [[TMP3]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+ ret <4 x double> %3
+}
+
+define <4 x double> @undef_test_permvar_df_256(<4 x double> %a0) {
+; CHECK-LABEL: @undef_test_permvar_df_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x double> [[TMP1]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+ ret <4 x double> %1
+}
+
+define <4 x double> @undef_test_permvar_df_256_mask(<4 x double> %a0, <4 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_df_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A0:%.*]], <4 x double> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[EXTRACT:%.*]] = shufflevector <8 x i1> [[TMP2]], <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = select <4 x i1> [[EXTRACT]], <4 x double> [[TMP1]], <4 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <4 x double> [[TMP3]]
+;
+ %1 = call <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double> %a0, <4 x i64> <i64 undef, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %3 = select <4 x i1> %extract, <4 x double> %1, <4 x double> %passthru
+ ret <4 x double> %3
+}
+
+declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32> @identity_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @identity_test_permvar_si_512(
+; CHECK-NEXT: ret <16 x i32> [[A0:%.*]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @identity_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_si_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i32> [[A0:%.*]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i32> [[TMP2]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+ ret <16 x i32> %3
+}
+
+define <16 x i32> @zero_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @zero_test_permvar_si_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @zero_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_si_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i32> [[TMP3]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> zeroinitializer)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+ ret <16 x i32> %3
+}
+
+define <16 x i32> @shuffle_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_si_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @shuffle_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_si_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i32> [[TMP3]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+ ret <16 x i32> %3
+}
+
+define <16 x i32> @undef_test_permvar_si_512(<16 x i32> %a0) {
+; CHECK-LABEL: @undef_test_permvar_si_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @undef_test_permvar_si_512_mask(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_si_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A0:%.*]], <16 x i32> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i32> [[TMP1]], <16 x i32> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i32> [[TMP3]]
+;
+ %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %passthru
+ ret <16 x i32> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>)
+
+define <16 x float> @identity_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @identity_test_permvar_sf_512(
+; CHECK-NEXT: ret <16 x float> [[A0:%.*]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+ ret <16 x float> %1
+}
+
+define <16 x float> @identity_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_sf_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x float> [[A0:%.*]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP2]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+ ret <16 x float> %3
+}
+
+define <16 x float> @zero_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @zero_test_permvar_sf_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> zeroinitializer)
+ ret <16 x float> %1
+}
+
+define <16 x float> @zero_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_sf_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> zeroinitializer)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+ ret <16 x float> %3
+}
+
+define <16 x float> @shuffle_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <16 x float> %1
+}
+
+define <16 x float> @shuffle_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_sf_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+ ret <16 x float> %3
+}
+
+define <16 x float> @undef_test_permvar_sf_512(<16 x float> %a0) {
+; CHECK-LABEL: @undef_test_permvar_sf_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ ret <16 x float> %1
+}
+
+define <16 x float> @undef_test_permvar_sf_512_mask(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_sf_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x float> [[A0:%.*]], <16 x float> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %a0, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
+ ret <16 x float> %3
+}
+
+declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64> @identity_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @identity_test_permvar_di_512(
+; CHECK-NEXT: ret <8 x i64> [[A0:%.*]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @identity_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_di_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i64> [[A0:%.*]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i64> [[TMP2]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+ ret <8 x i64> %3
+}
+
+define <8 x i64> @zero_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @zero_test_permvar_di_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @zero_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_di_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i64> [[TMP3]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+ ret <8 x i64> %3
+}
+
+define <8 x i64> @shuffle_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_di_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @shuffle_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_di_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i64> [[TMP3]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+ ret <8 x i64> %3
+}
+
+define <8 x i64> @undef_test_permvar_di_512(<8 x i64> %a0) {
+; CHECK-LABEL: @undef_test_permvar_di_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @undef_test_permvar_di_512_mask(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_di_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i64> [[A0:%.*]], <8 x i64> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i64> [[TMP1]], <8 x i64> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i64> [[TMP3]]
+;
+ %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %passthru
+ ret <8 x i64> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>)
+
+define <8 x double> @identity_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @identity_test_permvar_df_512(
+; CHECK-NEXT: ret <8 x double> [[A0:%.*]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+ ret <8 x double> %1
+}
+
+define <8 x double> @identity_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_df_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x double> [[A0:%.*]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP2]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+ ret <8 x double> %3
+}
+
+define <8 x double> @zero_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @zero_test_permvar_df_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> zeroinitializer)
+ ret <8 x double> %1
+}
+
+define <8 x double> @zero_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_df_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+ ret <8 x double> %3
+}
+
+define <8 x double> @shuffle_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_df_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ ret <8 x double> %1
+}
+
+define <8 x double> @shuffle_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_df_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+ ret <8 x double> %3
+}
+
+define <8 x double> @undef_test_permvar_df_512(<8 x double> %a0) {
+; CHECK-LABEL: @undef_test_permvar_df_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ ret <8 x double> %1
+}
+
+define <8 x double> @undef_test_permvar_df_512_mask(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_df_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x double> [[A0:%.*]], <8 x double> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %a0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
+ ret <8 x double> %3
+}
+
+declare <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @identity_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_128(
+; CHECK-NEXT: ret <8 x i16> [[A0:%.*]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @identity_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i16> [[A0:%.*]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+ ret <8 x i16> %3
+}
+
+define <8 x i16> @zero_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @zero_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> zeroinitializer)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+ ret <8 x i16> %3
+}
+
+define <8 x i16> @shuffle_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @shuffle_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+ ret <8 x i16> %3
+}
+
+define <8 x i16> @undef_test_permvar_hi_128(<8 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @undef_test_permvar_hi_128_mask(<8 x i16> %a0, <8 x i16> %passthru, i8 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A0:%.*]], <8 x i16> undef, <8 x i32> <i32 undef, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x i16> [[TMP1]], <8 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = call <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16> %a0, <8 x i16> <i16 undef, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passthru
+ ret <8 x i16> %3
+}
+
+declare <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16>, <16 x i16>)
+
+define <16 x i16> @identity_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_256(
+; CHECK-NEXT: ret <16 x i16> [[A0:%.*]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @identity_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i16> [[A0:%.*]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i16> [[TMP2]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+ ret <16 x i16> %3
+}
+
+define <16 x i16> @zero_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @zero_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> zeroinitializer)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+ ret <16 x i16> %3
+}
+
+define <16 x i16> @shuffle_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @shuffle_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+ ret <16 x i16> %3
+}
+
+define <16 x i16> @undef_test_permvar_hi_256(<16 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @undef_test_permvar_hi_256_mask(<16 x i16> %a0, <16 x i16> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A0:%.*]], <16 x i16> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i16> [[TMP1]], <16 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = call <16 x i16> @llvm.x86.avx512.permvar.hi.256(<16 x i16> %a0, <16 x i16> <i16 undef, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passthru
+ ret <16 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @identity_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @identity_test_permvar_hi_512(
+; CHECK-NEXT: ret <32 x i16> [[A0:%.*]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @identity_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @identity_test_permvar_hi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <32 x i1> [[TMP1]], <32 x i16> [[A0:%.*]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i16> [[TMP2]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 20, i16 21, i16 22, i16 23, i16 24, i16 25, i16 26, i16 27, i16 28, i16 29, i16 30, i16 31>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+ ret <32 x i16> %3
+}
+
+define <32 x i16> @zero_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @zero_test_permvar_hi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @zero_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @zero_test_permvar_hi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> zeroinitializer)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+ ret <32 x i16> %3
+}
+
+define <32 x i16> @shuffle_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @shuffle_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_hi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+ ret <32 x i16> %3
+}
+
+define <32 x i16> @undef_test_permvar_hi_512(<32 x i16> %a0) {
+; CHECK-LABEL: @undef_test_permvar_hi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @undef_test_permvar_hi_512_mask(<32 x i16> %a0, <32 x i16> %passthru, i32 %mask) {
+; CHECK-LABEL: @undef_test_permvar_hi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i16> [[A0:%.*]], <32 x i16> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i16> [[TMP1]], <32 x i16> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 undef, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passthru
+ ret <32 x i16> %3
+}
+
+declare <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8>, <16 x i8>)
+
+define <16 x i8> @identity_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_128(
+; CHECK-NEXT: ret <16 x i8> [[A0:%.*]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @identity_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <16 x i1> [[TMP1]], <16 x i8> [[A0:%.*]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i8> [[TMP2]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+ ret <16 x i8> %3
+}
+
+define <16 x i8> @zero_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: ret <16 x i8> [[TMP1]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> zeroinitializer)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @zero_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i8> [[TMP3]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> zeroinitializer)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+ ret <16 x i8> %3
+}
+
+define <16 x i8> @shuffle_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i8> [[TMP1]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @shuffle_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i8> [[TMP3]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+ ret <16 x i8> %3
+}
+
+define <16 x i8> @undef_test_permvar_qi_128(<16 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_128(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <16 x i8> [[TMP1]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @undef_test_permvar_qi_128_mask(<16 x i8> %a0, <16 x i8> %passthru, i16 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_128_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i8> [[A0:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x i8> [[TMP1]], <16 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <16 x i8> [[TMP3]]
+;
+ %1 = call <16 x i8> @llvm.x86.avx512.permvar.qi.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passthru
+ ret <16 x i8> %3
+}
+
+declare <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8>, <32 x i8>)
+
+define <32 x i8> @identity_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_256(
+; CHECK-NEXT: ret <32 x i8> [[A0:%.*]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @identity_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <32 x i1> [[TMP1]], <32 x i8> [[A0:%.*]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i8> [[TMP2]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+ ret <32 x i8> %3
+}
+
+define <32 x i8> @zero_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: ret <32 x i8> [[TMP1]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> zeroinitializer)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @zero_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i8> [[TMP3]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> zeroinitializer)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+ ret <32 x i8> %3
+}
+
+define <32 x i8> @shuffle_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <32 x i8> [[TMP1]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @shuffle_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i8> [[TMP3]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+ ret <32 x i8> %3
+}
+
+define <32 x i8> @undef_test_permvar_qi_256(<32 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <32 x i8> [[TMP1]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @undef_test_permvar_qi_256_mask(<32 x i8> %a0, <32 x i8> %passthru, i32 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_256_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <32 x i8> [[A0:%.*]], <32 x i8> undef, <32 x i32> <i32 undef, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[MASK:%.*]] to <32 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <32 x i1> [[TMP2]], <32 x i8> [[TMP1]], <32 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <32 x i8> [[TMP3]]
+;
+ %1 = call <32 x i8> @llvm.x86.avx512.permvar.qi.256(<32 x i8> %a0, <32 x i8> <i8 undef, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i32 %mask to <32 x i1>
+ %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passthru
+ ret <32 x i8> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8>, <64 x i8>)
+
+define <64 x i8> @identity_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @identity_test_permvar_qi_512(
+; CHECK-NEXT: ret <64 x i8> [[A0:%.*]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @identity_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @identity_test_permvar_qi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT: [[TMP2:%.*]] = select <64 x i1> [[TMP1]], <64 x i8> [[A0:%.*]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <64 x i8> [[TMP2]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 20, i8 21, i8 22, i8 23, i8 24, i8 25, i8 26, i8 27, i8 28, i8 29, i8 30, i8 31, i8 32, i8 33, i8 34, i8 35, i8 36, i8 37, i8 38, i8 39, i8 40, i8 41, i8 42, i8 43, i8 44, i8 45, i8 46, i8 47, i8 48, i8 49, i8 50, i8 51, i8 52, i8 53, i8 54, i8 55, i8 56, i8 57, i8 58, i8 59, i8 60, i8 61, i8 62, i8 63>)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+ ret <64 x i8> %3
+}
+
+define <64 x i8> @zero_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @zero_test_permvar_qi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT: ret <64 x i8> [[TMP1]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> zeroinitializer)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @zero_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @zero_test_permvar_qi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <64 x i8> [[TMP3]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> zeroinitializer)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+ ret <64 x i8> %3
+}
+
+define <64 x i8> @shuffle_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <64 x i8> [[TMP1]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 63, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @shuffle_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @shuffle_test_permvar_qi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 63, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <64 x i8> [[TMP3]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 63, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+ ret <64 x i8> %3
+}
+
+define <64 x i8> @undef_test_permvar_qi_512(<64 x i8> %a0) {
+; CHECK-LABEL: @undef_test_permvar_qi_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <64 x i8> [[TMP1]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @undef_test_permvar_qi_512_mask(<64 x i8> %a0, <64 x i8> %passthru, i64 %mask) {
+; CHECK-LABEL: @undef_test_permvar_qi_512_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <64 x i8> [[A0:%.*]], <64 x i8> undef, <64 x i32> <i32 undef, i32 62, i32 61, i32 60, i32 59, i32 58, i32 57, i32 56, i32 55, i32 54, i32 53, i32 52, i32 51, i32 50, i32 49, i32 48, i32 47, i32 46, i32 45, i32 44, i32 43, i32 42, i32 41, i32 40, i32 39, i32 38, i32 37, i32 36, i32 35, i32 34, i32 33, i32 32, i32 31, i32 30, i32 29, i32 28, i32 27, i32 26, i32 25, i32 24, i32 23, i32 22, i32 21, i32 20, i32 19, i32 18, i32 17, i32 16, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[MASK:%.*]] to <64 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <64 x i1> [[TMP2]], <64 x i8> [[TMP1]], <64 x i8> [[PASSTHRU:%.*]]
+; CHECK-NEXT: ret <64 x i8> [[TMP3]]
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.permvar.qi.512(<64 x i8> %a0, <64 x i8> <i8 undef, i8 62, i8 61, i8 60, i8 59, i8 58, i8 57, i8 56, i8 55, i8 54, i8 53, i8 52, i8 51, i8 50, i8 49, i8 48, i8 47, i8 46, i8 45, i8 44, i8 43, i8 42, i8 41, i8 40, i8 39, i8 38, i8 37, i8 36, i8 35, i8 34, i8 33, i8 32, i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16, i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+ %2 = bitcast i64 %mask to <64 x i1>
+ %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %passthru
+ ret <64 x i8> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_add_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_add_ps(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_add_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_add_ps_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_add_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_add_ps_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+define <16 x float> @test_add_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_add_ps_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_add_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_add_pd(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_add_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_add_pd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_add_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_pd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+define <8 x double> @test_add_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_add_pd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.add.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_sub_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_sub_ps(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_sub_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_sub_ps_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_sub_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_sub_ps_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+define <16 x float> @test_sub_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_sub_ps_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_sub_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_sub_pd(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_sub_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_sub_pd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_sub_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_pd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+define <8 x double> @test_sub_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_sub_pd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.sub.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_mul_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_mul_ps(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_mul_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_mul_ps_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_mul_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_mul_ps_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+define <16 x float> @test_mul_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_mul_ps_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_mul_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_mul_pd(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_mul_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_mul_pd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_mul_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_pd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+define <8 x double> @test_mul_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_mul_pd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float>, <16 x float>, i32)
+
+define <16 x float> @test_div_ps(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_div_ps(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_div_ps_round(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @test_div_ps_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <16 x float> [[TMP1]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ ret <16 x float> %1
+}
+
+define <16 x float> @test_div_ps_mask(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_div_ps_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 4)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+define <16 x float> @test_div_ps_mask_round(<16 x float> %a, <16 x float> %b, <16 x float> %c, i16 %mask) {
+; CHECK-LABEL: @test_div_ps_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> [[A:%.*]], <16 x float> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[MASK:%.*]] to <16 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <16 x i1> [[TMP2]], <16 x float> [[TMP1]], <16 x float> [[C:%.*]]
+; CHECK-NEXT: ret <16 x float> [[TMP3]]
+;
+ %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a, <16 x float> %b, i32 8)
+ %2 = bitcast i16 %mask to <16 x i1>
+ %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %c
+ ret <16 x float> %3
+}
+
+declare <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double>, <8 x double>, i32)
+
+define <8 x double> @test_div_pd(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_div_pd(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_div_pd_round(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @test_div_pd_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: ret <8 x double> [[TMP1]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ ret <8 x double> %1
+}
+
+define <8 x double> @test_div_pd_mask(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_pd_mask(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 4)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+define <8 x double> @test_div_pd_mask_round(<8 x double> %a, <8 x double> %b, <8 x double> %c, i8 %mask) {
+; CHECK-LABEL: @test_div_pd_mask_round(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> [[A:%.*]], <8 x double> [[B:%.*]], i32 8)
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT: [[TMP3:%.*]] = select <8 x i1> [[TMP2]], <8 x double> [[TMP1]], <8 x double> [[C:%.*]]
+; CHECK-NEXT: ret <8 x double> [[TMP3]]
+;
+ %1 = call <8 x double> @llvm.x86.avx512.div.pd.512(<8 x double> %a, <8 x double> %b, i32 8)
+ %2 = bitcast i8 %mask to <8 x i1>
+ %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %c
+ ret <8 x double> %3
+}
+
+declare i32 @llvm.x86.avx512.vcomi.ss(<4 x float>, <4 x float>, i32, i32)
+
+define i32 @test_comi_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comi_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]], i32 0, i32 4)
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> %4, <4 x float> %8, i32 0, i32 4)
+ ret i32 %9
+}
+
+declare i32 @llvm.x86.avx512.vcomi.sd(<2 x double>, <2 x double>, i32, i32)
+
+define i32 @test_comi_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comi_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]], i32 0, i32 4)
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %2, <2 x double> %4, i32 0, i32 4)
+ ret i32 %5
+}
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
new file mode 100644
index 000000000000..6deedec15860
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-pack-inseltpoison.ll
@@ -0,0 +1,635 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+;
+; UNDEF Elts
+;
+
+define <8 x i16> @undef_packssdw_128() {
+; CHECK-LABEL: @undef_packssdw_128(
+; CHECK-NEXT: ret <8 x i16> undef
+;
+ %1 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> undef, <4 x i32> undef)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @undef_packusdw_128() {
+; CHECK-LABEL: @undef_packusdw_128(
+; CHECK-NEXT: ret <8 x i16> undef
+;
+ %1 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> undef, <4 x i32> undef)
+ ret <8 x i16> %1
+}
+
+define <16 x i8> @undef_packsswb_128() {
+; CHECK-LABEL: @undef_packsswb_128(
+; CHECK-NEXT: ret <16 x i8> undef
+;
+ %1 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> undef, <8 x i16> undef)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @undef_packuswb_128() {
+; CHECK-LABEL: @undef_packuswb_128(
+; CHECK-NEXT: ret <16 x i8> undef
+;
+ %1 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> undef, <8 x i16> undef)
+ ret <16 x i8> %1
+}
+
+define <16 x i16> @undef_packssdw_256() {
+; CHECK-LABEL: @undef_packssdw_256(
+; CHECK-NEXT: ret <16 x i16> undef
+;
+ %1 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> undef, <8 x i32> undef)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @undef_packusdw_256() {
+; CHECK-LABEL: @undef_packusdw_256(
+; CHECK-NEXT: ret <16 x i16> undef
+;
+ %1 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> undef, <8 x i32> undef)
+ ret <16 x i16> %1
+}
+
+define <32 x i8> @undef_packsswb_256() {
+; CHECK-LABEL: @undef_packsswb_256(
+; CHECK-NEXT: ret <32 x i8> undef
+;
+ %1 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> undef, <16 x i16> undef)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @undef_packuswb_256() {
+; CHECK-LABEL: @undef_packuswb_256(
+; CHECK-NEXT: ret <32 x i8> undef
+;
+ %1 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> undef, <16 x i16> undef)
+ ret <32 x i8> %1
+}
+
+define <32 x i16> @undef_packssdw_512() {
+; CHECK-LABEL: @undef_packssdw_512(
+; CHECK-NEXT: ret <32 x i16> undef
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> undef, <16 x i32> undef)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @undef_packusdw_512() {
+; CHECK-LABEL: @undef_packusdw_512(
+; CHECK-NEXT: ret <32 x i16> undef
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> undef, <16 x i32> undef)
+ ret <32 x i16> %1
+}
+
+define <64 x i8> @undef_packsswb_512() {
+; CHECK-LABEL: @undef_packsswb_512(
+; CHECK-NEXT: ret <64 x i8> undef
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> undef, <32 x i16> undef)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @undef_packuswb_512() {
+; CHECK-LABEL: @undef_packuswb_512(
+; CHECK-NEXT: ret <64 x i8> undef
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> undef, <32 x i16> undef)
+ ret <64 x i8> %1
+}
+
+;
+; Constant Folding
+;
+
+define <8 x i16> @fold_packssdw_128() {
+; CHECK-LABEL: @fold_packssdw_128(
+; CHECK-NEXT: ret <8 x i16> <i16 0, i16 -1, i16 32767, i16 -32768, i16 0, i16 0, i16 0, i16 0>
+;
+ %1 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> <i32 0, i32 -1, i32 65536, i32 -131072>, <4 x i32> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @fold_packusdw_128() {
+; CHECK-LABEL: @fold_packusdw_128(
+; CHECK-NEXT: ret <8 x i16> <i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 0, i16 -32768, i16 -1>
+;
+ %1 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> undef, <4 x i32> <i32 0, i32 -1, i32 32768, i32 65537>)
+ ret <8 x i16> %1
+}
+
+define <16 x i8> @fold_packsswb_128() {
+; CHECK-LABEL: @fold_packsswb_128(
+; CHECK-NEXT: ret <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+;
+ %1 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> zeroinitializer, <8 x i16> undef)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @fold_packuswb_128() {
+; CHECK-LABEL: @fold_packuswb_128(
+; CHECK-NEXT: ret <16 x i8> <i8 0, i8 1, i8 0, i8 -1, i8 0, i8 0, i8 0, i8 15, i8 0, i8 127, i8 0, i8 1, i8 0, i8 1, i8 0, i8 0>
+;
+ %1 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> <i16 0, i16 1, i16 -1, i16 255, i16 65535, i16 -32768, i16 -127, i16 15>, <8 x i16> <i16 -15, i16 127, i16 32768, i16 -65535, i16 -255, i16 1, i16 -1, i16 0>)
+ ret <16 x i8> %1
+}
+
+define <16 x i16> @fold_packssdw_256() {
+; CHECK-LABEL: @fold_packssdw_256(
+; CHECK-NEXT: ret <16 x i16> <i16 0, i16 256, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+ %1 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> <i32 0, i32 256, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>, <8 x i32> undef)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @fold_packusdw_256() {
+; CHECK-LABEL: @fold_packusdw_256(
+; CHECK-NEXT: ret <16 x i16> <i16 0, i16 0, i16 0, i16 -1, i16 0, i16 256, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767>
+;
+ %1 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> <i32 0, i32 -256, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767>, <8 x i32> <i32 0, i32 256, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>)
+ ret <16 x i16> %1
+}
+
+define <32 x i8> @fold_packsswb_256() {
+; CHECK-LABEL: @fold_packsswb_256(
+; CHECK-NEXT: ret <32 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+ %1 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> undef, <16 x i16> zeroinitializer)
+ ret <32 x i8> %1
+}
+
+define <32 x i8> @fold_packuswb_256() {
+; CHECK-LABEL: @fold_packuswb_256(
+; CHECK-NEXT: ret <32 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64>
+;
+ %1 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> zeroinitializer, <16 x i16> <i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 256, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>)
+ ret <32 x i8> %1
+}
+
+define <32 x i16> @fold_packssdw_512() {
+; CHECK-LABEL: @fold_packssdw_512(
+; CHECK-NEXT: ret <32 x i16> <i16 0, i16 512, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef, i16 0, i16 512, i16 32767, i16 -32768, i16 undef, i16 undef, i16 undef, i16 undef, i16 -127, i16 -32768, i16 -32767, i16 32767, i16 undef, i16 undef, i16 undef, i16 undef>
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> <i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767, i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>, <16 x i32> undef)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @fold_packusdw_512() {
+; CHECK-LABEL: @fold_packusdw_512(
+; CHECK-NEXT: ret <32 x i16> <i16 0, i16 0, i16 0, i16 -1, i16 0, i16 512, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767, i16 0, i16 0, i16 0, i16 -1, i16 0, i16 512, i16 -1, i16 0, i16 127, i16 -32768, i16 32767, i16 0, i16 0, i16 0, i16 0, i16 32767>
+;
+ %1 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> <i32 0, i32 -512, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767, i32 0, i32 -512, i32 -65535, i32 65536, i32 127, i32 32768, i32 32767, i32 -32767>, <16 x i32> <i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767, i32 0, i32 512, i32 65535, i32 -65536, i32 -127, i32 -32768, i32 -32767, i32 32767>)
+ ret <32 x i16> %1
+}
+
+define <64 x i8> @fold_packsswb_512() {
+; CHECK-LABEL: @fold_packsswb_512(
+; CHECK-NEXT: ret <64 x i8> <i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> undef, <32 x i16> zeroinitializer)
+ ret <64 x i8> %1
+}
+
+define <64 x i8> @fold_packuswb_512() {
+; CHECK-LABEL: @fold_packuswb_512(
+; CHECK-NEXT: ret <64 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 -1, i8 -1, i8 -1, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 1, i8 2, i8 4, i8 8, i8 16, i8 32, i8 64>
+;
+ %1 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> zeroinitializer, <32 x i16> <i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 512, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 0, i16 -127, i16 -128, i16 -32768, i16 65536, i16 255, i16 512, i16 512, i16 -1, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>)
+ ret <64 x i8> %1
+}
+
+;
+; Demanded Elts
+;
+
+define <8 x i16> @elts_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_128(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[A0:%.*]], <4 x i32> undef)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 3, i32 1, i32 undef, i32 undef>
+ %2 = shufflevector <4 x i32> %a1, <4 x i32> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 undef>
+ %3 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %1, <4 x i32> %2)
+ %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 7, i32 7, i32 7, i32 7>
+ ret <8 x i16> %4
+}
+
+define <8 x i16> @elts_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_128(
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> [[A0:%.*]], <4 x i32> [[A1:%.*]])
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = insertelement <4 x i32> %a0, i32 0, i32 0
+ %2 = insertelement <4 x i32> %a1, i32 0, i32 3
+ %3 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %1, <4 x i32> %2)
+ %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <8 x i32> <i32 undef, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 undef>
+ ret <8 x i16> %4
+}
+
+define <16 x i8> @elts_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_128(
+; CHECK-NEXT: ret <16 x i8> zeroinitializer
+;
+ %1 = insertelement <8 x i16> %a0, i16 0, i32 0
+ %2 = insertelement <8 x i16> %a1, i16 0, i32 0
+ %3 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %1, <8 x i16> %2)
+ %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+ ret <16 x i8> %4
+}
+
+define <16 x i8> @elts_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_128(
+; CHECK-NEXT: ret <16 x i8> undef
+;
+ %1 = insertelement <8 x i16> poison, i16 0, i32 0
+ %2 = insertelement <8 x i16> poison, i16 0, i32 0
+ %3 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %1, <8 x i16> %2)
+ %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+ ret <16 x i8> %4
+}
+
+define <16 x i16> @elts_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_256(
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[A0:%.*]], <8 x i32> undef)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[TMP1]], <16 x i16> undef, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <16 x i16> [[TMP2]]
+;
+ %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef>
+ %3 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %1, <8 x i32> %2)
+ %4 = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 7, i32 8, i32 undef, i32 undef, i32 11, i32 12, i32 undef, i32 undef, i32 15>
+ ret <16 x i16> %4
+}
+
+define <16 x i16> @elts_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_256(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A1:%.*]], <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> undef, <8 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <16 x i16> [[TMP2]], <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = shufflevector <8 x i32> %a1, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %3 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %1, <8 x i32> %2)
+ %4 = shufflevector <16 x i16> %3, <16 x i16> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+ ret <16 x i16> %4
+}
+
+define <32 x i8> @elts_packsswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_256(
+; CHECK-NEXT: ret <32 x i8> zeroinitializer
+;
+ %1 = insertelement <16 x i16> %a0, i16 0, i32 0
+ %2 = insertelement <16 x i16> %a1, i16 0, i32 8
+ %3 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %1, <16 x i16> %2)
+ %4 = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24>
+ ret <32 x i8> %4
+}
+
+define <32 x i8> @elts_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_256(
+; CHECK-NEXT: ret <32 x i8> undef
+;
+ %1 = insertelement <16 x i16> poison, i16 0, i32 1
+ %2 = insertelement <16 x i16> poison, i16 0, i32 0
+ %3 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %1, <16 x i16> %2)
+ %4 = shufflevector <32 x i8> %3, <32 x i8> undef, <32 x i32> zeroinitializer
+ ret <32 x i8> %4
+}
+
+define <32 x i16> @elts_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @elts_packssdw_512(
+; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[A0:%.*]], <16 x i32> undef)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <32 x i16> [[TMP1]], <32 x i16> undef, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 8, i32 undef, i32 undef, i32 11, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 18, i32 19, i32 undef, i32 undef, i32 undef, i32 undef, i32 24, i32 undef, i32 undef, i32 27, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <32 x i16> [[TMP2]]
+;
+ %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 undef, i32 2, i32 1, i32 undef, i32 undef, i32 6, i32 5, i32 undef, i32 undef, i32 10, i32 9, i32 undef, i32 undef, i32 14, i32 13, i32 undef>
+ %3 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %1, <16 x i32> %2)
+ %4 = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 4, i32 undef, i32 undef, i32 7, i32 8, i32 undef, i32 undef, i32 11, i32 12, i32 undef, i32 undef, i32 15, i32 undef, i32 undef, i32 18, i32 19, i32 20, i32 undef, i32 undef, i32 23, i32 24, i32 undef, i32 undef, i32 27, i32 28, i32 undef, i32 undef, i32 31>
+ ret <32 x i16> %4
+}
+
+define <32 x i16> @elts_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @elts_packusdw_512(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> [[A1:%.*]], <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+; CHECK-NEXT: [[TMP2:%.*]] = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> undef, <16 x i32> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <32 x i16> [[TMP2]], <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = shufflevector <16 x i32> %a0, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+ %2 = shufflevector <16 x i32> %a1, <16 x i32> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
+ %3 = call <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32> %1, <16 x i32> %2)
+ %4 = shufflevector <32 x i16> %3, <32 x i16> undef, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 20, i32 21, i32 22, i32 23, i32 undef, i32 undef, i32 undef, i32 undef, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef>
+ ret <32 x i16> %4
+}
+
+define <64 x i8> @elts_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @elts_packsswb_512(
+; CHECK-NEXT: ret <64 x i8> zeroinitializer
+;
+ %1 = insertelement <32 x i16> %a0, i16 0, i32 0
+ %2 = insertelement <32 x i16> %a1, i16 0, i32 8
+ %3 = insertelement <32 x i16> %1, i16 0, i32 16
+ %4 = insertelement <32 x i16> %2, i16 0, i32 24
+ %5 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %3, <32 x i16> %4)
+ %6 = shufflevector <64 x i8> %5, <64 x i8> undef, <64 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 24, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56, i32 56>
+ ret <64 x i8> %6
+}
+
+define <64 x i8> @elts_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @elts_packuswb_512(
+; CHECK-NEXT: ret <64 x i8> undef
+;
+ %1 = insertelement <32 x i16> poison, i16 0, i32 1
+ %2 = insertelement <32 x i16> poison, i16 0, i32 0
+ %3 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %1, <32 x i16> %2)
+ %4 = shufflevector <64 x i8> %3, <64 x i8> undef, <64 x i32> zeroinitializer
+ ret <64 x i8> %4
+}
+
+;
+; Truncation (without Saturation)
+;
+
+define <8 x i16> @trunc_packssdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @trunc_packssdw_128(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[A1:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = ashr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+ %2 = and <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+ %3 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %1, <4 x i32> %2)
+ ret <8 x i16> %3
+}
+
+define <8 x i16> @trunc_packusdw_128(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: @trunc_packusdw_128(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[A1:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[TMP1]], <4 x i32> [[TMP2]])
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = lshr <4 x i32> %a0, <i32 17, i32 17, i32 17, i32 17>
+ %2 = and <4 x i32> %a1, <i32 15, i32 15, i32 15, i32 15>
+ %3 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %1, <4 x i32> %2)
+ ret <8 x i16> %3
+}
+
+define <16 x i8> @trunc_packsswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @trunc_packsswb_128(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> [[TMP1]], <8 x i16> [[TMP2]])
+; CHECK-NEXT: ret <16 x i8> [[TMP3]]
+;
+ %1 = ashr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <8 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %1, <8 x i16> %2)
+ ret <16 x i8> %3
+}
+
+define <16 x i8> @trunc_packuswb_128(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: @trunc_packuswb_128(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> [[TMP1]], <8 x i16> [[TMP2]])
+; CHECK-NEXT: ret <16 x i8> [[TMP3]]
+;
+ %1 = lshr <8 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <8 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %1, <8 x i16> %2)
+ ret <16 x i8> %3
+}
+
+define <16 x i16> @trunc_packssdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @trunc_packssdw_256(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <8 x i32> [[A1:%.*]], <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = ashr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+ %2 = ashr <8 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+ %3 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %1, <8 x i32> %2)
+ ret <16 x i16> %3
+}
+
+define <16 x i16> @trunc_packusdw_256(<8 x i32> %a0, <8 x i32> %a1) {
+; CHECK-LABEL: @trunc_packusdw_256(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = and <8 x i32> [[A1:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[TMP1]], <8 x i32> [[TMP2]])
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = lshr <8 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+ %2 = and <8 x i32> %a1, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+ %3 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %1, <8 x i32> %2)
+ ret <16 x i16> %3
+}
+
+define <32 x i8> @trunc_packsswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @trunc_packsswb_256(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <16 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT: ret <32 x i8> [[TMP3]]
+;
+ %1 = ashr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <16 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %1, <16 x i16> %2)
+ ret <32 x i8> %3
+}
+
+define <32 x i8> @trunc_packuswb_256(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: @trunc_packuswb_256(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <16 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> [[TMP1]], <16 x i16> [[TMP2]])
+; CHECK-NEXT: ret <32 x i8> [[TMP3]]
+;
+ %1 = lshr <16 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <16 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %1, <16 x i16> %2)
+ ret <32 x i8> %3
+}
+
+define <32 x i16> @trunc_packssdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @trunc_packssdw_512(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <16 x i32> [[A1:%.*]], <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+; CHECK-NEXT: [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = ashr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+ %2 = ashr <16 x i32> %a1, <i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23, i32 23>
+ %3 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %1, <16 x i32> %2)
+ ret <32 x i16> %3
+}
+
+define <32 x i16> @trunc_packusdw_512(<16 x i32> %a0, <16 x i32> %a1) {
+; CHECK-LABEL: @trunc_packusdw_512(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i32> [[A0:%.*]], <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: [[TMP2:%.*]] = and <16 x i32> [[A1:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = lshr <16 x i32> %a0, <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+ %2 = and <16 x i32> %a1, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+ %3 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %1, <16 x i32> %2)
+ ret <32 x i16> %3
+}
+
+define <64 x i8> @trunc_packsswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @trunc_packsswb_512(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <32 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; CHECK-NEXT: ret <64 x i8> [[TMP3]]
+;
+ %1 = ashr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <32 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %1, <32 x i16> %2)
+ ret <64 x i8> %3
+}
+
+define <64 x i8> @trunc_packuswb_512(<32 x i16> %a0, <32 x i16> %a1) {
+; CHECK-LABEL: @trunc_packuswb_512(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <32 x i16> [[A0:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: [[TMP2:%.*]] = and <32 x i16> [[A1:%.*]], <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+; CHECK-NEXT: [[TMP3:%.*]] = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> [[TMP1]], <32 x i16> [[TMP2]])
+; CHECK-NEXT: ret <64 x i8> [[TMP3]]
+;
+ %1 = lshr <32 x i16> %a0, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+ %2 = and <32 x i16> %a1, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+ %3 = call <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16> %1, <32 x i16> %2)
+ ret <64 x i8> %3
+}
+
+;
+; Signed Pack Comparison Results
+;
+
+define <8 x i16> @cmp_packssdw_128(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x i32> %a3) {
+; CHECK-LABEL: @cmp_packssdw_128(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <4 x i1> [[TMP2]] to <4 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> [[TMP3]], <4 x i32> [[TMP4]])
+; CHECK-NEXT: ret <8 x i16> [[TMP5]]
+;
+ %1 = icmp eq <4 x i32> %a0, %a1
+ %2 = icmp eq <4 x i32> %a2, %a3
+ %3 = sext <4 x i1> %1 to <4 x i32>
+ %4 = sext <4 x i1> %2 to <4 x i32>
+ %5 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %3, <4 x i32> %4)
+ ret <8 x i16> %5
+}
+
+define <16 x i8> @cmp_packsswb_128(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2, <8 x i16> %a3) {
+; CHECK-LABEL: @cmp_packsswb_128(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <8 x i16> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i1> [[TMP2]] to <8 x i16>
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> [[TMP3]], <8 x i16> [[TMP4]])
+; CHECK-NEXT: ret <16 x i8> [[TMP5]]
+;
+ %1 = icmp eq <8 x i16> %a0, %a1
+ %2 = icmp eq <8 x i16> %a2, %a3
+ %3 = sext <8 x i1> %1 to <8 x i16>
+ %4 = sext <8 x i1> %2 to <8 x i16>
+ %5 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %3, <8 x i16> %4)
+ ret <16 x i8> %5
+}
+
+define <16 x i16> @cmp_packssdw_256(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x i32> %a3) {
+; CHECK-LABEL: @cmp_packssdw_256(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <8 x i32> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <8 x i1> [[TMP2]] to <8 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> [[TMP3]], <8 x i32> [[TMP4]])
+; CHECK-NEXT: ret <16 x i16> [[TMP5]]
+;
+ %1 = icmp eq <8 x i32> %a0, %a1
+ %2 = icmp eq <8 x i32> %a2, %a3
+ %3 = sext <8 x i1> %1 to <8 x i32>
+ %4 = sext <8 x i1> %2 to <8 x i32>
+ %5 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %3, <8 x i32> %4)
+ ret <16 x i16> %5
+}
+
+define <32 x i8> @cmp_packsswb_256(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> %a2, <16 x i16> %a3) {
+; CHECK-LABEL: @cmp_packsswb_256(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i16> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i1> [[TMP1]] to <16 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i1> [[TMP2]] to <16 x i16>
+; CHECK-NEXT: [[TMP5:%.*]] = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> [[TMP3]], <16 x i16> [[TMP4]])
+; CHECK-NEXT: ret <32 x i8> [[TMP5]]
+;
+ %1 = icmp eq <16 x i16> %a0, %a1
+ %2 = icmp eq <16 x i16> %a2, %a3
+ %3 = sext <16 x i1> %1 to <16 x i16>
+ %4 = sext <16 x i1> %2 to <16 x i16>
+ %5 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %3, <16 x i16> %4)
+ ret <32 x i8> %5
+}
+
+define <32 x i16> @cmp_packssdw_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, <16 x i32> %a3) {
+; CHECK-LABEL: @cmp_packssdw_512(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <16 x i32> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <16 x i32> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <16 x i1> [[TMP1]] to <16 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <16 x i1> [[TMP2]] to <16 x i32>
+; CHECK-NEXT: [[TMP5:%.*]] = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> [[TMP3]], <16 x i32> [[TMP4]])
+; CHECK-NEXT: ret <32 x i16> [[TMP5]]
+;
+ %1 = icmp eq <16 x i32> %a0, %a1
+ %2 = icmp eq <16 x i32> %a2, %a3
+ %3 = sext <16 x i1> %1 to <16 x i32>
+ %4 = sext <16 x i1> %2 to <16 x i32>
+ %5 = call <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32> %3, <16 x i32> %4)
+ ret <32 x i16> %5
+}
+
+define <64 x i8> @cmp_packsswb_512(<32 x i16> %a0, <32 x i16> %a1, <32 x i16> %a2, <32 x i16> %a3) {
+; CHECK-LABEL: @cmp_packsswb_512(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <32 x i16> [[A0:%.*]], [[A1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <32 x i16> [[A2:%.*]], [[A3:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = sext <32 x i1> [[TMP1]] to <32 x i16>
+; CHECK-NEXT: [[TMP4:%.*]] = sext <32 x i1> [[TMP2]] to <32 x i16>
+; CHECK-NEXT: [[TMP5:%.*]] = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> [[TMP3]], <32 x i16> [[TMP4]])
+; CHECK-NEXT: ret <64 x i8> [[TMP5]]
+;
+ %1 = icmp eq <32 x i16> %a0, %a1
+ %2 = icmp eq <32 x i16> %a2, %a3
+ %3 = sext <32 x i1> %1 to <32 x i16>
+ %4 = sext <32 x i1> %2 to <32 x i16>
+ %5 = call <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16> %3, <32 x i16> %4)
+ ret <64 x i8> %5
+}
+
+declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
+declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
+
+declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
+declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
+declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
+declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
+
+declare <32 x i16> @llvm.x86.avx512.packssdw.512(<16 x i32>, <16 x i32>) nounwind readnone
+declare <32 x i16> @llvm.x86.avx512.packusdw.512(<16 x i32>, <16 x i32>) nounwind readnone
+declare <64 x i8> @llvm.x86.avx512.packsswb.512(<32 x i16>, <32 x i16>) nounwind readnone
+declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse-inseltpoison.ll
new file mode 100644
index 000000000000..0142925a5e03
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse-inseltpoison.ll
@@ -0,0 +1,694 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @test_rcp_ss_0(float %a) {
+; CHECK-LABEL: @test_rcp_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 0
+ ret float %6
+}
+
+define float @test_rcp_ss_1(float %a) {
+; CHECK-LABEL: @test_rcp_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 1
+ ret float %6
+}
+
+define float @test_sqrt_ss_0(float %a) {
+; CHECK-LABEL: @test_sqrt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[A:%.*]])
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 0
+ ret float %6
+}
+
+define float @test_sqrt_ss_2(float %a) {
+; CHECK-LABEL: @test_sqrt_ss_2(
+; CHECK-NEXT: ret float 2.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 2
+ ret float %6
+}
+
+define float @test_rsqrt_ss_0(float %a) {
+; CHECK-LABEL: @test_rsqrt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 0
+ ret float %6
+}
+
+define float @test_rsqrt_ss_3(float %a) {
+; CHECK-LABEL: @test_rsqrt_ss_3(
+; CHECK-NEXT: ret float 3.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 3
+ ret float %6
+}
+
+define float @test_add_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %4, <4 x float> %8)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_add_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_add_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 1
+ ret float %7
+}
+
+define float @test_add_ss_2(float %a) {
+; CHECK-LABEL: @test_add_ss_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd float [[A:%.*]], [[A]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.add.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define float @test_sub_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %4, <4 x float> %8)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_sub_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_sub_ss_2(
+; CHECK-NEXT: ret float 2.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 2
+ ret float %7
+}
+
+define float @test_sub_ss_3(float %a) {
+; CHECK-LABEL: @test_sub_ss_3(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub float [[A:%.*]], [[A]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.sub.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define float @test_mul_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %4, <4 x float> %8)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_mul_ss_3(float %a, float %b) {
+; CHECK-LABEL: @test_mul_ss_3(
+; CHECK-NEXT: ret float 3.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 3
+ ret float %7
+}
+
+define float @test_mul_ss_4(float %a) {
+; CHECK-LABEL: @test_mul_ss_4(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul float [[A:%.*]], [[A]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.mul.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define float @test_div_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %4, <4 x float> %8)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_div_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_div_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 1
+ ret float %7
+}
+
+define float @test_div_ss_2(float %a) {
+; CHECK-LABEL: @test_div_ss_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv float [[A:%.*]], [[A]]
+; CHECK-NEXT: ret float [[TMP1]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.div.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define <4 x float> @test_min_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_min_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a, <4 x float> %3)
+ ret <4 x float> %4
+}
+
+define float @test_min_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: ret float [[TMP4]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %4, <4 x float> %8)
+ %10 = extractelement <4 x float> %9, i32 0
+ ret float %10
+}
+
+define float @test_min_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_min_ss_2(
+; CHECK-NEXT: ret float 2.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 2
+ ret float %7
+}
+
+define float @test_min_ss_3(float %a) {
+; CHECK-LABEL: @test_min_ss_3(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define <4 x float> @test_max_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_max_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]])
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a, <4 x float> %3)
+ ret <4 x float> %4
+}
+
+define float @test_max_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: ret float [[TMP4]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %4, <4 x float> %8)
+ %10 = extractelement <4 x float> %9, i32 0
+ ret float %10
+}
+
+define float @test_max_ss_3(float %a, float %b) {
+; CHECK-LABEL: @test_max_ss_3(
+; CHECK-NEXT: ret float 3.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %4, <4 x float> %5)
+ %7 = extractelement <4 x float> %6, i32 3
+ ret float %7
+}
+
+define float @test_max_ss_4(float %a) {
+; CHECK-LABEL: @test_max_ss_4(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %1, <4 x float> %1)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define <4 x float> @test_cmp_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_cmp_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], i8 0)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %a, <4 x float> %3, i8 0)
+ ret <4 x float> %4
+}
+
+define float @test_cmp_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_cmp_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]], i8 0)
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %4, <4 x float> %8, i8 0)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_cmp_ss_1(float %a, float %b) {
+; CHECK-LABEL: @test_cmp_ss_1(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %4, <4 x float> %5, i8 0)
+ %7 = extractelement <4 x float> %6, i32 1
+ ret float %7
+}
+
+define float @test_cmp_ss_2(float %a) {
+; CHECK-LABEL: @test_cmp_ss_2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> <float undef, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00>, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> [[TMP1]], <4 x float> [[TMP1]], i8 3)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %1, <4 x float> %1, i8 3)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+define i32 @test_comieq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comieq_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comieq.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_comige_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comige_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comige.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_comigt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comigt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comigt.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_comile_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comile_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comile.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_comilt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comilt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comilt.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_comineq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_comineq_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.comineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.comineq.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomieq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomieq_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomige_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomige_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomige.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomige.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomigt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomigt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomigt.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomile_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomile_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomile.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomile.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomilt_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomilt_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomilt.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+define i32 @test_ucomineq_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_ucomineq_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> [[TMP1]], <4 x float> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call i32 @llvm.x86.sse.ucomineq.ss(<4 x float> %4, <4 x float> %8)
+ ret i32 %9
+}
+
+declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>)
+declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>)
+
+declare <4 x float> @llvm.x86.sse.add.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.sub.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.mul.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.div.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8)
+
+declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comige.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comigt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comile.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comilt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.comineq.ss(<4 x float>, <4 x float>)
+
+declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomige.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomigt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomile.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomilt.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.ucomineq.ss(<4 x float>, <4 x float>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse2-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse2-inseltpoison.ll
new file mode 100644
index 000000000000..ac4ff788730e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse2-inseltpoison.ll
@@ -0,0 +1,541 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define double @test_sqrt_sd_0(double %a) {
+; CHECK-LABEL: @test_sqrt_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[A:%.*]])
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %2)
+ %4 = extractelement <2 x double> %3, i32 0
+ ret double %4
+}
+
+define double @test_sqrt_sd_1(double %a) {
+; CHECK-LABEL: @test_sqrt_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = tail call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %2)
+ %4 = extractelement <2 x double> %3, i32 1
+ ret double %4
+}
+
+define double @test_add_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_add_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_add_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_add_sd_2(double %a) {
+; CHECK-LABEL: @test_add_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd double [[A:%.*]], [[A]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.add.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_sub_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_sub_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_sub_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_sub_sd_2(double %a) {
+; CHECK-LABEL: @test_sub_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub double [[A:%.*]], [[A]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.sub.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_mul_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_mul_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_mul_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_mul_sd_2(double %a) {
+; CHECK-LABEL: @test_mul_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul double [[A:%.*]], [[A]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.mul.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define double @test_div_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_div_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_div_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_div_sd_2(double %a) {
+; CHECK-LABEL: @test_div_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv double [[A:%.*]], [[A]]
+; CHECK-NEXT: ret double [[TMP1]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.div.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define <2 x double> @test_min_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_min_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]])
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a, <2 x double> %1)
+ ret <2 x double> %2
+}
+
+define double @test_min_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT: ret double [[TMP4]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_min_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_min_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_min_sd_2(double %a) {
+; CHECK-LABEL: @test_min_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define <2 x double> @test_max_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_max_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]])
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a, <2 x double> %1)
+ ret <2 x double> %2
+}
+
+define double @test_max_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT: ret double [[TMP4]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_max_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_max_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %2, <2 x double> %4)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_max_sd_2(double %a) {
+; CHECK-LABEL: @test_max_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %1, <2 x double> %1)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define <2 x double> @test_cmp_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_cmp_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i8 0)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %a, <2 x double> %1, i8 0)
+ ret <2 x double> %2
+}
+
+define double @test_cmp_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_cmp_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]], i8 0)
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-NEXT: ret double [[TMP4]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %2, <2 x double> %4, i8 0)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_cmp_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_cmp_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %2, <2 x double> %4, i8 0)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_cmp_sd_2(double %a) {
+; CHECK-LABEL: @test_cmp_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> [[TMP1]], <2 x double> [[TMP1]], i8 3)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %1, <2 x double> %1, i8 3)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define i32 @test_comieq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comieq_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_comige_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comige_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comige.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_comigt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comigt_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comigt.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_comile_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comile_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comile.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_comilt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comilt_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comilt.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_comineq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_comineq_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.comineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.comineq.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomieq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomieq_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomige_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomige_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomige.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomigt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomigt_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomigt.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomile_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomile_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomile.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomilt_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomilt_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomilt.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+define i32 @test_ucomineq_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_ucomineq_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> [[TMP1]], <2 x double> [[TMP2]])
+; CHECK-NEXT: ret i32 [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call i32 @llvm.x86.sse2.ucomineq.sd(<2 x double> %2, <2 x double> %4)
+ ret i32 %5
+}
+
+declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
+
+declare <2 x double> @llvm.x86.sse2.add.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.sub.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.mul.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.div.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8)
+
+declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comige.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comigt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comile.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comilt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.comineq.sd(<2 x double>, <2 x double>)
+
+declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomige.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomigt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomile.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomilt.sd(<2 x double>, <2 x double>)
+declare i32 @llvm.x86.sse2.ucomineq.sd(<2 x double>, <2 x double>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-sse41-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-sse41-inseltpoison.ll
new file mode 100644
index 000000000000..94f71510f379
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-sse41-inseltpoison.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define <2 x double> @test_round_sd(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @test_round_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> [[A:%.*]], <2 x double> [[B:%.*]], i32 10)
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 0
+ %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
+ %3 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %1, <2 x double> %2, i32 10)
+ ret <2 x double> %3
+}
+
+define double @test_round_sd_0(double %a, double %b) {
+; CHECK-LABEL: @test_round_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> undef, <2 x double> [[TMP1]], i32 10)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %2, <2 x double> %4, i32 10)
+ %6 = extractelement <2 x double> %5, i32 0
+ ret double %6
+}
+
+define double @test_round_sd_1(double %a, double %b) {
+; CHECK-LABEL: @test_round_sd_1(
+; CHECK-NEXT: ret double 1.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = insertelement <2 x double> poison, double %b, i32 0
+ %4 = insertelement <2 x double> %3, double 2.000000e+00, i32 1
+ %5 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %2, <2 x double> %4, i32 10)
+ %6 = extractelement <2 x double> %5, i32 1
+ ret double %6
+}
+
+define double @test_round_sd_2(double %a) {
+; CHECK-LABEL: @test_round_sd_2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> undef, <2 x double> [[TMP1]], i32 10)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> zeroinitializer, double %a, i32 0
+ %2 = tail call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %1, <2 x double> %1, i32 10)
+ %3 = extractelement <2 x double> %2, i32 0
+ ret double %3
+}
+
+define <4 x float> @test_round_ss(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_round_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, <4 x float> [[B:%.*]], i32 10)
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = insertelement <4 x float> %b, float 1.000000e+00, i32 1
+ %5 = insertelement <4 x float> %4, float 2.000000e+00, i32 2
+ %6 = insertelement <4 x float> %5, float 3.000000e+00, i32 3
+ %7 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %3, <4 x float> %6, i32 10)
+ ret <4 x float> %7
+}
+
+define float @test_round_ss_0(float %a, float %b) {
+; CHECK-LABEL: @test_round_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> undef, <4 x float> [[TMP1]], i32 10)
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %4, <4 x float> %8, i32 10)
+ %r = extractelement <4 x float> %9, i32 0
+ ret float %r
+}
+
+define float @test_round_ss_2(float %a, float %b) {
+; CHECK-LABEL: @test_round_ss_2(
+; CHECK-NEXT: ret float 2.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = insertelement <4 x float> poison, float %b, i32 0
+ %6 = insertelement <4 x float> %5, float 4.000000e+00, i32 1
+ %7 = insertelement <4 x float> %6, float 5.000000e+00, i32 2
+ %8 = insertelement <4 x float> %7, float 6.000000e+00, i32 3
+ %9 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %4, <4 x float> %8, i32 10)
+ %r = extractelement <4 x float> %9, i32 2
+ ret float %r
+}
+
+define float @test_round_ss_3(float %a) {
+; CHECK-LABEL: @test_round_ss_3(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> undef, <4 x float> [[TMP1]], i32 10)
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> zeroinitializer, float %a, i32 0
+ %2 = tail call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %1, <4 x float> %1, i32 10)
+ %3 = extractelement <4 x float> %2, i32 0
+ ret float %3
+}
+
+declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
+declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts-inseltpoison.ll
new file mode 100644
index 000000000000..070a15c46d7f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vec_demanded_elts-inseltpoison.ll
@@ -0,0 +1,110 @@
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i16 @test1(float %f) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: [[TMP281:%.*]] = fadd float %f, -1.000000e+00
+; CHECK-NEXT: [[TMP373:%.*]] = fmul float [[TMP281]], 5.000000e-01
+; CHECK-NEXT: [[TMP374:%.*]] = insertelement <4 x float> undef, float [[TMP373]], i32 0
+; CHECK-NEXT: [[TMP48:%.*]] = tail call <4 x float> @llvm.x86.sse.min.ss(<4 x float> [[TMP374]], <4 x float> <float 6.553500e+04, float undef, float undef, float undef>)
+; CHECK-NEXT: [[TMP59:%.*]] = tail call <4 x float> @llvm.x86.sse.max.ss(<4 x float> [[TMP48]], <4 x float> <float 0.000000e+00, float undef, float undef, float undef>)
+; CHECK-NEXT: [[TMP_UPGRD_1:%.*]] = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> [[TMP59]])
+; CHECK-NEXT: [[TMP69:%.*]] = trunc i32 [[TMP_UPGRD_1]] to i16
+; CHECK-NEXT: ret i16 [[TMP69]]
+;
+ %tmp = insertelement <4 x float> poison, float %f, i32 0
+ %tmp10 = insertelement <4 x float> %tmp, float 0.000000e+00, i32 1
+ %tmp11 = insertelement <4 x float> %tmp10, float 0.000000e+00, i32 2
+ %tmp12 = insertelement <4 x float> %tmp11, float 0.000000e+00, i32 3
+ %tmp28 = tail call <4 x float> @llvm.x86.sse.sub.ss( <4 x float> %tmp12, <4 x float> < float 1.000000e+00, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+ %tmp37 = tail call <4 x float> @llvm.x86.sse.mul.ss( <4 x float> %tmp28, <4 x float> < float 5.000000e-01, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+ %tmp48 = tail call <4 x float> @llvm.x86.sse.min.ss( <4 x float> %tmp37, <4 x float> < float 6.553500e+04, float 0.000000e+00, float 0.000000e+00, float 0.000000e+00 > )
+ %tmp59 = tail call <4 x float> @llvm.x86.sse.max.ss( <4 x float> %tmp48, <4 x float> zeroinitializer )
+ %tmp.upgrd.1 = tail call i32 @llvm.x86.sse.cvttss2si( <4 x float> %tmp59 )
+ %tmp69 = trunc i32 %tmp.upgrd.1 to i16
+ ret i16 %tmp69
+}
+
+define i64 @test3(float %f, double %d) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: [[V00:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> [[V00]])
+; CHECK-NEXT: [[V10:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> [[V10]])
+; CHECK-NEXT: [[V20:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> [[V20]])
+; CHECK-NEXT: [[V30:%.*]] = insertelement <4 x float> undef, float %f, i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> [[V30]])
+; CHECK-NEXT: [[V40:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = tail call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> [[V40]])
+; CHECK-NEXT: [[V50:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = tail call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> [[V50]])
+; CHECK-NEXT: [[V60:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> [[V60]])
+; CHECK-NEXT: [[V70:%.*]] = insertelement <2 x double> undef, double %d, i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> [[V70]])
+; CHECK-NEXT: [[TMP8:%.*]] = add i32 [[TMP0]], [[TMP2]]
+; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP4]], [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[TMP5]], [[TMP7]]
+; CHECK-NEXT: [[TMP14:%.*]] = add i64 [[TMP12]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = add i64 [[TMP14]], [[TMP11]]
+; CHECK-NEXT: ret i64 [[TMP15]]
+;
+ %v00 = insertelement <4 x float> poison, float %f, i32 0
+ %v01 = insertelement <4 x float> %v00, float 0.000000e+00, i32 1
+ %v02 = insertelement <4 x float> %v01, float 0.000000e+00, i32 2
+ %v03 = insertelement <4 x float> %v02, float 0.000000e+00, i32 3
+ %tmp0 = tail call i32 @llvm.x86.sse.cvtss2si(<4 x float> %v03)
+ %v10 = insertelement <4 x float> poison, float %f, i32 0
+ %v11 = insertelement <4 x float> %v10, float 0.000000e+00, i32 1
+ %v12 = insertelement <4 x float> %v11, float 0.000000e+00, i32 2
+ %v13 = insertelement <4 x float> %v12, float 0.000000e+00, i32 3
+ %tmp1 = tail call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %v13)
+ %v20 = insertelement <4 x float> poison, float %f, i32 0
+ %v21 = insertelement <4 x float> %v20, float 0.000000e+00, i32 1
+ %v22 = insertelement <4 x float> %v21, float 0.000000e+00, i32 2
+ %v23 = insertelement <4 x float> %v22, float 0.000000e+00, i32 3
+ %tmp2 = tail call i32 @llvm.x86.sse.cvttss2si(<4 x float> %v23)
+ %v30 = insertelement <4 x float> poison, float %f, i32 0
+ %v31 = insertelement <4 x float> %v30, float 0.000000e+00, i32 1
+ %v32 = insertelement <4 x float> %v31, float 0.000000e+00, i32 2
+ %v33 = insertelement <4 x float> %v32, float 0.000000e+00, i32 3
+ %tmp3 = tail call i64 @llvm.x86.sse.cvttss2si64(<4 x float> %v33)
+ %v40 = insertelement <2 x double> poison, double %d, i32 0
+ %v41 = insertelement <2 x double> %v40, double 0.000000e+00, i32 1
+ %tmp4 = tail call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %v41)
+ %v50 = insertelement <2 x double> poison, double %d, i32 0
+ %v51 = insertelement <2 x double> %v50, double 0.000000e+00, i32 1
+ %tmp5 = tail call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %v51)
+ %v60 = insertelement <2 x double> poison, double %d, i32 0
+ %v61 = insertelement <2 x double> %v60, double 0.000000e+00, i32 1
+ %tmp6 = tail call i32 @llvm.x86.sse2.cvttsd2si(<2 x double> %v61)
+ %v70 = insertelement <2 x double> poison, double %d, i32 0
+ %v71 = insertelement <2 x double> %v70, double 0.000000e+00, i32 1
+ %tmp7 = tail call i64 @llvm.x86.sse2.cvttsd2si64(<2 x double> %v71)
+ %tmp8 = add i32 %tmp0, %tmp2
+ %tmp9 = add i32 %tmp4, %tmp6
+ %tmp10 = add i32 %tmp8, %tmp9
+ %tmp11 = sext i32 %tmp10 to i64
+ %tmp12 = add i64 %tmp1, %tmp3
+ %tmp13 = add i64 %tmp5, %tmp7
+ %tmp14 = add i64 %tmp12, %tmp13
+ %tmp15 = add i64 %tmp11, %tmp14
+ ret i64 %tmp15
+}
+
+declare <4 x float> @llvm.x86.sse.sub.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.mul.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>)
+declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>)
+declare i32 @llvm.x86.sse.cvtss2si(<4 x float>)
+declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>)
+declare i32 @llvm.x86.sse.cvttss2si(<4 x float>)
+declare i64 @llvm.x86.sse.cvttss2si64(<4 x float>)
+declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>)
+declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>)
+declare i32 @llvm.x86.sse2.cvttsd2si(<2 x double>)
+declare i64 @llvm.x86.sse2.cvttsd2si64(<2 x double>)
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts-inseltpoison.ll
new file mode 100644
index 000000000000..fe9fe3b975cc
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts-inseltpoison.ll
@@ -0,0 +1,3783 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+;
+; ASHR - Immediate
+;
+
+define <8 x i16> @sse2_psrai_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 0)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrai_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 15)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrai_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrai_w_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 64)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrai_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 0)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrai_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 15)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrai_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrai_d_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %v, i32 64)
+ ret <4 x i32> %1
+}
+
+define <16 x i16> @avx2_psrai_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 0)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrai_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 15)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrai_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrai_w_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %v, i32 64)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrai_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 0)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrai_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 15)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrai_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrai_d_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 64)
+ ret <8 x i32> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 0)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_15(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 15)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psrai_q_128_64(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_128_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 63>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64> %v, i32 64)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 0)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 15)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psrai_q_256_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_256_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64> %v, i32 64)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 0)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 15)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrai_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrai_w_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %v, i32 64)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 0)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 15)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrai_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrai_d_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %v, i32 64)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 0)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 15)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrai_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrai_q_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 64)
+ ret <8 x i64> %1
+}
+
+;
+; LSHR - Immediate
+;
+
+define <8 x i16> @sse2_psrli_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 0)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrli_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 15)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrli_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrli_w_64(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %v, i32 64)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrli_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 0)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrli_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 15)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrli_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrli_d_64(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 64)
+ ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psrli_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 0)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrli_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 15)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrli_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrli_q_64(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %v, i32 64)
+ ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psrli_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 0)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrli_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 15)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrli_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrli_w_64(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %v, i32 64)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrli_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 0)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrli_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 15)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrli_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrli_d_64(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %v, i32 64)
+ ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psrli_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 0)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrli_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 15)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrli_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrli_q_64(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 64)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 0)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 15)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrli_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrli_w_512_64(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 64)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 0)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 15)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrli_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrli_d_512_64(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %v, i32 64)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 0)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 15)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrli_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrli_q_512_64(
+; CHECK-NEXT: ret <8 x i64> zeroinitializer
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %v, i32 64)
+ ret <8 x i64> %1
+}
+
+;
+; SHL - Immediate
+;
+
+define <8 x i16> @sse2_pslli_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 0)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_pslli_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 15)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_pslli_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_pslli_w_64(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %v, i32 64)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_pslli_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 0)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_pslli_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 15)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_pslli_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_pslli_d_64(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %v, i32 64)
+ ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_pslli_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 0)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_pslli_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 15)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_pslli_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_pslli_q_64(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 64)
+ ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_pslli_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 0)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_pslli_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 15)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_pslli_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_pslli_w_64(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 64)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_pslli_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 0)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_pslli_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 15)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_pslli_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_pslli_d_64(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %v, i32 64)
+ ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_pslli_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 0)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_pslli_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 15)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_pslli_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_pslli_q_64(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %v, i32 64)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 0)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 15)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_pslli_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_pslli_w_512_64(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16> %v, i32 64)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 0)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 15)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_pslli_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_pslli_d_512_64(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 64)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 0)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 15)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_pslli_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_pslli_q_512_64(
+; CHECK-NEXT: ret <8 x i64> zeroinitializer
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %v, i32 64)
+ ret <8 x i64> %1
+}
+
+;
+; ASHR - Constant Vector
+;
+
+define <8 x i16> @sse2_psra_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psra_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psra_w_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psra_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psra_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psra_d_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <16 x i16> @avx2_psra_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psra_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psra_w_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psra_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psra_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psra_d_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_15(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @avx512_psra_q_128_64(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_128_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 63>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx512_psra_q_256_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_256_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_15_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psra_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psra_w_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_15_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psra_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psra_d_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psra_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psra_q_512_64(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <8 x i64> %1
+}
+
+;
+; LSHR - Constant Vector
+;
+
+define <8 x i16> @sse2_psrl_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_15_splat(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psrl_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psrl_w_64(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psrl_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_15_splat(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psrl_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psrl_d_64(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psrl_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrl_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psrl_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psrl_q_64(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psrl_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_15_splat(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psrl_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psrl_w_64(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psrl_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_15_splat(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psrl_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrl_d_64(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psrl_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrl_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psrl_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrl_q_64(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_15_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_15_splat(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrl_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrl_w_512_64(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_15_splat(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrl_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrl_d_512_64(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrl_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrl_q_512_64(
+; CHECK-NEXT: ret <8 x i64> zeroinitializer
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <8 x i64> %1
+}
+
+;
+; SHL - Constant Vector
+;
+
+define <8 x i16> @sse2_psll_w_0(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_15(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_15_splat(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_15_splat(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @sse2_psll_w_64(<8 x i16> %v) {
+; CHECK-LABEL: @sse2_psll_w_64(
+; CHECK-NEXT: ret <8 x i16> zeroinitializer
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <8 x i16> %1
+}
+
+define <4 x i32> @sse2_psll_d_0(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_15(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_15_splat(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_15_splat(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @sse2_psll_d_64(<4 x i32> %v) {
+; CHECK-LABEL: @sse2_psll_d_64(
+; CHECK-NEXT: ret <4 x i32> zeroinitializer
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <4 x i32> %1
+}
+
+define <2 x i64> @sse2_psll_q_0(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psll_q_15(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 15, i64 15>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @sse2_psll_q_64(<2 x i64> %v) {
+; CHECK-LABEL: @sse2_psll_q_64(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <2 x i64> %1
+}
+
+define <16 x i16> @avx2_psll_w_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_15(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_15_splat(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_15_splat(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx2_psll_w_64(<16 x i16> %v) {
+; CHECK-LABEL: @avx2_psll_w_64(
+; CHECK-NEXT: ret <16 x i16> zeroinitializer
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <16 x i16> %1
+}
+
+define <8 x i32> @avx2_psll_d_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_15(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_15_splat(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_15_splat(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <8 x i32> %1
+}
+
+define <8 x i32> @avx2_psll_d_64(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psll_d_64(
+; CHECK-NEXT: ret <8 x i32> zeroinitializer
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <8 x i32> %1
+}
+
+define <4 x i64> @avx2_psll_q_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psll_q_15(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <4 x i64> @avx2_psll_q_64(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psll_q_64(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <4 x i64> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_15(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_15_512_splat(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_15_512_splat(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psll_w_512_64(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psll_w_512_64(
+; CHECK-NEXT: ret <32 x i16> zeroinitializer
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> <i16 64, i16 0, i16 0, i16 0, i16 9999, i16 9999, i16 9999, i16 9999>)
+ ret <32 x i16> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_15(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_15_splat(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_15_splat(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psll_d_512_64(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psll_d_512_64(
+; CHECK-NEXT: ret <16 x i32> zeroinitializer
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> <i32 64, i32 0, i32 9999, i32 9999>)
+ ret <16 x i32> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_15(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_15(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15, i64 15>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> <i64 15, i64 9999>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psll_q_512_64(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psll_q_512_64(
+; CHECK-NEXT: ret <8 x i64> zeroinitializer
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> <i64 64, i64 9999>)
+ ret <8 x i64> %1
+}
+
+;
+; ASHR - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psrav_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 31, i32 31, i32 31, i32 undef>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrav_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <8 x i32> %1
+}
+
+define <16 x i32> @avx512_psrav_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 undef, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <16 x i32> %1
+}
+
+define <4 x i32> @avx2_psrav_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 64>, i32 undef, i32 0
+ %2 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psrav_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrav_d_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <16 x i32> @avx512_psrav_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrav_d_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 32, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %v, <16 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <2 x i64> @avx512_psrav_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 0, i64 8>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> <i64 64, i64 undef>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx512_psrav_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx512_psrav_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i64> [[V:%.*]], <i64 undef, i64 8>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 0
+ %2 = tail call <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @avx512_psrav_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64> %v, <4 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <8 x i64> @avx512_psrav_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 63, i64 undef, i64 63, i64 63, i64 63, i64 undef, i64 63, i64 63>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrav_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrav_q_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %v, <8 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psrav_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrav_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+ %2 = tail call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psrav_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrav_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+ %2 = tail call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %v, <16 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psrav_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_allbig(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 undef, i16 15, i16 15, i16 undef, i16 15, i16 15>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrav_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrav_w_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> %1)
+ ret <32 x i16> %2
+}
+
+;
+; LSHR - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psrlv_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_big(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> [[V:%.*]], <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_big(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> [[V:%.*]], <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_allbig(
+; CHECK-NEXT: ret <4 x i32> <i32 0, i32 0, i32 0, i32 undef>
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psrlv_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_allbig(
+; CHECK-NEXT: ret <8 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psrlv_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
+ %2 = tail call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psrlv_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <2 x i64> @avx2_psrlv_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i64> [[V:%.*]], <i64 0, i64 8>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_big(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> [[V:%.*]], <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_big(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> [[V:%.*]], <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psrlv_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_allbig(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> <i64 128, i64 -64>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psrlv_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_allbig(
+; CHECK-NEXT: ret <4 x i64> <i64 0, i64 undef, i64 0, i64 0>
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <4 x i64> %1
+}
+
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
+define <2 x i64> @avx2_psrlv_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_128_undef(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
+ %2 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @avx2_psrlv_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psrlv_q_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %v, <4 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <16 x i32> @avx2_psrlv_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx2_psrlv_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_big(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> [[V:%.*]], <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_allbig(
+; CHECK-NEXT: ret <16 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psrlv_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psrlv_d_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %v, <16 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psrlv_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_big(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> [[V:%.*]], <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_allbig(
+; CHECK-NEXT: ret <8 x i64> <i64 0, i64 undef, i64 0, i64 0, i64 0, i64 undef, i64 0, i64 0>
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psrlv_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psrlv_q_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psrlv_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_big(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> [[V:%.*]], <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_allbig(
+; CHECK-NEXT: ret <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef>
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psrlv_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+ %2 = tail call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psrlv_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_big(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> [[V:%.*]], <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_allbig(
+; CHECK-NEXT: ret <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psrlv_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+ %2 = tail call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> %v, <16 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psrlv_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_big(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> [[V:%.*]], <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_allbig(
+; CHECK-NEXT: ret <32 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 undef, i16 0, i16 0>
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psrlv_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psrlv_w_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %v, <32 x i16> %1)
+ ret <32 x i16> %2
+}
+
+;
+; SHL - Constant Per-Element Vector
+;
+
+define <4 x i32> @avx2_psllv_d_128_0(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_0(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> zeroinitializer)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_0(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_0(
+; CHECK-NEXT: ret <8 x i32> [[V:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> zeroinitializer)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_var(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 31>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_var(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_big(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> [[V:%.*]], <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 0, i32 8, i32 16, i32 64>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_big(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> [[V:%.*]], <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_allbig(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_allbig(
+; CHECK-NEXT: ret <4 x i32> <i32 0, i32 0, i32 0, i32 undef>
+;
+ %1 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> <i32 32, i32 100, i32 -255, i32 undef>)
+ ret <4 x i32> %1
+}
+
+define <8 x i32> @avx2_psllv_d_256_allbig(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_allbig(
+; CHECK-NEXT: ret <8 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <8 x i32> %1
+}
+
+define <4 x i32> @avx2_psllv_d_128_undef(<4 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 undef, i32 8, i32 16, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = insertelement <4 x i32> <i32 0, i32 8, i32 16, i32 31>, i32 undef, i32 0
+ %2 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <8 x i32> @avx2_psllv_d_256_undef(<8 x i32> %v) {
+; CHECK-LABEL: @avx2_psllv_d_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = insertelement <8 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %v, <8 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <2 x i64> @avx2_psllv_q_128_0(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_0(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> zeroinitializer)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_0(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_0(
+; CHECK-NEXT: ret <4 x i64> [[V:%.*]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> zeroinitializer)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_var(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[V:%.*]], <i64 0, i64 8>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 8>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_var(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 31>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_big(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> [[V:%.*]], <2 x i64> <i64 0, i64 128>)
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 0, i64 128>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_big(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> [[V:%.*]], <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 0, i64 8, i64 16, i64 64>)
+ ret <4 x i64> %1
+}
+
+define <2 x i64> @avx2_psllv_q_128_allbig(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_allbig(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %1 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> <i64 128, i64 -64>)
+ ret <2 x i64> %1
+}
+
+define <4 x i64> @avx2_psllv_q_256_allbig(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_allbig(
+; CHECK-NEXT: ret <4 x i64> <i64 0, i64 undef, i64 0, i64 0>
+;
+ %1 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> <i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <4 x i64> %1
+}
+
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
+define <2 x i64> @avx2_psllv_q_128_undef(<2 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_128_undef(
+; CHECK-NEXT: ret <2 x i64> [[V:%.*]]
+;
+ %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
+ %2 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @avx2_psllv_q_256_undef(<4 x i64> %v) {
+; CHECK-LABEL: @avx2_psllv_q_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = insertelement <4 x i64> <i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <16 x i32> @avx512_psllv_d_512_0(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[V:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> zeroinitializer)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_var(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 24, i32 31, i32 24, i32 8, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_big(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> [[V:%.*]], <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 64, i32 31, i32 24, i32 8, i32 0>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_allbig(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_allbig(
+; CHECK-NEXT: ret <16 x i32> <i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 undef, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> <i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555, i32 undef, i32 100, i32 255, i32 55555, i32 -32, i32 -100, i32 -255, i32 -55555>)
+ ret <16 x i32> %1
+}
+
+define <16 x i32> @avx512_psllv_d_512_undef(<16 x i32> %v) {
+; CHECK-LABEL: @avx512_psllv_d_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i32> [[V:%.*]], <i32 0, i32 undef, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = insertelement <16 x i32> <i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0, i32 0, i32 8, i32 16, i32 31, i32 31, i32 24, i32 8, i32 0>, i32 undef, i32 1
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %v, <16 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psllv_q_512_0(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_0(
+; CHECK-NEXT: ret <8 x i64> [[V:%.*]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> zeroinitializer)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_var(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_big(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> [[V:%.*]], <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 0, i64 8, i64 16, i64 64, i64 0, i64 8, i64 16, i64 64>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_allbig(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_allbig(
+; CHECK-NEXT: ret <8 x i64> <i64 0, i64 undef, i64 0, i64 0, i64 0, i64 undef, i64 0, i64 0>
+;
+ %1 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> <i64 64, i64 undef, i64 -128, i64 -60, i64 64, i64 undef, i64 -128, i64 -60>)
+ ret <8 x i64> %1
+}
+
+define <8 x i64> @avx512_psllv_q_512_undef(<8 x i64> %v) {
+; CHECK-LABEL: @avx512_psllv_q_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i64> [[V:%.*]], <i64 undef, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = insertelement <8 x i64> <i64 0, i64 8, i64 16, i64 31, i64 0, i64 8, i64 16, i64 31>, i64 undef, i64 0
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %v, <8 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <8 x i16> @avx512_psllv_w_128_0(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_0(
+; CHECK-NEXT: ret <8 x i16> [[V:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> zeroinitializer)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_var(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_big(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> [[V:%.*]], <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 16>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_allbig(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_allbig(
+; CHECK-NEXT: ret <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef>
+;
+ %1 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 undef>)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @avx512_psllv_w_128_undef(<8 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_128_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <8 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = insertelement <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>, i16 undef, i64 0
+ %2 = tail call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <16 x i16> @avx512_psllv_w_256_0(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_0(
+; CHECK-NEXT: ret <16 x i16> [[V:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> zeroinitializer)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_var(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_big(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> [[V:%.*]], <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 16>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_allbig(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_allbig(
+; CHECK-NEXT: ret <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 256, i16 16, i16 28, i16 65535, i16 32767>)
+ ret <16 x i16> %1
+}
+
+define <16 x i16> @avx512_psllv_w_256_undef(<16 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_256_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <16 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = insertelement <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>, i16 undef, i64 0
+ %2 = tail call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %v, <16 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <32 x i16> @avx512_psllv_w_512_0(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[V:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> zeroinitializer)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_var(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_big(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_big(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> [[V:%.*]], <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_allbig(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_allbig(
+; CHECK-NEXT: ret <32 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 undef, i16 0, i16 0>
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> <i16 20, i16 -1, i16 -2, i16 33, i16 44, i16 55, i16 66, i16 -7, i16 undef, i16 64, i16 -10, i16 128, i16 16, i16 28, i16 65535, i16 32767, i16 56, i16 -14, i16 undef, i16 16, i16 67, i16 567, i16 -32768, i16 4096, i16 8192, i16 -12345, i16 undef, i16 345, i16 123, i16 undef, i16 1024, i16 54321>)
+ ret <32 x i16> %1
+}
+
+define <32 x i16> @avx512_psllv_w_512_undef(<32 x i16> %v) {
+; CHECK-LABEL: @avx512_psllv_w_512_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <32 x i16> [[V:%.*]], <i16 undef, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = insertelement <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, i16 undef, i64 0
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> %1)
+ ret <32 x i16> %2
+}
+
+;
+; Vector Masked Shift Amounts
+;
+
+define <8 x i16> @sse2_psra_w_128_masked(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psra_w_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i16> [[A:%.*]], <i16 15, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <8 x i16> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = and <8 x i16> %a, <i16 15, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef>
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <8 x i32> @avx2_psra_d_256_masked(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psra_d_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 31, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <8 x i32> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %1 = and <4 x i32> %a, <i32 31, i32 0, i32 undef, i32 undef>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <8 x i64> @avx512_psra_q_512_masked(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i64> [[A:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <8 x i64> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i64> [[TMP3]]
+;
+ %1 = and <2 x i64> %a, <i64 63, i64 undef>
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <4 x i32> @sse2_psrl_d_128_masked(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psrl_d_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 31, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i32> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %1 = and <4 x i32> %a, <i32 31, i32 0, i32 undef, i32 undef>
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <4 x i64> @avx2_psrl_q_256_masked(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrl_q_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i64> [[A:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i64> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %1 = and <2 x i64> %a, <i64 63, i64 undef>
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psrl_w_512_masked(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psrl_w_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i16> [[A:%.*]], <i16 15, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <32 x i16> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = and <8 x i16> %a, <i16 15, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %1)
+ ret <32 x i16> %2
+}
+
+define <2 x i64> @sse2_psll_q_128_masked(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psll_q_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i64> [[A:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shl <2 x i64> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %1 = and <2 x i64> %a, <i64 63, i64 undef>
+ %2 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_psll_w_256_masked(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psll_w_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i16> [[A:%.*]], <i16 15, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shl <16 x i16> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = and <8 x i16> %a, <i16 15, i16 0, i16 0, i16 0, i16 undef, i16 undef, i16 undef, i16 undef>
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <16 x i32> @avx512_psll_d_512_masked(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psll_d_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 31, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shl <16 x i32> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <16 x i32> [[TMP3]]
+;
+ %1 = and <4 x i32> %a, <i32 31, i32 0, i32 undef, i32 undef>
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <8 x i16> @sse2_psrai_w_128_masked(<8 x i16> %v, i32 %a) {
+; CHECK-LABEL: @sse2_psrai_w_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> undef, i16 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <8 x i16> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %1 = and i32 %a, 15
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %v, i32 %1)
+ ret <8 x i16> %2
+}
+
+define <8 x i32> @avx2_psrai_d_256_masked(<8 x i32> %v, i32 %a) {
+; CHECK-LABEL: @avx2_psrai_d_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 31
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i32> [[DOTSPLATINSERT]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <8 x i32> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <8 x i32> [[TMP2]]
+;
+ %1 = and i32 %a, 31
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %v, i32 %1)
+ ret <8 x i32> %2
+}
+
+define <8 x i64> @avx512_psrai_q_512_masked(<8 x i64> %v, i32 %a) {
+; CHECK-LABEL: @avx512_psrai_q_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 63
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <8 x i64> undef, i64 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <8 x i64> [[DOTSPLATINSERT]], <8 x i64> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <8 x i64> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <8 x i64> [[TMP3]]
+;
+ %1 = and i32 %a, 63
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %v, i32 %1)
+ ret <8 x i64> %2
+}
+
+define <4 x i32> @sse2_psrli_d_128_masked(<4 x i32> %v, i32 %a) {
+; CHECK-LABEL: @sse2_psrli_d_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 31
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <4 x i32> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = and i32 %a, 31
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %v, i32 %1)
+ ret <4 x i32> %2
+}
+
+define <4 x i64> @avx2_psrli_q_256_masked(<4 x i64> %v, i32 %a) {
+; CHECK-LABEL: @avx2_psrli_q_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 63
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <4 x i64> undef, i64 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i64> [[DOTSPLATINSERT]], <4 x i64> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <4 x i64> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %1 = and i32 %a, 63
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %v, i32 %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psrli_w_512_masked(<32 x i16> %v, i32 %a) {
+; CHECK-LABEL: @avx512_psrli_w_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <32 x i16> undef, i16 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <32 x i16> [[DOTSPLATINSERT]], <32 x i16> undef, <32 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = lshr <32 x i16> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <32 x i16> [[TMP3]]
+;
+ %1 = and i32 %a, 15
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16> %v, i32 %1)
+ ret <32 x i16> %2
+}
+
+define <2 x i64> @sse2_pslli_q_128_masked(<2 x i64> %v, i32 %a) {
+; CHECK-LABEL: @sse2_pslli_q_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 63
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <2 x i64> undef, i64 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT]], <2 x i64> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shl <2 x i64> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %1 = and i32 %a, 63
+ %2 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %v, i32 %1)
+ ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_pslli_w_256_masked(<16 x i16> %v, i32 %a) {
+; CHECK-LABEL: @avx2_pslli_w_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[A:%.*]] to i16
+; CHECK-NEXT: [[TMP2:%.*]] = and i16 [[TMP1]], 15
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <16 x i16> undef, i16 [[TMP2]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <16 x i16> [[DOTSPLATINSERT]], <16 x i16> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = shl <16 x i16> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %1 = and i32 %a, 15
+ %2 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %v, i32 %1)
+ ret <16 x i16> %2
+}
+
+define <16 x i32> @avx512_pslli_d_512_masked(<16 x i32> %v, i32 %a) {
+; CHECK-LABEL: @avx512_pslli_d_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[A:%.*]], 31
+; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <16 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <16 x i32> [[DOTSPLATINSERT]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = shl <16 x i32> [[V:%.*]], [[DOTSPLAT]]
+; CHECK-NEXT: ret <16 x i32> [[TMP2]]
+;
+ %1 = and i32 %a, 31
+ %2 = tail call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %v, i32 %1)
+ ret <16 x i32> %2
+}
+
+define <4 x i32> @avx2_psrav_d_128_masked(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psrav_d_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <4 x i32> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = and <4 x i32> %a, <i32 31, i32 31, i32 31, i32 31>
+ %2 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <4 x i32> @avx2_psrav_d_128_masked_shuffle(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psrav_d_128_masked_shuffle(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 undef, i32 undef, i32 15, i32 31>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[V:%.*]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %1 = and <4 x i32> %a, <i32 undef, i32 undef, i32 15, i32 31>
+ %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 2, i32 3>
+ %3 = tail call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %v, <4 x i32> %2)
+ ret <4 x i32> %3
+}
+
+define <8 x i32> @avx2_psrav_d_256_masked(<8 x i32> %v, <8 x i32> %a) {
+; CHECK-LABEL: @avx2_psrav_d_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i32> [[A:%.*]], <i32 0, i32 1, i32 7, i32 15, i32 16, i32 30, i32 31, i32 31>
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <8 x i32> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <8 x i32> [[TMP2]]
+;
+ %1 = and <8 x i32> %a, <i32 0, i32 1, i32 7, i32 15, i32 16, i32 30, i32 31, i32 31>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %v, <8 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <32 x i16> @avx512_psrav_w_512_masked(<32 x i16> %v, <32 x i16> %a) {
+; CHECK-LABEL: @avx512_psrav_w_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <32 x i16> [[A:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: [[TMP2:%.*]] = ashr <32 x i16> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <32 x i16> [[TMP2]]
+;
+ %1 = and <32 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %v, <32 x i16> %1)
+ ret <32 x i16> %2
+}
+
+define <2 x i64> @avx2_psrlv_q_128_masked(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrlv_q_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i64> [[A:%.*]], <i64 32, i64 63>
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <2 x i64> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %1 = and <2 x i64> %a, <i64 32, i64 63>
+ %2 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <8 x i32> @avx2_psrlv_d_256_masked(<8 x i32> %v, <8 x i32> %a) {
+; CHECK-LABEL: @avx2_psrlv_d_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i32> [[A:%.*]], <i32 0, i32 1, i32 7, i32 15, i32 16, i32 30, i32 31, i32 31>
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <8 x i32> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <8 x i32> [[TMP2]]
+;
+ %1 = and <8 x i32> %a, <i32 0, i32 1, i32 7, i32 15, i32 16, i32 30, i32 31, i32 31>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %v, <8 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <8 x i64> @avx512_psrlv_q_512_masked(<8 x i64> %v, <8 x i64> %a) {
+; CHECK-LABEL: @avx512_psrlv_q_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <8 x i64> [[A:%.*]], <i64 0, i64 1, i64 4, i64 16, i64 32, i64 47, i64 62, i64 63>
+; CHECK-NEXT: [[TMP2:%.*]] = lshr <8 x i64> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <8 x i64> [[TMP2]]
+;
+ %1 = and <8 x i64> %a, <i64 0, i64 1, i64 4, i64 16, i64 32, i64 47, i64 62, i64 63>
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %v, <8 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <4 x i32> @avx2_psllv_d_128_masked(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psllv_d_128_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[A:%.*]], <i32 0, i32 15, i32 16, i32 31>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i32> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = and <4 x i32> %a, <i32 0, i32 15, i32 16, i32 31>
+ %2 = tail call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <4 x i64> @avx2_psllv_q_256_masked(<4 x i64> %v, <4 x i64> %a) {
+; CHECK-LABEL: @avx2_psllv_q_256_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i64> [[A:%.*]], <i64 0, i64 16, i64 32, i64 63>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i64> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <4 x i64> [[TMP2]]
+;
+ %1 = and <4 x i64> %a, <i64 0, i64 16, i64 32, i64 63>
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %v, <4 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psllv_w_512_masked(<32 x i16> %v, <32 x i16> %a) {
+; CHECK-LABEL: @avx512_psllv_w_512_masked(
+; CHECK-NEXT: [[TMP1:%.*]] = and <32 x i16> [[A:%.*]], <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <32 x i16> [[V:%.*]], [[TMP1]]
+; CHECK-NEXT: ret <32 x i16> [[TMP2]]
+;
+ %1 = and <32 x i16> %a, <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %v, <32 x i16> %1)
+ ret <32 x i16> %2
+}
+
+;
+; Vector Demanded Bits
+;
+
+define <8 x i16> @sse2_psra_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psra_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <8 x i16> @sse2_psra_w_var_bc(<8 x i16> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psra_w_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = bitcast <2 x i64> %1 to <8 x i16>
+ %3 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %v, <8 x i16> %2)
+ ret <8 x i16> %3
+}
+
+define <4 x i32> @sse2_psra_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psra_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <4 x i32> @sse2_psra_d_var_bc(<4 x i32> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psra_d_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = bitcast <8 x i16> %1 to <4 x i32>
+ %3 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %v, <4 x i32> %2)
+ ret <4 x i32> %3
+}
+
+define <16 x i16> @avx2_psra_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psra_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %v, <8 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <8 x i32> @avx2_psra_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psra_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %v, <4 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <2 x i64> @avx512_psra_q_128_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_128_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <4 x i64> @avx512_psra_q_256_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_256_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64> %v, <2 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psra_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psra_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %v, <8 x i16> %1)
+ ret <32 x i16> %2
+}
+
+define <16 x i32> @avx512_psra_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psra_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %v, <4 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psra_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psra_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %v, <2 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <8 x i16> @sse2_psrl_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psrl_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <4 x i32> @sse2_psrl_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psrl_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <2 x i64> @sse2_psrl_q_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psrl_q_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_psrl_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psrl_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <16 x i16> @avx2_psrl_w_var_bc(<16 x i16> %v, <16 x i8> %a) {
+; CHECK-LABEL: @avx2_psrl_w_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
+; CHECK-NEXT: ret <16 x i16> [[TMP2]]
+;
+ %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = bitcast <16 x i8> %1 to <8 x i16>
+ %3 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %v, <8 x i16> %2)
+ ret <16 x i16> %3
+}
+
+define <8 x i32> @avx2_psrl_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psrl_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <8 x i32> @avx2_psrl_d_var_bc(<8 x i32> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrl_d_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
+; CHECK-NEXT: ret <8 x i32> [[TMP2]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = bitcast <2 x i64> %1 to <4 x i32>
+ %3 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %v, <4 x i32> %2)
+ ret <8 x i32> %3
+}
+
+define <4 x i64> @avx2_psrl_q_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psrl_q_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %v, <2 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psrl_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psrl_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %1)
+ ret <32 x i16> %2
+}
+
+define <32 x i16> @avx512_psrl_w_512_var_bc(<32 x i16> %v, <16 x i8> %a) {
+; CHECK-LABEL: @avx512_psrl_w_512_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[TMP1]])
+; CHECK-NEXT: ret <32 x i16> [[TMP2]]
+;
+ %1 = shufflevector <16 x i8> %a, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %2 = bitcast <16 x i8> %1 to <8 x i16>
+ %3 = tail call <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16> %v, <8 x i16> %2)
+ ret <32 x i16> %3
+}
+
+define <16 x i32> @avx512_psrl_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psrl_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <16 x i32> @avx512_psrl_d_512_var_bc(<16 x i32> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psrl_d_512_var_bc(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[TMP1]])
+; CHECK-NEXT: ret <16 x i32> [[TMP2]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = bitcast <2 x i64> %1 to <4 x i32>
+ %3 = tail call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %v, <4 x i32> %2)
+ ret <16 x i32> %3
+}
+
+define <8 x i64> @avx512_psrl_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psrl_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %v, <2 x i64> %1)
+ ret <8 x i64> %2
+}
+
+define <8 x i16> @sse2_psll_w_var(<8 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @sse2_psll_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %v, <8 x i16> %1)
+ ret <8 x i16> %2
+}
+
+define <4 x i32> @sse2_psll_d_var(<4 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @sse2_psll_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %v, <4 x i32> %1)
+ ret <4 x i32> %2
+}
+
+define <2 x i64> @sse2_psll_q_var(<2 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @sse2_psll_q_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %v, <2 x i64> %1)
+ ret <2 x i64> %2
+}
+
+define <16 x i16> @avx2_psll_w_var(<16 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx2_psll_w_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %v, <8 x i16> %1)
+ ret <16 x i16> %2
+}
+
+define <8 x i32> @avx2_psll_d_var(<8 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx2_psll_d_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %v, <4 x i32> %1)
+ ret <8 x i32> %2
+}
+
+define <4 x i64> @avx2_psll_q_var(<4 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx2_psll_q_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <4 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %v, <2 x i64> %1)
+ ret <4 x i64> %2
+}
+
+define <32 x i16> @avx512_psll_w_512_var(<32 x i16> %v, <8 x i16> %a) {
+; CHECK-LABEL: @avx512_psll_w_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> [[V:%.*]], <8 x i16> [[A:%.*]])
+; CHECK-NEXT: ret <32 x i16> [[TMP1]]
+;
+ %1 = shufflevector <8 x i16> %a, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16> %v, <8 x i16> %1)
+ ret <32 x i16> %2
+}
+
+define <16 x i32> @avx512_psll_d_512_var(<16 x i32> %v, <4 x i32> %a) {
+; CHECK-LABEL: @avx512_psll_d_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> [[V:%.*]], <4 x i32> [[A:%.*]])
+; CHECK-NEXT: ret <16 x i32> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32> %a, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %v, <4 x i32> %1)
+ ret <16 x i32> %2
+}
+
+define <8 x i64> @avx512_psll_q_512_var(<8 x i64> %v, <2 x i64> %a) {
+; CHECK-LABEL: @avx512_psll_q_512_var(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> [[V:%.*]], <2 x i64> [[A:%.*]])
+; CHECK-NEXT: ret <8 x i64> [[TMP1]]
+;
+ %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> <i32 0, i32 0>
+ %2 = tail call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %v, <2 x i64> %1)
+ ret <8 x i64> %2
+}
+
+;
+; Constant Folding
+;
+
+define <8 x i16> @test_sse2_psra_w_0(<8 x i16> %A) {
+; CHECK-LABEL: @test_sse2_psra_w_0(
+; CHECK-NEXT: ret <8 x i16> [[A:%.*]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %A, i32 0)
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %3 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %2, i32 0)
+ ret <8 x i16> %3
+}
+
+define <8 x i16> @test_sse2_psra_w_8() {
+; CHECK-LABEL: @test_sse2_psra_w_8(
+; CHECK-NEXT: ret <8 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+ %1 = bitcast <2 x i64> <i64 1152956690052710400, i64 1152956690052710400> to <8 x i16>
+ %2 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %1, i32 3)
+ %3 = tail call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %4 = tail call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %3, i32 2)
+ ret <8 x i16> %4
+}
+
+define <4 x i32> @test_sse2_psra_d_0(<4 x i32> %A) {
+; CHECK-LABEL: @test_sse2_psra_d_0(
+; CHECK-NEXT: ret <4 x i32> [[A:%.*]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %A, i32 0)
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+ %3 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %1, i32 0)
+ ret <4 x i32> %3
+}
+
+define <4 x i32> @sse2_psra_d_8() {
+; CHECK-LABEL: @sse2_psra_d_8(
+; CHECK-NEXT: ret <4 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+ %1 = bitcast <2 x i64> <i64 1152956690052710400, i64 1152956690052710400> to <4 x i32>
+ %2 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %1, i32 3)
+ %3 = tail call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+ %4 = tail call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %3, i32 2)
+ ret <4 x i32> %4
+}
+
+define <16 x i16> @test_avx2_psra_w_0(<16 x i16> %A) {
+; CHECK-LABEL: @test_avx2_psra_w_0(
+; CHECK-NEXT: ret <16 x i16> [[A:%.*]]
+;
+ %1 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %A, i32 0)
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %3 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %2, i32 0)
+ ret <16 x i16> %3
+}
+
+define <16 x i16> @test_avx2_psra_w_8(<16 x i16> %A) {
+; CHECK-LABEL: @test_avx2_psra_w_8(
+; CHECK-NEXT: ret <16 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+ %1 = bitcast <4 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <16 x i16>
+ %2 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %1, i32 3)
+ %3 = tail call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %4 = tail call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %3, i32 2)
+ ret <16 x i16> %4
+}
+
+define <8 x i32> @test_avx2_psra_d_0(<8 x i32> %A) {
+; CHECK-LABEL: @test_avx2_psra_d_0(
+; CHECK-NEXT: ret <8 x i32> [[A:%.*]]
+;
+ %1 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %A, i32 0)
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+ %3 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %2, i32 0)
+ ret <8 x i32> %3
+}
+
+define <8 x i32> @test_avx2_psra_d_8() {
+; CHECK-LABEL: @test_avx2_psra_d_8(
+; CHECK-NEXT: ret <8 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+ %1 = bitcast <4 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <8 x i32>
+ %2 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %1, i32 3)
+ %3 = tail call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+ %4 = tail call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %3, i32 2)
+ ret <8 x i32> %4
+}
+
+define <32 x i16> @test_avx512_psra_w_512_0(<32 x i16> %A) {
+; CHECK-LABEL: @test_avx512_psra_w_512_0(
+; CHECK-NEXT: ret <32 x i16> [[A:%.*]]
+;
+ %1 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %A, i32 0)
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %1, <8 x i16> <i16 0, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %3 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %2, i32 0)
+ ret <32 x i16> %3
+}
+
+define <32 x i16> @test_avx512_psra_w_512_8(<32 x i16> %A) {
+; CHECK-LABEL: @test_avx512_psra_w_512_8(
+; CHECK-NEXT: ret <32 x i16> <i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16, i16 -128, i16 64, i16 32, i16 16>
+;
+ %1 = bitcast <8 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <32 x i16>
+ %2 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %1, i32 3)
+ %3 = tail call <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16> %2, <8 x i16> <i16 3, i16 0, i16 0, i16 0, i16 7, i16 0, i16 0, i16 0>)
+ %4 = tail call <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16> %3, i32 2)
+ ret <32 x i16> %4
+}
+
+define <16 x i32> @test_avx512_psra_d_512_0(<16 x i32> %A) {
+; CHECK-LABEL: @test_avx512_psra_d_512_0(
+; CHECK-NEXT: ret <16 x i32> [[A:%.*]]
+;
+ %1 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %A, i32 0)
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %1, <4 x i32> <i32 0, i32 0, i32 7, i32 0>)
+ %3 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %2, i32 0)
+ ret <16 x i32> %3
+}
+
+define <16 x i32> @test_avx512_psra_d_512_8() {
+; CHECK-LABEL: @test_avx512_psra_d_512_8(
+; CHECK-NEXT: ret <16 x i32> <i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608, i32 4194432, i32 1048608>
+;
+ %1 = bitcast <8 x i64> <i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400, i64 1152956690052710400> to <16 x i32>
+ %2 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %1, i32 3)
+ %3 = tail call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %2, <4 x i32> <i32 3, i32 0, i32 7, i32 0>)
+ %4 = tail call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %3, i32 2)
+ ret <16 x i32> %4
+}
+
+;
+; Old Tests
+;
+
+define <2 x i64> @test_sse2_1() {
+; CHECK-LABEL: @test_sse2_1(
+; CHECK-NEXT: ret <2 x i64> <i64 72058418680037440, i64 144117112246370624>
+;
+ %S = bitcast i32 1 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> %4)
+ %6 = bitcast <8 x i16> %5 to <4 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <4 x i32> %8 to <2 x i64>
+ %10 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <2 x i64> %10 to <8 x i16>
+ %12 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %11, i32 %S)
+ %13 = bitcast <8 x i16> %12 to <4 x i32>
+ %14 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %13, i32 %S)
+ %15 = bitcast <4 x i32> %14 to <2 x i64>
+ %16 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %15, i32 %S)
+ ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_1() {
+; CHECK-LABEL: @test_avx2_1(
+; CHECK-NEXT: ret <4 x i64> <i64 64, i64 128, i64 192, i64 256>
+;
+ %S = bitcast i32 1 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> <i16 1, i16 0, i16 0, i16 0, i16 2, i16 0, i16 0, i16 0, i16 3, i16 0, i16 0, i16 0, i16 4, i16 0, i16 0, i16 0>, <8 x i16> %4)
+ %6 = bitcast <16 x i16> %5 to <8 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <8 x i32> %8 to <4 x i64>
+ %10 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <4 x i64> %10 to <16 x i16>
+ %12 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %11, i32 %S)
+ %13 = bitcast <16 x i16> %12 to <8 x i32>
+ %14 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %13, i32 %S)
+ %15 = bitcast <8 x i32> %14 to <4 x i64>
+ %16 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %15, i32 %S)
+ ret <4 x i64> %16
+}
+
+define <2 x i64> @test_sse2_0() {
+; CHECK-LABEL: @test_sse2_0(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %S = bitcast i32 128 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> %4)
+ %6 = bitcast <8 x i16> %5 to <4 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <4 x i32> %8 to <2 x i64>
+ %10 = tail call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <2 x i64> %10 to <8 x i16>
+ %12 = tail call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %11, i32 %S)
+ %13 = bitcast <8 x i16> %12 to <4 x i32>
+ %14 = tail call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %13, i32 %S)
+ %15 = bitcast <4 x i32> %14 to <2 x i64>
+ %16 = tail call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %15, i32 %S)
+ ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_0() {
+; CHECK-LABEL: @test_avx2_0(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %S = bitcast i32 128 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> <i16 1, i16 0, i16 0, i16 0, i16 2, i16 0, i16 0, i16 0, i16 3, i16 0, i16 0, i16 0, i16 4, i16 0, i16 0, i16 0>, <8 x i16> %4)
+ %6 = bitcast <16 x i16> %5 to <8 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <8 x i32> %8 to <4 x i64>
+ %10 = tail call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <4 x i64> %10 to <16 x i16>
+ %12 = tail call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %11, i32 %S)
+ %13 = bitcast <16 x i16> %12 to <8 x i32>
+ %14 = tail call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %13, i32 %S)
+ %15 = bitcast <8 x i32> %14 to <4 x i64>
+ %16 = tail call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %15, i32 %S)
+ ret <4 x i64> %16
+}
+define <2 x i64> @test_sse2_psrl_1() {
+; CHECK-LABEL: @test_sse2_psrl_1(
+; CHECK-NEXT: ret <2 x i64> <i64 562954248421376, i64 9007267974742020>
+;
+ %S = bitcast i32 1 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> <i16 16, i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048>, <8 x i16> %4)
+ %6 = bitcast <8 x i16> %5 to <4 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <4 x i32> %8 to <2 x i64>
+ %10 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <2 x i64> %10 to <8 x i16>
+ %12 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %11, i32 %S)
+ %13 = bitcast <8 x i16> %12 to <4 x i32>
+ %14 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %13, i32 %S)
+ %15 = bitcast <4 x i32> %14 to <2 x i64>
+ %16 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %15, i32 %S)
+ ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_psrl_1() {
+; CHECK-LABEL: @test_avx2_psrl_1(
+; CHECK-NEXT: ret <4 x i64> <i64 16, i64 32, i64 64, i64 128>
+;
+ %S = bitcast i32 1 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> <i16 1024, i16 0, i16 0, i16 0, i16 2048, i16 0, i16 0, i16 0, i16 4096, i16 0, i16 0, i16 0, i16 8192, i16 0, i16 0, i16 0>, <8 x i16> %4)
+ %6 = bitcast <16 x i16> %5 to <8 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <8 x i32> %8 to <4 x i64>
+ %10 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <4 x i64> %10 to <16 x i16>
+ %12 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %11, i32 %S)
+ %13 = bitcast <16 x i16> %12 to <8 x i32>
+ %14 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %13, i32 %S)
+ %15 = bitcast <8 x i32> %14 to <4 x i64>
+ %16 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %15, i32 %S)
+ ret <4 x i64> %16
+}
+
+define <2 x i64> @test_sse2_psrl_0() {
+; CHECK-LABEL: @test_sse2_psrl_0(
+; CHECK-NEXT: ret <2 x i64> zeroinitializer
+;
+ %S = bitcast i32 128 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> <i16 32, i16 64, i16 128, i16 256, i16 512, i16 1024, i16 2048, i16 4096>, <8 x i16> %4)
+ %6 = bitcast <8 x i16> %5 to <4 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <4 x i32> %8 to <2 x i64>
+ %10 = tail call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <2 x i64> %10 to <8 x i16>
+ %12 = tail call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %11, i32 %S)
+ %13 = bitcast <8 x i16> %12 to <4 x i32>
+ %14 = tail call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %13, i32 %S)
+ %15 = bitcast <4 x i32> %14 to <2 x i64>
+ %16 = tail call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %15, i32 %S)
+ ret <2 x i64> %16
+}
+
+define <4 x i64> @test_avx2_psrl_0() {
+; CHECK-LABEL: @test_avx2_psrl_0(
+; CHECK-NEXT: ret <4 x i64> zeroinitializer
+;
+ %S = bitcast i32 128 to i32
+ %1 = zext i32 %S to i64
+ %2 = insertelement <2 x i64> poison, i64 %1, i32 0
+ %3 = insertelement <2 x i64> %2, i64 0, i32 1
+ %4 = bitcast <2 x i64> %3 to <8 x i16>
+ %5 = tail call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> <i16 1024, i16 0, i16 0, i16 0, i16 2048, i16 0, i16 0, i16 0, i16 4096, i16 0, i16 0, i16 0, i16 8192, i16 0, i16 0, i16 0>, <8 x i16> %4)
+ %6 = bitcast <16 x i16> %5 to <8 x i32>
+ %7 = bitcast <2 x i64> %3 to <4 x i32>
+ %8 = tail call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %6, <4 x i32> %7)
+ %9 = bitcast <8 x i32> %8 to <4 x i64>
+ %10 = tail call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %9, <2 x i64> %3)
+ %11 = bitcast <4 x i64> %10 to <16 x i16>
+ %12 = tail call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %11, i32 %S)
+ %13 = bitcast <16 x i16> %12 to <8 x i32>
+ %14 = tail call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %13, i32 %S)
+ %15 = bitcast <8 x i32> %14 to <4 x i64>
+ %16 = tail call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %15, i32 %S)
+ ret <4 x i64> %16
+}
+
+declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.pslli.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) #1
+
+declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.psrli.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psrl.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) #1
+
+declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) #1
+declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) #1
+declare <32 x i16> @llvm.x86.avx512.psrai.w.512(<32 x i16>, i32) #1
+declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) #1
+declare <32 x i16> @llvm.x86.avx512.psra.w.512(<32 x i16>, <8 x i16>) #1
+declare <4 x i64> @llvm.x86.avx512.psrai.q.256(<4 x i64>, i32) #1
+declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) #1
+declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) #1
+declare <4 x i64> @llvm.x86.avx512.psra.q.256(<4 x i64>, <2 x i64>) #1
+declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) #1
+declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) #1
+declare <2 x i64> @llvm.x86.avx512.psrai.q.128(<2 x i64>, i32) #1
+declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) #1
+declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) #1
+declare <2 x i64> @llvm.x86.avx512.psra.q.128(<2 x i64>, <2 x i64>) #1
+declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) #1
+declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) #1
+declare <2 x i64> @llvm.x86.avx512.psrav.q.128(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx512.psrav.q.256(<4 x i64>, <4 x i64>) #1
+declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) #1
+declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) #1
+declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) #1
+declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) #1
+declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) #1
+declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) #1
+declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) #1
+declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) #1
+
+declare <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>) #1
+declare <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) #1
+declare <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16>, <8 x i16>) #1
+declare <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16>, <16 x i16>) #1
+declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) #1
+
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-xop-inseltpoison.ll b/llvm/test/Transforms/InstCombine/X86/x86-xop-inseltpoison.ll
new file mode 100644
index 000000000000..e09b8e2d3c15
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/X86/x86-xop-inseltpoison.ll
@@ -0,0 +1,305 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -mtriple=x86_64-unknown-unknown -S | FileCheck %s
+
+define <2 x double> @test_vfrcz_sd(<2 x double> %a) {
+; CHECK-LABEL: @test_vfrcz_sd(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> [[A:%.*]])
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
+ %2 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %1)
+ ret <2 x double> %2
+}
+
+define double @test_vfrcz_sd_0(double %a) {
+; CHECK-LABEL: @test_vfrcz_sd_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT: ret double [[TMP3]]
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %2)
+ %4 = extractelement <2 x double> %3, i32 0
+ ret double %4
+}
+
+define double @test_vfrcz_sd_1(double %a) {
+; CHECK-LABEL: @test_vfrcz_sd_1(
+; CHECK-NEXT: ret double 0.000000e+00
+;
+ %1 = insertelement <2 x double> poison, double %a, i32 0
+ %2 = insertelement <2 x double> %1, double 1.000000e+00, i32 1
+ %3 = tail call <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double> %2)
+ %4 = extractelement <2 x double> %3, i32 1
+ ret double %4
+}
+
+define <4 x float> @test_vfrcz_ss(<4 x float> %a) {
+; CHECK-LABEL: @test_vfrcz_ss(
+; CHECK-NEXT: [[TMP1:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> [[A:%.*]])
+; CHECK-NEXT: ret <4 x float> [[TMP1]]
+;
+ %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
+ %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
+ %3 = insertelement <4 x float> %2, float 3.000000e+00, i32 3
+ %4 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %3)
+ ret <4 x float> %4
+}
+
+define float @test_vfrcz_ss_0(float %a) {
+; CHECK-LABEL: @test_vfrcz_ss_0(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> [[TMP1]])
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: ret float [[TMP3]]
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 0
+ ret float %6
+}
+
+define float @test_vfrcz_ss_3(float %a) {
+; CHECK-LABEL: @test_vfrcz_ss_3(
+; CHECK-NEXT: ret float 0.000000e+00
+;
+ %1 = insertelement <4 x float> poison, float %a, i32 0
+ %2 = insertelement <4 x float> %1, float 1.000000e+00, i32 1
+ %3 = insertelement <4 x float> %2, float 2.000000e+00, i32 2
+ %4 = insertelement <4 x float> %3, float 3.000000e+00, i32 3
+ %5 = tail call <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float> %4)
+ %6 = extractelement <4 x float> %5, i32 3
+ ret float %6
+}
+
+define <2 x i64> @cmp_slt_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_slt_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.xop.vpcomltq(<2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_ult_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_ult_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ult <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.xop.vpcomltuq(<2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_sle_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_sle_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sle <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.xop.vpcomleq(<2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %1
+}
+
+define <2 x i64> @cmp_ule_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @cmp_ule_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ule <2 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <2 x i1> [[TMP1]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP2]]
+;
+ %1 = tail call <2 x i64> @llvm.x86.xop.vpcomleuq(<2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %1
+}
+
+define <4 x i32> @cmp_sgt_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_sgt_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sgt <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgtd(<4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_ugt_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_ugt_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ugt <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgtud(<4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_sge_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_sge_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp sge <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.xop.vpcomged(<4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %1
+}
+
+define <4 x i32> @cmp_uge_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @cmp_uge_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp uge <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[TMP2]]
+;
+ %1 = tail call <4 x i32> @llvm.x86.xop.vpcomgeud(<4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %1
+}
+
+define <8 x i16> @cmp_seq_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_seq_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.xop.vpcomeqw(<8 x i16> %a, <8 x i16> %b)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_ueq_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_ueq_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <8 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16> %a, <8 x i16> %b)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_sne_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_sne_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.xop.vpcomnew(<8 x i16> %a, <8 x i16> %b)
+ ret <8 x i16> %1
+}
+
+define <8 x i16> @cmp_une_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @cmp_une_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i16> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext <8 x i1> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TMP2]]
+;
+ %1 = tail call <8 x i16> @llvm.x86.xop.vpcomneuw(<8 x i16> %a, <8 x i16> %b)
+ ret <8 x i16> %1
+}
+
+define <16 x i8> @cmp_strue_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_strue_v16i8(
+; CHECK-NEXT: ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+;
+ %1 = tail call <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8> %a, <16 x i8> %b)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_utrue_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_utrue_v16i8(
+; CHECK-NEXT: ret <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+;
+ %1 = tail call <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8> %a, <16 x i8> %b)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_sfalse_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_sfalse_v16i8(
+; CHECK-NEXT: ret <16 x i8> zeroinitializer
+;
+ %1 = tail call <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8> %a, <16 x i8> %b)
+ ret <16 x i8> %1
+}
+
+define <16 x i8> @cmp_ufalse_v16i8(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @cmp_ufalse_v16i8(
+; CHECK-NEXT: ret <16 x i8> zeroinitializer
+;
+ %1 = tail call <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8> %a, <16 x i8> %b)
+ ret <16 x i8> %1
+}
+
+declare <2 x double> @llvm.x86.xop.vfrcz.sd(<2 x double>) nounwind readnone
+declare <4 x float> @llvm.x86.xop.vfrcz.ss(<4 x float>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomltb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomltw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomltd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomltq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomltub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomltuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomltud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomltuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomleb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomlew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomled(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomleq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomleub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomleuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomleud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomleuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomgtb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgtw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgtd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgtq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomgtub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgtuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgtud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgtuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomgeb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomged(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgeq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomgeub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomgeuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomgeud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomgeuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomeqb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomeqw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomeqd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomeqq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomequb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomequd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomequq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomneb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomnew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomned(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomneq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomneub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomneuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomneud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomneuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomfalsew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomfalsed(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomfalseq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomfalseuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomfalseud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomfalseuq(<2 x i64>, <2 x i64>) nounwind readnone
+
+declare <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomtrued(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomtrueq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomtruew(<8 x i16>, <8 x i16>) nounwind readnone
+declare <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8>, <16 x i8>) nounwind readnone
+declare <8 x i16> @llvm.x86.xop.vpcomtrueuw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <4 x i32> @llvm.x86.xop.vpcomtrueud(<4 x i32>, <4 x i32>) nounwind readnone
+declare <2 x i64> @llvm.x86.xop.vpcomtrueuq(<2 x i64>, <2 x i64>) nounwind readnone
diff --git a/llvm/test/Transforms/InstCombine/bitcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/bitcast-inseltpoison.ll
new file mode 100644
index 000000000000..7de03c3bbae3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-inseltpoison.ll
@@ -0,0 +1,573 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+; Bitcasts between vectors and scalars are valid.
+; PR4487
+define i32 @test1(i64 %a) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret i32 0
+;
+ %t1 = bitcast i64 %a to <2 x i32>
+ %t2 = bitcast i64 %a to <2 x i32>
+ %t3 = xor <2 x i32> %t1, %t2
+ %t4 = extractelement <2 x i32> %t3, i32 0
+ ret i32 %t4
+}
+
+; Perform the bitwise logic in the source type of the operands to eliminate bitcasts.
+
+define <2 x i32> @xor_two_vector_bitcasts(<1 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: @xor_two_vector_bitcasts(
+; CHECK-NEXT: [[T31:%.*]] = xor <1 x i64> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[T3:%.*]] = bitcast <1 x i64> [[T31]] to <2 x i32>
+; CHECK-NEXT: ret <2 x i32> [[T3]]
+;
+ %t1 = bitcast <1 x i64> %a to <2 x i32>
+ %t2 = bitcast <1 x i64> %b to <2 x i32>
+ %t3 = xor <2 x i32> %t1, %t2
+ ret <2 x i32> %t3
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @xor_bitcast_vec_to_vec(<1 x i64> %a) {
+; CHECK-LABEL: @xor_bitcast_vec_to_vec(
+; CHECK-NEXT: [[T1:%.*]] = bitcast <1 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT: [[T2:%.*]] = xor <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x i32> [[T2]]
+;
+ %t1 = bitcast <1 x i64> %a to <2 x i32>
+ %t2 = xor <2 x i32> <i32 1, i32 2>, %t1
+ ret <2 x i32> %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define i64 @and_bitcast_vec_to_int(<2 x i32> %a) {
+; CHECK-LABEL: @and_bitcast_vec_to_int(
+; CHECK-NEXT: [[T1:%.*]] = bitcast <2 x i32> [[A:%.*]] to i64
+; CHECK-NEXT: [[T2:%.*]] = and i64 [[T1]], 3
+; CHECK-NEXT: ret i64 [[T2]]
+;
+ %t1 = bitcast <2 x i32> %a to i64
+ %t2 = and i64 %t1, 3
+ ret i64 %t2
+}
+
+; No change. Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i32> @or_bitcast_int_to_vec(i64 %a) {
+; CHECK-LABEL: @or_bitcast_int_to_vec(
+; CHECK-NEXT: [[T1:%.*]] = bitcast i64 [[A:%.*]] to <2 x i32>
+; CHECK-NEXT: [[T2:%.*]] = or <2 x i32> [[T1]], <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x i32> [[T2]]
+;
+ %t1 = bitcast i64 %a to <2 x i32>
+ %t2 = or <2 x i32> %t1, <i32 1, i32 2>
+ ret <2 x i32> %t2
+}
+
+; PR26702 - https://bugs.llvm.org//show_bug.cgi?id=26702
+; Bitcast is canonicalized above logic, so we can see the not-not pattern.
+
+define <2 x i64> @is_negative(<4 x i32> %x) {
+; CHECK-LABEL: @is_negative(
+; CHECK-NEXT: [[LOBIT:%.*]] = ashr <4 x i32> [[X:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i32> [[LOBIT]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %lobit = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+ %not = xor <4 x i32> %lobit, <i32 -1, i32 -1, i32 -1, i32 -1>
+ %bc = bitcast <4 x i32> %not to <2 x i64>
+ %notnot = xor <2 x i64> %bc, <i64 -1, i64 -1>
+ ret <2 x i64> %notnot
+}
+
+; This variation has an extra bitcast at the end. This means that the 2nd xor
+; can be done in <4 x i32> to eliminate a bitcast regardless of canonicalizaion.
+
+define <4 x i32> @is_negative_bonus_bitcast(<4 x i32> %x) {
+; CHECK-LABEL: @is_negative_bonus_bitcast(
+; CHECK-NEXT: [[LOBIT:%.*]] = ashr <4 x i32> [[X:%.*]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT: ret <4 x i32> [[LOBIT]]
+;
+ %lobit = ashr <4 x i32> %x, <i32 31, i32 31, i32 31, i32 31>
+ %not = xor <4 x i32> %lobit, <i32 -1, i32 -1, i32 -1, i32 -1>
+ %bc = bitcast <4 x i32> %not to <2 x i64>
+ %notnot = xor <2 x i64> %bc, <i64 -1, i64 -1>
+ %bc2 = bitcast <2 x i64> %notnot to <4 x i32>
+ ret <4 x i32> %bc2
+}
+
+; Bitcasts are canonicalized above bitwise logic.
+
+define <2 x i8> @canonicalize_bitcast_logic_with_constant(<4 x i4> %x) {
+; CHECK-LABEL: @canonicalize_bitcast_logic_with_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x i4> [[X:%.*]] to <2 x i8>
+; CHECK-NEXT: [[B:%.*]] = and <2 x i8> [[TMP1]], <i8 -128, i8 -128>
+; CHECK-NEXT: ret <2 x i8> [[B]]
+;
+ %a = and <4 x i4> %x, <i4 0, i4 8, i4 0, i4 8>
+ %b = bitcast <4 x i4> %a to <2 x i8>
+ ret <2 x i8> %b
+}
+
+; PR27925 - https://llvm.org/bugs/show_bug.cgi?id=27925
+
+define <4 x i32> @bitcasts_and_bitcast(<4 x i32> %a, <8 x i16> %b) {
+; CHECK-LABEL: @bitcasts_and_bitcast(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i16> [[B:%.*]] to <4 x i32>
+; CHECK-NEXT: [[BC3:%.*]] = and <4 x i32> [[TMP1]], [[A:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[BC3]]
+;
+ %bc1 = bitcast <4 x i32> %a to <2 x i64>
+ %bc2 = bitcast <8 x i16> %b to <2 x i64>
+ %and = and <2 x i64> %bc2, %bc1
+ %bc3 = bitcast <2 x i64> %and to <4 x i32>
+ ret <4 x i32> %bc3
+}
+
+; The destination must have an integer element type.
+; FIXME: We can still eliminate one bitcast in this test by doing the logic op
+; in the type of the input that has an integer element type.
+
+define <4 x float> @bitcasts_and_bitcast_to_fp(<4 x float> %a, <8 x i16> %b) {
+; CHECK-LABEL: @bitcasts_and_bitcast_to_fp(
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <4 x float> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT: [[BC2:%.*]] = bitcast <8 x i16> [[B:%.*]] to <2 x i64>
+; CHECK-NEXT: [[AND:%.*]] = and <2 x i64> [[BC2]], [[BC1]]
+; CHECK-NEXT: [[BC3:%.*]] = bitcast <2 x i64> [[AND]] to <4 x float>
+; CHECK-NEXT: ret <4 x float> [[BC3]]
+;
+ %bc1 = bitcast <4 x float> %a to <2 x i64>
+ %bc2 = bitcast <8 x i16> %b to <2 x i64>
+ %and = and <2 x i64> %bc2, %bc1
+ %bc3 = bitcast <2 x i64> %and to <4 x float>
+ ret <4 x float> %bc3
+}
+
+; FIXME: Transform limited from changing vector op to integer op to avoid codegen problems.
+
+define i128 @bitcast_or_bitcast(i128 %a, <2 x i64> %b) {
+; CHECK-LABEL: @bitcast_or_bitcast(
+; CHECK-NEXT: [[BC1:%.*]] = bitcast i128 [[A:%.*]] to <2 x i64>
+; CHECK-NEXT: [[OR:%.*]] = or <2 x i64> [[BC1]], [[B:%.*]]
+; CHECK-NEXT: [[BC2:%.*]] = bitcast <2 x i64> [[OR]] to i128
+; CHECK-NEXT: ret i128 [[BC2]]
+;
+ %bc1 = bitcast i128 %a to <2 x i64>
+ %or = or <2 x i64> %b, %bc1
+ %bc2 = bitcast <2 x i64> %or to i128
+ ret i128 %bc2
+}
+
+; FIXME: Transform limited from changing integer op to vector op to avoid codegen problems.
+
+define <4 x i32> @bitcast_xor_bitcast(<4 x i32> %a, i128 %b) {
+; CHECK-LABEL: @bitcast_xor_bitcast(
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <4 x i32> [[A:%.*]] to i128
+; CHECK-NEXT: [[XOR:%.*]] = xor i128 [[BC1]], [[B:%.*]]
+; CHECK-NEXT: [[BC2:%.*]] = bitcast i128 [[XOR]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[BC2]]
+;
+ %bc1 = bitcast <4 x i32> %a to i128
+ %xor = xor i128 %bc1, %b
+ %bc2 = bitcast i128 %xor to <4 x i32>
+ ret <4 x i32> %bc2
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=6137#c6
+
+define <4 x float> @bitcast_vector_select(<4 x float> %x, <2 x i64> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <4 x float>
+; CHECK-NEXT: [[T7:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x float> [[X:%.*]], <4 x float> [[TMP1]]
+; CHECK-NEXT: ret <4 x float> [[T7]]
+;
+ %t4 = bitcast <4 x float> %x to <4 x i32>
+ %t5 = bitcast <2 x i64> %y to <4 x i32>
+ %t6 = select <4 x i1> %cmp, <4 x i32> %t4, <4 x i32> %t5
+ %t7 = bitcast <4 x i32> %t6 to <4 x float>
+ ret <4 x float> %t7
+}
+
+define float @bitcast_scalar_select_of_scalars(float %x, i32 %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_of_scalars(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[Y:%.*]] to float
+; CHECK-NEXT: [[T7:%.*]] = select i1 [[CMP:%.*]], float [[X:%.*]], float [[TMP1]]
+; CHECK-NEXT: ret float [[T7]]
+;
+ %t4 = bitcast float %x to i32
+ %t6 = select i1 %cmp, i32 %t4, i32 %y
+ %t7 = bitcast i32 %t6 to float
+ ret float %t7
+}
+
+; FIXME: We should change the select operand types to scalars, but we need to make
+; sure the backend can reverse that transform if needed.
+
+define float @bitcast_scalar_select_type_mismatch1(float %x, <4 x i8> %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_type_mismatch1(
+; CHECK-NEXT: [[T4:%.*]] = bitcast float [[X:%.*]] to <4 x i8>
+; CHECK-NEXT: [[T6:%.*]] = select i1 [[CMP:%.*]], <4 x i8> [[T4]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT: [[T7:%.*]] = bitcast <4 x i8> [[T6]] to float
+; CHECK-NEXT: ret float [[T7]]
+;
+ %t4 = bitcast float %x to <4 x i8>
+ %t6 = select i1 %cmp, <4 x i8> %t4, <4 x i8> %y
+ %t7 = bitcast <4 x i8> %t6 to float
+ ret float %t7
+}
+
+; FIXME: We should change the select operand types to vectors, but we need to make
+; sure the backend can reverse that transform if needed.
+
+define <4 x i8> @bitcast_scalar_select_type_mismatch2(<4 x i8> %x, float %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_type_mismatch2(
+; CHECK-NEXT: [[T4:%.*]] = bitcast <4 x i8> [[X:%.*]] to float
+; CHECK-NEXT: [[T6:%.*]] = select i1 [[CMP:%.*]], float [[T4]], float [[Y:%.*]]
+; CHECK-NEXT: [[T7:%.*]] = bitcast float [[T6]] to <4 x i8>
+; CHECK-NEXT: ret <4 x i8> [[T7]]
+;
+ %t4 = bitcast <4 x i8> %x to float
+ %t6 = select i1 %cmp, float %t4, float %y
+ %t7 = bitcast float %t6 to <4 x i8>
+ ret <4 x i8> %t7
+}
+
+define <4 x float> @bitcast_scalar_select_of_vectors(<4 x float> %x, <2 x i64> %y, i1 %cmp) {
+; CHECK-LABEL: @bitcast_scalar_select_of_vectors(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x i64> [[Y:%.*]] to <4 x float>
+; CHECK-NEXT: [[T7:%.*]] = select i1 [[CMP:%.*]], <4 x float> [[X:%.*]], <4 x float> [[TMP1]]
+; CHECK-NEXT: ret <4 x float> [[T7]]
+;
+ %t4 = bitcast <4 x float> %x to <4 x i32>
+ %t5 = bitcast <2 x i64> %y to <4 x i32>
+ %t6 = select i1 %cmp, <4 x i32> %t4, <4 x i32> %t5
+ %t7 = bitcast <4 x i32> %t6 to <4 x float>
+ ret <4 x float> %t7
+}
+
+; Can't change the type of the vector select if the dest type is scalar.
+
+define float @bitcast_vector_select_no_fold1(float %x, <2 x i16> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select_no_fold1(
+; CHECK-NEXT: [[T4:%.*]] = bitcast float [[X:%.*]] to <4 x i8>
+; CHECK-NEXT: [[T5:%.*]] = bitcast <2 x i16> [[Y:%.*]] to <4 x i8>
+; CHECK-NEXT: [[T6:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i8> [[T4]], <4 x i8> [[T5]]
+; CHECK-NEXT: [[T7:%.*]] = bitcast <4 x i8> [[T6]] to float
+; CHECK-NEXT: ret float [[T7]]
+;
+ %t4 = bitcast float %x to <4 x i8>
+ %t5 = bitcast <2 x i16> %y to <4 x i8>
+ %t6 = select <4 x i1> %cmp, <4 x i8> %t4, <4 x i8> %t5
+ %t7 = bitcast <4 x i8> %t6 to float
+ ret float %t7
+}
+
+; Can't change the type of the vector select if the number of elements in the dest type is not the same.
+
+define <2 x float> @bitcast_vector_select_no_fold2(<2 x float> %x, <4 x i16> %y, <8 x i1> %cmp) {
+; CHECK-LABEL: @bitcast_vector_select_no_fold2(
+; CHECK-NEXT: [[T4:%.*]] = bitcast <2 x float> [[X:%.*]] to <8 x i8>
+; CHECK-NEXT: [[T5:%.*]] = bitcast <4 x i16> [[Y:%.*]] to <8 x i8>
+; CHECK-NEXT: [[T6:%.*]] = select <8 x i1> [[CMP:%.*]], <8 x i8> [[T4]], <8 x i8> [[T5]]
+; CHECK-NEXT: [[T7:%.*]] = bitcast <8 x i8> [[T6]] to <2 x float>
+; CHECK-NEXT: ret <2 x float> [[T7]]
+;
+ %t4 = bitcast <2 x float> %x to <8 x i8>
+ %t5 = bitcast <4 x i16> %y to <8 x i8>
+ %t6 = select <8 x i1> %cmp, <8 x i8> %t4, <8 x i8> %t5
+ %t7 = bitcast <8 x i8> %t6 to <2 x float>
+ ret <2 x float> %t7
+}
+
+; Optimize bitcasts that are extracting low element of vector. This happens because of SRoA.
+; rdar://7892780
+define float @test2(<2 x float> %A, <2 x i32> %B) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[BC:%.*]] = bitcast <2 x i32> [[B:%.*]] to <2 x float>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[BC]], i32 0
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT: ret float [[ADD]]
+;
+ %tmp28 = bitcast <2 x float> %A to i64 ; <i64> [#uses=2]
+ %tmp23 = trunc i64 %tmp28 to i32 ; <i32> [#uses=1]
+ %tmp24 = bitcast i32 %tmp23 to float ; <float> [#uses=1]
+
+ %tmp = bitcast <2 x i32> %B to i64
+ %tmp2 = trunc i64 %tmp to i32 ; <i32> [#uses=1]
+ %tmp4 = bitcast i32 %tmp2 to float ; <float> [#uses=1]
+
+ %add = fadd float %tmp24, %tmp4
+ ret float %add
+}
+
+; Optimize bitcasts that are extracting other elements of a vector. This happens because of SRoA.
+; rdar://7892780
+define float @test3(<2 x float> %A, <2 x i64> %B) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x float> [[A:%.*]], i32 1
+; CHECK-NEXT: [[BC2:%.*]] = bitcast <2 x i64> [[B:%.*]] to <4 x float>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[BC2]], i32 2
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP24]], [[TMP4]]
+; CHECK-NEXT: ret float [[ADD]]
+;
+ %tmp28 = bitcast <2 x float> %A to i64
+ %tmp29 = lshr i64 %tmp28, 32
+ %tmp23 = trunc i64 %tmp29 to i32
+ %tmp24 = bitcast i32 %tmp23 to float
+
+ %tmp = bitcast <2 x i64> %B to i128
+ %tmp1 = lshr i128 %tmp, 64
+ %tmp2 = trunc i128 %tmp1 to i32
+ %tmp4 = bitcast i32 %tmp2 to float
+
+ %add = fadd float %tmp24, %tmp4
+ ret float %add
+}
+
+; Both bitcasts are unnecessary; change the extractelement.
+
+define float @bitcast_extelt1(<2 x float> %A) {
+; CHECK-LABEL: @bitcast_extelt1(
+; CHECK-NEXT: [[BC2:%.*]] = extractelement <2 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: ret float [[BC2]]
+;
+ %bc1 = bitcast <2 x float> %A to <2 x i32>
+ %ext = extractelement <2 x i32> %bc1, i32 0
+ %bc2 = bitcast i32 %ext to float
+ ret float %bc2
+}
+
+; Second bitcast can be folded into the first.
+
+define i64 @bitcast_extelt2(<4 x float> %A) {
+; CHECK-LABEL: @bitcast_extelt2(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x float> [[A:%.*]] to <2 x i64>
+; CHECK-NEXT: [[BC2:%.*]] = extractelement <2 x i64> [[BC]], i32 1
+; CHECK-NEXT: ret i64 [[BC2]]
+;
+ %bc1 = bitcast <4 x float> %A to <2 x double>
+ %ext = extractelement <2 x double> %bc1, i32 1
+ %bc2 = bitcast double %ext to i64
+ ret i64 %bc2
+}
+
+; TODO: This should return %A.
+
+define <2 x i32> @bitcast_extelt3(<2 x i32> %A) {
+; CHECK-LABEL: @bitcast_extelt3(
+; CHECK-NEXT: [[BC1:%.*]] = bitcast <2 x i32> [[A:%.*]] to <1 x i64>
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <1 x i64> [[BC1]], i32 0
+; CHECK-NEXT: [[BC2:%.*]] = bitcast i64 [[EXT]] to <2 x i32>
+; CHECK-NEXT: ret <2 x i32> [[BC2]]
+;
+ %bc1 = bitcast <2 x i32> %A to <1 x i64>
+ %ext = extractelement <1 x i64> %bc1, i32 0
+ %bc2 = bitcast i64 %ext to <2 x i32>
+ ret <2 x i32> %bc2
+}
+
+; Handle the case where the input is not a vector.
+
+define double @bitcast_extelt4(i128 %A) {
+; CHECK-LABEL: @bitcast_extelt4(
+; CHECK-NEXT: [[BC:%.*]] = bitcast i128 [[A:%.*]] to <2 x double>
+; CHECK-NEXT: [[BC2:%.*]] = extractelement <2 x double> [[BC]], i32 0
+; CHECK-NEXT: ret double [[BC2]]
+;
+ %bc1 = bitcast i128 %A to <2 x i64>
+ %ext = extractelement <2 x i64> %bc1, i32 0
+ %bc2 = bitcast i64 %ext to double
+ ret double %bc2
+}
+
+define <2 x i32> @test4(i32 %A, i32 %B){
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT: ret <2 x i32> [[TMP2]]
+;
+ %tmp38 = zext i32 %A to i64
+ %tmp32 = zext i32 %B to i64
+ %tmp33 = shl i64 %tmp32, 32
+ %ins35 = or i64 %tmp33, %tmp38
+ %tmp43 = bitcast i64 %ins35 to <2 x i32>
+ ret <2 x i32> %tmp43
+}
+
+; rdar://8360454
+define <2 x float> @test5(float %A, float %B) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> [[TMP1]], float [[B:%.*]], i32 1
+; CHECK-NEXT: ret <2 x float> [[TMP2]]
+;
+ %tmp37 = bitcast float %A to i32
+ %tmp38 = zext i32 %tmp37 to i64
+ %tmp31 = bitcast float %B to i32
+ %tmp32 = zext i32 %tmp31 to i64
+ %tmp33 = shl i64 %tmp32, 32
+ %ins35 = or i64 %tmp33, %tmp38
+ %tmp43 = bitcast i64 %ins35 to <2 x float>
+ ret <2 x float> %tmp43
+}
+
+define <2 x float> @test6(float %A){
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> <float 4.200000e+01, float undef>, float [[A:%.*]], i32 1
+; CHECK-NEXT: ret <2 x float> [[TMP1]]
+;
+ %tmp23 = bitcast float %A to i32
+ %tmp24 = zext i32 %tmp23 to i64
+ %tmp25 = shl i64 %tmp24, 32
+ %mask20 = or i64 %tmp25, 1109917696
+ %tmp35 = bitcast i64 %mask20 to <2 x float>
+ ret <2 x float> %tmp35
+}
+
+define i64 @ISPC0(i64 %in) {
+; CHECK-LABEL: @ISPC0(
+; CHECK-NEXT: ret i64 0
+;
+ %out = and i64 %in, xor (i64 bitcast (<4 x i16> <i16 -1, i16 -1, i16 -1, i16 -1> to i64), i64 -1)
+ ret i64 %out
+}
+
+
+define i64 @Vec2(i64 %in) {
+; CHECK-LABEL: @Vec2(
+; CHECK-NEXT: ret i64 0
+;
+ %out = and i64 %in, xor (i64 bitcast (<4 x i16> <i16 0, i16 0, i16 0, i16 0> to i64), i64 0)
+ ret i64 %out
+}
+
+define i64 @All11(i64 %in) {
+; CHECK-LABEL: @All11(
+; CHECK-NEXT: ret i64 0
+;
+ %out = and i64 %in, xor (i64 bitcast (<2 x float> bitcast (i64 -1 to <2 x float>) to i64), i64 -1)
+ ret i64 %out
+}
+
+
+define i32 @All111(i32 %in) {
+; CHECK-LABEL: @All111(
+; CHECK-NEXT: ret i32 0
+;
+ %out = and i32 %in, xor (i32 bitcast (<1 x float> bitcast (i32 -1 to <1 x float>) to i32), i32 -1)
+ ret i32 %out
+}
+
+define <2 x i16> @BitcastInsert(i32 %a) {
+; CHECK-LABEL: @BitcastInsert(
+; CHECK-NEXT: [[R:%.*]] = bitcast i32 [[A:%.*]] to <2 x i16>
+; CHECK-NEXT: ret <2 x i16> [[R]]
+;
+ %v = insertelement <1 x i32> poison, i32 %a, i32 0
+ %r = bitcast <1 x i32> %v to <2 x i16>
+ ret <2 x i16> %r
+}
+
+; PR17293
+define <2 x i64> @test7(<2 x i8*>* %arg) nounwind {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: [[CAST:%.*]] = bitcast <2 x i8*>* [[ARG:%.*]] to <2 x i64>*
+; CHECK-NEXT: [[LOAD:%.*]] = load <2 x i64>, <2 x i64>* [[CAST]], align 16
+; CHECK-NEXT: ret <2 x i64> [[LOAD]]
+;
+ %cast = bitcast <2 x i8*>* %arg to <2 x i64>*
+ %load = load <2 x i64>, <2 x i64>* %cast, align 16
+ ret <2 x i64> %load
+}
+
+define i8 @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT: ret i8 -85
+;
+ %res = bitcast <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true> to i8
+ ret i8 %res
+}
+
+ at g = internal unnamed_addr global i32 undef
+
+define void @constant_fold_vector_to_double() {
+; CHECK-LABEL: @constant_fold_vector_to_double(
+; CHECK-NEXT: store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 1.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0xFFFFFFFFFFFFFFFF, double* undef, align 8
+; CHECK-NEXT: store volatile double 0x162E000004D2, double* undef, align 8
+; CHECK-NEXT: store volatile double bitcast (<2 x i32> <i32 1234, i32 ptrtoint (i32* @g to i32)> to double), double* undef, align 8
+; CHECK-NEXT: store volatile double 0x400000003F800000, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: store volatile double 0.000000e+00, double* undef, align 8
+; CHECK-NEXT: ret void
+;
+ store volatile double bitcast (<1 x i64> <i64 4607182418800017408> to double), double* undef
+ store volatile double bitcast (<2 x i32> <i32 0, i32 1072693248> to double), double* undef
+ store volatile double bitcast (<4 x i16> <i16 0, i16 0, i16 0, i16 16368> to double), double* undef
+ store volatile double bitcast (<8 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 240, i8 63> to double), double* undef
+
+ store volatile double bitcast (<2 x i32> <i32 -1, i32 -1> to double), double* undef
+ store volatile double bitcast (<2 x i32> <i32 1234, i32 5678> to double), double* undef
+
+ store volatile double bitcast (<2 x i32> <i32 1234, i32 ptrtoint (i32* @g to i32)> to double), double* undef
+ store volatile double bitcast (<2 x float> <float 1.0, float 2.0> to double), double* undef
+
+ store volatile double bitcast (<2 x i32> zeroinitializer to double), double* undef
+ store volatile double bitcast (<4 x i16> zeroinitializer to double), double* undef
+ store volatile double bitcast (<8 x i8> zeroinitializer to double), double* undef
+ store volatile double bitcast (<16 x i4> zeroinitializer to double), double* undef
+ store volatile double bitcast (<32 x i2> zeroinitializer to double), double* undef
+ store volatile double bitcast (<64 x i1> zeroinitializer to double), double* undef
+ ret void
+}
+
+define void @constant_fold_vector_to_float() {
+; CHECK-LABEL: @constant_fold_vector_to_float(
+; CHECK-NEXT: store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT: store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT: store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT: store volatile float 1.000000e+00, float* undef, align 4
+; CHECK-NEXT: ret void
+;
+ store volatile float bitcast (<1 x i32> <i32 1065353216> to float), float* undef
+ store volatile float bitcast (<2 x i16> <i16 0, i16 16256> to float), float* undef
+ store volatile float bitcast (<4 x i8> <i8 0, i8 0, i8 128, i8 63> to float), float* undef
+ store volatile float bitcast (<32 x i1> <i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 0> to float), float* undef
+
+ ret void
+}
+
+define void @constant_fold_vector_to_half() {
+; CHECK-LABEL: @constant_fold_vector_to_half(
+; CHECK-NEXT: store volatile half 0xH4000, half* undef, align 2
+; CHECK-NEXT: store volatile half 0xH4000, half* undef, align 2
+; CHECK-NEXT: ret void
+;
+ store volatile half bitcast (<2 x i8> <i8 0, i8 64> to half), half* undef
+ store volatile half bitcast (<4 x i4> <i4 0, i4 0, i4 0, i4 4> to half), half* undef
+ ret void
+}
+
+; Ensure that we do not crash when looking at such a weird bitcast.
+define i8* @bitcast_from_single_element_pointer_vector_to_pointer(<1 x i8*> %ptrvec) {
+; CHECK-LABEL: @bitcast_from_single_element_pointer_vector_to_pointer(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <1 x i8*> [[PTRVEC:%.*]], i32 0
+; CHECK-NEXT: ret i8* [[TMP1]]
+;
+ %ptr = bitcast <1 x i8*> %ptrvec to i8*
+ ret i8* %ptr
+}
diff --git a/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll b/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll
new file mode 100644
index 000000000000..c2882758cb51
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-vec-canon-inseltpoison.ll
@@ -0,0 +1,167 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define double @a(<1 x i64> %y) {
+; CHECK-LABEL: @a(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <1 x i64> [[Y:%.*]] to <1 x double>
+; CHECK-NEXT: [[C:%.*]] = extractelement <1 x double> [[BC]], i32 0
+; CHECK-NEXT: ret double [[C]]
+;
+ %c = bitcast <1 x i64> %y to double
+ ret double %c
+}
+
+define i64 @b(<1 x i64> %y) {
+; CHECK-LABEL: @b(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <1 x i64> [[Y:%.*]], i32 0
+; CHECK-NEXT: ret i64 [[TMP1]]
+;
+ %c = bitcast <1 x i64> %y to i64
+ ret i64 %c
+}
+
+define <1 x i64> @c(double %y) {
+; CHECK-LABEL: @c(
+; CHECK-NEXT: [[C:%.*]] = bitcast double [[Y:%.*]] to <1 x i64>
+; CHECK-NEXT: ret <1 x i64> [[C]]
+;
+ %c = bitcast double %y to <1 x i64>
+ ret <1 x i64> %c
+}
+
+define <1 x i64> @d(i64 %y) {
+; CHECK-LABEL: @d(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <1 x i64> undef, i64 [[Y:%.*]], i32 0
+; CHECK-NEXT: ret <1 x i64> [[TMP1]]
+;
+ %c = bitcast i64 %y to <1 x i64>
+ ret <1 x i64> %c
+}
+
+define x86_mmx @e(<1 x i64> %y) {
+; CHECK-LABEL: @e(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <1 x i64> [[Y:%.*]], i32 0
+; CHECK-NEXT: [[C:%.*]] = bitcast i64 [[TMP1]] to x86_mmx
+; CHECK-NEXT: ret x86_mmx [[C]]
+;
+ %c = bitcast <1 x i64> %y to x86_mmx
+ ret x86_mmx %c
+}
+
+define <1 x i64> @f(x86_mmx %y) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast x86_mmx [[Y:%.*]] to i64
+; CHECK-NEXT: [[C:%.*]] = insertelement <1 x i64> undef, i64 [[TMP1]], i32 0
+; CHECK-NEXT: ret <1 x i64> [[C]]
+;
+ %c = bitcast x86_mmx %y to <1 x i64>
+ ret <1 x i64> %c
+}
+
+define double @g(x86_mmx %x) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast x86_mmx [[X:%.*]] to double
+; CHECK-NEXT: ret double [[TMP0]]
+;
+entry:
+ %0 = bitcast x86_mmx %x to <1 x i64>
+ %1 = bitcast <1 x i64> %0 to double
+ ret double %1
+}
+
+; FP source is ok.
+
+define <3 x i64> @bitcast_inselt_undef(double %x, i32 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[X:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT: [[I:%.*]] = bitcast <3 x double> [[TMP1]] to <3 x i64>
+; CHECK-NEXT: ret <3 x i64> [[I]]
+;
+ %xb = bitcast double %x to i64
+ %i = insertelement <3 x i64> poison, i64 %xb, i32 %idx
+ ret <3 x i64> %i
+}
+
+; Integer source is ok; index is anything.
+
+define <3 x float> @bitcast_inselt_undef_fp(i32 %x, i567 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef_fp(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i32> undef, i32 [[X:%.*]], i567 [[IDX:%.*]]
+; CHECK-NEXT: [[I:%.*]] = bitcast <3 x i32> [[TMP1]] to <3 x float>
+; CHECK-NEXT: ret <3 x float> [[I]]
+;
+ %xb = bitcast i32 %x to float
+ %i = insertelement <3 x float> poison, float %xb, i567 %idx
+ ret <3 x float> %i
+}
+
+define <vscale x 3 x float> @bitcast_inselt_undef_vscale(i32 %x, i567 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef_vscale(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 3 x i32> undef, i32 [[X:%.*]], i567 [[IDX:%.*]]
+; CHECK-NEXT: [[I:%.*]] = bitcast <vscale x 3 x i32> [[TMP1]] to <vscale x 3 x float>
+; CHECK-NEXT: ret <vscale x 3 x float> [[I]]
+;
+ %xb = bitcast i32 %x to float
+ %i = insertelement <vscale x 3 x float> poison, float %xb, i567 %idx
+ ret <vscale x 3 x float> %i
+}
+
+declare void @use(i64)
+
+; Negative test - extra use prevents canonicalization
+
+define <3 x i64> @bitcast_inselt_undef_extra_use(double %x, i32 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef_extra_use(
+; CHECK-NEXT: [[XB:%.*]] = bitcast double [[X:%.*]] to i64
+; CHECK-NEXT: call void @use(i64 [[XB]])
+; CHECK-NEXT: [[I:%.*]] = insertelement <3 x i64> poison, i64 [[XB]], i32 [[IDX:%.*]]
+; CHECK-NEXT: ret <3 x i64> [[I]]
+;
+ %xb = bitcast double %x to i64
+ call void @use(i64 %xb)
+ %i = insertelement <3 x i64> poison, i64 %xb, i32 %idx
+ ret <3 x i64> %i
+}
+
+; Negative test - source type must be scalar
+
+define <3 x i64> @bitcast_inselt_undef_vec_src(<2 x i32> %x, i32 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef_vec_src(
+; CHECK-NEXT: [[XB:%.*]] = bitcast <2 x i32> [[X:%.*]] to i64
+; CHECK-NEXT: [[I:%.*]] = insertelement <3 x i64> poison, i64 [[XB]], i32 [[IDX:%.*]]
+; CHECK-NEXT: ret <3 x i64> [[I]]
+;
+ %xb = bitcast <2 x i32> %x to i64
+ %i = insertelement <3 x i64> poison, i64 %xb, i32 %idx
+ ret <3 x i64> %i
+}
+
+; Negative test - source type must be scalar
+
+define <3 x i64> @bitcast_inselt_undef_from_mmx(x86_mmx %x, i32 %idx) {
+; CHECK-LABEL: @bitcast_inselt_undef_from_mmx(
+; CHECK-NEXT: [[XB:%.*]] = bitcast x86_mmx [[X:%.*]] to i64
+; CHECK-NEXT: [[I:%.*]] = insertelement <3 x i64> poison, i64 [[XB]], i32 [[IDX:%.*]]
+; CHECK-NEXT: ret <3 x i64> [[I]]
+;
+ %xb = bitcast x86_mmx %x to i64
+ %i = insertelement <3 x i64> poison, i64 %xb, i32 %idx
+ ret <3 x i64> %i
+}
+
+; Reduce number of casts
+
+define <2 x i64> @PR45748(double %x, double %y) {
+; CHECK-LABEL: @PR45748(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> undef, double [[X:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP1]], double [[Y:%.*]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = bitcast <2 x double> [[TMP2]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[I1]]
+;
+ %xb = bitcast double %x to i64
+ %i0 = insertelement <2 x i64> poison, i64 %xb, i32 0
+ %yb = bitcast double %y to i64
+ %i1 = insertelement <2 x i64> %i0, i64 %yb, i32 1
+ ret <2 x i64> %i1
+}
diff --git a/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll b/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
new file mode 100644
index 000000000000..b41e3f2b744a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/broadcast-inseltpoison.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define <4 x float> @good1(float %arg) {
+; CHECK-LABEL: @good1(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <4 x float> @good2(float %arg) {
+; CHECK-LABEL: @good2(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 1
+ %t4 = insertelement <4 x float> %t, float %arg, i32 2
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 0
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <4 x float> @good3(float %arg) {
+; CHECK-LABEL: @good3(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> zeroinitializer, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <4 x float> @good4(float %arg) {
+; CHECK-LABEL: @good4(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[T]], [[T]]
+; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x float> [[T7]]
+;
+ %t = insertelement <4 x float> zeroinitializer, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ %t7 = fadd <4 x float> %t6, %t6
+ ret <4 x float> %t7
+}
+
+define <4 x float> @good5(float %v) {
+; CHECK-LABEL: @good5(
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> poison, float [[V:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]]
+; CHECK-NEXT: [[INS4:%.*]] = shufflevector <4 x float> [[INS1]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = fadd <4 x float> [[A1]], [[INS4]]
+; CHECK-NEXT: ret <4 x float> [[RES]]
+;
+ %ins1 = insertelement <4 x float> poison, float %v, i32 0
+ %a1 = fadd <4 x float> %ins1, %ins1
+ %ins2 = insertelement<4 x float> %ins1, float %v, i32 1
+ %ins3 = insertelement<4 x float> %ins2, float %v, i32 2
+ %ins4 = insertelement<4 x float> %ins3, float %v, i32 3
+ %res = fadd <4 x float> %a1, %ins4
+ ret <4 x float> %res
+}
+
+; The insert is changed to allow the canonical shuffle-splat pattern from element 0.
+
+define <4 x float> @splat_undef1(float %arg) {
+; CHECK-LABEL: @splat_undef1(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 1
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+; Re-uses the existing first insertelement.
+
+define <4 x float> @splat_undef2(float %arg) {
+; CHECK-LABEL: @splat_undef2(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 0>
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 0
+ %t5 = insertelement <4 x float> %t, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <4 x float> @bad3(float %arg, float %arg2) {
+; CHECK-LABEL: @bad3(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG2:%.*]], i32 1
+; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i32 2
+; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i32 3
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg2, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <1 x float> @bad4(float %arg) {
+; CHECK-LABEL: @bad4(
+; CHECK-NEXT: [[T:%.*]] = insertelement <1 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: ret <1 x float> [[T]]
+;
+ %t = insertelement <1 x float> poison, float %arg, i32 0
+ ret <1 x float> %t
+}
+
+; Multiple undef elements are ok.
+; TODO: Multiple uses triggers the transform at %t4, but we should sink/scalarize/CSE the splats?
+
+define <4 x float> @splat_undef3(float %arg) {
+; CHECK-LABEL: @splat_undef3(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T4:%.*]] = shufflevector <4 x float> [[T]], <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: [[T6:%.*]] = shufflevector <4 x float> [[T]], <4 x float> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[T7:%.*]] = fadd <4 x float> [[T6]], [[T4]]
+; CHECK-NEXT: ret <4 x float> [[T7]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 2
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ %t7 = fadd <4 x float> %t6, %t4
+ ret <4 x float> %t7
+}
+
+define <4 x float> @bad6(float %arg, i32 %k) {
+; CHECK-LABEL: @bad6(
+; CHECK-NEXT: [[T:%.*]] = insertelement <4 x float> poison, float [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[T4:%.*]] = insertelement <4 x float> [[T]], float [[ARG]], i32 1
+; CHECK-NEXT: [[T5:%.*]] = insertelement <4 x float> [[T4]], float [[ARG]], i32 [[K:%.*]]
+; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x float> [[T5]], float [[ARG]], i32 3
+; CHECK-NEXT: ret <4 x float> [[T6]]
+;
+ %t = insertelement <4 x float> poison, float %arg, i32 0
+ %t4 = insertelement <4 x float> %t, float %arg, i32 1
+ %t5 = insertelement <4 x float> %t4, float %arg, i32 %k
+ %t6 = insertelement <4 x float> %t5, float %arg, i32 3
+ ret <4 x float> %t6
+}
+
+define <4 x float> @bad7(float %v) {
+; CHECK-LABEL: @bad7(
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> poison, float [[V:%.*]], i32 1
+; CHECK-NEXT: [[A1:%.*]] = fadd <4 x float> [[INS1]], [[INS1]]
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float [[V]], i32 2
+; CHECK-NEXT: [[INS3:%.*]] = insertelement <4 x float> [[INS2]], float [[V]], i32 3
+; CHECK-NEXT: [[INS4:%.*]] = insertelement <4 x float> [[INS3]], float [[V]], i32 0
+; CHECK-NEXT: [[RES:%.*]] = fadd <4 x float> [[A1]], [[INS4]]
+; CHECK-NEXT: ret <4 x float> [[RES]]
+;
+ %ins1 = insertelement <4 x float> poison, float %v, i32 1
+ %a1 = fadd <4 x float> %ins1, %ins1
+ %ins2 = insertelement<4 x float> %ins1, float %v, i32 2
+ %ins3 = insertelement<4 x float> %ins2, float %v, i32 3
+ %ins4 = insertelement<4 x float> %ins3, float %v, i32 0
+ %res = fadd <4 x float> %a1, %ins4
+ ret <4 x float> %res
+}
diff --git a/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
new file mode 100644
index 000000000000..d614bb06879e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/extractelement-inseltpoison.ll
@@ -0,0 +1,332 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e" | FileCheck %s --check-prefixes=ANY,LE
+; RUN: opt < %s -instcombine -S -data-layout="E" | FileCheck %s --check-prefixes=ANY,BE
+
+define i32 @extractelement_out_of_range(<2 x i32> %x) {
+; ANY-LABEL: @extractelement_out_of_range(
+; ANY-NEXT: ret i32 undef
+;
+ %E1 = extractelement <2 x i32> %x, i8 16
+ ret i32 %E1
+}
+
+define i32 @extractelement_type_out_of_range(<2 x i32> %x) {
+; ANY-LABEL: @extractelement_type_out_of_range(
+; ANY-NEXT: [[E1:%.*]] = extractelement <2 x i32> [[X:%.*]], i128 0
+; ANY-NEXT: ret i32 [[E1]]
+;
+ %E1 = extractelement <2 x i32> %x, i128 0
+ ret i32 %E1
+}
+
+define i32 @bitcasted_inselt_equal_num_elts(float %f) {
+; ANY-LABEL: @bitcasted_inselt_equal_num_elts(
+; ANY-NEXT: [[R:%.*]] = bitcast float [[F:%.*]] to i32
+; ANY-NEXT: ret i32 [[R]]
+;
+ %vf = insertelement <4 x float> poison, float %f, i32 0
+ %vi = bitcast <4 x float> %vf to <4 x i32>
+ %r = extractelement <4 x i32> %vi, i32 0
+ ret i32 %r
+}
+
+define i64 @test2(i64 %in) {
+; ANY-LABEL: @test2(
+; ANY-NEXT: ret i64 [[IN:%.*]]
+;
+ %vec = insertelement <8 x i64> poison, i64 %in, i32 0
+ %splat = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> zeroinitializer
+ %add = add <8 x i64> %splat, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>
+ %r = extractelement <8 x i64> %add, i32 0
+ ret i64 %r
+}
+
+define i32 @bitcasted_inselt_wide_source_zero_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_zero_elt(
+; LE-NEXT: [[R:%.*]] = trunc i64 [[X:%.*]] to i32
+; LE-NEXT: ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_zero_elt(
+; BE-NEXT: [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; BE-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; BE-NEXT: ret i32 [[R]]
+;
+ %i = insertelement <2 x i64> zeroinitializer, i64 %x, i32 0
+ %b = bitcast <2 x i64> %i to <4 x i32>
+ %r = extractelement <4 x i32> %b, i32 0
+ ret i32 %r
+}
+
+define i16 @bitcasted_inselt_wide_source_modulo_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_modulo_elt(
+; LE-NEXT: [[R:%.*]] = trunc i64 [[X:%.*]] to i16
+; LE-NEXT: ret i16 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_modulo_elt(
+; BE-NEXT: [[TMP1:%.*]] = lshr i64 [[X:%.*]], 48
+; BE-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i16
+; BE-NEXT: ret i16 [[R]]
+;
+ %i = insertelement <2 x i64> poison, i64 %x, i32 1
+ %b = bitcast <2 x i64> %i to <8 x i16>
+ %r = extractelement <8 x i16> %b, i32 4
+ ret i16 %r
+}
+
+define i32 @bitcasted_inselt_wide_source_not_modulo_elt(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt(
+; LE-NEXT: [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; LE-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; LE-NEXT: ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt(
+; BE-NEXT: [[R:%.*]] = trunc i64 [[X:%.*]] to i32
+; BE-NEXT: ret i32 [[R]]
+;
+ %i = insertelement <2 x i64> poison, i64 %x, i32 0
+ %b = bitcast <2 x i64> %i to <4 x i32>
+ %r = extractelement <4 x i32> %b, i32 1
+ ret i32 %r
+}
+
+define i8 @bitcasted_inselt_wide_source_not_modulo_elt_not_half(i32 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half(
+; LE-NEXT: [[TMP1:%.*]] = lshr i32 [[X:%.*]], 16
+; LE-NEXT: [[R:%.*]] = trunc i32 [[TMP1]] to i8
+; LE-NEXT: ret i8 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half(
+; BE-NEXT: [[TMP1:%.*]] = lshr i32 [[X:%.*]], 8
+; BE-NEXT: [[R:%.*]] = trunc i32 [[TMP1]] to i8
+; BE-NEXT: ret i8 [[R]]
+;
+ %i = insertelement <2 x i32> poison, i32 %x, i32 0
+ %b = bitcast <2 x i32> %i to <8 x i8>
+ %r = extractelement <8 x i8> %b, i32 2
+ ret i8 %r
+}
+
+define i3 @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(i15 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(
+; LE-NEXT: [[TMP1:%.*]] = lshr i15 [[X:%.*]], 3
+; LE-NEXT: [[R:%.*]] = trunc i15 [[TMP1]] to i3
+; LE-NEXT: ret i3 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_not_modulo_elt_not_half_weird_types(
+; BE-NEXT: [[TMP1:%.*]] = lshr i15 [[X:%.*]], 9
+; BE-NEXT: [[R:%.*]] = trunc i15 [[TMP1]] to i3
+; BE-NEXT: ret i3 [[R]]
+;
+ %i = insertelement <3 x i15> poison, i15 %x, i32 0
+ %b = bitcast <3 x i15> %i to <15 x i3>
+ %r = extractelement <15 x i3> %b, i32 1
+ ret i3 %r
+}
+
+; Negative test for the above fold, but we can remove the insert here.
+
+define i8 @bitcasted_inselt_wide_source_wrong_insert(<2 x i32> %v, i32 %x) {
+; ANY-LABEL: @bitcasted_inselt_wide_source_wrong_insert(
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x i32> [[V:%.*]] to <8 x i8>
+; ANY-NEXT: [[R:%.*]] = extractelement <8 x i8> [[B]], i32 2
+; ANY-NEXT: ret i8 [[R]]
+;
+ %i = insertelement <2 x i32> %v, i32 %x, i32 1
+ %b = bitcast <2 x i32> %i to <8 x i8>
+ %r = extractelement <8 x i8> %b, i32 2
+ ret i8 %r
+}
+
+; Partial negative test for the above fold, extra uses are not allowed if shift is needed.
+
+declare void @use(<8 x i8>)
+
+define i8 @bitcasted_inselt_wide_source_uses(i32 %x) {
+; LE-LABEL: @bitcasted_inselt_wide_source_uses(
+; LE-NEXT: [[I:%.*]] = insertelement <2 x i32> poison, i32 [[X:%.*]], i32 0
+; LE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8>
+; LE-NEXT: call void @use(<8 x i8> [[B]])
+; LE-NEXT: [[R:%.*]] = extractelement <8 x i8> [[B]], i32 3
+; LE-NEXT: ret i8 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_wide_source_uses(
+; BE-NEXT: [[I:%.*]] = insertelement <2 x i32> poison, i32 [[X:%.*]], i32 0
+; BE-NEXT: [[B:%.*]] = bitcast <2 x i32> [[I]] to <8 x i8>
+; BE-NEXT: call void @use(<8 x i8> [[B]])
+; BE-NEXT: [[R:%.*]] = trunc i32 [[X]] to i8
+; BE-NEXT: ret i8 [[R]]
+;
+ %i = insertelement <2 x i32> poison, i32 %x, i32 0
+ %b = bitcast <2 x i32> %i to <8 x i8>
+ call void @use(<8 x i8> %b)
+ %r = extractelement <8 x i8> %b, i32 3
+ ret i8 %r
+}
+
+define float @bitcasted_inselt_to_FP(i64 %x) {
+; LE-LABEL: @bitcasted_inselt_to_FP(
+; LE-NEXT: [[TMP1:%.*]] = lshr i64 [[X:%.*]], 32
+; LE-NEXT: [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; LE-NEXT: [[R:%.*]] = bitcast i32 [[TMP2]] to float
+; LE-NEXT: ret float [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_to_FP(
+; BE-NEXT: [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
+; BE-NEXT: [[R:%.*]] = bitcast i32 [[TMP1]] to float
+; BE-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x i64> poison, i64 %x, i32 0
+ %b = bitcast <2 x i64> %i to <4 x float>
+ %r = extractelement <4 x float> %b, i32 1
+ ret float %r
+}
+
+declare void @use_v2i128(<2 x i128>)
+declare void @use_v8f32(<8 x float>)
+
+define float @bitcasted_inselt_to_FP_uses(i128 %x) {
+; ANY-LABEL: @bitcasted_inselt_to_FP_uses(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> poison, i128 [[X:%.*]], i32 0
+; ANY-NEXT: call void @use_v2i128(<2 x i128> [[I]])
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float>
+; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i32 1
+; ANY-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x i128> poison, i128 %x, i32 0
+ call void @use_v2i128(<2 x i128> %i)
+ %b = bitcast <2 x i128> %i to <8 x float>
+ %r = extractelement <8 x float> %b, i32 1
+ ret float %r
+}
+
+define float @bitcasted_inselt_to_FP_uses2(i128 %x) {
+; ANY-LABEL: @bitcasted_inselt_to_FP_uses2(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x i128> poison, i128 [[X:%.*]], i32 0
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x i128> [[I]] to <8 x float>
+; ANY-NEXT: call void @use_v8f32(<8 x float> [[B]])
+; ANY-NEXT: [[R:%.*]] = extractelement <8 x float> [[B]], i32 1
+; ANY-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x i128> poison, i128 %x, i32 0
+ %b = bitcast <2 x i128> %i to <8 x float>
+ call void @use_v8f32(<8 x float> %b)
+ %r = extractelement <8 x float> %b, i32 1
+ ret float %r
+}
+
+define i32 @bitcasted_inselt_from_FP(double %x) {
+; LE-LABEL: @bitcasted_inselt_from_FP(
+; LE-NEXT: [[TMP1:%.*]] = bitcast double [[X:%.*]] to i64
+; LE-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 32
+; LE-NEXT: [[R:%.*]] = trunc i64 [[TMP2]] to i32
+; LE-NEXT: ret i32 [[R]]
+;
+; BE-LABEL: @bitcasted_inselt_from_FP(
+; BE-NEXT: [[TMP1:%.*]] = bitcast double [[X:%.*]] to i64
+; BE-NEXT: [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; BE-NEXT: ret i32 [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ %b = bitcast <2 x double> %i to <4 x i32>
+ %r = extractelement <4 x i32> %b, i32 1
+ ret i32 %r
+}
+
+declare void @use_v2f64(<2 x double>)
+declare void @use_v8i16(<8 x i16>)
+
+define i16 @bitcasted_inselt_from_FP_uses(double %x) {
+; ANY-LABEL: @bitcasted_inselt_from_FP_uses(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i32 0
+; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]])
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16>
+; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; ANY-NEXT: ret i16 [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ call void @use_v2f64(<2 x double> %i)
+ %b = bitcast <2 x double> %i to <8 x i16>
+ %r = extractelement <8 x i16> %b, i32 1
+ ret i16 %r
+}
+
+define i16 @bitcasted_inselt_from_FP_uses2(double %x) {
+; ANY-LABEL: @bitcasted_inselt_from_FP_uses2(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i32 0
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <8 x i16>
+; ANY-NEXT: call void @use_v8i16(<8 x i16> [[B]])
+; ANY-NEXT: [[R:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; ANY-NEXT: ret i16 [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ %b = bitcast <2 x double> %i to <8 x i16>
+ call void @use_v8i16(<8 x i16> %b)
+ %r = extractelement <8 x i16> %b, i32 1
+ ret i16 %r
+}
+
+define float @bitcasted_inselt_to_and_from_FP(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i32 0
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ %b = bitcast <2 x double> %i to <4 x float>
+ %r = extractelement <4 x float> %b, i32 1
+ ret float %r
+}
+
+define float @bitcasted_inselt_to_and_from_FP_uses(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i32 0
+; ANY-NEXT: call void @use_v2f64(<2 x double> [[I]])
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ call void @use_v2f64(<2 x double> %i)
+ %b = bitcast <2 x double> %i to <4 x float>
+ %r = extractelement <4 x float> %b, i32 1
+ ret float %r
+}
+
+declare void @use_v4f32(<4 x float>)
+
+define float @bitcasted_inselt_to_and_from_FP_uses2(double %x) {
+; ANY-LABEL: @bitcasted_inselt_to_and_from_FP_uses2(
+; ANY-NEXT: [[I:%.*]] = insertelement <2 x double> poison, double [[X:%.*]], i32 0
+; ANY-NEXT: [[B:%.*]] = bitcast <2 x double> [[I]] to <4 x float>
+; ANY-NEXT: call void @use_v4f32(<4 x float> [[B]])
+; ANY-NEXT: [[R:%.*]] = extractelement <4 x float> [[B]], i32 1
+; ANY-NEXT: ret float [[R]]
+;
+ %i = insertelement <2 x double> poison, double %x, i32 0
+ %b = bitcast <2 x double> %i to <4 x float>
+ call void @use_v4f32(<4 x float> %b)
+ %r = extractelement <4 x float> %b, i32 1
+ ret float %r
+}
+
+; This would crash/assert because the logic for collectShuffleElements()
+; does not consider the possibility of invalid insert/extract operands.
+
+define <4 x double> @invalid_extractelement(<2 x double> %a, <4 x double> %b, double* %p) {
+; ANY-LABEL: @invalid_extractelement(
+; ANY-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; ANY-NEXT: [[T4:%.*]] = shufflevector <4 x double> [[B:%.*]], <4 x double> [[TMP1]], <4 x i32> <i32 undef, i32 1, i32 4, i32 3>
+; ANY-NEXT: [[E:%.*]] = extractelement <4 x double> [[B]], i32 1
+; ANY-NEXT: store double [[E]], double* [[P:%.*]], align 8
+; ANY-NEXT: [[R:%.*]] = insertelement <4 x double> [[T4]], double undef, i64 0
+; ANY-NEXT: ret <4 x double> [[R]]
+;
+ %t3 = extractelement <2 x double> %a, i32 0
+ %t4 = insertelement <4 x double> %b, double %t3, i32 2
+ %e = extractelement <4 x double> %t4, i32 1
+ store double %e, double* %p
+ %e1 = extractelement <2 x double> %a, i32 4 ; invalid index
+ %r = insertelement <4 x double> %t4, double %e1, i64 0
+ ret <4 x double> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/fold-vector-zero-inseltpoison.ll b/llvm/test/Transforms/InstCombine/fold-vector-zero-inseltpoison.ll
new file mode 100644
index 000000000000..0584116b2ff3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fold-vector-zero-inseltpoison.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -instcombine -S | not grep zeroinitializer
+
+define void @foo(i64 %A, i64 %B) {
+bb8:
+ br label %bb30
+
+bb30:
+ %s0 = phi i64 [ 0, %bb8 ], [ %r21, %bb30 ]
+ %l0 = phi i64 [ -2222, %bb8 ], [ %r23, %bb30 ]
+ %r2 = add i64 %s0, %B
+ %r3 = inttoptr i64 %r2 to <2 x double>*
+ %r4 = load <2 x double>, <2 x double>* %r3, align 8
+ %r6 = bitcast <2 x double> %r4 to <2 x i64>
+ %r7 = bitcast <2 x double> zeroinitializer to <2 x i64>
+ %r8 = insertelement <2 x i64> poison, i64 9223372036854775807, i32 0
+ %r9 = insertelement <2 x i64> poison, i64 -9223372036854775808, i32 0
+ %r10 = insertelement <2 x i64> %r8, i64 9223372036854775807, i32 1
+ %r11 = insertelement <2 x i64> %r9, i64 -9223372036854775808, i32 1
+ %r12 = and <2 x i64> %r6, %r10
+ %r13 = and <2 x i64> %r7, %r11
+ %r14 = or <2 x i64> %r12, %r13
+ %r15 = bitcast <2 x i64> %r14 to <2 x double>
+ %r18 = add i64 %s0, %A
+ %r19 = inttoptr i64 %r18 to <2 x double>*
+ store <2 x double> %r15, <2 x double>* %r19, align 8
+ %r21 = add i64 16, %s0
+ %r23 = add i64 1, %l0
+ %r25 = icmp slt i64 %r23, 0
+ %r26 = zext i1 %r25 to i64
+ %r27 = icmp ne i64 %r26, 0
+ br i1 %r27, label %bb30, label %bb5
+
+bb5:
+ ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-bc-vec-inseltpoison.ll b/llvm/test/Transforms/InstCombine/icmp-bc-vec-inseltpoison.ll
new file mode 100644
index 000000000000..5df11093b147
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/icmp-bc-vec-inseltpoison.ll
@@ -0,0 +1,127 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Tests to verify proper functioning of the icmp folding implemented in
+; InstCombiner::foldICmpBitCastConstant
+; Specifically, folding:
+; icmp <pred> iN X, C
+; where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
+; and C is a splat of a K-bit pattern
+; and SC is a constant vector = <C', C', C', ..., C'>
+; Into:
+; %E = extractelement <M x iK> %vec, i32 C'
+; icmp <pred> iK %E, trunc(C)
+
+define i1 @test_i1_0(i1 %val) {
+; CHECK-LABEL: @test_i1_0(
+; CHECK-NEXT: [[COND:%.*]] = xor i1 [[VAL:%.*]], true
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i1> poison, i1 %val, i32 0
+ %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> zeroinitializer
+ %cast = bitcast <4 x i1> %vec to i4
+ %cond = icmp eq i4 %cast, 0
+ ret i1 %cond
+}
+
+define i1 @test_i1_0_2(i1 %val) {
+; CHECK-LABEL: @test_i1_0_2(
+; CHECK-NEXT: [[COND:%.*]] = xor i1 [[VAL:%.*]], true
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i1> poison, i1 %val, i32 2
+ %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %cast = bitcast <4 x i1> %vec to i4
+ %cond = icmp eq i4 %cast, 0
+ ret i1 %cond
+}
+
+define i1 @test_i1_m1(i1 %val) {
+; CHECK-LABEL: @test_i1_m1(
+; CHECK-NEXT: ret i1 [[VAL:%.*]]
+;
+ %insvec = insertelement <4 x i1> poison, i1 %val, i32 0
+ %vec = shufflevector <4 x i1> %insvec, <4 x i1> undef, <4 x i32> zeroinitializer
+ %cast = bitcast <4 x i1> %vec to i4
+ %cond = icmp eq i4 %cast, -1
+ ret i1 %cond
+}
+
+define i1 @test_i8_pattern(i8 %val) {
+; CHECK-LABEL: @test_i8_pattern(
+; CHECK-NEXT: [[COND:%.*]] = icmp eq i8 [[VAL:%.*]], 72
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i8> poison, i8 %val, i32 0
+ %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+ %cast = bitcast <4 x i8> %vec to i32
+ %cond = icmp eq i32 %cast, 1212696648
+ ret i1 %cond
+}
+
+define i1 @test_i8_pattern_2(i8 %val) {
+; CHECK-LABEL: @test_i8_pattern_2(
+; CHECK-NEXT: [[COND:%.*]] = icmp eq i8 [[VAL:%.*]], 72
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i8> poison, i8 %val, i32 2
+ %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %cast = bitcast <4 x i8> %vec to i32
+ %cond = icmp eq i32 %cast, 1212696648
+ ret i1 %cond
+}
+
+; Make sure we don't try to fold if the shufflemask has
diff ering element values
+define i1 @test_i8_pattern_3(<4 x i8> %invec) {
+; CHECK-LABEL: @test_i8_pattern_3(
+; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INVEC:%.*]], <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32
+; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696648
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %vec = shufflevector <4 x i8> %invec, <4 x i8> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+ %cast = bitcast <4 x i8> %vec to i32
+ %cond = icmp eq i32 %cast, 1212696648
+ ret i1 %cond
+}
+
+; Make sure we don't try to fold if the compared-to constant isn't a splatted value
+define i1 @test_i8_nopattern(i8 %val) {
+; CHECK-LABEL: @test_i8_nopattern(
+; CHECK-NEXT: [[INSVEC:%.*]] = insertelement <4 x i8> poison, i8 [[VAL:%.*]], i32 0
+; CHECK-NEXT: [[VEC:%.*]] = shufflevector <4 x i8> [[INSVEC]], <4 x i8> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[CAST:%.*]] = bitcast <4 x i8> [[VEC]] to i32
+; CHECK-NEXT: [[COND:%.*]] = icmp eq i32 [[CAST]], 1212696647
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i8> poison, i8 %val, i32 0
+ %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+ %cast = bitcast <4 x i8> %vec to i32
+ %cond = icmp eq i32 %cast, 1212696647
+ ret i1 %cond
+}
+
+; Verify that we fold more than just the eq predicate
+define i1 @test_i8_ult_pattern(i8 %val) {
+; CHECK-LABEL: @test_i8_ult_pattern(
+; CHECK-NEXT: [[COND:%.*]] = icmp ult i8 [[VAL:%.*]], 72
+; CHECK-NEXT: ret i1 [[COND]]
+;
+ %insvec = insertelement <4 x i8> poison, i8 %val, i32 0
+ %vec = shufflevector <4 x i8> %insvec, <4 x i8> undef, <4 x i32> zeroinitializer
+ %cast = bitcast <4 x i8> %vec to i32
+ %cond = icmp ult i32 %cast, 1212696648
+ ret i1 %cond
+}
+
+define i1 @extending_shuffle_with_weird_types(<2 x i9> %v) {
+; CHECK-LABEL: @extending_shuffle_with_weird_types(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i9> [[V:%.*]], i32 0
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i9 [[TMP1]], 1
+; CHECK-NEXT: ret i1 [[CMP]]
+;
+ %splat = shufflevector <2 x i9> %v, <2 x i9> undef, <3 x i32> zeroinitializer
+ %cast = bitcast <3 x i9> %splat to i27
+ %cmp = icmp slt i27 %cast, 262657 ; 0x040201
+ ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/InstCombine/inselt-binop-inseltpoison.ll b/llvm/test/Transforms/InstCombine/inselt-binop-inseltpoison.ll
new file mode 100644
index 000000000000..856eb61ba0ba
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/inselt-binop-inseltpoison.ll
@@ -0,0 +1,635 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+define <2 x i8> @add_constant(i8 %x) {
+; CHECK-LABEL: @add_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = add <2 x i8> %ins, <i8 42, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @add_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @add_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = add <2 x i8> %ins, <i8 42, i8 -42>
+ ret <2 x i8> %bo
+}
+
+; IR flags are not required, but they should propagate.
+
+define <2 x i8> @sub_constant_op0(i8 %x) {
+; CHECK-LABEL: @sub_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = sub nuw nsw <2 x i8> <i8 undef, i8 -42>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = sub nsw nuw <2 x i8> <i8 undef, i8 -42>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sub_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = sub nuw <2 x i8> <i8 42, i8 -42>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = sub nuw <2 x i8> <i8 42, i8 -42>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op1(i8 %x) {
+; CHECK-LABEL: @sub_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], <i8 -42, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = sub nuw <2 x i8> %ins, <i8 42, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sub_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sub_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = add <2 x i8> [[INS]], <i8 -42, i8 42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = sub nuw <2 x i8> %ins, <i8 42, i8 -42>
+ ret <2 x i8> %bo
+}
+
+define <3 x i8> @mul_constant(i8 %x) {
+; CHECK-LABEL: @mul_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> poison, i8 [[X:%.*]], i32 2
+; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], <i8 undef, i8 undef, i8 -42>
+; CHECK-NEXT: ret <3 x i8> [[BO]]
+;
+ %ins = insertelement <3 x i8> poison, i8 %x, i32 2
+ %bo = mul <3 x i8> %ins, <i8 undef, i8 undef, i8 -42>
+ ret <3 x i8> %bo
+}
+
+define <3 x i8> @mul_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @mul_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <3 x i8> poison, i8 [[X:%.*]], i32 2
+; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[INS]], <i8 42, i8 undef, i8 -42>
+; CHECK-NEXT: ret <3 x i8> [[BO]]
+;
+ %ins = insertelement <3 x i8> poison, i8 %x, i32 2
+ %bo = mul <3 x i8> %ins, <i8 42, i8 undef, i8 -42>
+ ret <3 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op0(i8 %x) {
+; CHECK-LABEL: @shl_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> <i8 undef, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = shl <2 x i8> <i8 undef, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @shl_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = shl <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = shl <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op1(i8 %x) {
+; CHECK-LABEL: @shl_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], <i8 5, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = shl nuw <2 x i8> %ins, <i8 5, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @shl_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @shl_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = shl nuw <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op0(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = ashr exact <2 x i8> <i8 undef, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = ashr exact <2 x i8> <i8 undef, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = ashr exact <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op1(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], <i8 5, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = ashr <2 x i8> %ins, <i8 5, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @ashr_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @ashr_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = ashr <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = ashr <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op0(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = lshr <2 x i8> <i8 5, i8 undef>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = lshr <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = lshr <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op1(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], <i8 undef, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = lshr exact <2 x i8> %ins, <i8 undef, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @lshr_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @lshr_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = lshr exact <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = lshr exact <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op0(i8 %x) {
+; CHECK-LABEL: @urem_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = urem <2 x i8> <i8 5, i8 undef>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @urem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = urem <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op1(i8 %x) {
+; CHECK-LABEL: @urem_constant_op1(
+; CHECK-NEXT: ret <2 x i8> undef
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = urem <2 x i8> %ins, <i8 undef, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @urem_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @urem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = urem <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = urem <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op0(i8 %x) {
+; CHECK-LABEL: @srem_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = srem <2 x i8> <i8 5, i8 undef>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @srem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = srem <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op1(i8 %x) {
+; CHECK-LABEL: @srem_constant_op1(
+; CHECK-NEXT: ret <2 x i8> undef
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = srem <2 x i8> %ins, <i8 undef, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @srem_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @srem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = srem <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = srem <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op0(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = udiv exact <2 x i8> <i8 5, i8 undef>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = udiv exact <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = udiv exact <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op1(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op1(
+; CHECK-NEXT: ret <2 x i8> undef
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = udiv <2 x i8> %ins, <i8 undef, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @udiv_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @udiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = udiv <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = udiv <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op0(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> <i8 5, i8 undef>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = sdiv <2 x i8> <i8 5, i8 undef>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op0_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = sdiv <2 x i8> <i8 5, i8 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = sdiv <2 x i8> <i8 5, i8 2>, %ins
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op1(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op1(
+; CHECK-NEXT: ret <2 x i8> undef
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = sdiv exact <2 x i8> %ins, <i8 undef, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @sdiv_constant_op1_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @sdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = sdiv exact <2 x i8> [[INS]], <i8 5, i8 2>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = sdiv exact <2 x i8> %ins, <i8 5, i8 2>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @and_constant(i8 %x) {
+; CHECK-LABEL: @and_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = and <2 x i8> %ins, <i8 42, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @and_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @and_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = and <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = and <2 x i8> %ins, <i8 42, i8 -42>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @or_constant(i8 %x) {
+; CHECK-LABEL: @or_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], <i8 undef, i8 -42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = or <2 x i8> %ins, <i8 undef, i8 -42>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @or_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @or_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = or <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 1
+ %bo = or <2 x i8> %ins, <i8 42, i8 -42>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @xor_constant(i8 %x) {
+; CHECK-LABEL: @xor_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], <i8 42, i8 undef>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = xor <2 x i8> %ins, <i8 42, i8 undef>
+ ret <2 x i8> %bo
+}
+
+define <2 x i8> @xor_constant_not_undef_lane(i8 %x) {
+; CHECK-LABEL: @xor_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i8> poison, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = xor <2 x i8> [[INS]], <i8 42, i8 -42>
+; CHECK-NEXT: ret <2 x i8> [[BO]]
+;
+ %ins = insertelement <2 x i8> poison, i8 %x, i32 0
+ %bo = xor <2 x i8> %ins, <i8 42, i8 -42>
+ ret <2 x i8> %bo
+}
+
+define <2 x float> @fadd_constant(float %x) {
+; CHECK-LABEL: @fadd_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fadd <2 x float> %ins, <float 42.0, float undef>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fadd_constant_not_undef_lane(float %x) {
+; CHECK-LABEL: @fadd_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = fadd <2 x float> %ins, <float 42.0, float -42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op0(float %x) {
+; CHECK-LABEL: @fsub_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fsub fast <2 x float> <float 4.200000e+01, float undef>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fsub fast <2 x float> <float 42.0, float undef>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @fsub_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = fsub nsz <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = fsub nsz <2 x float> <float 42.0, float -42.0>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op1(float %x) {
+; CHECK-LABEL: @fsub_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], <float undef, float -4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = fsub <2 x float> %ins, <float undef, float 42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fsub_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @fsub_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fadd <2 x float> [[INS]], <float -4.200000e+01, float 4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fsub <2 x float> %ins, <float 42.0, float -42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fmul_constant(float %x) {
+; CHECK-LABEL: @fmul_constant(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fmul reassoc <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fmul reassoc <2 x float> %ins, <float 42.0, float undef>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fmul_constant_not_undef_lane(float %x) {
+; CHECK-LABEL: @fmul_constant_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = fmul <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = fmul <2 x float> %ins, <float 42.0, float -42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op0(float %x) {
+; CHECK-LABEL: @fdiv_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = fdiv nnan <2 x float> <float undef, float 4.200000e+01>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = fdiv nnan <2 x float> <float undef, float 42.0>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @fdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fdiv ninf <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fdiv ninf <2 x float> <float 42.0, float -42.0>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op1(float %x) {
+; CHECK-LABEL: @fdiv_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fdiv <2 x float> %ins, <float 42.0, float undef>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @fdiv_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @fdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = fdiv <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = fdiv <2 x float> %ins, <float 42.0, float -42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op0(float %x) {
+; CHECK-LABEL: @frem_constant_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = frem fast <2 x float> <float 4.200000e+01, float undef>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = frem fast <2 x float> <float 42.0, float undef>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op0_not_undef_lane(float %x) {
+; CHECK-LABEL: @frem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = frem <2 x float> <float 4.200000e+01, float -4.200000e+01>, [[INS]]
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = frem <2 x float> <float 42.0, float -42.0>, %ins
+ ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op1(float %x) {
+; CHECK-LABEL: @frem_constant_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = frem ninf <2 x float> [[INS]], <float undef, float 4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 1
+ %bo = frem ninf <2 x float> %ins, <float undef, float 42.0>
+ ret <2 x float> %bo
+}
+
+define <2 x float> @frem_constant_op1_not_undef_lane(float %x) {
+; CHECK-LABEL: @frem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = frem nnan <2 x float> [[INS]], <float 4.200000e+01, float -4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[BO]]
+;
+ %ins = insertelement <2 x float> poison, float %x, i32 0
+ %bo = frem nnan <2 x float> %ins, <float 42.0, float -42.0>
+ ret <2 x float> %bo
+}
+
diff --git a/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
new file mode 100644
index 000000000000..7af57b76d369
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/insert-extract-shuffle-inseltpoison.ll
@@ -0,0 +1,735 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine %s | FileCheck %s
+
+define <1 x i8> @test1(<8 x i8> %in) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: [[VEC:%.*]] = shufflevector <8 x i8> [[IN:%.*]], <8 x i8> undef, <1 x i32> <i32 5>
+; CHECK-NEXT: ret <1 x i8> [[VEC]]
+;
+ %val = extractelement <8 x i8> %in, i32 5
+ %vec = insertelement <1 x i8> poison, i8 %val, i32 0
+ ret <1 x i8> %vec
+}
+
+define <4 x i16> @test2(<8 x i16> %in, <8 x i16> %in2) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[VEC_3:%.*]] = shufflevector <8 x i16> [[IN2:%.*]], <8 x i16> [[IN:%.*]], <4 x i32> <i32 11, i32 9, i32 0, i32 10>
+; CHECK-NEXT: ret <4 x i16> [[VEC_3]]
+;
+ %elt0 = extractelement <8 x i16> %in, i32 3
+ %elt1 = extractelement <8 x i16> %in, i32 1
+ %elt2 = extractelement <8 x i16> %in2, i32 0
+ %elt3 = extractelement <8 x i16> %in, i32 2
+
+ %vec.0 = insertelement <4 x i16> poison, i16 %elt0, i32 0
+ %vec.1 = insertelement <4 x i16> %vec.0, i16 %elt1, i32 1
+ %vec.2 = insertelement <4 x i16> %vec.1, i16 %elt2, i32 2
+ %vec.3 = insertelement <4 x i16> %vec.2, i16 %elt3, i32 3
+
+ ret <4 x i16> %vec.3
+}
+
+define <2 x i64> @test_vcopyq_lane_p64(<2 x i64> %a, <1 x i64> %b) {
+; CHECK-LABEL: @test_vcopyq_lane_p64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <1 x i64> [[B:%.*]], <1 x i64> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: [[RES:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[TMP1]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: ret <2 x i64> [[RES]]
+;
+ %elt = extractelement <1 x i64> %b, i32 0
+ %res = insertelement <2 x i64> %a, i64 %elt, i32 1
+ ret <2 x i64> %res
+}
+
+; PR2109: https://llvm.org/bugs/show_bug.cgi?id=2109
+
+define <4 x float> @widen_extract2(<4 x float> %ins, <2 x float> %ext) {
+; CHECK-LABEL: @widen_extract2(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[I2:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 4, i32 2, i32 5>
+; CHECK-NEXT: ret <4 x float> [[I2]]
+;
+ %e1 = extractelement <2 x float> %ext, i32 0
+ %e2 = extractelement <2 x float> %ext, i32 1
+ %i1 = insertelement <4 x float> %ins, float %e1, i32 1
+ %i2 = insertelement <4 x float> %i1, float %e2, i32 3
+ ret <4 x float> %i2
+}
+
+define <4 x float> @widen_extract3(<4 x float> %ins, <3 x float> %ext) {
+; CHECK-LABEL: @widen_extract3(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <3 x float> [[EXT:%.*]], <3 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: [[I3:%.*]] = shufflevector <4 x float> [[INS:%.*]], <4 x float> [[TMP1]], <4 x i32> <i32 6, i32 5, i32 4, i32 3>
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %e1 = extractelement <3 x float> %ext, i32 0
+ %e2 = extractelement <3 x float> %ext, i32 1
+ %e3 = extractelement <3 x float> %ext, i32 2
+ %i1 = insertelement <4 x float> %ins, float %e1, i32 2
+ %i2 = insertelement <4 x float> %i1, float %e2, i32 1
+ %i3 = insertelement <4 x float> %i2, float %e3, i32 0
+ ret <4 x float> %i3
+}
+
+define <8 x float> @widen_extract4(<8 x float> %ins, <2 x float> %ext) {
+; CHECK-LABEL: @widen_extract4(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x float> [[EXT:%.*]], <2 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[I1:%.*]] = shufflevector <8 x float> [[INS:%.*]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <8 x float> [[I1]]
+;
+ %e1 = extractelement <2 x float> %ext, i32 0
+ %i1 = insertelement <8 x float> %ins, float %e1, i32 2
+ ret <8 x float> %i1
+}
+
+; PR26015: https://llvm.org/bugs/show_bug.cgi?id=26015
+; The widening shuffle must be inserted before any uses.
+
+define <8 x i16> @pr26015(<4 x i16> %t0) {
+; CHECK-LABEL: @pr26015(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i16> [[T0:%.*]], <4 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[T5:%.*]] = shufflevector <8 x i16> <i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 undef>, <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 10, i32 4, i32 5, i32 6, i32 11>
+; CHECK-NEXT: ret <8 x i16> [[T5]]
+;
+ %t1 = extractelement <4 x i16> %t0, i32 2
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i32 3
+ %t3 = insertelement <8 x i16> %t2, i16 0, i32 6
+ %t4 = extractelement <4 x i16> %t0, i32 3
+ %t5 = insertelement <8 x i16> %t3, i16 %t4, i32 7
+ ret <8 x i16> %t5
+}
+
+; PR25999: https://llvm.org/bugs/show_bug.cgi?id=25999
+; TODO: The widening shuffle could be inserted at the start of the function to allow the first extract to use it.
+
+define <8 x i16> @pr25999(<4 x i16> %t0, i1 %b) {
+; CHECK-LABEL: @pr25999(
+; CHECK-NEXT: [[T1:%.*]] = extractelement <4 x i16> [[T0:%.*]], i32 2
+; CHECK-NEXT: br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK: if:
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i16> [[T0]], <4 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[T3:%.*]] = insertelement <8 x i16> <i16 0, i16 0, i16 0, i16 undef, i16 0, i16 0, i16 0, i16 undef>, i16 [[T1]], i32 3
+; CHECK-NEXT: [[T5:%.*]] = shufflevector <8 x i16> [[T3]], <8 x i16> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 11>
+; CHECK-NEXT: ret <8 x i16> [[T5]]
+; CHECK: end:
+; CHECK-NEXT: [[A1:%.*]] = add i16 [[T1]], 4
+; CHECK-NEXT: [[T6:%.*]] = insertelement <8 x i16> <i16 undef, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>, i16 [[A1]], i32 0
+; CHECK-NEXT: ret <8 x i16> [[T6]]
+;
+
+ %t1 = extractelement <4 x i16> %t0, i32 2
+ br i1 %b, label %if, label %end
+
+if:
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i32 3
+ %t3 = insertelement <8 x i16> %t2, i16 0, i32 6
+ %t4 = extractelement <4 x i16> %t0, i32 3
+ %t5 = insertelement <8 x i16> %t3, i16 %t4, i32 7
+ ret <8 x i16> %t5
+
+end:
+ %a1 = add i16 %t1, 4
+ %t6 = insertelement <8 x i16> zeroinitializer, i16 %a1, i32 0
+ ret <8 x i16> %t6
+}
+
+; The widening shuffle must be inserted at a valid point (after the PHIs).
+
+define <4 x double> @pr25999_phis1(i1 %c, <2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @pr25999_phis1(
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: br i1 [[C:%.*]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK: bb2:
+; CHECK-NEXT: [[R:%.*]] = call <2 x double> @dummy(<2 x double> [[A:%.*]])
+; CHECK-NEXT: br label [[BB3]]
+; CHECK: bb3:
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[TMP1]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+; CHECK-NEXT: ret <4 x double> [[TMP4]]
+;
+bb1:
+ br i1 %c, label %bb2, label %bb3
+
+bb2:
+ %r = call <2 x double> @dummy(<2 x double> %a)
+ br label %bb3
+
+bb3:
+ %tmp1 = phi <2 x double> [ %a, %bb1 ], [ %r, %bb2 ]
+ %tmp2 = phi <4 x double> [ %b, %bb1 ], [ zeroinitializer, %bb2 ]
+ %tmp3 = extractelement <2 x double> %tmp1, i32 0
+ %tmp4 = insertelement <4 x double> %tmp2, double %tmp3, i32 2
+ ret <4 x double> %tmp4
+}
+
+declare <2 x double> @dummy(<2 x double>)
+
+define <4 x double> @pr25999_phis2(i1 %c, <2 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @pr25999_phis2(
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: br i1 [[C:%.*]], label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK: bb2:
+; CHECK-NEXT: [[R:%.*]] = call <2 x double> @dummy(<2 x double> [[A:%.*]])
+; CHECK-NEXT: br label [[BB3]]
+; CHECK: bb3:
+; CHECK-NEXT: [[TMP1:%.*]] = phi <2 x double> [ [[A]], [[BB1:%.*]] ], [ [[R]], [[BB2]] ]
+; CHECK-NEXT: [[TMP2:%.*]] = phi <4 x double> [ [[B:%.*]], [[BB1]] ], [ zeroinitializer, [[BB2]] ]
+; CHECK-NEXT: [[D:%.*]] = fadd <2 x double> [[TMP1]], [[TMP1]]
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[D]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 4, i32 3>
+; CHECK-NEXT: ret <4 x double> [[TMP4]]
+;
+bb1:
+ br i1 %c, label %bb2, label %bb3
+
+bb2:
+ %r = call <2 x double> @dummy(<2 x double> %a)
+ br label %bb3
+
+bb3:
+ %tmp1 = phi <2 x double> [ %a, %bb1 ], [ %r, %bb2 ]
+ %tmp2 = phi <4 x double> [ %b, %bb1 ], [ zeroinitializer, %bb2 ]
+ %d = fadd <2 x double> %tmp1, %tmp1
+ %tmp3 = extractelement <2 x double> %d, i32 0
+ %tmp4 = insertelement <4 x double> %tmp2, double %tmp3, i32 2
+ ret <4 x double> %tmp4
+}
+
+; PR26354: https://llvm.org/bugs/show_bug.cgi?id=26354
+; Don't create a shufflevector if we know that we're not going to replace the insertelement.
+
+define double @pr26354(<2 x double>* %tmp, i1 %B) {
+; CHECK-LABEL: @pr26354(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[LD:%.*]] = load <2 x double>, <2 x double>* [[TMP:%.*]], align 16
+; CHECK-NEXT: br i1 [[B:%.*]], label [[IF:%.*]], label [[END:%.*]]
+; CHECK: if:
+; CHECK-NEXT: [[E2:%.*]] = extractelement <2 x double> [[LD]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x double> <double 0.000000e+00, double 0.000000e+00, double 0.000000e+00, double undef>, double [[E2]], i32 3
+; CHECK-NEXT: br label [[END]]
+; CHECK: end:
+; CHECK-NEXT: [[PH:%.*]] = phi <4 x double> [ undef, [[ENTRY:%.*]] ], [ [[I1]], [[IF]] ]
+; CHECK-NEXT: [[E1:%.*]] = extractelement <2 x double> [[LD]], i32 0
+; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x double> [[PH]], i32 1
+; CHECK-NEXT: [[MU:%.*]] = fmul double [[E1]], [[E3]]
+; CHECK-NEXT: ret double [[MU]]
+;
+
+entry:
+ %ld = load <2 x double>, <2 x double>* %tmp
+ %e1 = extractelement <2 x double> %ld, i32 0
+ %e2 = extractelement <2 x double> %ld, i32 1
+ br i1 %B, label %if, label %end
+
+if:
+ %i1 = insertelement <4 x double> zeroinitializer, double %e2, i32 3
+ br label %end
+
+end:
+ %ph = phi <4 x double> [ undef, %entry ], [ %i1, %if ]
+ %e3 = extractelement <4 x double> %ph, i32 1
+ %mu = fmul double %e1, %e3
+ ret double %mu
+}
+
+; https://llvm.org/bugs/show_bug.cgi?id=30923
+; Delete the widening shuffle if we're not going to reduce the extract/insert to a shuffle.
+
+define <4 x float> @PR30923(<2 x float> %x) {
+; CHECK-LABEL: @PR30923(
+; CHECK-NEXT: bb1:
+; CHECK-NEXT: [[EXT1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 1
+; CHECK-NEXT: store float [[EXT1]], float* undef, align 4
+; CHECK-NEXT: br label [[BB2:%.*]]
+; CHECK: bb2:
+; CHECK-NEXT: [[EXT2:%.*]] = extractelement <2 x float> [[X]], i32 0
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> <float 0.000000e+00, float 0.000000e+00, float undef, float undef>, float [[EXT2]], i32 2
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float [[EXT1]], i32 3
+; CHECK-NEXT: ret <4 x float> [[INS2]]
+;
+bb1:
+ %ext1 = extractelement <2 x float> %x, i32 1
+ store float %ext1, float* undef, align 4
+ br label %bb2
+
+bb2:
+ %widen = shufflevector <2 x float> %x, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %ext2 = extractelement <4 x float> %widen, i32 0
+ %ins1 = insertelement <4 x float> <float 0.0, float 0.0, float undef, float undef>, float %ext2, i32 2
+ %ins2 = insertelement <4 x float> %ins1, float %ext1, i32 3
+ ret <4 x float> %ins2
+}
+
+; Don't insert extractelements from the wider vector before the def of the index operand.
+
+define <4 x i32> @extractelt_insertion(<2 x i32> %x, i32 %y) {
+; CHECK-LABEL: @extractelt_insertion(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[B:%.*]] = shufflevector <4 x i32> <i32 0, i32 0, i32 0, i32 undef>, <4 x i32> [[TMP0]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+; CHECK-NEXT: [[C:%.*]] = add i32 [[Y:%.*]], 3
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[TMP0]], i32 [[C]]
+; CHECK-NEXT: [[E:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[RET:%.*]] = select i1 [[E]], <4 x i32> [[B]], <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[RET]]
+;
+entry:
+ %a = extractelement <2 x i32> %x, i32 1
+ %b = insertelement <4 x i32> zeroinitializer, i32 %a, i64 3
+ %c = add i32 %y, 3
+ %d = extractelement <2 x i32> %x, i32 %c
+ %e = icmp eq i32 %d, 0
+ %ret = select i1 %e, <4 x i32> %b, <4 x i32> zeroinitializer
+ ret <4 x i32> %ret
+}
+
+; PR34724: https://bugs.llvm.org/show_bug.cgi?id=34724
+
+define <4 x float> @collectShuffleElts(<2 x float> %x, float %y) {
+; CHECK-LABEL: @collectShuffleElts(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <2 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x float> [[X]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> poison, float [[X0]], i32 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[X1]], i32 2
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[Y:%.*]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %x0 = extractelement <2 x float> %x, i32 0
+ %x1 = extractelement <2 x float> %x, i32 1
+ %v1 = insertelement <4 x float> poison, float %x0, i32 1
+ %v2 = insertelement <4 x float> %v1, float %x1, i32 2
+ %v3 = insertelement <4 x float> %v2, float %y, i32 3
+ ret <4 x float> %v3
+}
+
+; Simplest case - insert scalar into undef, then shuffle that value in place into another vector.
+
+define <4 x float> @insert_shuffle(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_shuffle(
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 0
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+ ret <4 x float> %r
+}
+
+; Insert scalar into some element of a dummy vector, then move it to a
diff erent element in another vector.
+
+define <4 x float> @insert_shuffle_translate(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_shuffle_translate(
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 4, i32 0, i32 6, i32 7>
+ ret <4 x float> %r
+}
+
+; The vector operand of the insert is irrelevant.
+
+define <4 x float> @insert_not_undef_shuffle_translate(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate(
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 2
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> %q, float %x, i32 3
+ %r = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 4, i32 5, i32 3, i32 7>
+ ret <4 x float> %r
+}
+
+; The insert may be the 2nd operand of the shuffle. The shuffle mask can include undef elements.
+
+define <4 x float> @insert_not_undef_shuffle_translate_commute(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute(
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> %q, float %x, i32 2
+ %r = shufflevector <4 x float> %y, <4 x float> %xv, <4 x i32> <i32 0, i32 6, i32 2, i32 undef>
+ ret <4 x float> %r
+}
+
+; Both shuffle operands may be inserts - choose the correct side.
+
+define <4 x float> @insert_insert_shuffle_translate(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate(
+; CHECK-NEXT: [[XV2:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X2:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[XV2]], float [[X1:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+ %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+ %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 4, i32 0, i32 6, i32 7>
+ ret <4 x float> %r
+}
+
+; Both shuffle operands may be inserts - choose the correct side.
+
+define <4 x float> @insert_insert_shuffle_translate_commute(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate_commute(
+; CHECK-NEXT: [[XV1:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X1:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[XV1]], float [[X2:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+ %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+ %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 0, i32 6, i32 2, i32 3>
+ ret <4 x float> %r
+}
+
+; Negative test - this only works if the shuffle is choosing exactly 1 element from 1 of the inputs.
+; TODO: But this could be a special-case because we're inserting into the same base vector.
+
+define <4 x float> @insert_insert_shuffle_translate_wrong_mask(float %x1, float %x2, <4 x float> %q) {
+; CHECK-LABEL: @insert_insert_shuffle_translate_wrong_mask(
+; CHECK-NEXT: [[XV1:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X1:%.*]], i32 0
+; CHECK-NEXT: [[XV2:%.*]] = insertelement <4 x float> [[Q]], float [[X2:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[XV1]], <4 x float> [[XV2]], <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv1 = insertelement <4 x float> %q, float %x1, i32 0
+ %xv2 = insertelement <4 x float> %q, float %x2, i32 2
+ %r = shufflevector <4 x float> %xv1, <4 x float> %xv2, <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+ ret <4 x float> %r
+}
+
+; The insert may have other uses.
+
+declare void @use(<4 x float>)
+
+define <4 x float> @insert_not_undef_shuffle_translate_commute_uses(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_uses(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> [[Q:%.*]], float [[X:%.*]], i32 2
+; CHECK-NEXT: call void @use(<4 x float> [[XV]])
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X]], i32 0
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> %q, float %x, i32 2
+ call void @use(<4 x float> %xv)
+ %r = shufflevector <4 x float> %y, <4 x float> %xv, <4 x i32> <i32 6, i32 undef, i32 2, i32 3>
+ ret <4 x float> %r
+}
+
+; Negative test - size-changing shuffle.
+
+define <5 x float> @insert_not_undef_shuffle_translate_commute_lengthen(float %x, <4 x float> %y, <4 x float> %q) {
+; CHECK-LABEL: @insert_not_undef_shuffle_translate_commute_lengthen(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[XV]], <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+; CHECK-NEXT: ret <5 x float> [[R]]
+;
+ %xv = insertelement <4 x float> %q, float %x, i32 2
+ %r = shufflevector <4 x float> %y, <4 x float> %xv, <5 x i32> <i32 0, i32 6, i32 2, i32 undef, i32 undef>
+ ret <5 x float> %r
+}
+
+define <4 x float> @insert_nonzero_index_splat(float %x) {
+; CHECK-LABEL: @insert_nonzero_index_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> undef, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[SPLAT]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 2
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 2, i32 undef>
+ ret <4 x float> %splat
+}
+
+define <3 x double> @insert_nonzero_index_splat_narrow(double %x) {
+; CHECK-LABEL: @insert_nonzero_index_splat_narrow(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x double> undef, double [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <3 x double> [[TMP1]], <3 x double> undef, <3 x i32> <i32 0, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x double> [[SPLAT]]
+;
+ %xv = insertelement <4 x double> poison, double %x, i32 3
+ %splat = shufflevector <4 x double> %xv, <4 x double> undef, <3 x i32> <i32 3, i32 undef, i32 3>
+ ret <3 x double> %splat
+}
+
+define <5 x i7> @insert_nonzero_index_splat_widen(i7 %x) {
+; CHECK-LABEL: @insert_nonzero_index_splat_widen(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <5 x i7> undef, i7 [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <5 x i7> [[TMP1]], <5 x i7> undef, <5 x i32> <i32 undef, i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT: ret <5 x i7> [[SPLAT]]
+;
+ %xv = insertelement <4 x i7> poison, i7 %x, i32 1
+ %splat = shufflevector <4 x i7> %xv, <4 x i7> undef, <5 x i32> <i32 undef, i32 1, i32 1, i32 undef, i32 1>
+ ret <5 x i7> %splat
+}
+
+; Negative test - don't increase instruction count
+
+define <4 x float> @insert_nonzero_index_splat_extra_use(float %x) {
+; CHECK-LABEL: @insert_nonzero_index_splat_extra_use(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 2
+; CHECK-NEXT: call void @use(<4 x float> [[XV]])
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[SPLAT]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 2
+ call void @use(<4 x float> %xv)
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 2, i32 undef>
+ ret <4 x float> %splat
+}
+
+; Negative test - non-undef base vector
+
+define <4 x float> @insert_nonzero_index_splat_wrong_base(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_nonzero_index_splat_wrong_base(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 2
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[SPLAT]]
+;
+ %xv = insertelement <4 x float> %y, float %x, i32 2
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 2, i32 3, i32 undef>
+ ret <4 x float> %splat
+}
+
+; Negative test - non-constant insert index
+
+define <4 x float> @insert_nonzero_index_splat_wrong_index(float %x, i32 %index) {
+; CHECK-LABEL: @insert_nonzero_index_splat_wrong_index(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 [[INDEX:%.*]]
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[SPLAT]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 %index
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+ ret <4 x float> %splat
+}
+
+define <4 x float> @insert_in_splat(float %x) {
+; CHECK-LABEL: @insert_in_splat(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+ %r = insertelement <4 x float> %splat, float %x, i32 3
+ ret <4 x float> %r
+}
+
+define <4 x float> @insert_in_splat_extra_uses(float %x) {
+; CHECK-LABEL: @insert_in_splat_extra_uses(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use(<4 x float> [[XV]])
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT: call void @use(<4 x float> [[SPLAT]])
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ call void @use(<4 x float> %xv)
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+ call void @use(<4 x float> %splat)
+ %r = insertelement <4 x float> %splat, float %x, i32 3
+ ret <4 x float> %r
+}
+
+; Negative test - not a constant index insert
+
+define <4 x float> @insert_in_splat_variable_index(float %x, i32 %y) {
+; CHECK-LABEL: @insert_in_splat_variable_index(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 [[Y:%.*]]
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 undef>
+ %r = insertelement <4 x float> %splat, float %x, i32 %y
+ ret <4 x float> %r
+}
+
+; Negative test - not a splat shuffle
+
+define <4 x float> @insert_in_nonsplat(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_in_nonsplat(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> [[Y:%.*]], <4 x i32> <i32 undef, i32 0, i32 4, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> poison, float %x, i32 0
+ %splat = shufflevector <4 x float> %xv, <4 x float> %y, <4 x i32> <i32 undef, i32 0, i32 4, i32 undef>
+ %r = insertelement <4 x float> %splat, float %x, i32 3
+ ret <4 x float> %r
+}
+
+; Negative test - not a splat shuffle
+
+define <4 x float> @insert_in_nonsplat2(float %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_in_nonsplat2(
+; CHECK-NEXT: [[XV:%.*]] = insertelement <4 x float> [[Y:%.*]], float [[X:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <4 x float> [[XV]], <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> [[SPLAT]], float [[X]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %xv = insertelement <4 x float> %y, float %x, i32 0
+ %splat = shufflevector <4 x float> %xv, <4 x float> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 undef>
+ %r = insertelement <4 x float> %splat, float %x, i32 3
+ ret <4 x float> %r
+}
+
+define <4 x i8> @shuf_identity_padding(<2 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_padding(
+; CHECK-NEXT: [[V1:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT: ret <4 x i8> [[V2]]
+;
+ %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %x1 = extractelement <2 x i8> %x, i32 1
+ %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 1
+ %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+ ret <4 x i8> %v2
+}
+
+define <3 x i8> @shuf_identity_extract(<4 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_extract(
+; CHECK-NEXT: [[V1:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: [[V2:%.*]] = insertelement <3 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT: ret <3 x i8> [[V2]]
+;
+ %v0 = shufflevector <4 x i8> %x, <4 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 undef>
+ %x1 = extractelement <4 x i8> %x, i32 1
+ %v1 = insertelement <3 x i8> %v0, i8 %x1, i32 1
+ %v2 = insertelement <3 x i8> %v1, i8 %y, i32 2
+ ret <3 x i8> %v2
+}
+
+define <4 x float> @shuf_identity_extract_extra_use(<6 x float> %x, float %y) {
+; CHECK-LABEL: @shuf_identity_extract_extra_use(
+; CHECK-NEXT: [[V0:%.*]] = shufflevector <6 x float> [[X:%.*]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+; CHECK-NEXT: call void @use(<4 x float> [[V0]])
+; CHECK-NEXT: [[V1:%.*]] = shufflevector <6 x float> [[X]], <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 2, i32 3>
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[Y:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[V2]]
+;
+ %v0 = shufflevector <6 x float> %x, <6 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 3>
+ call void @use(<4 x float> %v0)
+ %x1 = extractelement <6 x float> %x, i32 2
+ %v1 = insertelement <4 x float> %v0, float %x1, i32 2
+ %v2 = insertelement <4 x float> %v1, float %y, i32 1
+ ret <4 x float> %v2
+}
+
+; Negative test - can't map variable index to shuffle mask.
+
+define <4 x i8> @shuf_identity_padding_variable_index(<2 x i8> %x, i8 %y, i32 %index) {
+; CHECK-LABEL: @shuf_identity_padding_variable_index(
+; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 [[INDEX:%.*]]
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 [[INDEX]]
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT: ret <4 x i8> [[V2]]
+;
+ %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %x1 = extractelement <2 x i8> %x, i32 %index
+ %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 %index
+ %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+ ret <4 x i8> %v2
+}
+
+; Negative test - don't create arbitrary shuffle masks.
+
+define <4 x i8> @shuf_identity_padding_wrong_source_vec(<2 x i8> %x, i8 %y, <2 x i8> %other) {
+; CHECK-LABEL: @shuf_identity_padding_wrong_source_vec(
+; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[OTHER:%.*]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 2
+; CHECK-NEXT: ret <4 x i8> [[V2]]
+;
+ %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %x1 = extractelement <2 x i8> %other, i32 1
+ %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 1
+ %v2 = insertelement <4 x i8> %v1, i8 %y, i32 2
+ ret <4 x i8> %v2
+}
+
+; Negative test - don't create arbitrary shuffle masks.
+
+define <4 x i8> @shuf_identity_padding_wrong_index(<2 x i8> %x, i8 %y) {
+; CHECK-LABEL: @shuf_identity_padding_wrong_index(
+; CHECK-NEXT: [[V0:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[X1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i8> [[V0]], i8 [[X1]], i32 2
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i8> [[V1]], i8 [[Y:%.*]], i32 3
+; CHECK-NEXT: ret <4 x i8> [[V2]]
+;
+ %v0 = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %x1 = extractelement <2 x i8> %x, i32 1
+ %v1 = insertelement <4 x i8> %v0, i8 %x1, i32 2
+ %v2 = insertelement <4 x i8> %v1, i8 %y, i32 3
+ ret <4 x i8> %v2
+}
+
+define <4 x float> @insert_undemanded_element_op0(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_undemanded_element_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT: call void @use(<4 x float> [[INS]])
+; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[X]], <4 x float> [[Y:%.*]], <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT: ret <4 x float> [[S]]
+;
+ %ins = insertelement <4 x float> %x, float 42.0, i32 3
+ call void @use(<4 x float> %ins)
+ %s = shufflevector <4 x float> %ins, <4 x float> %y, <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+ ret <4 x float> %s
+}
+
+define <4 x float> @insert_undemanded_element_op1(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_undemanded_element_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT: call void @use(<4 x float> [[INS]])
+; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[X]], <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT: ret <4 x float> [[S]]
+;
+ %ins = insertelement <4 x float> %x, float 42.0, i32 3
+ call void @use(<4 x float> %ins)
+ %s = shufflevector <4 x float> %y, <4 x float> %ins, <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+ ret <4 x float> %s
+}
+
+; Negative test - shuffle chooses the inserted constant.
+
+define <4 x float> @insert_demanded_element_op0(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_demanded_element_op0(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.200000e+01, i32 3
+; CHECK-NEXT: call void @use(<4 x float> [[INS]])
+; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[INS]], <4 x float> [[Y:%.*]], <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+; CHECK-NEXT: ret <4 x float> [[S]]
+;
+ %ins = insertelement <4 x float> %x, float 42.0, i32 3
+ call void @use(<4 x float> %ins)
+ %s = shufflevector <4 x float> %ins, <4 x float> %y, <4 x i32> <i32 3, i32 2, i32 1, i32 4>
+ ret <4 x float> %s
+}
+
+; Negative test - shuffle chooses the inserted constant.
+
+define <4 x float> @insert_demanded_element_op1(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @insert_demanded_element_op1(
+; CHECK-NEXT: [[INS:%.*]] = insertelement <4 x float> [[X:%.*]], float 4.300000e+01, i32 3
+; CHECK-NEXT: call void @use(<4 x float> [[INS]])
+; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x float> [[Y:%.*]], <4 x float> [[INS]], <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+; CHECK-NEXT: ret <4 x float> [[S]]
+;
+ %ins = insertelement <4 x float> %x, float 43.0, i32 3
+ call void @use(<4 x float> %ins)
+ %s = shufflevector <4 x float> %y, <4 x float> %ins, <4 x i32> <i32 0, i32 7, i32 1, i32 4>
+ ret <4 x float> %s
+}
+
+define <4 x float> @splat_constant(<4 x float> %x) {
+; CHECK-LABEL: @splat_constant(
+; CHECK-NEXT: [[INS3:%.*]] = insertelement <4 x float> [[X:%.*]], float 3.000000e+00, i32 3
+; CHECK-NEXT: [[R:%.*]] = fadd <4 x float> [[INS3]], <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %ins3 = insertelement <4 x float> %x, float 3.0, i32 3
+ %splat3 = shufflevector <4 x float> %ins3, <4 x float> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+ %r = fadd <4 x float> %ins3, %splat3
+ ret <4 x float> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
new file mode 100644
index 000000000000..8e49e29eecf1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics-inseltpoison.ll
@@ -0,0 +1,271 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
+declare void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptrs, i32, <2 x i1> %mask)
+declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32, <2 x i1> %mask, <2 x double> %passthru)
+declare <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32, <4 x i1> %mask, <4 x double> %passthru)
+declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32, <2 x i1> %mask)
+
+define <2 x double> @load_zeromask(<2 x double>* %ptr, <2 x double> %passthru) {
+; CHECK-LABEL: @load_zeromask(
+; CHECK-NEXT: ret <2 x double> [[PASSTHRU:%.*]]
+;
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 1, <2 x i1> zeroinitializer, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+define <2 x double> @load_onemask(<2 x double>* %ptr, <2 x double> %passthru) {
+; CHECK-LABEL: @load_onemask(
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT: ret <2 x double> [[UNMASKEDLOAD]]
+;
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 1>, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+define <2 x double> @load_undefmask(<2 x double>* %ptr, <2 x double> %passthru) {
+; CHECK-LABEL: @load_undefmask(
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT: ret <2 x double> [[UNMASKEDLOAD]]
+;
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 undef>, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+ at G = external global i8
+
+define <2 x double> @load_cemask(<2 x double>* %ptr, <2 x double> %passthru) {
+; CHECK-LABEL: @load_cemask(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 2, <2 x i1> <i1 true, i1 ptrtoint (i8* @G to i1)>, <2 x double> [[PASSTHRU:%.*]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 ptrtoint (i8* @G to i1)>, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+define <2 x double> @load_lane0(<2 x double>* %ptr, double %pt) {
+; CHECK-LABEL: @load_lane0(
+; CHECK-NEXT: [[PTV2:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 1
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> [[PTV2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+define double @load_all(double* %base, double %pt) {
+; CHECK-LABEL: @load_all(
+; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <4 x i64> <i64 0, i64 undef, i64 2, i64 3>
+; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> [[PTRS]], i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x double> undef)
+; CHECK-NEXT: [[ELT:%.*]] = extractelement <4 x double> [[RES]], i64 2
+; CHECK-NEXT: ret double [[ELT]]
+;
+ %ptrs = getelementptr double, double* %base, <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+ %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32 4, <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x double> undef)
+ %elt = extractelement <4 x double> %res, i64 2
+ ret double %elt
+}
+
+define <2 x double> @load_generic(<2 x double>* %ptr, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @load_generic(
+; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+define <2 x double> @load_speculative(<2 x double>* dereferenceable(16) align 4 %ptr, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @load_speculative(
+; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = select <2 x i1> [[MASK:%.*]], <2 x double> [[UNMASKEDLOAD]], <2 x double> [[PTV2]]
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+define <2 x double> @load_speculative_less_aligned(<2 x double>* dereferenceable(16) %ptr, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @load_speculative_less_aligned(
+; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = select <2 x i1> [[MASK:%.*]], <2 x double> [[UNMASKEDLOAD]], <2 x double> [[PTV2]]
+; CHECK-NEXT: ret <2 x double> [[TMP1]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+; Can't speculate since only half of required size is known deref
+
+define <2 x double> @load_spec_neg_size(<2 x double>* dereferenceable(8) %ptr, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @load_spec_neg_size(
+; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PTV2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+; Can only speculate one lane (but it's the only one active)
+define <2 x double> @load_spec_lan0(<2 x double>* dereferenceable(8) %ptr, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @load_spec_lan0(
+; CHECK-NEXT: [[PTV1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* nonnull [[PTR:%.*]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PTV2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptv1 = insertelement <2 x double> poison, double %pt, i64 0
+ %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+ %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+ %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 4, <2 x i1> %mask2, <2 x double> %ptv2)
+ ret <2 x double> %res
+}
+
+define void @store_zeromask(<2 x double>* %ptr, <2 x double> %val) {
+; CHECK-LABEL: @store_zeromask(
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 4, <2 x i1> zeroinitializer)
+ ret void
+}
+
+define void @store_onemask(<2 x double>* %ptr, <2 x double> %val) {
+; CHECK-LABEL: @store_onemask(
+; CHECK-NEXT: store <2 x double> [[VAL:%.*]], <2 x double>* [[PTR:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 4, <2 x i1> <i1 1, i1 1>)
+ ret void
+}
+
+define void @store_demandedelts(<2 x double>* %ptr, double %val) {
+; CHECK-LABEL: @store_demandedelts(
+; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> poison, double [[VAL:%.*]], i32 0
+; CHECK-NEXT: call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> [[VALVEC1]], <2 x double>* [[PTR:%.*]], i32 4, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT: ret void
+;
+ %valvec1 = insertelement <2 x double> poison, double %val, i32 0
+ %valvec2 = insertelement <2 x double> %valvec1, double %val, i32 1
+ call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %valvec2, <2 x double>* %ptr, i32 4, <2 x i1> <i1 true, i1 false>)
+ ret void
+}
+
+define <2 x double> @gather_generic(<2 x double*> %ptrs, <2 x i1> %mask, <2 x double> %passthru) {
+; CHECK-LABEL: @gather_generic(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS:%.*]], i32 4, <2 x i1> [[MASK:%.*]], <2 x double> [[PASSTHRU:%.*]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+
+define <2 x double> @gather_zeromask(<2 x double*> %ptrs, <2 x double> %passthru) {
+; CHECK-LABEL: @gather_zeromask(
+; CHECK-NEXT: ret <2 x double> [[PASSTHRU:%.*]]
+;
+ %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> zeroinitializer, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+
+define <2 x double> @gather_onemask(<2 x double*> %ptrs, <2 x double> %passthru) {
+; CHECK-LABEL: @gather_onemask(
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS:%.*]], i32 4, <2 x i1> <i1 true, i1 true>, <2 x double> undef)
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> <i1 true, i1 true>, <2 x double> %passthru)
+ ret <2 x double> %res
+}
+
+define <4 x double> @gather_lane2(double* %base, double %pt) {
+; CHECK-LABEL: @gather_lane2(
+; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <4 x i64> <i64 undef, i64 undef, i64 2, i64 undef>
+; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <4 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <4 x double> [[PT_V1]], <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 0>
+; CHECK-NEXT: [[RES:%.*]] = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> [[PTRS]], i32 4, <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x double> [[PT_V2]])
+; CHECK-NEXT: ret <4 x double> [[RES]]
+;
+ %ptrs = getelementptr double, double *%base, <4 x i64> <i64 0, i64 1, i64 2, i64 3>
+ %pt_v1 = insertelement <4 x double> poison, double %pt, i64 0
+ %pt_v2 = shufflevector <4 x double> %pt_v1, <4 x double> undef, <4 x i32> zeroinitializer
+ %res = call <4 x double> @llvm.masked.gather.v4f64.v4p0f64(<4 x double*> %ptrs, i32 4, <4 x i1> <i1 false, i1 false, i1 true, i1 false>, <4 x double> %pt_v2)
+ ret <4 x double> %res
+}
+
+define <2 x double> @gather_lane0_maybe(double* %base, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @gather_lane0_maybe(
+; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+ %pt_v1 = insertelement <2 x double> poison, double %pt, i64 0
+ %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+ %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+ %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask2, <2 x double> %pt_v2)
+ ret <2 x double> %res
+}
+
+define <2 x double> @gather_lane0_maybe_spec(double* %base, double %pt, <2 x i1> %mask) {
+; CHECK-LABEL: @gather_lane0_maybe_spec(
+; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT: [[PT_V1:%.*]] = insertelement <2 x double> poison, double [[PT:%.*]], i64 0
+; CHECK-NEXT: [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[MASK2:%.*]] = insertelement <2 x i1> [[MASK:%.*]], i1 false, i64 1
+; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 4, <2 x i1> [[MASK2]], <2 x double> [[PT_V2]])
+; CHECK-NEXT: ret <2 x double> [[RES]]
+;
+ %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+ %pt_v1 = insertelement <2 x double> poison, double %pt, i64 0
+ %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+ %mask2 = insertelement <2 x i1> %mask, i1 false, i64 1
+ %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask2, <2 x double> %pt_v2)
+ ret <2 x double> %res
+}
+
+
+define void @scatter_zeromask(<2 x double*> %ptrs, <2 x double> %val) {
+; CHECK-LABEL: @scatter_zeromask(
+; CHECK-NEXT: ret void
+;
+ call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32 8, <2 x i1> zeroinitializer)
+ ret void
+}
+
+define void @scatter_demandedelts(double* %ptr, double %val) {
+; CHECK-LABEL: @scatter_demandedelts(
+; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, double* [[PTR:%.*]], <2 x i64> <i64 0, i64 undef>
+; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> poison, double [[VAL:%.*]], i32 0
+; CHECK-NEXT: call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> [[VALVEC1]], <2 x double*> [[PTRS]], i32 8, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT: ret void
+;
+ %ptrs = getelementptr double, double* %ptr, <2 x i64> <i64 0, i64 1>
+ %valvec1 = insertelement <2 x double> poison, double %val, i32 0
+ %valvec2 = insertelement <2 x double> %valvec1, double %val, i32 1
+ call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %valvec2, <2 x double*> %ptrs, i32 8, <2 x i1> <i1 true, i1 false>)
+ ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll b/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
new file mode 100644
index 000000000000..ec69291fb87c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr38984-inseltpoison.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "p:16:16"
+
+ at a = external global [21 x i16], align 1
+ at offsets = external global [4 x i16], align 1
+
+; The "same gep" optimization should work with vector icmp.
+define <4 x i1> @PR38984_1() {
+; CHECK-LABEL: @PR38984_1(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret <4 x i1> <i1 true, i1 true, i1 true, i1 true>
+;
+entry:
+ %0 = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef), align 1
+ %1 = insertelement <4 x i16> poison, i16 %0, i32 3
+ %2 = getelementptr i32, i32* null, <4 x i16> %1
+ %3 = getelementptr i32, i32* null, <4 x i16> %1
+ %4 = icmp eq <4 x i32*> %2, %3
+ ret <4 x i1> %4
+}
+
+; The "compare base pointers" optimization should not kick in for vector icmp.
+define <4 x i1> @PR38984_2() {
+; CHECK-LABEL: @PR38984_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef), align 2
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i16> poison, i16 [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i16, i16* getelementptr inbounds ([21 x i16], [21 x i16]* @a, i16 1, i16 0), <4 x i16> [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i16, i16* null, <4 x i16> [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i16*> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: ret <4 x i1> [[TMP4]]
+;
+entry:
+ %0 = load i16, i16* getelementptr ([4 x i16], [4 x i16]* @offsets, i16 0, i16 undef)
+ %1 = insertelement <4 x i16> poison, i16 %0, i32 3
+ %2 = getelementptr i16, i16* getelementptr ([21 x i16], [21 x i16]* @a, i64 1, i32 0), <4 x i16> %1
+ %3 = getelementptr i16, i16* null, <4 x i16> %1
+ %4 = icmp eq <4 x i16*> %2, %3
+ ret <4 x i1> %4
+}
diff --git a/llvm/test/Transforms/InstCombine/scalarization-inseltpoison.ll b/llvm/test/Transforms/InstCombine/scalarization-inseltpoison.ll
new file mode 100644
index 000000000000..b2f815971c46
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/scalarization-inseltpoison.ll
@@ -0,0 +1,335 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define i32 @extract_load(<4 x i32>* %p) {
+; CHECK-LABEL: @extract_load(
+; CHECK-NEXT: [[X:%.*]] = load <4 x i32>, <4 x i32>* [[P:%.*]], align 4
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x i32> [[X]], i32 1
+; CHECK-NEXT: ret i32 [[EXT]]
+;
+ %x = load <4 x i32>, <4 x i32>* %p, align 4
+ %ext = extractelement <4 x i32> %x, i32 1
+ ret i32 %ext
+}
+
+define double @extract_load_fp(<4 x double>* %p) {
+; CHECK-LABEL: @extract_load_fp(
+; CHECK-NEXT: [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 3
+; CHECK-NEXT: ret double [[EXT]]
+;
+ %x = load <4 x double>, <4 x double>* %p, align 32
+ %ext = extractelement <4 x double> %x, i32 3
+ ret double %ext
+}
+
+define double @extract_load_volatile(<4 x double>* %p) {
+; CHECK-LABEL: @extract_load_volatile(
+; CHECK-NEXT: [[X:%.*]] = load volatile <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 2
+; CHECK-NEXT: ret double [[EXT]]
+;
+ %x = load volatile <4 x double>, <4 x double>* %p
+ %ext = extractelement <4 x double> %x, i32 2
+ ret double %ext
+}
+
+define double @extract_load_extra_use(<4 x double>* %p, <4 x double>* %p2) {
+; CHECK-LABEL: @extract_load_extra_use(
+; CHECK-NEXT: [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 8
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 0
+; CHECK-NEXT: store <4 x double> [[X]], <4 x double>* [[P2:%.*]], align 32
+; CHECK-NEXT: ret double [[EXT]]
+;
+ %x = load <4 x double>, <4 x double>* %p, align 8
+ %ext = extractelement <4 x double> %x, i32 0
+ store <4 x double> %x, <4 x double>* %p2
+ ret double %ext
+}
+
+define double @extract_load_variable_index(<4 x double>* %p, i32 %y) {
+; CHECK-LABEL: @extract_load_variable_index(
+; CHECK-NEXT: [[X:%.*]] = load <4 x double>, <4 x double>* [[P:%.*]], align 32
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <4 x double> [[X]], i32 [[Y:%.*]]
+; CHECK-NEXT: ret double [[EXT]]
+;
+ %x = load <4 x double>, <4 x double>* %p
+ %ext = extractelement <4 x double> %x, i32 %y
+ ret double %ext
+}
+
+define void @scalarize_phi(i32 * %n, float * %inout) {
+; CHECK-LABEL: @scalarize_phi(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[T0:%.*]] = load volatile float, float* [[INOUT:%.*]], align 4
+; CHECK-NEXT: br label [[FOR_COND:%.*]]
+; CHECK: for.cond:
+; CHECK-NEXT: [[TMP0:%.*]] = phi float [ [[T0]], [[ENTRY:%.*]] ], [ [[TMP1:%.*]], [[FOR_BODY:%.*]] ]
+; CHECK-NEXT: [[I_0:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[T1:%.*]] = load i32, i32* [[N:%.*]], align 4
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[I_0]], [[T1]]
+; CHECK-NEXT: br i1 [[CMP_NOT]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK: for.body:
+; CHECK-NEXT: store volatile float [[TMP0]], float* [[INOUT]], align 4
+; CHECK-NEXT: [[TMP1]] = fmul float [[TMP0]], 0x4002A3D700000000
+; CHECK-NEXT: [[INC]] = add nuw nsw i32 [[I_0]], 1
+; CHECK-NEXT: br label [[FOR_COND]]
+; CHECK: for.end:
+; CHECK-NEXT: ret void
+;
+entry:
+ %t0 = load volatile float, float * %inout, align 4
+ %insert = insertelement <4 x float> poison, float %t0, i32 0
+ %splat = shufflevector <4 x float> %insert, <4 x float> undef, <4 x i32> zeroinitializer
+ %insert1 = insertelement <4 x float> poison, float 3.0, i32 0
+ br label %for.cond
+
+for.cond:
+ %x.0 = phi <4 x float> [ %splat, %entry ], [ %mul, %for.body ]
+ %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+ %t1 = load i32, i32 * %n, align 4
+ %cmp = icmp ne i32 %i.0, %t1
+ br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+ %t2 = extractelement <4 x float> %x.0, i32 1
+ store volatile float %t2, float * %inout, align 4
+ %mul = fmul <4 x float> %x.0, <float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000>
+ %inc = add nsw i32 %i.0, 1
+ br label %for.cond
+
+for.end:
+ ret void
+}
+
+define float @extract_element_binop_splat_constant_index(<4 x float> %x) {
+; CHECK-LABEL: @extract_element_binop_splat_constant_index(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], 0x4002A3D700000000
+; CHECK-NEXT: ret float [[R]]
+;
+ %b = fadd <4 x float> %x, <float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000, float 0x4002A3D700000000>
+ %r = extractelement <4 x float> %b, i32 2
+ ret float %r
+}
+
+define double @extract_element_binop_splat_with_undef_constant_index(<2 x double> %x) {
+; CHECK-LABEL: @extract_element_binop_splat_with_undef_constant_index(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x double> [[X:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = fdiv double 4.200000e+01, [[TMP1]]
+; CHECK-NEXT: ret double [[R]]
+;
+ %b = fdiv <2 x double> <double 42.0, double undef>, %x
+ %r = extractelement <2 x double> %b, i32 0
+ ret double %r
+}
+
+define float @extract_element_binop_nonsplat_constant_index(<2 x float> %x) {
+; CHECK-LABEL: @extract_element_binop_nonsplat_constant_index(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[X:%.*]], i32 1
+; CHECK-NEXT: [[R:%.*]] = fmul float [[TMP1]], 4.300000e+01
+; CHECK-NEXT: ret float [[R]]
+;
+ %b = fmul <2 x float> %x, <float 42.0, float 43.0>
+ %r = extractelement <2 x float> %b, i32 1
+ ret float %r
+}
+
+define i8 @extract_element_binop_splat_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_splat_variable_index(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sdiv i8 [[TMP1]], 42
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %b = sdiv <4 x i8> %x, <i8 42, i8 42, i8 42, i8 42>
+ %r = extractelement <4 x i8> %b, i32 %y
+ ret i8 %r
+}
+
+define i8 @extract_element_binop_splat_with_undef_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_splat_with_undef_variable_index(
+; CHECK-NEXT: [[B:%.*]] = mul <4 x i8> [[X:%.*]], <i8 42, i8 42, i8 undef, i8 42>
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i8> [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %b = mul <4 x i8> %x, <i8 42, i8 42, i8 undef, i8 42>
+ %r = extractelement <4 x i8> %b, i32 %y
+ ret i8 %r
+}
+
+define i8 @extract_element_binop_nonsplat_variable_index(<4 x i8> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_binop_nonsplat_variable_index(
+; CHECK-NEXT: [[B:%.*]] = lshr <4 x i8> [[X:%.*]], <i8 4, i8 3, i8 undef, i8 2>
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i8> [[B]], i32 [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %b = lshr <4 x i8> %x, <i8 4, i8 3, i8 undef, i8 2>
+ %r = extractelement <4 x i8> %b, i32 %y
+ ret i8 %r
+}
+
+define float @extract_element_load(<4 x float> %x, <4 x float>* %ptr) {
+; CHECK-LABEL: @extract_element_load(
+; CHECK-NEXT: [[LOAD:%.*]] = load <4 x float>, <4 x float>* [[PTR:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[LOAD]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %load = load <4 x float>, <4 x float>* %ptr
+ %add = fadd <4 x float> %x, %load
+ %r = extractelement <4 x float> %add, i32 2
+ ret float %r
+}
+
+define float @extract_element_multi_Use_load(<4 x float> %x, <4 x float>* %ptr0, <4 x float>* %ptr1) {
+; CHECK-LABEL: @extract_element_multi_Use_load(
+; CHECK-NEXT: [[LOAD:%.*]] = load <4 x float>, <4 x float>* [[PTR0:%.*]], align 16
+; CHECK-NEXT: store <4 x float> [[LOAD]], <4 x float>* [[PTR1:%.*]], align 16
+; CHECK-NEXT: [[ADD:%.*]] = fadd <4 x float> [[LOAD]], [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[ADD]], i32 2
+; CHECK-NEXT: ret float [[R]]
+;
+ %load = load <4 x float>, <4 x float>* %ptr0
+ store <4 x float> %load, <4 x float>* %ptr1
+ %add = fadd <4 x float> %x, %load
+ %r = extractelement <4 x float> %add, i32 2
+ ret float %r
+}
+
+define float @extract_element_variable_index(<4 x float> %x, i32 %y) {
+; CHECK-LABEL: @extract_element_variable_index(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = fadd float [[TMP1]], 1.000000e+00
+; CHECK-NEXT: ret float [[R]]
+;
+ %add = fadd <4 x float> %x, <float 1.0, float 1.0, float 1.0, float 1.0>
+ %r = extractelement <4 x float> %add, i32 %y
+ ret float %r
+}
+
+define float @extelt_binop_insertelt(<4 x float> %A, <4 x float> %B, float %f) {
+; CHECK-LABEL: @extelt_binop_insertelt(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT: [[E:%.*]] = fmul nnan float [[TMP1]], [[F:%.*]]
+; CHECK-NEXT: ret float [[E]]
+;
+ %C = insertelement <4 x float> %A, float %f, i32 0
+ %D = fmul nnan <4 x float> %C, %B
+ %E = extractelement <4 x float> %D, i32 0
+ ret float %E
+}
+
+; We recurse to find a scalarizable operand.
+; FIXME: We should propagate the IR flags including wrapping flags.
+
+define i32 @extelt_binop_binop_insertelt(<4 x i32> %A, <4 x i32> %B, i32 %f) {
+; CHECK-LABEL: @extelt_binop_binop_insertelt(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], [[F:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[B]], i32 0
+; CHECK-NEXT: [[E:%.*]] = mul i32 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %v = insertelement <4 x i32> %A, i32 %f, i32 0
+ %C = add <4 x i32> %v, %B
+ %D = mul nsw <4 x i32> %C, %B
+ %E = extractelement <4 x i32> %D, i32 0
+ ret i32 %E
+}
+
+define float @extract_element_constant_vector_variable_index(i32 %y) {
+; CHECK-LABEL: @extract_element_constant_vector_variable_index(
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> <float 1.000000e+00, float 2.000000e+00, float 3.000000e+00, float 4.000000e+00>, i32 [[Y:%.*]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %r = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %y
+ ret float %r
+}
+
+define i1 @cheap_to_extract_icmp(<4 x i32> %x, <4 x i1> %y) {
+; CHECK-LABEL: @cheap_to_extract_icmp(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[Y:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp = icmp eq <4 x i32> %x, zeroinitializer
+ %and = and <4 x i1> %cmp, %y
+ %r = extractelement <4 x i1> %and, i32 2
+ ret i1 %r
+}
+
+define i1 @cheap_to_extract_fcmp(<4 x float> %x, <4 x i1> %y) {
+; CHECK-LABEL: @cheap_to_extract_fcmp(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[X:%.*]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[Y:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = and i1 [[TMP2]], [[TMP3]]
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp = fcmp oeq <4 x float> %x, zeroinitializer
+ %and = and <4 x i1> %cmp, %y
+ %r = extractelement <4 x i1> %and, i32 2
+ ret i1 %r
+}
+
+define i1 @extractelt_vector_icmp_constrhs(<2 x i32> %arg) {
+; CHECK-LABEL: @extractelt_vector_icmp_constrhs(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[EXT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: ret i1 [[EXT]]
+;
+ %cmp = icmp eq <2 x i32> %arg, zeroinitializer
+ %ext = extractelement <2 x i1> %cmp, i32 0
+ ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_constrhs(<2 x float> %arg) {
+; CHECK-LABEL: @extractelt_vector_fcmp_constrhs(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[ARG:%.*]], i32 0
+; CHECK-NEXT: [[EXT:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT: ret i1 [[EXT]]
+;
+ %cmp = fcmp oeq <2 x float> %arg, zeroinitializer
+ %ext = extractelement <2 x i1> %cmp, i32 0
+ ret i1 %ext
+}
+
+define i1 @extractelt_vector_icmp_constrhs_dynidx(<2 x i32> %arg, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_icmp_constrhs_dynidx(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i32> [[ARG:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT: [[EXT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: ret i1 [[EXT]]
+;
+ %cmp = icmp eq <2 x i32> %arg, zeroinitializer
+ %ext = extractelement <2 x i1> %cmp, i32 %idx
+ ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_constrhs_dynidx(<2 x float> %arg, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_fcmp_constrhs_dynidx(
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[ARG:%.*]], i32 [[IDX:%.*]]
+; CHECK-NEXT: [[EXT:%.*]] = fcmp oeq float [[TMP1]], 0.000000e+00
+; CHECK-NEXT: ret i1 [[EXT]]
+;
+ %cmp = fcmp oeq <2 x float> %arg, zeroinitializer
+ %ext = extractelement <2 x i1> %cmp, i32 %idx
+ ret i1 %ext
+}
+
+define i1 @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(<2 x float> %arg0, <2 x float> %arg1, <2 x float> %arg2, i32 %idx) {
+; CHECK-LABEL: @extractelt_vector_fcmp_not_cheap_to_scalarize_multi_use(
+; CHECK-NEXT: [[ADD:%.*]] = fadd <2 x float> [[ARG1:%.*]], [[ARG2:%.*]]
+; CHECK-NEXT: store volatile <2 x float> [[ADD]], <2 x float>* undef, align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp oeq <2 x float> [[ADD]], [[ARG0:%.*]]
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <2 x i1> [[CMP]], i32 0
+; CHECK-NEXT: ret i1 [[EXT]]
+;
+ %add = fadd <2 x float> %arg1, %arg2
+ store volatile <2 x float> %add, <2 x float>* undef
+ %cmp = fcmp oeq <2 x float> %arg0, %add
+ %ext = extractelement <2 x i1> %cmp, i32 0
+ ret i1 %ext
+}
diff --git a/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
new file mode 100644
index 000000000000..a59782378b9a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/select-extractelement-inseltpoison.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare void @v4float_user(<4 x float>) #0
+
+define float @extract_one_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_one_select(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT: ret float [[EXTRACT]]
+;
+ %cmp = icmp ne i32 %c, 0
+ %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+ %extract = extractelement <4 x float> %sel, i32 2
+ ret float %extract
+}
+
+; Multiple extractelements
+define <2 x float> @extract_two_select(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_two_select(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> [[BUILD2]]
+;
+ %cmp = icmp ne i32 %c, 0
+ %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+ %extract1 = extractelement <4 x float> %sel, i32 1
+ %extract2 = extractelement <4 x float> %sel, i32 2
+ %build1 = insertelement <2 x float> poison, float %extract1, i32 0
+ %build2 = insertelement <2 x float> %build1, float %extract2, i32 1
+ ret <2 x float> %build2
+}
+
+; Select has an extra non-extractelement user, don't change it
+define float @extract_one_select_user(<4 x float> %a, <4 x float> %b, i32 %c) #0 {
+; CHECK-LABEL: @extract_one_select_user(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i32 [[C:%.*]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT: call void @v4float_user(<4 x float> [[SEL]])
+; CHECK-NEXT: ret float [[EXTRACT]]
+;
+ %cmp = icmp ne i32 %c, 0
+ %sel = select i1 %cmp, <4 x float> %a, <4 x float> %b
+ %extract = extractelement <4 x float> %sel, i32 2
+ call void @v4float_user(<4 x float> %sel)
+ ret float %extract
+}
+
+define float @extract_one_vselect_user(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_one_vselect_user(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <4 x float> [[SEL]], i32 2
+; CHECK-NEXT: call void @v4float_user(<4 x float> [[SEL]])
+; CHECK-NEXT: ret float [[EXTRACT]]
+;
+ %cmp = icmp ne <4 x i32> %c, zeroinitializer
+ %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+ %extract = extractelement <4 x float> %sel, i32 2
+ call void @v4float_user(<4 x float> %sel)
+ ret float %extract
+}
+
+; Do not convert the vector select into a scalar select. That would increase
+; the instruction count and potentially obfuscate a vector min/max idiom.
+
+define float @extract_one_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_one_vselect(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[SELECT:%.*]] = select <4 x i1> [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <4 x float> [[SELECT]], i32 0
+; CHECK-NEXT: ret float [[EXTRACT]]
+;
+ %cmp = icmp ne <4 x i32> %c, zeroinitializer
+ %select = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+ %extract = extractelement <4 x float> %select, i32 0
+ ret float %extract
+}
+
+; Multiple extractelements from a vector select
+define <2 x float> @extract_two_vselect(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @extract_two_vselect(
+; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[BUILD2:%.*]] = shufflevector <4 x float> [[SEL]], <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> [[BUILD2]]
+;
+ %cmp = icmp ne <4 x i32> %c, zeroinitializer
+ %sel = select <4 x i1> %cmp, <4 x float> %a, <4 x float> %b
+ %extract1 = extractelement <4 x float> %sel, i32 1
+ %extract2 = extractelement <4 x float> %sel, i32 2
+ %build1 = insertelement <2 x float> poison, float %extract1, i32 0
+ %build2 = insertelement <2 x float> %build1, float %extract2, i32 1
+ ret <2 x float> %build2
+}
+
+; The vector selects are not decomposed into scalar selects because that would increase
+; the instruction count. Extract+insert is converted to non-lane-crossing shuffles.
+; Test multiple extractelements
+define <4 x float> @simple_vector_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_vector_select(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT: [[A_SINK:%.*]] = select i1 [[TOBOOL_NOT]], <4 x float> [[B:%.*]], <4 x float> [[A:%.*]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT: [[TOBOOL1_NOT:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT: [[A_SINK1:%.*]] = select i1 [[TOBOOL1_NOT]], <4 x float> [[B]], <4 x float> [[A]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[A_SINK]], <4 x float> [[A_SINK1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT: [[TOBOOL6_NOT:%.*]] = icmp eq i32 [[TMP3]], 0
+; CHECK-NEXT: [[A_SINK2:%.*]] = select i1 [[TOBOOL6_NOT]], <4 x float> [[B]], <4 x float> [[A]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[A_SINK2]], <4 x i32> <i32 0, i32 1, i32 6, i32 undef>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[TOBOOL11_NOT:%.*]] = icmp eq i32 [[TMP5]], 0
+; CHECK-NEXT: [[A_SINK3:%.*]] = select i1 [[TOBOOL11_NOT]], <4 x float> [[B]], <4 x float> [[A]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> [[A_SINK3]], <4 x i32> <i32 0, i32 1, i32 2, i32 7>
+; CHECK-NEXT: ret <4 x float> [[TMP6]]
+;
+entry:
+ %0 = extractelement <4 x i32> %c, i32 0
+ %tobool = icmp ne i32 %0, 0
+ %a.sink = select i1 %tobool, <4 x float> %a, <4 x float> %b
+ %1 = extractelement <4 x float> %a.sink, i32 0
+ %2 = insertelement <4 x float> poison, float %1, i32 0
+ %3 = extractelement <4 x i32> %c, i32 1
+ %tobool1 = icmp ne i32 %3, 0
+ %a.sink1 = select i1 %tobool1, <4 x float> %a, <4 x float> %b
+ %4 = extractelement <4 x float> %a.sink1, i32 1
+ %5 = insertelement <4 x float> %2, float %4, i32 1
+ %6 = extractelement <4 x i32> %c, i32 2
+ %tobool6 = icmp ne i32 %6, 0
+ %a.sink2 = select i1 %tobool6, <4 x float> %a, <4 x float> %b
+ %7 = extractelement <4 x float> %a.sink2, i32 2
+ %8 = insertelement <4 x float> %5, float %7, i32 2
+ %9 = extractelement <4 x i32> %c, i32 3
+ %tobool11 = icmp ne i32 %9, 0
+ %a.sink3 = select i1 %tobool11, <4 x float> %a, <4 x float> %b
+ %10 = extractelement <4 x float> %a.sink3, i32 3
+ %11 = insertelement <4 x float> %8, float %10, i32 3
+ ret <4 x float> %11
+}
+
+define <4 x i32> @extract_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @extract_cond(
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %cond = extractelement <4 x i1> %condv, i32 3
+ %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_cond(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @splat_cond(
+; CHECK-NEXT: [[SPLATCOND:%.*]] = shufflevector <4 x i1> [[CONDV:%.*]], <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[SPLATCOND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatcond = shufflevector <4 x i1> %condv, <4 x i1> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+ %r = select <4 x i1> %splatcond, <4 x i32> %x, <4 x i32> %y
+ ret <4 x i32> %r
+}
+
+declare void @extra_use(i1)
+
+; Negative test
+
+define <4 x i32> @extract_cond_extra_use(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv) {
+; CHECK-LABEL: @extract_cond_extra_use(
+; CHECK-NEXT: [[COND:%.*]] = extractelement <4 x i1> [[CONDV:%.*]], i32 3
+; CHECK-NEXT: call void @extra_use(i1 [[COND]])
+; CHECK-NEXT: [[R:%.*]] = select i1 [[COND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %cond = extractelement <4 x i1> %condv, i32 3
+ call void @extra_use(i1 %cond)
+ %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+ ret <4 x i32> %r
+}
+
+; Negative test
+
+define <4 x i32> @extract_cond_variable_index(<4 x i32> %x, <4 x i32> %y, <4 x i1> %condv, i32 %index) {
+; CHECK-LABEL: @extract_cond_variable_index(
+; CHECK-NEXT: [[COND:%.*]] = extractelement <4 x i1> [[CONDV:%.*]], i32 [[INDEX:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select i1 [[COND]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %cond = extractelement <4 x i1> %condv, i32 %index
+ %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+ ret <4 x i32> %r
+}
+
+; IR shuffle can alter the number of elements in the vector, so this is ok.
+
+define <4 x i32> @extract_cond_type_mismatch(<4 x i32> %x, <4 x i32> %y, <5 x i1> %condv) {
+; CHECK-LABEL: @extract_cond_type_mismatch(
+; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <5 x i1> [[CONDV:%.*]], <5 x i1> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[DOTSPLAT]], <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %cond = extractelement <5 x i1> %condv, i32 1
+ %r = select i1 %cond, <4 x i32> %x, <4 x i32> %y
+ ret <4 x i32> %r
+}
+
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/InstCombine/shift-add-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shift-add-inseltpoison.ll
new file mode 100644
index 000000000000..8d1f068136f1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-add-inseltpoison.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; This test makes sure that these instructions are properly eliminated.
+;
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @shl_C1_add_A_C2_i32(i16 %A) {
+; CHECK-LABEL: @shl_C1_add_A_C2_i32(
+; CHECK-NEXT: [[B:%.*]] = zext i16 [[A:%.*]] to i32
+; CHECK-NEXT: [[D:%.*]] = shl i32 192, [[B]]
+; CHECK-NEXT: ret i32 [[D]]
+;
+ %B = zext i16 %A to i32
+ %C = add i32 %B, 5
+ %D = shl i32 6, %C
+ ret i32 %D
+}
+
+define i32 @ashr_C1_add_A_C2_i32(i32 %A) {
+; CHECK-LABEL: @ashr_C1_add_A_C2_i32(
+; CHECK-NEXT: ret i32 0
+;
+ %B = and i32 %A, 65535
+ %C = add i32 %B, 5
+ %D = ashr i32 6, %C
+ ret i32 %D
+}
+
+define i32 @lshr_C1_add_A_C2_i32(i32 %A) {
+; CHECK-LABEL: @lshr_C1_add_A_C2_i32(
+; CHECK-NEXT: [[B:%.*]] = and i32 [[A:%.*]], 65535
+; CHECK-NEXT: [[D:%.*]] = shl i32 192, [[B]]
+; CHECK-NEXT: ret i32 [[D]]
+;
+ %B = and i32 %A, 65535
+ %C = add i32 %B, 5
+ %D = shl i32 6, %C
+ ret i32 %D
+}
+
+define <4 x i32> @shl_C1_add_A_C2_v4i32(<4 x i16> %A) {
+; CHECK-LABEL: @shl_C1_add_A_C2_v4i32(
+; CHECK-NEXT: [[B:%.*]] = zext <4 x i16> [[A:%.*]] to <4 x i32>
+; CHECK-NEXT: [[D:%.*]] = shl <4 x i32> <i32 6, i32 4, i32 poison, i32 -458752>, [[B]]
+; CHECK-NEXT: ret <4 x i32> [[D]]
+;
+ %B = zext <4 x i16> %A to <4 x i32>
+ %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+ %D = shl <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+ ret <4 x i32> %D
+}
+
+define <4 x i32> @ashr_C1_add_A_C2_v4i32(<4 x i32> %A) {
+; CHECK-LABEL: @ashr_C1_add_A_C2_v4i32(
+; CHECK-NEXT: [[B:%.*]] = and <4 x i32> [[A:%.*]], <i32 0, i32 15, i32 255, i32 65535>
+; CHECK-NEXT: [[D:%.*]] = ashr <4 x i32> <i32 6, i32 1, i32 poison, i32 -1>, [[B]]
+; CHECK-NEXT: ret <4 x i32> [[D]]
+;
+ %B = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+ %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+ %D = ashr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+ ret <4 x i32> %D
+}
+
+define <4 x i32> @lshr_C1_add_A_C2_v4i32(<4 x i32> %A) {
+; CHECK-LABEL: @lshr_C1_add_A_C2_v4i32(
+; CHECK-NEXT: [[B:%.*]] = and <4 x i32> [[A:%.*]], <i32 0, i32 15, i32 255, i32 65535>
+; CHECK-NEXT: [[D:%.*]] = lshr <4 x i32> <i32 6, i32 1, i32 poison, i32 65535>, [[B]]
+; CHECK-NEXT: ret <4 x i32> [[D]]
+;
+ %B = and <4 x i32> %A, <i32 0, i32 15, i32 255, i32 65535>
+ %C = add <4 x i32> %B, <i32 0, i32 1, i32 50, i32 16>
+ %D = lshr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %C
+ ret <4 x i32> %D
+}
+
+define <4 x i32> @shl_C1_add_A_C2_v4i32_splat(i16 %I) {
+; CHECK-LABEL: @shl_C1_add_A_C2_v4i32_splat(
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32
+; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
+; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[E:%.*]] = shl <4 x i32> <i32 6, i32 4, i32 poison, i32 -458752>, [[C]]
+; CHECK-NEXT: ret <4 x i32> [[E]]
+;
+ %A = zext i16 %I to i32
+ %B = insertelement <4 x i32> poison, i32 %A, i32 0
+ %C = shufflevector <4 x i32> %B, <4 x i32> undef, <4 x i32> zeroinitializer
+ %D = add <4 x i32> %C, <i32 0, i32 1, i32 50, i32 16>
+ %E = shl <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %D
+ ret <4 x i32> %E
+}
+
+define <4 x i32> @ashr_C1_add_A_C2_v4i32_splat(i16 %I) {
+; CHECK-LABEL: @ashr_C1_add_A_C2_v4i32_splat(
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32
+; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
+; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[E:%.*]] = ashr <4 x i32> <i32 6, i32 1, i32 poison, i32 -1>, [[C]]
+; CHECK-NEXT: ret <4 x i32> [[E]]
+;
+ %A = zext i16 %I to i32
+ %B = insertelement <4 x i32> poison, i32 %A, i32 0
+ %C = shufflevector <4 x i32> %B, <4 x i32> undef, <4 x i32> zeroinitializer
+ %D = add <4 x i32> %C, <i32 0, i32 1, i32 50, i32 16>
+ %E = ashr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %D
+ ret <4 x i32> %E
+}
+
+define <4 x i32> @lshr_C1_add_A_C2_v4i32_splat(i16 %I) {
+; CHECK-LABEL: @lshr_C1_add_A_C2_v4i32_splat(
+; CHECK-NEXT: [[A:%.*]] = zext i16 [[I:%.*]] to i32
+; CHECK-NEXT: [[B:%.*]] = insertelement <4 x i32> poison, i32 [[A]], i32 0
+; CHECK-NEXT: [[C:%.*]] = shufflevector <4 x i32> [[B]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[E:%.*]] = lshr <4 x i32> <i32 6, i32 1, i32 poison, i32 65535>, [[C]]
+; CHECK-NEXT: ret <4 x i32> [[E]]
+;
+ %A = zext i16 %I to i32
+ %B = insertelement <4 x i32> poison, i32 %A, i32 0
+ %C = shufflevector <4 x i32> %B, <4 x i32> undef, <4 x i32> zeroinitializer
+ %D = add <4 x i32> %C, <i32 0, i32 1, i32 50, i32 16>
+ %E = lshr <4 x i32> <i32 6, i32 2, i32 1, i32 -7>, %D
+ ret <4 x i32> %E
+}
diff --git a/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
new file mode 100644
index 000000000000..2a62bec2e1ed
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shufflevector-div-rem-inseltpoison.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -o - | FileCheck %s
+
+; This test case was added as a reproducer for a miscompile, where instcombine
+; introduced an
+; srem <2 x i16> %1, <i16 undef, i16 2>
+; instruction, which makes the whole srem undefined (even if we only end up
+; extracting the second element in the vector).
+define i16 @test_srem_orig(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_srem_orig(
+; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 undef, i16 1>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT: [[T3:%.*]] = extractelement <2 x i16> [[T2]], i32 1
+; CHECK-NEXT: ret i16 [[T3]]
+;
+ %splatinsert = insertelement <2 x i16> poison, i16 %a, i32 0
+ %splat = shufflevector <2 x i16> %splatinsert, <2 x i16> undef, <2 x i32> zeroinitializer
+ %t1 = select i1 %cmp, <2 x i16> <i16 1, i16 1>, <2 x i16> %splat
+ %t2 = srem <2 x i16> %t1, <i16 2, i16 2>
+ %t3 = extractelement <2 x i16> %t2, i32 1
+ ret i16 %t3
+}
+
+; This is basically a reduced version of test_srem_orig (based on what the
+; code would look like after a few iterations of instcombine, just before we
+; try to transform the shufflevector by doing
+; "evaluateInDifferentElementOrder".
+define <2 x i16> @test_srem(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_srem(
+; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = srem <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT: ret <2 x i16> [[T2]]
+;
+ %splatinsert = insertelement <2 x i16> poison, i16 %a, i32 0
+ %t1 = srem <2 x i16> %splatinsert, <i16 2, i16 1>
+ %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+ ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_urem(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_urem(
+; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = urem <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
+; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT: ret <2 x i16> [[T2]]
+;
+ %splatinsert = insertelement <2 x i16> poison, i16 %a, i32 0
+ %t1 = urem <2 x i16> %splatinsert, <i16 3, i16 1>
+ %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+ ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_sdiv(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_sdiv(
+; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = sdiv <2 x i16> [[SPLATINSERT]], <i16 2, i16 1>
+; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT: ret <2 x i16> [[T2]]
+;
+ %splatinsert = insertelement <2 x i16> poison, i16 %a, i32 0
+ %t1 = sdiv <2 x i16> %splatinsert, <i16 2, i16 1>
+ %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+ ret <2 x i16> %t2
+}
+
+define <2 x i16> @test_udiv(i16 %a, i1 %cmp) {
+; CHECK-LABEL: @test_udiv(
+; CHECK-NEXT: [[SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[A:%.*]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = udiv <2 x i16> [[SPLATINSERT]], <i16 3, i16 1>
+; CHECK-NEXT: [[SPLAT_OP:%.*]] = shufflevector <2 x i16> [[T1]], <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x i16> <i16 77, i16 99>, <2 x i16> [[SPLAT_OP]]
+; CHECK-NEXT: ret <2 x i16> [[T2]]
+;
+ %splatinsert = insertelement <2 x i16> poison, i16 %a, i32 0
+ %t1 = udiv <2 x i16> %splatinsert, <i16 3, i16 1>
+ %splat.op = shufflevector <2 x i16> %t1, <2 x i16> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x i16> <i16 77, i16 99>, <2 x i16> %splat.op
+ ret <2 x i16> %t2
+}
+
+; For fdiv we do not need to worry about div by undef. Verify that the
+; shufflevector is eliminated here.
+define <2 x float> @test_fdiv(float %a, float %b, i1 %cmp) {
+; CHECK-LABEL: @test_fdiv(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = fdiv <2 x float> [[TMP1]], <float undef, float 3.000000e+00>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> <float 7.700000e+01, float 9.900000e+01>, <2 x float> [[TMP2]]
+; CHECK-NEXT: ret <2 x float> [[T2]]
+;
+ %splatinsert = insertelement <2 x float> poison, float %a, i32 0
+ %denom = insertelement <2 x float> <float 3.0, float undef>, float 1.0, i32 1
+ %t1 = fdiv <2 x float> %splatinsert, %denom
+ %splat.op = shufflevector <2 x float> %t1, <2 x float> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x float> <float 77.0, float 99.0>, <2 x float> %splat.op
+ ret <2 x float> %t2
+}
+
+; For frem we do not need to worry about div by undef. Verify that the
+; shufflevector is eliminated here.
+define <2 x float> @test_frem(float %a, float %b, i1 %cmp) {
+; CHECK-LABEL: @test_frem(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> undef, float [[A:%.*]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = frem <2 x float> [[TMP1]], <float undef, float 3.000000e+00>
+; CHECK-NEXT: [[T2:%.*]] = select i1 [[CMP:%.*]], <2 x float> <float 7.700000e+01, float 9.900000e+01>, <2 x float> [[TMP2]]
+; CHECK-NEXT: ret <2 x float> [[T2]]
+;
+ %splatinsert = insertelement <2 x float> poison, float %a, i32 0
+ %denom = insertelement <2 x float> <float 3.0, float undef>, float 1.0, i32 1
+ %t1 = frem <2 x float> %splatinsert, %denom
+ %splat.op = shufflevector <2 x float> %t1, <2 x float> undef, <2 x i32> <i32 undef, i32 0>
+ %t2 = select i1 %cmp, <2 x float> <float 77.0, float 99.0>, <2 x float> %splat.op
+ ret <2 x float> %t2
+}
diff --git a/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll
new file mode 100644
index 000000000000..580854a432bf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/trunc-extractelement-inseltpoison.ll
@@ -0,0 +1,195 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S -data-layout="e" | FileCheck %s --check-prefixes=ANY,LE
+; RUN: opt < %s -instcombine -S -data-layout="E" | FileCheck %s --check-prefixes=ANY,BE
+
+define i32 @shrinkExtractElt_i64_to_i32_0(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_0(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 0
+; LE-NEXT: ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_0(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 1
+; BE-NEXT: ret i32 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i32 0
+ %t = trunc i64 %e to i32
+ ret i32 %t
+}
+
+define i32 @vscale_shrinkExtractElt_i64_to_i32_0(<vscale x 3 x i64> %x) {
+; LE-LABEL: @vscale_shrinkExtractElt_i64_to_i32_0(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 3 x i64> [[X:%.*]] to <vscale x 6 x i32>
+; LE-NEXT: [[T:%.*]] = extractelement <vscale x 6 x i32> [[TMP1]], i32 0
+; LE-NEXT: ret i32 [[T]]
+;
+; BE-LABEL: @vscale_shrinkExtractElt_i64_to_i32_0(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <vscale x 3 x i64> [[X:%.*]] to <vscale x 6 x i32>
+; BE-NEXT: [[T:%.*]] = extractelement <vscale x 6 x i32> [[TMP1]], i32 1
+; BE-NEXT: ret i32 [[T]]
+;
+ %e = extractelement <vscale x 3 x i64> %x, i32 0
+ %t = trunc i64 %e to i32
+ ret i32 %t
+}
+
+
+define i32 @shrinkExtractElt_i64_to_i32_1(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_1(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 2
+; LE-NEXT: ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_1(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 3
+; BE-NEXT: ret i32 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i32 1
+ %t = trunc i64 %e to i32
+ ret i32 %t
+}
+
+define i32 @shrinkExtractElt_i64_to_i32_2(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i32_2(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; LE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 4
+; LE-NEXT: ret i32 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i32_2(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <6 x i32>
+; BE-NEXT: [[T:%.*]] = extractelement <6 x i32> [[TMP1]], i32 5
+; BE-NEXT: ret i32 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i32 2
+ %t = trunc i64 %e to i32
+ ret i32 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_0(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_0(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 0
+; LE-NEXT: ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_0(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 3
+; BE-NEXT: ret i16 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i16 0
+ %t = trunc i64 %e to i16
+ ret i16 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_1(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_1(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 4
+; LE-NEXT: ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_1(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 7
+; BE-NEXT: ret i16 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i16 1
+ %t = trunc i64 %e to i16
+ ret i16 %t
+}
+
+define i16 @shrinkExtractElt_i64_to_i16_2(<3 x i64> %x) {
+; LE-LABEL: @shrinkExtractElt_i64_to_i16_2(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; LE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 8
+; LE-NEXT: ret i16 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i64_to_i16_2(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i64> [[X:%.*]] to <12 x i16>
+; BE-NEXT: [[T:%.*]] = extractelement <12 x i16> [[TMP1]], i32 11
+; BE-NEXT: ret i16 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i16 2
+ %t = trunc i64 %e to i16
+ ret i16 %t
+}
+
+; Crazy types may be ok.
+define i11 @shrinkExtractElt_i33_to_11_2(<3 x i33> %x) {
+; LE-LABEL: @shrinkExtractElt_i33_to_11_2(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <3 x i33> [[X:%.*]] to <9 x i11>
+; LE-NEXT: [[T:%.*]] = extractelement <9 x i11> [[TMP1]], i32 6
+; LE-NEXT: ret i11 [[T]]
+;
+; BE-LABEL: @shrinkExtractElt_i33_to_11_2(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <3 x i33> [[X:%.*]] to <9 x i11>
+; BE-NEXT: [[T:%.*]] = extractelement <9 x i11> [[TMP1]], i32 8
+; BE-NEXT: ret i11 [[T]]
+;
+ %e = extractelement <3 x i33> %x, i16 2
+ %t = trunc i33 %e to i11
+ ret i11 %t
+}
+
+; Do not optimize if it would result in an invalid bitcast instruction.
+define i13 @shrinkExtractElt_i67_to_i13_2(<3 x i67> %x) {
+; ANY-LABEL: @shrinkExtractElt_i67_to_i13_2(
+; ANY-NEXT: [[E:%.*]] = extractelement <3 x i67> [[X:%.*]], i459 2
+; ANY-NEXT: [[T:%.*]] = trunc i67 [[E]] to i13
+; ANY-NEXT: ret i13 [[T]]
+;
+ %e = extractelement <3 x i67> %x, i459 2
+ %t = trunc i67 %e to i13
+ ret i13 %t
+}
+
+; Do not optimize if the bitcast instruction would be valid, but the
+; transform would be wrong.
+define i30 @shrinkExtractElt_i40_to_i30_1(<3 x i40> %x) {
+; ANY-LABEL: @shrinkExtractElt_i40_to_i30_1(
+; ANY-NEXT: [[E:%.*]] = extractelement <3 x i40> [[X:%.*]], i32 1
+; ANY-NEXT: [[T:%.*]] = trunc i40 [[E]] to i30
+; ANY-NEXT: ret i30 [[T]]
+;
+ %e = extractelement <3 x i40> %x, i32 1
+ %t = trunc i40 %e to i30
+ ret i30 %t
+}
+
+; Do not canonicalize if that would increase the instruction count.
+declare void @use(i64)
+define i16 @shrinkExtractElt_i64_to_i16_2_extra_use(<3 x i64> %x) {
+; ANY-LABEL: @shrinkExtractElt_i64_to_i16_2_extra_use(
+; ANY-NEXT: [[E:%.*]] = extractelement <3 x i64> [[X:%.*]], i64 2
+; ANY-NEXT: call void @use(i64 [[E]])
+; ANY-NEXT: [[T:%.*]] = trunc i64 [[E]] to i16
+; ANY-NEXT: ret i16 [[T]]
+;
+ %e = extractelement <3 x i64> %x, i64 2
+ call void @use(i64 %e)
+ %t = trunc i64 %e to i16
+ ret i16 %t
+}
+
+; Check to ensure PR45314 remains fixed.
+define <4 x i64> @PR45314(<4 x i64> %x) {
+; LE-LABEL: @PR45314(
+; LE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32>
+; LE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; LE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64>
+; LE-NEXT: ret <4 x i64> [[B]]
+;
+; BE-LABEL: @PR45314(
+; BE-NEXT: [[TMP1:%.*]] = bitcast <4 x i64> [[X:%.*]] to <8 x i32>
+; BE-NEXT: [[S:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; BE-NEXT: [[B:%.*]] = bitcast <8 x i32> [[S]] to <4 x i64>
+; BE-NEXT: ret <4 x i64> [[B]]
+;
+ %e = extractelement <4 x i64> %x, i32 0
+ %t = trunc i64 %e to i32
+ %i = insertelement <8 x i32> poison, i32 %t, i32 0
+ %s = shufflevector <8 x i32> %i, <8 x i32> undef, <8 x i32> zeroinitializer
+ %b = bitcast <8 x i32> %s to <4 x i64>
+ ret <4 x i64> %b
+}
diff --git a/llvm/test/Transforms/InstCombine/udiv-pow2-vscale-inseltpoison.ll b/llvm/test/Transforms/InstCombine/udiv-pow2-vscale-inseltpoison.ll
new file mode 100644
index 000000000000..a01c53305f5e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/udiv-pow2-vscale-inseltpoison.ll
@@ -0,0 +1,27 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+; This vscale udiv with a power-of-2 spalt on the rhs should not crash opt
+
+; CHECK: define <vscale x 2 x i32> @udiv_pow2_vscale(<vscale x 2 x i32> %lhs)
+define <vscale x 2 x i32> @udiv_pow2_vscale(<vscale x 2 x i32> %lhs) {
+ %splatter = insertelement <vscale x 2 x i32> poison, i32 2, i32 0
+ %rhs = shufflevector <vscale x 2 x i32> %splatter,
+ <vscale x 2 x i32> undef,
+ <vscale x 2 x i32> zeroinitializer
+ %res = udiv <vscale x 2 x i32> %lhs, %rhs
+ ret <vscale x 2 x i32> %res
+}
+
+; This fixed width udiv with a power-of-2 splat on the rhs should also not
+; crash, and instcombine should eliminate the udiv
+
+; CHECK-LABEL: define <2 x i32> @udiv_pow2_fixed(<2 x i32> %lhs)
+; CHECK-NOT: udiv
+define <2 x i32> @udiv_pow2_fixed(<2 x i32> %lhs) {
+ %splatter = insertelement <2 x i32> poison, i32 2, i32 0
+ %rhs = shufflevector <2 x i32> %splatter,
+ <2 x i32> undef,
+ <2 x i32> zeroinitializer
+ %res = udiv <2 x i32> %lhs, %rhs
+ ret <2 x i32> %res
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
new file mode 100644
index 000000000000..78238a9044cd
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts-inseltpoison.ll
@@ -0,0 +1,850 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define i32 @test2(float %f) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[T5:%.*]] = fmul float [[F:%.*]], [[F]]
+; CHECK-NEXT: [[T21:%.*]] = bitcast float [[T5]] to i32
+; CHECK-NEXT: ret i32 [[T21]]
+;
+ %t5 = fmul float %f, %f
+ %t9 = insertelement <4 x float> poison, float %t5, i32 0
+ %t10 = insertelement <4 x float> %t9, float 0.000000e+00, i32 1
+ %t11 = insertelement <4 x float> %t10, float 0.000000e+00, i32 2
+ %t12 = insertelement <4 x float> %t11, float 0.000000e+00, i32 3
+ %t19 = bitcast <4 x float> %t12 to <4 x i32>
+ %t21 = extractelement <4 x i32> %t19, i32 0
+ ret i32 %t21
+}
+
+define void @get_image() nounwind {
+; CHECK-LABEL: @get_image(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @fgetc(i8* null) [[ATTR0:#.*]]
+; CHECK-NEXT: br i1 false, label [[BB2:%.*]], label [[BB3:%.*]]
+; CHECK: bb2:
+; CHECK-NEXT: br label [[BB3]]
+; CHECK: bb3:
+; CHECK-NEXT: unreachable
+;
+entry:
+ %0 = call i32 @fgetc(i8* null) nounwind
+ %1 = trunc i32 %0 to i8
+ %t2 = insertelement <100 x i8> zeroinitializer, i8 %1, i32 1
+ %t1 = extractelement <100 x i8> %t2, i32 0
+ %2 = icmp eq i8 %t1, 80
+ br i1 %2, label %bb2, label %bb3
+
+bb2: ; preds = %entry
+ br label %bb3
+
+bb3: ; preds = %bb2, %entry
+ unreachable
+}
+
+; PR4340
+define void @vac(<4 x float>* nocapture %a) nounwind {
+; CHECK-LABEL: @vac(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: store <4 x float> zeroinitializer, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+ %t1 = load <4 x float>, <4 x float>* %a ; <<4 x float>> [#uses=1]
+ %vecins = insertelement <4 x float> %t1, float 0.000000e+00, i32 0 ; <<4 x float>> [#uses=1]
+ %vecins4 = insertelement <4 x float> %vecins, float 0.000000e+00, i32 1; <<4 x float>> [#uses=1]
+ %vecins6 = insertelement <4 x float> %vecins4, float 0.000000e+00, i32 2; <<4 x float>> [#uses=1]
+ %vecins8 = insertelement <4 x float> %vecins6, float 0.000000e+00, i32 3; <<4 x float>> [#uses=1]
+ store <4 x float> %vecins8, <4 x float>* %a
+ ret void
+}
+
+declare i32 @fgetc(i8*)
+
+define <4 x float> @dead_shuffle_elt(<4 x float> %x, <2 x float> %y) nounwind {
+; CHECK-LABEL: @dead_shuffle_elt(
+; CHECK-NEXT: [[SHUFFLE_I:%.*]] = shufflevector <2 x float> [[Y:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[SHUFFLE9_I:%.*]] = shufflevector <4 x float> [[SHUFFLE_I]], <4 x float> [[X:%.*]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT: ret <4 x float> [[SHUFFLE9_I]]
+;
+ %shuffle.i = shufflevector <2 x float> %y, <2 x float> %y, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %shuffle9.i = shufflevector <4 x float> %x, <4 x float> %shuffle.i, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+ ret <4 x float> %shuffle9.i
+}
+
+define <2 x float> @test_fptrunc(double %f) {
+; CHECK-LABEL: @test_fptrunc(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> <double undef, double 0.000000e+00>, double [[F:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = fptrunc <2 x double> [[TMP1]] to <2 x float>
+; CHECK-NEXT: ret <2 x float> [[TMP2]]
+;
+ %t9 = insertelement <4 x double> poison, double %f, i32 0
+ %t10 = insertelement <4 x double> %t9, double 0.000000e+00, i32 1
+ %t11 = insertelement <4 x double> %t10, double 0.000000e+00, i32 2
+ %t12 = insertelement <4 x double> %t11, double 0.000000e+00, i32 3
+ %t5 = fptrunc <4 x double> %t12 to <4 x float>
+ %ret = shufflevector <4 x float> %t5, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x float> %ret
+}
+
+define <2 x double> @test_fpext(float %f) {
+; CHECK-LABEL: @test_fpext(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> <float undef, float 0.000000e+00>, float [[F:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
+; CHECK-NEXT: ret <2 x double> [[TMP2]]
+;
+ %t9 = insertelement <4 x float> poison, float %f, i32 0
+ %t10 = insertelement <4 x float> %t9, float 0.000000e+00, i32 1
+ %t11 = insertelement <4 x float> %t10, float 0.000000e+00, i32 2
+ %t12 = insertelement <4 x float> %t11, float 0.000000e+00, i32 3
+ %t5 = fpext <4 x float> %t12 to <4 x double>
+ %ret = shufflevector <4 x double> %t5, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x double> %ret
+}
+
+define <4 x double> @test_shuffle(<4 x double> %f) {
+; CHECK-LABEL: @test_shuffle(
+; CHECK-NEXT: [[RET1:%.*]] = insertelement <4 x double> [[F:%.*]], double 1.000000e+00, i32 3
+; CHECK-NEXT: ret <4 x double> [[RET1]]
+;
+ %ret = shufflevector <4 x double> %f, <4 x double> <double undef, double 1.0, double undef, double undef>, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+ ret <4 x double> %ret
+}
+
+define <4 x float> @test_select(float %f, float %g) {
+; CHECK-LABEL: @test_select(
+; CHECK-NEXT: [[A3:%.*]] = insertelement <4 x float> <float undef, float undef, float undef, float 3.000000e+00>, float [[F:%.*]], i32 0
+; CHECK-NEXT: [[RET:%.*]] = shufflevector <4 x float> [[A3]], <4 x float> <float undef, float 4.000000e+00, float 5.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: ret <4 x float> [[RET]]
+;
+ %a0 = insertelement <4 x float> poison, float %f, i32 0
+ %a1 = insertelement <4 x float> %a0, float 1.000000e+00, i32 1
+ %a2 = insertelement <4 x float> %a1, float 2.000000e+00, i32 2
+ %a3 = insertelement <4 x float> %a2, float 3.000000e+00, i32 3
+ %b0 = insertelement <4 x float> poison, float %g, i32 0
+ %b1 = insertelement <4 x float> %b0, float 4.000000e+00, i32 1
+ %b2 = insertelement <4 x float> %b1, float 5.000000e+00, i32 2
+ %b3 = insertelement <4 x float> %b2, float 6.000000e+00, i32 3
+ %ret = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x float> %a3, <4 x float> %b3
+ ret <4 x float> %ret
+}
+
+; Check that instcombine doesn't wrongly fold away the select completely.
+
+define <2 x i64> @PR24922(<2 x i64> %v) {
+; CHECK-LABEL: @PR24922(
+; CHECK-NEXT: [[RESULT1:%.*]] = insertelement <2 x i64> [[V:%.*]], i64 0, i32 0
+; CHECK-NEXT: ret <2 x i64> [[RESULT1]]
+;
+ %result = select <2 x i1> <i1 icmp eq (i64 extractelement (<2 x i64> bitcast (<4 x i32> <i32 15, i32 15, i32 15, i32 15> to <2 x i64>), i64 0), i64 0), i1 true>, <2 x i64> %v, <2 x i64> zeroinitializer
+ ret <2 x i64> %result
+}
+
+; The shuffle only demands the 0th (undef) element of 'out123', so everything should fold away.
+
+define <4 x float> @inselt_shuf_no_demand(float %a1, float %a2, float %a3) {
+; CHECK-LABEL: @inselt_shuf_no_demand(
+; CHECK-NEXT: ret <4 x float> undef
+;
+ %out1 = insertelement <4 x float> poison, float %a1, i32 1
+ %out12 = insertelement <4 x float> %out1, float %a2, i32 2
+ %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+ %shuffle = shufflevector <4 x float> %out123, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ ret <4 x float> %shuffle
+}
+
+; The shuffle only demands the 0th (undef) element of 'out123', so everything should fold away.
+
+define <4 x float> @inselt_shuf_no_demand_commute(float %a1, float %a2, float %a3) {
+; CHECK-LABEL: @inselt_shuf_no_demand_commute(
+; CHECK-NEXT: ret <4 x float> undef
+;
+ %out1 = insertelement <4 x float> poison, float %a1, i32 1
+ %out12 = insertelement <4 x float> %out1, float %a2, i32 2
+ %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+ %shuffle = shufflevector <4 x float> undef, <4 x float> %out123, <4 x i32> <i32 4, i32 undef, i32 undef, i32 undef>
+ ret <4 x float> %shuffle
+}
+
+; The add uses 'out012' giving it multiple uses after the shuffle is transformed to also
+; use 'out012'. The analysis should be able to see past that.
+
+define <4 x i32> @inselt_shuf_no_demand_multiuse(i32 %a0, i32 %a1, <4 x i32> %b) {
+; CHECK-LABEL: @inselt_shuf_no_demand_multiuse(
+; CHECK-NEXT: [[OUT0:%.*]] = insertelement <4 x i32> poison, i32 [[A0:%.*]], i32 0
+; CHECK-NEXT: [[OUT01:%.*]] = insertelement <4 x i32> [[OUT0]], i32 [[A1:%.*]], i32 1
+; CHECK-NEXT: [[FOO:%.*]] = add <4 x i32> [[OUT01]], [[B:%.*]]
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[FOO]], <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i32> [[SHUFFLE]]
+;
+ %out0 = insertelement <4 x i32> poison, i32 %a0, i32 0
+ %out01 = insertelement <4 x i32> %out0, i32 %a1, i32 1
+ %out012 = insertelement <4 x i32> %out01, i32 %a0, i32 2
+ %foo = add <4 x i32> %out012, %b
+ %out0123 = insertelement <4 x i32> %foo, i32 %a1, i32 3
+ %shuffle = shufflevector <4 x i32> %out0123, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ ret <4 x i32> %shuffle
+}
+
+define <4 x float> @inselt_shuf_no_demand_bogus_insert_index_in_chain(float %a1, float %a2, float %a3, i32 %variable_index) {
+; CHECK-LABEL: @inselt_shuf_no_demand_bogus_insert_index_in_chain(
+; CHECK-NEXT: [[OUT12:%.*]] = insertelement <4 x float> poison, float [[A2:%.*]], i32 [[VARIABLE_INDEX:%.*]]
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[OUT12]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[SHUFFLE]]
+;
+ %out1 = insertelement <4 x float> poison, float %a1, i32 1
+ %out12 = insertelement <4 x float> %out1, float %a2, i32 %variable_index ; something unexpected
+ %out123 = insertelement <4 x float> %out12, float %a3, i32 3
+ %shuffle = shufflevector <4 x float> %out123, <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ ret <4 x float> %shuffle
+}
+
+; Test undef replacement in constant vector elements with binops.
+
+define <3 x i8> @shuf_add(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_add(
+; CHECK-NEXT: [[BO:%.*]] = add <3 x i8> [[X:%.*]], <i8 undef, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = add nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sub(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sub(
+; CHECK-NEXT: [[BO:%.*]] = sub <3 x i8> <i8 1, i8 undef, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = sub nuw <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 2>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_mul(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_mul(
+; CHECK-NEXT: [[BO:%.*]] = mul <3 x i8> [[X:%.*]], <i8 1, i8 undef, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 2, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = mul nsw <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 2, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_and(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_and(
+; CHECK-NEXT: [[BO:%.*]] = and <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = and <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_or(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_or(
+; CHECK-NEXT: [[BO:%.*]] = or <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = or <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_xor(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_xor(
+; CHECK-NEXT: [[BO:%.*]] = xor <3 x i8> [[X:%.*]], <i8 1, i8 undef, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = xor <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_lshr_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_lshr_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = lshr <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_lshr_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_lshr_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = lshr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = lshr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_ashr_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_ashr_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = lshr <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = ashr <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_ashr_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_ashr_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = ashr exact <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = ashr exact <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_shl_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_shl_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = shl nsw <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_shl_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_shl_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = shl nuw <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = shl nuw <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sdiv_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sdiv_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = sdiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 0, i32 undef, i32 1>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_sdiv_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_sdiv_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = sdiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = sdiv <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_srem_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_srem_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = srem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = srem <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_srem_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_srem_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = srem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 1>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = srem <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 1>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_udiv_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_udiv_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = udiv exact <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_udiv_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_udiv_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = udiv <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = udiv <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 undef, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_urem_const_op0(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_urem_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = urem <3 x i8> <i8 1, i8 2, i8 3>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = urem <3 x i8> <i8 1, i8 2, i8 3>, %x
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 2, i32 1, i32 undef>
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @shuf_urem_const_op1(<3 x i8> %x) {
+; CHECK-LABEL: @shuf_urem_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = urem <3 x i8> [[X:%.*]], <i8 1, i8 2, i8 3>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x i8> [[BO]], <3 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %bo = urem <3 x i8> %x, <i8 1, i8 2, i8 3>
+ %r = shufflevector <3 x i8> %bo, <3 x i8> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+ ret <3 x i8> %r
+}
+
+define <3 x float> @shuf_fadd(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fadd(
+; CHECK-NEXT: [[BO:%.*]] = fadd <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = fadd <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fsub(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fsub(
+; CHECK-NEXT: [[BO:%.*]] = fsub fast <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = fsub fast <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fmul(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fmul(
+; CHECK-NEXT: [[BO:%.*]] = fmul reassoc <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = fmul reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fdiv_const_op0(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fdiv_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = fdiv reassoc ninf <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = fdiv ninf reassoc <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 0, i32 2>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_fdiv_const_op1(<3 x float> %x) {
+; CHECK-LABEL: @shuf_fdiv_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = fdiv nnan ninf <3 x float> [[X:%.*]], <float 1.000000e+00, float 2.000000e+00, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = fdiv ninf nnan <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 1, i32 0>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_frem_const_op0(<3 x float> %x) {
+; CHECK-LABEL: @shuf_frem_const_op0(
+; CHECK-NEXT: [[BO:%.*]] = frem nnan <3 x float> <float 1.000000e+00, float undef, float 3.000000e+00>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 undef, i32 2, i32 0>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = frem nnan <3 x float> <float 1.0, float 2.0, float 3.0>, %x
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 undef, i32 2, i32 0>
+ ret <3 x float> %r
+}
+
+define <3 x float> @shuf_frem_const_op1(<3 x float> %x) {
+; CHECK-LABEL: @shuf_frem_const_op1(
+; CHECK-NEXT: [[BO:%.*]] = frem reassoc ninf <3 x float> [[X:%.*]], <float undef, float 2.000000e+00, float 3.000000e+00>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <3 x float> [[BO]], <3 x float> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+; CHECK-NEXT: ret <3 x float> [[R]]
+;
+ %bo = frem ninf reassoc <3 x float> %x, <float 1.0, float 2.0, float 3.0>
+ %r = shufflevector <3 x float> %bo, <3 x float> undef, <3 x i32> <i32 1, i32 undef, i32 2>
+ ret <3 x float> %r
+}
+
+;; TODO: getelementptr tests below show missing simplifications for
+;; vector demanded elements on vector geps.
+
+define i32* @gep_vbase_w_s_idx(<2 x i32*> %base) {
+; CHECK-LABEL: @gep_vbase_w_s_idx(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASE:%.*]], i64 1
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %gep = getelementptr i32, <2 x i32*> %base, i64 1
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_s_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_s_idx(
+; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], i64 1
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %basevec1 = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+ %gep = getelementptr i32, <2 x i32*> %basevec2, i64 1
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+
+define i32* @gep_splat_base_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_cv_idx(
+; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %basevec1 = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+ %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> <i64 0, i64 1>
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_vidx(i32* %base, <2 x i64> %idxvec) {
+; CHECK-LABEL: @gep_splat_base_w_vidx(
+; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]]
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %basevec1 = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+ %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+
+ at GLOBAL = internal global i32 zeroinitializer
+
+define i32* @gep_cvbase_w_s_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_s_idx(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> <i32* undef, i32* @GLOBAL>, i64 [[RAW_ADDR:%.*]]
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, i64 %raw_addr
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+define i32* @gep_cvbase_w_cv_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_cv_idx(
+; CHECK-NEXT: ret i32* getelementptr inbounds (i32, i32* @GLOBAL, i64 1)
+;
+ %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+
+define i32* @gep_sbase_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_sbase_w_cv_idx(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %gep = getelementptr i32, i32* %base, <2 x i64> <i64 0, i64 1>
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+define i32* @gep_sbase_w_splat_idx(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_sbase_w_splat_idx(
+; CHECK-NEXT: [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %idxvec1 = insertelement <2 x i64> poison, i64 %idx, i32 0
+ %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+ %gep = getelementptr i32, i32* %base, <2 x i64> %idxvec2
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+define i32* @gep_splat_both(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_splat_both(
+; CHECK-NEXT: [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT: [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT: [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT: ret i32* [[EE]]
+;
+ %basevec1 = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+ %idxvec1 = insertelement <2 x i64> poison, i64 %idx, i32 0
+ %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+ %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec2
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+define <2 x i32*> @gep_all_lanes_undef(i32* %base, i64 %idx) {;
+; CHECK-LABEL: @gep_all_lanes_undef(
+; CHECK-NEXT: ret <2 x i32*> undef
+;
+ %basevec = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %idxvec = insertelement <2 x i64> poison, i64 %idx, i32 1
+ %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+ ret <2 x i32*> %gep
+}
+
+define i32* @gep_demanded_lane_undef(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_demanded_lane_undef(
+; CHECK-NEXT: ret i32* undef
+;
+ %basevec = insertelement <2 x i32*> poison, i32* %base, i32 0
+ %idxvec = insertelement <2 x i64> poison, i64 %idx, i32 1
+ %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+ %ee = extractelement <2 x i32*> %gep, i32 1
+ ret i32* %ee
+}
+
+
+;; LangRef has an odd quirk around FCAs which make it illegal to use undef
+;; indices.
+define i32* @PR41624(<2 x { i32, i32 }*> %a) {
+; CHECK-LABEL: @PR41624(
+; CHECK-NEXT: [[W:%.*]] = getelementptr { i32, i32 }, <2 x { i32, i32 }*> [[A:%.*]], <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <2 x i32*> [[W]], i32 0
+; CHECK-NEXT: ret i32* [[R]]
+;
+ %w = getelementptr { i32, i32 }, <2 x { i32, i32 }*> %a, <2 x i64> <i64 5, i64 5>, <2 x i32> zeroinitializer
+ %r = extractelement <2 x i32*> %w, i32 0
+ ret i32* %r
+}
+
+ at global = external global [0 x i32], align 4
+
+; Make sure we don't get stuck in a loop turning the zeroinitializer into
+; <0, undef, undef, undef> and then changing it back.
+define i32* @zero_sized_type_extract(<4 x i64> %arg, i64 %arg1) {
+; CHECK-LABEL: @zero_sized_type_extract(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[T:%.*]] = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* undef, [0 x i32]* undef, [0 x i32]* undef>, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i64> [[ARG:%.*]]
+; CHECK-NEXT: [[T2:%.*]] = extractelement <4 x i32*> [[T]], i64 0
+; CHECK-NEXT: ret i32* [[T2]]
+;
+bb:
+ %t = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global>, <4 x i64> zeroinitializer, <4 x i64> %arg
+ %t2 = extractelement <4 x i32*> %t, i64 0
+ ret i32* %t2
+}
+
+; The non-zero elements of the result are always 'y', so the splat is unnecessary.
+
+define <4 x i8> @select_cond_with_eq_true_false_elts(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @select_cond_with_eq_true_false_elts(
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[SEL]], <4 x i8> [[Y]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %tval = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+ %splat = shufflevector <4 x i1> %cmp, <4 x i1> undef, <4 x i32> zeroinitializer
+ %r = select <4 x i1> %splat, <4 x i8> %tval, <4 x i8> %y
+ ret <4 x i8> %r
+}
+
+; First element of the result is always x[0], so first element of select condition is unnecessary.
+
+define <4 x i8> @select_cond_with_eq_true_false_elts2(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @select_cond_with_eq_true_false_elts2(
+; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[Y:%.*]], <4 x i8> [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[SEL]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %tval = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+ %cond = shufflevector <4 x i1> %cmp, <4 x i1> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %r = select <4 x i1> %cond, <4 x i8> %tval, <4 x i8> %x
+ ret <4 x i8> %r
+}
+
+; Second element of the result is always x[3], so second element of select condition is unnecessary.
+; Fourth element of the result is always undef, so fourth element of select condition is unnecessary.
+
+define <4 x float> @select_cond_with_eq_true_false_elts3(<4 x float> %x, <4 x float> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @select_cond_with_eq_true_false_elts3(
+; CHECK-NEXT: [[TVAL:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 3, i32 5, i32 undef>
+; CHECK-NEXT: [[FVAL:%.*]] = shufflevector <4 x float> [[Y]], <4 x float> [[X]], <4 x i32> <i32 0, i32 7, i32 6, i32 undef>
+; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[TVAL]], <4 x float> [[FVAL]]
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %tval = shufflevector <4 x float> %x, <4 x float> %y, <4 x i32> <i32 1, i32 3, i32 5, i32 undef>
+ %fval = shufflevector <4 x float> %y, <4 x float> %x, <4 x i32> <i32 0, i32 7, i32 6, i32 undef>
+ %cond = shufflevector <4 x i1> %cmp, <4 x i1> undef, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+ %r = select <4 x i1> %cond, <4 x float> %tval, <4 x float> %fval
+ ret <4 x float> %r
+}
+
+define <4 x i8> @select_cond_with_undef_true_false_elts(<4 x i8> %x, <4 x i8> %y, <4 x i1> %cmp) {
+; CHECK-LABEL: @select_cond_with_undef_true_false_elts(
+; CHECK-NEXT: [[TVAL:%.*]] = shufflevector <4 x i8> [[Y:%.*]], <4 x i8> undef, <4 x i32> <i32 undef, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[COND:%.*]] = shufflevector <4 x i1> [[CMP:%.*]], <4 x i1> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x i8> [[TVAL]], <4 x i8> [[X:%.*]]
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %tval = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 undef, i32 5, i32 6, i32 7>
+ %cond = shufflevector <4 x i1> %cmp, <4 x i1> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+ %r = select <4 x i1> %cond, <4 x i8> %tval, <4 x i8> %x
+ ret <4 x i8> %r
+}
+
+; The insert can be safely eliminated because the shuffle blocks poison from cmp[0].
+
+define <4 x i8> @select_cond_(<4 x i8> %x, <4 x i8> %min, <4 x i1> %cmp, i1 %poison_blocker) {
+; CHECK-LABEL: @select_cond_(
+; CHECK-NEXT: [[SEL:%.*]] = select <4 x i1> [[CMP:%.*]], <4 x i8> [[MIN:%.*]], <4 x i8> [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i8> [[X]], <4 x i8> [[SEL]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %ins = insertelement <4 x i1> %cmp, i1 %poison_blocker, i32 0
+ %vecins = shufflevector <4 x i8> %x, <4 x i8> %min, <4 x i32> <i32 0, i32 5, i32 6, i32 7>
+ %r = select <4 x i1> %ins, <4 x i8> %vecins, <4 x i8> %x
+ ret <4 x i8> %r
+}
+
+define <4 x float> @ins_of_ext(<4 x float> %x, float %y) {
+; CHECK-LABEL: @ins_of_ext(
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[X:%.*]], float [[Y:%.*]], i32 1
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i32 2
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[Y]], i32 3
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %e0 = extractelement <4 x float> %x, i32 0
+ %i0 = insertelement <4 x float> poison, float %e0, i32 0
+ %i1 = insertelement <4 x float> %i0, float %y, i32 1
+ %i2 = insertelement <4 x float> %i1, float %y, i32 2
+ %i3 = insertelement <4 x float> %i2, float %y, i32 3
+ ret <4 x float> %i3
+}
+
+define <4 x float> @ins_of_ext_twice(<4 x float> %x, float %y) {
+; CHECK-LABEL: @ins_of_ext_twice(
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[X:%.*]], float [[Y:%.*]], i32 2
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[Y]], i32 3
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %e0 = extractelement <4 x float> %x, i32 0
+ %i0 = insertelement <4 x float> poison, float %e0, i32 0
+ %e1 = extractelement <4 x float> %x, i32 1
+ %i1 = insertelement <4 x float> %i0, float %e1, i32 1
+ %i2 = insertelement <4 x float> %i1, float %y, i32 2
+ %i3 = insertelement <4 x float> %i2, float %y, i32 3
+ ret <4 x float> %i3
+}
+
+; Negative test - element 3 of the result must be undef to be poison safe.
+; TODO: Could convert insert/extract to identity shuffle with undef mask elements.
+
+define <4 x float> @ins_of_ext_wrong_demand(<4 x float> %x, float %y) {
+; CHECK-LABEL: @ins_of_ext_wrong_demand(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[E0]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[Y:%.*]], i32 1
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i32 2
+; CHECK-NEXT: ret <4 x float> [[I2]]
+;
+ %e0 = extractelement <4 x float> %x, i32 0
+ %i0 = insertelement <4 x float> poison, float %e0, i32 0
+ %i1 = insertelement <4 x float> %i0, float %y, i32 1
+ %i2 = insertelement <4 x float> %i1, float %y, i32 2
+ ret <4 x float> %i2
+}
+
+; Negative test - can't replace i0 with x.
+; TODO: Could convert insert/extract to identity shuffle with undef mask elements.
+
+define <4 x float> @ins_of_ext_wrong_type(<5 x float> %x, float %y) {
+; CHECK-LABEL: @ins_of_ext_wrong_type(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <5 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[E0]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[Y:%.*]], i32 1
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[Y]], i32 2
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[Y]], i32 3
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %e0 = extractelement <5 x float> %x, i32 0
+ %i0 = insertelement <4 x float> poison, float %e0, i32 0
+ %i1 = insertelement <4 x float> %i0, float %y, i32 1
+ %i2 = insertelement <4 x float> %i1, float %y, i32 2
+ %i3 = insertelement <4 x float> %i2, float %y, i32 3
+ ret <4 x float> %i3
+}
+
+; This should reduce, but the shuffle mask must remain as-is (no extra undef).
+
+define <4 x i4> @ins_of_ext_undef_elts_propagation(<4 x i4> %v, <4 x i4> %v2, i4 %x) {
+; CHECK-LABEL: @ins_of_ext_undef_elts_propagation(
+; CHECK-NEXT: [[T2:%.*]] = insertelement <4 x i4> [[V:%.*]], i4 [[X:%.*]], i32 2
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i4> [[T2]], <4 x i4> [[V2:%.*]], <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+; CHECK-NEXT: ret <4 x i4> [[R]]
+;
+ %v0 = extractelement <4 x i4> %v, i32 0
+ %t0 = insertelement <4 x i4> poison, i4 %v0, i32 0
+ %t2 = insertelement <4 x i4> %t0, i4 %x, i32 2
+ %r = shufflevector <4 x i4> %t2, <4 x i4> %v2, <4 x i32> <i32 0, i32 6, i32 2, i32 7>
+ ret <4 x i4> %r
+}
+
+; Similar to above, but more ops/uses to verify things work in more complicated cases.
+
+define <8 x i4> @ins_of_ext_undef_elts_propagation2(<8 x i4> %v, <8 x i4> %v2, i4 %x) {
+; CHECK-LABEL: @ins_of_ext_undef_elts_propagation2(
+; CHECK-NEXT: [[I19:%.*]] = insertelement <8 x i4> [[V:%.*]], i4 [[X:%.*]], i32 2
+; CHECK-NEXT: [[I20:%.*]] = shufflevector <8 x i4> [[I19]], <8 x i4> [[V2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 undef>
+; CHECK-NEXT: [[I21:%.*]] = shufflevector <8 x i4> [[I20]], <8 x i4> [[V]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; CHECK-NEXT: ret <8 x i4> [[I21]]
+;
+ %i15 = extractelement <8 x i4> %v, i32 0
+ %i16 = insertelement <8 x i4> poison, i4 %i15, i32 0
+ %i17 = extractelement <8 x i4> %v, i32 1
+ %i18 = insertelement <8 x i4> %i16, i4 %i17, i32 1
+ %i19 = insertelement <8 x i4> %i18, i4 %x, i32 2
+ %i20 = shufflevector <8 x i4> %i19, <8 x i4> %v2, <8 x i32> <i32 0, i32 1, i32 2, i32 11, i32 10, i32 9, i32 8, i32 undef>
+ %i21 = shufflevector <8 x i4> %i20, <8 x i4> %v, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+ ret <8 x i4> %i21
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_extract_var_elt-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_extract_var_elt-inseltpoison.ll
new file mode 100644
index 000000000000..48b2513a00d7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_extract_var_elt-inseltpoison.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @test (float %b, <8 x float> * %p) {
+; CHECK: extractelement
+; CHECK: fptosi
+ %1 = load <8 x float> , <8 x float> * %p
+ %2 = bitcast <8 x float> %1 to <8 x i32>
+ %3 = bitcast <8 x i32> %2 to <8 x float>
+ %a = fptosi <8 x float> %3 to <8 x i32>
+ %4 = fptosi float %b to i32
+ %5 = add i32 %4, -2
+ %6 = extractelement <8 x i32> %a, i32 %5
+ %7 = insertelement <8 x i32> poison, i32 %6, i32 7
+ %8 = sitofp <8 x i32> %7 to <8 x float>
+ store <8 x float> %8, <8 x float>* %p
+ ret void
+}
+
+; PR18600
+define i32 @test2(i32 %i) {
+ %e = extractelement <4 x i32> bitcast (<2 x i64> <i64 1, i64 2> to <4 x i32>), i32 %i
+ ret i32 %e
+
+; CHECK-LABEL: @test2
+; CHECK: extractelement
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
new file mode 100644
index 000000000000..e39b3e46801f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_gep_scalar_arg-inseltpoison.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+define <4 x i16*> @PR41270([4 x i16]* %x) {
+; CHECK-LABEL: @PR41270(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x [4 x i16]*> undef, [4 x i16]* [[X:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [4 x i16], <4 x [4 x i16]*> [[TMP1]], i64 0, i64 3
+; CHECK-NEXT: ret <4 x i16*> [[TMP2]]
+;
+ %ins = insertelement <4 x [4 x i16]*> poison, [4 x i16]* %x, i32 0
+ %splat = shufflevector <4 x [4 x i16]*> %ins, <4 x [4 x i16]*> undef, <4 x i32> zeroinitializer
+ %t2 = getelementptr inbounds [4 x i16], <4 x [4 x i16]*> %splat, i32 0, i32 3
+ %t3 = extractelement <4 x i16*> %t2, i32 3
+ %ins2 = insertelement <4 x i16*> poison, i16* %t3, i32 0
+ ret <4 x i16*> %ins2
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll
new file mode 100644
index 000000000000..ba4f88d87d3b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_phi_extract-inseltpoison.ll
@@ -0,0 +1,107 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define void @f(i64 %val, i32 %limit, i32 *%ptr) {
+; CHECK-LABEL: @f
+; CHECK: %0 = trunc i64 %val to i32
+; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ]
+entry:
+ %tempvector = insertelement <16 x i64> poison, i64 %val, i32 0
+ %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+ %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+ %1 = trunc <16 x i64> %0 to <16 x i32>
+ br label %loop
+
+loop:
+ %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+ %elt = extractelement <16 x i32> %2, i32 0
+ %end = icmp ult i32 %elt, %limit
+ %3 = add i32 10, %elt
+ %4 = sext i32 %elt to i64
+ %5 = getelementptr i32, i32* %ptr, i64 %4
+ store i32 %3, i32* %5
+ %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+ br i1 %end, label %loop, label %ret
+
+ret:
+ ret void
+}
+
+define void @copy(i64 %val, i32 %limit, i32 *%ptr) {
+; CHECK-LABEL: @copy
+; CHECK: %0 = trunc i64 %val to i32
+; CHECK: %1 = phi i32 [ %0, %entry ], [ {{.*}}, %loop ]
+entry:
+ %tempvector = insertelement <16 x i64> poison, i64 %val, i32 0
+ %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+ %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+ %1 = trunc <16 x i64> %0 to <16 x i32>
+ br label %loop
+
+loop:
+ %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+ %elt = extractelement <16 x i32> %2, i32 0
+ %eltcopy = extractelement <16 x i32> %2, i32 0
+ %end = icmp ult i32 %elt, %limit
+ %3 = add i32 10, %eltcopy
+ %4 = sext i32 %elt to i64
+ %5 = getelementptr i32, i32* %ptr, i64 %4
+ store i32 %3, i32* %5
+ %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+ br i1 %end, label %loop, label %ret
+
+ret:
+ ret void
+}
+
+define void @nocopy(i64 %val, i32 %limit, i32 *%ptr) {
+; CHECK-LABEL: @nocopy
+; CHECK-NOT: phi i32
+; CHECK: phi <16 x i32> [ %3, %entry ], [ %inc, %loop ]
+entry:
+ %tempvector = insertelement <16 x i64> poison, i64 %val, i32 0
+ %vector = shufflevector <16 x i64> %tempvector, <16 x i64> undef, <16 x i32> zeroinitializer
+ %0 = add <16 x i64> %vector, <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11, i64 12, i64 13, i64 14, i64 15>
+ %1 = trunc <16 x i64> %0 to <16 x i32>
+ br label %loop
+
+loop:
+ %2 = phi <16 x i32> [ %1, %entry ], [ %inc, %loop ]
+ %elt = extractelement <16 x i32> %2, i32 0
+ %eltcopy = extractelement <16 x i32> %2, i32 1
+ %end = icmp ult i32 %elt, %limit
+ %3 = add i32 10, %eltcopy
+ %4 = sext i32 %elt to i64
+ %5 = getelementptr i32, i32* %ptr, i64 %4
+ store i32 %3, i32* %5
+ %inc = add <16 x i32> %2, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+ br i1 %end, label %loop, label %ret
+
+ret:
+ ret void
+}
+
+define i1 @g(<3 x i32> %input_2) {
+; CHECK-LABEL: @g
+; CHECK: extractelement <3 x i32> %input_2, i32 0
+entry:
+ br label %for.cond
+
+for.cond:
+ %input_2.addr.0 = phi <3 x i32> [ %input_2, %entry ], [ %div45, %for.body ]
+ %input_1.addr.1 = phi <3 x i32> [ undef, %entry ], [ %dec43, %for.body ]
+ br i1 undef, label %for.end, label %for.body
+
+; CHECK-NOT: extractelement <3 x i32> %{{.*}}, i32 0
+for.body:
+ %dec43 = add <3 x i32> %input_1.addr.1, <i32 -1, i32 -1, i32 -1>
+ %sub44 = sub <3 x i32> <i32 -1, i32 -1, i32 -1>, %dec43
+ %div45 = sdiv <3 x i32> %input_2.addr.0, %sub44
+ br label %for.cond
+
+for.end:
+ %0 = extractelement <3 x i32> %input_2.addr.0, i32 0
+ %.89 = select i1 false, i32 0, i32 %0
+ %tobool313 = icmp eq i32 %.89, 0
+ ret i1 %tobool313
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
new file mode 100644
index 000000000000..efdb40a80aea
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vec_shuffle-inseltpoison.ll
@@ -0,0 +1,1790 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define <4 x float> @test1(<4 x float> %v1) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret <4 x float> [[V1:%.*]]
+;
+ %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ ret <4 x float> %v2
+}
+
+define <4 x float> @test2(<4 x float> %v1) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret <4 x float> [[V1:%.*]]
+;
+ %v2 = shufflevector <4 x float> %v1, <4 x float> %v1, <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+ ret <4 x float> %v2
+}
+
+define float @test3(<4 x float> %A, <4 x float> %B, float %f) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: ret float [[F:%.*]]
+;
+ %C = insertelement <4 x float> %A, float %f, i32 0
+ %D = shufflevector <4 x float> %C, <4 x float> %B, <4 x i32> <i32 5, i32 0, i32 2, i32 7>
+ %E = extractelement <4 x float> %D, i32 1
+ ret float %E
+}
+
+define i32 @test4(<4 x i32> %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 0
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %t = shufflevector <4 x i32> %X, <4 x i32> undef, <4 x i32> zeroinitializer
+ %r = extractelement <4 x i32> %t, i32 0
+ ret i32 %r
+}
+
+define i32 @test5(<4 x i32> %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 3
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %t = shufflevector <4 x i32> %X, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 undef, i32 undef>
+ %r = extractelement <4 x i32> %t, i32 0
+ ret i32 %r
+}
+
+define float @test6(<4 x float> %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %X1 = bitcast <4 x float> %X to <4 x i32>
+ %t = shufflevector <4 x i32> %X1, <4 x i32> undef, <4 x i32> zeroinitializer
+ %t2 = bitcast <4 x i32> %t to <4 x float>
+ %r = extractelement <4 x float> %t2, i32 0
+ ret float %r
+}
+
+define float @testvscale6(<vscale x 4 x float> %X) {
+; CHECK-LABEL: @testvscale6(
+; CHECK-NEXT: [[T2:%.*]] = shufflevector <vscale x 4 x float> [[X:%.*]], <vscale x 4 x float> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x float> [[T2]], i32 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %X1 = bitcast <vscale x 4 x float> %X to <vscale x 4 x i32>
+ %t = shufflevector <vscale x 4 x i32> %X1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %t2 = bitcast <vscale x 4 x i32> %t to <vscale x 4 x float>
+ %r = extractelement <vscale x 4 x float> %t2, i32 0
+ ret float %r
+}
+
+
+define <4 x float> @test7(<4 x float> %x) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> < i32 0, i32 1, i32 6, i32 7 >
+ ret <4 x float> %r
+}
+
+; This should turn into a single shuffle.
+define <4 x float> @test8(<4 x float> %x, <4 x float> %y) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT: [[T134:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]], <4 x i32> <i32 1, i32 undef, i32 3, i32 4>
+; CHECK-NEXT: ret <4 x float> [[T134]]
+;
+ %t4 = extractelement <4 x float> %x, i32 1
+ %t2 = extractelement <4 x float> %x, i32 3
+ %t1 = extractelement <4 x float> %y, i32 0
+ %t128 = insertelement <4 x float> poison, float %t4, i32 0
+ %t130 = insertelement <4 x float> %t128, float undef, i32 1
+ %t132 = insertelement <4 x float> %t130, float %t2, i32 2
+ %t134 = insertelement <4 x float> %t132, float %t1, i32 3
+ ret <4 x float> %t134
+}
+
+; Test fold of two shuffles where the first shuffle vectors inputs are a
+;
diff erent length then the second.
+define <4 x i8> @test9(<16 x i8> %t6) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT: [[T9:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 13, i32 9, i32 4, i32 13>
+; CHECK-NEXT: ret <4 x i8> [[T9]]
+;
+ %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 13, i32 9, i32 4, i32 13 >
+ %t9 = shufflevector <4 x i8> %t7, <4 x i8> undef, <4 x i32> < i32 3, i32 1, i32 2, i32 0 >
+ ret <4 x i8> %t9
+}
+
+; Same as test9, but make sure that "undef" mask values are not confused with
+; mask values of 2*N, where N is the mask length. These shuffles should not
+; be folded (because [8,9,4,8] may not be a mask supported by the target).
+
+define <4 x i8> @test9a(<16 x i8> %t6) {
+; CHECK-LABEL: @test9a(
+; CHECK-NEXT: [[T7:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <4 x i32> <i32 undef, i32 9, i32 4, i32 8>
+; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T7]], <4 x i8> undef, <4 x i32> <i32 3, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x i8> [[T9]]
+;
+ %t7 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> < i32 undef, i32 9, i32 4, i32 8 >
+ %t9 = shufflevector <4 x i8> %t7, <4 x i8> undef, <4 x i32> < i32 3, i32 1, i32 2, i32 0 >
+ ret <4 x i8> %t9
+}
+
+; Test fold of two shuffles where the first shuffle vectors inputs are a
+;
diff erent length then the second.
+define <4 x i8> @test9b(<4 x i8> %t6, <4 x i8> %t7) {
+; CHECK-LABEL: @test9b(
+; CHECK-NEXT: [[T9:%.*]] = shufflevector <4 x i8> [[T6:%.*]], <4 x i8> [[T7:%.*]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: ret <4 x i8> [[T9]]
+;
+ %t1 = shufflevector <4 x i8> %t6, <4 x i8> %t7, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 4, i32 5, i32 2, i32 3>
+ %t9 = shufflevector <8 x i8> %t1, <8 x i8> undef, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+ ret <4 x i8> %t9
+}
+
+; Redundant vector splats should be removed. Radar 8597790.
+define <4 x i32> @test10(<4 x i32> %t5) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT: [[T7:%.*]] = shufflevector <4 x i32> [[T5:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[T7]]
+;
+ %t6 = shufflevector <4 x i32> %t5, <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %t7 = shufflevector <4 x i32> %t6, <4 x i32> undef, <4 x i32> zeroinitializer
+ ret <4 x i32> %t7
+}
+
+; Test fold of two shuffles where the two shufflevector inputs's op1 are the same.
+
+define <8 x i8> @test11(<16 x i8> %t6) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT: [[T3:%.*]] = shufflevector <16 x i8> [[T6:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: ret <8 x i8> [[T3]]
+;
+ %t1 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+ %t2 = shufflevector <16 x i8> %t6, <16 x i8> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+ %t3 = shufflevector <4 x i8> %t1, <4 x i8> %t2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ ret <8 x i8> %t3
+}
+
+; Test fold of two shuffles where the first shufflevector's inputs are the same as the second.
+
+define <8 x i8> @test12(<8 x i8> %t6, <8 x i8> %t2) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT: [[T3:%.*]] = shufflevector <8 x i8> [[T6:%.*]], <8 x i8> [[T2:%.*]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 9, i32 8, i32 11, i32 12>
+; CHECK-NEXT: ret <8 x i8> [[T3]]
+;
+ %t1 = shufflevector <8 x i8> %t6, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 undef, i32 7>
+ %t3 = shufflevector <8 x i8> %t1, <8 x i8> %t2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 9, i32 8, i32 11, i32 12>
+ ret <8 x i8> %t3
+}
+
+; Test fold of two shuffles where the first shufflevector's inputs are the same as the second.
+
+define <8 x i8> @test12a(<8 x i8> %t6, <8 x i8> %t2) {
+; CHECK-LABEL: @test12a(
+; CHECK-NEXT: [[T3:%.*]] = shufflevector <8 x i8> [[T2:%.*]], <8 x i8> [[T6:%.*]], <8 x i32> <i32 0, i32 3, i32 1, i32 4, i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: ret <8 x i8> [[T3]]
+;
+ %t1 = shufflevector <8 x i8> %t6, <8 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 undef, i32 7>
+ %t3 = shufflevector <8 x i8> %t2, <8 x i8> %t1, <8 x i32> <i32 0, i32 3, i32 1, i32 4, i32 8, i32 9, i32 10, i32 11>
+ ret <8 x i8> %t3
+}
+
+; The mask length of the 1st shuffle can be reduced to eliminate the 2nd shuffle.
+
+define <2 x i8> @extract_subvector_of_shuffle(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle(
+; CHECK-NEXT: [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: ret <2 x i8> [[EXTRACT_SUBV]]
+;
+ %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <3 x i32> <i32 0, i32 2, i32 0>
+ %extract_subv = shufflevector <3 x i8> %shuf, <3 x i8> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x i8> %extract_subv
+}
+
+; Undef elements in either mask are ok. Undefs from the 2nd shuffle mask should propagate to the new shuffle.
+; The type of the inputs does not have to match the output type.
+
+define <4 x i8> @extract_subvector_of_shuffle_undefs_types(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle_undefs_types(
+; CHECK-NEXT: [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT: ret <4 x i8> [[EXTRACT_SUBV]]
+;
+ %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+ %extract_subv = shufflevector <5 x i8> %shuf, <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+ ret <4 x i8> %extract_subv
+}
+
+; Extra uses are not ok - we only do the transform when we can eliminate an instruction.
+
+declare void @use_v5i8(<5 x i8>)
+
+define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use(
+; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+; CHECK-NEXT: call void @use_v5i8(<5 x i8> [[SHUF]])
+; CHECK-NEXT: [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x i8> [[EXTRACT_SUBV]]
+;
+ %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+ call void @use_v5i8(<5 x i8> %shuf)
+ %extract_subv = shufflevector <5 x i8> %shuf, <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+ ret <4 x i8> %extract_subv
+}
+
+define <2 x i8> @test13a(i8 %x1, i8 %x2) {
+; CHECK-LABEL: @test13a(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i8> [[TMP2]], <i8 7, i8 5>
+; CHECK-NEXT: ret <2 x i8> [[TMP3]]
+;
+ %A = insertelement <2 x i8> poison, i8 %x1, i32 0
+ %B = insertelement <2 x i8> %A, i8 %x2, i32 1
+ %C = add <2 x i8> %B, <i8 5, i8 7>
+ %D = shufflevector <2 x i8> %C, <2 x i8> undef, <2 x i32> <i32 1, i32 0>
+ ret <2 x i8> %D
+}
+
+; Increasing length of vector ops is not a good canonicalization.
+
+define <3 x i32> @add_wider(i32 %y, i32 %z) {
+; CHECK-LABEL: @add_wider(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT: [[A:%.*]] = add <2 x i32> [[I1]], <i32 255, i32 255>
+; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x i32> [[EXT]]
+;
+ %i0 = insertelement <2 x i32> poison, i32 %y, i32 0
+ %i1 = insertelement <2 x i32> %i0, i32 %z, i32 1
+ %a = add <2 x i32> %i1, <i32 255, i32 255>
+ %ext = shufflevector <2 x i32> %a, <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+ ret <3 x i32> %ext
+}
+
+; Increasing length of vector ops must be safe from illegal undef propagation.
+
+define <3 x i32> @div_wider(i32 %y, i32 %z) {
+; CHECK-LABEL: @div_wider(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x i32> poison, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <2 x i32> [[I0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT: [[A:%.*]] = sdiv <2 x i32> [[I1]], <i32 255, i32 255>
+; CHECK-NEXT: [[EXT:%.*]] = shufflevector <2 x i32> [[A]], <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+; CHECK-NEXT: ret <3 x i32> [[EXT]]
+;
+ %i0 = insertelement <2 x i32> poison, i32 %y, i32 0
+ %i1 = insertelement <2 x i32> %i0, i32 %z, i32 1
+ %a = sdiv <2 x i32> %i1, <i32 255, i32 255>
+ %ext = shufflevector <2 x i32> %a, <2 x i32> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+ ret <3 x i32> %ext
+}
+
+; Increasing length of insertelements (no math ops) is a good canonicalization.
+
+define <3 x i8> @fold_inselts_with_widening_shuffle(i8 %x, i8 %y) {
+; CHECK-LABEL: @fold_inselts_with_widening_shuffle(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <3 x i8> undef, i8 [[X:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <3 x i8> [[TMP1]], i8 [[Y:%.*]], i32 1
+; CHECK-NEXT: ret <3 x i8> [[TMP2]]
+;
+ %ins0 = insertelement <2 x i8> poison, i8 %x, i32 0
+ %ins1 = insertelement <2 x i8> %ins0, i8 %y, i32 1
+ %widen = shufflevector <2 x i8> %ins1, <2 x i8> undef, <3 x i32> <i32 0, i32 1, i32 undef>
+ ret <3 x i8> %widen
+}
+
+define <2 x i8> @test13b(i8 %x) {
+; CHECK-LABEL: @test13b(
+; CHECK-NEXT: [[B:%.*]] = insertelement <2 x i8> undef, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[B]]
+;
+ %A = insertelement <2 x i8> poison, i8 %x, i32 0
+ %B = shufflevector <2 x i8> %A, <2 x i8> undef, <2 x i32> <i32 undef, i32 0>
+ ret <2 x i8> %B
+}
+
+define <2 x i8> @test13c(i8 %x1, i8 %x2) {
+; CHECK-LABEL: @test13c(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i8> undef, i8 [[X1:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> [[TMP1]], i8 [[X2:%.*]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[TMP2]]
+;
+ %A = insertelement <4 x i8> poison, i8 %x1, i32 0
+ %B = insertelement <4 x i8> %A, i8 %x2, i32 2
+ %C = shufflevector <4 x i8> %B, <4 x i8> undef, <2 x i32> <i32 0, i32 2>
+ ret <2 x i8> %C
+}
+
+define void @test14(i16 %conv10) {
+; CHECK-LABEL: @test14(
+; CHECK-NEXT: store <4 x i16> <i16 poison, i16 poison, i16 poison, i16 23>, <4 x i16>* undef, align 8
+; CHECK-NEXT: ret void
+;
+ %t = alloca <4 x i16>, align 8
+ %vecinit6 = insertelement <4 x i16> poison, i16 23, i32 3
+ store <4 x i16> %vecinit6, <4 x i16>* undef
+ %t1 = load <4 x i16>, <4 x i16>* undef
+ %vecinit11 = insertelement <4 x i16> poison, i16 %conv10, i32 3
+ %div = udiv <4 x i16> %t1, %vecinit11
+ store <4 x i16> %div, <4 x i16>* %t
+ %t4 = load <4 x i16>, <4 x i16>* %t
+ %t5 = shufflevector <4 x i16> %t4, <4 x i16> undef, <2 x i32> <i32 2, i32 0>
+ %cmp = icmp ule <2 x i16> %t5, undef
+ %sext = sext <2 x i1> %cmp to <2 x i16>
+ ret void
+}
+
+; Check that sequences of insert/extract element are
+; collapsed into valid shuffle instruction with correct shuffle indexes.
+
+define <4 x float> @test15a(<4 x float> %LHS, <4 x float> %RHS) {
+; CHECK-LABEL: @test15a(
+; CHECK-NEXT: [[T4:%.*]] = shufflevector <4 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]], <4 x i32> <i32 4, i32 0, i32 6, i32 6>
+; CHECK-NEXT: ret <4 x float> [[T4]]
+;
+ %t1 = extractelement <4 x float> %LHS, i32 0
+ %t2 = insertelement <4 x float> %RHS, float %t1, i32 1
+ %t3 = extractelement <4 x float> %RHS, i32 2
+ %t4 = insertelement <4 x float> %t2, float %t3, i32 3
+ ret <4 x float> %t4
+}
+
+define <4 x float> @test15b(<4 x float> %LHS, <4 x float> %RHS) {
+; CHECK-LABEL: @test15b(
+; CHECK-NEXT: [[T5:%.*]] = shufflevector <4 x float> [[LHS:%.*]], <4 x float> [[RHS:%.*]], <4 x i32> <i32 4, i32 3, i32 6, i32 6>
+; CHECK-NEXT: ret <4 x float> [[T5]]
+;
+ %t0 = extractelement <4 x float> %LHS, i32 3
+ %t1 = insertelement <4 x float> %RHS, float %t0, i32 0
+ %t2 = extractelement <4 x float> %t1, i32 0
+ %t3 = insertelement <4 x float> %RHS, float %t2, i32 1
+ %t4 = extractelement <4 x float> %RHS, i32 2
+ %t5 = insertelement <4 x float> %t3, float %t4, i32 3
+ ret <4 x float> %t5
+}
+
+define <1 x i32> @test16a(i32 %ele) {
+; CHECK-LABEL: @test16a(
+; CHECK-NEXT: ret <1 x i32> <i32 2>
+;
+ %t0 = insertelement <2 x i32> <i32 1, i32 undef>, i32 %ele, i32 1
+ %t1 = shl <2 x i32> %t0, <i32 1, i32 1>
+ %t2 = shufflevector <2 x i32> %t1, <2 x i32> undef, <1 x i32> <i32 0>
+ ret <1 x i32> %t2
+}
+
+define <4 x i8> @test16b(i8 %ele) {
+; CHECK-LABEL: @test16b(
+; CHECK-NEXT: ret <4 x i8> <i8 2, i8 2, i8 2, i8 2>
+;
+ %t0 = insertelement <8 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 undef, i8 1>, i8 %ele, i32 6
+ %t1 = shl <8 x i8> %t0, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+ %t2 = shufflevector <8 x i8> %t1, <8 x i8> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
+ ret <4 x i8> %t2
+}
+
+; If composition of two shuffles is identity, shuffles can be removed.
+define <4 x i32> @shuffle_17ident(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17ident(
+; CHECK-NEXT: ret <4 x i32> [[V:%.*]]
+;
+ %shuffle = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %shuffle2 = shufflevector <4 x i32> %shuffle, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+ ret <4 x i32> %shuffle2
+}
+
+; swizzle can be put after operation
+define <4 x i32> @shuffle_17and(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17and(
+; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = and <4 x i32> %t1, %t2
+ ret <4 x i32> %r
+}
+
+declare void @use(<2 x float>)
+
+; One extra use is ok to transform.
+
+define <2 x float> @shuffle_fadd_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fadd_multiuse(
+; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[V1]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: call void @use(<2 x float> [[T1]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %r = fadd <2 x float> %t1, %t2
+ call void @use(<2 x float> %t1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @shuffle_fdiv_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fdiv_multiuse(
+; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <2 x float> [[V1:%.*]], [[V2]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: call void @use(<2 x float> [[T2]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %r = fdiv <2 x float> %t1, %t2
+ call void @use(<2 x float> %t2)
+ ret <2 x float> %r
+}
+
+; But 2 extra uses would require an extra instruction.
+
+define <2 x float> @shuffle_fsub_multiuse(<2 x float> %v1, <2 x float> %v2) {
+; CHECK-LABEL: @shuffle_fsub_multiuse(
+; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V1:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[T2:%.*]] = shufflevector <2 x float> [[V2:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[R:%.*]] = fsub <2 x float> [[T1]], [[T2]]
+; CHECK-NEXT: call void @use(<2 x float> [[T1]])
+; CHECK-NEXT: call void @use(<2 x float> [[T2]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %t1 = shufflevector <2 x float> %v1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %t2 = shufflevector <2 x float> %v2, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %r = fsub <2 x float> %t1, %t2
+ call void @use(<2 x float> %t1)
+ call void @use(<2 x float> %t2)
+ ret <2 x float> %r
+}
+
+define <4 x i32> @shuffle_17add(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17add(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = add <4 x i32> %t1, %t2
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17addnsw(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17addnsw(
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = add nsw <4 x i32> %t1, %t2
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17addnuw(<4 x i32> %v1, <4 x i32> %v2) {
+; CHECK-LABEL: @shuffle_17addnuw(
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw <4 x i32> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %t2 = shufflevector <4 x i32> %v2, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = add nuw <4 x i32> %t1, %t2
+ ret <4 x i32> %r
+}
+
+define <4 x float> @shuffle_17fsub_fast(<4 x float> %v1, <4 x float> %v2) {
+; CHECK-LABEL: @shuffle_17fsub_fast(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub fast <4 x float> [[V1:%.*]], [[V2:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %t1 = shufflevector <4 x float> %v1, <4 x float> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %t2 = shufflevector <4 x float> %v2, <4 x float> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = fsub fast <4 x float> %t1, %t2
+ ret <4 x float> %r
+}
+
+define <4 x i32> @add_const(<4 x i32> %v) {
+; CHECK-LABEL: @add_const(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[V:%.*]], <i32 44, i32 41, i32 42, i32 43>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+ %r = add <4 x i32> %t1, <i32 41, i32 42, i32 43, i32 44>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @sub_const(<4 x i32> %v) {
+; CHECK-LABEL: @sub_const(
+; CHECK-NEXT: [[TMP1:%.*]] = sub <4 x i32> <i32 44, i32 43, i32 42, i32 41>, [[V:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %r = sub <4 x i32> <i32 41, i32 42, i32 43, i32 44>, %t1
+ ret <4 x i32> %r
+}
+
+; Math before shuffle requires an extra shuffle.
+
+define <2 x float> @fadd_const_multiuse(<2 x float> %v) {
+; CHECK-LABEL: @fadd_const_multiuse(
+; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[R:%.*]] = fadd <2 x float> [[T1]], <float 4.100000e+01, float 4.200000e+01>
+; CHECK-NEXT: call void @use(<2 x float> [[T1]])
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %t1 = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 1, i32 0>
+ %r = fadd <2 x float> %t1, <float 41.0, float 42.0>
+ call void @use(<2 x float> %t1)
+ ret <2 x float> %r
+}
+
+; Math before splat allows replacing constant elements with undef lanes.
+
+define <4 x i32> @mul_const_splat(<4 x i32> %v) {
+; CHECK-LABEL: @mul_const_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <4 x i32> [[V:%.*]], <i32 undef, i32 42, i32 undef, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %r = mul <4 x i32> <i32 42, i32 42, i32 42, i32 42>, %t1
+ ret <4 x i32> %r
+}
+
+; Take 2 elements of a vector and shift each of those by a
diff erent amount
+
+define <4 x i32> @lshr_const_half_splat(<4 x i32> %v) {
+; CHECK-LABEL: @lshr_const_half_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <4 x i32> <i32 undef, i32 8, i32 9, i32 undef>, [[V:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %t1 = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 2>
+ %r = lshr <4 x i32> <i32 8, i32 8, i32 9, i32 9>, %t1
+ ret <4 x i32> %r
+}
+
+; We can't change this because there's no pre-shuffle version of the fmul constant.
+
+define <2 x float> @fmul_const_invalid_constant(<2 x float> %v) {
+; CHECK-LABEL: @fmul_const_invalid_constant(
+; CHECK-NEXT: [[T1:%.*]] = shufflevector <2 x float> [[V:%.*]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = fmul <2 x float> [[T1]], <float 4.100000e+01, float 4.200000e+01>
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %t1 = shufflevector <2 x float> %v, <2 x float> undef, <2 x i32> <i32 0, i32 0>
+ %r = fmul <2 x float> %t1, <float 41.0, float 42.0>
+ ret <2 x float> %r
+}
+
+; Reduce the width of the binop by moving it ahead of a shuffle.
+
+define <4 x i8> @widening_shuffle_add_1(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_1(
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 42, i8 43>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+ ret <4 x i8> %r
+}
+
+; Reduce the width of the binop by moving it ahead of a shuffle.
+
+define <4 x i8> @widening_shuffle_add_2(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_2(
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i8> [[X:%.*]], <i8 43, i8 42>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 0, i32 undef, i32 undef>
+ %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+ ret <4 x i8> %r
+}
+
+; Negative test - widening shuffles have the same mask/constant constraint as non-size-changing shuffles.
+
+define <4 x i8> @widening_shuffle_add_invalid_constant(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_invalid_constant(
+; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 1, i32 1, i32 undef, i32 undef>
+ %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+ ret <4 x i8> %r
+}
+
+; Negative test - widening shuffles have an additional constraint: they must not extend with anything but undefs.
+
+define <4 x i8> @widening_shuffle_add_invalid_mask(<2 x i8> %x) {
+; CHECK-LABEL: @widening_shuffle_add_invalid_mask(
+; CHECK-NEXT: [[WIDEX:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i8> [[WIDEX]], <i8 42, i8 43, i8 44, i8 45>
+; CHECK-NEXT: ret <4 x i8> [[R]]
+;
+ %widex = shufflevector <2 x i8> %x, <2 x i8> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 0>
+ %r = add <4 x i8> %widex, <i8 42, i8 43, i8 44, i8 45>
+ ret <4 x i8> %r
+}
+
+; A binop that produces undef in the high lanes can be moved before the shuffle.
+; This is ok because 'shl C, undef --> undef'.
+
+define <4 x i16> @widening_shuffle_shl_constant_op0(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op0(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> <i16 42, i16 -42>, [[V:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i16> [[BO]]
+;
+ %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %bo = shl <4 x i16> <i16 42, i16 -42, i16 -1, i16 -1>, %shuf
+ ret <4 x i16> %bo
+}
+
+; A binop that produces undef in the high lanes can be moved before the shuffle.
+; This is ok because 'shl undef, 0 --> undef'.
+
+define <4 x i16> @widening_shuffle_shl_constant_op1(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op1(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i16> [[V:%.*]], <i16 2, i16 4>
+; CHECK-NEXT: [[BO:%.*]] = shufflevector <2 x i16> [[TMP1]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i16> [[BO]]
+;
+ %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %bo = shl <4 x i16> %shuf, <i16 2, i16 4, i16 0, i16 0>
+ ret <4 x i16> %bo
+}
+
+; A binop that does not produce undef in the high lanes can not be moved before the shuffle.
+; This is not ok because 'shl undef, 1 (or 2)' --> 0' but moving the shuffle results in undef instead.
+
+define <4 x i16> @widening_shuffle_shl_constant_op1_non0(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_shl_constant_op1_non0(
+; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[BO:%.*]] = shl <4 x i16> [[SHUF]], <i16 2, i16 4, i16 1, i16 2>
+; CHECK-NEXT: ret <4 x i16> [[BO]]
+;
+ %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %bo = shl <4 x i16> %shuf, <i16 2, i16 4, i16 1, i16 2>
+ ret <4 x i16> %bo
+}
+
+; A binop that does not produce undef in the high lanes can not be moved before the shuffle.
+; This is not ok because 'or -1, undef --> -1' but moving the shuffle results in undef instead.
+
+define <4 x i16> @widening_shuffle_or(<2 x i16> %v) {
+; CHECK-LABEL: @widening_shuffle_or(
+; CHECK-NEXT: [[SHUF:%.*]] = shufflevector <2 x i16> [[V:%.*]], <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[BO:%.*]] = or <4 x i16> [[SHUF]], <i16 42, i16 -42, i16 -1, i16 -1>
+; CHECK-NEXT: ret <4 x i16> [[BO]]
+;
+ %shuf = shufflevector <2 x i16> %v, <2 x i16> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %bo = or <4 x i16> %shuf, <i16 42, i16 -42, i16 -1, i16 -1>
+ ret <4 x i16> %bo
+}
+
+define <4 x i32> @shuffle_17add2(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17add2(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[V:%.*]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[TMP1]]
+;
+ %t1 = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %t2 = add <4 x i32> %t1, %t1
+ %r = shufflevector <4 x i32> %t2, <4 x i32> zeroinitializer, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @shuffle_17mulsplat(<4 x i32> %v) {
+; CHECK-LABEL: @shuffle_17mulsplat(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <4 x i32> [[V:%.*]], [[V]]
+; CHECK-NEXT: [[M1:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: ret <4 x i32> [[M1]]
+;
+ %s1 = shufflevector <4 x i32> %v, <4 x i32> zeroinitializer, <4 x i32> zeroinitializer
+ %m1 = mul <4 x i32> %s1, %s1
+ %s2 = shufflevector <4 x i32> %m1, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ ret <4 x i32> %s2
+}
+
+; Do not reorder shuffle and binop if LHS of shuffles are of
diff erent size
+define <2 x i32> @pr19717(<4 x i32> %in0, <2 x i32> %in1) {
+; CHECK-LABEL: @pr19717(
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[IN0:%.*]], <4 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[SHUFFLE4:%.*]] = shufflevector <2 x i32> [[IN1:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[MUL:%.*]] = mul <2 x i32> [[SHUFFLE]], [[SHUFFLE4]]
+; CHECK-NEXT: ret <2 x i32> [[MUL]]
+;
+ %shuffle = shufflevector <4 x i32> %in0, <4 x i32> %in0, <2 x i32> zeroinitializer
+ %shuffle4 = shufflevector <2 x i32> %in1, <2 x i32> %in1, <2 x i32> zeroinitializer
+ %mul = mul <2 x i32> %shuffle, %shuffle4
+ ret <2 x i32> %mul
+}
+
+define <4 x i16> @pr19717a(<8 x i16> %in0, <8 x i16> %in1) {
+; CHECK-LABEL: @pr19717a(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <8 x i16> [[IN0:%.*]], [[IN1:%.*]]
+; CHECK-NEXT: [[MUL:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+; CHECK-NEXT: ret <4 x i16> [[MUL]]
+;
+ %shuffle = shufflevector <8 x i16> %in0, <8 x i16> %in0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+ %shuffle1 = shufflevector <8 x i16> %in1, <8 x i16> %in1, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+ %mul = mul <4 x i16> %shuffle, %shuffle1
+ ret <4 x i16> %mul
+}
+
+define <8 x i8> @pr19730(<16 x i8> %in0) {
+; CHECK-LABEL: @pr19730(
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[IN0:%.*]], <16 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <8 x i8> [[SHUFFLE]], <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x i8> [[SHUFFLE1]]
+;
+ %shuffle = shufflevector <16 x i8> %in0, <16 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %shuffle1 = shufflevector <8 x i8> %shuffle, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ ret <8 x i8> %shuffle1
+}
+
+define i32 @pr19737(<4 x i32> %in0) {
+; CHECK-LABEL: @pr19737(
+; CHECK-NEXT: [[RV:%.*]] = extractelement <4 x i32> [[IN0:%.*]], i32 0
+; CHECK-NEXT: ret i32 [[RV]]
+;
+ %shuffle.i = shufflevector <4 x i32> zeroinitializer, <4 x i32> %in0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+ %neg.i = xor <4 x i32> %shuffle.i, <i32 -1, i32 -1, i32 -1, i32 -1>
+ %and.i = and <4 x i32> %in0, %neg.i
+ %rv = extractelement <4 x i32> %and.i, i32 0
+ ret i32 %rv
+}
+
+; In PR20059 ( http://llvm.org/pr20059 ), shufflevector operations are reordered/removed
+; for an srem operation. This is not a valid optimization because it may cause a trap
+; on div-by-zero.
+
+define <4 x i32> @pr20059(<4 x i32> %p1, <4 x i32> %p2) {
+; CHECK-LABEL: @pr20059(
+; CHECK-NEXT: [[SPLAT1:%.*]] = shufflevector <4 x i32> [[P1:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[SPLAT2:%.*]] = shufflevector <4 x i32> [[P2:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[RETVAL:%.*]] = srem <4 x i32> [[SPLAT1]], [[SPLAT2]]
+; CHECK-NEXT: ret <4 x i32> [[RETVAL]]
+;
+ %splat1 = shufflevector <4 x i32> %p1, <4 x i32> undef, <4 x i32> zeroinitializer
+ %splat2 = shufflevector <4 x i32> %p2, <4 x i32> undef, <4 x i32> zeroinitializer
+ %retval = srem <4 x i32> %splat1, %splat2
+ ret <4 x i32> %retval
+}
+
+define <4 x i32> @pr20114(<4 x i32> %__mask) {
+; CHECK-LABEL: @pr20114(
+; CHECK-NEXT: [[MASK01_I:%.*]] = shufflevector <4 x i32> [[__MASK:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[MASKED_NEW_I_I_I:%.*]] = and <4 x i32> [[MASK01_I]], bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64)> to <4 x i32>)
+; CHECK-NEXT: ret <4 x i32> [[MASKED_NEW_I_I_I]]
+;
+ %mask01.i = shufflevector <4 x i32> %__mask, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+ %masked_new.i.i.i = and <4 x i32> bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @pr20114 to i64)> to <4 x i32>), %mask01.i
+ ret <4 x i32> %masked_new.i.i.i
+}
+
+define <2 x i32*> @pr23113(<4 x i32*> %A) {
+; CHECK-LABEL: @pr23113(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32*> [[A:%.*]], <4 x i32*> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: ret <2 x i32*> [[TMP1]]
+;
+ %1 = shufflevector <4 x i32*> %A, <4 x i32*> undef, <2 x i32> <i32 0, i32 1>
+ ret <2 x i32*> %1
+}
+
+; Unused lanes in the new binop should not kill the entire op (although it may simplify anyway as shown here).
+
+define <2 x i32> @PR37648(<2 x i32> %x) {
+; CHECK-LABEL: @PR37648(
+; CHECK-NEXT: ret <2 x i32> zeroinitializer
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = urem <2 x i32> %splat, <i32 1, i32 1>
+ ret <2 x i32> %r
+}
+
+; Test shuffle followed by binop with splat constant for all 18 binop opcodes.
+; Test with constant as operand 0 and operand 1 for non-commutative opcodes.
+
+define <2 x i32> @add_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @add_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = add <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @sub_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @sub_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = sub <2 x i32> <i32 42, i32 undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = sub <2 x i32> <i32 42, i32 42>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @sub_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @sub_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = add <2 x i32> [[X:%.*]], <i32 -42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = sub <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @mul_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @mul_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = mul <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @shl_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @shl_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = shl <2 x i32> <i32 5, i32 5>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @shl_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @shl_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = shl <2 x i32> %splat, <i32 5, i32 5>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @ashr_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = ashr <2 x i32> <i32 5, i32 5>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @ashr_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @ashr_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = ashr <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = ashr <2 x i32> %splat, <i32 5, i32 5>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @lshr_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @lshr_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> <i32 5, i32 undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = lshr <2 x i32> <i32 5, i32 5>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @lshr_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @lshr_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = lshr <2 x i32> [[X:%.*]], <i32 5, i32 0>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = lshr <2 x i32> %splat, <i32 5, i32 5>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @urem_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @urem_splat_constant0(
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = urem <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = urem <2 x i32> <i32 42, i32 42>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @urem_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @urem_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = urem <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = urem <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @srem_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @srem_splat_constant0(
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = srem <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = srem <2 x i32> <i32 42, i32 42>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @srem_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @srem_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = srem <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = srem <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @udiv_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @udiv_splat_constant0(
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = udiv <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = udiv <2 x i32> <i32 42, i32 42>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @udiv_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @udiv_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = udiv <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = udiv <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @sdiv_splat_constant0(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv_splat_constant0(
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <2 x i32> [[X:%.*]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = sdiv <2 x i32> <i32 42, i32 42>, [[SPLAT]]
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = sdiv <2 x i32> <i32 42, i32 42>, %splat
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @sdiv_splat_constant1(<2 x i32> %x) {
+; CHECK-LABEL: @sdiv_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv <2 x i32> [[X:%.*]], <i32 42, i32 1>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = sdiv <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @and_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @and_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = and <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+; AND does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @and_constant_mask_undef(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 0, i16 0, i16 -1, i16 -1>
+; CHECK-NEXT: ret <4 x i16> [[AND]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+ %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+ ret <4 x i16> %and
+}
+
+; AND does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @and_constant_mask_undef_2(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[ADD:%.*]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+; CHECK-NEXT: [[AND:%.*]] = and <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 -1, i16 0>
+; CHECK-NEXT: ret <4 x i16> [[AND]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 undef>
+ %and = and <4 x i16> %shuffle, <i16 -1, i16 -1, i16 -1, i16 -0>
+ ret <4 x i16> %and
+}
+
+; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes.
+define <4 x i16> @and_constant_mask_undef_3(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret <4 x i16> <i16 0, i16 0, i16 0, i16 undef>
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+ %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 0, i16 -1>
+ ret <4 x i16> %and
+}
+
+; We can move the AND across the shuffle, as -1 (AND identity value) is used for undef lanes.
+define <4 x i16> @and_constant_mask_undef_4(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_undef_4(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 9, i16 20, i16 undef, i16 undef>
+; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+; CHECK-NEXT: ret <4 x i16> [[AND]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 undef>
+ %and = and <4 x i16> %shuffle, <i16 9, i16 20, i16 20, i16 -1>
+ ret <4 x i16> %and
+}
+
+define <4 x i16> @and_constant_mask_not_undef(<4 x i16> %add) {
+; CHECK-LABEL: @and_constant_mask_not_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = and <4 x i16> [[ADD:%.*]], <i16 undef, i16 -1, i16 0, i16 0>
+; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i16> [[AND]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %add, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+ %and = and <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+ ret <4 x i16> %and
+}
+
+; OR does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @or_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 -1, i16 0, i16 0>
+; CHECK-NEXT: ret <4 x i16> [[OR]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+ %or = or <4 x i16> %shuffle, <i16 -1, i16 -1, i16 0, i16 0>
+ ret <4 x i16> %or
+}
+
+; OR does not fold to undef for undef operands, we cannot move it
+; across a shuffle with undef masks.
+define <4 x i16> @or_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT: [[OR:%.*]] = or <4 x i16> [[SHUFFLE]], <i16 -1, i16 0, i16 0, i16 -1>
+; CHECK-NEXT: ret <4 x i16> [[OR]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+ %or = or <4 x i16> %shuffle, <i16 -1, i16 0, i16 0, i16 -1>
+ ret <4 x i16> %or
+}
+
+; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes.
+define <4 x i16> @or_constant_mask_undef_3(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_3(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: ret <4 x i16> <i16 undef, i16 -1, i16 -1, i16 undef>
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+ %or = or <4 x i16> %shuffle, <i16 0, i16 -1, i16 -1, i16 0>
+ ret <4 x i16> %or
+}
+
+; We can move the OR across the shuffle, as 0 (OR identity value) is used for undef lanes.
+define <4 x i16> @or_constant_mask_undef_4(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_undef_4(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 undef, i16 99, i16 undef, i16 undef>
+; CHECK-NEXT: [[OR:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+; CHECK-NEXT: ret <4 x i16> [[OR]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 1, i32 1, i32 undef>
+ %or = or <4 x i16> %shuffle, <i16 0, i16 99, i16 99, i16 0>
+ ret <4 x i16> %or
+}
+
+define <4 x i16> @or_constant_mask_not_undef(<4 x i16> %in) {
+; CHECK-LABEL: @or_constant_mask_not_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = or <4 x i16> [[IN:%.*]], <i16 undef, i16 -1, i16 0, i16 0>
+; CHECK-NEXT: [[AND:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i16> [[AND]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 2, i32 3, i32 1, i32 1>
+ %and = or <4 x i16> %shuffle, <i16 0, i16 0, i16 -1, i16 -1>
+ ret <4 x i16> %and
+}
+
+define <4 x i16> @shl_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @shl_constant_mask_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+; CHECK-NEXT: [[SHL:%.*]] = shl <4 x i16> [[SHUFFLE]], <i16 10, i16 3, i16 0, i16 0>
+; CHECK-NEXT: ret <4 x i16> [[SHL]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 1>
+ %shl = shl <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+ ret <4 x i16> %shl
+}
+
+define <4 x i16> @add_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @add_constant_mask_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[ADD:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i16> [[ADD]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+ %add = add <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+ ret <4 x i16> %add
+}
+
+define <4 x i16> @add_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @add_constant_mask_undef_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 undef, i16 0, i16 3, i16 undef>
+; CHECK-NEXT: [[ADD:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i16> [[ADD]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 2, i32 1, i32 1>
+ %add = add <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+ ret <4 x i16> %add
+}
+
+define <4 x i16> @sub_constant_mask_undef(<4 x i16> %in) {
+; CHECK-LABEL: @sub_constant_mask_undef(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SUB:%.*]] = shufflevector <4 x i16> [[IN:%.*]], <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+; CHECK-NEXT: ret <4 x i16> [[SUB]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 1>
+ %sub = sub <4 x i16> %shuffle, <i16 10, i16 3, i16 0, i16 0>
+ ret <4 x i16> %sub
+}
+
+define <4 x i16> @sub_constant_mask_undef_2(<4 x i16> %in) {
+; CHECK-LABEL: @sub_constant_mask_undef_2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = add <4 x i16> [[IN:%.*]], <i16 undef, i16 0, i16 -10, i16 undef>
+; CHECK-NEXT: [[SUB:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+; CHECK-NEXT: ret <4 x i16> [[SUB]]
+;
+entry:
+ %shuffle = shufflevector <4 x i16> %in, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 2, i32 undef>
+ %sub = sub <4 x i16> %shuffle, <i16 0, i16 0, i16 10, i16 99>
+ ret <4 x i16> %sub
+}
+
+define <2 x i32> @or_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @or_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = or <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x i32> @xor_splat_constant(<2 x i32> %x) {
+; CHECK-LABEL: @xor_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i32> [[X:%.*]], <i32 42, i32 undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32> [[R]]
+;
+ %splat = shufflevector <2 x i32> %x, <2 x i32> undef, <2 x i32> zeroinitializer
+ %r = xor <2 x i32> %splat, <i32 42, i32 42>
+ ret <2 x i32> %r
+}
+
+define <2 x float> @fadd_splat_constant(<2 x float> %x) {
+; CHECK-LABEL: @fadd_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fadd <2 x float> %splat, <float 42.0, float 42.0>
+ ret <2 x float> %r
+}
+
+define <2 x float> @fsub_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @fsub_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fsub <2 x float> <float 42.0, float 42.0>, %splat
+ ret <2 x float> %r
+}
+
+define <2 x float> @fsub_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @fsub_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[X:%.*]], <float -4.200000e+01, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fsub <2 x float> %splat, <float 42.0, float 42.0>
+ ret <2 x float> %r
+}
+
+define <2 x float> @fneg(<2 x float> %x) {
+; CHECK-LABEL: @fneg(
+; CHECK-NEXT: [[TMP1:%.*]] = fneg <2 x float> [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fsub <2 x float> <float -0.0, float -0.0>, %splat
+ ret <2 x float> %r
+}
+
+define <2 x float> @fmul_splat_constant(<2 x float> %x) {
+; CHECK-LABEL: @fmul_splat_constant(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fmul <2 x float> %splat, <float 42.0, float 42.0>
+ ret <2 x float> %r
+}
+
+define <2 x float> @fdiv_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @fdiv_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fdiv <2 x float> <float 42.0, float 42.0>, %splat
+ ret <2 x float> %r
+}
+
+define <2 x float> @fdiv_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @fdiv_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = fdiv <2 x float> %splat, <float 42.0, float 42.0>
+ ret <2 x float> %r
+}
+
+define <2 x float> @frem_splat_constant0(<2 x float> %x) {
+; CHECK-LABEL: @frem_splat_constant0(
+; CHECK-NEXT: [[TMP1:%.*]] = frem <2 x float> <float 4.200000e+01, float undef>, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = frem <2 x float> <float 42.0, float 42.0>, %splat
+ ret <2 x float> %r
+}
+
+define <2 x float> @frem_splat_constant1(<2 x float> %x) {
+; CHECK-LABEL: @frem_splat_constant1(
+; CHECK-NEXT: [[TMP1:%.*]] = frem <2 x float> [[X:%.*]], <float 4.200000e+01, float undef>
+; CHECK-NEXT: [[R:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splat = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %r = frem <2 x float> %splat, <float 42.0, float 42.0>
+ ret <2 x float> %r
+}
+
+; Equivalent shuffle masks, but only one is a narrowing op.
+
+define <2 x i1> @PR40734(<1 x i1> %x, <4 x i1> %y) {
+; CHECK-LABEL: @PR40734(
+; CHECK-NEXT: [[WIDEN:%.*]] = shufflevector <1 x i1> zeroinitializer, <1 x i1> [[X:%.*]], <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[NARROW:%.*]] = shufflevector <4 x i1> [[Y:%.*]], <4 x i1> undef, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[R:%.*]] = and <2 x i1> [[WIDEN]], [[NARROW]]
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %widen = shufflevector <1 x i1> zeroinitializer, <1 x i1> %x, <2 x i32> <i32 0, i32 1>
+ %narrow = shufflevector <4 x i1> %y, <4 x i1> undef, <2 x i32> <i32 0, i32 1>
+ %r = and <2 x i1> %widen, %narrow
+ ret <2 x i1> %r
+}
+
+; Negative test - do not transform non-power-of-2 unless we know the backend handles these sequences identically.
+
+define <7 x i8> @insert_subvector_shuffles(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @insert_subvector_shuffles(
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <7 x i32> <i32 0, i32 8, i32 1, i32 undef, i32 8, i32 1, i32 9>
+; CHECK-NEXT: ret <7 x i8> [[S3]]
+;
+ %s1 = shufflevector <3 x i8> %x, <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s2 = shufflevector <3 x i8> %y, <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <7 x i8> %s1, <7 x i8> %s2, <7 x i32> <i32 0, i32 8, i32 1, i32 undef, i32 8, i32 1, i32 9>
+ ret <7 x i8> %s3
+}
+
+define <8 x i8> @insert_subvector_shuffles_pow2elts(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @insert_subvector_shuffles_pow2elts(
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <8 x i32> <i32 0, i32 2, i32 1, i32 undef, i32 2, i32 1, i32 3, i32 0>
+; CHECK-NEXT: ret <8 x i8> [[S3]]
+;
+ %s1 = shufflevector <2 x i8> %x, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s2 = shufflevector <2 x i8> %y, <2 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <8 x i32> <i32 0, i32 8, i32 1, i32 undef, i32 8, i32 1, i32 9, i32 0>
+ ret <8 x i8> %s3
+}
+
+; The last shuffle may change the vector type.
+; Negative test - do not transform non-power-of-2 unless we know the backend handles these sequences identically.
+
+define <2 x i8> @insert_subvector_shuffles_narrowing(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @insert_subvector_shuffles_narrowing(
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <7 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x i8> [[Y:%.*]], <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <7 x i8> [[S1]], <7 x i8> [[S2]], <2 x i32> <i32 0, i32 8>
+; CHECK-NEXT: ret <2 x i8> [[S3]]
+;
+ %s1 = shufflevector <3 x i8> %x, <3 x i8> undef, <7 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s2 = shufflevector <3 x i8> %y, <3 x i8> undef, <7 x i32> <i32 undef, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <7 x i8> %s1, <7 x i8> %s2, <2 x i32> <i32 0, i32 8>
+ ret <2 x i8> %s3
+}
+
+define <2 x i8> @insert_subvector_shuffles_narrowing_pow2elts(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @insert_subvector_shuffles_narrowing_pow2elts(
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x i8> [[X:%.*]], <4 x i8> [[Y:%.*]], <2 x i32> <i32 0, i32 4>
+; CHECK-NEXT: ret <2 x i8> [[S3]]
+;
+ %s1 = shufflevector <4 x i8> %x, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s2 = shufflevector <4 x i8> %y, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <2 x i32> <i32 0, i32 8>
+ ret <2 x i8> %s3
+}
+
+; Similar to above, but this reduces to a widen with undefs of 'x'.
+
+define <4 x double> @insert_subvector_shuffles_identity(<2 x double> %x) {
+; CHECK-LABEL: @insert_subvector_shuffles_identity(
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x double> [[S3]]
+;
+ %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
+ %s2 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <4 x double> %s2, <4 x double> %s1, <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+ ret <4 x double> %s3
+}
+
+; Negative test - not identity with padding (although this could be folded with better analysis).
+
+define <4 x double> @not_insert_subvector_shuffle(<2 x double> %x) {
+; CHECK-LABEL: @not_insert_subvector_shuffle(
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <2 x double> [[X]], <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 7, i32 undef>
+; CHECK-NEXT: ret <4 x double> [[S3]]
+;
+ %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 1>
+ %s2 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <4 x double> %s2, <4 x double> %s1, <4 x i32> <i32 0, i32 5, i32 7, i32 undef>
+ ret <4 x double> %s3
+}
+
+; Negative test - operands are not the same size (although this could be partly folded with better analysis).
+
+define <4 x double> @not_insert_subvector_shuffles_with_same_size(<2 x double> %x, <3 x double> %y) {
+; CHECK-LABEL: @not_insert_subvector_shuffles_with_same_size(
+; CHECK-NEXT: [[S1:%.*]] = shufflevector <2 x double> [[X:%.*]], <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[S2:%.*]] = shufflevector <3 x double> [[Y:%.*]], <3 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[S3:%.*]] = shufflevector <4 x double> [[S2]], <4 x double> [[S1]], <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x double> [[S3]]
+;
+ %s1 = shufflevector <2 x double> %x, <2 x double> undef, <4 x i32> <i32 undef, i32 1, i32 undef, i32 undef>
+ %s2 = shufflevector <3 x double> %y, <3 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %s3 = shufflevector <4 x double> %s2, <4 x double> %s1, <4 x i32> <i32 0, i32 5, i32 undef, i32 undef>
+ ret <4 x double> %s3
+}
+
+; Demanded vector elements may not be able to simplify a shuffle mask
+; before we try to narrow it. This used to crash.
+
+define <4 x float> @insert_subvector_crash_invalid_mask_elt(<2 x float> %x, <4 x float>* %p) {
+; CHECK-LABEL: @insert_subvector_crash_invalid_mask_elt(
+; CHECK-NEXT: [[WIDEN:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[I:%.*]] = shufflevector <2 x float> [[X]], <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: store <4 x float> [[I]], <4 x float>* [[P:%.*]], align 16
+; CHECK-NEXT: ret <4 x float> [[WIDEN]]
+;
+ %widen = shufflevector <2 x float> %x, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %ext2 = extractelement <2 x float> %x, i32 0
+ %I = insertelement <4 x float> %widen, float %ext2, i16 0
+ store <4 x float> %I, <4 x float>* %p
+ ret <4 x float> %widen
+}
+
+define <4 x i32> @splat_assoc_add(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], <i32 317426, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+ %a = add <4 x i32> %y, <i32 317426, i32 317426, i32 317426, i32 317426>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <vscale x 4 x i32> @vsplat_assoc_add(<vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; CHECK-LABEL: @vsplat_assoc_add(
+; CHECK-NEXT: [[TMP1:%.*]] = add <vscale x 4 x i32> [[X:%.*]], shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 317426, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <vscale x 4 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+
+ %splatx = shufflevector <vscale x 4 x i32> %x, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %a = add <vscale x 4 x i32> %y, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 317426, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+ %r = add <vscale x 4 x i32> %splatx, %a
+ ret <vscale x 4 x i32> %r
+}
+
+; Undefs in splat mask are replaced with defined splat index
+
+define <4 x i32> @splat_assoc_add_undef_mask_elts(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_mask_elts(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 0, i32 undef, i32 undef>
+ %a = add <4 x i32> %y, <i32 42, i32 42, i32 42, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+; Undefs in splat mask are replaced with defined splat index
+
+define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+ %a = add <4 x i32> %y, <i32 42, i32 42, i32 42, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_assoc_add_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_constant_elts(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+ %a = add <4 x i32> %y, <i32 42, i32 undef, i32 undef, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_assoc_add_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_constant_elt_at_splat_index(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 undef, i32 42, i32 undef, i32 42>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+ %a = add <4 x i32> %y, <i32 undef, i32 42, i32 undef, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_assoc_add_undef_mask_elts_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_mask_elts_undef_constant_elts(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 undef>
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 0, i32 undef>
+ %a = add <4 x i32> %y, <i32 42, i32 undef, i32 undef, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elts(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 undef, i32 undef, i32 42>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+ %a = add <4 x i32> %y, <i32 42, i32 undef, i32 undef, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_undef_mask_elt_at_splat_index_undef_constant_elt_at_splat_index(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 undef, i32 42, i32 undef, i32 42>
+; CHECK-NEXT: [[R:%.*]] = add <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 0, i32 0>
+ %a = add <4 x i32> %y, <i32 undef, i32 42, i32 undef, i32 42>
+ %r = add <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+; Non-zero splat index; commute operands; FMF intersect
+
+define <2 x float> @splat_assoc_fmul(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @splat_assoc_fmul(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul reassoc nsz <2 x float> [[X:%.*]], <float undef, float 3.000000e+00>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[R:%.*]] = fmul reassoc nsz <2 x float> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splatx = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+ %a = fmul reassoc nsz <2 x float> %y, <float 3.0, float 3.0>
+ %r = fmul reassoc nsz nnan <2 x float> %a, %splatx
+ ret <2 x float> %r
+}
+
+; Two splat shuffles; drop poison-generating flags
+
+define <3 x i8> @splat_assoc_mul(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_mul(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <3 x i8> [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i8> [[TMP1]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[R:%.*]] = mul <3 x i8> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %a = mul nsw <3 x i8> %y, %splatz
+ %r = mul <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @splat_assoc_mul_undef_elt1(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_mul_undef_elt1(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <3 x i8> [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i8> [[TMP1]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[R:%.*]] = mul <3 x i8> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 undef, i32 2, i32 2>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %a = mul nsw <3 x i8> %y, %splatz
+ %r = mul nsw nuw <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @splat_assoc_mul_undef_elt2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_mul_undef_elt2(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 undef, i32 2, i32 2>
+; CHECK-NEXT: [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 undef, i32 2, i32 2>
+ %a = mul nsw <3 x i8> %y, %splatz
+ %r = mul nsw nuw <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index1(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_mul_undef_elt_at_splat_index1(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <3 x i8> [[Z:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <3 x i8> [[TMP1]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[R:%.*]] = mul <3 x i8> [[TMP2]], [[Y:%.*]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 undef>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %a = mul nsw <3 x i8> %y, %splatz
+ %r = mul nsw nuw <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+define <3 x i8> @splat_assoc_mul_undef_elt_at_splat_index2(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_mul_undef_elt_at_splat_index2(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 undef>
+; CHECK-NEXT: [[A:%.*]] = mul nsw <3 x i8> [[SPLATZ]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = mul nuw nsw <3 x i8> [[A]], [[SPLATX]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 undef>
+ %a = mul nsw <3 x i8> %y, %splatz
+ %r = mul nsw nuw <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+; Negative test - mismatched splat elements
+
+define <3 x i8> @splat_assoc_or(<3 x i8> %x, <3 x i8> %y, <3 x i8> %z) {
+; CHECK-LABEL: @splat_assoc_or(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <3 x i8> [[X:%.*]], <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 1>
+; CHECK-NEXT: [[SPLATZ:%.*]] = shufflevector <3 x i8> [[Z:%.*]], <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+; CHECK-NEXT: [[A:%.*]] = or <3 x i8> [[SPLATZ]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = or <3 x i8> [[A]], [[SPLATX]]
+; CHECK-NEXT: ret <3 x i8> [[R]]
+;
+ %splatx = shufflevector <3 x i8> %x, <3 x i8> undef, <3 x i32> <i32 1, i32 1, i32 1>
+ %splatz = shufflevector <3 x i8> %z, <3 x i8> undef, <3 x i32> <i32 2, i32 2, i32 2>
+ %a = or <3 x i8> %y, %splatz
+ %r = or <3 x i8> %a, %splatx
+ ret <3 x i8> %r
+}
+
+; Negative test - not associative
+
+define <2 x float> @splat_assoc_fdiv(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @splat_assoc_fdiv(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = fdiv reassoc nsz <2 x float> [[Y:%.*]], <float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT: [[R:%.*]] = fdiv reassoc nsz <2 x float> [[A]], [[SPLATX]]
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splatx = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> zeroinitializer
+ %a = fdiv reassoc nsz <2 x float> %y, <float 3.0, float 3.0>
+ %r = fdiv reassoc nsz <2 x float> %a, %splatx
+ ret <2 x float> %r
+}
+
+; Negative test - extra use
+
+define <2 x float> @splat_assoc_fadd(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @splat_assoc_fadd(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <2 x float> [[X:%.*]], <2 x float> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[A:%.*]] = fadd fast <2 x float> [[Y:%.*]], <float 3.000000e+00, float 3.000000e+00>
+; CHECK-NEXT: call void @use(<2 x float> [[A]])
+; CHECK-NEXT: [[R:%.*]] = fadd fast <2 x float> [[A]], [[SPLATX]]
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %splatx = shufflevector <2 x float> %x, <2 x float> undef, <2 x i32> <i32 1, i32 1>
+ %a = fadd fast <2 x float> %y, <float 3.0, float 3.0>
+ call void @use(<2 x float> %a)
+ %r = fadd fast <2 x float> %a, %splatx
+ ret <2 x float> %r
+}
+
+; Negative test - narrowing splat
+
+define <3 x i32> @splat_assoc_and(<4 x i32> %x, <3 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_and(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <3 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = and <3 x i32> [[Y:%.*]], <i32 42, i32 42, i32 42>
+; CHECK-NEXT: [[R:%.*]] = and <3 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <3 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <3 x i32> zeroinitializer
+ %a = and <3 x i32> %y, <i32 42, i32 42, i32 42>
+ %r = and <3 x i32> %splatx, %a
+ ret <3 x i32> %r
+}
+
+; Negative test - widening splat
+
+define <5 x i32> @splat_assoc_xor(<4 x i32> %x, <5 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_xor(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <5 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = xor <5 x i32> [[Y:%.*]], <i32 42, i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT: [[R:%.*]] = xor <5 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <5 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <5 x i32> zeroinitializer
+ %a = xor <5 x i32> %y, <i32 42, i32 42, i32 42, i32 42, i32 42>
+ %r = xor <5 x i32> %splatx, %a
+ ret <5 x i32> %r
+}
+
+; Negative test - opcode mismatch
+
+define <4 x i32> @splat_assoc_add_mul(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @splat_assoc_add_mul(
+; CHECK-NEXT: [[SPLATX:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[A:%.*]] = add <4 x i32> [[Y:%.*]], <i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT: [[R:%.*]] = mul <4 x i32> [[SPLATX]], [[A]]
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %splatx = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> zeroinitializer
+ %a = add <4 x i32> %y, <i32 42, i32 42, i32 42, i32 42>
+ %r = mul <4 x i32> %splatx, %a
+ ret <4 x i32> %r
+}
+
+
+; Do not crash on constant expressions.
+
+define <4 x i32> @PR46872(<4 x i32> %x) {
+; CHECK-LABEL: @PR46872(
+; CHECK-NEXT: [[S:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[A:%.*]] = and <4 x i32> [[S]], bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64)> to <4 x i32>)
+; CHECK-NEXT: ret <4 x i32> [[A]]
+;
+ %s = shufflevector <4 x i32> %x, <4 x i32> undef, <4 x i32> <i32 undef, i32 0, i32 1, i32 1>
+ %a = and <4 x i32> %s, bitcast (<2 x i64> <i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64), i64 ptrtoint (<4 x i32> (<4 x i32>)* @PR46872 to i64)> to <4 x i32>)
+ ret <4 x i32> %a
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll
new file mode 100644
index 000000000000..5ffb47fd6e83
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector-casts-inseltpoison.ll
@@ -0,0 +1,413 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; Can't get smaller than this.
+
+define <2 x i1> @trunc(<2 x i64> %a) {
+; CHECK-LABEL: @trunc(
+; CHECK-NEXT: [[T:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT: ret <2 x i1> [[T]]
+;
+ %t = trunc <2 x i64> %a to <2 x i1>
+ ret <2 x i1> %t
+}
+
+; This is trunc.
+
+define <2 x i1> @and_cmp_is_trunc(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc(
+; CHECK-NEXT: [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %t = and <2 x i64> %a, <i64 1, i64 1>
+ %r = icmp ne <2 x i64> %t, zeroinitializer
+ ret <2 x i1> %r
+}
+
+; This is trunc.
+
+define <2 x i1> @and_cmp_is_trunc_even_with_undef_elt(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elt(
+; CHECK-NEXT: [[R:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i1>
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %t = and <2 x i64> %a, <i64 undef, i64 1>
+ %r = icmp ne <2 x i64> %t, zeroinitializer
+ ret <2 x i1> %r
+}
+
+; TODO: This could be just 1 instruction (trunc), but our undef matching is incomplete.
+
+define <2 x i1> @and_cmp_is_trunc_even_with_undef_elts(<2 x i64> %a) {
+; CHECK-LABEL: @and_cmp_is_trunc_even_with_undef_elts(
+; CHECK-NEXT: [[T:%.*]] = and <2 x i64> [[A:%.*]], <i64 undef, i64 1>
+; CHECK-NEXT: [[R:%.*]] = icmp ne <2 x i64> [[T]], <i64 undef, i64 0>
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %t = and <2 x i64> %a, <i64 undef, i64 1>
+ %r = icmp ne <2 x i64> %t, <i64 undef, i64 0>
+ ret <2 x i1> %r
+}
+
+; The ashr turns into an lshr.
+define <2 x i64> @test2(<2 x i64> %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: [[B:%.*]] = lshr <2 x i64> [[A:%.*]], <i64 1, i64 1>
+; CHECK-NEXT: [[TMP1:%.*]] = and <2 x i64> [[B]], <i64 32767, i64 32767>
+; CHECK-NEXT: ret <2 x i64> [[TMP1]]
+;
+ %b = and <2 x i64> %a, <i64 65535, i64 65535>
+ %t = ashr <2 x i64> %b, <i64 1, i64 1>
+ ret <2 x i64> %t
+}
+
+define <2 x i64> @test3(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp ord <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[AND:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[CONV]]
+;
+ %cmp = fcmp ord <4 x float> %a, zeroinitializer
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %cmp4 = fcmp ord <4 x float> %b, zeroinitializer
+ %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+ %and = and <4 x i32> %sext, %sext5
+ %conv = bitcast <4 x i32> %and to <2 x i64>
+ ret <2 x i64> %conv
+}
+
+define <2 x i64> @test4(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: [[TMP1:%.*]] = fcmp uno <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[OR:%.*]] = sext <4 x i1> [[TMP1]] to <4 x i32>
+; CHECK-NEXT: [[CONV:%.*]] = bitcast <4 x i32> [[OR]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[CONV]]
+;
+ %cmp = fcmp uno <4 x float> %a, zeroinitializer
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %cmp4 = fcmp uno <4 x float> %b, zeroinitializer
+ %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+ %or = or <4 x i32> %sext, %sext5
+ %conv = bitcast <4 x i32> %or to <2 x i64>
+ ret <2 x i64> %conv
+}
+
+; rdar://7434900
+define <2 x i64> @test5(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT: [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT: [[AND1:%.*]] = and <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT: [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT: [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[CONV]]
+;
+ %cmp = fcmp ult <4 x float> %a, zeroinitializer
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+ %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+ %and = and <4 x i32> %sext, %sext5
+ %conv = bitcast <4 x i32> %and to <2 x i64>
+ ret <2 x i64> %conv
+}
+
+define <2 x i64> @test6(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT: [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT: [[AND1:%.*]] = or <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT: [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT: [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[CONV]]
+;
+ %cmp = fcmp ult <4 x float> %a, zeroinitializer
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+ %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+ %and = or <4 x i32> %sext, %sext5
+ %conv = bitcast <4 x i32> %and to <2 x i64>
+ ret <2 x i64> %conv
+}
+
+define <2 x i64> @test7(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ult <4 x float> [[A:%.*]], zeroinitializer
+; CHECK-NEXT: [[CMP4:%.*]] = fcmp ult <4 x float> [[B:%.*]], zeroinitializer
+; CHECK-NEXT: [[AND1:%.*]] = xor <4 x i1> [[CMP]], [[CMP4]]
+; CHECK-NEXT: [[AND:%.*]] = sext <4 x i1> [[AND1]] to <4 x i32>
+; CHECK-NEXT: [[CONV:%.*]] = bitcast <4 x i32> [[AND]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[CONV]]
+;
+ %cmp = fcmp ult <4 x float> %a, zeroinitializer
+ %sext = sext <4 x i1> %cmp to <4 x i32>
+ %cmp4 = fcmp ult <4 x float> %b, zeroinitializer
+ %sext5 = sext <4 x i1> %cmp4 to <4 x i32>
+ %and = xor <4 x i32> %sext, %sext5
+ %conv = bitcast <4 x i32> %and to <2 x i64>
+ ret <2 x i64> %conv
+}
+
+define void @convert(<2 x i32>* %dst.addr, <2 x i64> %src) {
+; CHECK-LABEL: @convert(
+; CHECK-NEXT: [[VAL:%.*]] = trunc <2 x i64> [[SRC:%.*]] to <2 x i32>
+; CHECK-NEXT: [[ADD:%.*]] = add <2 x i32> [[VAL]], <i32 1, i32 1>
+; CHECK-NEXT: store <2 x i32> [[ADD]], <2 x i32>* [[DST_ADDR:%.*]], align 8
+; CHECK-NEXT: ret void
+;
+ %val = trunc <2 x i64> %src to <2 x i32>
+ %add = add <2 x i32> %val, <i32 1, i32 1>
+ store <2 x i32> %add, <2 x i32>* %dst.addr
+ ret void
+}
+
+define <2 x i65> @foo(<2 x i64> %t) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: [[A_MASK:%.*]] = and <2 x i64> [[T:%.*]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT: [[B:%.*]] = zext <2 x i64> [[A_MASK]] to <2 x i65>
+; CHECK-NEXT: ret <2 x i65> [[B]]
+;
+ %a = trunc <2 x i64> %t to <2 x i32>
+ %b = zext <2 x i32> %a to <2 x i65>
+ ret <2 x i65> %b
+}
+
+define <2 x i64> @bar(<2 x i65> %t) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: [[TMP1:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT: [[B:%.*]] = and <2 x i64> [[TMP1]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT: ret <2 x i64> [[B]]
+;
+ %a = trunc <2 x i65> %t to <2 x i32>
+ %b = zext <2 x i32> %a to <2 x i64>
+ ret <2 x i64> %b
+}
+
+define <2 x i64> @bars(<2 x i65> %t) {
+; CHECK-LABEL: @bars(
+; CHECK-NEXT: [[A:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i32>
+; CHECK-NEXT: [[B:%.*]] = sext <2 x i32> [[A]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[B]]
+;
+ %a = trunc <2 x i65> %t to <2 x i32>
+ %b = sext <2 x i32> %a to <2 x i64>
+ ret <2 x i64> %b
+}
+
+define <2 x i64> @quxs(<2 x i64> %t) {
+; CHECK-LABEL: @quxs(
+; CHECK-NEXT: [[TMP1:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
+; CHECK-NEXT: [[B:%.*]] = ashr exact <2 x i64> [[TMP1]], <i64 32, i64 32>
+; CHECK-NEXT: ret <2 x i64> [[B]]
+;
+ %a = trunc <2 x i64> %t to <2 x i32>
+ %b = sext <2 x i32> %a to <2 x i64>
+ ret <2 x i64> %b
+}
+
+define <2 x i64> @quxt(<2 x i64> %t) {
+; CHECK-LABEL: @quxt(
+; CHECK-NEXT: [[A:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
+; CHECK-NEXT: [[B:%.*]] = ashr exact <2 x i64> [[A]], <i64 32, i64 32>
+; CHECK-NEXT: ret <2 x i64> [[B]]
+;
+ %a = shl <2 x i64> %t, <i64 32, i64 32>
+ %b = ashr <2 x i64> %a, <i64 32, i64 32>
+ ret <2 x i64> %b
+}
+
+define <2 x double> @fa(<2 x double> %t) {
+; CHECK-LABEL: @fa(
+; CHECK-NEXT: [[A:%.*]] = fptrunc <2 x double> [[T:%.*]] to <2 x float>
+; CHECK-NEXT: [[B:%.*]] = fpext <2 x float> [[A]] to <2 x double>
+; CHECK-NEXT: ret <2 x double> [[B]]
+;
+ %a = fptrunc <2 x double> %t to <2 x float>
+ %b = fpext <2 x float> %a to <2 x double>
+ ret <2 x double> %b
+}
+
+define <2 x double> @fb(<2 x double> %t) {
+; CHECK-LABEL: @fb(
+; CHECK-NEXT: [[A:%.*]] = fptoui <2 x double> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT: [[B:%.*]] = uitofp <2 x i64> [[A]] to <2 x double>
+; CHECK-NEXT: ret <2 x double> [[B]]
+;
+ %a = fptoui <2 x double> %t to <2 x i64>
+ %b = uitofp <2 x i64> %a to <2 x double>
+ ret <2 x double> %b
+}
+
+define <2 x double> @fc(<2 x double> %t) {
+; CHECK-LABEL: @fc(
+; CHECK-NEXT: [[A:%.*]] = fptosi <2 x double> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT: [[B:%.*]] = sitofp <2 x i64> [[A]] to <2 x double>
+; CHECK-NEXT: ret <2 x double> [[B]]
+;
+ %a = fptosi <2 x double> %t to <2 x i64>
+ %b = sitofp <2 x i64> %a to <2 x double>
+ ret <2 x double> %b
+}
+
+; PR9228
+define <4 x float> @f(i32 %a) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: ret <4 x float> undef
+;
+ %dim = insertelement <4 x i32> poison, i32 %a, i32 0
+ %dim30 = insertelement <4 x i32> %dim, i32 %a, i32 1
+ %dim31 = insertelement <4 x i32> %dim30, i32 %a, i32 2
+ %dim32 = insertelement <4 x i32> %dim31, i32 %a, i32 3
+
+ %offset_ptr = getelementptr <4 x float>, <4 x float>* null, i32 1
+ %offset_int = ptrtoint <4 x float>* %offset_ptr to i64
+ %sizeof32 = trunc i64 %offset_int to i32
+
+ %smearinsert33 = insertelement <4 x i32> poison, i32 %sizeof32, i32 0
+ %smearinsert34 = insertelement <4 x i32> %smearinsert33, i32 %sizeof32, i32 1
+ %smearinsert35 = insertelement <4 x i32> %smearinsert34, i32 %sizeof32, i32 2
+ %smearinsert36 = insertelement <4 x i32> %smearinsert35, i32 %sizeof32, i32 3
+
+ %delta_scale = mul <4 x i32> %dim32, %smearinsert36
+ %offset_delta = add <4 x i32> zeroinitializer, %delta_scale
+
+ %offset_varying_delta = add <4 x i32> %offset_delta, undef
+
+ ret <4 x float> undef
+}
+
+define <8 x i32> @pr24458(<8 x float> %n) {
+; CHECK-LABEL: @pr24458(
+; CHECK-NEXT: ret <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+;
+ %notequal_b_load_.i = fcmp une <8 x float> %n, zeroinitializer
+ %equal_a_load72_.i = fcmp ueq <8 x float> %n, zeroinitializer
+ %notequal_b_load__to_boolvec.i = sext <8 x i1> %notequal_b_load_.i to <8 x i32>
+ %equal_a_load72__to_boolvec.i = sext <8 x i1> %equal_a_load72_.i to <8 x i32>
+ %wrong = or <8 x i32> %notequal_b_load__to_boolvec.i, %equal_a_load72__to_boolvec.i
+ ret <8 x i32> %wrong
+}
+
+; Hoist a trunc to a scalar if we're inserting into an undef vector.
+; trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index
+
+define <3 x i16> @trunc_inselt_undef(i32 %x) {
+; CHECK-LABEL: @trunc_inselt_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
+; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <3 x i16> undef, i16 [[TMP1]], i32 1
+; CHECK-NEXT: ret <3 x i16> [[TRUNC]]
+;
+ %vec = insertelement <3 x i32> poison, i32 %x, i32 1
+ %trunc = trunc <3 x i32> %vec to <3 x i16>
+ ret <3 x i16> %trunc
+}
+
+; Hoist a trunc to a scalar if we're inserting into an undef vector.
+; trunc (inselt undef, X, Index) --> inselt undef, (trunc X), Index
+
+define <2 x float> @fptrunc_inselt_undef(double %x, i32 %index) {
+; CHECK-LABEL: @fptrunc_inselt_undef(
+; CHECK-NEXT: [[TMP1:%.*]] = fptrunc double [[X:%.*]] to float
+; CHECK-NEXT: [[TRUNC:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 [[INDEX:%.*]]
+; CHECK-NEXT: ret <2 x float> [[TRUNC]]
+;
+ %vec = insertelement <2 x double> <double undef, double undef>, double %x, i32 %index
+ %trunc = fptrunc <2 x double> %vec to <2 x float>
+ ret <2 x float> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar int into a constant vector and truncate:
+; trunc (inselt C, X, Index) --> inselt C, (trunc X), Index
+
+define <3 x i16> @trunc_inselt1(i32 %x) {
+; CHECK-LABEL: @trunc_inselt1(
+; CHECK-NEXT: [[VEC:%.*]] = insertelement <3 x i32> <i32 3, i32 undef, i32 65536>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc <3 x i32> [[VEC]] to <3 x i16>
+; CHECK-NEXT: ret <3 x i16> [[TRUNC]]
+;
+ %vec = insertelement <3 x i32> <i32 3, i32 -2, i32 65536>, i32 %x, i32 1
+ %trunc = trunc <3 x i32> %vec to <3 x i16>
+ ret <3 x i16> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar FP into a constant vector and FP truncate:
+; fptrunc (inselt C, X, Index) --> inselt C, (fptrunc X), Index
+
+define <2 x float> @fptrunc_inselt1(double %x, i32 %index) {
+; CHECK-LABEL: @fptrunc_inselt1(
+; CHECK-NEXT: [[VEC:%.*]] = insertelement <2 x double> <double undef, double 3.000000e+00>, double [[X:%.*]], i32 [[INDEX:%.*]]
+; CHECK-NEXT: [[TRUNC:%.*]] = fptrunc <2 x double> [[VEC]] to <2 x float>
+; CHECK-NEXT: ret <2 x float> [[TRUNC]]
+;
+ %vec = insertelement <2 x double> <double undef, double 3.0>, double %x, i32 %index
+ %trunc = fptrunc <2 x double> %vec to <2 x float>
+ ret <2 x float> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar int constant into a vector and truncate:
+; trunc (inselt X, C, Index) --> inselt (trunc X), C', Index
+
+define <8 x i16> @trunc_inselt2(<8 x i32> %x, i32 %index) {
+; CHECK-LABEL: @trunc_inselt2(
+; CHECK-NEXT: [[VEC:%.*]] = insertelement <8 x i32> [[X:%.*]], i32 1048576, i32 [[INDEX:%.*]]
+; CHECK-NEXT: [[TRUNC:%.*]] = trunc <8 x i32> [[VEC]] to <8 x i16>
+; CHECK-NEXT: ret <8 x i16> [[TRUNC]]
+;
+ %vec = insertelement <8 x i32> %x, i32 1048576, i32 %index
+ %trunc = trunc <8 x i32> %vec to <8 x i16>
+ ret <8 x i16> %trunc
+}
+
+; TODO: Strengthen the backend, so we can have this canonicalization.
+; Insert a scalar FP constant into a vector and FP truncate:
+; fptrunc (inselt X, C, Index) --> inselt (fptrunc X), C', Index
+
+define <3 x float> @fptrunc_inselt2(<3 x double> %x) {
+; CHECK-LABEL: @fptrunc_inselt2(
+; CHECK-NEXT: [[VEC:%.*]] = insertelement <3 x double> [[X:%.*]], double 4.000000e+00, i32 2
+; CHECK-NEXT: [[TRUNC:%.*]] = fptrunc <3 x double> [[VEC]] to <3 x float>
+; CHECK-NEXT: ret <3 x float> [[TRUNC]]
+;
+ %vec = insertelement <3 x double> %x, double 4.0, i32 2
+ %trunc = fptrunc <3 x double> %vec to <3 x float>
+ ret <3 x float> %trunc
+}
+
+; Converting to a wide type might reduce instruction count,
+; but we can not do that unless the backend can recover from
+; the creation of a potentially illegal op (like a 64-bit vmul).
+; PR40032 - https://bugs.llvm.org/show_bug.cgi?id=40032
+
+define <2 x i64> @sext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @sext_less_casting_with_wideop(
+; CHECK-NEXT: [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT: [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT: [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT: [[R:%.*]] = sext <2 x i32> [[MUL]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %xnarrow = trunc <2 x i64> %x to <2 x i32>
+ %ynarrow = trunc <2 x i64> %y to <2 x i32>
+ %mul = mul <2 x i32> %xnarrow, %ynarrow
+ %r = sext <2 x i32> %mul to <2 x i64>
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @zext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: @zext_less_casting_with_wideop(
+; CHECK-NEXT: [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT: [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT: [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT: [[R:%.*]] = zext <2 x i32> [[MUL]] to <2 x i64>
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %xnarrow = trunc <2 x i64> %x to <2 x i32>
+ %ynarrow = trunc <2 x i64> %y to <2 x i32>
+ %mul = mul <2 x i32> %xnarrow, %ynarrow
+ %r = zext <2 x i32> %mul to <2 x i64>
+ ret <2 x i64> %r
+}
+
diff --git a/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
new file mode 100644
index 000000000000..29c4471183d3
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector_gep1-inseltpoison.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at G1 = global i8 zeroinitializer
+
+define <2 x i1> @test(<2 x i8*> %a, <2 x i8*> %b) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: [[C:%.*]] = icmp eq <2 x i8*> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: ret <2 x i1> [[C]]
+;
+ %c = icmp eq <2 x i8*> %a, %b
+ ret <2 x i1> %c
+}
+
+define <2 x i1> @test2(<2 x i8*> %a) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret <2 x i1> zeroinitializer
+;
+ %c = inttoptr <2 x i32> <i32 1, i32 2> to <2 x i8*>
+ %d = icmp ult <2 x i8*> %c, zeroinitializer
+ ret <2 x i1> %d
+}
+
+define <2 x i1> @test3(<2 x i8*> %a) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: ret <2 x i1> zeroinitializer
+;
+ %g = getelementptr i8, <2 x i8*> %a, <2 x i32> <i32 1, i32 0>
+ %B = icmp ult <2 x i8*> %g, zeroinitializer
+ ret <2 x i1> %B
+}
+
+define <1 x i1> @test4(<1 x i8*> %a) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: ret <1 x i1> zeroinitializer
+;
+ %g = getelementptr i8, <1 x i8*> %a, <1 x i32> <i32 1>
+ %B = icmp ult <1 x i8*> %g, zeroinitializer
+ ret <1 x i1> %B
+}
+
+define <2 x i1> @test5(<2 x i8*> %a) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: ret <2 x i1> zeroinitializer
+;
+ %w = getelementptr i8, <2 x i8*> %a, <2 x i32> zeroinitializer
+ %e = getelementptr i8, <2 x i8*> %w, <2 x i32> <i32 5, i32 9>
+ %g = getelementptr i8, <2 x i8*> %e, <2 x i32> <i32 1, i32 0>
+ %B = icmp ult <2 x i8*> %g, zeroinitializer
+ ret <2 x i1> %B
+}
+
+define <2 x i32*> @test7(<2 x {i32, i32}*> %a) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: [[W:%.*]] = getelementptr { i32, i32 }, <2 x { i32, i32 }*> [[A:%.*]], <2 x i64> <i64 5, i64 9>, <2 x i32> zeroinitializer
+; CHECK-NEXT: ret <2 x i32*> [[W]]
+;
+ %w = getelementptr {i32, i32}, <2 x {i32, i32}*> %a, <2 x i32> <i32 5, i32 9>, <2 x i32> zeroinitializer
+ ret <2 x i32*> %w
+}
+
+define <vscale x 2 x i1> @test8() {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT: ret <vscale x 2 x i1> icmp ult (<vscale x 2 x i64> zext (<vscale x 2 x i32> shufflevector (<vscale x 2 x i32> insertelement (<vscale x 2 x i32> poison, i32 1, i32 0), <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer) to <vscale x 2 x i64>), <vscale x 2 x i64> zeroinitializer)
+;
+ %ins = insertelement <vscale x 2 x i32> poison, i32 1, i32 0
+ %b = shufflevector <vscale x 2 x i32> %ins, <vscale x 2 x i32> undef, <vscale x 2 x i32> zeroinitializer
+ %c = inttoptr <vscale x 2 x i32> %b to <vscale x 2 x i8*>
+ %d = icmp ult <vscale x 2 x i8*> %c, zeroinitializer
+ ret <vscale x 2 x i1> %d
+}
diff --git a/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle-inseltpoison.ll
new file mode 100644
index 000000000000..36ca9eda1d1c
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vector_insertelt_shuffle-inseltpoison.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; insertelements should fold to shuffle
+define <4 x float> @foo(<4 x float> %x) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: [[INS2:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: ret <4 x float> [[INS2]]
+;
+ %ins1 = insertelement<4 x float> %x, float 1.0, i32 1
+ %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+ ret <4 x float> %ins2
+}
+
+; Insert of a constant is canonicalized ahead of insert of a variable.
+
+define <4 x float> @bar(<4 x float> %x, float %a) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x float> [[X:%.*]], float 2.000000e+00, i32 2
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x float> [[TMP1]], float [[A:%.*]], i32 1
+; CHECK-NEXT: ret <4 x float> [[INS2]]
+;
+ %ins1 = insertelement<4 x float> %x, float %a, i32 1
+ %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+ ret <4 x float> %ins2
+}
+
+define <4 x float> @baz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @baz(
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> [[X:%.*]], float 1.000000e+00, i32 1
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float 2.000000e+00, i32 [[A:%.*]]
+; CHECK-NEXT: ret <4 x float> [[INS2]]
+;
+ %ins1 = insertelement<4 x float> %x, float 1.0, i32 1
+ %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 %a
+ ret <4 x float> %ins2
+}
+
+; insertelements should fold to shuffle
+define <4 x float> @bazz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @bazz(
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x float> [[X:%.*]], float 1.000000e+00, i32 3
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x float> [[INS1]], float 5.000000e+00, i32 [[A:%.*]]
+; CHECK-NEXT: [[INS5:%.*]] = shufflevector <4 x float> [[INS2]], <4 x float> <float undef, float 1.000000e+00, float 2.000000e+00, float undef>, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[INS6:%.*]] = insertelement <4 x float> [[INS5]], float 7.000000e+00, i32 [[A]]
+; CHECK-NEXT: ret <4 x float> [[INS6]]
+;
+ %ins1 = insertelement<4 x float> %x, float 1.0, i32 3
+ %ins2 = insertelement<4 x float> %ins1, float 5.0, i32 %a
+ %ins3 = insertelement<4 x float> %ins2, float 3.0, i32 2
+ %ins4 = insertelement<4 x float> %ins3, float 1.0, i32 1
+ %ins5 = insertelement<4 x float> %ins4, float 2.0, i32 2
+ %ins6 = insertelement<4 x float> %ins5, float 7.0, i32 %a
+ ret <4 x float> %ins6
+}
+
+; Out of bounds index folds to undef
+define <4 x float> @bazzz(<4 x float> %x) {
+; CHECK-LABEL: @bazzz(
+; CHECK-NEXT: ret <4 x float> <float undef, float undef, float 2.000000e+00, float undef>
+;
+ %ins1 = insertelement<4 x float> %x, float 1.0, i32 5
+ %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+ ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzz(<4 x float> %x) {
+; CHECK-LABEL: @bazzzz(
+; CHECK-NEXT: ret <4 x float> <float undef, float undef, float 2.000000e+00, float undef>
+;
+ %ins1 = insertelement<4 x float> %x, float 1.0, i32 undef
+ %ins2 = insertelement<4 x float> %ins1, float 2.0, i32 2
+ ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzzz() {
+; CHECK-LABEL: @bazzzzz(
+; CHECK-NEXT: ret <4 x float> <float 1.000000e+00, float 5.000000e+00, float 1.000000e+01, float 4.000000e+00>
+;
+ %ins1 = insertelement <4 x float> insertelement (<4 x float> <float 1.0, float 2.0, float 3.0, float undef>, float 4.0, i32 3), float 5.0, i32 1
+ %ins2 = insertelement<4 x float> %ins1, float 10.0, i32 2
+ ret <4 x float> %ins2
+}
+
+define <4 x float> @bazzzzzz(<4 x float> %x, i32 %a) {
+; CHECK-LABEL: @bazzzzzz(
+; CHECK-NEXT: ret <4 x float> <float poison, float 5.000000e+00, float undef, float 4.000000e+00>
+;
+ %ins1 = insertelement <4 x float> insertelement (<4 x float> shufflevector (<4 x float> poison, <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0> , <4 x i32> <i32 0, i32 5, i32 undef, i32 6> ), float 4.0, i32 3), float 5.0, i32 1
+ ret <4 x float> %ins1
+}
+
+
diff --git a/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
new file mode 100644
index 000000000000..43dc22c20cd2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vscale_extractelement-inseltpoison.ll
@@ -0,0 +1,185 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+define i32 @extractelement_in_range(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_in_range(
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i64 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = extractelement <vscale x 4 x i32> %a, i64 1
+ ret i32 %r
+}
+
+define i32 @extractelement_maybe_out_of_range(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_maybe_out_of_range(
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i64 4
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = extractelement <vscale x 4 x i32> %a, i64 4
+ ret i32 %r
+}
+
+define i32 @extractelement_bitcast(float %f) {
+; CHECK-LABEL: @extractelement_bitcast(
+; CHECK-NEXT: [[R:%.*]] = bitcast float [[F:%.*]] to i32
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %vec_float = insertelement <vscale x 4 x float> poison, float %f, i32 0
+ %vec_int = bitcast <vscale x 4 x float> %vec_float to <vscale x 4 x i32>
+ %r = extractelement <vscale x 4 x i32> %vec_int, i32 0
+ ret i32 %r
+}
+
+define i8 @extractelement_bitcast_to_trunc(<vscale x 2 x i32> %a, i32 %x) {
+; CHECK-LABEL: @extractelement_bitcast_to_trunc(
+; CHECK-NEXT: [[R:%.*]] = trunc i32 [[X:%.*]] to i8
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %vec = insertelement <vscale x 2 x i32> %a, i32 %x, i32 1
+ %vec_cast = bitcast <vscale x 2 x i32> %vec to <vscale x 8 x i8>
+ %r = extractelement <vscale x 8 x i8> %vec_cast, i32 4
+ ret i8 %r
+}
+
+; TODO: Instcombine could remove the insert.
+define i8 @extractelement_bitcast_wrong_insert(<vscale x 2 x i32> %a, i32 %x) {
+; CHECK-LABEL: @extractelement_bitcast_wrong_insert(
+; CHECK-NEXT: [[VEC:%.*]] = insertelement <vscale x 2 x i32> [[A:%.*]], i32 [[X:%.*]], i32 1
+; CHECK-NEXT: [[VEC_CAST:%.*]] = bitcast <vscale x 2 x i32> [[VEC]] to <vscale x 8 x i8>
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 8 x i8> [[VEC_CAST]], i32 2
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %vec = insertelement <vscale x 2 x i32> %a, i32 %x, i32 1 ; <- This insert could be removed.
+ %vec_cast = bitcast <vscale x 2 x i32> %vec to <vscale x 8 x i8>
+ %r = extractelement <vscale x 8 x i8> %vec_cast, i32 2
+ ret i8 %r
+}
+
+; TODO: Instcombine could optimize to return %v.
+define i32 @extractelement_shuffle_in_range(i32 %v) {
+; CHECK-LABEL: @extractelement_shuffle_in_range(
+; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i32 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
+ %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %r = extractelement <vscale x 4 x i32> %splat, i32 1
+ ret i32 %r
+}
+
+define i32 @extractelement_shuffle_maybe_out_of_range(i32 %v) {
+; CHECK-LABEL: @extractelement_shuffle_maybe_out_of_range(
+; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i32 4
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
+ %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %r = extractelement <vscale x 4 x i32> %splat, i32 4
+ ret i32 %r
+}
+
+define i32 @extractelement_shuffle_invalid_index(i32 %v) {
+; CHECK-LABEL: @extractelement_shuffle_invalid_index(
+; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i32 -1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
+ %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %r = extractelement <vscale x 4 x i32> %splat, i32 -1
+ ret i32 %r
+}
+
+
+define i32 @extractelement_shuffle_symbolic_index(i32 %v, i32 %idx) {
+; CHECK-LABEL: @extractelement_shuffle_symbolic_index(
+; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[SPLAT]], i32 [[IDX:%.*]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
+ %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %r = extractelement <vscale x 4 x i32> %splat, i32 %idx
+ ret i32 %r
+}
+
+define <vscale x 4 x i32> @extractelement_insertelement_same_positions(<vscale x 4 x i32> %vec) {
+; CHECK-LABEL: @extractelement_insertelement_same_positions(
+; CHECK-NEXT: ret <vscale x 4 x i32> [[VEC:%.*]]
+;
+ %vec.e0 = extractelement <vscale x 4 x i32> %vec, i32 0
+ %vec.e1 = extractelement <vscale x 4 x i32> %vec, i32 1
+ %vec.e2 = extractelement <vscale x 4 x i32> %vec, i32 2
+ %vec.e3 = extractelement <vscale x 4 x i32> %vec, i32 3
+ %1 = insertelement <vscale x 4 x i32> %vec, i32 %vec.e0, i32 0
+ %2 = insertelement <vscale x 4 x i32> %1, i32 %vec.e1, i32 1
+ %3 = insertelement <vscale x 4 x i32> %2, i32 %vec.e2, i32 2
+ %4 = insertelement <vscale x 4 x i32> %3, i32 %vec.e3, i32 3
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @extractelement_insertelement_
diff _positions(<vscale x 4 x i32> %vec) {
+; CHECK-LABEL: @extractelement_insertelement_
diff _positions(
+; CHECK-NEXT: [[VEC_E0:%.*]] = extractelement <vscale x 4 x i32> [[VEC:%.*]], i32 4
+; CHECK-NEXT: [[VEC_E1:%.*]] = extractelement <vscale x 4 x i32> [[VEC]], i32 5
+; CHECK-NEXT: [[VEC_E2:%.*]] = extractelement <vscale x 4 x i32> [[VEC]], i32 6
+; CHECK-NEXT: [[VEC_E3:%.*]] = extractelement <vscale x 4 x i32> [[VEC]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 4 x i32> [[VEC]], i32 [[VEC_E0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <vscale x 4 x i32> [[TMP1]], i32 [[VEC_E1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <vscale x 4 x i32> [[TMP2]], i32 [[VEC_E2]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <vscale x 4 x i32> [[TMP3]], i32 [[VEC_E3]], i32 3
+; CHECK-NEXT: ret <vscale x 4 x i32> [[TMP4]]
+;
+ %vec.e0 = extractelement <vscale x 4 x i32> %vec, i32 4
+ %vec.e1 = extractelement <vscale x 4 x i32> %vec, i32 5
+ %vec.e2 = extractelement <vscale x 4 x i32> %vec, i32 6
+ %vec.e3 = extractelement <vscale x 4 x i32> %vec, i32 7
+ %1 = insertelement <vscale x 4 x i32> %vec, i32 %vec.e0, i32 0
+ %2 = insertelement <vscale x 4 x i32> %1, i32 %vec.e1, i32 1
+ %3 = insertelement <vscale x 4 x i32> %2, i32 %vec.e2, i32 2
+ %4 = insertelement <vscale x 4 x i32> %3, i32 %vec.e3, i32 3
+ ret <vscale x 4 x i32> %4
+}
+
+define i32 @bitcast_of_extractelement( <vscale x 2 x float> %d) {
+; CHECK-LABEL: @bitcast_of_extractelement(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <vscale x 2 x float> [[D:%.*]] to <vscale x 2 x i32>
+; CHECK-NEXT: [[CAST:%.*]] = extractelement <vscale x 2 x i32> [[BC]], i32 0
+; CHECK-NEXT: ret i32 [[CAST]]
+;
+ %ext = extractelement <vscale x 2 x float> %d, i32 0
+ %cast = bitcast float %ext to i32
+ ret i32 %cast
+}
+
+define i1 @extractelement_is_zero(<vscale x 2 x i32> %d, i1 %b, i32 %z) {
+; CHECK-LABEL: @extractelement_is_zero(
+; CHECK-NEXT: [[EXT:%.*]] = extractelement <vscale x 2 x i32> [[D:%.*]], i32 0
+; CHECK-NEXT: [[BB:%.*]] = icmp eq i32 [[EXT]], 0
+; CHECK-NEXT: ret i1 [[BB]]
+;
+ %ext = extractelement <vscale x 2 x i32> %d, i32 0
+ %bb = icmp eq i32 %ext, 0
+ ret i1 %bb
+}
+
+; OSS-Fuzz #25272
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=25272
+define i32 @ossfuzz_25272(float %f) {
+; CHECK-LABEL: @ossfuzz_25272(
+; CHECK-NEXT: [[VEC_FLOAT:%.*]] = insertelement <vscale x 4 x float> poison, float [[F:%.*]], i32 0
+; CHECK-NEXT: [[VEC_INT:%.*]] = bitcast <vscale x 4 x float> [[VEC_FLOAT]] to <vscale x 4 x i32>
+; CHECK-NEXT: [[E:%.*]] = extractelement <vscale x 4 x i32> [[VEC_INT]], i32 2147483647
+; CHECK-NEXT: ret i32 [[E]]
+;
+ %vec_float = insertelement <vscale x 4 x float> poison, float %f, i32 0
+ %vec_int = bitcast <vscale x 4 x float> %vec_float to <vscale x 4 x i32>
+ %E = extractelement <vscale x 4 x i32> %vec_int, i32 2147483647
+ ret i32 %E
+}
diff --git a/llvm/test/Transforms/InstCombine/vscale_insertelement-inseltpoison.ll b/llvm/test/Transforms/InstCombine/vscale_insertelement-inseltpoison.ll
new file mode 100644
index 000000000000..252483664f9b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/vscale_insertelement-inseltpoison.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+; This test checks that bitcast is moved after insertelement when both vector and scalar are
+; bitcast from the same element type.
+; inselt (bitcast VecSrc), (bitcast ScalarSrc), IdxOp
+; --> bitcast (inselt VecSrc, ScalarSrc, IdxOp)
+define <vscale x 4 x float> @insertelement_bitcast(<vscale x 4 x i32> %vec, i32 %x) {
+; CHECK-LABEL: @insertelement_bitcast(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <vscale x 4 x i32> [[VEC:%.*]], i32 [[X:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = bitcast <vscale x 4 x i32> [[TMP1]] to <vscale x 4 x float>
+; CHECK-NEXT: ret <vscale x 4 x float> [[R]]
+;
+ %x_cast = bitcast i32 %x to float
+ %vec_cast = bitcast <vscale x 4 x i32> %vec to <vscale x 4 x float>
+ %r = insertelement <vscale x 4 x float> %vec_cast, float %x_cast, i32 0
+ ret <vscale x 4 x float> %r
+}
+
+; This test checks that code-path "Try to form a shuffle from a chain of extract-insert ops" is
+; not taken when both extract and insert are scalable type.
+; For scalable type, the vector length needed to create shuffle mask is not a compile-time constant.
+; Meanwhile, for scalable type shuffle mask only support splat and undef in the current code base.
+; Otherwise we crash at:
+; "Assertion `isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"' failed."
+define <vscale x 4 x i32> @insertelement_extractelement(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: @insertelement_extractelement(
+; CHECK-NEXT: [[T0:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT: [[T1:%.*]] = insertelement <vscale x 4 x i32> [[B:%.*]], i32 [[T0]], i32 0
+; CHECK-NEXT: ret <vscale x 4 x i32> [[T1]]
+;
+ %t0 = extractelement <vscale x 4 x i32> %a, i32 1
+ %t1 = insertelement <vscale x 4 x i32> %b, i32 %t0, i32 0
+ ret <vscale x 4 x i32> %t1
+}
+
+; This test checks that we are not attempting to create a shuffle from extract/insert chain,
+; when extract is from a scalable type, and the insert vector is fixed-length.
+define <4 x i32> @insertelement_extractelement_fixed_vec_extract_from_scalable(<vscale x 4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @insertelement_extractelement_fixed_vec_extract_from_scalable(
+; CHECK-NEXT: [[T0:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT: [[T1:%.*]] = insertelement <4 x i32> [[B:%.*]], i32 [[T0]], i32 0
+; CHECK-NEXT: ret <4 x i32> [[T1]]
+;
+ %t0 = extractelement <vscale x 4 x i32> %a, i32 1
+ %t1 = insertelement <4 x i32> %b, i32 %t0, i32 0
+ ret <4 x i32> %t1
+}
+
+; This test checks that the optimization "foldConstantInsEltInfoShuffle" is not taken for scalable type.
+; Particularly the fold:
+; insertelt (insertelt X, C1, CIndex1), C, CIndex
+; --> shufflevector X, CVec', Mask'
+; For scalable type, the vector length needed to create shuffle mask is not a compile-time constant.
+; Meanwhile, for scalable type shuffle mask only support splat and undef in the current code base.
+; Otherwise we crash at:
+; "Assertion `isValidOperands(V1, V2, Mask) && "Invalid shuffle vector instruction operands!"' failed."
+define <vscale x 4 x i32> @insertelement_insertelement(<vscale x 4 x i32> %vec) {
+; CHECK-LABEL: @insertelement_insertelement(
+; CHECK-NEXT: [[T0:%.*]] = insertelement <vscale x 4 x i32> [[VEC:%.*]], i32 1, i32 1
+; CHECK-NEXT: [[T1:%.*]] = insertelement <vscale x 4 x i32> [[T0]], i32 2, i32 2
+; CHECK-NEXT: ret <vscale x 4 x i32> [[T1]]
+;
+ %t0 = insertelement <vscale x 4 x i32> %vec, i32 1, i32 1
+ %t1 = insertelement <vscale x 4 x i32> %t0, i32 2, i32 2
+ ret <vscale x 4 x i32> %t1
+}
+
+; This test checks that the following insertelement sequence is not folded into shuffle splat.
+; The length of scalable vector is unknown at compile-time. Therefore the following insertelements
+; may not form a valid splat.
+define <vscale x 4 x float> @insertelement_sequene_may_not_be_splat(float %x) {
+; CHECK-LABEL: @insertelement_sequene_may_not_be_splat(
+; CHECK-NEXT: [[T0:%.*]] = insertelement <vscale x 4 x float> poison, float [[X:%.*]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <vscale x 4 x float> [[T0]], float [[X]], i32 1
+; CHECK-NEXT: [[T2:%.*]] = insertelement <vscale x 4 x float> [[T1]], float [[X]], i32 2
+; CHECK-NEXT: [[T3:%.*]] = insertelement <vscale x 4 x float> [[T2]], float [[X]], i32 3
+; CHECK-NEXT: ret <vscale x 4 x float> [[T3]]
+;
+ %t0 = insertelement <vscale x 4 x float> poison, float %x, i32 0
+ %t1 = insertelement <vscale x 4 x float> %t0, float %x, i32 1
+ %t2 = insertelement <vscale x 4 x float> %t1, float %x, i32 2
+ %t3 = insertelement <vscale x 4 x float> %t2, float %x, i32 3
+ ret <vscale x 4 x float> %t3
+}
+
+; OSS-Fuzz #27416
+; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=27416
+define void @ossfuzz_27416(i32 %v) {
+; CHECK-LABEL: @ossfuzz_27416(
+; CHECK-NEXT: [[IN:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i32 0
+; CHECK-NEXT: [[SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[IN]], <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT: [[I1:%.*]] = insertelement <vscale x 4 x i32> [[SPLAT]], i32 undef, i8 -128
+; CHECK-NEXT: store <vscale x 4 x i32> [[I1]], <vscale x 4 x i32>* undef, align 16
+; CHECK-NEXT: ret void
+;
+ %in = insertelement <vscale x 4 x i32> poison, i32 %v, i32 0
+ %splat = shufflevector <vscale x 4 x i32> %in, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %I1 = insertelement <vscale x 4 x i32> %splat, i32 undef, i8 -128
+ store <vscale x 4 x i32> %I1, <vscale x 4 x i32>* undef, align 16
+ ret void
+}
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/InsertElement-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/ConstProp/InsertElement-inseltpoison.ll
new file mode 100644
index 000000000000..54b862c8514a
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/InsertElement-inseltpoison.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i32 @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret i32 2139171423
+;
+ %A = bitcast i32 2139171423 to float
+ %B = insertelement <1 x float> poison, float %A, i32 0
+ %C = extractelement <1 x float> %B, i32 0
+ %D = bitcast float %C to i32
+ ret i32 %D
+}
+
+define <4 x i64> @insertelement() {
+; CHECK-LABEL: @insertelement(
+; CHECK-NEXT: ret <4 x i64> <i64 -1, i64 -2, i64 -3, i64 -4>
+;
+ %vec1 = insertelement <4 x i64> poison, i64 -1, i32 0
+ %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+ %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+ %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+ ret <4 x i64> %vec4
+}
+
+define <4 x i64> @insertelement_undef() {
+; CHECK-LABEL: @insertelement_undef(
+; CHECK-NEXT: ret <4 x i64> poison
+;
+ %vec1 = insertelement <4 x i64> poison, i64 -1, i32 0
+ %vec2 = insertelement <4 x i64> %vec1, i64 -2, i32 1
+ %vec3 = insertelement <4 x i64> %vec2, i64 -3, i32 2
+ %vec4 = insertelement <4 x i64> %vec3, i64 -4, i32 3
+ %vec5 = insertelement <4 x i64> %vec3, i64 -5, i32 4
+ ret <4 x i64> %vec5
+}
+
+define i64 @extract_undef_index_from_zero_vec() {
+; CHECK-LABEL: @extract_undef_index_from_zero_vec(
+; CHECK-NEXT: ret i64 poison
+;
+ %E = extractelement <2 x i64> zeroinitializer, i64 undef
+ ret i64 %E
+}
+
+define i64 @extract_undef_index_from_nonzero_vec() {
+; CHECK-LABEL: @extract_undef_index_from_nonzero_vec(
+; CHECK-NEXT: ret i64 poison
+;
+ %E = extractelement <2 x i64> <i64 -1, i64 -1>, i64 undef
+ ret i64 %E
+}
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll
new file mode 100644
index 000000000000..e91d943c5eb7
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll
@@ -0,0 +1,301 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S -verify | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Unary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x double> @fneg(<vscale x 2 x double> %val) {
+; CHECK-LABEL: @fneg(
+; CHECK-NEXT: ret <vscale x 2 x double> undef
+;
+ %r = fneg <vscale x 2 x double> undef
+ ret <vscale x 2 x double> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Binary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @add() {
+; CHECK-LABEL: @add(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = add <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fadd() {
+; CHECK-LABEL: @fadd(
+; CHECK-NEXT: ret <vscale x 4 x float> undef
+;
+ %r = fadd <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @sub() {
+; CHECK-LABEL: @sub(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = sub <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @sub_splat() {
+; CHECK-LABEL: @sub_splat(
+; CHECK-NEXT: ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 -16, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %r = sub <vscale x 4 x i32> zeroinitializer, shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 16, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fsub() {
+; CHECK-LABEL: @fsub(
+; CHECK-NEXT: ret <vscale x 4 x float> undef
+;
+ %r = fsub <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @mul() {
+; CHECK-LABEL: @mul(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = mul <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fmul() {
+; CHECK-LABEL: @fmul(
+; CHECK-NEXT: ret <vscale x 4 x float> undef
+;
+ %r = fmul <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @udiv() {
+; CHECK-LABEL: @udiv(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = udiv <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @udiv_splat_zero() {
+; CHECK-LABEL: @udiv_splat_zero(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = udiv <vscale x 4 x i32> zeroinitializer, zeroinitializer
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @sdiv() {
+; CHECK-LABEL: @sdiv(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = sdiv <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @fdiv() {
+; CHECK-LABEL: @fdiv(
+; CHECK-NEXT: ret <vscale x 4 x float> undef
+;
+ %r = fdiv <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x float> %r
+}
+
+define <vscale x 4 x i32> @urem() {
+; CHECK-LABEL: @urem(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = urem <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @srem() {
+; CHECK-LABEL: @srem(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = srem <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x float> @frem() {
+; CHECK-LABEL: @frem(
+; CHECK-NEXT: ret <vscale x 4 x float> undef
+;
+ %r = frem <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x float> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Bitwise Binary Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @shl() {
+; CHECK-LABEL: @shl(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = shl <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @lshr() {
+; CHECK-LABEL: @lshr(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = lshr <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @ashr() {
+; CHECK-LABEL: @ashr(
+; CHECK-NEXT: ret <vscale x 4 x i32> poison
+;
+ %r = ashr <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @and() {
+; CHECK-LABEL: @and(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = and <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @or() {
+; CHECK-LABEL: @or(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = or <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @xor() {
+; CHECK-LABEL: @xor(
+; CHECK-NEXT: ret <vscale x 4 x i32> zeroinitializer
+;
+ %r = xor <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i32> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Vector Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @insertelement() {
+; CHECK-LABEL: @insertelement(
+; CHECK-NEXT: ret <vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0)
+;
+ %i = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ ret <vscale x 4 x i32> %i
+}
+
+define <vscale x 4 x i32> @shufflevector() {
+; CHECK-LABEL: @shufflevector(
+; CHECK-NEXT: ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %i = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ %i2 = shufflevector <vscale x 4 x i32> %i, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ ret <vscale x 4 x i32> %i2
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Memory Access and Addressing Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 2 x double> @load() {
+; CHECK-LABEL: @load(
+; CHECK-NEXT: [[R:%.*]] = load <vscale x 2 x double>, <vscale x 2 x double>* getelementptr (<vscale x 2 x double>, <vscale x 2 x double>* null, i64 1), align 16
+; CHECK-NEXT: ret <vscale x 2 x double> [[R]]
+;
+ %r = load <vscale x 2 x double>, <vscale x 2 x double>* getelementptr (<vscale x 2 x double>, <vscale x 2 x double>* null, i64 1)
+ ret <vscale x 2 x double> %r
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Conversion Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x float> @bitcast() {
+; CHECK-LABEL: @bitcast(
+; CHECK-NEXT: ret <vscale x 4 x float> bitcast (<vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer) to <vscale x 4 x float>)
+;
+ %i1 = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ %i2 = shufflevector <vscale x 4 x i32> %i1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %i3 = bitcast <vscale x 4 x i32> %i2 to <vscale x 4 x float>
+ ret <vscale x 4 x float> %i3
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Other Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+define <vscale x 4 x i32> @select() {
+; CHECK-LABEL: @select(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = select <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer, <vscale x 4 x i32> undef
+ ret <vscale x 4 x i32> %r
+}
+
+declare <vscale x 16 x i8> @llvm.something(<vscale x 16 x i8>, <vscale x 16 x i8>)
+
+define <vscale x 16 x i8> @call() {
+; CHECK-LABEL: @call(
+; CHECK-NEXT: [[R:%.*]] = call <vscale x 16 x i8> @llvm.something(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+; CHECK-NEXT: ret <vscale x 16 x i8> [[R]]
+;
+ %r = call <vscale x 16 x i8> @llvm.something(<vscale x 16 x i8> undef, <vscale x 16 x i8> undef)
+ ret <vscale x 16 x i8> %r
+}
+
+define <vscale x 4 x i1> @icmp_undef() {
+; CHECK-LABEL: @icmp_undef(
+; CHECK-NEXT: ret <vscale x 4 x i1> undef
+;
+ %r = icmp eq <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @icmp_zero() {
+; CHECK-LABEL: @icmp_zero(
+; CHECK-NEXT: ret <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> undef, i1 true, i32 0), <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %r = icmp eq <vscale x 4 x i32> zeroinitializer, zeroinitializer
+ ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @fcmp_true() {
+; CHECK-LABEL: @fcmp_true(
+; CHECK-NEXT: ret <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> undef, i1 true, i32 0), <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %r = fcmp true <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @fcmp_false() {
+; CHECK-LABEL: @fcmp_false(
+; CHECK-NEXT: ret <vscale x 4 x i1> zeroinitializer
+;
+ %r = fcmp false <vscale x 4 x float> undef, undef
+ ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @fcmp_undef() {
+; CHECK-LABEL: @fcmp_undef(
+; CHECK-NEXT: ret <vscale x 4 x i1> undef
+;
+ %r = icmp ne <vscale x 4 x i32> undef, undef
+ ret <vscale x 4 x i1> %r
+}
+
+define <vscale x 4 x i1> @fcmp_not_equality() {
+; CHECK-LABEL: @fcmp_not_equality(
+; CHECK-NEXT: ret <vscale x 4 x i1> shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> undef, i1 true, i32 0), <vscale x 4 x i1> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %r = icmp ule <vscale x 4 x i32> undef, zeroinitializer
+ ret <vscale x 4 x i1> %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/ConstProp/vscale-shufflevector-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-shufflevector-inseltpoison.ll
new file mode 100644
index 000000000000..48ec29c95f61
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ConstProp/vscale-shufflevector-inseltpoison.ll
@@ -0,0 +1,39 @@
+; RUN: opt -early-cse -earlycse-debug-hash -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+; This test checks that SimplifyInstruction does not blow up in the face of
+; a scalable shufflevector. vscale is a constant value known only at runtime.
+; Therefore, it is not possible to know the concrete value of, or the length
+; of the mask at compile time. Simplifications that depend on the value
+; of the mask cannot be performed.
+
+; Given the fact that the value of the mask is unknown at compile time for
+; scalable vectors, very few simplifications will be done. Here, we want to
+; see that the instruction can be passed to SimplifyInstruction and not crash
+; the compiler. It happens to be the case that this will be the result.
+
+; CHECK-LABEL: define <vscale x 8 x i1> @vscale_version()
+; CHECK-NEXT: ret <vscale x 8 x i1> shufflevector (<vscale x 8 x i1> insertelement (<vscale x 8 x i1> poison, i1 true, i32 0), <vscale x 8 x i1> undef, <vscale x 8 x i32> zeroinitializer)
+
+define <vscale x 8 x i1> @vscale_version() {
+ %splatter = insertelement <vscale x 8 x i1> poison, i1 true, i32 0
+ %foo = shufflevector <vscale x 8 x i1> %splatter,
+ <vscale x 8 x i1> undef,
+ <vscale x 8 x i32> zeroinitializer
+ ret <vscale x 8 x i1> %foo
+}
+
+; The non-scalable version should be optimized as normal.
+
+; CHECK-LABEL: define <8 x i1> @fixed_length_version() {
+; CHECK-NEXT: ret <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
+define <8 x i1> @fixed_length_version() {
+ %splatter = insertelement <8 x i1> poison, i1 true, i32 0
+ %foo = shufflevector <8 x i1> %splatter,
+ <8 x i1> undef,
+ <8 x i32> zeroinitializer
+ ret <8 x i1> %foo
+}
+
diff --git a/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll
new file mode 100644
index 000000000000..a949740e8429
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/select-inseltpoison.ll
@@ -0,0 +1,1007 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+define i1 @bool_true_or_false(i1 %cond) {
+; CHECK-LABEL: @bool_true_or_false(
+; CHECK-NEXT: ret i1 [[COND:%.*]]
+;
+ %s = select i1 %cond, i1 true, i1 false
+ ret i1 %s
+}
+
+define <2 x i1> @bool_true_or_false_vec(<2 x i1> %cond) {
+; CHECK-LABEL: @bool_true_or_false_vec(
+; CHECK-NEXT: ret <2 x i1> [[COND:%.*]]
+;
+ %s = select <2 x i1> %cond, <2 x i1> <i1 true, i1 true>, <2 x i1> zeroinitializer
+ ret <2 x i1> %s
+}
+
+define <2 x i1> @bool_true_or_false_vec_undef(<2 x i1> %cond) {
+; CHECK-LABEL: @bool_true_or_false_vec_undef(
+; CHECK-NEXT: ret <2 x i1> [[COND:%.*]]
+;
+ %s = select <2 x i1> %cond, <2 x i1> <i1 undef, i1 true>, <2 x i1> <i1 false, i1 undef>
+ ret <2 x i1> %s
+}
+
+define i32 @cond_is_false(i32 %A, i32 %B) {
+; CHECK-LABEL: @cond_is_false(
+; CHECK-NEXT: ret i32 [[B:%.*]]
+;
+ %C = select i1 false, i32 %A, i32 %B
+ ret i32 %C
+}
+
+define i32 @cond_is_true(i32 %A, i32 %B) {
+; CHECK-LABEL: @cond_is_true(
+; CHECK-NEXT: ret i32 [[A:%.*]]
+;
+ %C = select i1 true, i32 %A, i32 %B
+ ret i32 %C
+}
+
+define i32 @equal_arms(i1 %cond, i32 %x) {
+; CHECK-LABEL: @equal_arms(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %V = select i1 %cond, i32 %x, i32 %x
+ ret i32 %V
+}
+
+define <2 x i32> @equal_arms_vec(<2 x i1> %cond, <2 x i32> %x) {
+; CHECK-LABEL: @equal_arms_vec(
+; CHECK-NEXT: ret <2 x i32> [[X:%.*]]
+;
+ %V = select <2 x i1> %cond, <2 x i32> %x, <2 x i32> %x
+ ret <2 x i32> %V
+}
+
+define <2 x i32> @equal_arms_vec_undef(<2 x i1> %cond) {
+; CHECK-LABEL: @equal_arms_vec_undef(
+; CHECK-NEXT: ret <2 x i32> <i32 42, i32 42>
+;
+ %V = select <2 x i1> %cond, <2 x i32> <i32 42, i32 undef>, <2 x i32> <i32 undef, i32 42>
+ ret <2 x i32> %V
+}
+
+define <3 x float> @equal_arms_vec_less_undef(<3 x i1> %cond) {
+; CHECK-LABEL: @equal_arms_vec_less_undef(
+; CHECK-NEXT: ret <3 x float> <float 4.200000e+01, float 4.200000e+01, float 4.300000e+01>
+;
+ %V = select <3 x i1> %cond, <3 x float> <float 42.0, float undef, float 43.0>, <3 x float> <float 42.0, float 42.0, float 43.0>
+ ret <3 x float> %V
+}
+
+define <3 x float> @equal_arms_vec_more_undef(<3 x i1> %cond) {
+; CHECK-LABEL: @equal_arms_vec_more_undef(
+; CHECK-NEXT: ret <3 x float> <float 4.200000e+01, float undef, float 4.300000e+01>
+;
+ %V = select <3 x i1> %cond, <3 x float> <float 42.0, float undef, float undef>, <3 x float> <float undef, float undef, float 43.0>
+ ret <3 x float> %V
+}
+
+define <2 x i8> @vsel_tvec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @vsel_tvec(
+; CHECK-NEXT: ret <2 x i8> [[X:%.*]]
+;
+ %s = select <2 x i1><i1 true, i1 true>, <2 x i8> %x, <2 x i8> %y
+ ret <2 x i8> %s
+}
+
+define <2 x i8> @vsel_fvec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @vsel_fvec(
+; CHECK-NEXT: ret <2 x i8> [[Y:%.*]]
+;
+ %s = select <2 x i1><i1 false, i1 false>, <2 x i8> %x, <2 x i8> %y
+ ret <2 x i8> %s
+}
+
+define <2 x i8> @vsel_mixedvec() {
+; CHECK-LABEL: @vsel_mixedvec(
+; CHECK-NEXT: ret <2 x i8> <i8 0, i8 3>
+;
+ %s = select <2 x i1><i1 true, i1 false>, <2 x i8> <i8 0, i8 1>, <2 x i8> <i8 2, i8 3>
+ ret <2 x i8> %s
+}
+
+; FIXME: Allow for undef elements in a constant vector condition.
+
+define <3 x i8> @vsel_undef_true_op(<3 x i8> %x, <3 x i8> %y) {
+; CHECK-LABEL: @vsel_undef_true_op(
+; CHECK-NEXT: [[S:%.*]] = select <3 x i1> <i1 true, i1 undef, i1 true>, <3 x i8> [[X:%.*]], <3 x i8> [[Y:%.*]]
+; CHECK-NEXT: ret <3 x i8> [[S]]
+;
+ %s = select <3 x i1><i1 1, i1 undef, i1 1>, <3 x i8> %x, <3 x i8> %y
+ ret <3 x i8> %s
+}
+
+define <3 x i4> @vsel_undef_false_op(<3 x i4> %x, <3 x i4> %y) {
+; CHECK-LABEL: @vsel_undef_false_op(
+; CHECK-NEXT: [[S:%.*]] = select <3 x i1> <i1 false, i1 undef, i1 undef>, <3 x i4> [[X:%.*]], <3 x i4> [[Y:%.*]]
+; CHECK-NEXT: ret <3 x i4> [[S]]
+;
+ %s = select <3 x i1><i1 0, i1 undef, i1 undef>, <3 x i4> %x, <3 x i4> %y
+ ret <3 x i4> %s
+}
+
+define i32 @test1(i32 %x) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 1
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i32 %x, -2
+ %and1.x = select i1 %cmp, i32 %and1, i32 %x
+ ret i32 %and1.x
+}
+
+define i32 @test2(i32 %x) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 1
+ %cmp = icmp ne i32 %and, 0
+ %and1 = and i32 %x, -2
+ %and1.x = select i1 %cmp, i32 %x, i32 %and1
+ ret i32 %and1.x
+}
+
+define i32 @test3(i32 %x) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[X:%.*]], -2
+; CHECK-NEXT: ret i32 [[AND1]]
+;
+ %and = and i32 %x, 1
+ %cmp = icmp ne i32 %and, 0
+ %and1 = and i32 %x, -2
+ %and1.x = select i1 %cmp, i32 %and1, i32 %x
+ ret i32 %and1.x
+}
+
+define i32 @test4(i32 %X) {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %cmp = icmp slt i32 %X, 0
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %X, i32 %or
+ ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test4noncanon(i32 %X) {
+; CHECK-LABEL: @test4noncanon(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %cmp = icmp sle i32 %X, -1
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %X, i32 %or
+ ret i32 %cond
+}
+
+define i32 @test5(i32 %X) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %cmp = icmp slt i32 %X, 0
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %or, i32 %X
+ ret i32 %cond
+}
+
+define i32 @test6(i32 %X) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp slt i32 %X, 0
+ %and = and i32 %X, 2147483647
+ %cond = select i1 %cmp, i32 %and, i32 %X
+ ret i32 %cond
+}
+
+define i32 @test7(i32 %X) {
+; CHECK-LABEL: @test7(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %cmp = icmp slt i32 %X, 0
+ %and = and i32 %X, 2147483647
+ %cond = select i1 %cmp, i32 %X, i32 %and
+ ret i32 %cond
+}
+
+define i32 @test8(i32 %X) {
+; CHECK-LABEL: @test8(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %cmp = icmp sgt i32 %X, -1
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %X, i32 %or
+ ret i32 %cond
+}
+
+define i32 @test9(i32 %X) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %cmp = icmp sgt i32 %X, -1
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %or, i32 %X
+ ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test9noncanon(i32 %X) {
+; CHECK-LABEL: @test9noncanon(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], -2147483648
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %cmp = icmp sge i32 %X, 0
+ %or = or i32 %X, -2147483648
+ %cond = select i1 %cmp, i32 %or, i32 %X
+ ret i32 %cond
+}
+
+define i32 @test10(i32 %X) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %cmp = icmp sgt i32 %X, -1
+ %and = and i32 %X, 2147483647
+ %cond = select i1 %cmp, i32 %and, i32 %X
+ ret i32 %cond
+}
+
+define i32 @test11(i32 %X) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 2147483647
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp sgt i32 %X, -1
+ %and = and i32 %X, 2147483647
+ %cond = select i1 %cmp, i32 %X, i32 %and
+ ret i32 %cond
+}
+
+define <2 x i8> @test11vec(<2 x i8> %X) {
+; CHECK-LABEL: @test11vec(
+; CHECK-NEXT: [[AND:%.*]] = and <2 x i8> [[X:%.*]], <i8 127, i8 127>
+; CHECK-NEXT: ret <2 x i8> [[AND]]
+;
+ %cmp = icmp sgt <2 x i8> %X, <i8 -1, i8 -1>
+ %and = and <2 x i8> %X, <i8 127, i8 127>
+ %sel = select <2 x i1> %cmp, <2 x i8> %X, <2 x i8> %and
+ ret <2 x i8> %sel
+}
+
+define i32 @test12(i32 %X) {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp ult i32 %X, 4
+ %and = and i32 %X, 3
+ %cond = select i1 %cmp, i32 %X, i32 %and
+ ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test12noncanon(i32 %X) {
+; CHECK-LABEL: @test12noncanon(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp ule i32 %X, 3
+ %and = and i32 %X, 3
+ %cond = select i1 %cmp, i32 %X, i32 %and
+ ret i32 %cond
+}
+
+define i32 @test13(i32 %X) {
+; CHECK-LABEL: @test13(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp ugt i32 %X, 3
+ %and = and i32 %X, 3
+ %cond = select i1 %cmp, i32 %and, i32 %X
+ ret i32 %cond
+}
+
+; Same as above, but the compare isn't canonical
+define i32 @test13noncanon(i32 %X) {
+; CHECK-LABEL: @test13noncanon(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %cmp = icmp uge i32 %X, 4
+ %and = and i32 %X, 3
+ %cond = select i1 %cmp, i32 %and, i32 %X
+ ret i32 %cond
+}
+
+define i32 @select_icmp_and_8_eq_0_or_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], 8
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %or = or i32 %x, 8
+ %sel = select i1 %cmp, i32 %or, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_or_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_or_8_alt(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], 8
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp ne i32 %and, 0
+ %or = or i32 %x, 8
+ %sel = select i1 %cmp, i32 %x, i32 %or
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_or_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp ne i32 %and, 0
+ %or = or i32 %x, 8
+ %sel = select i1 %cmp, i32 %or, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_or_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_or_8_alt(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %or = or i32 %x, 8
+ %sel = select i1 %cmp, i32 %x, i32 %or
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8(
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[X:%.*]], -9
+; CHECK-NEXT: ret i32 [[AND1]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %x, i32 %and1
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_eq_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_eq_0_and_not_8_alt(
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[X:%.*]], -9
+; CHECK-NEXT: ret i32 [[AND1]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp ne i32 %and, 0
+ %and1 = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %and1, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp ne i32 %and, 0
+ %and1 = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %x, i32 %and1
+ ret i32 %sel
+}
+
+define i32 @select_icmp_and_8_ne_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_alt(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %and1, i32 %x
+ ret i32 %sel
+}
+
+; PR28466: https://llvm.org/bugs/show_bug.cgi?id=28466
+; Each of the previous 8 patterns has a variant that replaces the
+; 'and' with a 'trunc' and the icmp eq/ne with icmp slt/sgt.
+
+define i32 @select_icmp_trunc_8_ne_0_or_128(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], 128
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %trunc = trunc i32 %x to i8
+ %cmp = icmp sgt i8 %trunc, -1
+ %or = or i32 %x, 128
+ %sel = select i1 %cmp, i32 %or, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_or_128_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_or_128_alt(
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[X:%.*]], 128
+; CHECK-NEXT: ret i32 [[OR]]
+;
+ %trunc = trunc i32 %x to i8
+ %cmp = icmp slt i8 %trunc, 0
+ %or = or i32 %x, 128
+ %sel = select i1 %cmp, i32 %x, i32 %or
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_or_128(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %trunc = trunc i32 %x to i8
+ %cmp = icmp slt i8 %trunc, 0
+ %or = or i32 %x, 128
+ %sel = select i1 %cmp, i32 %or, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_or_128_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_or_128_alt(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %trunc = trunc i32 %x to i8
+ %cmp = icmp sgt i8 %trunc, -1
+ %or = or i32 %x, 128
+ %sel = select i1 %cmp, i32 %x, i32 %or
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], -9
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %trunc = trunc i32 %x to i4
+ %cmp = icmp sgt i4 %trunc, -1
+ %and = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %x, i32 %and
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_eq_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_eq_0_and_not_8_alt(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], -9
+; CHECK-NEXT: ret i32 [[AND]]
+;
+ %trunc = trunc i32 %x to i4
+ %cmp = icmp slt i4 %trunc, 0
+ %and = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %and, i32 %x
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_and_not_8(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %trunc = trunc i32 %x to i4
+ %cmp = icmp slt i4 %trunc, 0
+ %and = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %x, i32 %and
+ ret i32 %sel
+}
+
+define i32 @select_icmp_trunc_8_ne_0_and_not_8_alt(i32 %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %trunc = trunc i32 %x to i4
+ %cmp = icmp sgt i4 %trunc, -1
+ %and = and i32 %x, -9
+ %sel = select i1 %cmp, i32 %and, i32 %x
+ ret i32 %sel
+}
+
+; Make sure that at least a few of the same patterns are repeated with vector types.
+
+define <2 x i32> @select_icmp_and_8_ne_0_and_not_8_vec(<2 x i32> %x) {
+; CHECK-LABEL: @select_icmp_and_8_ne_0_and_not_8_vec(
+; CHECK-NEXT: ret <2 x i32> [[X:%.*]]
+;
+ %and = and <2 x i32> %x, <i32 8, i32 8>
+ %cmp = icmp ne <2 x i32> %and, zeroinitializer
+ %and1 = and <2 x i32> %x, <i32 -9, i32 -9>
+ %sel = select <2 x i1> %cmp, <2 x i32> %x, <2 x i32> %and1
+ ret <2 x i32> %sel
+}
+
+define <2 x i32> @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(<2 x i32> %x) {
+; CHECK-LABEL: @select_icmp_trunc_8_ne_0_and_not_8_alt_vec(
+; CHECK-NEXT: ret <2 x i32> [[X:%.*]]
+;
+ %trunc = trunc <2 x i32> %x to <2 x i4>
+ %cmp = icmp sgt <2 x i4> %trunc, <i4 -1, i4 -1>
+ %and = and <2 x i32> %x, <i32 -9, i32 -9>
+ %sel = select <2 x i1> %cmp, <2 x i32> %and, <2 x i32> %x
+ ret <2 x i32> %sel
+}
+
+; Insert a bit from x into y? This should be possible in InstCombine, but not InstSimplify?
+
+define i32 @select_icmp_x_and_8_eq_0_y_and_not_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y_and_not_8(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT: [[AND1:%.*]] = and i32 [[Y:%.*]], -9
+; CHECK-NEXT: [[Y_AND1:%.*]] = select i1 [[CMP]], i32 [[Y]], i32 [[AND1]]
+; CHECK-NEXT: ret i32 [[Y_AND1]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i32 %y, -9
+ %y.and1 = select i1 %cmp, i32 %y, i32 %and1
+ ret i32 %y.and1
+}
+
+define i64 @select_icmp_x_and_8_eq_0_y64_and_not_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_eq_0_y64_and_not_8(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT: [[AND1:%.*]] = and i64 [[Y:%.*]], -9
+; CHECK-NEXT: [[Y_AND1:%.*]] = select i1 [[CMP]], i64 [[Y]], i64 [[AND1]]
+; CHECK-NEXT: ret i64 [[Y_AND1]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i64 %y, -9
+ %y.and1 = select i1 %cmp, i64 %y, i64 %and1
+ ret i64 %y.and1
+}
+
+define i64 @select_icmp_x_and_8_ne_0_y64_and_not_8(i32 %x, i64 %y) {
+; CHECK-LABEL: @select_icmp_x_and_8_ne_0_y64_and_not_8(
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[X:%.*]], 8
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT: [[AND1:%.*]] = and i64 [[Y:%.*]], -9
+; CHECK-NEXT: [[AND1_Y:%.*]] = select i1 [[CMP]], i64 [[AND1]], i64 [[Y]]
+; CHECK-NEXT: ret i64 [[AND1_Y]]
+;
+ %and = and i32 %x, 8
+ %cmp = icmp eq i32 %and, 0
+ %and1 = and i64 %y, -9
+ %and1.y = select i1 %cmp, i64 %and1, i64 %y
+ ret i64 %and1.y
+}
+
+; Don't crash on a pointer or aggregate type.
+
+define i32* @select_icmp_pointers(i32* %x, i32* %y) {
+; CHECK-LABEL: @select_icmp_pointers(
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32* [[X:%.*]], null
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i32* [[X]], i32* [[Y:%.*]]
+; CHECK-NEXT: ret i32* [[SEL]]
+;
+ %cmp = icmp slt i32* %x, null
+ %sel = select i1 %cmp, i32* %x, i32* %y
+ ret i32* %sel
+}
+
+; If the condition is known, we don't need to select, but we're not
+; doing this fold here to avoid compile-time cost.
+
+declare void @llvm.assume(i1)
+
+define i8 @assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @assume_sel_cond(
+; CHECK-NEXT: call void @llvm.assume(i1 [[COND:%.*]])
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X:%.*]], i8 [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[SEL]]
+;
+ call void @llvm.assume(i1 %cond)
+ %sel = select i1 %cond, i8 %x, i8 %y
+ ret i8 %sel
+}
+
+define i8 @do_not_assume_sel_cond(i1 %cond, i8 %x, i8 %y) {
+; CHECK-LABEL: @do_not_assume_sel_cond(
+; CHECK-NEXT: [[NOTCOND:%.*]] = icmp eq i1 [[COND:%.*]], false
+; CHECK-NEXT: call void @llvm.assume(i1 [[NOTCOND]])
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[COND]], i8 [[X:%.*]], i8 [[Y:%.*]]
+; CHECK-NEXT: ret i8 [[SEL]]
+;
+ %notcond = icmp eq i1 %cond, false
+ call void @llvm.assume(i1 %notcond)
+ %sel = select i1 %cond, i8 %x, i8 %y
+ ret i8 %sel
+}
+
+define i32* @select_icmp_eq_0_gep_operand(i32* %base, i64 %n) {
+; CHECK-LABEL: @select_icmp_eq_0_gep_operand(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[N:%.*]]
+; CHECK-NEXT: ret i32* [[GEP]]
+;
+ %cond = icmp eq i64 %n, 0
+ %gep = getelementptr i32, i32* %base, i64 %n
+ %r = select i1 %cond, i32* %base, i32* %gep
+ ret i32* %r
+}
+
+define i32* @select_icmp_ne_0_gep_operand(i32* %base, i64 %n) {
+; CHECK-LABEL: @select_icmp_ne_0_gep_operand(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], i64 [[N:%.*]]
+; CHECK-NEXT: ret i32* [[GEP]]
+;
+ %cond = icmp ne i64 %n, 0
+ %gep = getelementptr i32, i32* %base, i64 %n
+ %r = select i1 %cond, i32* %gep, i32* %base
+ ret i32* %r
+}
+
+define i1 @and_cmps(i32 %x) {
+; CHECK-LABEL: @and_cmps(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 92
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[X]], 11
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP1]], i1 [[CMP2]], i1 false
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp1 = icmp slt i32 %x, 92
+ %cmp2 = icmp slt i32 %x, 11
+ %r = select i1 %cmp1, i1 %cmp2, i1 false
+ ret i1 %r
+}
+
+define <2 x i1> @and_cmps_vector(<2 x i32> %x) {
+; CHECK-LABEL: @and_cmps_vector(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt <2 x i32> [[X:%.*]], <i32 92, i32 92>
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt <2 x i32> [[X]], <i32 11, i32 11>
+; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[CMP1]], <2 x i1> [[CMP2]], <2 x i1> zeroinitializer
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %cmp1 = icmp slt <2 x i32> %x, <i32 92, i32 92>
+ %cmp2 = icmp slt <2 x i32> %x, <i32 11, i32 11>
+ %r = select <2 x i1> %cmp1, <2 x i1> %cmp2, <2 x i1> <i1 false, i1 false>
+ ret <2 x i1> %r
+}
+
+define i1 @or_cmps(float %x) {
+; CHECK-LABEL: @or_cmps(
+; CHECK-NEXT: [[CMP1:%.*]] = fcmp uno float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[CMP2:%.*]] = fcmp uno float [[X]], 5.200000e+01
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP1]], i1 true, i1 [[CMP2]]
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp1 = fcmp uno float %x, 42.0
+ %cmp2 = fcmp uno float %x, 52.0
+ %r = select i1 %cmp1, i1 true, i1 %cmp2
+ ret i1 %r
+}
+
+define <2 x i1> @or_logic_vector(<2 x i1> %x, <2 x i1> %y) {
+; CHECK-LABEL: @or_logic_vector(
+; CHECK-NEXT: [[A:%.*]] = and <2 x i1> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = select <2 x i1> [[X]], <2 x i1> <i1 true, i1 true>, <2 x i1> [[A]]
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %a = and <2 x i1> %x, %y
+ %r = select <2 x i1> %x, <2 x i1> <i1 true, i1 true>, <2 x i1> %a
+ ret <2 x i1> %r
+}
+
+define i1 @and_not_cmps(i32 %x) {
+; CHECK-LABEL: @and_not_cmps(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 92
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[X]], 11
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP1]], i1 false, i1 [[CMP2]]
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp1 = icmp slt i32 %x, 92
+ %cmp2 = icmp slt i32 %x, 11
+ %r = select i1 %cmp1, i1 false, i1 %cmp2
+ ret i1 %r
+}
+
+define i1 @or_not_cmps(i32 %x) {
+; CHECK-LABEL: @or_not_cmps(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 92
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[X]], 11
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP1]], i1 [[CMP2]], i1 true
+; CHECK-NEXT: ret i1 [[R]]
+;
+ %cmp1 = icmp slt i32 %x, 92
+ %cmp2 = icmp slt i32 %x, 11
+ %r = select i1 %cmp1, i1 %cmp2, i1 true
+ ret i1 %r
+}
+
+define i8 @and_cmps_wrong_type(i32 %x) {
+; CHECK-LABEL: @and_cmps_wrong_type(
+; CHECK-NEXT: [[CMP1:%.*]] = icmp slt i32 [[X:%.*]], 92
+; CHECK-NEXT: [[CMP2:%.*]] = icmp slt i32 [[X]], 11
+; CHECK-NEXT: [[S:%.*]] = sext i1 [[CMP2]] to i8
+; CHECK-NEXT: [[R:%.*]] = select i1 [[CMP1]], i8 [[S]], i8 0
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %cmp1 = icmp slt i32 %x, 92
+ %cmp2 = icmp slt i32 %x, 11
+ %s = sext i1 %cmp2 to i8
+ %r = select i1 %cmp1, i8 %s, i8 0
+ ret i8 %r
+}
+
+define i1 @y_might_be_poison(float %x, float %y) {
+; CHECK-LABEL: @y_might_be_poison(
+; CHECK-NEXT: [[C1:%.*]] = fcmp ord float 0.000000e+00, [[X:%.*]]
+; CHECK-NEXT: [[C2:%.*]] = fcmp ord float [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[C3:%.*]] = select i1 [[C1]], i1 [[C2]], i1 false
+; CHECK-NEXT: ret i1 [[C3]]
+;
+ %c1 = fcmp ord float 0.0, %x
+ %c2 = fcmp ord float %x, %y
+ %c3 = select i1 %c1, i1 %c2, i1 false
+ ret i1 %c3
+}
+
+; Negative tests to ensure we don't remove selects with undef true/false values.
+; See https://bugs.llvm.org/show_bug.cgi?id=31633
+; https://lists.llvm.org/pipermail/llvm-dev/2016-October/106182.html
+; https://reviews.llvm.org/D83360
+define i32 @false_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef(
+; CHECK-NEXT: [[S:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 undef
+; CHECK-NEXT: ret i32 [[S]]
+;
+ %s = select i1 %cond, i32 %x, i32 undef
+ ret i32 %s
+}
+
+define i32 @true_undef(i1 %cond, i32 %x) {
+; CHECK-LABEL: @true_undef(
+; CHECK-NEXT: [[S:%.*]] = select i1 [[COND:%.*]], i32 undef, i32 [[X:%.*]]
+; CHECK-NEXT: ret i32 [[S]]
+;
+ %s = select i1 %cond, i32 undef, i32 %x
+ ret i32 %s
+}
+
+define <2 x i32> @false_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @false_undef_vec(
+; CHECK-NEXT: [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> undef
+; CHECK-NEXT: ret <2 x i32> [[S]]
+;
+ %s = select i1 %cond, <2 x i32> %x, <2 x i32> undef
+ ret <2 x i32> %s
+}
+
+define <2 x i32> @true_undef_vec(i1 %cond, <2 x i32> %x) {
+; CHECK-LABEL: @true_undef_vec(
+; CHECK-NEXT: [[S:%.*]] = select i1 [[COND:%.*]], <2 x i32> undef, <2 x i32> [[X:%.*]]
+; CHECK-NEXT: ret <2 x i32> [[S]]
+;
+ %s = select i1 %cond, <2 x i32> undef, <2 x i32> %x
+ ret <2 x i32> %s
+}
+
+; These can be folded because the other value is guaranteed not to be poison.
+define i32 @false_undef_true_constant(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constant(
+; CHECK-NEXT: ret i32 10
+;
+ %s = select i1 %cond, i32 10, i32 undef
+ ret i32 %s
+}
+
+define i32 @true_undef_false_constant(i1 %cond) {
+; CHECK-LABEL: @true_undef_false_constant(
+; CHECK-NEXT: ret i32 20
+;
+ %s = select i1 %cond, i32 undef, i32 20
+ ret i32 %s
+}
+
+define <2 x i32> @false_undef_true_constant_vec(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constant_vec(
+; CHECK-NEXT: ret <2 x i32> <i32 42, i32 -42>
+;
+ %s = select i1 %cond, <2 x i32> <i32 42, i32 -42>, <2 x i32> undef
+ ret <2 x i32> %s
+}
+
+define <2 x i32> @true_undef_false_constant_vec(i1 %cond) {
+; CHECK-LABEL: @true_undef_false_constant_vec(
+; CHECK-NEXT: ret <2 x i32> <i32 -42, i32 42>
+;
+ %s = select i1 %cond, <2 x i32> undef, <2 x i32> <i32 -42, i32 42>
+ ret <2 x i32> %s
+}
+
+; If one input is undef and the other is freeze, we can fold it to the freeze.
+define i32 @false_undef_true_freeze(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef_true_freeze(
+; CHECK-NEXT: [[XF:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT: ret i32 [[XF]]
+;
+ %xf = freeze i32 %x
+ %s = select i1 %cond, i32 %xf, i32 undef
+ ret i32 %s
+}
+
+define i32 @false_undef_false_freeze(i1 %cond, i32 %x) {
+; CHECK-LABEL: @false_undef_false_freeze(
+; CHECK-NEXT: [[XF:%.*]] = freeze i32 [[X:%.*]]
+; CHECK-NEXT: ret i32 [[XF]]
+;
+ %xf = freeze i32 %x
+ %s = select i1 %cond, i32 undef, i32 %xf
+ ret i32 %s
+}
+
+ at g = external global i32, align 1
+
+define <2 x i32> @false_undef_true_constextpr_vec(i1 %cond) {
+; CHECK-LABEL: @false_undef_true_constextpr_vec(
+; CHECK-NEXT: ret <2 x i32> <i32 ptrtoint (i32* @g to i32), i32 ptrtoint (i32* @g to i32)>
+;
+ %s = select i1 %cond, <2 x i32> <i32 undef, i32 ptrtoint (i32* @g to i32)>, <2 x i32> <i32 ptrtoint (i32* @g to i32), i32 undef>
+ ret <2 x i32> %s
+}
+
+define i32 @all_constant_true_undef() {
+; CHECK-LABEL: @all_constant_true_undef(
+; CHECK-NEXT: ret i32 1
+;
+ %s = select i1 ptrtoint (i32 ()* @all_constant_true_undef to i1), i32 undef, i32 1
+ ret i32 %s
+}
+
+define float @all_constant_false_undef() {
+; CHECK-LABEL: @all_constant_false_undef(
+; CHECK-NEXT: ret float 1.000000e+00
+;
+ %s = select i1 ptrtoint (float ()* @all_constant_false_undef to i1), float undef, float 1.0
+ ret float %s
+}
+
+define <2 x i32> @all_constant_true_undef_vec() {
+; CHECK-LABEL: @all_constant_true_undef_vec(
+; CHECK-NEXT: ret <2 x i32> <i32 1, i32 -1>
+;
+ %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_vec to i1), <2 x i32> undef, <2 x i32> <i32 1, i32 -1>
+ ret <2 x i32> %s
+}
+
+define <2 x float> @all_constant_false_undef_vec() {
+; CHECK-LABEL: @all_constant_false_undef_vec(
+; CHECK-NEXT: ret <2 x float> <float 1.000000e+00, float -1.000000e+00>
+;
+ %s = select i1 ptrtoint (<2 x float> ()* @all_constant_false_undef_vec to i1), <2 x float> undef, <2 x float> <float 1.0, float -1.0>
+ ret <2 x float> %s
+}
+
+; Negative tests. Don't fold if the non-undef operand is a constexpr.
+define i32 @all_constant_false_undef_true_constexpr() {
+; CHECK-LABEL: @all_constant_false_undef_true_constexpr(
+; CHECK-NEXT: [[S:%.*]] = select i1 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i1), i32 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i32), i32 undef
+; CHECK-NEXT: ret i32 [[S]]
+;
+ %s = select i1 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i1), i32 ptrtoint (i32 ()* @all_constant_false_undef_true_constexpr to i32), i32 undef
+ ret i32 %s
+}
+
+define i32 @all_constant_true_undef_false_constexpr() {
+; CHECK-LABEL: @all_constant_true_undef_false_constexpr(
+; CHECK-NEXT: [[S:%.*]] = select i1 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i1), i32 undef, i32 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i32)
+; CHECK-NEXT: ret i32 [[S]]
+;
+ %s = select i1 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i1), i32 undef, i32 ptrtoint (i32 ()* @all_constant_true_undef_false_constexpr to i32)
+ ret i32 %s
+}
+
+; Negative tests. Don't fold if the non-undef operand is a vector containing a constexpr.
+define <2 x i32> @all_constant_false_undef_true_constexpr_vec() {
+; CHECK-LABEL: @all_constant_false_undef_true_constexpr_vec(
+; CHECK-NEXT: [[S:%.*]] = select i1 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i1), <2 x i32> <i32 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i32), i32 -1>, <2 x i32> undef
+; CHECK-NEXT: ret <2 x i32> [[S]]
+;
+ %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i1), <2 x i32> <i32 ptrtoint (<2 x i32> ()* @all_constant_false_undef_true_constexpr_vec to i32), i32 -1>, <2 x i32> undef
+ ret <2 x i32> %s
+}
+
+define <2 x i32> @all_constant_true_undef_false_constexpr_vec() {
+; CHECK-LABEL: @all_constant_true_undef_false_constexpr_vec(
+; CHECK-NEXT: [[S:%.*]] = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i1), <2 x i32> undef, <2 x i32> <i32 -1, i32 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i32)>
+; CHECK-NEXT: ret <2 x i32> [[S]]
+;
+ %s = select i1 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i1), <2 x i32> undef, <2 x i32><i32 -1, i32 ptrtoint (<2 x i32> ()* @all_constant_true_undef_false_constexpr_vec to i32)>
+ ret <2 x i32> %s
+}
+
+define i1 @expand_binop_undef(i32 %x, i32 %y) {
+; CHECK-LABEL: @expand_binop_undef(
+; CHECK-NEXT: [[CMP9_NOT_1:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[CMP15:%.*]] = icmp slt i32 [[X]], [[Y]]
+; CHECK-NEXT: [[SPEC_SELECT39:%.*]] = select i1 [[CMP9_NOT_1]], i1 undef, i1 [[CMP15]]
+; CHECK-NEXT: [[SPEC_SELECT40:%.*]] = xor i1 [[CMP9_NOT_1]], true
+; CHECK-NEXT: [[SPEC_SELECT:%.*]] = and i1 [[SPEC_SELECT39]], [[SPEC_SELECT40]]
+; CHECK-NEXT: ret i1 [[SPEC_SELECT]]
+;
+ %cmp9.not.1 = icmp eq i32 %x, %y
+ %cmp15 = icmp slt i32 %x, %y
+ %spec.select39 = select i1 %cmp9.not.1, i1 undef, i1 %cmp15
+ %spec.select40 = xor i1 %cmp9.not.1, 1
+ %spec.select = and i1 %spec.select39, %spec.select40
+ ret i1 %spec.select
+}
+
+define i32 @pr47322_more_poisonous_replacement(i32 %arg) {
+; CHECK-LABEL: @pr47322_more_poisonous_replacement(
+; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[ARG:%.*]], 0
+; CHECK-NEXT: [[TRAILING:%.*]] = call i32 @llvm.cttz.i32(i32 [[ARG]], i1 immarg true)
+; CHECK-NEXT: [[SHIFTED:%.*]] = lshr i32 [[ARG]], [[TRAILING]]
+; CHECK-NEXT: [[R1_SROA_0_1:%.*]] = select i1 [[CMP]], i32 0, i32 [[SHIFTED]]
+; CHECK-NEXT: ret i32 [[R1_SROA_0_1]]
+;
+ %cmp = icmp eq i32 %arg, 0
+ %trailing = call i32 @llvm.cttz.i32(i32 %arg, i1 immarg true)
+ %shifted = lshr i32 %arg, %trailing
+ %r1.sroa.0.1 = select i1 %cmp, i32 0, i32 %shifted
+ ret i32 %r1.sroa.0.1
+}
+declare i32 @llvm.cttz.i32(i32, i1 immarg)
+
+; Partial undef scalable vectors should be ignored.
+define <vscale x 2 x i1> @ignore_scalable_undef(<vscale x 2 x i1> %cond) {
+; CHECK-LABEL: @ignore_scalable_undef(
+; CHECK-NEXT: [[S:%.*]] = select <vscale x 2 x i1> [[COND:%.*]], <vscale x 2 x i1> undef, <vscale x 2 x i1> insertelement (<vscale x 2 x i1> poison, i1 true, i32 0)
+; CHECK-NEXT: ret <vscale x 2 x i1> [[S]]
+;
+ %vec = insertelement <vscale x 2 x i1> poison, i1 true, i32 0
+ %s = select <vscale x 2 x i1> %cond, <vscale x 2 x i1> undef, <vscale x 2 x i1> %vec
+ ret <vscale x 2 x i1> %s
+}
+
+; TODO: these can be optimized more
+
+define i32 @poison(i32 %x, i32 %y) {
+; CHECK-LABEL: @poison(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %v = select i1 undef, i32 %x, i32 %y
+ ret i32 %v
+}
+
+define i32 @poison2(i1 %cond, i32 %x) {
+; CHECK-LABEL: @poison2(
+; CHECK-NEXT: [[V:%.*]] = select i1 [[COND:%.*]], i32 poison, i32 [[X:%.*]]
+; CHECK-NEXT: ret i32 [[V]]
+;
+ %v = select i1 %cond, i32 poison, i32 %x
+ ret i32 %v
+}
+
+define i32 @poison3(i1 %cond, i32 %x) {
+; CHECK-LABEL: @poison3(
+; CHECK-NEXT: [[V:%.*]] = select i1 [[COND:%.*]], i32 [[X:%.*]], i32 poison
+; CHECK-NEXT: ret i32 [[V]]
+;
+ %v = select i1 %cond, i32 %x, i32 poison
+ ret i32 %v
+}
+
+define <2 x i32> @poison4(<2 x i1> %cond, <2 x i32> %x) {
+; CHECK-LABEL: @poison4(
+; CHECK-NEXT: [[V:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i32> [[X:%.*]], <2 x i32> poison
+; CHECK-NEXT: ret <2 x i32> [[V]]
+;
+ %v = select <2 x i1> %cond, <2 x i32> %x, <2 x i32> poison
+ ret <2 x i32> %v
+}
diff --git a/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll b/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll
new file mode 100644
index 000000000000..dac1ae37f7d8
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/vscale-inseltpoison.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S -verify | FileCheck %s
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Vector Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; insertelement
+
+define <vscale x 4 x i32> @insertelement_idx_undef(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @insertelement_idx_undef(
+; CHECK-NEXT: ret <vscale x 4 x i32> undef
+;
+ %r = insertelement <vscale x 4 x i32> %a, i32 5, i64 undef
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @insertelement_value_undef(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @insertelement_value_undef(
+; CHECK-NEXT: [[R:%.*]] = insertelement <vscale x 4 x i32> [[A:%.*]], i32 undef, i64 0
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %r = insertelement <vscale x 4 x i32> %a, i32 undef, i64 0
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @insertelement_idx_maybe_out_of_bound(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @insertelement_idx_maybe_out_of_bound(
+; CHECK-NEXT: [[R:%.*]] = insertelement <vscale x 4 x i32> [[A:%.*]], i32 5, i64 4
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %r = insertelement <vscale x 4 x i32> %a, i32 5, i64 4
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @insertelement_idx_large_bound(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @insertelement_idx_large_bound(
+; CHECK-NEXT: [[R:%.*]] = insertelement <vscale x 4 x i32> [[A:%.*]], i32 5, i64 12345
+; CHECK-NEXT: ret <vscale x 4 x i32> [[R]]
+;
+ %r = insertelement <vscale x 4 x i32> %a, i32 5, i64 12345
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @insert_extract_element_same_vec_idx_1(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @insert_extract_element_same_vec_idx_1(
+; CHECK-NEXT: ret <vscale x 4 x i32> [[A:%.*]]
+;
+ %v = extractelement <vscale x 4 x i32> %a, i64 1
+ %r = insertelement <vscale x 4 x i32> %a, i32 %v, i64 1
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 4 x i32> @insertelement_inline_to_ret() {
+; CHECK-LABEL: @insertelement_inline_to_ret(
+; CHECK-NEXT: ret <vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0)
+;
+ %i = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ ret <vscale x 4 x i32> %i
+}
+
+define <vscale x 4 x i32> @insertelement_shufflevector_inline_to_ret() {
+; CHECK-LABEL: @insertelement_shufflevector_inline_to_ret(
+; CHECK-NEXT: ret <vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer)
+;
+ %i = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ %i2 = shufflevector <vscale x 4 x i32> %i, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ ret <vscale x 4 x i32> %i2
+}
+
+; extractelement
+
+define i32 @extractelement_idx_undef(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_idx_undef(
+; CHECK-NEXT: ret i32 undef
+;
+ %r = extractelement <vscale x 4 x i32> %a, i64 undef
+ ret i32 %r
+}
+
+define i32 @extractelement_vec_undef(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_vec_undef(
+; CHECK-NEXT: ret i32 undef
+;
+ %r = extractelement <vscale x 4 x i32> undef, i64 1
+ ret i32 %r
+}
+
+define i32 @extractelement_idx_maybe_out_of_bound(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_idx_maybe_out_of_bound(
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i64 4
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = extractelement <vscale x 4 x i32> %a, i64 4
+ ret i32 %r
+}
+define i32 @extractelement_idx_large_bound(<vscale x 4 x i32> %a) {
+; CHECK-LABEL: @extractelement_idx_large_bound(
+; CHECK-NEXT: [[R:%.*]] = extractelement <vscale x 4 x i32> [[A:%.*]], i64 12345
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = extractelement <vscale x 4 x i32> %a, i64 12345
+ ret i32 %r
+}
+
+define i32 @insert_extract_element_same_vec_idx_2() {
+; CHECK-LABEL: @insert_extract_element_same_vec_idx_2(
+; CHECK-NEXT: ret i32 1
+;
+ %v = insertelement <vscale x 4 x i32> poison, i32 1, i64 4
+ %r = extractelement <vscale x 4 x i32> %v, i64 4
+ ret i32 %r
+}
+
+define i32 @insert_extract_element_same_vec_idx_3() {
+; CHECK-LABEL: @insert_extract_element_same_vec_idx_3(
+; CHECK-NEXT: ret i32 1
+;
+ %r = extractelement <vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i64 4), i64 4
+ ret i32 %r
+}
+
+define i32 @insert_extract_element_same_vec_idx_4() {
+; CHECK-LABEL: @insert_extract_element_same_vec_idx_4(
+; CHECK-NEXT: ret i32 1
+;
+ %r = extractelement <vscale x 4 x i32> insertelement (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> undef, i32 1, i32 4), i32 2, i64 3), i64 4
+ ret i32 %r
+}
+
+; more complicated expressions
+
+define <vscale x 2 x i1> @cmp_le_smax_always_true(<vscale x 2 x i64> %x) {
+; CHECK-LABEL: @cmp_le_smax_always_true(
+; CHECK-NEXT: ret <vscale x 2 x i1> shufflevector (<vscale x 2 x i1> insertelement (<vscale x 2 x i1> undef, i1 true, i32 0), <vscale x 2 x i1> undef, <vscale x 2 x i32> zeroinitializer)
+;
+ %cmp = icmp sle <vscale x 2 x i64> %x, shufflevector (<vscale x 2 x i64> insertelement (<vscale x 2 x i64> undef, i64 9223372036854775807, i32 0), <vscale x 2 x i64> undef, <vscale x 2 x i32> zeroinitializer)
+ ret <vscale x 2 x i1> %cmp
+}
+
+define <vscale x 4 x float> @bitcast() {
+; CHECK-LABEL: @bitcast(
+; CHECK-NEXT: ret <vscale x 4 x float> bitcast (<vscale x 4 x i32> shufflevector (<vscale x 4 x i32> insertelement (<vscale x 4 x i32> poison, i32 1, i32 0), <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer) to <vscale x 4 x float>)
+;
+ %i1 = insertelement <vscale x 4 x i32> poison, i32 1, i32 0
+ %i2 = shufflevector <vscale x 4 x i32> %i1, <vscale x 4 x i32> undef, <vscale x 4 x i32> zeroinitializer
+ %i3 = bitcast <vscale x 4 x i32> %i2 to <vscale x 4 x float>
+ ret <vscale x 4 x float> %i3
+}
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Memory Access and Addressing Operations
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+; getelementptr
+
+define <vscale x 4 x i32*> @getelementptr_constant_foldable_1() {
+; CHECK-LABEL: @getelementptr_constant_foldable_1(
+; CHECK-NEXT: ret <vscale x 4 x i32*> zeroinitializer
+;
+ %ptr = getelementptr i32, <vscale x 4 x i32*> zeroinitializer, <vscale x 4 x i64> undef
+ ret <vscale x 4 x i32*> %ptr
+}
+
+define <vscale x 4 x <vscale x 4 x i32>*> @getelementptr_constant_foldable_2() {
+; CHECK-LABEL: @getelementptr_constant_foldable_2(
+; CHECK-NEXT: ret <vscale x 4 x <vscale x 4 x i32>*> zeroinitializer
+;
+ %ptr = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* null, <vscale x 4 x i64> undef
+ ret <vscale x 4 x <vscale x 4 x i32>*> %ptr
+}
+
+; fold getelementptr P, 0 -> P.
+define <vscale x 4 x i32>* @getelementptr_constant_foldable_3() {
+; CHECK-LABEL: @getelementptr_constant_foldable_3(
+; CHECK-NEXT: ret <vscale x 4 x i32>* null
+;
+ %ptr = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* null, i64 0
+ ret <vscale x 4 x i32>* %ptr
+}
+
+define <vscale x 4 x i32>* @getelementptr_not_constant_foldable(i64 %x) {
+; CHECK-LABEL: @getelementptr_not_constant_foldable(
+; CHECK-NEXT: [[PTR:%.*]] = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* null, i64 [[X:%.*]]
+; CHECK-NEXT: ret <vscale x 4 x i32>* [[PTR]]
+;
+ %ptr = getelementptr <vscale x 4 x i32>, <vscale x 4 x i32>* null, i64 %x
+ ret <vscale x 4 x i32>* %ptr
+}
+
+; Check GEP's result is known to be non-null.
+define i1 @getelementptr_check_non_null(<vscale x 16 x i8>* %ptr) {
+; CHECK-LABEL: @getelementptr_check_non_null(
+; CHECK-NEXT: ret i1 false
+;
+ %x = getelementptr inbounds <vscale x 16 x i8>, <vscale x 16 x i8>* %ptr, i32 1
+ %cmp = icmp eq <vscale x 16 x i8>* %x, null
+ ret i1 %cmp
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
new file mode 100644
index 000000000000..8405a4a70ce9
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
@@ -0,0 +1,95 @@
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -load-store-vectorizer -dce -S -o - %s | FileCheck %s
+
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5"
+
+define void @base_case(i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b, <3 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @base_case
+; CHECK: load <3 x i32>
+entry:
+ %gep1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 1
+ %gep2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 2
+ %gep4 = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 1
+ %gep5 = getelementptr inbounds i32, i32 addrspace(1)* %b, i64 2
+ %selected = select i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b
+ %selected14 = select i1 %cnd, i32 addrspace(1)* %gep1, i32 addrspace(1)* %gep4
+ %selected25 = select i1 %cnd, i32 addrspace(1)* %gep2, i32 addrspace(1)* %gep5
+ %val0 = load i32, i32 addrspace(1)* %selected, align 4
+ %val1 = load i32, i32 addrspace(1)* %selected14, align 4
+ %val2 = load i32, i32 addrspace(1)* %selected25, align 4
+ %t0 = insertelement <3 x i32> poison, i32 %val0, i32 0
+ %t1 = insertelement <3 x i32> %t0, i32 %val1, i32 1
+ %t2 = insertelement <3 x i32> %t1, i32 %val2, i32 2
+ store <3 x i32> %t2, <3 x i32> addrspace(1)* %out
+ ret void
+}
+
+define void @scev_targeting_complex_case(i1 %cnd, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %base, <2 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @scev_targeting_complex_case
+; CHECK: load <2 x i32>
+entry:
+ %base.x4 = shl i32 %base, 2
+ %base.x4.p1 = add i32 %base.x4, 1
+ %base.x4.p2 = add i32 %base.x4, 2
+ %base.x4.p3 = add i32 %base.x4, 3
+ %zext.x4 = zext i32 %base.x4 to i64
+ %zext.x4.p1 = zext i32 %base.x4.p1 to i64
+ %zext.x4.p2 = zext i32 %base.x4.p2 to i64
+ %zext.x4.p3 = zext i32 %base.x4.p3 to i64
+ %base.x16 = mul i64 %zext.x4, 4
+ %base.x16.p4 = shl i64 %zext.x4.p1, 2
+ %base.x16.p8 = shl i64 %zext.x4.p2, 2
+ %base.x16.p12 = mul i64 %zext.x4.p3, 4
+ %a.pi8 = bitcast i32 addrspace(1)* %a to i8 addrspace(1)*
+ %b.pi8 = bitcast i32 addrspace(1)* %b to i8 addrspace(1)*
+ %gep.a.base.x16 = getelementptr inbounds i8, i8 addrspace(1)* %a.pi8, i64 %base.x16
+ %gep.b.base.x16.p4 = getelementptr inbounds i8, i8 addrspace(1)* %b.pi8, i64 %base.x16.p4
+ %gep.a.base.x16.p8 = getelementptr inbounds i8, i8 addrspace(1)* %a.pi8, i64 %base.x16.p8
+ %gep.b.base.x16.p12 = getelementptr inbounds i8, i8 addrspace(1)* %b.pi8, i64 %base.x16.p12
+ %a.base.x16 = bitcast i8 addrspace(1)* %gep.a.base.x16 to i32 addrspace(1)*
+ %b.base.x16.p4 = bitcast i8 addrspace(1)* %gep.b.base.x16.p4 to i32 addrspace(1)*
+ %selected.base.x16.p0.or.4 = select i1 %cnd, i32 addrspace(1)* %a.base.x16, i32 addrspace(1)* %b.base.x16.p4
+ %gep.selected.base.x16.p8.or.12 = select i1 %cnd, i8 addrspace(1)* %gep.a.base.x16.p8, i8 addrspace(1)* %gep.b.base.x16.p12
+ %selected.base.x16.p8.or.12 = bitcast i8 addrspace(1)* %gep.selected.base.x16.p8.or.12 to i32 addrspace(1)*
+ %selected.base.x16.p40.or.44 = getelementptr inbounds i32, i32 addrspace(1)* %selected.base.x16.p0.or.4, i64 10
+ %selected.base.x16.p44.or.48 = getelementptr inbounds i32, i32 addrspace(1)* %selected.base.x16.p8.or.12, i64 9
+ %val0 = load i32, i32 addrspace(1)* %selected.base.x16.p40.or.44, align 4
+ %val1 = load i32, i32 addrspace(1)* %selected.base.x16.p44.or.48, align 4
+ %t0 = insertelement <2 x i32> poison, i32 %val0, i32 0
+ %t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
+ store <2 x i32> %t1, <2 x i32> addrspace(1)* %out
+ ret void
+}
+
+define void @nested_selects(i1 %cnd0, i1 %cnd1, i32 addrspace(1)* %a, i32 addrspace(1)* %b, i32 %base, <2 x i32> addrspace(1)* %out) {
+; CHECK-LABEL: @nested_selects
+; CHECK: load <2 x i32>
+entry:
+ %base.p1 = add nsw i32 %base, 1
+ %base.p2 = add i32 %base, 2
+ %base.p3 = add nsw i32 %base, 3
+ %base.x4 = mul i32 %base, 4
+ %base.x4.p5 = add i32 %base.x4, 5
+ %base.x4.p6 = add i32 %base.x4, 6
+ %sext = sext i32 %base to i64
+ %sext.p1 = sext i32 %base.p1 to i64
+ %sext.p2 = sext i32 %base.p2 to i64
+ %sext.p3 = sext i32 %base.p3 to i64
+ %sext.x4.p5 = sext i32 %base.x4.p5 to i64
+ %sext.x4.p6 = sext i32 %base.x4.p6 to i64
+ %gep.a.base = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext
+ %gep.a.base.p1 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p1
+ %gep.a.base.p2 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p2
+ %gep.a.base.p3 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.p3
+ %gep.b.base.x4.p5 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.x4.p5
+ %gep.b.base.x4.p6 = getelementptr inbounds i32, i32 addrspace(1)* %a, i64 %sext.x4.p6
+ %selected.1.L = select i1 %cnd1, i32 addrspace(1)* %gep.a.base.p2, i32 addrspace(1)* %gep.b.base.x4.p5
+ %selected.1.R = select i1 %cnd1, i32 addrspace(1)* %gep.a.base.p3, i32 addrspace(1)* %gep.b.base.x4.p6
+ %selected.0.L = select i1 %cnd0, i32 addrspace(1)* %gep.a.base, i32 addrspace(1)* %selected.1.L
+ %selected.0.R = select i1 %cnd0, i32 addrspace(1)* %gep.a.base.p1, i32 addrspace(1)* %selected.1.R
+ %val0 = load i32, i32 addrspace(1)* %selected.0.L, align 4
+ %val1 = load i32, i32 addrspace(1)* %selected.0.R, align 4
+ %t0 = insertelement <2 x i32> poison, i32 %val0, i32 0
+ %t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
+ store <2 x i32> %t1, <2 x i32> addrspace(1)* %out
+ ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
new file mode 100644
index 000000000000..41e9f0b590be
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
@@ -0,0 +1,40 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -load-store-vectorizer -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
+
+define <8 x double> @loadwidth_insert_extract(double* %ptr) {
+ %a = bitcast double* %ptr to <2 x double> *
+ %b = getelementptr <2 x double>, <2 x double>* %a, i32 1
+ %c = getelementptr <2 x double>, <2 x double>* %a, i32 2
+ %d = getelementptr <2 x double>, <2 x double>* %a, i32 3
+; CHECK-HSW: load <4 x double>
+; CHECK-HSW: load <4 x double>
+; CHECK-HSW-NOT: load
+; CHECK-KNL: load <8 x double>
+; CHECK-KNL-NOT: load
+ %la = load <2 x double>, <2 x double> *%a
+ %lb = load <2 x double>, <2 x double> *%b
+ %lc = load <2 x double>, <2 x double> *%c
+ %ld = load <2 x double>, <2 x double> *%d
+ ; Scalarize everything - Explicitly not a shufflevector to test this code
+ ; path in the LSV
+ %v1 = extractelement <2 x double> %la, i32 0
+ %v2 = extractelement <2 x double> %la, i32 1
+ %v3 = extractelement <2 x double> %lb, i32 0
+ %v4 = extractelement <2 x double> %lb, i32 1
+ %v5 = extractelement <2 x double> %lc, i32 0
+ %v6 = extractelement <2 x double> %lc, i32 1
+ %v7 = extractelement <2 x double> %ld, i32 0
+ %v8 = extractelement <2 x double> %ld, i32 1
+ ; Make a vector again
+ %i1 = insertelement <8 x double> poison, double %v1, i32 0
+ %i2 = insertelement <8 x double> %i1, double %v2, i32 1
+ %i3 = insertelement <8 x double> %i2, double %v3, i32 2
+ %i4 = insertelement <8 x double> %i3, double %v4, i32 3
+ %i5 = insertelement <8 x double> %i4, double %v5, i32 4
+ %i6 = insertelement <8 x double> %i5, double %v6, i32 5
+ %i7 = insertelement <8 x double> %i6, double %v7, i32 6
+ %i8 = insertelement <8 x double> %i7, double %v8, i32 7
+ ret <8 x double> %i8
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
new file mode 100644
index 000000000000..babee83fd0e4
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -o - -S -load-store-vectorizer -dce %s | FileCheck %s
+
+; Make sure LoadStoreVectorizer vectorizes the loads below.
+; In order to prove that the vectorization is safe, it tries to
+; match nested adds and find an expression that adds a constant
+; value to an existing index and the result doesn't overflow.
+
+target triple = "x86_64--"
+
+define void @ld_v4i8_add_nsw(i32 %v0, i32 %v1, i8* %src, <4 x i8>* %dst) {
+; CHECK-LABEL: @ld_v4i8_add_nsw(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[TMP:%.*]] = add nsw i32 [[V0:%.*]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[V1:%.*]], [[TMP]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[SRC:%.*]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i8* [[TMP3]] to <4 x i8>*
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* [[TMP0]], align 1
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP41]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP82]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP133]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP184]], i32 3
+; CHECK-NEXT: store <4 x i8> [[TMP22]], <4 x i8>* [[DST:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+bb:
+ %tmp = add nsw i32 %v0, -1
+ %tmp1 = add nsw i32 %v1, %tmp
+ %tmp2 = sext i32 %tmp1 to i64
+ %tmp3 = getelementptr inbounds i8, i8* %src, i64 %tmp2
+ %tmp4 = load i8, i8* %tmp3, align 1
+ %tmp5 = add nsw i32 %v1, %v0
+ %tmp6 = sext i32 %tmp5 to i64
+ %tmp7 = getelementptr inbounds i8, i8* %src, i64 %tmp6
+ %tmp8 = load i8, i8* %tmp7, align 1
+ %tmp9 = add nsw i32 %v0, 1
+ %tmp10 = add nsw i32 %v1, %tmp9
+ %tmp11 = sext i32 %tmp10 to i64
+ %tmp12 = getelementptr inbounds i8, i8* %src, i64 %tmp11
+ %tmp13 = load i8, i8* %tmp12, align 1
+ %tmp14 = add nsw i32 %v0, 2
+ %tmp15 = add nsw i32 %v1, %tmp14
+ %tmp16 = sext i32 %tmp15 to i64
+ %tmp17 = getelementptr inbounds i8, i8* %src, i64 %tmp16
+ %tmp18 = load i8, i8* %tmp17, align 1
+ %tmp19 = insertelement <4 x i8> poison, i8 %tmp4, i32 0
+ %tmp20 = insertelement <4 x i8> %tmp19, i8 %tmp8, i32 1
+ %tmp21 = insertelement <4 x i8> %tmp20, i8 %tmp13, i32 2
+ %tmp22 = insertelement <4 x i8> %tmp21, i8 %tmp18, i32 3
+ store <4 x i8> %tmp22, <4 x i8>* %dst
+ ret void
+}
+
+define void @ld_v4i8_add_nuw(i32 %v0, i32 %v1, i8* %src, <4 x i8>* %dst) {
+; CHECK-LABEL: @ld_v4i8_add_nuw(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[TMP:%.*]] = add nuw i32 [[V0:%.*]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw i32 [[V1:%.*]], [[TMP]]
+; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[SRC:%.*]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i8* [[TMP3]] to <4 x i8>*
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* [[TMP0]], align 1
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP41]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP82]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP133]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP184]], i32 3
+; CHECK-NEXT: store <4 x i8> [[TMP22]], <4 x i8>* [[DST:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+bb:
+ %tmp = add nuw i32 %v0, -1
+ %tmp1 = add nuw i32 %v1, %tmp
+ %tmp2 = zext i32 %tmp1 to i64
+ %tmp3 = getelementptr inbounds i8, i8* %src, i64 %tmp2
+ %tmp4 = load i8, i8* %tmp3, align 1
+ %tmp5 = add nuw i32 %v1, %v0
+ %tmp6 = zext i32 %tmp5 to i64
+ %tmp7 = getelementptr inbounds i8, i8* %src, i64 %tmp6
+ %tmp8 = load i8, i8* %tmp7, align 1
+ %tmp9 = add nuw i32 %v0, 1
+ %tmp10 = add nuw i32 %v1, %tmp9
+ %tmp11 = zext i32 %tmp10 to i64
+ %tmp12 = getelementptr inbounds i8, i8* %src, i64 %tmp11
+ %tmp13 = load i8, i8* %tmp12, align 1
+ %tmp14 = add nuw i32 %v0, 2
+ %tmp15 = add nuw i32 %v1, %tmp14
+ %tmp16 = zext i32 %tmp15 to i64
+ %tmp17 = getelementptr inbounds i8, i8* %src, i64 %tmp16
+ %tmp18 = load i8, i8* %tmp17, align 1
+ %tmp19 = insertelement <4 x i8> poison, i8 %tmp4, i32 0
+ %tmp20 = insertelement <4 x i8> %tmp19, i8 %tmp8, i32 1
+ %tmp21 = insertelement <4 x i8> %tmp20, i8 %tmp13, i32 2
+ %tmp22 = insertelement <4 x i8> %tmp21, i8 %tmp18, i32 3
+ store <4 x i8> %tmp22, <4 x i8>* %dst
+ ret void
+}
+
+; Make sure we don't vectorize the loads below because the source of
+; sext instructions doesn't have the nsw flag.
+
+define void @ld_v4i8_add_not_safe(i32 %v0, i32 %v1, i8* %src, <4 x i8>* %dst) {
+; CHECK-LABEL: @ld_v4i8_add_not_safe(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[TMP:%.*]] = add nsw i32 [[V0:%.*]], -1
+; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[V1:%.*]], [[TMP]]
+; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, i8* [[SRC:%.*]], i64 [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = load i8, i8* [[TMP3]], align 1
+; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
+; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 [[TMP6]]
+; CHECK-NEXT: [[TMP8:%.*]] = load i8, i8* [[TMP7]], align 1
+; CHECK-NEXT: [[TMP9:%.*]] = add nsw i32 [[V0]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[V1]], [[TMP9]]
+; CHECK-NEXT: [[TMP11:%.*]] = sext i32 [[TMP10]] to i64
+; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = load i8, i8* [[TMP12]], align 1
+; CHECK-NEXT: [[TMP14:%.*]] = add nsw i32 [[V0]], 2
+; CHECK-NEXT: [[TMP15:%.*]] = add i32 [[V1]], [[TMP14]]
+; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[TMP15]] to i64
+; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 [[TMP16]]
+; CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[TMP17]], align 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP13]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP18]], i32 3
+; CHECK-NEXT: store <4 x i8> [[TMP22]], <4 x i8>* [[DST:%.*]], align 4
+; CHECK-NEXT: ret void
+;
+bb:
+ %tmp = add nsw i32 %v0, -1
+ %tmp1 = add i32 %v1, %tmp
+ %tmp2 = sext i32 %tmp1 to i64
+ %tmp3 = getelementptr inbounds i8, i8* %src, i64 %tmp2
+ %tmp4 = load i8, i8* %tmp3, align 1
+ %tmp5 = add i32 %v1, %v0
+ %tmp6 = sext i32 %tmp5 to i64
+ %tmp7 = getelementptr inbounds i8, i8* %src, i64 %tmp6
+ %tmp8 = load i8, i8* %tmp7, align 1
+ %tmp9 = add nsw i32 %v0, 1
+ %tmp10 = add i32 %v1, %tmp9
+ %tmp11 = sext i32 %tmp10 to i64
+ %tmp12 = getelementptr inbounds i8, i8* %src, i64 %tmp11
+ %tmp13 = load i8, i8* %tmp12, align 1
+ %tmp14 = add nsw i32 %v0, 2
+ %tmp15 = add i32 %v1, %tmp14
+ %tmp16 = sext i32 %tmp15 to i64
+ %tmp17 = getelementptr inbounds i8, i8* %src, i64 %tmp16
+ %tmp18 = load i8, i8* %tmp17, align 1
+ %tmp19 = insertelement <4 x i8> poison, i8 %tmp4, i32 0
+ %tmp20 = insertelement <4 x i8> %tmp19, i8 %tmp8, i32 1
+ %tmp21 = insertelement <4 x i8> %tmp20, i8 %tmp13, i32 2
+ %tmp22 = insertelement <4 x i8> %tmp21, i8 %tmp18, i32 3
+ store <4 x i8> %tmp22, <4 x i8>* %dst
+ ret void
+}
diff --git a/llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains-inseltpoison.ll b/llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains-inseltpoison.ll
new file mode 100644
index 000000000000..6c21bcae05c6
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains-inseltpoison.ll
@@ -0,0 +1,257 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve %s -S -loop-reduce -o - | FileCheck %s
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m-arm-none-eabi"
+
+define float @vctp8(float* %0, i32 %1) {
+; CHECK-LABEL: @vctp8(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], <i32 -32, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: br label [[TMP11:%.*]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = tail call <16 x i1> @llvm.arm.mve.vctp8(i32 [[TMP12]])
+; CHECK-NEXT: [[MASK:%.*]] = tail call <4 x i1> @v16i1_to_v4i1(<16 x i1> [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[MASK]])
+; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0
+; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[MASK]], <4 x float> [[TMP13]])
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4
+; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4
+; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]]
+; CHECK: 22:
+; CHECK-NEXT: [[TMP23:%.*]] = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float
+; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]])
+; CHECK-NEXT: ret float [[TMP25]]
+;
+ %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+ %4 = extractvalue { <4 x i32>, i32 } %3, 0
+ %5 = add nsw i32 %1, -1
+ %6 = ptrtoint float* %0 to i32
+ %7 = insertelement <4 x i32> poison, i32 %6, i32 0
+ %8 = add <4 x i32> %7, <i32 -32, i32 undef, i32 undef, i32 undef>
+ %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer
+ %10 = add <4 x i32> %4, %9
+ br label %11
+
+11: ; preds = %11, %2
+ %12 = phi i32 [ %5, %2 ], [ %20, %11 ]
+ %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ]
+ %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ]
+ %15 = tail call <16 x i1> @llvm.arm.mve.vctp8(i32 %12)
+ %mask = tail call <4 x i1> @v16i1_to_v4i1(<16 x i1> %15)
+ %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %mask)
+ %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1
+ %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0
+ %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %mask, <4 x float> %13)
+ %20 = add nsw i32 %12, -4
+ %21 = icmp sgt i32 %12, 4
+ br i1 %21, label %11, label %22
+
+22: ; preds = %11
+ %23 = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19)
+ %24 = sitofp i32 %23 to float
+ %25 = tail call float @llvm.fabs.f32(float %24)
+ ret float %25
+}
+
+define float @vctp16(float* %0, i32 %1) {
+; CHECK-LABEL: @vctp16(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], <i32 -32, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: br label [[TMP11:%.*]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP12]])
+; CHECK-NEXT: [[MASK:%.*]] = tail call <4 x i1> @v8i1_to_v4i1(<8 x i1> [[TMP15]])
+; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[MASK]])
+; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0
+; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[MASK]], <4 x float> [[TMP13]])
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4
+; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4
+; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]]
+; CHECK: 22:
+; CHECK-NEXT: [[TMP23:%.*]] = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float
+; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]])
+; CHECK-NEXT: ret float [[TMP25]]
+;
+ %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+ %4 = extractvalue { <4 x i32>, i32 } %3, 0
+ %5 = add nsw i32 %1, -1
+ %6 = ptrtoint float* %0 to i32
+ %7 = insertelement <4 x i32> poison, i32 %6, i32 0
+ %8 = add <4 x i32> %7, <i32 -32, i32 undef, i32 undef, i32 undef>
+ %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer
+ %10 = add <4 x i32> %4, %9
+ br label %11
+
+11: ; preds = %11, %2
+ %12 = phi i32 [ %5, %2 ], [ %20, %11 ]
+ %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ]
+ %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ]
+ %15 = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 %12)
+ %mask = tail call <4 x i1> @v8i1_to_v4i1(<8 x i1> %15)
+ %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %mask)
+ %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1
+ %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0
+ %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %mask, <4 x float> %13)
+ %20 = add nsw i32 %12, -4
+ %21 = icmp sgt i32 %12, 4
+ br i1 %21, label %11, label %22
+
+22: ; preds = %11
+ %23 = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19)
+ %24 = sitofp i32 %23 to float
+ %25 = tail call float @llvm.fabs.f32(float %24)
+ ret float %25
+}
+
+define float @vctpi32(float* %0, i32 %1) {
+; CHECK-LABEL: @vctpi32(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], <i32 -32, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: br label [[TMP11:%.*]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP12]])
+; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[TMP15]])
+; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0
+; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[TMP15]], <4 x float> [[TMP13]])
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4
+; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4
+; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]]
+; CHECK: 22:
+; CHECK-NEXT: [[TMP23:%.*]] = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float
+; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]])
+; CHECK-NEXT: ret float [[TMP25]]
+;
+ %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+ %4 = extractvalue { <4 x i32>, i32 } %3, 0
+ %5 = add nsw i32 %1, -1
+ %6 = ptrtoint float* %0 to i32
+ %7 = insertelement <4 x i32> poison, i32 %6, i32 0
+ %8 = add <4 x i32> %7, <i32 -32, i32 undef, i32 undef, i32 undef>
+ %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer
+ %10 = add <4 x i32> %4, %9
+ br label %11
+
+11: ; preds = %11, %2
+ %12 = phi i32 [ %5, %2 ], [ %20, %11 ]
+ %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ]
+ %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ]
+ %15 = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 %12)
+ %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %15)
+ %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1
+ %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0
+ %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %15, <4 x float> %13)
+ %20 = add nsw i32 %12, -4
+ %21 = icmp sgt i32 %12, 4
+ br i1 %21, label %11, label %22
+
+22: ; preds = %11
+ %23 = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19)
+ %24 = sitofp i32 %23 to float
+ %25 = tail call float @llvm.fabs.f32(float %24)
+ ret float %25
+}
+
+
+define float @vctpi64(float* %0, i32 %1) {
+; CHECK-LABEL: @vctpi64(
+; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1
+; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], <i32 -32, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]]
+; CHECK-NEXT: br label [[TMP11:%.*]]
+; CHECK: 11:
+; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ]
+; CHECK-NEXT: [[TMP15:%.*]] = tail call <4 x i1> @llvm.arm.mve.vctp64(i32 [[TMP12]])
+; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[TMP15]])
+; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1
+; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0
+; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[TMP15]], <4 x float> [[TMP13]])
+; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4
+; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4
+; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]]
+; CHECK: 22:
+; CHECK-NEXT: [[TMP23:%.*]] = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]])
+; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float
+; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]])
+; CHECK-NEXT: ret float [[TMP25]]
+;
+ %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8)
+ %4 = extractvalue { <4 x i32>, i32 } %3, 0
+ %5 = add nsw i32 %1, -1
+ %6 = ptrtoint float* %0 to i32
+ %7 = insertelement <4 x i32> poison, i32 %6, i32 0
+ %8 = add <4 x i32> %7, <i32 -32, i32 undef, i32 undef, i32 undef>
+ %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer
+ %10 = add <4 x i32> %4, %9
+ br label %11
+
+11: ; preds = %11, %2
+ %12 = phi i32 [ %5, %2 ], [ %20, %11 ]
+ %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ]
+ %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ]
+ %15 = tail call <4 x i1> @llvm.arm.mve.vctp64(i32 %12)
+ %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %15)
+ %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1
+ %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0
+ %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %15, <4 x float> %13)
+ %20 = add nsw i32 %12, -4
+ %21 = icmp sgt i32 %12, 4
+ br i1 %21, label %11, label %22
+
+22: ; preds = %11
+ %23 = tail call i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19)
+ %24 = sitofp i32 %23 to float
+ %25 = tail call float @llvm.fabs.f32(float %24)
+ ret float %25
+}
+
+declare { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32, i32)
+declare <16 x i1> @llvm.arm.mve.vctp8(i32)
+declare <8 x i1> @llvm.arm.mve.vctp16(i32)
+declare <4 x i1> @llvm.arm.mve.vctp32(i32)
+declare <4 x i1> @llvm.arm.mve.vctp64(i32)
+declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
+declare <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float>, <4 x float>, <4 x i1>, <4 x float>)
+declare i32 @vecAddAcrossF32Mve(...)
+declare <4 x i1> @v8i1_to_v4i1(<8 x i1>)
+declare <4 x i1> @v16i1_to_v4i1(<16 x i1>)
+declare float @llvm.fabs.f32(float)
diff --git a/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll
new file mode 100644
index 000000000000..9571f12098b6
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/PowerPC/p8-unrolling-legalize-vectors-inseltpoison.ll
@@ -0,0 +1,256 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -loop-unroll | FileCheck %s
+; RUN: opt < %s -S -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -loop-unroll | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-n32:64"
+target triple = "powerpc64le-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind
+define i8* @f(i8* returned %s, i32 zeroext %x, i32 signext %k) local_unnamed_addr #0 {
+; CHECK-LABEL: @f(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[CMP10:%.*]] = icmp sgt i32 [[K:%.*]], 0
+; CHECK-NEXT: br i1 [[CMP10]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK: for.body.lr.ph:
+; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[K]] to i64
+; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[K]], 16
+; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK: vector.ph:
+; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 4294967280
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> undef, <16 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[N_VEC]], -16
+; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 4
+; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT: [[XTRAITER1:%.*]] = and i64 [[TMP2]], 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 [[TMP1]], 1
+; CHECK-NEXT: br i1 [[TMP3]], label [[MIDDLE_BLOCK_UNR_LCSSA:%.*]], label [[VECTOR_PH_NEW:%.*]]
+; CHECK: vector.ph.new:
+; CHECK-NEXT: [[UNROLL_ITER:%.*]] = sub i64 [[TMP2]], [[XTRAITER1]]
+; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
+; CHECK: vector.body:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH_NEW]] ], [ [[INDEX_NEXT_1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND12:%.*]] = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, [[VECTOR_PH_NEW]] ], [ [[VEC_IND_NEXT13_1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[NITER:%.*]] = phi i64 [ [[UNROLL_ITER]], [[VECTOR_PH_NEW]] ], [ [[NITER_NSUB_1:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = shl <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[VEC_IND12]]
+; CHECK-NEXT: [[TMP5:%.*]] = and <16 x i32> [[TMP4]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <16 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = select <16 x i1> [[TMP6]], <16 x i8> <i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48>, <16 x i8> <i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49>
+; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, i8* [[S:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i8* [[TMP8]] to <16 x i8>*
+; CHECK-NEXT: store <16 x i8> [[TMP7]], <16 x i8>* [[TMP9]], align 1
+; CHECK-NEXT: [[INDEX_NEXT:%.*]] = add nuw nsw i64 [[INDEX]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT13:%.*]] = add <16 x i32> [[VEC_IND12]], <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT: [[NITER_NSUB:%.*]] = sub i64 [[NITER]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = shl <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[VEC_IND_NEXT13]]
+; CHECK-NEXT: [[TMP11:%.*]] = and <16 x i32> [[TMP10]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP12:%.*]] = icmp eq <16 x i32> [[TMP11]], zeroinitializer
+; CHECK-NEXT: [[TMP13:%.*]] = select <16 x i1> [[TMP12]], <16 x i8> <i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48>, <16 x i8> <i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49>
+; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDEX_NEXT]]
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast i8* [[TMP14]] to <16 x i8>*
+; CHECK-NEXT: store <16 x i8> [[TMP13]], <16 x i8>* [[TMP15]], align 1
+; CHECK-NEXT: [[INDEX_NEXT_1]] = add i64 [[INDEX_NEXT]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT13_1]] = add <16 x i32> [[VEC_IND_NEXT13]], <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT: [[NITER_NSUB_1]] = sub i64 [[NITER_NSUB]], 1
+; CHECK-NEXT: [[NITER_NCMP_1:%.*]] = icmp eq i64 [[NITER_NSUB_1]], 0
+; CHECK-NEXT: br i1 [[NITER_NCMP_1]], label [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT:%.*]], label [[VECTOR_BODY]]
+; CHECK: middle.block.unr-lcssa.loopexit:
+; CHECK-NEXT: [[INDEX_UNR_PH:%.*]] = phi i64 [ [[INDEX_NEXT_1]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND12_UNR_PH:%.*]] = phi <16 x i32> [ [[VEC_IND_NEXT13_1]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: br label [[MIDDLE_BLOCK_UNR_LCSSA]]
+; CHECK: middle.block.unr-lcssa:
+; CHECK-NEXT: [[INDEX_UNR:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ]
+; CHECK-NEXT: [[VEC_IND12_UNR:%.*]] = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, [[VECTOR_PH]] ], [ [[VEC_IND12_UNR_PH]], [[MIDDLE_BLOCK_UNR_LCSSA_LOOPEXIT]] ]
+; CHECK-NEXT: [[LCMP_MOD2:%.*]] = icmp ne i64 [[XTRAITER1]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD2]], label [[VECTOR_BODY_EPIL_PREHEADER:%.*]], label [[MIDDLE_BLOCK:%.*]]
+; CHECK: vector.body.epil.preheader:
+; CHECK-NEXT: br label [[VECTOR_BODY_EPIL:%.*]]
+; CHECK: vector.body.epil:
+; CHECK-NEXT: [[INDEX_EPIL:%.*]] = phi i64 [ [[INDEX_UNR]], [[VECTOR_BODY_EPIL_PREHEADER]] ]
+; CHECK-NEXT: [[VEC_IND12_EPIL:%.*]] = phi <16 x i32> [ [[VEC_IND12_UNR]], [[VECTOR_BODY_EPIL_PREHEADER]] ]
+; CHECK-NEXT: [[TMP16:%.*]] = shl <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[VEC_IND12_EPIL]]
+; CHECK-NEXT: [[TMP17:%.*]] = and <16 x i32> [[TMP16]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT: [[TMP18:%.*]] = icmp eq <16 x i32> [[TMP17]], zeroinitializer
+; CHECK-NEXT: [[TMP19:%.*]] = select <16 x i1> [[TMP18]], <16 x i8> <i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48>, <16 x i8> <i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49>
+; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDEX_EPIL]]
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast i8* [[TMP20]] to <16 x i8>*
+; CHECK-NEXT: store <16 x i8> [[TMP19]], <16 x i8>* [[TMP21]], align 1
+; CHECK-NEXT: [[INDEX_NEXT_EPIL:%.*]] = add i64 [[INDEX_EPIL]], 16
+; CHECK-NEXT: [[VEC_IND_NEXT13_EPIL:%.*]] = add <16 x i32> [[VEC_IND12_EPIL]], <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT_EPIL]], [[N_VEC]]
+; CHECK-NEXT: br label [[MIDDLE_BLOCK_EPILOG_LCSSA:%.*]]
+; CHECK: middle.block.epilog-lcssa:
+; CHECK-NEXT: br label [[MIDDLE_BLOCK]]
+; CHECK: middle.block:
+; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER]]
+; CHECK: for.body.preheader:
+; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_LR_PH]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[INDVARS_IV_PH]]
+; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[WIDE_TRIP_COUNT]], -1
+; CHECK-NEXT: [[TMP25:%.*]] = sub i64 [[TMP24]], [[INDVARS_IV_PH]]
+; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP23]], 7
+; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i64 [[XTRAITER]], 0
+; CHECK-NEXT: br i1 [[LCMP_MOD]], label [[FOR_BODY_PROL_PREHEADER:%.*]], label [[FOR_BODY_PROL_LOOPEXIT:%.*]]
+; CHECK: for.body.prol.preheader:
+; CHECK-NEXT: br label [[FOR_BODY_PROL:%.*]]
+; CHECK: for.body.prol:
+; CHECK-NEXT: [[INDVARS_IV_PROL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL:%.*]], [[FOR_BODY_PROL]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PROL_PREHEADER]] ]
+; CHECK-NEXT: [[PROL_ITER:%.*]] = phi i64 [ [[XTRAITER]], [[FOR_BODY_PROL_PREHEADER]] ], [ [[PROL_ITER_SUB:%.*]], [[FOR_BODY_PROL]] ]
+; CHECK-NEXT: [[TMP26:%.*]] = trunc i64 [[INDVARS_IV_PROL]] to i32
+; CHECK-NEXT: [[SHL_PROL:%.*]] = shl i32 1, [[TMP26]]
+; CHECK-NEXT: [[AND_PROL:%.*]] = and i32 [[SHL_PROL]], [[X]]
+; CHECK-NEXT: [[TOBOOL_PROL:%.*]] = icmp eq i32 [[AND_PROL]], 0
+; CHECK-NEXT: [[CONV_PROL:%.*]] = select i1 [[TOBOOL_PROL]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_PROL]]
+; CHECK-NEXT: store i8 [[CONV_PROL]], i8* [[ARRAYIDX_PROL]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 1
+; CHECK-NEXT: [[EXITCOND_PROL:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_PROL]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: [[PROL_ITER_SUB]] = sub i64 [[PROL_ITER]], 1
+; CHECK-NEXT: [[PROL_ITER_CMP:%.*]] = icmp ne i64 [[PROL_ITER_SUB]], 0
+; CHECK-NEXT: br i1 [[PROL_ITER_CMP]], label [[FOR_BODY_PROL]], label [[FOR_BODY_PROL_LOOPEXIT_UNR_LCSSA:%.*]], [[LOOP0:!llvm.loop !.*]]
+; CHECK: for.body.prol.loopexit.unr-lcssa:
+; CHECK-NEXT: [[INDVARS_IV_UNR_PH:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL]], [[FOR_BODY_PROL]] ]
+; CHECK-NEXT: br label [[FOR_BODY_PROL_LOOPEXIT]]
+; CHECK: for.body.prol.loopexit:
+; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_UNR_PH]], [[FOR_BODY_PROL_LOOPEXIT_UNR_LCSSA]] ]
+; CHECK-NEXT: [[TMP27:%.*]] = icmp ult i64 [[TMP25]], 7
+; CHECK-NEXT: br i1 [[TMP27]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_PREHEADER_NEW:%.*]]
+; CHECK: for.body.preheader.new:
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_PREHEADER_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[TMP28:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[SHL:%.*]] = shl i32 1, [[TMP28]]
+; CHECK-NEXT: [[AND:%.*]] = and i32 [[SHL]], [[X]]
+; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[AND]], 0
+; CHECK-NEXT: [[CONV:%.*]] = select i1 [[TOBOOL]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV]]
+; CHECK-NEXT: store i8 [[CONV]], i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[TMP29:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[SHL_1:%.*]] = shl i32 1, [[TMP29]]
+; CHECK-NEXT: [[AND_1:%.*]] = and i32 [[SHL_1]], [[X]]
+; CHECK-NEXT: [[TOBOOL_1:%.*]] = icmp eq i32 [[AND_1]], 0
+; CHECK-NEXT: [[CONV_1:%.*]] = select i1 [[TOBOOL_1]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT: store i8 [[CONV_1]], i8* [[ARRAYIDX_1]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT]], 1
+; CHECK-NEXT: [[TMP30:%.*]] = trunc i64 [[INDVARS_IV_NEXT_1]] to i32
+; CHECK-NEXT: [[SHL_2:%.*]] = shl i32 1, [[TMP30]]
+; CHECK-NEXT: [[AND_2:%.*]] = and i32 [[SHL_2]], [[X]]
+; CHECK-NEXT: [[TOBOOL_2:%.*]] = icmp eq i32 [[AND_2]], 0
+; CHECK-NEXT: [[CONV_2:%.*]] = select i1 [[TOBOOL_2]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_1]]
+; CHECK-NEXT: store i8 [[CONV_2]], i8* [[ARRAYIDX_2]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_1]], 1
+; CHECK-NEXT: [[TMP31:%.*]] = trunc i64 [[INDVARS_IV_NEXT_2]] to i32
+; CHECK-NEXT: [[SHL_3:%.*]] = shl i32 1, [[TMP31]]
+; CHECK-NEXT: [[AND_3:%.*]] = and i32 [[SHL_3]], [[X]]
+; CHECK-NEXT: [[TOBOOL_3:%.*]] = icmp eq i32 [[AND_3]], 0
+; CHECK-NEXT: [[CONV_3:%.*]] = select i1 [[TOBOOL_3]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_2]]
+; CHECK-NEXT: store i8 [[CONV_3]], i8* [[ARRAYIDX_3]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_2]], 1
+; CHECK-NEXT: [[TMP32:%.*]] = trunc i64 [[INDVARS_IV_NEXT_3]] to i32
+; CHECK-NEXT: [[SHL_4:%.*]] = shl i32 1, [[TMP32]]
+; CHECK-NEXT: [[AND_4:%.*]] = and i32 [[SHL_4]], [[X]]
+; CHECK-NEXT: [[TOBOOL_4:%.*]] = icmp eq i32 [[AND_4]], 0
+; CHECK-NEXT: [[CONV_4:%.*]] = select i1 [[TOBOOL_4]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_3]]
+; CHECK-NEXT: store i8 [[CONV_4]], i8* [[ARRAYIDX_4]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_3]], 1
+; CHECK-NEXT: [[TMP33:%.*]] = trunc i64 [[INDVARS_IV_NEXT_4]] to i32
+; CHECK-NEXT: [[SHL_5:%.*]] = shl i32 1, [[TMP33]]
+; CHECK-NEXT: [[AND_5:%.*]] = and i32 [[SHL_5]], [[X]]
+; CHECK-NEXT: [[TOBOOL_5:%.*]] = icmp eq i32 [[AND_5]], 0
+; CHECK-NEXT: [[CONV_5:%.*]] = select i1 [[TOBOOL_5]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_4]]
+; CHECK-NEXT: store i8 [[CONV_5]], i8* [[ARRAYIDX_5]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_4]], 1
+; CHECK-NEXT: [[TMP34:%.*]] = trunc i64 [[INDVARS_IV_NEXT_5]] to i32
+; CHECK-NEXT: [[SHL_6:%.*]] = shl i32 1, [[TMP34]]
+; CHECK-NEXT: [[AND_6:%.*]] = and i32 [[SHL_6]], [[X]]
+; CHECK-NEXT: [[TOBOOL_6:%.*]] = icmp eq i32 [[AND_6]], 0
+; CHECK-NEXT: [[CONV_6:%.*]] = select i1 [[TOBOOL_6]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_5]]
+; CHECK-NEXT: store i8 [[CONV_6]], i8* [[ARRAYIDX_6]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV_NEXT_5]], 1
+; CHECK-NEXT: [[TMP35:%.*]] = trunc i64 [[INDVARS_IV_NEXT_6]] to i32
+; CHECK-NEXT: [[SHL_7:%.*]] = shl i32 1, [[TMP35]]
+; CHECK-NEXT: [[AND_7:%.*]] = and i32 [[SHL_7]], [[X]]
+; CHECK-NEXT: [[TOBOOL_7:%.*]] = icmp eq i32 [[AND_7]], 0
+; CHECK-NEXT: [[CONV_7:%.*]] = select i1 [[TOBOOL_7]], i8 48, i8 49
+; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[INDVARS_IV_NEXT_6]]
+; CHECK-NEXT: store i8 [[CONV_7]], i8* [[ARRAYIDX_7]], align 1
+; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV_NEXT_6]], 1
+; CHECK-NEXT: [[EXITCOND_7:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_7]], [[WIDE_TRIP_COUNT]]
+; CHECK-NEXT: br i1 [[EXITCOND_7]], label [[FOR_END_LOOPEXIT_UNR_LCSSA:%.*]], label [[FOR_BODY]]
+; CHECK: for.end.loopexit.unr-lcssa:
+; CHECK-NEXT: br label [[FOR_END_LOOPEXIT]]
+; CHECK: for.end.loopexit:
+; CHECK-NEXT: br label [[FOR_END]]
+; CHECK: for.end:
+; CHECK-NEXT: [[IDXPROM1:%.*]] = sext i32 [[K]] to i64
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, i8* [[S]], i64 [[IDXPROM1]]
+; CHECK-NEXT: store i8 0, i8* [[ARRAYIDX2]], align 1
+; CHECK-NEXT: ret i8* [[S]]
+;
+entry:
+ %cmp10 = icmp sgt i32 %k, 0
+ br i1 %cmp10, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph: ; preds = %entry
+ %wide.trip.count = zext i32 %k to i64
+ %min.iters.check = icmp ult i32 %k, 16
+ br i1 %min.iters.check, label %for.body.preheader, label %vector.ph
+
+vector.ph: ; preds = %for.body.lr.ph
+ %n.vec = and i64 %wide.trip.count, 4294967280
+ %broadcast.splatinsert = insertelement <16 x i32> poison, i32 %x, i32 0
+ %broadcast.splat = shufflevector <16 x i32> %broadcast.splatinsert, <16 x i32> undef, <16 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body: ; preds = %vector.body, %vector.ph
+ %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+ %vec.ind12 = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, %vector.ph ], [ %vec.ind.next13, %vector.body ]
+ %0 = shl <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %vec.ind12
+ %1 = and <16 x i32> %0, %broadcast.splat
+ %2 = icmp eq <16 x i32> %1, zeroinitializer
+ %3 = select <16 x i1> %2, <16 x i8> <i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48, i8 48>, <16 x i8> <i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49, i8 49>
+ %4 = getelementptr inbounds i8, i8* %s, i64 %index
+ %5 = bitcast i8* %4 to <16 x i8>*
+ store <16 x i8> %3, <16 x i8>* %5, align 1
+ %index.next = add i64 %index, 16
+ %vec.ind.next13 = add <16 x i32> %vec.ind12, <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+ %6 = icmp eq i64 %index.next, %n.vec
+ br i1 %6, label %middle.block, label %vector.body
+
+middle.block: ; preds = %vector.body
+ %cmp.n = icmp eq i64 %n.vec, %wide.trip.count
+ br i1 %cmp.n, label %for.end, label %for.body.preheader
+
+for.body.preheader: ; preds = %middle.block, %for.body.lr.ph
+ %indvars.iv.ph = phi i64 [ 0, %for.body.lr.ph ], [ %n.vec, %middle.block ]
+ br label %for.body
+
+for.body: ; preds = %for.body.preheader, %for.body
+ %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ %indvars.iv.ph, %for.body.preheader ]
+ %7 = trunc i64 %indvars.iv to i32
+ %shl = shl i32 1, %7
+ %and = and i32 %shl, %x
+ %tobool = icmp eq i32 %and, 0
+ %conv = select i1 %tobool, i8 48, i8 49
+ %arrayidx = getelementptr inbounds i8, i8* %s, i64 %indvars.iv
+ store i8 %conv, i8* %arrayidx, align 1
+ %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+ %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+ br i1 %exitcond, label %for.end, label %for.body
+
+for.end: ; preds = %for.body, %middle.block, %entry
+ %idxprom1 = sext i32 %k to i64
+ %arrayidx2 = getelementptr inbounds i8, i8* %s, i64 %idxprom1
+ store i8 0, i8* %arrayidx2, align 1
+ ret i8* %s
+}
+
diff --git a/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather-xfail-inseltpoison.ll b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather-xfail-inseltpoison.ll
new file mode 100644
index 000000000000..816062032454
--- /dev/null
+++ b/llvm/test/Transforms/NewGVN/2016-08-30-MaskedScatterGather-xfail-inseltpoison.ll
@@ -0,0 +1,43 @@
+; XFAIL: *
+; RUN: opt < %s -basic-aa -newgvn -S | FileCheck %s
+
+declare void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> , <2 x i32*> , i32 , <2 x i1> )
+declare <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*>, i32, <2 x i1>, <2 x i32>)
+
+; This test ensures that masked scatter and gather operations, which take vectors of pointers,
+; do not have pointer aliasing ignored when being processed.
+; No scatter/gather calls should end up eliminated
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+; CHECK: llvm.masked.scatter
+; CHECK: llvm.masked.gather
+define spir_kernel void @test(<2 x i32*> %in1, <2 x i32*> %in2, i32* %out) {
+entry:
+ ; Just some temporary storage
+ %tmp.0 = alloca i32
+ %tmp.1 = alloca i32
+ %tmp.i = insertelement <2 x i32*> poison, i32* %tmp.0, i32 0
+ %tmp = insertelement <2 x i32*> %tmp.i, i32* %tmp.1, i32 1
+ ; Read from in1 and in2
+ %in1.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in1, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ %in2.v = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %in2, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in1 to the allocas
+ call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in1.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+ ; Read in1 from the allocas
+ ; This gather should alias the scatter we just saw
+ %tmp.v.0 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in2 to the allocas
+ call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %in2.v, <2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>);
+ ; Read in2 from the allocas
+ ; This gather should alias the scatter we just saw, and not be eliminated
+ %tmp.v.1 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0i32(<2 x i32*> %tmp, i32 1, <2 x i1> <i1 true, i1 true>, <2 x i32> undef) #1
+ ; Store in2 to out for good measure
+ %tmp.v.1.0 = extractelement <2 x i32> %tmp.v.1, i32 0
+ %tmp.v.1.1 = extractelement <2 x i32> %tmp.v.1, i32 1
+ store i32 %tmp.v.1.0, i32* %out
+ %out.1 = getelementptr i32, i32* %out, i32 1
+ store i32 %tmp.v.1.1, i32* %out.1
+ ret void
+}
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_nest-inseltpoison.ll b/llvm/test/Transforms/PGOProfile/counter_promo_nest-inseltpoison.ll
new file mode 100644
index 000000000000..925204149928
--- /dev/null
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_nest-inseltpoison.ll
@@ -0,0 +1,165 @@
+; TEST that counter updates are promoted outside the whole loop nest
+; RUN: opt < %s -pgo-instr-gen -instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO %s
+; RUN: opt < %s --passes=pgo-instr-gen,instrprof -do-counter-promotion=true -S | FileCheck --check-prefix=PROMO %s
+
+ at g = common local_unnamed_addr global i32 0, align 4
+ at c = local_unnamed_addr global i32 10, align 4
+
+; Function Attrs: noinline norecurse nounwind uwtable
+define void @bar() local_unnamed_addr #0 {
+bb:
+ %tmp2 = load i32, i32* @g, align 4, !tbaa !2
+ %tmp3 = add nsw i32 %tmp2, 1
+ store i32 %tmp3, i32* @g, align 4, !tbaa !2
+ ret void
+}
+
+; Function Attrs: norecurse nounwind uwtable
+define i32 @main() local_unnamed_addr #1 {
+bb:
+ store i32 0, i32* @g, align 4, !tbaa !2
+ %tmp = load i32, i32* @c, align 4, !tbaa !2
+ %tmp1 = icmp sgt i32 %tmp, 0
+ br i1 %tmp1, label %bb2_1, label %bb84
+
+bb2_1:
+ br label %bb2
+
+bb2: ; preds = %bb39, %bb
+ %tmp3 = phi i32 [ %tmp40, %bb39 ], [ %tmp, %bb2_1 ]
+ %tmp5 = phi i32 [ %tmp43, %bb39 ], [ 0, %bb2_1 ]
+ %tmp7 = icmp sgt i32 %tmp3, 0
+ br i1 %tmp7, label %bb14_1, label %bb39
+
+bb8: ; preds = %bb39
+; PROMO-LABEL: bb8
+; PROMO: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+; PROMO-NEXT: load {{.*}} @__profc_main{{.*}}
+; PROMO-NEXT: add
+; PROMO-NEXT: store {{.*}}@__profc_main{{.*}}
+
+ %tmp13 = icmp sgt i32 %tmp40, 0
+ br i1 %tmp13, label %bb45, label %bb84
+
+bb14_1:
+ br label %bb14
+
+bb14: ; preds = %bb29, %bb2
+ %tmp15 = phi i32 [ %tmp30, %bb29 ], [ %tmp3, %bb14_1 ]
+ %tmp16 = phi i64 [ %tmp31, %bb29 ], [ 0, %bb14_1 ]
+ %tmp17 = phi i64 [ %tmp32, %bb29 ], [ 0, %bb14_1 ]
+ %tmp18 = phi i32 [ %tmp33, %bb29 ], [ 0, %bb14_1 ]
+ %tmp19 = icmp sgt i32 %tmp15, 0
+ br i1 %tmp19, label %bb20_split, label %bb29
+
+bb20_split:
+ br label %bb20
+
+bb20: ; preds = %bb20, %bb14
+ %tmp21 = phi i64 [ %tmp23, %bb20 ], [ 0, %bb20_split ]
+ %tmp22 = phi i32 [ %tmp24, %bb20 ], [ 0, %bb20_split ]
+ %tmp23 = add nuw i64 %tmp21, 1
+ tail call void @bar()
+ %tmp24 = add nuw nsw i32 %tmp22, 1
+ %tmp25 = load i32, i32* @c, align 4, !tbaa !2
+ %tmp26 = icmp slt i32 %tmp24, %tmp25
+ br i1 %tmp26, label %bb20, label %bb27
+
+bb27: ; preds = %bb20
+ %tmp28 = add i64 %tmp23, %tmp16
+ br label %bb29
+
+bb29: ; preds = %bb27, %bb14
+ %tmp30 = phi i32 [ %tmp25, %bb27 ], [ %tmp15, %bb14 ]
+ %tmp31 = phi i64 [ %tmp28, %bb27 ], [ %tmp16, %bb14 ]
+ %tmp32 = add nuw i64 %tmp17, 1
+ %tmp33 = add nuw nsw i32 %tmp18, 1
+ %tmp34 = icmp slt i32 %tmp33, %tmp30
+ br i1 %tmp34, label %bb14, label %bb35
+
+bb35: ; preds = %bb29
+ %tmp36 = insertelement <2 x i64> poison, i64 %tmp31, i32 0
+ br label %bb39
+
+bb39: ; preds = %bb35, %bb2
+ %tmp40 = phi i32 [ %tmp30, %bb35 ], [ %tmp3, %bb2 ]
+ %tmp43 = add nuw nsw i32 %tmp5, 1
+ %tmp44 = icmp slt i32 %tmp43, %tmp40
+ br i1 %tmp44, label %bb2, label %bb8
+
+bb45: ; preds = %bb67, %bb8
+ %tmp46 = phi i32 [ %tmp68, %bb67 ], [ %tmp40, %bb8 ]
+ %tmp47 = phi i64 [ %tmp69, %bb67 ], [ 0, %bb8 ]
+ %tmp48 = phi i64 [ %tmp70, %bb67 ], [ 0, %bb8 ]
+ %tmp49 = phi i32 [ %tmp71, %bb67 ], [ 0, %bb8 ]
+ %tmp50 = icmp sgt i32 %tmp46, 0
+ br i1 %tmp50, label %bb57, label %bb67
+
+bb51: ; preds = %bb67
+ %tmp56 = icmp sgt i32 %tmp68, 0
+ br i1 %tmp56, label %bb73, label %bb84
+
+bb57: ; preds = %bb57, %bb45
+ %tmp58 = phi i64 [ %tmp60, %bb57 ], [ 0, %bb45 ]
+ %tmp59 = phi i32 [ %tmp61, %bb57 ], [ 0, %bb45 ]
+ %tmp60 = add nuw i64 %tmp58, 1
+ tail call void @bar()
+ %tmp61 = add nuw nsw i32 %tmp59, 1
+ %tmp62 = load i32, i32* @c, align 4, !tbaa !2
+ %tmp63 = mul nsw i32 %tmp62, 10
+ %tmp64 = icmp slt i32 %tmp61, %tmp63
+ br i1 %tmp64, label %bb57, label %bb65
+
+bb65: ; preds = %bb57
+ %tmp66 = add i64 %tmp60, %tmp47
+ br label %bb67
+
+bb67: ; preds = %bb65, %bb45
+ %tmp68 = phi i32 [ %tmp62, %bb65 ], [ %tmp46, %bb45 ]
+ %tmp69 = phi i64 [ %tmp66, %bb65 ], [ %tmp47, %bb45 ]
+ %tmp70 = add nuw i64 %tmp48, 1
+ %tmp71 = add nuw nsw i32 %tmp49, 1
+ %tmp72 = icmp slt i32 %tmp71, %tmp68
+ br i1 %tmp72, label %bb45, label %bb51
+
+bb73: ; preds = %bb73, %bb51
+ %tmp74 = phi i64 [ %tmp76, %bb73 ], [ 0, %bb51 ]
+ %tmp75 = phi i32 [ %tmp77, %bb73 ], [ 0, %bb51 ]
+ %tmp76 = add nuw i64 %tmp74, 1
+ tail call void @bar()
+ %tmp77 = add nuw nsw i32 %tmp75, 1
+ %tmp78 = load i32, i32* @c, align 4, !tbaa !2
+ %tmp79 = mul nsw i32 %tmp78, 100
+ %tmp80 = icmp slt i32 %tmp77, %tmp79
+ br i1 %tmp80, label %bb73, label %bb81
+
+bb81: ; preds = %bb73
+ br label %bb84
+
+bb84: ; preds = %bb81, %bb51, %bb8, %bb
+ ret i32 0
+}
+
+attributes #0 = { noinline }
+attributes #1 = { norecurse nounwind uwtable }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 5.0.0 (trunk 307355)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
new file mode 100644
index 000000000000..71444a9251c0
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/addsub-inseltpoison.ll
@@ -0,0 +1,101 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -O3 -S | FileCheck %s
+; RUN: opt < %s -passes='default<O3>' -aa-pipeline=default -S | FileCheck %s
+
+target triple = "x86_64--"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; Ideally, this should reach the backend with 1 fsub, 1 fadd, and 1 shuffle.
+; That may require some coordination between VectorCombine, SLP, and other passes.
+; The end goal is to get a single "vaddsubps" instruction for x86 with AVX.
+
+define <4 x float> @PR45015(<4 x float> %arg, <4 x float> %arg1) {
+; CHECK-LABEL: @PR45015(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <4 x float> [[ARG:%.*]], [[ARG1:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[ARG]], [[ARG1]]
+; CHECK-NEXT: [[T16:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: ret <4 x float> [[T16]]
+;
+ %t = extractelement <4 x float> %arg, i32 0
+ %t2 = extractelement <4 x float> %arg1, i32 0
+ %t3 = fsub float %t, %t2
+ %t4 = insertelement <4 x float> poison, float %t3, i32 0
+ %t5 = extractelement <4 x float> %arg, i32 1
+ %t6 = extractelement <4 x float> %arg1, i32 1
+ %t7 = fadd float %t5, %t6
+ %t8 = insertelement <4 x float> %t4, float %t7, i32 1
+ %t9 = extractelement <4 x float> %arg, i32 2
+ %t10 = extractelement <4 x float> %arg1, i32 2
+ %t11 = fsub float %t9, %t10
+ %t12 = insertelement <4 x float> %t8, float %t11, i32 2
+ %t13 = extractelement <4 x float> %arg, i32 3
+ %t14 = extractelement <4 x float> %arg1, i32 3
+ %t15 = fadd float %t13, %t14
+ %t16 = insertelement <4 x float> %t12, float %t15, i32 3
+ ret <4 x float> %t16
+}
+
+; PR42022 - https://bugs.llvm.org/show_bug.cgi?id=42022
+
+%struct.Vector4 = type { float, float, float, float }
+
+define { <2 x float>, <2 x float> } @add_aggregate(<2 x float> %a0, <2 x float> %a1, <2 x float> %b0, <2 x float> %b1) {
+; CHECK-LABEL: @add_aggregate(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[A0:%.*]], [[B0:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[A1:%.*]], [[B1:%.*]]
+; CHECK-NEXT: [[FCA_0_INSERT:%.*]] = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> [[TMP1]], 0
+; CHECK-NEXT: [[FCA_1_INSERT:%.*]] = insertvalue { <2 x float>, <2 x float> } [[FCA_0_INSERT]], <2 x float> [[TMP2]], 1
+; CHECK-NEXT: ret { <2 x float>, <2 x float> } [[FCA_1_INSERT]]
+;
+ %a00 = extractelement <2 x float> %a0, i32 0
+ %b00 = extractelement <2 x float> %b0, i32 0
+ %add = fadd float %a00, %b00
+ %retval.0.0.insert = insertelement <2 x float> poison, float %add, i32 0
+ %a01 = extractelement <2 x float> %a0, i32 1
+ %b01 = extractelement <2 x float> %b0, i32 1
+ %add4 = fadd float %a01, %b01
+ %retval.0.1.insert = insertelement <2 x float> %retval.0.0.insert, float %add4, i32 1
+ %a10 = extractelement <2 x float> %a1, i32 0
+ %b10 = extractelement <2 x float> %b1, i32 0
+ %add7 = fadd float %a10, %b10
+ %retval.1.0.insert = insertelement <2 x float> poison, float %add7, i32 0
+ %a11 = extractelement <2 x float> %a1, i32 1
+ %b11 = extractelement <2 x float> %b1, i32 1
+ %add10 = fadd float %a11, %b11
+ %retval.1.1.insert = insertelement <2 x float> %retval.1.0.insert, float %add10, i32 1
+ %fca.0.insert = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> %retval.0.1.insert, 0
+ %fca.1.insert = insertvalue { <2 x float>, <2 x float> } %fca.0.insert, <2 x float> %retval.1.1.insert, 1
+ ret { <2 x float>, <2 x float> } %fca.1.insert
+}
+
+define void @add_aggregate_store(<2 x float> %a0, <2 x float> %a1, <2 x float> %b0, <2 x float> %b1, %struct.Vector4* nocapture dereferenceable(16) %r) {
+; CHECK-LABEL: @add_aggregate_store(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[A0:%.*]], [[B0:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[A1:%.*]], [[B1:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast %struct.Vector4* [[R:%.*]] to <4 x float>*
+; CHECK-NEXT: store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT: ret void
+;
+ %a00 = extractelement <2 x float> %a0, i32 0
+ %b00 = extractelement <2 x float> %b0, i32 0
+ %add = fadd float %a00, %b00
+ %r0 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %r, i64 0, i32 0
+ store float %add, float* %r0, align 4
+ %a01 = extractelement <2 x float> %a0, i32 1
+ %b01 = extractelement <2 x float> %b0, i32 1
+ %add4 = fadd float %a01, %b01
+ %r1 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %r, i64 0, i32 1
+ store float %add4, float* %r1, align 4
+ %a10 = extractelement <2 x float> %a1, i32 0
+ %b10 = extractelement <2 x float> %b1, i32 0
+ %add7 = fadd float %a10, %b10
+ %r2 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %r, i64 0, i32 2
+ store float %add7, float* %r2, align 4
+ %a11 = extractelement <2 x float> %a1, i32 1
+ %b11 = extractelement <2 x float> %b1, i32 1
+ %add10 = fadd float %a11, %b11
+ %r3 = getelementptr inbounds %struct.Vector4, %struct.Vector4* %r, i64 0, i32 3
+ store float %add10, float* %r3, align 4
+ ret void
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/horiz-math-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/X86/horiz-math-inseltpoison.ll
new file mode 100644
index 000000000000..9358f32a38ed
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/horiz-math-inseltpoison.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O3 -S < %s | FileCheck %s
+; RUN: opt -passes='default<O3>' -S < %s | FileCheck %s
+
+target triple = "x86_64--"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; PR41813 - https://bugs.llvm.org/show_bug.cgi?id=41813
+
+define <4 x float> @hadd_reverse_v4f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: @hadd_reverse_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i32> <i32 3, i32 1, i32 7, i32 5>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[B]], <4 x i32> <i32 2, i32 0, i32 6, i32 4>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x float> [[TMP3]]
+;
+ %shuffle = shufflevector <4 x float> %a, <4 x float> %a, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %shuffle1 = shufflevector <4 x float> %b, <4 x float> %b, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ %vecext = extractelement <4 x float> %shuffle, i32 0
+ %vecext2 = extractelement <4 x float> %shuffle, i32 1
+ %add = fadd float %vecext, %vecext2
+ %vecinit = insertelement <4 x float> poison, float %add, i32 0
+ %vecext3 = extractelement <4 x float> %shuffle, i32 2
+ %vecext4 = extractelement <4 x float> %shuffle, i32 3
+ %add5 = fadd float %vecext3, %vecext4
+ %vecinit6 = insertelement <4 x float> %vecinit, float %add5, i32 1
+ %vecext7 = extractelement <4 x float> %shuffle1, i32 0
+ %vecext8 = extractelement <4 x float> %shuffle1, i32 1
+ %add9 = fadd float %vecext7, %vecext8
+ %vecinit10 = insertelement <4 x float> %vecinit6, float %add9, i32 2
+ %vecext11 = extractelement <4 x float> %shuffle1, i32 2
+ %vecext12 = extractelement <4 x float> %shuffle1, i32 3
+ %add13 = fadd float %vecext11, %vecext12
+ %vecinit14 = insertelement <4 x float> %vecinit10, float %add13, i32 3
+ ret <4 x float> %vecinit14
+}
+
+define <4 x float> @reverse_hadd_v4f32(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: @reverse_hadd_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> [[A:%.*]], <4 x i32> <i32 2, i32 0, i32 6, i32 4>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> [[A]], <4 x i32> <i32 3, i32 1, i32 7, i32 5>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x float> [[TMP3]]
+;
+ %vecext = extractelement <4 x float> %a, i32 0
+ %vecext1 = extractelement <4 x float> %a, i32 1
+ %add = fadd float %vecext, %vecext1
+ %vecinit = insertelement <4 x float> poison, float %add, i32 0
+ %vecext2 = extractelement <4 x float> %a, i32 2
+ %vecext3 = extractelement <4 x float> %a, i32 3
+ %add4 = fadd float %vecext2, %vecext3
+ %vecinit5 = insertelement <4 x float> %vecinit, float %add4, i32 1
+ %vecext6 = extractelement <4 x float> %b, i32 0
+ %vecext7 = extractelement <4 x float> %b, i32 1
+ %add8 = fadd float %vecext6, %vecext7
+ %vecinit9 = insertelement <4 x float> %vecinit5, float %add8, i32 2
+ %vecext10 = extractelement <4 x float> %b, i32 2
+ %vecext11 = extractelement <4 x float> %b, i32 3
+ %add12 = fadd float %vecext10, %vecext11
+ %vecinit13 = insertelement <4 x float> %vecinit9, float %add12, i32 3
+ %shuffle = shufflevector <4 x float> %vecinit13, <4 x float> %a, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+ ret <4 x float> %shuffle
+}
+
+define <8 x float> @hadd_reverse_v8f32(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: @hadd_reverse_v8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 7, i32 5, i32 15, i32 13, i32 3, i32 1, i32 11, i32 9>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 6, i32 4, i32 14, i32 12, i32 2, i32 0, i32 10, i32 8>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <8 x float> [[TMP3]]
+;
+ %shuffle = shufflevector <8 x float> %a, <8 x float> %a, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %shuffle1 = shufflevector <8 x float> %b, <8 x float> %b, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ %vecext = extractelement <8 x float> %shuffle, i32 0
+ %vecext2 = extractelement <8 x float> %shuffle, i32 1
+ %add = fadd float %vecext, %vecext2
+ %vecinit = insertelement <8 x float> poison, float %add, i32 0
+ %vecext3 = extractelement <8 x float> %shuffle, i32 2
+ %vecext4 = extractelement <8 x float> %shuffle, i32 3
+ %add5 = fadd float %vecext3, %vecext4
+ %vecinit6 = insertelement <8 x float> %vecinit, float %add5, i32 1
+ %vecext7 = extractelement <8 x float> %shuffle1, i32 0
+ %vecext8 = extractelement <8 x float> %shuffle1, i32 1
+ %add9 = fadd float %vecext7, %vecext8
+ %vecinit10 = insertelement <8 x float> %vecinit6, float %add9, i32 2
+ %vecext11 = extractelement <8 x float> %shuffle1, i32 2
+ %vecext12 = extractelement <8 x float> %shuffle1, i32 3
+ %add13 = fadd float %vecext11, %vecext12
+ %vecinit14 = insertelement <8 x float> %vecinit10, float %add13, i32 3
+ %vecext15 = extractelement <8 x float> %shuffle, i32 4
+ %vecext16 = extractelement <8 x float> %shuffle, i32 5
+ %add17 = fadd float %vecext15, %vecext16
+ %vecinit18 = insertelement <8 x float> %vecinit14, float %add17, i32 4
+ %vecext19 = extractelement <8 x float> %shuffle, i32 6
+ %vecext20 = extractelement <8 x float> %shuffle, i32 7
+ %add21 = fadd float %vecext19, %vecext20
+ %vecinit22 = insertelement <8 x float> %vecinit18, float %add21, i32 5
+ %vecext23 = extractelement <8 x float> %shuffle1, i32 4
+ %vecext24 = extractelement <8 x float> %shuffle1, i32 5
+ %add25 = fadd float %vecext23, %vecext24
+ %vecinit26 = insertelement <8 x float> %vecinit22, float %add25, i32 6
+ %vecext27 = extractelement <8 x float> %shuffle1, i32 6
+ %vecext28 = extractelement <8 x float> %shuffle1, i32 7
+ %add29 = fadd float %vecext27, %vecext28
+ %vecinit30 = insertelement <8 x float> %vecinit26, float %add29, i32 7
+ ret <8 x float> %vecinit30
+}
+
+define <8 x float> @reverse_hadd_v8f32(<8 x float> %a, <8 x float> %b) #0 {
+; CHECK-LABEL: @reverse_hadd_v8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: ret <8 x float> [[SHUFFLE]]
+;
+ %vecext = extractelement <8 x float> %a, i32 0
+ %vecext1 = extractelement <8 x float> %a, i32 1
+ %add = fadd float %vecext, %vecext1
+ %vecinit = insertelement <8 x float> poison, float %add, i32 0
+ %vecext2 = extractelement <8 x float> %a, i32 2
+ %vecext3 = extractelement <8 x float> %a, i32 3
+ %add4 = fadd float %vecext2, %vecext3
+ %vecinit5 = insertelement <8 x float> %vecinit, float %add4, i32 1
+ %vecext6 = extractelement <8 x float> %b, i32 0
+ %vecext7 = extractelement <8 x float> %b, i32 1
+ %add8 = fadd float %vecext6, %vecext7
+ %vecinit9 = insertelement <8 x float> %vecinit5, float %add8, i32 2
+ %vecext10 = extractelement <8 x float> %b, i32 2
+ %vecext11 = extractelement <8 x float> %b, i32 3
+ %add12 = fadd float %vecext10, %vecext11
+ %vecinit13 = insertelement <8 x float> %vecinit9, float %add12, i32 3
+ %vecext14 = extractelement <8 x float> %a, i32 4
+ %vecext15 = extractelement <8 x float> %a, i32 5
+ %add16 = fadd float %vecext14, %vecext15
+ %vecinit17 = insertelement <8 x float> %vecinit13, float %add16, i32 4
+ %vecext18 = extractelement <8 x float> %a, i32 6
+ %vecext19 = extractelement <8 x float> %a, i32 7
+ %add20 = fadd float %vecext18, %vecext19
+ %vecinit21 = insertelement <8 x float> %vecinit17, float %add20, i32 5
+ %vecext22 = extractelement <8 x float> %b, i32 4
+ %vecext23 = extractelement <8 x float> %b, i32 5
+ %add24 = fadd float %vecext22, %vecext23
+ %vecinit25 = insertelement <8 x float> %vecinit21, float %add24, i32 6
+ %vecext26 = extractelement <8 x float> %b, i32 6
+ %vecext27 = extractelement <8 x float> %b, i32 7
+ %add28 = fadd float %vecext26, %vecext27
+ %vecinit29 = insertelement <8 x float> %vecinit25, float %add28, i32 7
+ %shuffle = shufflevector <8 x float> %vecinit29, <8 x float> %a, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+ ret <8 x float> %shuffle
+}
+
+attributes #0 = { "min-legal-vector-width"="128" "target-cpu"="btver2" "target-features"="+avx,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+sse4a,+ssse3" }
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/scalarization-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/X86/scalarization-inseltpoison.ll
new file mode 100644
index 000000000000..c01c9562274a
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/scalarization-inseltpoison.ll
@@ -0,0 +1,71 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O3 -S < %s | FileCheck %s
+; RUN: opt -passes='default<O3>' -S < %s | FileCheck %s
+
+target triple = "x86_64--"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; PR42174 - https://bugs.llvm.org/show_bug.cgi?id=42174
+; This test should match the IR produced by clang after running -mem2reg.
+; All math before the final 'add' should be scalarized.
+
+define <4 x i32> @square(<4 x i32> %num, i32 %y, i32 %x, i32 %h, i32 %k, i32 %w, i32 %p, i32 %j, i32 %u) {
+; CHECK-LABEL: @square(
+; CHECK-NEXT: [[DIV:%.*]] = sdiv i32 [[K:%.*]], 2
+; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[P:%.*]], 6234
+; CHECK-NEXT: [[MUL5:%.*]] = mul nsw i32 [[H:%.*]], 75
+; CHECK-NEXT: [[DIV9:%.*]] = sdiv i32 [[J:%.*]], 3452
+; CHECK-NEXT: [[MUL13:%.*]] = mul nsw i32 [[W:%.*]], 53
+; CHECK-NEXT: [[DIV17:%.*]] = sdiv i32 [[X:%.*]], 820
+; CHECK-NEXT: [[MUL21:%.*]] = shl nsw i32 [[U:%.*]], 2
+; CHECK-NEXT: [[DOTSCALAR:%.*]] = add i32 [[Y:%.*]], 1
+; CHECK-NEXT: [[DOTSCALAR1:%.*]] = add i32 [[DOTSCALAR]], [[DIV17]]
+; CHECK-NEXT: [[DOTSCALAR2:%.*]] = add i32 [[DOTSCALAR1]], [[MUL5]]
+; CHECK-NEXT: [[DOTSCALAR3:%.*]] = add i32 [[DOTSCALAR2]], [[DIV]]
+; CHECK-NEXT: [[DOTSCALAR4:%.*]] = add i32 [[DOTSCALAR3]], [[MUL13]]
+; CHECK-NEXT: [[DOTSCALAR5:%.*]] = add i32 [[DOTSCALAR4]], [[MUL]]
+; CHECK-NEXT: [[DOTSCALAR6:%.*]] = add i32 [[DOTSCALAR5]], [[DIV9]]
+; CHECK-NEXT: [[DOTSCALAR7:%.*]] = add i32 [[DOTSCALAR6]], [[MUL21]]
+; CHECK-NEXT: [[DOTSCALAR8:%.*]] = add i32 [[DOTSCALAR7]], 317425
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[DOTSCALAR8]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT: [[ADD29:%.*]] = add <4 x i32> [[TMP2]], [[NUM:%.*]]
+; CHECK-NEXT: ret <4 x i32> [[ADD29]]
+;
+ %add = add <4 x i32> %num, <i32 1, i32 1, i32 1, i32 1>
+ %div = sdiv i32 %k, 2
+ %splatinsert = insertelement <4 x i32> poison, i32 %div, i32 0
+ %splat = shufflevector <4 x i32> %splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add1 = add <4 x i32> %add, %splat
+ %mul = mul nsw i32 %p, 6234
+ %splatinsert2 = insertelement <4 x i32> poison, i32 %mul, i32 0
+ %splat3 = shufflevector <4 x i32> %splatinsert2, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add4 = add <4 x i32> %add1, %splat3
+ %mul5 = mul nsw i32 75, %h
+ %splatinsert6 = insertelement <4 x i32> poison, i32 %mul5, i32 0
+ %splat7 = shufflevector <4 x i32> %splatinsert6, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add8 = add <4 x i32> %add4, %splat7
+ %div9 = sdiv i32 %j, 3452
+ %splatinsert10 = insertelement <4 x i32> poison, i32 %div9, i32 0
+ %splat11 = shufflevector <4 x i32> %splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add12 = add <4 x i32> %add8, %splat11
+ %mul13 = mul nsw i32 53, %w
+ %splatinsert14 = insertelement <4 x i32> poison, i32 %mul13, i32 0
+ %splat15 = shufflevector <4 x i32> %splatinsert14, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add16 = add <4 x i32> %add12, %splat15
+ %div17 = sdiv i32 %x, 820
+ %splatinsert18 = insertelement <4 x i32> poison, i32 %div17, i32 0
+ %splat19 = shufflevector <4 x i32> %splatinsert18, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add20 = add <4 x i32> %add16, %splat19
+ %mul21 = mul nsw i32 4, %u
+ %splatinsert22 = insertelement <4 x i32> poison, i32 %mul21, i32 0
+ %splat23 = shufflevector <4 x i32> %splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add24 = add <4 x i32> %add20, %splat23
+ %splatinsert25 = insertelement <4 x i32> poison, i32 %y, i32 0
+ %splat26 = shufflevector <4 x i32> %splatinsert25, <4 x i32> undef, <4 x i32> zeroinitializer
+ %add27 = add <4 x i32> %add24, %splat26
+ %add28 = add <4 x i32> %add27, <i32 25, i32 25, i32 25, i32 25>
+ %add29 = add <4 x i32> %add28, <i32 317400, i32 317400, i32 317400, i32 317400>
+ ret <4 x i32> %add29
+}
+
diff --git a/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll b/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
new file mode 100644
index 000000000000..b9cfc53bdac7
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/vector-trunc-inseltpoison.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O2 -S -data-layout="e" < %s | FileCheck %s
+; RUN: opt -passes='default<O2>' -S -data-layout="e" < %s | FileCheck %s
+
+define <4 x i16> @truncate(<4 x i32> %x) {
+; CHECK-LABEL: @truncate(
+; CHECK-NEXT: [[V3:%.*]] = trunc <4 x i32> [[X:%.*]] to <4 x i16>
+; CHECK-NEXT: ret <4 x i16> [[V3]]
+;
+ %x0 = extractelement <4 x i32> %x, i32 0
+ %t0 = trunc i32 %x0 to i16
+ %v0 = insertelement <4 x i16> poison, i16 %t0, i32 0
+ %x1 = extractelement <4 x i32> %x, i32 1
+ %t1 = trunc i32 %x1 to i16
+ %v1 = insertelement <4 x i16> %v0, i16 %t1, i32 1
+ %x2 = extractelement <4 x i32> %x, i32 2
+ %t2 = trunc i32 %x2 to i16
+ %v2 = insertelement <4 x i16> %v1, i16 %t2, i32 2
+ %x3 = extractelement <4 x i32> %x, i32 3
+ %t3 = trunc i32 %x3 to i16
+ %v3 = insertelement <4 x i16> %v2, i16 %t3, i32 3
+ ret <4 x i16> %v3
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
new file mode 100644
index 000000000000..b7f4e131d939
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/base-vector-inseltpoison.ll
@@ -0,0 +1,279 @@
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+
+define i64 addrspace(1)* @test(<2 x i64 addrspace(1)*> %vec, i32 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test
+; CHECK: extractelement
+; CHECK: extractelement
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%base_ee, %obj)
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%base_ee, %base_ee)
+; Note that the second extractelement is actually redundant here. A correct output would
+; be to reuse the existing obj as a base since it is actually a base pointer.
+entry:
+ %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx
+ call void @do_safepoint() [ "deopt"() ]
+ ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test2(<2 x i64 addrspace(1)*>* %ptr, i1 %cnd, i32 %idx1, i32 %idx2) gc "statepoint-example" {
+; CHECK-LABEL: test2
+entry:
+ br i1 %cnd, label %taken, label %untaken
+
+taken: ; preds = %entry
+ %obja = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ br label %merge
+
+untaken: ; preds = %entry
+ %objb = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ br label %merge
+
+merge: ; preds = %untaken, %taken
+ %vec = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
+ br i1 %cnd, label %taken2, label %untaken2
+
+taken2: ; preds = %merge
+ %obj0 = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx1
+ br label %merge2
+
+untaken2: ; preds = %merge
+ %obj1 = extractelement <2 x i64 addrspace(1)*> %vec, i32 %idx2
+ br label %merge2
+
+merge2: ; preds = %untaken2, %taken2
+; CHECK-LABEL: merge2:
+; CHECK: %obj.base = phi i64 addrspace(1)*
+; CHECK: %obj = phi i64 addrspace(1)*
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj)
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj.base)
+ %obj = phi i64 addrspace(1)* [ %obj0, %taken2 ], [ %obj1, %untaken2 ]
+ call void @do_safepoint() [ "deopt"() ]
+ ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test3(i64 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test3
+; CHECK: insertelement
+; CHECK: extractelement
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%obj.base, %obj)
+entry:
+ %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %ptr, i32 0
+ %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+ call void @do_safepoint() [ "deopt"() ]
+ ret i64 addrspace(1)* %obj
+}
+
+define i64 addrspace(1)* @test4(i64 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test4
+; CHECK: statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: ; (%obj.base, %obj)
+; When we can optimize an extractelement from a known
+; index and avoid introducing new base pointer instructions
+entry:
+ %derived = getelementptr i64, i64 addrspace(1)* %ptr, i64 16
+ %veca = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %derived, i32 0
+ %vec = insertelement <2 x i64 addrspace(1)*> %veca, i64 addrspace(1)* %ptr, i32 1
+ %obj = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+ call void @do_safepoint() [ "deopt"() ]
+ ret i64 addrspace(1)* %obj
+}
+
+declare void @use(i64 addrspace(1)*) "gc-leaf-function"
+declare void @use_vec(<4 x i64 addrspace(1)*>) "gc-leaf-function"
+
+define void @test5(i1 %cnd, i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: @test5
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+; When we fundementally have to duplicate
+entry:
+ %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+ %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+ %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i32 0
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ call void @use(i64 addrspace(1)* %bdv)
+ ret void
+}
+
+define void @test6(i1 %cnd, i64 addrspace(1)* %obj, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test6
+; CHECK: %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+; CHECK: %vec.base = insertelement <2 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 0, !is_base_value !0
+; CHECK: %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+; CHECK: %bdv.base = extractelement <2 x i64 addrspace(1)*> %vec.base, i64 %idx, !is_base_value !0
+; CHECK: %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+; A more complicated example involving vector and scalar bases.
+; This is derived from a failing test case when we didn't have correct
+; insertelement handling.
+entry:
+ %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+ %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+ %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ call void @use(i64 addrspace(1)* %bdv)
+ ret void
+}
+
+define i64 addrspace(1)* @test7(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test7
+entry:
+ %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %obj2, i32 0
+ br label %merge1
+
+merge1: ; preds = %merge1, %entry
+; CHECK-LABEL: merge1:
+; CHECK: vec2.base
+; CHECK: vec2
+; CHECK: gep
+; CHECK: vec3.base
+; CHECK: vec3
+ %vec2 = phi <2 x i64 addrspace(1)*> [ %vec, %entry ], [ %vec3, %merge1 ]
+ %gep = getelementptr i64, i64 addrspace(1)* %obj2, i64 1
+ %vec3 = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+ br i1 %cnd, label %merge1, label %next1
+
+next1: ; preds = %merge1
+; CHECK-LABEL: next1:
+; CHECK: bdv.base =
+; CHECK: bdv =
+ %bdv = extractelement <2 x i64 addrspace(1)*> %vec2, i32 0
+ br label %merge
+
+merge: ; preds = %merge, %next1
+; CHECK-LABEL: merge:
+; CHECK: %objb.base
+; CHECK: %objb
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%objb.base, %objb)
+ %objb = phi i64 addrspace(1)* [ %obj, %next1 ], [ %bdv, %merge ]
+ br i1 %cnd, label %merge, label %next
+
+next: ; preds = %merge
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ ret i64 addrspace(1)* %objb
+}
+
+; identify base for shufflevector
+define void @test8(i64 addrspace(1)* %obj, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test8
+; CHECK: %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+; CHECK: %gep2 = getelementptr i64, i64 addrspace(1)* %obj, i64 2
+; CHECK: %vec1.base = insertelement <4 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 0, !is_base_value !0
+; CHECK: %vec1 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+; CHECK: %vec2.base = insertelement <4 x i64 addrspace(1)*> zeroinitializer, i64 addrspace(1)* %obj, i32 2, !is_base_value !0
+; CHECK: %vec2 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep2, i32 2
+; CHECK: %vec.base = shufflevector <4 x i64 addrspace(1)*> %vec1.base, <4 x i64 addrspace(1)*> %vec2.base, <2 x i32> <i32 0, i32 2>, !is_base_value !0
+; CHECK: %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec2, <2 x i32> <i32 0, i32 2>
+; CHECK: %bdv.base = extractelement <2 x i64 addrspace(1)*> %vec.base, i64 %idx, !is_base_value !0
+; CHECK: %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+entry:
+ %gep = getelementptr i64, i64 addrspace(1)* %obj, i64 1
+ %gep2 = getelementptr i64, i64 addrspace(1)* %obj, i64 2
+ %vec1 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep, i32 0
+ %vec2 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %gep2, i32 2
+ %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec2, <2 x i32> <i32 0, i32 2>
+ %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ call void @use(i64 addrspace(1)* %bdv)
+ ret void
+}
+
+; Since the same 'base' vector is used in the shuffle operands, we do not need
+; create a shufflevector base.
+define void @test9(<4 x i64 addrspace(1)*> %vec1, i64 %idx) gc "statepoint-example" {
+; CHECK-LABEL: @test9
+; CHECK: %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec1, <2 x i32> <i32 0, i32 2>
+; CHECK: %base_ee = extractelement <4 x i64 addrspace(1)*> %vec1, i64 %idx, !is_base_value !0
+; CHECK: %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%base_ee, %bdv)
+entry:
+ ; shrinking vec1 into vec
+ %vec = shufflevector <4 x i64 addrspace(1)*> %vec1, <4 x i64 addrspace(1)*> %vec1, <2 x i32> <i32 0, i32 2>
+ %bdv = extractelement <2 x i64 addrspace(1)*> %vec, i64 %idx
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ call void @use(i64 addrspace(1)* %bdv)
+ ret void
+}
+
+; vector operand of shufflevector is a phi
+define i64 addrspace(1)* @test10(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj2) gc "statepoint-example" {
+; CHECK-LABEL: @test10
+entry:
+ %vec1 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %obj, i32 0
+ br i1 %cnd, label %here, label %merge
+
+here:
+ %vec2 = insertelement <4 x i64 addrspace(1)*> poison, i64 addrspace(1)* %obj2, i32 2
+ br label %merge
+
+merge: ; preds = %merge, %entry, %here
+; CHECK-LABEL: merge:
+; CHECK: %vec.base = phi <4 x i64 addrspace(1)*> [ %vec1.base, %entry ], [ %vec2.base, %here ], [ %vec3.base, %merge ], !is_base_value !0
+; CHECK: vec
+; CHECK: vec3.base = shufflevector <4 x i64 addrspace(1)*> %vec.base, <4 x i64 addrspace(1)*> %vec.base
+; CHECK: vec3
+; CHECK: bdv.base
+; CHECK: bdv
+ %vec = phi <4 x i64 addrspace(1)*> [ %vec1, %entry ], [ %vec2, %here], [ %vec3, %merge]
+ %vec3 = shufflevector <4 x i64 addrspace(1)*> %vec, <4 x i64 addrspace(1)*> %vec, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
+ %bdv = extractelement <4 x i64 addrspace(1)*> %vec3, i32 0
+ br i1 %cnd, label %merge, label %next
+
+next:
+; CHECK-LABEL: next:
+; CHECK: gc.statepoint
+; CHECK: gc.relocate
+; CHECK-DAG: (%bdv.base, %bdv)
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ ret i64 addrspace(1)* %bdv
+}
+declare void @do_safepoint()
+
+define void @test11(<4 x i64 addrspace(1)*> %vec1) gc "statepoint-example" {
+; CHECK-LABEL: @test11(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf{{.*}}<4 x i64 addrspace(1)*> %vec1)
+; CHECK: %vec1.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8
+; CHECK: %vec1.relocated.casted = bitcast <4 x i8 addrspace(1)*> %vec1.relocated to <4 x i64 addrspace(1)*>
+; CHECK: %vec2.remat = getelementptr i64, <4 x i64 addrspace(1)*> %vec1.relocated.casted, i32 1024
+; CHECK: call void @use_vec(<4 x i64 addrspace(1)*> %vec2.remat)
+entry:
+ %vec2 = getelementptr i64, <4 x i64 addrspace(1)*> %vec1, i32 1024
+ call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+ call void @use_vec(<4 x i64 addrspace(1) *> %vec2)
+ ret void
+}
+
+declare <4 x i64 addrspace(1)*> @def_vec() "gc-leaf-function"
+
+define void @test12(<4 x i64 addrspace(1)*> %vec1) gc "statepoint-example" {
+; CHECK-LABEL: @test12(
+; CHECK: @llvm.experimental.gc.statepoint.p0f_isVoidf{{.*}}<4 x i64 addrspace(1)*> %vec)
+; CHECK-NEXT: %vec.relocated = call coldcc <4 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v4p1i8(
+; CHECK-NEXT: %vec.relocated.casted = bitcast <4 x i8 addrspace(1)*> %vec.relocated to <4 x i64 addrspace(1)*>
+; CHECK-NEXT: call void @use_vec(<4 x i64 addrspace(1)*> %vec.relocated.casted)
+; CHECK-NEXT: ret void
+entry:
+ %vec = call <4 x i64 addrspace(1)*> @def_vec()
+ call void @do_safepoint() [ "deopt"() ]
+ call void @use_vec(<4 x i64 addrspace(1)*> %vec)
+ ret void
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order-inseltpoison.ll
new file mode 100644
index 000000000000..7a7d65117cab
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/check_traversal_order-inseltpoison.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -rewrite-statepoints-for-gc < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare void @f()
+declare void @g(i8 addrspace(1)*, i8 addrspace(1)*)
+declare i32 @personality_function()
+
+; Make sure that we do not fail assertion because we process call of @g before
+; we process the call of @f.
+
+define void @test_01(i8 addrspace(1)* %p, i1 %cond) gc "statepoint-example" personality i32 ()* @personality_function {
+
+; CHECK-LABEL: @test_01(
+
+entry:
+ %tmp0 = insertelement <2 x i8 addrspace(1)*> poison, i8 addrspace(1)* %p, i32 0
+ %tmp1 = insertelement <2 x i8 addrspace(1)*> %tmp0, i8 addrspace(1)* %p, i32 1
+ %tmp2 = extractelement <2 x i8 addrspace(1)*> %tmp1, i32 1
+ %tmp3 = extractelement <2 x i8 addrspace(1)*> %tmp1, i32 0
+ br label %loop
+
+loop:
+ br i1 %cond, label %cond_block, label %exit
+
+cond_block:
+ br i1 %cond, label %backedge, label %exit
+
+exit:
+ %tmp4 = phi i8 addrspace(1)* [ %tmp2, %loop ], [ %tmp2, %cond_block ]
+ call void @g(i8 addrspace(1)* %tmp3, i8 addrspace(1)* %tmp4)
+ ret void
+
+backedge:
+ call void @f()
+ br label %loop
+}
diff --git a/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
new file mode 100644
index 000000000000..3051655f4e29
--- /dev/null
+++ b/llvm/test/Transforms/RewriteStatepointsForGC/live-vector-nosplit-inseltpoison.ll
@@ -0,0 +1,119 @@
+; Test that we can correctly handle vectors of pointers in statepoint
+; rewriting.
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck %s
+
+; A non-vector relocation for comparison
+define i64 addrspace(1)* @test(i64 addrspace(1)* %obj) gc "statepoint-example" {
+; CHECK-LABEL: test
+; CHECK: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret i64 addrspace(1)*
+; A base vector from a argument
+entry:
+ call void @do_safepoint() [ "deopt"() ]
+ ret i64 addrspace(1)* %obj
+}
+
+; A vector argument
+define <2 x i64 addrspace(1)*> @test2(<2 x i64 addrspace(1)*> %obj) gc "statepoint-example" {
+; CHECK-LABEL: test2
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+ call void @do_safepoint() [ "deopt"() ]
+ ret <2 x i64 addrspace(1)*> %obj
+}
+
+; A load
+define <2 x i64 addrspace(1)*> @test3(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test3
+; CHECK: load
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+entry:
+ %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ call void @do_safepoint() [ "deopt"() ]
+ ret <2 x i64 addrspace(1)*> %obj
+}
+
+declare i32 @fake_personality_function()
+
+; When a statepoint is an invoke rather than a call
+define <2 x i64 addrspace(1)*> @test4(<2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" personality i32 ()* @fake_personality_function {
+; CHECK-LABEL: test4
+; CHECK: load
+; CHECK-NEXT: gc.statepoint
+entry:
+ %obj = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ invoke void @do_safepoint() [ "deopt"() ]
+ to label %normal_return unwind label %exceptional_return
+
+normal_return: ; preds = %entry
+; CHECK-LABEL: normal_return:
+; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+ ret <2 x i64 addrspace(1)*> %obj
+
+exceptional_return: ; preds = %entry
+; CHECK-LABEL: exceptional_return:
+; CHECK: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+ %landing_pad4 = landingpad token
+ cleanup
+ ret <2 x i64 addrspace(1)*> %obj
+}
+
+; A newly created vector
+define <2 x i64 addrspace(1)*> @test5(i64 addrspace(1)* %p) gc "statepoint-example" {
+; CHECK-LABEL: test5
+; CHECK: insertelement
+; CHECK-NEXT: insertelement
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*> %vec.relocated.casted
+entry:
+ %vec = insertelement <2 x i64 addrspace(1)*> poison, i64 addrspace(1)* %p, i32 0
+ call void @do_safepoint() [ "deopt"() ]
+ ret <2 x i64 addrspace(1)*> %vec
+}
+
+; A merge point
+define <2 x i64 addrspace(1)*> @test6(i1 %cnd, <2 x i64 addrspace(1)*>* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test6
+entry:
+ br i1 %cnd, label %taken, label %untaken
+
+taken: ; preds = %entry
+ %obja = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ br label %merge
+
+untaken: ; preds = %entry
+ %objb = load <2 x i64 addrspace(1)*>, <2 x i64 addrspace(1)*>* %ptr
+ br label %merge
+
+merge: ; preds = %untaken, %taken
+; CHECK-LABEL: merge:
+; CHECK-NEXT: = phi
+; CHECK-NEXT: = phi
+; CHECK-NEXT: gc.statepoint
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: gc.relocate
+; CHECK-NEXT: bitcast
+; CHECK-NEXT: ret <2 x i64 addrspace(1)*>
+ %obj = phi <2 x i64 addrspace(1)*> [ %obja, %taken ], [ %objb, %untaken ]
+ call void @do_safepoint() [ "deopt"() ]
+ ret <2 x i64 addrspace(1)*> %obj
+}
+
+declare void @do_safepoint()
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
new file mode 100644
index 000000000000..2e437ead6d2c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/accelerate-vector-functions-inseltpoison.ll
@@ -0,0 +1,1300 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -inject-tli-mappings -slp-vectorizer -vector-library=Accelerate -S %s | FileCheck %s
+; RUN: opt -inject-tli-mappings -slp-vectorizer -S %s | FileCheck --check-prefix NOACCELERATE %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios14.0.0"
+
+declare float @llvm.sin.f32(float)
+
+; Accelerate provides sin() for <4 x float>
+define <4 x float> @int_sin_4x(<4 x float>* %a) {
+; CHECK-LABEL: @int_sin_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vsinf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @int_sin_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @llvm.sin.f32(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @llvm.sin.f32(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @llvm.sin.f32(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @llvm.sin.f32(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+declare float @ceilf(float) readonly
+
+define <4 x float> @ceil_4x(<4 x float>* %a) {
+; CHECK-LABEL: @ceil_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @ceil_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.ceil.v4f32(<4 x float> [[TMP0]])
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; NOACCELERATE-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @ceilf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @ceilf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @ceilf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @ceilf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+declare float @fabsf(float) readonly
+
+define <4 x float> @fabs_4x(<4 x float>* %a) {
+; CHECK-LABEL: @fabs_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @fabs_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP0]])
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; NOACCELERATE-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @fabsf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @fabsf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @fabsf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @fabsf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @llvm.fabs.f32(float)
+define <4 x float> @int_fabs_4x(<4 x float>* %a) {
+; CHECK-LABEL: @int_fabs_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @int_fabs_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.fabs.v4f32(<4 x float> [[TMP0]])
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; NOACCELERATE-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @llvm.fabs.f32(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @llvm.fabs.f32(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @llvm.fabs.f32(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @llvm.fabs.f32(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @floorf(float) readonly
+define <4 x float> @floor_4x(<4 x float>* %a) {
+; CHECK-LABEL: @floor_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @floor_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.floor.v4f32(<4 x float> [[TMP0]])
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; NOACCELERATE-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @floorf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @floorf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @floorf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @floorf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @sqrtf(float) readonly
+define <4 x float> @sqrt_4x(<4 x float>* %a) {
+; CHECK-LABEL: @sqrt_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @sqrt_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[TMP0]])
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; NOACCELERATE-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @sqrtf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @sqrtf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @sqrtf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @sqrtf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @expf(float) readonly
+define <4 x float> @exp_4x(<4 x float>* %a) {
+; CHECK-LABEL: @exp_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vexpf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @exp_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @expf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @expf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @expf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @expf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @expf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @expf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @expf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @expf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @expm1f(float) readonly
+define <4 x float> @expm1_4x(<4 x float>* %a) {
+; CHECK-LABEL: @expm1_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vexpm1f(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @expm1_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @expm1f(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @expm1f(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @expm1f(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @expm1f(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @expm1f(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @expm1f(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @expm1f(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @expm1f(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @logf(float) readonly
+define <4 x float> @log_4x(<4 x float>* %a) {
+; CHECK-LABEL: @log_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vlogf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @log_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @logf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @logf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @logf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @logf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @logf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @logf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @logf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @logf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @log1pf(float) readonly
+define <4 x float> @log1p_4x(<4 x float>* %a) {
+; CHECK-LABEL: @log1p_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vlog1pf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @log1p_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @log1pf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @log1pf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @log1pf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @log1pf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @log1pf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @log1pf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @log1pf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @log1pf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @log10pf(float) readonly
+define <4 x float> @log10p_4x(<4 x float>* %a) {
+; CHECK-LABEL: @log10p_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = tail call fast float @log10pf(float [[VECEXT]])
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = tail call fast float @log10pf(float [[VECEXT_1]])
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; CHECK-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = tail call fast float @log10pf(float [[VECEXT_2]])
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; CHECK-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = tail call fast float @log10pf(float [[VECEXT_3]])
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @log10p_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @log10pf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @log10pf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @log10pf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @log10pf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @log10pf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @log10pf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @log10pf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @log10pf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @logbf(float) readonly
+define <4 x float> @logb_4x(<4 x float>* %a) {
+; CHECK-LABEL: @logb_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vlogbf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @logb_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @logbf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @logbf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @logbf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @logbf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @logbf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @logbf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @logbf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @logbf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @sinf(float) readonly
+define <4 x float> @sin_4x(<4 x float>* %a) {
+; CHECK-LABEL: @sin_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vsinf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @sin_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @sinf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @sinf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @sinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @sinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @sinf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @sinf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @sinf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @sinf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @cosf(float) readonly
+define <4 x float> @cos_4x(<4 x float>* %a) {
+; CHECK-LABEL: @cos_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vcosf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @cos_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @cosf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @cosf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @cosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @cosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @cosf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @cosf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @cosf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @cosf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @tanf(float) readonly
+define <4 x float> @tan_4x(<4 x float>* %a) {
+; CHECK-LABEL: @tan_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vtanf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @tan_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @tanf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @tanf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @tanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @tanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @tanf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @tanf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @tanf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @tanf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @asinf(float) readonly
+define <4 x float> @asin_4x(<4 x float>* %a) {
+; CHECK-LABEL: @asin_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vasinf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @asin_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @asinf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @asinf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @asinf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @asinf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @asinf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @asinf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @asinf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @asinf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @acosf(float) readonly
+define <4 x float> @acos_4x(<4 x float>* %a) {
+; CHECK-LABEL: @acos_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vacosf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @acos_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @acosf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @acosf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @acosf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @acosf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @acosf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @acosf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @acosf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @acosf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @atanf(float) readonly
+define <4 x float> @atan_4x(<4 x float>* %a) {
+; CHECK-LABEL: @atan_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vatanf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @atan_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @atanf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @atanf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @atanf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @atanf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @atanf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @atanf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @atanf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @atanf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @sinhf(float) readonly
+define <4 x float> @sinh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @sinh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vsinhf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @sinh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @sinhf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @sinhf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @sinhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @sinhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @sinhf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @sinhf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @sinhf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @sinhf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @coshf(float) readonly
+define <4 x float> @cosh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @cosh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vcoshf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @cosh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @coshf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @coshf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @coshf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @coshf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @coshf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @coshf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @coshf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @coshf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @tanhf(float) readonly
+define <4 x float> @tanh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @tanh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vtanhf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @tanh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @tanhf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @tanhf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @tanhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @tanhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @tanhf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @tanhf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @tanhf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @tanhf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @asinhf(float) readonly
+define <4 x float> @asinh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @asinh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vasinhf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @asinh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @asinhf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @asinhf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @asinhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @asinhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @asinhf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @asinhf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @asinhf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @asinhf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @acoshf(float) readonly
+define <4 x float> @acosh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @acosh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vacoshf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @acosh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @acoshf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @acoshf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @acoshf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @acoshf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @acoshf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @acoshf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @acoshf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @acoshf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+declare float @atanhf(float) readonly
+define <4 x float> @atanh_4x(<4 x float>* %a) {
+; CHECK-LABEL: @atanh_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vatanhf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @atanh_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @atanhf(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @atanhf(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @atanhf(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @atanhf(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @atanhf(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @atanhf(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @atanhf(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @atanhf(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+; Accelerate *does not* provide sin() for <2 x float>.
+define <2 x float> @sin_2x(<2 x float>* %a) {
+; CHECK-LABEL: @sin_2x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, <2 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]]) [[ATTR2:#.*]]
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]]) [[ATTR2]]
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
+; CHECK-NEXT: ret <2 x float> [[VECINS_1]]
+;
+; NOACCELERATE-LABEL: @sin_2x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <2 x float>, <2 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.sin.f32(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: ret <2 x float> [[VECINS_1]]
+;
+entry:
+ %0 = load <2 x float>, <2 x float>* %a, align 16
+ %vecext = extractelement <2 x float> %0, i32 0
+ %1 = tail call fast float @llvm.sin.f32(float %vecext)
+ %vecins = insertelement <2 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <2 x float> %0, i32 1
+ %2 = tail call fast float @llvm.sin.f32(float %vecext.1)
+ %vecins.1 = insertelement <2 x float> %vecins, float %2, i32 1
+ ret <2 x float> %vecins.1
+}
+
+
+declare float @llvm.cos.f32(float)
+
+; Accelerate provides cos() for <4 x float>
+define <4 x float> @int_cos_4x(<4 x float>* %a) {
+; CHECK-LABEL: @int_cos_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vcosf(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+; NOACCELERATE-LABEL: @int_cos_4x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOACCELERATE-NEXT: [[TMP3:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_2]])
+; NOACCELERATE-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; NOACCELERATE-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOACCELERATE-NEXT: [[TMP4:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_3]])
+; NOACCELERATE-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; NOACCELERATE-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @llvm.cos.f32(float %vecext)
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @llvm.cos.f32(float %vecext.1)
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @llvm.cos.f32(float %vecext.2)
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @llvm.cos.f32(float %vecext.3)
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+; Accelerate *does not* provide cos() for <2 x float>.
+define <2 x float> @cos_2x(<2 x float>* %a) {
+; CHECK-LABEL: @cos_2x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, <2 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]]) [[ATTR3:#.*]]
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]]) [[ATTR3]]
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
+; CHECK-NEXT: ret <2 x float> [[VECINS_1]]
+;
+; NOACCELERATE-LABEL: @cos_2x(
+; NOACCELERATE-NEXT: entry:
+; NOACCELERATE-NEXT: [[TMP0:%.*]] = load <2 x float>, <2 x float>* [[A:%.*]], align 16
+; NOACCELERATE-NEXT: [[VECEXT:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; NOACCELERATE-NEXT: [[TMP1:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT]])
+; NOACCELERATE-NEXT: [[VECINS:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; NOACCELERATE-NEXT: [[VECEXT_1:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; NOACCELERATE-NEXT: [[TMP2:%.*]] = tail call fast float @llvm.cos.f32(float [[VECEXT_1]])
+; NOACCELERATE-NEXT: [[VECINS_1:%.*]] = insertelement <2 x float> [[VECINS]], float [[TMP2]], i32 1
+; NOACCELERATE-NEXT: ret <2 x float> [[VECINS_1]]
+;
+entry:
+ %0 = load <2 x float>, <2 x float>* %a, align 16
+ %vecext = extractelement <2 x float> %0, i32 0
+ %1 = tail call fast float @llvm.cos.f32(float %vecext)
+ %vecins = insertelement <2 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <2 x float> %0, i32 1
+ %2 = tail call fast float @llvm.cos.f32(float %vecext.1)
+ %vecins.1 = insertelement <2 x float> %vecins, float %2, i32 1
+ ret <2 x float> %vecins.1
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/insertelement-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/insertelement-inseltpoison.ll
new file mode 100644
index 000000000000..bf81c9e2f10d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/insertelement-inseltpoison.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S 2>%t | FileCheck %s
+; RUN: FileCheck --check-prefix=WARN --allow-empty %s <%t
+
+; WARN-NOT: warning
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-linux-gnu"
+
+define <2 x float> @insertelement-fixed-vector() {
+; CHECK-LABEL: @insertelement-fixed-vector(
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <2 x float> @llvm.fabs.v2f32(<2 x float> undef)
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[I0:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = insertelement <2 x float> [[I0]], float [[TMP3]], i32 1
+; CHECK-NEXT: ret <2 x float> [[I1]]
+;
+ %f0 = tail call fast float @llvm.fabs.f32(float undef)
+ %f1 = tail call fast float @llvm.fabs.f32(float undef)
+ %i0 = insertelement <2 x float> poison, float %f0, i32 0
+ %i1 = insertelement <2 x float> %i0, float %f1, i32 1
+ ret <2 x float> %i1
+}
+
+; TODO: llvm.fabs could be optimized in vector form. It's legal to extract
+; elements from fixed-length vector and insert into scalable vector.
+define <vscale x 2 x float> @insertelement-scalable-vector() {
+; CHECK-LABEL: @insertelement-scalable-vector(
+; CHECK-NEXT: [[F0:%.*]] = tail call fast float @llvm.fabs.f32(float undef)
+; CHECK-NEXT: [[F1:%.*]] = tail call fast float @llvm.fabs.f32(float undef)
+; CHECK-NEXT: [[I0:%.*]] = insertelement <vscale x 2 x float> poison, float [[F0]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <vscale x 2 x float> [[I0]], float [[F1]], i32 1
+; CHECK-NEXT: ret <vscale x 2 x float> [[I1]]
+;
+ %f0 = tail call fast float @llvm.fabs.f32(float undef)
+ %f1 = tail call fast float @llvm.fabs.f32(float undef)
+ %i0 = insertelement <vscale x 2 x float> poison, float %f0, i32 0
+ %i1 = insertelement <vscale x 2 x float> %i0, float %f1, i32 1
+ ret <vscale x 2 x float> %i1
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare float @llvm.fabs.f32(float)
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
new file mode 100644
index 000000000000..5f953aa6d995
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/transpose-inseltpoison.ll
@@ -0,0 +1,294 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-gnu"
+
+define <2 x i64> @build_vec_v2i64(<2 x i64> %v0, <2 x i64> %v1) {
+; CHECK-LABEL: @build_vec_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[V0:%.*]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[V1:%.*]], <2 x i64> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i64> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub <2 x i64> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP6]], <2 x i64> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i64> [[TMP8]], [[TMP5]]
+; CHECK-NEXT: ret <2 x i64> [[TMP9]]
+;
+ %v0.0 = extractelement <2 x i64> %v0, i32 0
+ %v0.1 = extractelement <2 x i64> %v0, i32 1
+ %v1.0 = extractelement <2 x i64> %v1, i32 0
+ %v1.1 = extractelement <2 x i64> %v1, i32 1
+ %tmp0.0 = add i64 %v0.0, %v1.0
+ %tmp0.1 = add i64 %v0.1, %v1.1
+ %tmp1.0 = sub i64 %v0.0, %v1.0
+ %tmp1.1 = sub i64 %v0.1, %v1.1
+ %tmp2.0 = add i64 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i64 %tmp1.0, %tmp1.1
+ %tmp3.0 = insertelement <2 x i64> poison, i64 %tmp2.0, i32 0
+ %tmp3.1 = insertelement <2 x i64> %tmp3.0, i64 %tmp2.1, i32 1
+ ret <2 x i64> %tmp3.1
+}
+
+define void @store_chain_v2i64(i64* %a, i64* %b, i64* %c) {
+; CHECK-LABEL: @store_chain_v2i64(
+; CHECK-NEXT: [[A_1:%.*]] = getelementptr i64, i64* [[A:%.*]], i64 1
+; CHECK-NEXT: [[B_1:%.*]] = getelementptr i64, i64* [[B:%.*]], i64 1
+; CHECK-NEXT: [[C_1:%.*]] = getelementptr i64, i64* [[C:%.*]], i64 1
+; CHECK-NEXT: [[V0_0:%.*]] = load i64, i64* [[A]], align 8
+; CHECK-NEXT: [[V0_1:%.*]] = load i64, i64* [[A_1]], align 8
+; CHECK-NEXT: [[V1_0:%.*]] = load i64, i64* [[B]], align 8
+; CHECK-NEXT: [[V1_1:%.*]] = load i64, i64* [[B_1]], align 8
+; CHECK-NEXT: [[TMP0_0:%.*]] = add i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP0_1:%.*]] = add i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP1_0:%.*]] = sub i64 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP1_1:%.*]] = sub i64 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP2_0:%.*]] = add i64 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT: [[TMP2_1:%.*]] = add i64 [[TMP1_0]], [[TMP1_1]]
+; CHECK-NEXT: store i64 [[TMP2_0]], i64* [[C]], align 8
+; CHECK-NEXT: store i64 [[TMP2_1]], i64* [[C_1]], align 8
+; CHECK-NEXT: ret void
+;
+ %a.0 = getelementptr i64, i64* %a, i64 0
+ %a.1 = getelementptr i64, i64* %a, i64 1
+ %b.0 = getelementptr i64, i64* %b, i64 0
+ %b.1 = getelementptr i64, i64* %b, i64 1
+ %c.0 = getelementptr i64, i64* %c, i64 0
+ %c.1 = getelementptr i64, i64* %c, i64 1
+ %v0.0 = load i64, i64* %a.0, align 8
+ %v0.1 = load i64, i64* %a.1, align 8
+ %v1.0 = load i64, i64* %b.0, align 8
+ %v1.1 = load i64, i64* %b.1, align 8
+ %tmp0.0 = add i64 %v0.0, %v1.0
+ %tmp0.1 = add i64 %v0.1, %v1.1
+ %tmp1.0 = sub i64 %v0.0, %v1.0
+ %tmp1.1 = sub i64 %v0.1, %v1.1
+ %tmp2.0 = add i64 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i64 %tmp1.0, %tmp1.1
+ store i64 %tmp2.0, i64* %c.0, align 8
+ store i64 %tmp2.1, i64* %c.1, align 8
+ ret void
+}
+
+define <4 x i32> @build_vec_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP6:%.*]] = add <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT: ret <4 x i32> [[TMP9]]
+;
+ %v0.0 = extractelement <4 x i32> %v0, i32 0
+ %v0.1 = extractelement <4 x i32> %v0, i32 1
+ %v0.2 = extractelement <4 x i32> %v0, i32 2
+ %v0.3 = extractelement <4 x i32> %v0, i32 3
+ %v1.0 = extractelement <4 x i32> %v1, i32 0
+ %v1.1 = extractelement <4 x i32> %v1, i32 1
+ %v1.2 = extractelement <4 x i32> %v1, i32 2
+ %v1.3 = extractelement <4 x i32> %v1, i32 3
+ %tmp0.0 = add i32 %v0.0, %v1.0
+ %tmp0.1 = add i32 %v0.1, %v1.1
+ %tmp0.2 = add i32 %v0.2, %v1.2
+ %tmp0.3 = add i32 %v0.3, %v1.3
+ %tmp1.0 = sub i32 %v0.0, %v1.0
+ %tmp1.1 = sub i32 %v0.1, %v1.1
+ %tmp1.2 = sub i32 %v0.2, %v1.2
+ %tmp1.3 = sub i32 %v0.3, %v1.3
+ %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+ %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+ %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+ %tmp3.0 = insertelement <4 x i32> poison, i32 %tmp2.0, i32 0
+ %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+ %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.2, i32 2
+ %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.3, i32 3
+ ret <4 x i32> %tmp3.3
+}
+
+define <4 x i32> @build_vec_v4i32_reuse_0(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_reuse_0(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i32> [[V0:%.*]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[V1:%.*]], <2 x i32> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <2 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = add <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP7:%.*]] = sub <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP6]], <2 x i32> [[TMP7]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT: [[TMP9:%.*]] = add <2 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT: [[TMP3_3:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+; CHECK-NEXT: ret <4 x i32> [[TMP3_3]]
+;
+ %v0.0 = extractelement <2 x i32> %v0, i32 0
+ %v0.1 = extractelement <2 x i32> %v0, i32 1
+ %v1.0 = extractelement <2 x i32> %v1, i32 0
+ %v1.1 = extractelement <2 x i32> %v1, i32 1
+ %tmp0.0 = add i32 %v0.0, %v1.0
+ %tmp0.1 = add i32 %v0.1, %v1.1
+ %tmp1.0 = sub i32 %v0.0, %v1.0
+ %tmp1.1 = sub i32 %v0.1, %v1.1
+ %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+ %tmp3.0 = insertelement <4 x i32> poison, i32 %tmp2.0, i32 0
+ %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+ %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.0, i32 2
+ %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.1, i32 3
+ ret <4 x i32> %tmp3.3
+}
+
+define <4 x i32> @build_vec_v4i32_reuse_1(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_reuse_1(
+; CHECK-NEXT: [[V0_0:%.*]] = extractelement <2 x i32> [[V0:%.*]], i32 0
+; CHECK-NEXT: [[V0_1:%.*]] = extractelement <2 x i32> [[V0]], i32 1
+; CHECK-NEXT: [[V1_0:%.*]] = extractelement <2 x i32> [[V1:%.*]], i32 0
+; CHECK-NEXT: [[V1_1:%.*]] = extractelement <2 x i32> [[V1]], i32 1
+; CHECK-NEXT: [[TMP0_0:%.*]] = add i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP0_1:%.*]] = add i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP0_2:%.*]] = xor i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP0_3:%.*]] = xor i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP1_0:%.*]] = sub i32 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT: [[TMP1_1:%.*]] = sub i32 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT: [[TMP1_2:%.*]] = sub i32 [[TMP0_2]], [[TMP0_3]]
+; CHECK-NEXT: [[TMP1_3:%.*]] = sub i32 [[TMP0_3]], [[TMP0_2]]
+; CHECK-NEXT: [[TMP2_0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1_0]], i32 0
+; CHECK-NEXT: [[TMP2_1:%.*]] = insertelement <4 x i32> [[TMP2_0]], i32 [[TMP1_1]], i32 1
+; CHECK-NEXT: [[TMP2_2:%.*]] = insertelement <4 x i32> [[TMP2_1]], i32 [[TMP1_2]], i32 2
+; CHECK-NEXT: [[TMP2_3:%.*]] = insertelement <4 x i32> [[TMP2_2]], i32 [[TMP1_3]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[TMP2_3]]
+;
+ %v0.0 = extractelement <2 x i32> %v0, i32 0
+ %v0.1 = extractelement <2 x i32> %v0, i32 1
+ %v1.0 = extractelement <2 x i32> %v1, i32 0
+ %v1.1 = extractelement <2 x i32> %v1, i32 1
+ %tmp0.0 = add i32 %v0.0, %v1.0
+ %tmp0.1 = add i32 %v0.1, %v1.1
+ %tmp0.2 = xor i32 %v0.0, %v1.0
+ %tmp0.3 = xor i32 %v0.1, %v1.1
+ %tmp1.0 = sub i32 %tmp0.0, %tmp0.1
+ %tmp1.1 = sub i32 %tmp0.0, %tmp0.1
+ %tmp1.2 = sub i32 %tmp0.2, %tmp0.3
+ %tmp1.3 = sub i32 %tmp0.3, %tmp0.2
+ %tmp2.0 = insertelement <4 x i32> poison, i32 %tmp1.0, i32 0
+ %tmp2.1 = insertelement <4 x i32> %tmp2.0, i32 %tmp1.1, i32 1
+ %tmp2.2 = insertelement <4 x i32> %tmp2.1, i32 %tmp1.2, i32 2
+ %tmp2.3 = insertelement <4 x i32> %tmp2.2, i32 %tmp1.3, i32 3
+ ret <4 x i32> %tmp2.3
+}
+
+define <4 x i32> @build_vec_v4i32_3_binops(<2 x i32> %v0, <2 x i32> %v1) {
+; CHECK-LABEL: @build_vec_v4i32_3_binops(
+; CHECK-NEXT: [[V0_0:%.*]] = extractelement <2 x i32> [[V0:%.*]], i32 0
+; CHECK-NEXT: [[V0_1:%.*]] = extractelement <2 x i32> [[V0]], i32 1
+; CHECK-NEXT: [[V1_0:%.*]] = extractelement <2 x i32> [[V1:%.*]], i32 0
+; CHECK-NEXT: [[V1_1:%.*]] = extractelement <2 x i32> [[V1]], i32 1
+; CHECK-NEXT: [[TMP0_0:%.*]] = add i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP0_1:%.*]] = add i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP1_0:%.*]] = mul i32 [[V0_0]], [[V1_0]]
+; CHECK-NEXT: [[TMP1_1:%.*]] = mul i32 [[V0_1]], [[V1_1]]
+; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = xor <2 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+; CHECK-NEXT: [[TMP2_0:%.*]] = add i32 [[TMP0_0]], [[TMP0_1]]
+; CHECK-NEXT: [[TMP2_1:%.*]] = add i32 [[TMP1_0]], [[TMP1_1]]
+; CHECK-NEXT: [[TMP5:%.*]] = add <2 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP3_0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2_0]], i32 0
+; CHECK-NEXT: [[TMP3_1:%.*]] = insertelement <4 x i32> [[TMP3_0]], i32 [[TMP2_1]], i32 1
+; CHECK-NEXT: [[TMP3_3:%.*]] = shufflevector <4 x i32> [[TMP3_1]], <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
+; CHECK-NEXT: ret <4 x i32> [[TMP3_3]]
+;
+ %v0.0 = extractelement <2 x i32> %v0, i32 0
+ %v0.1 = extractelement <2 x i32> %v0, i32 1
+ %v1.0 = extractelement <2 x i32> %v1, i32 0
+ %v1.1 = extractelement <2 x i32> %v1, i32 1
+ %tmp0.0 = add i32 %v0.0, %v1.0
+ %tmp0.1 = add i32 %v0.1, %v1.1
+ %tmp0.2 = xor i32 %v0.0, %v1.0
+ %tmp0.3 = xor i32 %v0.1, %v1.1
+ %tmp1.0 = mul i32 %v0.0, %v1.0
+ %tmp1.1 = mul i32 %v0.1, %v1.1
+ %tmp1.2 = xor i32 %v0.0, %v1.0
+ %tmp1.3 = xor i32 %v0.1, %v1.1
+ %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+ %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+ %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+ %tmp3.0 = insertelement <4 x i32> poison, i32 %tmp2.0, i32 0
+ %tmp3.1 = insertelement <4 x i32> %tmp3.0, i32 %tmp2.1, i32 1
+ %tmp3.2 = insertelement <4 x i32> %tmp3.1, i32 %tmp2.2, i32 2
+ %tmp3.3 = insertelement <4 x i32> %tmp3.2, i32 %tmp2.3, i32 3
+ ret <4 x i32> %tmp3.3
+}
+
+define i32 @reduction_v4i32(<4 x i32> %v0, <4 x i32> %v1) {
+; CHECK-LABEL: @reduction_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[V0:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[V1:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[TMP6:%.*]] = sub <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP7:%.*]] = add <4 x i32> [[V0]], [[V1]]
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> [[TMP7]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: [[TMP9:%.*]] = add <4 x i32> [[TMP8]], [[TMP5]]
+; CHECK-NEXT: [[TMP10:%.*]] = lshr <4 x i32> [[TMP9]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT: [[TMP11:%.*]] = and <4 x i32> [[TMP10]], <i32 65537, i32 65537, i32 65537, i32 65537>
+; CHECK-NEXT: [[TMP12:%.*]] = mul nuw <4 x i32> [[TMP11]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT: [[TMP13:%.*]] = add <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT: [[TMP14:%.*]] = xor <4 x i32> [[TMP13]], [[TMP12]]
+; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
+; CHECK-NEXT: ret i32 [[TMP15]]
+;
+ %v0.0 = extractelement <4 x i32> %v0, i32 0
+ %v0.1 = extractelement <4 x i32> %v0, i32 1
+ %v0.2 = extractelement <4 x i32> %v0, i32 2
+ %v0.3 = extractelement <4 x i32> %v0, i32 3
+ %v1.0 = extractelement <4 x i32> %v1, i32 0
+ %v1.1 = extractelement <4 x i32> %v1, i32 1
+ %v1.2 = extractelement <4 x i32> %v1, i32 2
+ %v1.3 = extractelement <4 x i32> %v1, i32 3
+ %tmp0.0 = add i32 %v0.0, %v1.0
+ %tmp0.1 = add i32 %v0.1, %v1.1
+ %tmp0.2 = add i32 %v0.2, %v1.2
+ %tmp0.3 = add i32 %v0.3, %v1.3
+ %tmp1.0 = sub i32 %v0.0, %v1.0
+ %tmp1.1 = sub i32 %v0.1, %v1.1
+ %tmp1.2 = sub i32 %v0.2, %v1.2
+ %tmp1.3 = sub i32 %v0.3, %v1.3
+ %tmp2.0 = add i32 %tmp0.0, %tmp0.1
+ %tmp2.1 = add i32 %tmp1.0, %tmp1.1
+ %tmp2.2 = add i32 %tmp0.2, %tmp0.3
+ %tmp2.3 = add i32 %tmp1.2, %tmp1.3
+ %tmp3.0 = lshr i32 %tmp2.0, 15
+ %tmp3.1 = lshr i32 %tmp2.1, 15
+ %tmp3.2 = lshr i32 %tmp2.2, 15
+ %tmp3.3 = lshr i32 %tmp2.3, 15
+ %tmp4.0 = and i32 %tmp3.0, 65537
+ %tmp4.1 = and i32 %tmp3.1, 65537
+ %tmp4.2 = and i32 %tmp3.2, 65537
+ %tmp4.3 = and i32 %tmp3.3, 65537
+ %tmp5.0 = mul nuw i32 %tmp4.0, 65535
+ %tmp5.1 = mul nuw i32 %tmp4.1, 65535
+ %tmp5.2 = mul nuw i32 %tmp4.2, 65535
+ %tmp5.3 = mul nuw i32 %tmp4.3, 65535
+ %tmp6.0 = add i32 %tmp5.0, %tmp2.0
+ %tmp6.1 = add i32 %tmp5.1, %tmp2.1
+ %tmp6.2 = add i32 %tmp5.2, %tmp2.2
+ %tmp6.3 = add i32 %tmp5.3, %tmp2.3
+ %tmp7.0 = xor i32 %tmp6.0, %tmp5.0
+ %tmp7.1 = xor i32 %tmp6.1, %tmp5.1
+ %tmp7.2 = xor i32 %tmp6.2, %tmp5.2
+ %tmp7.3 = xor i32 %tmp6.3, %tmp5.3
+ %reduce.0 = add i32 %tmp7.1, %tmp7.0
+ %reduce.1 = add i32 %reduce.0, %tmp7.2
+ %reduce.2 = add i32 %reduce.1, %tmp7.3
+ ret i32 %reduce.2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
new file mode 100644
index 000000000000..b9dabe966bcd
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/add_sub_sat-inseltpoison.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer -instcombine %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+define <2 x i16> @uadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v2i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @uadd_sat_v2i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[TMP0:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT: ret <2 x i16> [[TMP0]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+ %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %ins.0 = insertelement <2 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+ ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @usub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @usub_sat_v2i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.usub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @usub_sat_v2i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[TMP0:%.*]] = call <2 x i16> @llvm.usub.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT: ret <2 x i16> [[TMP0]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+ %add.0 = call i16 @llvm.usub.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.usub.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %ins.0 = insertelement <2 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+ ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @sadd_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @sadd_sat_v2i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @sadd_sat_v2i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[TMP0:%.*]] = call <2 x i16> @llvm.sadd.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT: ret <2 x i16> [[TMP0]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+ %add.0 = call i16 @llvm.sadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.sadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %ins.0 = insertelement <2 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+ ret <2 x i16> %ins.1
+}
+
+define <2 x i16> @ssub_sat_v2i16(<2 x i16> %arg0, <2 x i16> %arg1) {
+; GFX7-LABEL: @ssub_sat_v2i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.ssub.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <2 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <2 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: ret <2 x i16> [[INS_1]]
+;
+; GFX8-LABEL: @ssub_sat_v2i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[TMP0:%.*]] = call <2 x i16> @llvm.ssub.sat.v2i16(<2 x i16> [[ARG0:%.*]], <2 x i16> [[ARG1:%.*]])
+; GFX8-NEXT: ret <2 x i16> [[TMP0]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i16> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i16> %arg1, i64 1
+ %add.0 = call i16 @llvm.ssub.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.ssub.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %ins.0 = insertelement <2 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <2 x i16> %ins.0, i16 %add.1, i64 1
+ ret <2 x i16> %ins.1
+}
+
+define <2 x i32> @uadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @uadd_sat_v2i32(
+; GCN-NEXT: bb:
+; GCN-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i32> [[ARG0:%.*]], i64 0
+; GCN-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i32> [[ARG0]], i64 1
+; GCN-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i32> [[ARG1:%.*]], i64 0
+; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
+; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
+; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.uadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
+; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
+; GCN-NEXT: ret <2 x i32> [[INS_1]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+ %add.0 = call i32 @llvm.uadd.sat.i32(i32 %arg0.0, i32 %arg1.0)
+ %add.1 = call i32 @llvm.uadd.sat.i32(i32 %arg0.1, i32 %arg1.1)
+ %ins.0 = insertelement <2 x i32> poison, i32 %add.0, i64 0
+ %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+ ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @usub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @usub_sat_v2i32(
+; GCN-NEXT: bb:
+; GCN-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i32> [[ARG0:%.*]], i64 0
+; GCN-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i32> [[ARG0]], i64 1
+; GCN-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i32> [[ARG1:%.*]], i64 0
+; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
+; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
+; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.usub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
+; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
+; GCN-NEXT: ret <2 x i32> [[INS_1]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+ %add.0 = call i32 @llvm.usub.sat.i32(i32 %arg0.0, i32 %arg1.0)
+ %add.1 = call i32 @llvm.usub.sat.i32(i32 %arg0.1, i32 %arg1.1)
+ %ins.0 = insertelement <2 x i32> poison, i32 %add.0, i64 0
+ %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+ ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @sadd_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @sadd_sat_v2i32(
+; GCN-NEXT: bb:
+; GCN-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i32> [[ARG0:%.*]], i64 0
+; GCN-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i32> [[ARG0]], i64 1
+; GCN-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i32> [[ARG1:%.*]], i64 0
+; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
+; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
+; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.sadd.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
+; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
+; GCN-NEXT: ret <2 x i32> [[INS_1]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+ %add.0 = call i32 @llvm.sadd.sat.i32(i32 %arg0.0, i32 %arg1.0)
+ %add.1 = call i32 @llvm.sadd.sat.i32(i32 %arg0.1, i32 %arg1.1)
+ %ins.0 = insertelement <2 x i32> poison, i32 %add.0, i64 0
+ %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+ ret <2 x i32> %ins.1
+}
+
+define <2 x i32> @ssub_sat_v2i32(<2 x i32> %arg0, <2 x i32> %arg1) {
+; GCN-LABEL: @ssub_sat_v2i32(
+; GCN-NEXT: bb:
+; GCN-NEXT: [[ARG0_0:%.*]] = extractelement <2 x i32> [[ARG0:%.*]], i64 0
+; GCN-NEXT: [[ARG0_1:%.*]] = extractelement <2 x i32> [[ARG0]], i64 1
+; GCN-NEXT: [[ARG1_0:%.*]] = extractelement <2 x i32> [[ARG1:%.*]], i64 0
+; GCN-NEXT: [[ARG1_1:%.*]] = extractelement <2 x i32> [[ARG1]], i64 1
+; GCN-NEXT: [[ADD_0:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_0]], i32 [[ARG1_0]])
+; GCN-NEXT: [[ADD_1:%.*]] = call i32 @llvm.ssub.sat.i32(i32 [[ARG0_1]], i32 [[ARG1_1]])
+; GCN-NEXT: [[INS_0:%.*]] = insertelement <2 x i32> poison, i32 [[ADD_0]], i64 0
+; GCN-NEXT: [[INS_1:%.*]] = insertelement <2 x i32> [[INS_0]], i32 [[ADD_1]], i64 1
+; GCN-NEXT: ret <2 x i32> [[INS_1]]
+;
+bb:
+ %arg0.0 = extractelement <2 x i32> %arg0, i64 0
+ %arg0.1 = extractelement <2 x i32> %arg0, i64 1
+ %arg1.0 = extractelement <2 x i32> %arg1, i64 0
+ %arg1.1 = extractelement <2 x i32> %arg1, i64 1
+ %add.0 = call i32 @llvm.ssub.sat.i32(i32 %arg0.0, i32 %arg1.0)
+ %add.1 = call i32 @llvm.ssub.sat.i32(i32 %arg0.1, i32 %arg1.1)
+ %ins.0 = insertelement <2 x i32> poison, i32 %add.0, i64 0
+ %ins.1 = insertelement <2 x i32> %ins.0, i32 %add.1, i64 1
+ ret <2 x i32> %ins.1
+}
+
+define <3 x i16> @uadd_sat_v3i16(<3 x i16> %arg0, <3 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v3i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <3 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0]], i64 2
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <3 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1]], i64 2
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX7-NEXT: ret <3 x i16> [[INS_2]]
+;
+; GFX8-LABEL: @uadd_sat_v3i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[ARG0_2:%.*]] = extractelement <3 x i16> [[ARG0:%.*]], i64 2
+; GFX8-NEXT: [[ARG1_2:%.*]] = extractelement <3 x i16> [[ARG1:%.*]], i64 2
+; GFX8-NEXT: [[TMP0:%.*]] = shufflevector <3 x i16> [[ARG0]], <3 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT: [[TMP1:%.*]] = shufflevector <3 x i16> [[ARG1]], <3 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT: [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX8-NEXT: [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX8-NEXT: [[TMP3:%.*]] = extractelement <2 x i16> [[TMP2]], i32 0
+; GFX8-NEXT: [[INS_0:%.*]] = insertelement <3 x i16> poison, i16 [[TMP3]], i64 0
+; GFX8-NEXT: [[TMP4:%.*]] = extractelement <2 x i16> [[TMP2]], i32 1
+; GFX8-NEXT: [[INS_1:%.*]] = insertelement <3 x i16> [[INS_0]], i16 [[TMP4]], i64 1
+; GFX8-NEXT: [[INS_2:%.*]] = insertelement <3 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX8-NEXT: ret <3 x i16> [[INS_2]]
+;
+bb:
+ %arg0.0 = extractelement <3 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <3 x i16> %arg0, i64 1
+ %arg0.2 = extractelement <3 x i16> %arg0, i64 2
+ %arg1.0 = extractelement <3 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <3 x i16> %arg1, i64 1
+ %arg1.2 = extractelement <3 x i16> %arg1, i64 2
+ %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %add.2 = call i16 @llvm.uadd.sat.i16(i16 %arg0.2, i16 %arg1.2)
+ %ins.0 = insertelement <3 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <3 x i16> %ins.0, i16 %add.1, i64 1
+ %ins.2 = insertelement <3 x i16> %ins.1, i16 %add.2, i64 2
+ ret <3 x i16> %ins.2
+}
+
+define <4 x i16> @uadd_sat_v4i16(<4 x i16> %arg0, <4 x i16> %arg1) {
+; GFX7-LABEL: @uadd_sat_v4i16(
+; GFX7-NEXT: bb:
+; GFX7-NEXT: [[ARG0_0:%.*]] = extractelement <4 x i16> [[ARG0:%.*]], i64 0
+; GFX7-NEXT: [[ARG0_1:%.*]] = extractelement <4 x i16> [[ARG0]], i64 1
+; GFX7-NEXT: [[ARG0_2:%.*]] = extractelement <4 x i16> [[ARG0]], i64 2
+; GFX7-NEXT: [[ARG0_3:%.*]] = extractelement <4 x i16> [[ARG0]], i64 3
+; GFX7-NEXT: [[ARG1_0:%.*]] = extractelement <4 x i16> [[ARG1:%.*]], i64 0
+; GFX7-NEXT: [[ARG1_1:%.*]] = extractelement <4 x i16> [[ARG1]], i64 1
+; GFX7-NEXT: [[ARG1_2:%.*]] = extractelement <4 x i16> [[ARG1]], i64 2
+; GFX7-NEXT: [[ARG1_3:%.*]] = extractelement <4 x i16> [[ARG1]], i64 3
+; GFX7-NEXT: [[ADD_0:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_0]], i16 [[ARG1_0]])
+; GFX7-NEXT: [[ADD_1:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_1]], i16 [[ARG1_1]])
+; GFX7-NEXT: [[ADD_2:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_2]], i16 [[ARG1_2]])
+; GFX7-NEXT: [[ADD_3:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[ARG0_3]], i16 [[ARG1_3]])
+; GFX7-NEXT: [[INS_0:%.*]] = insertelement <4 x i16> poison, i16 [[ADD_0]], i64 0
+; GFX7-NEXT: [[INS_1:%.*]] = insertelement <4 x i16> [[INS_0]], i16 [[ADD_1]], i64 1
+; GFX7-NEXT: [[INS_2:%.*]] = insertelement <4 x i16> [[INS_1]], i16 [[ADD_2]], i64 2
+; GFX7-NEXT: [[INS_3:%.*]] = insertelement <4 x i16> [[INS_2]], i16 [[ADD_3]], i64 3
+; GFX7-NEXT: ret <4 x i16> [[INS_3]]
+;
+; GFX8-LABEL: @uadd_sat_v4i16(
+; GFX8-NEXT: bb:
+; GFX8-NEXT: [[TMP0:%.*]] = shufflevector <4 x i16> [[ARG0:%.*]], <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT: [[TMP1:%.*]] = shufflevector <4 x i16> [[ARG1:%.*]], <4 x i16> undef, <2 x i32> <i32 0, i32 1>
+; GFX8-NEXT: [[TMP2:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP0]], <2 x i16> [[TMP1]])
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i16> [[ARG0]], <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT: [[TMP4:%.*]] = shufflevector <4 x i16> [[ARG1]], <4 x i16> undef, <2 x i32> <i32 2, i32 3>
+; GFX8-NEXT: [[TMP5:%.*]] = call <2 x i16> @llvm.uadd.sat.v2i16(<2 x i16> [[TMP3]], <2 x i16> [[TMP4]])
+; GFX8-NEXT: [[INS_3:%.*]] = shufflevector <2 x i16> [[TMP2]], <2 x i16> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; GFX8-NEXT: ret <4 x i16> [[INS_3]]
+;
+bb:
+ %arg0.0 = extractelement <4 x i16> %arg0, i64 0
+ %arg0.1 = extractelement <4 x i16> %arg0, i64 1
+ %arg0.2 = extractelement <4 x i16> %arg0, i64 2
+ %arg0.3 = extractelement <4 x i16> %arg0, i64 3
+ %arg1.0 = extractelement <4 x i16> %arg1, i64 0
+ %arg1.1 = extractelement <4 x i16> %arg1, i64 1
+ %arg1.2 = extractelement <4 x i16> %arg1, i64 2
+ %arg1.3 = extractelement <4 x i16> %arg1, i64 3
+ %add.0 = call i16 @llvm.uadd.sat.i16(i16 %arg0.0, i16 %arg1.0)
+ %add.1 = call i16 @llvm.uadd.sat.i16(i16 %arg0.1, i16 %arg1.1)
+ %add.2 = call i16 @llvm.uadd.sat.i16(i16 %arg0.2, i16 %arg1.2)
+ %add.3 = call i16 @llvm.uadd.sat.i16(i16 %arg0.3, i16 %arg1.3)
+ %ins.0 = insertelement <4 x i16> poison, i16 %add.0, i64 0
+ %ins.1 = insertelement <4 x i16> %ins.0, i16 %add.1, i64 1
+ %ins.2 = insertelement <4 x i16> %ins.1, i16 %add.2, i64 2
+ %ins.3 = insertelement <4 x i16> %ins.2, i16 %add.3, i64 3
+ ret <4 x i16> %ins.3
+}
+
+declare i16 @llvm.uadd.sat.i16(i16, i16) #0
+declare i16 @llvm.usub.sat.i16(i16, i16) #0
+declare i16 @llvm.sadd.sat.i16(i16, i16) #0
+declare i16 @llvm.ssub.sat.i16(i16, i16) #0
+
+declare i32 @llvm.uadd.sat.i32(i32, i32) #0
+declare i32 @llvm.usub.sat.i32(i32, i32) #0
+declare i32 @llvm.sadd.sat.i32(i32, i32) #0
+declare i32 @llvm.ssub.sat.i32(i32, i32) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/bswap-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/bswap-inseltpoison.ll
new file mode 100644
index 000000000000..0dacc72541bf
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/bswap-inseltpoison.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+; GCN-LABEL: @bswap_v2i16(
+; GFX7: call i16 @llvm.bswap.i16(
+; GFX7: call i16 @llvm.bswap.i16(
+
+; GFX8: call <2 x i16> @llvm.bswap.v2i16(
+define <2 x i16> @bswap_v2i16(<2 x i16> %arg) {
+bb:
+ %tmp = extractelement <2 x i16> %arg, i64 0
+ %tmp1 = tail call i16 @llvm.bswap.i16(i16 %tmp)
+ %tmp2 = insertelement <2 x i16> poison, i16 %tmp1, i64 0
+ %tmp3 = extractelement <2 x i16> %arg, i64 1
+ %tmp4 = tail call i16 @llvm.bswap.i16(i16 %tmp3)
+ %tmp5 = insertelement <2 x i16> %tmp2, i16 %tmp4, i64 1
+ ret <2 x i16> %tmp5
+}
+
+; GCN-LABEL: @bswap_v2i32(
+; GCN: call i32 @llvm.bswap.i32
+; GCN: call i32 @llvm.bswap.i32
+define <2 x i32> @bswap_v2i32(<2 x i32> %arg) {
+bb:
+ %tmp = extractelement <2 x i32> %arg, i64 0
+ %tmp1 = tail call i32 @llvm.bswap.i32(i32 %tmp)
+ %tmp2 = insertelement <2 x i32> poison, i32 %tmp1, i64 0
+ %tmp3 = extractelement <2 x i32> %arg, i64 1
+ %tmp4 = tail call i32 @llvm.bswap.i32(i32 %tmp3)
+ %tmp5 = insertelement <2 x i32> %tmp2, i32 %tmp4, i64 1
+ ret <2 x i32> %tmp5
+}
+
+declare i16 @llvm.bswap.i16(i16) #0
+declare i32 @llvm.bswap.i32(i32) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/round-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/round-inseltpoison.ll
new file mode 100644
index 000000000000..b2e444931802
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/round-inseltpoison.ll
@@ -0,0 +1,38 @@
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -slp-vectorizer %s | FileCheck -check-prefixes=GCN,GFX8 %s
+
+; GCN-LABEL: @round_v2f16(
+; GFX7: call half @llvm.round.f16(
+; GFX7: call half @llvm.round.f16(
+
+; GFX8: call <2 x half> @llvm.round.v2f16(
+define <2 x half> @round_v2f16(<2 x half> %arg) {
+bb:
+ %tmp = extractelement <2 x half> %arg, i64 0
+ %tmp1 = tail call half @llvm.round.half(half %tmp)
+ %tmp2 = insertelement <2 x half> poison, half %tmp1, i64 0
+ %tmp3 = extractelement <2 x half> %arg, i64 1
+ %tmp4 = tail call half @llvm.round.half(half %tmp3)
+ %tmp5 = insertelement <2 x half> %tmp2, half %tmp4, i64 1
+ ret <2 x half> %tmp5
+}
+
+; GCN-LABEL: @round_v2f32(
+; GCN: call float @llvm.round.f32(
+; GCN: call float @llvm.round.f32(
+define <2 x float> @round_v2f32(<2 x float> %arg) {
+bb:
+ %tmp = extractelement <2 x float> %arg, i64 0
+ %tmp1 = tail call float @llvm.round.f32(float %tmp)
+ %tmp2 = insertelement <2 x float> poison, float %tmp1, i64 0
+ %tmp3 = extractelement <2 x float> %arg, i64 1
+ %tmp4 = tail call float @llvm.round.f32(float %tmp3)
+ %tmp5 = insertelement <2 x float> %tmp2, float %tmp4, i64 1
+ ret <2 x float> %tmp5
+}
+
+declare half @llvm.round.half(half) #0
+declare float @llvm.round.f32(float) #0
+
+attributes #0 = { nounwind readnone speculatable willreturn }
diff --git a/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert-inseltpoison.ll
new file mode 100644
index 000000000000..9d3102566258
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/ARM/extract-insert-inseltpoison.ll
@@ -0,0 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=thumb7 -mcpu=swift | FileCheck %s
+
+define <4 x i32> @PR13837(<4 x float> %in) {
+; CHECK-LABEL: @PR13837(
+; CHECK-NEXT: [[TMP1:%.*]] = fptosi <4 x float> [[IN:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[V3]]
+;
+ %t0 = extractelement <4 x float> %in, i64 0
+ %t1 = extractelement <4 x float> %in, i64 1
+ %t2 = extractelement <4 x float> %in, i64 2
+ %t3 = extractelement <4 x float> %in, i64 3
+ %c0 = fptosi float %t0 to i32
+ %c1 = fptosi float %t1 to i32
+ %c2 = fptosi float %t2 to i32
+ %c3 = fptosi float %t3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %c0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %c1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %c2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %c3, i32 3
+ ret <4 x i32> %v3
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic-inseltpoison.ll
new file mode 100644
index 000000000000..b55d0662cbd8
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic-inseltpoison.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -o - -S -slp-threshold=-1000 | FileCheck %s
+
+target datalayout = "e-p:32:32-i64:64-v16:16-v32:32-n16:32:64"
+target triple = "nvptx--nvidiacl"
+
+; CTLZ cannot be vectorized currently because the second argument is a scalar
+; for both the scalar and vector forms of the intrinsic. In the future it
+; should be possible to vectorize such functions.
+; Test causes an assert if LLVM tries to vectorize CTLZ.
+
+define <2 x i8> @cltz_test(<2 x i8> %x) #0 {
+; CHECK-LABEL: @cltz_test(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <2 x i8> poison, i8 [[CALL_I]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT: [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[VECINIT2]]
+;
+entry:
+ %0 = extractelement <2 x i8> %x, i32 0
+ %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
+ %vecinit = insertelement <2 x i8> poison, i8 %call.i, i32 0
+ %1 = extractelement <2 x i8> %x, i32 1
+ %call.i4 = call i8 @llvm.ctlz.i8(i8 %1, i1 false)
+ %vecinit2 = insertelement <2 x i8> %vecinit, i8 %call.i4, i32 1
+ ret <2 x i8> %vecinit2
+}
+
+define <2 x i8> @cltz_test2(<2 x i8> %x) #1 {
+; CHECK-LABEL: @cltz_test2(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT: [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT: [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <2 x i8> poison, i8 [[CALL_I]], i32 0
+; CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[VECINIT2]]
+;
+entry:
+ %0 = extractelement <2 x i8> %x, i32 0
+ %1 = extractelement <2 x i8> %x, i32 1
+ %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
+ %call.i4 = call i8 @llvm.ctlz.i8(i8 %1, i1 false)
+ %vecinit = insertelement <2 x i8> poison, i8 %call.i, i32 0
+ %vecinit2 = insertelement <2 x i8> %vecinit, i8 %call.i4, i32 1
+ ret <2 x i8> %vecinit2
+}
+
+declare i8 @llvm.ctlz.i8(i8, i1) #3
+
+attributes #0 = { alwaysinline nounwind "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
new file mode 100644
index 000000000000..b6d9c0dbbe26
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35865-inseltpoison.ll
@@ -0,0 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -o - -mtriple=x86_64-apple-macosx10.10.0 -mcpu=core2 | FileCheck %s
+
+define void @_Z10fooConvertPDv4_xS0_S0_PKS_() {
+; CHECK-LABEL: @_Z10fooConvertPDv4_xS0_S0_PKS_(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = extractelement <16 x half> undef, i32 4
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <16 x half> undef, i32 5
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x half> undef, half [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x half> [[TMP2]], half [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = fpext <2 x half> [[TMP3]] to <2 x float>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x float> [[TMP4]] to <2 x i32>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP5]], i32 0
+; CHECK-NEXT: [[VECINS_I_4_I:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i32> [[TMP5]], i32 1
+; CHECK-NEXT: [[VECINS_I_5_I:%.*]] = insertelement <8 x i32> [[VECINS_I_4_I]], i32 [[TMP7]], i32 5
+; CHECK-NEXT: ret void
+;
+entry:
+ %0 = extractelement <16 x half> undef, i32 4
+ %conv.i.4.i = fpext half %0 to float
+ %1 = bitcast float %conv.i.4.i to i32
+ %vecins.i.4.i = insertelement <8 x i32> poison, i32 %1, i32 4
+ %2 = extractelement <16 x half> undef, i32 5
+ %conv.i.5.i = fpext half %2 to float
+ %3 = bitcast float %conv.i.5.i to i32
+ %vecins.i.5.i = insertelement <8 x i32> %vecins.i.4.i, i32 %3, i32 5
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
new file mode 100644
index 000000000000..a3407151a332
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-calls-inseltpoison.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s
+
+define <8 x float> @ceil_floor(<8 x float> %a) {
+; CHECK-LABEL: @ceil_floor(
+; CHECK-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; CHECK-NEXT: [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; CHECK-NEXT: [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; CHECK-NEXT: [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; CHECK-NEXT: [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; CHECK-NEXT: [[AB0:%.*]] = call float @llvm.ceil.f32(float [[A0]])
+; CHECK-NEXT: [[AB1:%.*]] = call float @llvm.floor.f32(float [[A1]])
+; CHECK-NEXT: [[AB2:%.*]] = call float @llvm.floor.f32(float [[A2]])
+; CHECK-NEXT: [[AB3:%.*]] = call float @llvm.ceil.f32(float [[A3]])
+; CHECK-NEXT: [[AB4:%.*]] = call float @llvm.ceil.f32(float [[A4]])
+; CHECK-NEXT: [[AB5:%.*]] = call float @llvm.ceil.f32(float [[A5]])
+; CHECK-NEXT: [[AB6:%.*]] = call float @llvm.floor.f32(float [[A6]])
+; CHECK-NEXT: [[AB7:%.*]] = call float @llvm.floor.f32(float [[A7]])
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %ab0 = call float @llvm.ceil.f32(float %a0)
+ %ab1 = call float @llvm.floor.f32(float %a1)
+ %ab2 = call float @llvm.floor.f32(float %a2)
+ %ab3 = call float @llvm.ceil.f32(float %a3)
+ %ab4 = call float @llvm.ceil.f32(float %a4)
+ %ab5 = call float @llvm.ceil.f32(float %a5)
+ %ab6 = call float @llvm.floor.f32(float %a6)
+ %ab7 = call float @llvm.floor.f32(float %a7)
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
+
+declare float @llvm.ceil.f32(float)
+declare float @llvm.floor.f32(float)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
new file mode 100644
index 000000000000..23392e5fc747
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-cast-inseltpoison.ll
@@ -0,0 +1,466 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+
+define <8 x float> @sitofp_uitofp(<8 x i32> %a) {
+; SSE-LABEL: @sitofp_uitofp(
+; SSE-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
+; SSE-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT: [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SSE-NEXT: [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SSE-NEXT: [[AB2:%.*]] = sitofp i32 [[A2]] to float
+; SSE-NEXT: [[AB3:%.*]] = sitofp i32 [[A3]] to float
+; SSE-NEXT: [[AB4:%.*]] = uitofp i32 [[A4]] to float
+; SSE-NEXT: [[AB5:%.*]] = uitofp i32 [[A5]] to float
+; SSE-NEXT: [[AB6:%.*]] = uitofp i32 [[A6]] to float
+; SSE-NEXT: [[AB7:%.*]] = uitofp i32 [[A7]] to float
+; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SSE-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SSE-NEXT: ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @sitofp_uitofp(
+; SLM-NEXT: [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; SLM-NEXT: [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; SLM-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; SLM-NEXT: ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @sitofp_uitofp(
+; AVX-NEXT: [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; AVX-NEXT: [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; AVX-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX-NEXT: ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @sitofp_uitofp(
+; AVX512-NEXT: [[TMP1:%.*]] = sitofp <8 x i32> [[A:%.*]] to <8 x float>
+; AVX512-NEXT: [[TMP2:%.*]] = uitofp <8 x i32> [[A]] to <8 x float>
+; AVX512-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %ab0 = sitofp i32 %a0 to float
+ %ab1 = sitofp i32 %a1 to float
+ %ab2 = sitofp i32 %a2 to float
+ %ab3 = sitofp i32 %a3 to float
+ %ab4 = uitofp i32 %a4 to float
+ %ab5 = uitofp i32 %a5 to float
+ %ab6 = uitofp i32 %a6 to float
+ %ab7 = uitofp i32 %a7 to float
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x i32> @fptosi_fptoui(<8 x float> %a) {
+; SSE-LABEL: @fptosi_fptoui(
+; SSE-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SSE-NEXT: [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SSE-NEXT: [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SSE-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SSE-NEXT: [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SSE-NEXT: [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SSE-NEXT: [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SSE-NEXT: [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; SSE-NEXT: [[AB0:%.*]] = fptosi float [[A0]] to i32
+; SSE-NEXT: [[AB1:%.*]] = fptosi float [[A1]] to i32
+; SSE-NEXT: [[AB2:%.*]] = fptosi float [[A2]] to i32
+; SSE-NEXT: [[AB3:%.*]] = fptosi float [[A3]] to i32
+; SSE-NEXT: [[AB4:%.*]] = fptoui float [[A4]] to i32
+; SSE-NEXT: [[AB5:%.*]] = fptoui float [[A5]] to i32
+; SSE-NEXT: [[AB6:%.*]] = fptoui float [[A6]] to i32
+; SSE-NEXT: [[AB7:%.*]] = fptoui float [[A7]] to i32
+; SSE-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SSE-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SSE-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SSE-NEXT: ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @fptosi_fptoui(
+; SLM-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SLM-NEXT: [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SLM-NEXT: [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SLM-NEXT: [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SLM-NEXT: [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; SLM-NEXT: [[AB0:%.*]] = fptosi float [[A0]] to i32
+; SLM-NEXT: [[AB1:%.*]] = fptosi float [[A1]] to i32
+; SLM-NEXT: [[AB2:%.*]] = fptosi float [[A2]] to i32
+; SLM-NEXT: [[AB3:%.*]] = fptosi float [[A3]] to i32
+; SLM-NEXT: [[AB4:%.*]] = fptoui float [[A4]] to i32
+; SLM-NEXT: [[AB5:%.*]] = fptoui float [[A5]] to i32
+; SLM-NEXT: [[AB6:%.*]] = fptoui float [[A6]] to i32
+; SLM-NEXT: [[AB7:%.*]] = fptoui float [[A7]] to i32
+; SLM-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SLM-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; SLM-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SLM-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SLM-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX-LABEL: @fptosi_fptoui(
+; AVX-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; AVX-NEXT: [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; AVX-NEXT: [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; AVX-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; AVX-NEXT: [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; AVX-NEXT: [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; AVX-NEXT: [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; AVX-NEXT: [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; AVX-NEXT: [[AB0:%.*]] = fptosi float [[A0]] to i32
+; AVX-NEXT: [[AB1:%.*]] = fptosi float [[A1]] to i32
+; AVX-NEXT: [[AB2:%.*]] = fptosi float [[A2]] to i32
+; AVX-NEXT: [[AB3:%.*]] = fptosi float [[A3]] to i32
+; AVX-NEXT: [[AB4:%.*]] = fptoui float [[A4]] to i32
+; AVX-NEXT: [[AB5:%.*]] = fptoui float [[A5]] to i32
+; AVX-NEXT: [[AB6:%.*]] = fptoui float [[A6]] to i32
+; AVX-NEXT: [[AB7:%.*]] = fptoui float [[A7]] to i32
+; AVX-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
+; AVX-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; AVX-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @fptosi_fptoui(
+; AVX512-NEXT: [[TMP1:%.*]] = fptosi <8 x float> [[A:%.*]] to <8 x i32>
+; AVX512-NEXT: [[TMP2:%.*]] = fptoui <8 x float> [[A]] to <8 x i32>
+; AVX512-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %ab0 = fptosi float %a0 to i32
+ %ab1 = fptosi float %a1 to i32
+ %ab2 = fptosi float %a2 to i32
+ %ab3 = fptosi float %a3 to i32
+ %ab4 = fptoui float %a4 to i32
+ %ab5 = fptoui float %a5 to i32
+ %ab6 = fptoui float %a6 to i32
+ %ab7 = fptoui float %a7 to i32
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x float> @fneg_fabs(<8 x float> %a) {
+; CHECK-LABEL: @fneg_fabs(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x float> [[A:%.*]] to <8 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = xor <8 x i32> [[TMP1]], <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 undef, i32 undef, i32 undef, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i32> [[TMP4]] to <8 x float>
+; CHECK-NEXT: ret <8 x float> [[TMP5]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %aa0 = bitcast float %a0 to i32
+ %aa1 = bitcast float %a1 to i32
+ %aa2 = bitcast float %a2 to i32
+ %aa3 = bitcast float %a3 to i32
+ %aa4 = bitcast float %a4 to i32
+ %aa5 = bitcast float %a5 to i32
+ %aa6 = bitcast float %a6 to i32
+ %aa7 = bitcast float %a7 to i32
+ %ab0 = xor i32 %aa0, -2147483648
+ %ab1 = xor i32 %aa1, -2147483648
+ %ab2 = xor i32 %aa2, -2147483648
+ %ab3 = xor i32 %aa3, -2147483648
+ %ab4 = and i32 %aa4, 2147483647
+ %ab5 = and i32 %aa5, 2147483647
+ %ab6 = and i32 %aa6, 2147483647
+ %ab7 = and i32 %aa7, 2147483647
+ %ac0 = bitcast i32 %ab0 to float
+ %ac1 = bitcast i32 %ab1 to float
+ %ac2 = bitcast i32 %ab2 to float
+ %ac3 = bitcast i32 %ab3 to float
+ %ac4 = bitcast i32 %ab4 to float
+ %ac5 = bitcast i32 %ab5 to float
+ %ac6 = bitcast i32 %ab6 to float
+ %ac7 = bitcast i32 %ab7 to float
+ %r0 = insertelement <8 x float> poison, float %ac0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ac1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ac2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ac3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ac4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ac5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ac6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ac7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x i32> @sext_zext(<8 x i16> %a) {
+; CHECK-LABEL: @sext_zext(
+; CHECK-NEXT: [[TMP1:%.*]] = sext <8 x i16> [[A:%.*]] to <8 x i32>
+; CHECK-NEXT: [[TMP2:%.*]] = zext <8 x i16> [[A]] to <8 x i32>
+; CHECK-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i16> %a, i32 0
+ %a1 = extractelement <8 x i16> %a, i32 1
+ %a2 = extractelement <8 x i16> %a, i32 2
+ %a3 = extractelement <8 x i16> %a, i32 3
+ %a4 = extractelement <8 x i16> %a, i32 4
+ %a5 = extractelement <8 x i16> %a, i32 5
+ %a6 = extractelement <8 x i16> %a, i32 6
+ %a7 = extractelement <8 x i16> %a, i32 7
+ %ab0 = sext i16 %a0 to i32
+ %ab1 = sext i16 %a1 to i32
+ %ab2 = sext i16 %a2 to i32
+ %ab3 = sext i16 %a3 to i32
+ %ab4 = zext i16 %a4 to i32
+ %ab5 = zext i16 %a5 to i32
+ %ab6 = zext i16 %a6 to i32
+ %ab7 = zext i16 %a7 to i32
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x float> @sitofp_4i32_8i16(<4 x i32> %a, <8 x i16> %b) {
+; CHECK-LABEL: @sitofp_4i32_8i16(
+; CHECK-NEXT: [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; CHECK-NEXT: [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; CHECK-NEXT: [[B2:%.*]] = extractelement <8 x i16> [[B]], i32 2
+; CHECK-NEXT: [[B3:%.*]] = extractelement <8 x i16> [[B]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; CHECK-NEXT: [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; CHECK-NEXT: [[AB5:%.*]] = sitofp i16 [[B1]] to float
+; CHECK-NEXT: [[AB6:%.*]] = sitofp i16 [[B2]] to float
+; CHECK-NEXT: [[AB7:%.*]] = sitofp i16 [[B3]] to float
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <8 x i16> %b, i32 0
+ %b1 = extractelement <8 x i16> %b, i32 1
+ %b2 = extractelement <8 x i16> %b, i32 2
+ %b3 = extractelement <8 x i16> %b, i32 3
+ %ab0 = sitofp i32 %a0 to float
+ %ab1 = sitofp i32 %a1 to float
+ %ab2 = sitofp i32 %a2 to float
+ %ab3 = sitofp i32 %a3 to float
+ %ab4 = sitofp i16 %b0 to float
+ %ab5 = sitofp i16 %b1 to float
+ %ab6 = sitofp i16 %b2 to float
+ %ab7 = sitofp i16 %b3 to float
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
+
+; Inspired by PR38154
+define <8 x float> @sitofp_uitofp_4i32_8i16_16i8(<4 x i32> %a, <8 x i16> %b, <16 x i8> %c) {
+; SSE-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; SSE-NEXT: [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; SSE-NEXT: [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; SSE-NEXT: [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; SSE-NEXT: [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; SSE-NEXT: [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; SSE-NEXT: [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; SSE-NEXT: [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; SSE-NEXT: [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; SSE-NEXT: [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; SSE-NEXT: [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; SSE-NEXT: [[AB2:%.*]] = uitofp i32 [[A2]] to float
+; SSE-NEXT: [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; SSE-NEXT: [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; SSE-NEXT: [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; SSE-NEXT: [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; SSE-NEXT: [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; SSE-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SSE-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SSE-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SSE-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SSE-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SSE-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SSE-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SSE-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SSE-NEXT: ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; SLM-NEXT: [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; SLM-NEXT: [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; SLM-NEXT: [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; SLM-NEXT: [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; SLM-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[A]] to <4 x float>
+; SLM-NEXT: [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; SLM-NEXT: [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; SLM-NEXT: [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; SLM-NEXT: [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; SLM-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; SLM-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
+; SLM-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; SLM-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP4]], i32 1
+; SLM-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; SLM-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP5]], i32 2
+; SLM-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; SLM-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP6]], i32 3
+; SLM-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SLM-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SLM-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SLM-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SLM-NEXT: ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; AVX-NEXT: [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
+; AVX-NEXT: [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
+; AVX-NEXT: [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
+; AVX-NEXT: [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
+; AVX-NEXT: [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; AVX-NEXT: [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; AVX-NEXT: [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; AVX-NEXT: [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; AVX-NEXT: [[AB0:%.*]] = sitofp i32 [[A0]] to float
+; AVX-NEXT: [[AB1:%.*]] = sitofp i32 [[A1]] to float
+; AVX-NEXT: [[AB2:%.*]] = uitofp i32 [[A2]] to float
+; AVX-NEXT: [[AB3:%.*]] = uitofp i32 [[A3]] to float
+; AVX-NEXT: [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; AVX-NEXT: [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; AVX-NEXT: [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; AVX-NEXT: [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; AVX-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; AVX-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; AVX-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; AVX-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; AVX-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; AVX-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; AVX-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; AVX-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; AVX-NEXT: ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @sitofp_uitofp_4i32_8i16_16i8(
+; AVX512-NEXT: [[B0:%.*]] = extractelement <8 x i16> [[B:%.*]], i32 0
+; AVX512-NEXT: [[B1:%.*]] = extractelement <8 x i16> [[B]], i32 1
+; AVX512-NEXT: [[C0:%.*]] = extractelement <16 x i8> [[C:%.*]], i32 0
+; AVX512-NEXT: [[C1:%.*]] = extractelement <16 x i8> [[C]], i32 1
+; AVX512-NEXT: [[TMP1:%.*]] = sitofp <4 x i32> [[A:%.*]] to <4 x float>
+; AVX512-NEXT: [[TMP2:%.*]] = uitofp <4 x i32> [[A]] to <4 x float>
+; AVX512-NEXT: [[AB4:%.*]] = sitofp i16 [[B0]] to float
+; AVX512-NEXT: [[AB5:%.*]] = uitofp i16 [[B1]] to float
+; AVX512-NEXT: [[AB6:%.*]] = sitofp i8 [[C0]] to float
+; AVX512-NEXT: [[AB7:%.*]] = uitofp i8 [[C1]] to float
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; AVX512-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP3]], i32 0
+; AVX512-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP4]], i32 1
+; AVX512-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; AVX512-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP5]], i32 2
+; AVX512-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; AVX512-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP6]], i32 3
+; AVX512-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; AVX512-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; AVX512-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; AVX512-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <8 x i16> %b, i32 0
+ %b1 = extractelement <8 x i16> %b, i32 1
+ %c0 = extractelement <16 x i8> %c, i32 0
+ %c1 = extractelement <16 x i8> %c, i32 1
+ %ab0 = sitofp i32 %a0 to float
+ %ab1 = sitofp i32 %a1 to float
+ %ab2 = uitofp i32 %a2 to float
+ %ab3 = uitofp i32 %a3 to float
+ %ab4 = sitofp i16 %b0 to float
+ %ab5 = uitofp i16 %b1 to float
+ %ab6 = sitofp i8 %c0 to float
+ %ab7 = uitofp i8 %c1 to float
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
new file mode 100644
index 000000000000..638b2c3f75fc
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-fp-inseltpoison.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512
+
+define <8 x float> @fadd_fsub_v8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @fadd_fsub_v8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = fsub <8 x float> [[A]], [[B]]
+; CHECK-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %ab0 = fadd float %a0, %b0
+ %ab1 = fsub float %a1, %b1
+ %ab2 = fsub float %a2, %b2
+ %ab3 = fadd float %a3, %b3
+ %ab4 = fadd float %a4, %b4
+ %ab5 = fsub float %a5, %b5
+ %ab6 = fsub float %a6, %b6
+ %ab7 = fadd float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x float> @fmul_fdiv_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @fmul_fdiv_v8f32(
+; SSE-NEXT: [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT: [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; SSE-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; SSE-NEXT: ret <8 x float> [[R7]]
+;
+; SLM-LABEL: @fmul_fdiv_v8f32(
+; SLM-NEXT: [[A0:%.*]] = extractelement <8 x float> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <8 x float> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <8 x float> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <8 x float> [[A]], i32 3
+; SLM-NEXT: [[A4:%.*]] = extractelement <8 x float> [[A]], i32 4
+; SLM-NEXT: [[A5:%.*]] = extractelement <8 x float> [[A]], i32 5
+; SLM-NEXT: [[A6:%.*]] = extractelement <8 x float> [[A]], i32 6
+; SLM-NEXT: [[A7:%.*]] = extractelement <8 x float> [[A]], i32 7
+; SLM-NEXT: [[B0:%.*]] = extractelement <8 x float> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <8 x float> [[B]], i32 1
+; SLM-NEXT: [[B2:%.*]] = extractelement <8 x float> [[B]], i32 2
+; SLM-NEXT: [[B3:%.*]] = extractelement <8 x float> [[B]], i32 3
+; SLM-NEXT: [[B4:%.*]] = extractelement <8 x float> [[B]], i32 4
+; SLM-NEXT: [[B5:%.*]] = extractelement <8 x float> [[B]], i32 5
+; SLM-NEXT: [[B6:%.*]] = extractelement <8 x float> [[B]], i32 6
+; SLM-NEXT: [[B7:%.*]] = extractelement <8 x float> [[B]], i32 7
+; SLM-NEXT: [[AB0:%.*]] = fmul float [[A0]], [[B0]]
+; SLM-NEXT: [[AB1:%.*]] = fdiv float [[A1]], [[B1]]
+; SLM-NEXT: [[AB2:%.*]] = fdiv float [[A2]], [[B2]]
+; SLM-NEXT: [[AB3:%.*]] = fmul float [[A3]], [[B3]]
+; SLM-NEXT: [[AB4:%.*]] = fmul float [[A4]], [[B4]]
+; SLM-NEXT: [[AB5:%.*]] = fdiv float [[A5]], [[B5]]
+; SLM-NEXT: [[AB6:%.*]] = fdiv float [[A6]], [[B6]]
+; SLM-NEXT: [[AB7:%.*]] = fmul float [[A7]], [[B7]]
+; SLM-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[AB0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[AB1]], i32 1
+; SLM-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[AB2]], i32 2
+; SLM-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[AB4]], i32 4
+; SLM-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[AB5]], i32 5
+; SLM-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[AB6]], i32 6
+; SLM-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[AB7]], i32 7
+; SLM-NEXT: ret <8 x float> [[R7]]
+;
+; AVX-LABEL: @fmul_fdiv_v8f32(
+; AVX-NEXT: [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT: [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; AVX-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; AVX-NEXT: ret <8 x float> [[R7]]
+;
+; AVX512-LABEL: @fmul_fdiv_v8f32(
+; AVX512-NEXT: [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT: [[TMP2:%.*]] = fdiv <8 x float> [[A]], [[B]]
+; AVX512-NEXT: [[R7:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> [[TMP2]], <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 13, i32 14, i32 7>
+; AVX512-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %ab0 = fmul float %a0, %b0
+ %ab1 = fdiv float %a1, %b1
+ %ab2 = fdiv float %a2, %b2
+ %ab3 = fmul float %a3, %b3
+ %ab4 = fmul float %a4, %b4
+ %ab5 = fdiv float %a5, %b5
+ %ab6 = fdiv float %a6, %b6
+ %ab7 = fmul float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %ab3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %ab4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %ab5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %ab6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %ab7, i32 7
+ ret <8 x float> %r7
+}
+
+define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
+; SSE-LABEL: @fmul_fdiv_v4f32_const(
+; SSE-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; SSE-NEXT: ret <4 x float> [[TMP1]]
+;
+; SLM-LABEL: @fmul_fdiv_v4f32_const(
+; SLM-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; SLM-NEXT: [[AB0:%.*]] = fmul float [[A0]], 2.000000e+00
+; SLM-NEXT: [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
+; SLM-NEXT: [[R0:%.*]] = insertelement <4 x float> poison, float [[AB0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[A1]], i32 1
+; SLM-NEXT: [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[A2]], i32 2
+; SLM-NEXT: [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
+; SLM-NEXT: ret <4 x float> [[R3]]
+;
+; AVX-LABEL: @fmul_fdiv_v4f32_const(
+; AVX-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX-NEXT: ret <4 x float> [[TMP1]]
+;
+; AVX512-LABEL: @fmul_fdiv_v4f32_const(
+; AVX512-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], <float 2.000000e+00, float 1.000000e+00, float 1.000000e+00, float 2.000000e+00>
+; AVX512-NEXT: ret <4 x float> [[TMP1]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %ab0 = fmul float %a0, 2.0
+ %ab1 = fmul float %a1, 1.0
+ %ab2 = fdiv float %a2, 1.0
+ %ab3 = fdiv float %a3, 0.5
+ %r0 = insertelement <4 x float> poison, float %ab0, i32 0
+ %r1 = insertelement <4 x float> %r0, float %ab1, i32 1
+ %r2 = insertelement <4 x float> %r1, float %ab2, i32 2
+ %r3 = insertelement <4 x float> %r2, float %ab3, i32 3
+ ret <4 x float> %r3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
new file mode 100644
index 000000000000..d567524e8b5e
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -0,0 +1,497 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefixes=CHECK,AVX512
+
+define <8 x i32> @add_sub_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: @add_sub_v8i32(
+; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = sub <8 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %b0 = extractelement <8 x i32> %b, i32 0
+ %b1 = extractelement <8 x i32> %b, i32 1
+ %b2 = extractelement <8 x i32> %b, i32 2
+ %b3 = extractelement <8 x i32> %b, i32 3
+ %b4 = extractelement <8 x i32> %b, i32 4
+ %b5 = extractelement <8 x i32> %b, i32 5
+ %b6 = extractelement <8 x i32> %b, i32 6
+ %b7 = extractelement <8 x i32> %b, i32 7
+ %ab0 = add i32 %a0, %b0
+ %ab1 = add i32 %a1, %b1
+ %ab2 = add i32 %a2, %b2
+ %ab3 = add i32 %a3, %b3
+ %ab4 = sub i32 %a4, %b4
+ %ab5 = sub i32 %a5, %b5
+ %ab6 = sub i32 %a6, %b6
+ %ab7 = sub i32 %a7, %b7
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <4 x i32> @add_and_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @add_and_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = and <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 1, i32 6, i32 7>
+; CHECK-NEXT: ret <4 x i32> [[R3]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <4 x i32> %b, i32 0
+ %b1 = extractelement <4 x i32> %b, i32 1
+ %b2 = extractelement <4 x i32> %b, i32 2
+ %b3 = extractelement <4 x i32> %b, i32 3
+ %ab0 = add i32 %a0, %b0
+ %ab1 = add i32 %a1, %b1
+ %ab2 = and i32 %a2, %b2
+ %ab3 = and i32 %a3, %b3
+ %r0 = insertelement <4 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <4 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <4 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <4 x i32> %r2, i32 %ab3, i32 3
+ ret <4 x i32> %r3
+}
+
+define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @add_mul_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[A]], [[B]]
+; CHECK-NEXT: [[R3:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> [[TMP2]], <4 x i32> <i32 0, i32 5, i32 6, i32 3>
+; CHECK-NEXT: ret <4 x i32> [[R3]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <4 x i32> %b, i32 0
+ %b1 = extractelement <4 x i32> %b, i32 1
+ %b2 = extractelement <4 x i32> %b, i32 2
+ %b3 = extractelement <4 x i32> %b, i32 3
+ %ab0 = mul i32 %a0, %b0
+ %ab1 = add i32 %a1, %b1
+ %ab2 = add i32 %a2, %b2
+ %ab3 = mul i32 %a3, %b3
+ %r0 = insertelement <4 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <4 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <4 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <4 x i32> %r2, i32 %ab3, i32 3
+ ret <4 x i32> %r3
+}
+
+define <8 x i32> @ashr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @ashr_shl_v8i32(
+; SSE-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT: [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; SSE-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @ashr_shl_v8i32(
+; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT: [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; AVX1-NEXT: [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; AVX1-NEXT: [[B2:%.*]] = extractelement <8 x i32> [[B]], i32 2
+; AVX1-NEXT: [[B3:%.*]] = extractelement <8 x i32> [[B]], i32 3
+; AVX1-NEXT: [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; AVX1-NEXT: [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; AVX1-NEXT: [[AB2:%.*]] = ashr i32 [[A2]], [[B2]]
+; AVX1-NEXT: [[AB3:%.*]] = ashr i32 [[A3]], [[B3]]
+; AVX1-NEXT: [[TMP1:%.*]] = shl <8 x i32> [[A]], [[B]]
+; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[R3]], <8 x i32> [[TMP1]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX1-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @ashr_shl_v8i32(
+; AVX2-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; AVX2-NEXT: [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; AVX2-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_shl_v8i32(
+; AVX512-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT: [[TMP2:%.*]] = shl <8 x i32> [[A]], [[B]]
+; AVX512-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %b0 = extractelement <8 x i32> %b, i32 0
+ %b1 = extractelement <8 x i32> %b, i32 1
+ %b2 = extractelement <8 x i32> %b, i32 2
+ %b3 = extractelement <8 x i32> %b, i32 3
+ %b4 = extractelement <8 x i32> %b, i32 4
+ %b5 = extractelement <8 x i32> %b, i32 5
+ %b6 = extractelement <8 x i32> %b, i32 6
+ %b7 = extractelement <8 x i32> %b, i32 7
+ %ab0 = ashr i32 %a0, %b0
+ %ab1 = ashr i32 %a1, %b1
+ %ab2 = ashr i32 %a2, %b2
+ %ab3 = ashr i32 %a3, %b3
+ %ab4 = shl i32 %a4, %b4
+ %ab5 = shl i32 %a5, %b5
+ %ab6 = shl i32 %a6, %b6
+ %ab7 = shl i32 %a7, %b7
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x i32> @ashr_shl_v8i32_const(<8 x i32> %a) {
+; SSE-LABEL: @ashr_shl_v8i32_const(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT: [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], <i32 2, i32 2, i32 2, i32 2>
+; SSE-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 3, i32 3, i32 3, i32 3>
+; SSE-NEXT: [[R7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @ashr_shl_v8i32_const(
+; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX1-NEXT: [[TMP2:%.*]] = ashr <4 x i32> [[TMP1]], <i32 2, i32 2, i32 2, i32 2>
+; AVX1-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT: [[TMP4:%.*]] = shl <4 x i32> [[TMP3]], <i32 3, i32 3, i32 3, i32 3>
+; AVX1-NEXT: [[R7:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX1-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @ashr_shl_v8i32_const(
+; AVX2-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: [[TMP2:%.*]] = shl <8 x i32> [[A]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX2-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_shl_v8i32_const(
+; AVX512-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[A:%.*]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: [[TMP2:%.*]] = shl <8 x i32> [[A]], <i32 2, i32 2, i32 2, i32 2, i32 3, i32 3, i32 3, i32 3>
+; AVX512-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> [[TMP2]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %ab0 = ashr i32 %a0, 2
+ %ab1 = ashr i32 %a1, 2
+ %ab2 = ashr i32 %a2, 2
+ %ab3 = ashr i32 %a3, 2
+ %ab4 = shl i32 %a4, 3
+ %ab5 = shl i32 %a5, 3
+ %ab6 = shl i32 %a6, 3
+ %ab7 = shl i32 %a7, 3
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: @ashr_lshr_shl_v8i32(
+; SSE-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; SSE-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT: [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; SSE-NEXT: [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; SSE-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
+; SSE-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; SSE-NEXT: [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; SSE-NEXT: [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; SSE-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; SSE-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; SSE-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; SSE-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; SSE-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; SSE-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
+; SSE-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP2]], i32 2
+; SSE-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
+; SSE-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP3]], i32 3
+; SSE-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
+; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP4]], i32 4
+; SSE-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
+; SSE-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP5]], i32 5
+; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SSE-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @ashr_lshr_shl_v8i32(
+; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
+; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
+; AVX1-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT: [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
+; AVX1-NEXT: [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
+; AVX1-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
+; AVX1-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX1-NEXT: [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
+; AVX1-NEXT: [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
+; AVX1-NEXT: [[TMP1:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX1-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX1-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
+; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
+; AVX1-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
+; AVX1-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP2]], i32 2
+; AVX1-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
+; AVX1-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP3]], i32 3
+; AVX1-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
+; AVX1-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP4]], i32 4
+; AVX1-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
+; AVX1-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP5]], i32 5
+; AVX1-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX1-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @ashr_lshr_shl_v8i32(
+; AVX2-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; AVX2-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX2-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; AVX2-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX2-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX2-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX2-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX2-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX2-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i32 0
+; AVX2-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[TMP7]], i32 1
+; AVX2-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; AVX2-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP8]], i32 2
+; AVX2-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; AVX2-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP9]], i32 3
+; AVX2-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
+; AVX2-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP10]], i32 4
+; AVX2-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
+; AVX2-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP11]], i32 5
+; AVX2-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX2-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX2-NEXT: ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @ashr_lshr_shl_v8i32(
+; AVX512-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 6
+; AVX512-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX512-NEXT: [[B6:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 6
+; AVX512-NEXT: [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
+; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[B]], <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT: [[TMP4:%.*]] = lshr <4 x i32> [[TMP1]], [[TMP2]]
+; AVX512-NEXT: [[TMP5:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; AVX512-NEXT: [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
+; AVX512-NEXT: [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
+; AVX512-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX512-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP6]], i32 0
+; AVX512-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[TMP7]], i32 1
+; AVX512-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
+; AVX512-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP8]], i32 2
+; AVX512-NEXT: [[TMP9:%.*]] = extractelement <4 x i32> [[TMP4]], i32 3
+; AVX512-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP9]], i32 3
+; AVX512-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP5]], i32 4
+; AVX512-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP10]], i32 4
+; AVX512-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP5]], i32 5
+; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP11]], i32 5
+; AVX512-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX512-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX512-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %b0 = extractelement <8 x i32> %b, i32 0
+ %b1 = extractelement <8 x i32> %b, i32 1
+ %b2 = extractelement <8 x i32> %b, i32 2
+ %b3 = extractelement <8 x i32> %b, i32 3
+ %b4 = extractelement <8 x i32> %b, i32 4
+ %b5 = extractelement <8 x i32> %b, i32 5
+ %b6 = extractelement <8 x i32> %b, i32 6
+ %b7 = extractelement <8 x i32> %b, i32 7
+ %ab0 = ashr i32 %a0, %b0
+ %ab1 = ashr i32 %a1, %b1
+ %ab2 = lshr i32 %a2, %b2
+ %ab3 = lshr i32 %a3, %b3
+ %ab4 = lshr i32 %a4, %b4
+ %ab5 = lshr i32 %a5, %b5
+ %ab6 = shl i32 %a6, %b6
+ %ab7 = shl i32 %a7, %b7
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
+; CHECK-LABEL: @add_v8i32_undefs(
+; CHECK-NEXT: [[TMP1:%.*]] = add <8 x i32> [[A:%.*]], <i32 undef, i32 4, i32 8, i32 16, i32 undef, i32 4, i32 8, i32 16>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %ab0 = add i32 %a0, undef
+ %ab1 = add i32 %a1, 4
+ %ab2 = add i32 %a2, 8
+ %ab3 = add i32 %a3, 16
+ %ab4 = add i32 %a4, undef
+ %ab5 = add i32 %a5, 4
+ %ab6 = add i32 %a6, 8
+ %ab7 = add i32 %a7, 16
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
+; CHECK-LABEL: @sdiv_v8i32_undefs(
+; CHECK-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; CHECK-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; CHECK-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; CHECK-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; CHECK-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
+; CHECK-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
+; CHECK-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; CHECK-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
+; CHECK-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; CHECK-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> [[TMP1]], i32 [[AB2]], i32 2
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[AB3]], i32 3
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; CHECK-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %ab0 = sdiv i32 %a0, undef
+ %ab1 = sdiv i32 %a1, 4
+ %ab2 = sdiv i32 %a2, 8
+ %ab3 = sdiv i32 %a3, 16
+ %ab4 = sdiv i32 %a4, undef
+ %ab5 = sdiv i32 %a5, 4
+ %ab6 = sdiv i32 %a6, 8
+ %ab7 = sdiv i32 %a7, 16
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
+
+define <8 x i32> @add_sub_v8i32_splat(<8 x i32> %a, i32 %b) {
+; CHECK-LABEL: @add_sub_v8i32_splat(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <8 x i32> undef, i32 [[B:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[TMP4:%.*]] = sub <8 x i32> [[TMP2]], [[A]]
+; CHECK-NEXT: [[R7:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: ret <8 x i32> [[R7]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %ab0 = add i32 %a0, %b
+ %ab1 = add i32 %b, %a1
+ %ab2 = add i32 %a2, %b
+ %ab3 = add i32 %b, %a3
+ %ab4 = sub i32 %b, %a4
+ %ab5 = sub i32 %b, %a5
+ %ab6 = sub i32 %b, %a6
+ %ab7 = sub i32 %b, %a7
+ %r0 = insertelement <8 x i32> poison, i32 %ab0, i32 0
+ %r1 = insertelement <8 x i32> %r0, i32 %ab1, i32 1
+ %r2 = insertelement <8 x i32> %r1, i32 %ab2, i32 2
+ %r3 = insertelement <8 x i32> %r2, i32 %ab3, i32 3
+ %r4 = insertelement <8 x i32> %r3, i32 %ab4, i32 4
+ %r5 = insertelement <8 x i32> %r4, i32 %ab5, i32 5
+ %r6 = insertelement <8 x i32> %r5, i32 %ab6, i32 6
+ %r7 = insertelement <8 x i32> %r6, i32 %ab7, i32 7
+ ret <8 x i32> %r7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
new file mode 100644
index 000000000000..68a8c8684b68
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp-inseltpoison.ll
@@ -0,0 +1,1365 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=-prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -mattr=+prefer-128-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512
+
+;
+; 128-bit Vectors
+;
+
+define <2 x double> @buildvector_add_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_add_2f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: ret <2 x double> [[R1]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %c0 = fadd double %a0, %b0
+ %c1 = fadd double %a1, %b1
+ %r0 = insertelement <2 x double> poison, double %c0, i32 0
+ %r1 = insertelement <2 x double> %r0, double %c1, i32 1
+ ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_sub_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_2f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: ret <2 x double> [[R1]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %c0 = fsub double %a0, %b0
+ %c1 = fsub double %a1, %b1
+ %r0 = insertelement <2 x double> poison, double %c0, i32 0
+ %r1 = insertelement <2 x double> %r0, double %c1, i32 1
+ ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_mul_2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_2f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: ret <2 x double> [[R1]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %c0 = fmul double %a0, %b0
+ %c1 = fmul double %a1, %b1
+ %r0 = insertelement <2 x double> poison, double %c0, i32 0
+ %r1 = insertelement <2 x double> %r0, double %c1, i32 1
+ ret <2 x double> %r1
+}
+
+define <2 x double> @buildvector_div_2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @buildvector_div_2f64(
+; SSE-NEXT: [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; SSE-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; SSE-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; SSE-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT: ret <2 x double> [[R1]]
+;
+; SLM-LABEL: @buildvector_div_2f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT: [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT: [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT: [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[C0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT: ret <2 x double> [[R1]]
+;
+; AVX-LABEL: @buildvector_div_2f64(
+; AVX-NEXT: [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; AVX-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; AVX-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; AVX-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT: ret <2 x double> [[R1]]
+;
+; AVX512-LABEL: @buildvector_div_2f64(
+; AVX512-NEXT: [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT: [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; AVX512-NEXT: [[R0:%.*]] = insertelement <2 x double> poison, double [[TMP2]], i32 0
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; AVX512-NEXT: [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT: ret <2 x double> [[R1]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %c0 = fdiv double %a0, %b0
+ %c1 = fdiv double %a1, %b1
+ %r0 = insertelement <2 x double> poison, double %c0, i32 0
+ %r1 = insertelement <2 x double> %r0, double %c1, i32 1
+ ret <2 x double> %r1
+}
+
+define <4 x float> @buildvector_add_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_add_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c0 = fadd float %a0, %b0
+ %c1 = fadd float %a1, %b1
+ %c2 = fadd float %a2, %b2
+ %c3 = fadd float %a3, %b3
+ %r0 = insertelement <4 x float> poison, float %c0, i32 0
+ %r1 = insertelement <4 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <4 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <4 x float> %r2, float %c3, i32 3
+ ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_sub_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c0 = fsub float %a0, %b0
+ %c1 = fsub float %a1, %b1
+ %c2 = fsub float %a2, %b2
+ %c3 = fsub float %a3, %b3
+ %r0 = insertelement <4 x float> poison, float %c0, i32 0
+ %r1 = insertelement <4 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <4 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <4 x float> %r2, float %c3, i32 3
+ ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_mul_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c0 = fmul float %a0, %b0
+ %c1 = fmul float %a1, %b1
+ %c2 = fmul float %a2, %b2
+ %c3 = fmul float %a3, %b3
+ %r0 = insertelement <4 x float> poison, float %c0, i32 0
+ %r1 = insertelement <4 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <4 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <4 x float> %r2, float %c3, i32 3
+ ret <4 x float> %r3
+}
+
+define <4 x float> @buildvector_div_4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @buildvector_div_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[R3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c0 = fdiv float %a0, %b0
+ %c1 = fdiv float %a1, %b1
+ %c2 = fdiv float %a2, %b2
+ %c3 = fdiv float %a3, %b3
+ %r0 = insertelement <4 x float> poison, float %c0, i32 0
+ %r1 = insertelement <4 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <4 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <4 x float> %r2, float %c3, i32 3
+ ret <4 x float> %r3
+}
+
+;
+; 256-bit Vectors
+;
+
+define <4 x double> @buildvector_add_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_add_4f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x double> [[R3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %c0 = fadd double %a0, %b0
+ %c1 = fadd double %a1, %b1
+ %c2 = fadd double %a2, %b2
+ %c3 = fadd double %a3, %b3
+ %r0 = insertelement <4 x double> poison, double %c0, i32 0
+ %r1 = insertelement <4 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <4 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <4 x double> %r2, double %c3, i32 3
+ ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_sub_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_4f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x double> [[R3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %c0 = fsub double %a0, %b0
+ %c1 = fsub double %a1, %b1
+ %c2 = fsub double %a2, %b2
+ %c3 = fsub double %a3, %b3
+ %r0 = insertelement <4 x double> poison, double %c0, i32 0
+ %r1 = insertelement <4 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <4 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <4 x double> %r2, double %c3, i32 3
+ ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_mul_4f64(<4 x double> %a, <4 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_4f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <4 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x double> [[R3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %c0 = fmul double %a0, %b0
+ %c1 = fmul double %a1, %b1
+ %c2 = fmul double %a2, %b2
+ %c3 = fmul double %a3, %b3
+ %r0 = insertelement <4 x double> poison, double %c0, i32 0
+ %r1 = insertelement <4 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <4 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <4 x double> %r2, double %c3, i32 3
+ ret <4 x double> %r3
+}
+
+define <4 x double> @buildvector_div_4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @buildvector_div_4f64(
+; SSE-NEXT: [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; SSE-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; SSE-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; SSE-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; SSE-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; SSE-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; SSE-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; SSE-NEXT: ret <4 x double> [[R3]]
+;
+; SLM-LABEL: @buildvector_div_4f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <4 x double> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <4 x double> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <4 x double> [[A]], i32 3
+; SLM-NEXT: [[B0:%.*]] = extractelement <4 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <4 x double> [[B]], i32 1
+; SLM-NEXT: [[B2:%.*]] = extractelement <4 x double> [[B]], i32 2
+; SLM-NEXT: [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
+; SLM-NEXT: [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT: [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]]
+; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]]
+; SLM-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[C0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[C2]], i32 2
+; SLM-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[C3]], i32 3
+; SLM-NEXT: ret <4 x double> [[R3]]
+;
+; AVX-LABEL: @buildvector_div_4f64(
+; AVX-NEXT: [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; AVX-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; AVX-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; AVX-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; AVX-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; AVX-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; AVX-NEXT: ret <4 x double> [[R3]]
+;
+; AVX512-LABEL: @buildvector_div_4f64(
+; AVX512-NEXT: [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT: [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; AVX512-NEXT: [[R0:%.*]] = insertelement <4 x double> poison, double [[TMP2]], i32 0
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; AVX512-NEXT: [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; AVX512-NEXT: [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT: [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; AVX512-NEXT: [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT: ret <4 x double> [[R3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %c0 = fdiv double %a0, %b0
+ %c1 = fdiv double %a1, %b1
+ %c2 = fdiv double %a2, %b2
+ %c3 = fdiv double %a3, %b3
+ %r0 = insertelement <4 x double> poison, double %c0, i32 0
+ %r1 = insertelement <4 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <4 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <4 x double> %r2, double %c3, i32 3
+ ret <4 x double> %r3
+}
+
+define <8 x float> @buildvector_add_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_add_8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %c0 = fadd float %a0, %b0
+ %c1 = fadd float %a1, %b1
+ %c2 = fadd float %a2, %b2
+ %c3 = fadd float %a3, %b3
+ %c4 = fadd float %a4, %b4
+ %c5 = fadd float %a5, %b5
+ %c6 = fadd float %a6, %b6
+ %c7 = fadd float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %c0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %c3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %c4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %c5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %c6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %c7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_sub_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %c0 = fsub float %a0, %b0
+ %c1 = fsub float %a1, %b1
+ %c2 = fsub float %a2, %b2
+ %c3 = fsub float %a3, %b3
+ %c4 = fsub float %a4, %b4
+ %c5 = fsub float %a5, %b5
+ %c6 = fsub float %a6, %b6
+ %c7 = fsub float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %c0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %c3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %c4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %c5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %c6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %c7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_mul_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %c0 = fmul float %a0, %b0
+ %c1 = fmul float %a1, %b1
+ %c2 = fmul float %a2, %b2
+ %c3 = fmul float %a3, %b3
+ %c4 = fmul float %a4, %b4
+ %c5 = fmul float %a5, %b5
+ %c6 = fmul float %a6, %b6
+ %c7 = fmul float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %c0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %c3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %c4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %c5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %c6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %c7, i32 7
+ ret <8 x float> %r7
+}
+
+define <8 x float> @buildvector_div_8f32(<8 x float> %a, <8 x float> %b) {
+; CHECK-LABEL: @buildvector_div_8f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x float> [[R7]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %c0 = fdiv float %a0, %b0
+ %c1 = fdiv float %a1, %b1
+ %c2 = fdiv float %a2, %b2
+ %c3 = fdiv float %a3, %b3
+ %c4 = fdiv float %a4, %b4
+ %c5 = fdiv float %a5, %b5
+ %c6 = fdiv float %a6, %b6
+ %c7 = fdiv float %a7, %b7
+ %r0 = insertelement <8 x float> poison, float %c0, i32 0
+ %r1 = insertelement <8 x float> %r0, float %c1, i32 1
+ %r2 = insertelement <8 x float> %r1, float %c2, i32 2
+ %r3 = insertelement <8 x float> %r2, float %c3, i32 3
+ %r4 = insertelement <8 x float> %r3, float %c4, i32 4
+ %r5 = insertelement <8 x float> %r4, float %c5, i32 5
+ %r6 = insertelement <8 x float> %r5, float %c6, i32 6
+ %r7 = insertelement <8 x float> %r6, float %c7, i32 7
+ ret <8 x float> %r7
+}
+
+;
+; 512-bit Vectors
+;
+
+define <8 x double> @buildvector_add_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_add_8f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x double> [[R7]]
+;
+ %a0 = extractelement <8 x double> %a, i32 0
+ %a1 = extractelement <8 x double> %a, i32 1
+ %a2 = extractelement <8 x double> %a, i32 2
+ %a3 = extractelement <8 x double> %a, i32 3
+ %a4 = extractelement <8 x double> %a, i32 4
+ %a5 = extractelement <8 x double> %a, i32 5
+ %a6 = extractelement <8 x double> %a, i32 6
+ %a7 = extractelement <8 x double> %a, i32 7
+ %b0 = extractelement <8 x double> %b, i32 0
+ %b1 = extractelement <8 x double> %b, i32 1
+ %b2 = extractelement <8 x double> %b, i32 2
+ %b3 = extractelement <8 x double> %b, i32 3
+ %b4 = extractelement <8 x double> %b, i32 4
+ %b5 = extractelement <8 x double> %b, i32 5
+ %b6 = extractelement <8 x double> %b, i32 6
+ %b7 = extractelement <8 x double> %b, i32 7
+ %c0 = fadd double %a0, %b0
+ %c1 = fadd double %a1, %b1
+ %c2 = fadd double %a2, %b2
+ %c3 = fadd double %a3, %b3
+ %c4 = fadd double %a4, %b4
+ %c5 = fadd double %a5, %b5
+ %c6 = fadd double %a6, %b6
+ %c7 = fadd double %a7, %b7
+ %r0 = insertelement <8 x double> poison, double %c0, i32 0
+ %r1 = insertelement <8 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <8 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <8 x double> %r2, double %c3, i32 3
+ %r4 = insertelement <8 x double> %r3, double %c4, i32 4
+ %r5 = insertelement <8 x double> %r4, double %c5, i32 5
+ %r6 = insertelement <8 x double> %r5, double %c6, i32 6
+ %r7 = insertelement <8 x double> %r6, double %c7, i32 7
+ ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_sub_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_sub_8f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x double> [[R7]]
+;
+ %a0 = extractelement <8 x double> %a, i32 0
+ %a1 = extractelement <8 x double> %a, i32 1
+ %a2 = extractelement <8 x double> %a, i32 2
+ %a3 = extractelement <8 x double> %a, i32 3
+ %a4 = extractelement <8 x double> %a, i32 4
+ %a5 = extractelement <8 x double> %a, i32 5
+ %a6 = extractelement <8 x double> %a, i32 6
+ %a7 = extractelement <8 x double> %a, i32 7
+ %b0 = extractelement <8 x double> %b, i32 0
+ %b1 = extractelement <8 x double> %b, i32 1
+ %b2 = extractelement <8 x double> %b, i32 2
+ %b3 = extractelement <8 x double> %b, i32 3
+ %b4 = extractelement <8 x double> %b, i32 4
+ %b5 = extractelement <8 x double> %b, i32 5
+ %b6 = extractelement <8 x double> %b, i32 6
+ %b7 = extractelement <8 x double> %b, i32 7
+ %c0 = fsub double %a0, %b0
+ %c1 = fsub double %a1, %b1
+ %c2 = fsub double %a2, %b2
+ %c3 = fsub double %a3, %b3
+ %c4 = fsub double %a4, %b4
+ %c5 = fsub double %a5, %b5
+ %c6 = fsub double %a6, %b6
+ %c7 = fsub double %a7, %b7
+ %r0 = insertelement <8 x double> poison, double %c0, i32 0
+ %r1 = insertelement <8 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <8 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <8 x double> %r2, double %c3, i32 3
+ %r4 = insertelement <8 x double> %r3, double %c4, i32 4
+ %r5 = insertelement <8 x double> %r4, double %c5, i32 5
+ %r6 = insertelement <8 x double> %r5, double %c6, i32 6
+ %r7 = insertelement <8 x double> %r6, double %c7, i32 7
+ ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_mul_8f64(<8 x double> %a, <8 x double> %b) {
+; CHECK-LABEL: @buildvector_mul_8f64(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <8 x double> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x double> [[R7]]
+;
+ %a0 = extractelement <8 x double> %a, i32 0
+ %a1 = extractelement <8 x double> %a, i32 1
+ %a2 = extractelement <8 x double> %a, i32 2
+ %a3 = extractelement <8 x double> %a, i32 3
+ %a4 = extractelement <8 x double> %a, i32 4
+ %a5 = extractelement <8 x double> %a, i32 5
+ %a6 = extractelement <8 x double> %a, i32 6
+ %a7 = extractelement <8 x double> %a, i32 7
+ %b0 = extractelement <8 x double> %b, i32 0
+ %b1 = extractelement <8 x double> %b, i32 1
+ %b2 = extractelement <8 x double> %b, i32 2
+ %b3 = extractelement <8 x double> %b, i32 3
+ %b4 = extractelement <8 x double> %b, i32 4
+ %b5 = extractelement <8 x double> %b, i32 5
+ %b6 = extractelement <8 x double> %b, i32 6
+ %b7 = extractelement <8 x double> %b, i32 7
+ %c0 = fmul double %a0, %b0
+ %c1 = fmul double %a1, %b1
+ %c2 = fmul double %a2, %b2
+ %c3 = fmul double %a3, %b3
+ %c4 = fmul double %a4, %b4
+ %c5 = fmul double %a5, %b5
+ %c6 = fmul double %a6, %b6
+ %c7 = fmul double %a7, %b7
+ %r0 = insertelement <8 x double> poison, double %c0, i32 0
+ %r1 = insertelement <8 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <8 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <8 x double> %r2, double %c3, i32 3
+ %r4 = insertelement <8 x double> %r3, double %c4, i32 4
+ %r5 = insertelement <8 x double> %r4, double %c5, i32 5
+ %r6 = insertelement <8 x double> %r5, double %c6, i32 6
+ %r7 = insertelement <8 x double> %r6, double %c7, i32 7
+ ret <8 x double> %r7
+}
+
+define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
+; SSE-LABEL: @buildvector_div_8f64(
+; SSE-NEXT: [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; SSE-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; SSE-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; SSE-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; SSE-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; SSE-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; SSE-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; SSE-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; SSE-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; SSE-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; SSE-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; SSE-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; SSE-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; SSE-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; SSE-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; SSE-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; SSE-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; SSE-NEXT: ret <8 x double> [[R7]]
+;
+; SLM-LABEL: @buildvector_div_8f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <8 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <8 x double> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <8 x double> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <8 x double> [[A]], i32 3
+; SLM-NEXT: [[A4:%.*]] = extractelement <8 x double> [[A]], i32 4
+; SLM-NEXT: [[A5:%.*]] = extractelement <8 x double> [[A]], i32 5
+; SLM-NEXT: [[A6:%.*]] = extractelement <8 x double> [[A]], i32 6
+; SLM-NEXT: [[A7:%.*]] = extractelement <8 x double> [[A]], i32 7
+; SLM-NEXT: [[B0:%.*]] = extractelement <8 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <8 x double> [[B]], i32 1
+; SLM-NEXT: [[B2:%.*]] = extractelement <8 x double> [[B]], i32 2
+; SLM-NEXT: [[B3:%.*]] = extractelement <8 x double> [[B]], i32 3
+; SLM-NEXT: [[B4:%.*]] = extractelement <8 x double> [[B]], i32 4
+; SLM-NEXT: [[B5:%.*]] = extractelement <8 x double> [[B]], i32 5
+; SLM-NEXT: [[B6:%.*]] = extractelement <8 x double> [[B]], i32 6
+; SLM-NEXT: [[B7:%.*]] = extractelement <8 x double> [[B]], i32 7
+; SLM-NEXT: [[C0:%.*]] = fdiv double [[A0]], [[B0]]
+; SLM-NEXT: [[C1:%.*]] = fdiv double [[A1]], [[B1]]
+; SLM-NEXT: [[C2:%.*]] = fdiv double [[A2]], [[B2]]
+; SLM-NEXT: [[C3:%.*]] = fdiv double [[A3]], [[B3]]
+; SLM-NEXT: [[C4:%.*]] = fdiv double [[A4]], [[B4]]
+; SLM-NEXT: [[C5:%.*]] = fdiv double [[A5]], [[B5]]
+; SLM-NEXT: [[C6:%.*]] = fdiv double [[A6]], [[B6]]
+; SLM-NEXT: [[C7:%.*]] = fdiv double [[A7]], [[B7]]
+; SLM-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[C0]], i32 0
+; SLM-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[C1]], i32 1
+; SLM-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[C2]], i32 2
+; SLM-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[C3]], i32 3
+; SLM-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[C4]], i32 4
+; SLM-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[C5]], i32 5
+; SLM-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[C6]], i32 6
+; SLM-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[C7]], i32 7
+; SLM-NEXT: ret <8 x double> [[R7]]
+;
+; AVX-LABEL: @buildvector_div_8f64(
+; AVX-NEXT: [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; AVX-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; AVX-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; AVX-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; AVX-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; AVX-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; AVX-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; AVX-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; AVX-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; AVX-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; AVX-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; AVX-NEXT: ret <8 x double> [[R7]]
+;
+; AVX512-LABEL: @buildvector_div_8f64(
+; AVX512-NEXT: [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT: [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; AVX512-NEXT: [[R0:%.*]] = insertelement <8 x double> poison, double [[TMP2]], i32 0
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT: [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; AVX512-NEXT: [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT: [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; AVX512-NEXT: [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT: [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; AVX512-NEXT: [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; AVX512-NEXT: [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; AVX512-NEXT: [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; AVX512-NEXT: [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; AVX512-NEXT: [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; AVX512-NEXT: [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; AVX512-NEXT: ret <8 x double> [[R7]]
+;
+ %a0 = extractelement <8 x double> %a, i32 0
+ %a1 = extractelement <8 x double> %a, i32 1
+ %a2 = extractelement <8 x double> %a, i32 2
+ %a3 = extractelement <8 x double> %a, i32 3
+ %a4 = extractelement <8 x double> %a, i32 4
+ %a5 = extractelement <8 x double> %a, i32 5
+ %a6 = extractelement <8 x double> %a, i32 6
+ %a7 = extractelement <8 x double> %a, i32 7
+ %b0 = extractelement <8 x double> %b, i32 0
+ %b1 = extractelement <8 x double> %b, i32 1
+ %b2 = extractelement <8 x double> %b, i32 2
+ %b3 = extractelement <8 x double> %b, i32 3
+ %b4 = extractelement <8 x double> %b, i32 4
+ %b5 = extractelement <8 x double> %b, i32 5
+ %b6 = extractelement <8 x double> %b, i32 6
+ %b7 = extractelement <8 x double> %b, i32 7
+ %c0 = fdiv double %a0, %b0
+ %c1 = fdiv double %a1, %b1
+ %c2 = fdiv double %a2, %b2
+ %c3 = fdiv double %a3, %b3
+ %c4 = fdiv double %a4, %b4
+ %c5 = fdiv double %a5, %b5
+ %c6 = fdiv double %a6, %b6
+ %c7 = fdiv double %a7, %b7
+ %r0 = insertelement <8 x double> poison, double %c0, i32 0
+ %r1 = insertelement <8 x double> %r0, double %c1, i32 1
+ %r2 = insertelement <8 x double> %r1, double %c2, i32 2
+ %r3 = insertelement <8 x double> %r2, double %c3, i32 3
+ %r4 = insertelement <8 x double> %r3, double %c4, i32 4
+ %r5 = insertelement <8 x double> %r4, double %c5, i32 5
+ %r6 = insertelement <8 x double> %r5, double %c6, i32 6
+ %r7 = insertelement <8 x double> %r6, double %c7, i32 7
+ ret <8 x double> %r7
+}
+
+define <16 x float> @buildvector_add_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_add_16f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <16 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT: [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT: [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT: [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT: [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT: [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT: [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT: [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT: [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT: ret <16 x float> [[R15]]
+;
+ %a0 = extractelement <16 x float> %a, i32 0
+ %a1 = extractelement <16 x float> %a, i32 1
+ %a2 = extractelement <16 x float> %a, i32 2
+ %a3 = extractelement <16 x float> %a, i32 3
+ %a4 = extractelement <16 x float> %a, i32 4
+ %a5 = extractelement <16 x float> %a, i32 5
+ %a6 = extractelement <16 x float> %a, i32 6
+ %a7 = extractelement <16 x float> %a, i32 7
+ %a8 = extractelement <16 x float> %a, i32 8
+ %a9 = extractelement <16 x float> %a, i32 9
+ %a10 = extractelement <16 x float> %a, i32 10
+ %a11 = extractelement <16 x float> %a, i32 11
+ %a12 = extractelement <16 x float> %a, i32 12
+ %a13 = extractelement <16 x float> %a, i32 13
+ %a14 = extractelement <16 x float> %a, i32 14
+ %a15 = extractelement <16 x float> %a, i32 15
+ %b0 = extractelement <16 x float> %b, i32 0
+ %b1 = extractelement <16 x float> %b, i32 1
+ %b2 = extractelement <16 x float> %b, i32 2
+ %b3 = extractelement <16 x float> %b, i32 3
+ %b4 = extractelement <16 x float> %b, i32 4
+ %b5 = extractelement <16 x float> %b, i32 5
+ %b6 = extractelement <16 x float> %b, i32 6
+ %b7 = extractelement <16 x float> %b, i32 7
+ %b8 = extractelement <16 x float> %b, i32 8
+ %b9 = extractelement <16 x float> %b, i32 9
+ %b10 = extractelement <16 x float> %b, i32 10
+ %b11 = extractelement <16 x float> %b, i32 11
+ %b12 = extractelement <16 x float> %b, i32 12
+ %b13 = extractelement <16 x float> %b, i32 13
+ %b14 = extractelement <16 x float> %b, i32 14
+ %b15 = extractelement <16 x float> %b, i32 15
+ %c0 = fadd float %a0 , %b0
+ %c1 = fadd float %a1 , %b1
+ %c2 = fadd float %a2 , %b2
+ %c3 = fadd float %a3 , %b3
+ %c4 = fadd float %a4 , %b4
+ %c5 = fadd float %a5 , %b5
+ %c6 = fadd float %a6 , %b6
+ %c7 = fadd float %a7 , %b7
+ %c8 = fadd float %a8 , %b8
+ %c9 = fadd float %a9 , %b9
+ %c10 = fadd float %a10, %b10
+ %c11 = fadd float %a11, %b11
+ %c12 = fadd float %a12, %b12
+ %c13 = fadd float %a13, %b13
+ %c14 = fadd float %a14, %b14
+ %c15 = fadd float %a15, %b15
+ %r0 = insertelement <16 x float> poison, float %c0 , i32 0
+ %r1 = insertelement <16 x float> %r0 , float %c1 , i32 1
+ %r2 = insertelement <16 x float> %r1 , float %c2 , i32 2
+ %r3 = insertelement <16 x float> %r2 , float %c3 , i32 3
+ %r4 = insertelement <16 x float> %r3 , float %c4 , i32 4
+ %r5 = insertelement <16 x float> %r4 , float %c5 , i32 5
+ %r6 = insertelement <16 x float> %r5 , float %c6 , i32 6
+ %r7 = insertelement <16 x float> %r6 , float %c7 , i32 7
+ %r8 = insertelement <16 x float> %r7 , float %c8 , i32 8
+ %r9 = insertelement <16 x float> %r8 , float %c9 , i32 9
+ %r10 = insertelement <16 x float> %r9 , float %c10, i32 10
+ %r11 = insertelement <16 x float> %r10, float %c11, i32 11
+ %r12 = insertelement <16 x float> %r11, float %c12, i32 12
+ %r13 = insertelement <16 x float> %r12, float %c13, i32 13
+ %r14 = insertelement <16 x float> %r13, float %c14, i32 14
+ %r15 = insertelement <16 x float> %r14, float %c15, i32 15
+ ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_sub_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_sub_16f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <16 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT: [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT: [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT: [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT: [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT: [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT: [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT: [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT: [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT: ret <16 x float> [[R15]]
+;
+ %a0 = extractelement <16 x float> %a, i32 0
+ %a1 = extractelement <16 x float> %a, i32 1
+ %a2 = extractelement <16 x float> %a, i32 2
+ %a3 = extractelement <16 x float> %a, i32 3
+ %a4 = extractelement <16 x float> %a, i32 4
+ %a5 = extractelement <16 x float> %a, i32 5
+ %a6 = extractelement <16 x float> %a, i32 6
+ %a7 = extractelement <16 x float> %a, i32 7
+ %a8 = extractelement <16 x float> %a, i32 8
+ %a9 = extractelement <16 x float> %a, i32 9
+ %a10 = extractelement <16 x float> %a, i32 10
+ %a11 = extractelement <16 x float> %a, i32 11
+ %a12 = extractelement <16 x float> %a, i32 12
+ %a13 = extractelement <16 x float> %a, i32 13
+ %a14 = extractelement <16 x float> %a, i32 14
+ %a15 = extractelement <16 x float> %a, i32 15
+ %b0 = extractelement <16 x float> %b, i32 0
+ %b1 = extractelement <16 x float> %b, i32 1
+ %b2 = extractelement <16 x float> %b, i32 2
+ %b3 = extractelement <16 x float> %b, i32 3
+ %b4 = extractelement <16 x float> %b, i32 4
+ %b5 = extractelement <16 x float> %b, i32 5
+ %b6 = extractelement <16 x float> %b, i32 6
+ %b7 = extractelement <16 x float> %b, i32 7
+ %b8 = extractelement <16 x float> %b, i32 8
+ %b9 = extractelement <16 x float> %b, i32 9
+ %b10 = extractelement <16 x float> %b, i32 10
+ %b11 = extractelement <16 x float> %b, i32 11
+ %b12 = extractelement <16 x float> %b, i32 12
+ %b13 = extractelement <16 x float> %b, i32 13
+ %b14 = extractelement <16 x float> %b, i32 14
+ %b15 = extractelement <16 x float> %b, i32 15
+ %c0 = fsub float %a0 , %b0
+ %c1 = fsub float %a1 , %b1
+ %c2 = fsub float %a2 , %b2
+ %c3 = fsub float %a3 , %b3
+ %c4 = fsub float %a4 , %b4
+ %c5 = fsub float %a5 , %b5
+ %c6 = fsub float %a6 , %b6
+ %c7 = fsub float %a7 , %b7
+ %c8 = fsub float %a8 , %b8
+ %c9 = fsub float %a9 , %b9
+ %c10 = fsub float %a10, %b10
+ %c11 = fsub float %a11, %b11
+ %c12 = fsub float %a12, %b12
+ %c13 = fsub float %a13, %b13
+ %c14 = fsub float %a14, %b14
+ %c15 = fsub float %a15, %b15
+ %r0 = insertelement <16 x float> poison, float %c0 , i32 0
+ %r1 = insertelement <16 x float> %r0 , float %c1 , i32 1
+ %r2 = insertelement <16 x float> %r1 , float %c2 , i32 2
+ %r3 = insertelement <16 x float> %r2 , float %c3 , i32 3
+ %r4 = insertelement <16 x float> %r3 , float %c4 , i32 4
+ %r5 = insertelement <16 x float> %r4 , float %c5 , i32 5
+ %r6 = insertelement <16 x float> %r5 , float %c6 , i32 6
+ %r7 = insertelement <16 x float> %r6 , float %c7 , i32 7
+ %r8 = insertelement <16 x float> %r7 , float %c8 , i32 8
+ %r9 = insertelement <16 x float> %r8 , float %c9 , i32 9
+ %r10 = insertelement <16 x float> %r9 , float %c10, i32 10
+ %r11 = insertelement <16 x float> %r10, float %c11, i32 11
+ %r12 = insertelement <16 x float> %r11, float %c12, i32 12
+ %r13 = insertelement <16 x float> %r12, float %c13, i32 13
+ %r14 = insertelement <16 x float> %r13, float %c14, i32 14
+ %r15 = insertelement <16 x float> %r14, float %c15, i32 15
+ ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_mul_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_mul_16f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <16 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT: [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT: [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT: [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT: [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT: [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT: [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT: [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT: [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT: ret <16 x float> [[R15]]
+;
+ %a0 = extractelement <16 x float> %a, i32 0
+ %a1 = extractelement <16 x float> %a, i32 1
+ %a2 = extractelement <16 x float> %a, i32 2
+ %a3 = extractelement <16 x float> %a, i32 3
+ %a4 = extractelement <16 x float> %a, i32 4
+ %a5 = extractelement <16 x float> %a, i32 5
+ %a6 = extractelement <16 x float> %a, i32 6
+ %a7 = extractelement <16 x float> %a, i32 7
+ %a8 = extractelement <16 x float> %a, i32 8
+ %a9 = extractelement <16 x float> %a, i32 9
+ %a10 = extractelement <16 x float> %a, i32 10
+ %a11 = extractelement <16 x float> %a, i32 11
+ %a12 = extractelement <16 x float> %a, i32 12
+ %a13 = extractelement <16 x float> %a, i32 13
+ %a14 = extractelement <16 x float> %a, i32 14
+ %a15 = extractelement <16 x float> %a, i32 15
+ %b0 = extractelement <16 x float> %b, i32 0
+ %b1 = extractelement <16 x float> %b, i32 1
+ %b2 = extractelement <16 x float> %b, i32 2
+ %b3 = extractelement <16 x float> %b, i32 3
+ %b4 = extractelement <16 x float> %b, i32 4
+ %b5 = extractelement <16 x float> %b, i32 5
+ %b6 = extractelement <16 x float> %b, i32 6
+ %b7 = extractelement <16 x float> %b, i32 7
+ %b8 = extractelement <16 x float> %b, i32 8
+ %b9 = extractelement <16 x float> %b, i32 9
+ %b10 = extractelement <16 x float> %b, i32 10
+ %b11 = extractelement <16 x float> %b, i32 11
+ %b12 = extractelement <16 x float> %b, i32 12
+ %b13 = extractelement <16 x float> %b, i32 13
+ %b14 = extractelement <16 x float> %b, i32 14
+ %b15 = extractelement <16 x float> %b, i32 15
+ %c0 = fmul float %a0 , %b0
+ %c1 = fmul float %a1 , %b1
+ %c2 = fmul float %a2 , %b2
+ %c3 = fmul float %a3 , %b3
+ %c4 = fmul float %a4 , %b4
+ %c5 = fmul float %a5 , %b5
+ %c6 = fmul float %a6 , %b6
+ %c7 = fmul float %a7 , %b7
+ %c8 = fmul float %a8 , %b8
+ %c9 = fmul float %a9 , %b9
+ %c10 = fmul float %a10, %b10
+ %c11 = fmul float %a11, %b11
+ %c12 = fmul float %a12, %b12
+ %c13 = fmul float %a13, %b13
+ %c14 = fmul float %a14, %b14
+ %c15 = fmul float %a15, %b15
+ %r0 = insertelement <16 x float> poison, float %c0 , i32 0
+ %r1 = insertelement <16 x float> %r0 , float %c1 , i32 1
+ %r2 = insertelement <16 x float> %r1 , float %c2 , i32 2
+ %r3 = insertelement <16 x float> %r2 , float %c3 , i32 3
+ %r4 = insertelement <16 x float> %r3 , float %c4 , i32 4
+ %r5 = insertelement <16 x float> %r4 , float %c5 , i32 5
+ %r6 = insertelement <16 x float> %r5 , float %c6 , i32 6
+ %r7 = insertelement <16 x float> %r6 , float %c7 , i32 7
+ %r8 = insertelement <16 x float> %r7 , float %c8 , i32 8
+ %r9 = insertelement <16 x float> %r8 , float %c9 , i32 9
+ %r10 = insertelement <16 x float> %r9 , float %c10, i32 10
+ %r11 = insertelement <16 x float> %r10, float %c11, i32 11
+ %r12 = insertelement <16 x float> %r11, float %c12, i32 12
+ %r13 = insertelement <16 x float> %r12, float %c13, i32 13
+ %r14 = insertelement <16 x float> %r13, float %c14, i32 14
+ %r15 = insertelement <16 x float> %r14, float %c15, i32 15
+ ret <16 x float> %r15
+}
+
+define <16 x float> @buildvector_div_16f32(<16 x float> %a, <16 x float> %b) {
+; CHECK-LABEL: @buildvector_div_16f32(
+; CHECK-NEXT: [[TMP1:%.*]] = fdiv <16 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <16 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[R0:%.*]] = insertelement <16 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <16 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[R1:%.*]] = insertelement <16 x float> [[R0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <16 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[R2:%.*]] = insertelement <16 x float> [[R1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <16 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[R3:%.*]] = insertelement <16 x float> [[R2]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <16 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[R4:%.*]] = insertelement <16 x float> [[R3]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[R5:%.*]] = insertelement <16 x float> [[R4]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <16 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[R6:%.*]] = insertelement <16 x float> [[R5]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <16 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[R7:%.*]] = insertelement <16 x float> [[R6]], float [[TMP9]], i32 7
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <16 x float> [[TMP1]], i32 8
+; CHECK-NEXT: [[R8:%.*]] = insertelement <16 x float> [[R7]], float [[TMP10]], i32 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <16 x float> [[TMP1]], i32 9
+; CHECK-NEXT: [[R9:%.*]] = insertelement <16 x float> [[R8]], float [[TMP11]], i32 9
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <16 x float> [[TMP1]], i32 10
+; CHECK-NEXT: [[R10:%.*]] = insertelement <16 x float> [[R9]], float [[TMP12]], i32 10
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <16 x float> [[TMP1]], i32 11
+; CHECK-NEXT: [[R11:%.*]] = insertelement <16 x float> [[R10]], float [[TMP13]], i32 11
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <16 x float> [[TMP1]], i32 12
+; CHECK-NEXT: [[R12:%.*]] = insertelement <16 x float> [[R11]], float [[TMP14]], i32 12
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <16 x float> [[TMP1]], i32 13
+; CHECK-NEXT: [[R13:%.*]] = insertelement <16 x float> [[R12]], float [[TMP15]], i32 13
+; CHECK-NEXT: [[TMP16:%.*]] = extractelement <16 x float> [[TMP1]], i32 14
+; CHECK-NEXT: [[R14:%.*]] = insertelement <16 x float> [[R13]], float [[TMP16]], i32 14
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <16 x float> [[TMP1]], i32 15
+; CHECK-NEXT: [[R15:%.*]] = insertelement <16 x float> [[R14]], float [[TMP17]], i32 15
+; CHECK-NEXT: ret <16 x float> [[R15]]
+;
+ %a0 = extractelement <16 x float> %a, i32 0
+ %a1 = extractelement <16 x float> %a, i32 1
+ %a2 = extractelement <16 x float> %a, i32 2
+ %a3 = extractelement <16 x float> %a, i32 3
+ %a4 = extractelement <16 x float> %a, i32 4
+ %a5 = extractelement <16 x float> %a, i32 5
+ %a6 = extractelement <16 x float> %a, i32 6
+ %a7 = extractelement <16 x float> %a, i32 7
+ %a8 = extractelement <16 x float> %a, i32 8
+ %a9 = extractelement <16 x float> %a, i32 9
+ %a10 = extractelement <16 x float> %a, i32 10
+ %a11 = extractelement <16 x float> %a, i32 11
+ %a12 = extractelement <16 x float> %a, i32 12
+ %a13 = extractelement <16 x float> %a, i32 13
+ %a14 = extractelement <16 x float> %a, i32 14
+ %a15 = extractelement <16 x float> %a, i32 15
+ %b0 = extractelement <16 x float> %b, i32 0
+ %b1 = extractelement <16 x float> %b, i32 1
+ %b2 = extractelement <16 x float> %b, i32 2
+ %b3 = extractelement <16 x float> %b, i32 3
+ %b4 = extractelement <16 x float> %b, i32 4
+ %b5 = extractelement <16 x float> %b, i32 5
+ %b6 = extractelement <16 x float> %b, i32 6
+ %b7 = extractelement <16 x float> %b, i32 7
+ %b8 = extractelement <16 x float> %b, i32 8
+ %b9 = extractelement <16 x float> %b, i32 9
+ %b10 = extractelement <16 x float> %b, i32 10
+ %b11 = extractelement <16 x float> %b, i32 11
+ %b12 = extractelement <16 x float> %b, i32 12
+ %b13 = extractelement <16 x float> %b, i32 13
+ %b14 = extractelement <16 x float> %b, i32 14
+ %b15 = extractelement <16 x float> %b, i32 15
+ %c0 = fdiv float %a0 , %b0
+ %c1 = fdiv float %a1 , %b1
+ %c2 = fdiv float %a2 , %b2
+ %c3 = fdiv float %a3 , %b3
+ %c4 = fdiv float %a4 , %b4
+ %c5 = fdiv float %a5 , %b5
+ %c6 = fdiv float %a6 , %b6
+ %c7 = fdiv float %a7 , %b7
+ %c8 = fdiv float %a8 , %b8
+ %c9 = fdiv float %a9 , %b9
+ %c10 = fdiv float %a10, %b10
+ %c11 = fdiv float %a11, %b11
+ %c12 = fdiv float %a12, %b12
+ %c13 = fdiv float %a13, %b13
+ %c14 = fdiv float %a14, %b14
+ %c15 = fdiv float %a15, %b15
+ %r0 = insertelement <16 x float> poison, float %c0 , i32 0
+ %r1 = insertelement <16 x float> %r0 , float %c1 , i32 1
+ %r2 = insertelement <16 x float> %r1 , float %c2 , i32 2
+ %r3 = insertelement <16 x float> %r2 , float %c3 , i32 3
+ %r4 = insertelement <16 x float> %r3 , float %c4 , i32 4
+ %r5 = insertelement <16 x float> %r4 , float %c5 , i32 5
+ %r6 = insertelement <16 x float> %r5 , float %c6 , i32 6
+ %r7 = insertelement <16 x float> %r6 , float %c7 , i32 7
+ %r8 = insertelement <16 x float> %r7 , float %c8 , i32 8
+ %r9 = insertelement <16 x float> %r8 , float %c9 , i32 9
+ %r10 = insertelement <16 x float> %r9 , float %c10, i32 10
+ %r11 = insertelement <16 x float> %r10, float %c11, i32 11
+ %r12 = insertelement <16 x float> %r11, float %c12, i32 12
+ %r13 = insertelement <16 x float> %r12, float %c13, i32 13
+ %r14 = insertelement <16 x float> %r13, float %c14, i32 14
+ %r15 = insertelement <16 x float> %r14, float %c15, i32 15
+ ret <16 x float> %r15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
new file mode 100644
index 000000000000..cf47333e4b50
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/blending-shuffle-inseltpoison.ll
@@ -0,0 +1,200 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 -instcombine | FileCheck %s
+
+define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <2 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x i8> poison, i8 [[X0X0]], i32 0
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <2 x i8> [[INS1]], i8 [[Y1Y1]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[INS2]]
+;
+ %x0 = extractelement <2 x i8> %x, i32 0
+ %y1 = extractelement <2 x i8> %y, i32 1
+ %x0x0 = mul i8 %x0, %x0
+ %y1y1 = mul i8 %y1, %y1
+ %ins1 = insertelement <2 x i8> poison, i8 %x0x0, i32 0
+ %ins2 = insertelement <2 x i8> %ins1, i8 %y1y1, i32 1
+ ret <2 x i8> %ins2
+}
+
+define <4 x i8> @h(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @h(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <4 x i8> poison, i8 [[X0X0]], i32 0
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x i8> [[INS1]], i8 [[X3X3]], i32 1
+; CHECK-NEXT: [[INS3:%.*]] = insertelement <4 x i8> [[INS2]], i8 [[Y1Y1]], i32 2
+; CHECK-NEXT: [[INS4:%.*]] = insertelement <4 x i8> [[INS3]], i8 [[Y2Y2]], i32 3
+; CHECK-NEXT: ret <4 x i8> [[INS4]]
+;
+ %x0 = extractelement <4 x i8> %x, i32 0
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %y1 = extractelement <4 x i8> %y, i32 1
+ %y2 = extractelement <4 x i8> %y, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %y1y1 = mul i8 %y1, %y1
+ %y2y2 = mul i8 %y2, %y2
+ %ins1 = insertelement <4 x i8> poison, i8 %x0x0, i32 0
+ %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
+ %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
+ %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
+ ret <4 x i8> %ins4
+}
+
+define <4 x i8> @h_undef(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @h_undef(
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 3
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <4 x i8> undef, i8 [[X3X3]], i32 1
+; CHECK-NEXT: [[INS3:%.*]] = insertelement <4 x i8> [[INS2]], i8 [[Y1Y1]], i32 2
+; CHECK-NEXT: [[INS4:%.*]] = insertelement <4 x i8> [[INS3]], i8 [[Y2Y2]], i32 3
+; CHECK-NEXT: ret <4 x i8> [[INS4]]
+;
+ %x0 = extractelement <4 x i8> undef, i32 0
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %y1 = extractelement <4 x i8> %y, i32 1
+ %y2 = extractelement <4 x i8> %y, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %y1y1 = mul i8 %y1, %y1
+ %y2y2 = mul i8 %y2, %y2
+ %ins1 = insertelement <4 x i8> poison, i8 %x0x0, i32 0
+ %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
+ %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
+ %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
+ ret <4 x i8> %ins4
+}
+
+define i8 @i(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @i(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[Y1Y1]], [[Y2Y2]]
+; CHECK-NEXT: [[TMP3:%.*]] = add i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret i8 [[TMP3]]
+;
+ %x0 = extractelement <4 x i8> %x, i32 0
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %y1 = extractelement <4 x i8> %y, i32 1
+ %y2 = extractelement <4 x i8> %y, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %y1y1 = mul i8 %y1, %y1
+ %y2y2 = mul i8 %y2, %y2
+ %1 = add i8 %x0x0, %x3x3
+ %2 = add i8 %y1y1, %y2y2
+ %3 = add i8 %1, %2
+ ret i8 %3
+}
+
+define i8 @j(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: @j(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <4 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[Y]], i32 2
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[Y2Y2:%.*]] = mul i8 [[Y2]], [[Y2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[Y1Y1]], [[Y2Y2]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret i8 [[TMP3]]
+;
+ %x0 = extractelement <4 x i8> %x, i32 0
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %y1 = extractelement <4 x i8> %y, i32 1
+ %y2 = extractelement <4 x i8> %y, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %y1y1 = mul i8 %y1, %y1
+ %y2y2 = mul i8 %y2, %y2
+ %1 = add i8 %x0x0, %x3x3
+ %2 = add i8 %y1y1, %y2y2
+ %3 = sdiv i8 %1, %2
+ ret i8 %3
+}
+
+define i8 @k(<4 x i8> %x) {
+; CHECK-LABEL: @k(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i8> [[X]], i32 1
+; CHECK-NEXT: [[X2:%.*]] = extractelement <4 x i8> [[X]], i32 2
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[X1X1:%.*]] = mul i8 [[X1]], [[X1]]
+; CHECK-NEXT: [[X2X2:%.*]] = mul i8 [[X2]], [[X2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[X1X1]], [[X2X2]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret i8 [[TMP3]]
+;
+ %x0 = extractelement <4 x i8> %x, i32 0
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %x1 = extractelement <4 x i8> %x, i32 1
+ %x2 = extractelement <4 x i8> %x, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %x1x1 = mul i8 %x1, %x1
+ %x2x2 = mul i8 %x2, %x2
+ %1 = add i8 %x0x0, %x3x3
+ %2 = add i8 %x1x1, %x2x2
+ %3 = sdiv i8 %1, %2
+ ret i8 %3
+}
+
+define i8 @k_bb(<4 x i8> %x) {
+; CHECK-LABEL: @k_bb(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <4 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: br label [[BB1:%.*]]
+; CHECK: bb1:
+; CHECK-NEXT: [[X3:%.*]] = extractelement <4 x i8> [[X]], i32 3
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i8> [[X]], i32 1
+; CHECK-NEXT: [[X2:%.*]] = extractelement <4 x i8> [[X]], i32 2
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[X3X3:%.*]] = mul i8 [[X3]], [[X3]]
+; CHECK-NEXT: [[X1X1:%.*]] = mul i8 [[X1]], [[X1]]
+; CHECK-NEXT: [[X2X2:%.*]] = mul i8 [[X2]], [[X2]]
+; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X0X0]], [[X3X3]]
+; CHECK-NEXT: [[TMP2:%.*]] = add i8 [[X1X1]], [[X2X2]]
+; CHECK-NEXT: [[TMP3:%.*]] = sdiv i8 [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret i8 [[TMP3]]
+;
+ %x0 = extractelement <4 x i8> %x, i32 0
+ br label %bb1
+bb1:
+ %x3 = extractelement <4 x i8> %x, i32 3
+ %x1 = extractelement <4 x i8> %x, i32 1
+ %x2 = extractelement <4 x i8> %x, i32 2
+ %x0x0 = mul i8 %x0, %x0
+ %x3x3 = mul i8 %x3, %x3
+ %x1x1 = mul i8 %x1, %x1
+ %x2x2 = mul i8 %x2, %x2
+ %1 = add i8 %x0x0, %x3x3
+ %2 = add i8 %x1x1, %x2x2
+ %3 = sdiv i8 %1, %2
+ ret i8 %3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
new file mode 100644
index 000000000000..c096a0bac7a9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/cmp_commute-inseltpoison.ll
@@ -0,0 +1,283 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64--- -mattr=+sse2 | FileCheck %s
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64--- -mattr=+avx | FileCheck %s
+
+;
+; Check that we can commute operands based on the predicate.
+;
+
+define <4 x i32> @icmp_eq_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_eq_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp eq <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %p0 = getelementptr inbounds i32, i32* %b, i32 0
+ %p1 = getelementptr inbounds i32, i32* %b, i32 1
+ %p2 = getelementptr inbounds i32, i32* %b, i32 2
+ %p3 = getelementptr inbounds i32, i32* %b, i32 3
+ %b0 = load i32, i32* %p0, align 4
+ %b1 = load i32, i32* %p1, align 4
+ %b2 = load i32, i32* %p2, align 4
+ %b3 = load i32, i32* %p3, align 4
+ %c0 = icmp eq i32 %a0, %b0
+ %c1 = icmp eq i32 %b1, %a1
+ %c2 = icmp eq i32 %b2, %a2
+ %c3 = icmp eq i32 %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @icmp_ne_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_ne_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %p0 = getelementptr inbounds i32, i32* %b, i32 0
+ %p1 = getelementptr inbounds i32, i32* %b, i32 1
+ %p2 = getelementptr inbounds i32, i32* %b, i32 2
+ %p3 = getelementptr inbounds i32, i32* %b, i32 3
+ %b0 = load i32, i32* %p0, align 4
+ %b1 = load i32, i32* %p1, align 4
+ %b2 = load i32, i32* %p2, align 4
+ %b3 = load i32, i32* %p3, align 4
+ %c0 = icmp ne i32 %a0, %b0
+ %c1 = icmp ne i32 %b1, %a1
+ %c2 = icmp ne i32 %b2, %a2
+ %c3 = icmp ne i32 %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_oeq_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_oeq_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp oeq <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %p0 = getelementptr inbounds float, float* %b, i32 0
+ %p1 = getelementptr inbounds float, float* %b, i32 1
+ %p2 = getelementptr inbounds float, float* %b, i32 2
+ %p3 = getelementptr inbounds float, float* %b, i32 3
+ %b0 = load float, float* %p0, align 4
+ %b1 = load float, float* %p1, align 4
+ %b2 = load float, float* %p2, align 4
+ %b3 = load float, float* %p3, align 4
+ %c0 = fcmp oeq float %a0, %b0
+ %c1 = fcmp oeq float %b1, %a1
+ %c2 = fcmp oeq float %b2, %a2
+ %c3 = fcmp oeq float %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_uno_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_uno_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %p0 = getelementptr inbounds float, float* %b, i32 0
+ %p1 = getelementptr inbounds float, float* %b, i32 1
+ %p2 = getelementptr inbounds float, float* %b, i32 2
+ %p3 = getelementptr inbounds float, float* %b, i32 3
+ %b0 = load float, float* %p0, align 4
+ %b1 = load float, float* %p1, align 4
+ %b2 = load float, float* %p2, align 4
+ %b3 = load float, float* %p3, align 4
+ %c0 = fcmp uno float %a0, %b0
+ %c1 = fcmp uno float %b1, %a1
+ %c2 = fcmp uno float %b2, %a2
+ %c3 = fcmp uno float %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+;
+; Check that we can commute operands by swapping the predicate.
+;
+
+define <4 x i32> @icmp_sgt_slt_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_sgt_slt_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp slt <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %p0 = getelementptr inbounds i32, i32* %b, i32 0
+ %p1 = getelementptr inbounds i32, i32* %b, i32 1
+ %p2 = getelementptr inbounds i32, i32* %b, i32 2
+ %p3 = getelementptr inbounds i32, i32* %b, i32 3
+ %b0 = load i32, i32* %p0, align 4
+ %b1 = load i32, i32* %p1, align 4
+ %b2 = load i32, i32* %p2, align 4
+ %b3 = load i32, i32* %p3, align 4
+ %c0 = icmp sgt i32 %a0, %b0
+ %c1 = icmp slt i32 %b1, %a1
+ %c2 = icmp slt i32 %b2, %a2
+ %c3 = icmp sgt i32 %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @icmp_uge_ule_v4i32(<4 x i32> %a, i32* %b) {
+; CHECK-LABEL: @icmp_uge_ule_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ule <4 x i32> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %p0 = getelementptr inbounds i32, i32* %b, i32 0
+ %p1 = getelementptr inbounds i32, i32* %b, i32 1
+ %p2 = getelementptr inbounds i32, i32* %b, i32 2
+ %p3 = getelementptr inbounds i32, i32* %b, i32 3
+ %b0 = load i32, i32* %p0, align 4
+ %b1 = load i32, i32* %p1, align 4
+ %b2 = load i32, i32* %p2, align 4
+ %b3 = load i32, i32* %p3, align 4
+ %c0 = icmp uge i32 %a0, %b0
+ %c1 = icmp ule i32 %b1, %a1
+ %c2 = icmp ule i32 %b2, %a2
+ %c3 = icmp uge i32 %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_ogt_olt_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_ogt_olt_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fcmp olt <4 x float> [[TMP2]], [[A:%.*]]
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[TMP3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %p0 = getelementptr inbounds float, float* %b, i32 0
+ %p1 = getelementptr inbounds float, float* %b, i32 1
+ %p2 = getelementptr inbounds float, float* %b, i32 2
+ %p3 = getelementptr inbounds float, float* %b, i32 3
+ %b0 = load float, float* %p0, align 4
+ %b1 = load float, float* %p1, align 4
+ %b2 = load float, float* %p2, align 4
+ %b3 = load float, float* %p3, align 4
+ %c0 = fcmp ogt float %a0, %b0
+ %c1 = fcmp olt float %b1, %a1
+ %c2 = fcmp olt float %b2, %a2
+ %c3 = fcmp ogt float %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @fcmp_ord_uno_v4i32(<4 x float> %a, float* %b) {
+; CHECK-LABEL: @fcmp_ord_uno_v4i32(
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT: [[B0:%.*]] = load float, float* [[B]], align 4
+; CHECK-NEXT: [[B1:%.*]] = load float, float* [[P1]], align 4
+; CHECK-NEXT: [[B2:%.*]] = load float, float* [[P2]], align 4
+; CHECK-NEXT: [[B3:%.*]] = load float, float* [[P3]], align 4
+; CHECK-NEXT: [[C0:%.*]] = fcmp ord float [[A0]], [[B0]]
+; CHECK-NEXT: [[C1:%.*]] = fcmp uno float [[B1]], [[A1]]
+; CHECK-NEXT: [[C2:%.*]] = fcmp uno float [[B2]], [[A2]]
+; CHECK-NEXT: [[C3:%.*]] = fcmp ord float [[A3]], [[B3]]
+; CHECK-NEXT: [[D0:%.*]] = insertelement <4 x i1> poison, i1 [[C0]], i32 0
+; CHECK-NEXT: [[D1:%.*]] = insertelement <4 x i1> [[D0]], i1 [[C1]], i32 1
+; CHECK-NEXT: [[D2:%.*]] = insertelement <4 x i1> [[D1]], i1 [[C2]], i32 2
+; CHECK-NEXT: [[D3:%.*]] = insertelement <4 x i1> [[D2]], i1 [[C3]], i32 3
+; CHECK-NEXT: [[R:%.*]] = sext <4 x i1> [[D3]] to <4 x i32>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %p0 = getelementptr inbounds float, float* %b, i32 0
+ %p1 = getelementptr inbounds float, float* %b, i32 1
+ %p2 = getelementptr inbounds float, float* %b, i32 2
+ %p3 = getelementptr inbounds float, float* %b, i32 3
+ %b0 = load float, float* %p0, align 4
+ %b1 = load float, float* %p1, align 4
+ %b2 = load float, float* %p2, align 4
+ %b3 = load float, float* %p3, align 4
+ %c0 = fcmp ord float %a0, %b0
+ %c1 = fcmp uno float %b1, %a1
+ %c2 = fcmp uno float %b2, %a2
+ %c3 = fcmp ord float %a3, %b3
+ %d0 = insertelement <4 x i1> poison, i1 %c0, i32 0
+ %d1 = insertelement <4 x i1> %d0, i1 %c1, i32 1
+ %d2 = insertelement <4 x i1> %d1, i1 %c2, i32 2
+ %d3 = insertelement <4 x i1> %d2, i1 %c3, i32 3
+ %r = sext <4 x i1> %d3 to <4 x i32>
+ ret <4 x i32> %r
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
new file mode 100644
index 000000000000..348aed4a48e5
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/crash_scheduling-inseltpoison.ll
@@ -0,0 +1,81 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basic-aa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-darwin13.3.0"
+
+define void @_foo(double %p1, double %p2, double %p3) #0 {
+; CHECK-LABEL: @_foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TAB1:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT: [[TAB2:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT: br label [[BB1:%.*]]
+; CHECK: bb1:
+; CHECK-NEXT: [[MUL19:%.*]] = fmul double [[P1:%.*]], 1.638400e+04
+; CHECK-NEXT: [[MUL20:%.*]] = fmul double [[P3:%.*]], 1.638400e+04
+; CHECK-NEXT: [[ADD:%.*]] = fadd double [[MUL20]], 8.192000e+03
+; CHECK-NEXT: [[MUL21:%.*]] = fmul double [[P2:%.*]], 1.638400e+04
+; CHECK-NEXT: br label [[FOR_BODY:%.*]]
+; CHECK: for.body:
+; CHECK-NEXT: [[INDVARS_IV266:%.*]] = phi i64 [ 0, [[BB1]] ], [ [[INDVARS_IV_NEXT267:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[T_0259:%.*]] = phi double [ 0.000000e+00, [[BB1]] ], [ [[ADD27:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[P3_ADDR_0258:%.*]] = phi double [ [[ADD]], [[BB1]] ], [ [[ADD28:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT: [[VECINIT_I_I237:%.*]] = insertelement <2 x double> poison, double [[T_0259]], i32 0
+; CHECK-NEXT: [[X13:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I237]])
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB1]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT: store i32 [[X13]], i32* [[ARRAYIDX]], align 4, [[TBAA0:!tbaa !.*]]
+; CHECK-NEXT: [[VECINIT_I_I:%.*]] = insertelement <2 x double> poison, double [[P3_ADDR_0258]], i32 0
+; CHECK-NEXT: [[X14:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I]])
+; CHECK-NEXT: [[ARRAYIDX26:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB2]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT: store i32 [[X14]], i32* [[ARRAYIDX26]], align 4, [[TBAA0]]
+; CHECK-NEXT: [[ADD27]] = fadd double [[MUL19]], [[T_0259]]
+; CHECK-NEXT: [[ADD28]] = fadd double [[MUL21]], [[P3_ADDR_0258]]
+; CHECK-NEXT: [[INDVARS_IV_NEXT267]] = add nuw nsw i64 [[INDVARS_IV266]], 1
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT267]], 256
+; CHECK-NEXT: br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]]
+; CHECK: return:
+; CHECK-NEXT: ret void
+;
+entry:
+ %tab1 = alloca [256 x i32], align 16
+ %tab2 = alloca [256 x i32], align 16
+ br label %bb1
+
+
+bb1:
+ %mul19 = fmul double %p1, 1.638400e+04
+ %mul20 = fmul double %p3, 1.638400e+04
+ %add = fadd double %mul20, 8.192000e+03
+ %mul21 = fmul double %p2, 1.638400e+04
+ ; The SLPVectorizer crashed when scheduling this block after it inserted an
+ ; insertelement instruction (during vectorizing the for.body block) at this position.
+ br label %for.body
+
+for.body:
+ %indvars.iv266 = phi i64 [ 0, %bb1 ], [ %indvars.iv.next267, %for.body ]
+ %t.0259 = phi double [ 0.000000e+00, %bb1 ], [ %add27, %for.body ]
+ %p3.addr.0258 = phi double [ %add, %bb1 ], [ %add28, %for.body ]
+ %vecinit.i.i237 = insertelement <2 x double> poison, double %t.0259, i32 0
+ %x13 = tail call i32 @_xfn(<2 x double> %vecinit.i.i237) #2
+ %arrayidx = getelementptr inbounds [256 x i32], [256 x i32]* %tab1, i64 0, i64 %indvars.iv266
+ store i32 %x13, i32* %arrayidx, align 4, !tbaa !4
+ %vecinit.i.i = insertelement <2 x double> poison, double %p3.addr.0258, i32 0
+ %x14 = tail call i32 @_xfn(<2 x double> %vecinit.i.i) #2
+ %arrayidx26 = getelementptr inbounds [256 x i32], [256 x i32]* %tab2, i64 0, i64 %indvars.iv266
+ store i32 %x14, i32* %arrayidx26, align 4, !tbaa !4
+ %add27 = fadd double %mul19, %t.0259
+ %add28 = fadd double %mul21, %p3.addr.0258
+ %indvars.iv.next267 = add nuw nsw i64 %indvars.iv266, 1
+ %exitcond = icmp eq i64 %indvars.iv.next267, 256
+ br i1 %exitcond, label %return, label %for.body
+
+return:
+ ret void
+}
+
+declare i32 @_xfn(<2 x double>) #4
+
+!3 = !{!"int", !5, i64 0}
+!4 = !{!3, !3, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load-inseltpoison.ll
new file mode 100644
index 000000000000..45b65f165786
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/external_user_jumbled_load-inseltpoison.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -mtriple=x86_64-unknown -mattr=+avx -slp-vectorizer | FileCheck %s
+
+ at array = external global [20 x [13 x i32]]
+
+define void @hoge(i64 %idx, <4 x i32>* %sink) {
+; CHECK-LABEL: @hoge(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX:%.*]], i64 5
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 6
+; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 7
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 [[IDX]], i64 8
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32* [[TMP0]] to <4 x i32>*
+; CHECK-NEXT: [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 1, i32 2, i32 3, i32 0>
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 2
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 3
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x i32> [[TMP13]], <4 x i32>* [[SINK:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+bb:
+ %0 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 5
+ %1 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 6
+ %2 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 7
+ %3 = getelementptr inbounds [20 x [13 x i32]], [20 x [13 x i32]]* @array, i64 0, i64 %idx, i64 8
+ %4 = load i32, i32* %1, align 4
+ %5 = insertelement <4 x i32> poison, i32 %4, i32 0
+ %6 = load i32, i32* %2, align 4
+ %7 = insertelement <4 x i32> %5, i32 %6, i32 1
+ %8 = load i32, i32* %3, align 4
+ %9 = insertelement <4 x i32> %7, i32 %8, i32 2
+ %10 = load i32, i32* %0, align 4
+ %11 = insertelement <4 x i32> %9, i32 %10, i32 3
+ store <4 x i32> %11, <4 x i32>* %sink
+ ret void
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle-inseltpoison.ll
new file mode 100644
index 000000000000..f98b0dcd830b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/extract-shuffle-inseltpoison.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux -mcpu=bdver2 -slp-schedule-budget=1 | FileCheck %s
+
+define <2 x i8> @g(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @g(
+; CHECK-NEXT: [[X0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[Y1:%.*]] = extractelement <2 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[X0X0:%.*]] = mul i8 [[X0]], [[X0]]
+; CHECK-NEXT: [[Y1Y1:%.*]] = mul i8 [[Y1]], [[Y1]]
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x i8> poison, i8 [[X0X0]], i32 0
+; CHECK-NEXT: [[INS2:%.*]] = insertelement <2 x i8> [[INS1]], i8 [[Y1Y1]], i32 1
+; CHECK-NEXT: ret <2 x i8> [[INS2]]
+;
+ %x0 = extractelement <2 x i8> %x, i32 0
+ %y1 = extractelement <2 x i8> %y, i32 1
+ %x0x0 = mul i8 %x0, %x0
+ %y1y1 = mul i8 %y1, %y1
+ %ins1 = insertelement <2 x i8> poison, i8 %x0x0, i32 0
+ %ins2 = insertelement <2 x i8> %ins1, i8 %y1y1, i32 1
+ ret <2 x i8> %ins2
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fptosi-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/fptosi-inseltpoison.ll
new file mode 100644
index 000000000000..8bc8639a0eb6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fptosi-inseltpoison.ll
@@ -0,0 +1,534 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+ at src64 = common global [8 x double] zeroinitializer, align 64
+ at src32 = common global [16 x float] zeroinitializer, align 64
+ at dst64 = common global [8 x i64] zeroinitializer, align 64
+ at dst32 = common global [16 x i32] zeroinitializer, align 64
+ at dst16 = common global [32 x i16] zeroinitializer, align 64
+ at dst8 = common global [64 x i8] zeroinitializer, align 64
+
+;
+; FPTOSI vXf64
+;
+
+define void @fptosi_8f64_8i64() #0 {
+; SSE-LABEL: @fptosi_8f64_8i64(
+; SSE-NEXT: [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT: [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT: [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT: [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT: [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT: [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT: [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT: [[CVT0:%.*]] = fptosi double [[A0]] to i64
+; SSE-NEXT: [[CVT1:%.*]] = fptosi double [[A1]] to i64
+; SSE-NEXT: [[CVT2:%.*]] = fptosi double [[A2]] to i64
+; SSE-NEXT: [[CVT3:%.*]] = fptosi double [[A3]] to i64
+; SSE-NEXT: [[CVT4:%.*]] = fptosi double [[A4]] to i64
+; SSE-NEXT: [[CVT5:%.*]] = fptosi double [[A5]] to i64
+; SSE-NEXT: [[CVT6:%.*]] = fptosi double [[A6]] to i64
+; SSE-NEXT: [[CVT7:%.*]] = fptosi double [[A7]] to i64
+; SSE-NEXT: store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT: store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT: store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT: store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT: store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @fptosi_8f64_8i64(
+; AVX256NODQ-NEXT: [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT: [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT: [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT: [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT: [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = fptosi double [[A0]] to i64
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = fptosi double [[A1]] to i64
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = fptosi double [[A2]] to i64
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = fptosi double [[A3]] to i64
+; AVX256NODQ-NEXT: [[CVT4:%.*]] = fptosi double [[A4]] to i64
+; AVX256NODQ-NEXT: [[CVT5:%.*]] = fptosi double [[A5]] to i64
+; AVX256NODQ-NEXT: [[CVT6:%.*]] = fptosi double [[A6]] to i64
+; AVX256NODQ-NEXT: [[CVT7:%.*]] = fptosi double [[A7]] to i64
+; AVX256NODQ-NEXT: store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @fptosi_8f64_8i64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT: [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i64>
+; AVX512-NEXT: store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @fptosi_8f64_8i64(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; AVX256DQ-NEXT: [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; AVX256DQ-NEXT: [[TMP3:%.*]] = fptosi <4 x double> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT: [[TMP4:%.*]] = fptosi <4 x double> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT: store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT: store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT: ret void
+;
+ %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+ %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+ %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+ %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+ %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+ %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+ %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+ %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+ %cvt0 = fptosi double %a0 to i64
+ %cvt1 = fptosi double %a1 to i64
+ %cvt2 = fptosi double %a2 to i64
+ %cvt3 = fptosi double %a3 to i64
+ %cvt4 = fptosi double %a4 to i64
+ %cvt5 = fptosi double %a5 to i64
+ %cvt6 = fptosi double %a6 to i64
+ %cvt7 = fptosi double %a7 to i64
+ store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+ store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+ store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+ store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+ store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+ store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+ store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+ store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+define void @fptosi_8f64_8i32() #0 {
+; SSE-LABEL: @fptosi_8f64_8i32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x double>, <4 x double>* bitcast ([8 x double]* @src64 to <4 x double>*), align 8
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x double>, <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4) to <4 x double>*), align 8
+; SSE-NEXT: [[TMP3:%.*]] = fptosi <4 x double> [[TMP1]] to <4 x i32>
+; SSE-NEXT: [[TMP4:%.*]] = fptosi <4 x double> [[TMP2]] to <4 x i32>
+; SSE-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([16 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @fptosi_8f64_8i32(
+; AVX-NEXT: [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX-NEXT: [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i32>
+; AVX-NEXT: store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX-NEXT: ret void
+;
+ %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+ %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+ %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+ %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+ %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+ %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+ %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+ %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+ %cvt0 = fptosi double %a0 to i32
+ %cvt1 = fptosi double %a1 to i32
+ %cvt2 = fptosi double %a2 to i32
+ %cvt3 = fptosi double %a3 to i32
+ %cvt4 = fptosi double %a4 to i32
+ %cvt5 = fptosi double %a5 to i32
+ %cvt6 = fptosi double %a6 to i32
+ %cvt7 = fptosi double %a7 to i32
+ store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+ store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+ store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+ store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+ store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+ store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+ store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+ store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @fptosi_8f64_8i16() #0 {
+; CHECK-LABEL: @fptosi_8f64_8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT: ret void
+;
+ %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+ %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+ %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+ %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+ %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+ %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+ %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+ %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+ %cvt0 = fptosi double %a0 to i16
+ %cvt1 = fptosi double %a1 to i16
+ %cvt2 = fptosi double %a2 to i16
+ %cvt3 = fptosi double %a3 to i16
+ %cvt4 = fptosi double %a4 to i16
+ %cvt5 = fptosi double %a5 to i16
+ %cvt6 = fptosi double %a6 to i16
+ %cvt7 = fptosi double %a7 to i16
+ store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+ store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+ store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+ store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+ store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+ store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+ store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+ store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+ ret void
+}
+
+define void @fptosi_8f64_8i8() #0 {
+; CHECK-LABEL: @fptosi_8f64_8i8(
+; CHECK-NEXT: [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; CHECK-NEXT: [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT: [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; CHECK-NEXT: [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; CHECK-NEXT: [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; CHECK-NEXT: [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; CHECK-NEXT: [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; CHECK-NEXT: [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; CHECK-NEXT: [[CVT0:%.*]] = fptosi double [[A0]] to i8
+; CHECK-NEXT: [[CVT1:%.*]] = fptosi double [[A1]] to i8
+; CHECK-NEXT: [[CVT2:%.*]] = fptosi double [[A2]] to i8
+; CHECK-NEXT: [[CVT3:%.*]] = fptosi double [[A3]] to i8
+; CHECK-NEXT: [[CVT4:%.*]] = fptosi double [[A4]] to i8
+; CHECK-NEXT: [[CVT5:%.*]] = fptosi double [[A5]] to i8
+; CHECK-NEXT: [[CVT6:%.*]] = fptosi double [[A6]] to i8
+; CHECK-NEXT: [[CVT7:%.*]] = fptosi double [[A7]] to i8
+; CHECK-NEXT: store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT: store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT: store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT: store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT: store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT: store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT: store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT: store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT: ret void
+;
+ %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+ %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+ %a2 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+ %a3 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+ %a4 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+ %a5 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+ %a6 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+ %a7 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+ %cvt0 = fptosi double %a0 to i8
+ %cvt1 = fptosi double %a1 to i8
+ %cvt2 = fptosi double %a2 to i8
+ %cvt3 = fptosi double %a3 to i8
+ %cvt4 = fptosi double %a4 to i8
+ %cvt5 = fptosi double %a5 to i8
+ %cvt6 = fptosi double %a6 to i8
+ %cvt7 = fptosi double %a7 to i8
+ store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+ store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+ store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+ store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+ store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+ store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+ store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+ store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+ ret void
+}
+
+;
+; FPTOSI vXf32
+;
+
+define void @fptosi_8f32_8i64() #0 {
+; SSE-LABEL: @fptosi_8f32_8i64(
+; SSE-NEXT: [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT: [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT: [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT: [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT: [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT: [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT: [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT: [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT: [[CVT0:%.*]] = fptosi float [[A0]] to i64
+; SSE-NEXT: [[CVT1:%.*]] = fptosi float [[A1]] to i64
+; SSE-NEXT: [[CVT2:%.*]] = fptosi float [[A2]] to i64
+; SSE-NEXT: [[CVT3:%.*]] = fptosi float [[A3]] to i64
+; SSE-NEXT: [[CVT4:%.*]] = fptosi float [[A4]] to i64
+; SSE-NEXT: [[CVT5:%.*]] = fptosi float [[A5]] to i64
+; SSE-NEXT: [[CVT6:%.*]] = fptosi float [[A6]] to i64
+; SSE-NEXT: [[CVT7:%.*]] = fptosi float [[A7]] to i64
+; SSE-NEXT: store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; SSE-NEXT: store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; SSE-NEXT: store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; SSE-NEXT: store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; SSE-NEXT: store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @fptosi_8f32_8i64(
+; AVX256NODQ-NEXT: [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; AVX256NODQ-NEXT: [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT: [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; AVX256NODQ-NEXT: [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT: [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; AVX256NODQ-NEXT: [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT: [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; AVX256NODQ-NEXT: [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = fptosi float [[A0]] to i64
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = fptosi float [[A1]] to i64
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = fptosi float [[A2]] to i64
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = fptosi float [[A3]] to i64
+; AVX256NODQ-NEXT: [[CVT4:%.*]] = fptosi float [[A4]] to i64
+; AVX256NODQ-NEXT: [[CVT5:%.*]] = fptosi float [[A5]] to i64
+; AVX256NODQ-NEXT: [[CVT6:%.*]] = fptosi float [[A6]] to i64
+; AVX256NODQ-NEXT: [[CVT7:%.*]] = fptosi float [[A7]] to i64
+; AVX256NODQ-NEXT: store i64 [[CVT0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT: store i64 [[CVT7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @fptosi_8f32_8i64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX512-NEXT: [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i64>
+; AVX512-NEXT: store <8 x i64> [[TMP2]], <8 x i64>* bitcast ([8 x i64]* @dst64 to <8 x i64>*), align 8
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @fptosi_8f32_8i64(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; AVX256DQ-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; AVX256DQ-NEXT: [[TMP3:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i64>
+; AVX256DQ-NEXT: [[TMP4:%.*]] = fptosi <4 x float> [[TMP2]] to <4 x i64>
+; AVX256DQ-NEXT: store <4 x i64> [[TMP3]], <4 x i64>* bitcast ([8 x i64]* @dst64 to <4 x i64>*), align 8
+; AVX256DQ-NEXT: store <4 x i64> [[TMP4]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256DQ-NEXT: ret void
+;
+ %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+ %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+ %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+ %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+ %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+ %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+ %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+ %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+ %cvt0 = fptosi float %a0 to i64
+ %cvt1 = fptosi float %a1 to i64
+ %cvt2 = fptosi float %a2 to i64
+ %cvt3 = fptosi float %a3 to i64
+ %cvt4 = fptosi float %a4 to i64
+ %cvt5 = fptosi float %a5 to i64
+ %cvt6 = fptosi float %a6 to i64
+ %cvt7 = fptosi float %a7 to i64
+ store i64 %cvt0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 0), align 8
+ store i64 %cvt1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 1), align 8
+ store i64 %cvt2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 2), align 8
+ store i64 %cvt3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 3), align 8
+ store i64 %cvt4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 4), align 8
+ store i64 %cvt5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 5), align 8
+ store i64 %cvt6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 6), align 8
+ store i64 %cvt7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+define void @fptosi_8f32_8i32() #0 {
+; SSE-LABEL: @fptosi_8f32_8i32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([16 x float]* @src32 to <4 x float>*), align 4
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4) to <4 x float>*), align 4
+; SSE-NEXT: [[TMP3:%.*]] = fptosi <4 x float> [[TMP1]] to <4 x i32>
+; SSE-NEXT: [[TMP4:%.*]] = fptosi <4 x float> [[TMP2]] to <4 x i32>
+; SSE-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([16 x i32]* @dst32 to <4 x i32>*), align 4
+; SSE-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @fptosi_8f32_8i32(
+; AVX-NEXT: [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT: [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i32>
+; AVX-NEXT: store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([16 x i32]* @dst32 to <8 x i32>*), align 4
+; AVX-NEXT: ret void
+;
+ %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+ %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+ %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+ %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+ %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+ %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+ %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+ %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+ %cvt0 = fptosi float %a0 to i32
+ %cvt1 = fptosi float %a1 to i32
+ %cvt2 = fptosi float %a2 to i32
+ %cvt3 = fptosi float %a3 to i32
+ %cvt4 = fptosi float %a4 to i32
+ %cvt5 = fptosi float %a5 to i32
+ %cvt6 = fptosi float %a6 to i32
+ %cvt7 = fptosi float %a7 to i32
+ store i32 %cvt0, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 0), align 4
+ store i32 %cvt1, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 1), align 4
+ store i32 %cvt2, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 2), align 4
+ store i32 %cvt3, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 3), align 4
+ store i32 %cvt4, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 4), align 4
+ store i32 %cvt5, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 5), align 4
+ store i32 %cvt6, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 6), align 4
+ store i32 %cvt7, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @fptosi_8f32_8i16() #0 {
+; CHECK-LABEL: @fptosi_8f32_8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; CHECK-NEXT: [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i16>
+; CHECK-NEXT: store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([32 x i16]* @dst16 to <8 x i16>*), align 2
+; CHECK-NEXT: ret void
+;
+ %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+ %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+ %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+ %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+ %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+ %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+ %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+ %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+ %cvt0 = fptosi float %a0 to i16
+ %cvt1 = fptosi float %a1 to i16
+ %cvt2 = fptosi float %a2 to i16
+ %cvt3 = fptosi float %a3 to i16
+ %cvt4 = fptosi float %a4 to i16
+ %cvt5 = fptosi float %a5 to i16
+ %cvt6 = fptosi float %a6 to i16
+ %cvt7 = fptosi float %a7 to i16
+ store i16 %cvt0, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 0), align 2
+ store i16 %cvt1, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 1), align 2
+ store i16 %cvt2, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 2), align 2
+ store i16 %cvt3, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 3), align 2
+ store i16 %cvt4, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 4), align 2
+ store i16 %cvt5, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 5), align 2
+ store i16 %cvt6, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 6), align 2
+ store i16 %cvt7, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @dst16, i32 0, i64 7), align 2
+ ret void
+}
+
+define void @fptosi_8f32_8i8() #0 {
+; CHECK-LABEL: @fptosi_8f32_8i8(
+; CHECK-NEXT: [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; CHECK-NEXT: [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT: [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; CHECK-NEXT: [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; CHECK-NEXT: [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; CHECK-NEXT: [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; CHECK-NEXT: [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; CHECK-NEXT: [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; CHECK-NEXT: [[CVT0:%.*]] = fptosi float [[A0]] to i8
+; CHECK-NEXT: [[CVT1:%.*]] = fptosi float [[A1]] to i8
+; CHECK-NEXT: [[CVT2:%.*]] = fptosi float [[A2]] to i8
+; CHECK-NEXT: [[CVT3:%.*]] = fptosi float [[A3]] to i8
+; CHECK-NEXT: [[CVT4:%.*]] = fptosi float [[A4]] to i8
+; CHECK-NEXT: [[CVT5:%.*]] = fptosi float [[A5]] to i8
+; CHECK-NEXT: [[CVT6:%.*]] = fptosi float [[A6]] to i8
+; CHECK-NEXT: [[CVT7:%.*]] = fptosi float [[A7]] to i8
+; CHECK-NEXT: store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; CHECK-NEXT: store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; CHECK-NEXT: store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; CHECK-NEXT: store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; CHECK-NEXT: store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; CHECK-NEXT: store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; CHECK-NEXT: store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; CHECK-NEXT: store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT: ret void
+;
+ %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+ %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+ %a2 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+ %a3 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+ %a4 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+ %a5 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+ %a6 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+ %a7 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+ %cvt0 = fptosi float %a0 to i8
+ %cvt1 = fptosi float %a1 to i8
+ %cvt2 = fptosi float %a2 to i8
+ %cvt3 = fptosi float %a3 to i8
+ %cvt4 = fptosi float %a4 to i8
+ %cvt5 = fptosi float %a5 to i8
+ %cvt6 = fptosi float %a6 to i8
+ %cvt7 = fptosi float %a7 to i8
+ store i8 %cvt0, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+ store i8 %cvt1, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+ store i8 %cvt2, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+ store i8 %cvt3, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+ store i8 %cvt4, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+ store i8 %cvt5, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+ store i8 %cvt6, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+ store i8 %cvt7, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+ ret void
+}
+
+;
+; FPTOSI BUILDVECTOR
+;
+
+define <4 x i32> @fptosi_4xf64_4i32(double %a0, double %a1, double %a2, double %a3) #0 {
+; CHECK-LABEL: @fptosi_4xf64_4i32(
+; CHECK-NEXT: [[CVT0:%.*]] = fptosi double [[A0:%.*]] to i32
+; CHECK-NEXT: [[CVT1:%.*]] = fptosi double [[A1:%.*]] to i32
+; CHECK-NEXT: [[CVT2:%.*]] = fptosi double [[A2:%.*]] to i32
+; CHECK-NEXT: [[CVT3:%.*]] = fptosi double [[A3:%.*]] to i32
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <4 x i32> poison, i32 [[CVT0]], i32 0
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <4 x i32> [[RES0]], i32 [[CVT1]], i32 1
+; CHECK-NEXT: [[RES2:%.*]] = insertelement <4 x i32> [[RES1]], i32 [[CVT2]], i32 2
+; CHECK-NEXT: [[RES3:%.*]] = insertelement <4 x i32> [[RES2]], i32 [[CVT3]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[RES3]]
+;
+ %cvt0 = fptosi double %a0 to i32
+ %cvt1 = fptosi double %a1 to i32
+ %cvt2 = fptosi double %a2 to i32
+ %cvt3 = fptosi double %a3 to i32
+ %res0 = insertelement <4 x i32> poison, i32 %cvt0, i32 0
+ %res1 = insertelement <4 x i32> %res0, i32 %cvt1, i32 1
+ %res2 = insertelement <4 x i32> %res1, i32 %cvt2, i32 2
+ %res3 = insertelement <4 x i32> %res2, i32 %cvt3, i32 3
+ ret <4 x i32> %res3
+}
+
+define <4 x i32> @fptosi_4xf32_4i32(float %a0, float %a1, float %a2, float %a3) #0 {
+; CHECK-LABEL: @fptosi_4xf32_4i32(
+; CHECK-NEXT: [[CVT0:%.*]] = fptosi float [[A0:%.*]] to i32
+; CHECK-NEXT: [[CVT1:%.*]] = fptosi float [[A1:%.*]] to i32
+; CHECK-NEXT: [[CVT2:%.*]] = fptosi float [[A2:%.*]] to i32
+; CHECK-NEXT: [[CVT3:%.*]] = fptosi float [[A3:%.*]] to i32
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <4 x i32> poison, i32 [[CVT0]], i32 0
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <4 x i32> [[RES0]], i32 [[CVT1]], i32 1
+; CHECK-NEXT: [[RES2:%.*]] = insertelement <4 x i32> [[RES1]], i32 [[CVT2]], i32 2
+; CHECK-NEXT: [[RES3:%.*]] = insertelement <4 x i32> [[RES2]], i32 [[CVT3]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[RES3]]
+;
+ %cvt0 = fptosi float %a0 to i32
+ %cvt1 = fptosi float %a1 to i32
+ %cvt2 = fptosi float %a2 to i32
+ %cvt3 = fptosi float %a3 to i32
+ %res0 = insertelement <4 x i32> poison, i32 %cvt0, i32 0
+ %res1 = insertelement <4 x i32> %res0, i32 %cvt1, i32 1
+ %res2 = insertelement <4 x i32> %res1, i32 %cvt2, i32 2
+ %res3 = insertelement <4 x i32> %res2, i32 %cvt3, i32 3
+ ret <4 x i32> %res3
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
new file mode 100644
index 000000000000..97775135d025
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hadd-inseltpoison.ll
@@ -0,0 +1,433 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+;
+; 128-bit vectors
+;
+
+define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @test_v2f64(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT: ret <2 x double> [[TMP3]]
+;
+; SLM-LABEL: @test_v2f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT: [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT: [[R0:%.*]] = fadd double [[A0]], [[A1]]
+; SLM-NEXT: [[R1:%.*]] = fadd double [[B0]], [[B1]]
+; SLM-NEXT: [[R00:%.*]] = insertelement <2 x double> poison, double [[R0]], i32 0
+; SLM-NEXT: [[R01:%.*]] = insertelement <2 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT: ret <2 x double> [[R01]]
+;
+; AVX-LABEL: @test_v2f64(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <2 x double> [[TMP3]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %r0 = fadd double %a0, %a1
+ %r1 = fadd double %b0, %b1
+ %r00 = insertelement <2 x double> poison, double %r0, i32 0
+ %r01 = insertelement <2 x double> %r00, double %r1, i32 1
+ ret <2 x double> %r01
+}
+
+define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x float> [[TMP3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %r0 = fadd float %a0, %a1
+ %r1 = fadd float %a2, %a3
+ %r2 = fadd float %b0, %b1
+ %r3 = fadd float %b2, %b3
+ %r00 = insertelement <4 x float> poison, float %r0, i32 0
+ %r01 = insertelement <4 x float> %r00, float %r1, i32 1
+ %r02 = insertelement <4 x float> %r01, float %r2, i32 2
+ %r03 = insertelement <4 x float> %r02, float %r3, i32 3
+ ret <4 x float> %r03
+}
+
+define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @test_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %a0 = extractelement <2 x i64> %a, i32 0
+ %a1 = extractelement <2 x i64> %a, i32 1
+ %b0 = extractelement <2 x i64> %b, i32 0
+ %b1 = extractelement <2 x i64> %b, i32 1
+ %r0 = add i64 %a0, %a1
+ %r1 = add i64 %b0, %b1
+ %r00 = insertelement <2 x i64> poison, i64 %r0, i32 0
+ %r01 = insertelement <2 x i64> %r00, i64 %r1, i32 1
+ ret <2 x i64> %r01
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <4 x i32> %b, i32 0
+ %b1 = extractelement <4 x i32> %b, i32 1
+ %b2 = extractelement <4 x i32> %b, i32 2
+ %b3 = extractelement <4 x i32> %b, i32 3
+ %r0 = add i32 %a0, %a1
+ %r1 = add i32 %a2, %a3
+ %r2 = add i32 %b0, %b1
+ %r3 = add i32 %b2, %b3
+ %r00 = insertelement <4 x i32> poison, i32 %r0, i32 0
+ %r01 = insertelement <4 x i32> %r00, i32 %r1, i32 1
+ %r02 = insertelement <4 x i32> %r01, i32 %r2, i32 2
+ %r03 = insertelement <4 x i32> %r02, i32 %r3, i32 3
+ ret <4 x i32> %r03
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @test_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %a0 = extractelement <8 x i16> %a, i32 0
+ %a1 = extractelement <8 x i16> %a, i32 1
+ %a2 = extractelement <8 x i16> %a, i32 2
+ %a3 = extractelement <8 x i16> %a, i32 3
+ %a4 = extractelement <8 x i16> %a, i32 4
+ %a5 = extractelement <8 x i16> %a, i32 5
+ %a6 = extractelement <8 x i16> %a, i32 6
+ %a7 = extractelement <8 x i16> %a, i32 7
+ %b0 = extractelement <8 x i16> %b, i32 0
+ %b1 = extractelement <8 x i16> %b, i32 1
+ %b2 = extractelement <8 x i16> %b, i32 2
+ %b3 = extractelement <8 x i16> %b, i32 3
+ %b4 = extractelement <8 x i16> %b, i32 4
+ %b5 = extractelement <8 x i16> %b, i32 5
+ %b6 = extractelement <8 x i16> %b, i32 6
+ %b7 = extractelement <8 x i16> %b, i32 7
+ %r0 = add i16 %a0, %a1
+ %r1 = add i16 %a2, %a3
+ %r2 = add i16 %a4, %a5
+ %r3 = add i16 %a6, %a7
+ %r4 = add i16 %b0, %b1
+ %r5 = add i16 %b2, %b3
+ %r6 = add i16 %b4, %b5
+ %r7 = add i16 %b6, %b7
+ %r00 = insertelement <8 x i16> poison, i16 %r0, i32 0
+ %r01 = insertelement <8 x i16> %r00, i16 %r1, i32 1
+ %r02 = insertelement <8 x i16> %r01, i16 %r2, i32 2
+ %r03 = insertelement <8 x i16> %r02, i16 %r3, i32 3
+ %r04 = insertelement <8 x i16> %r03, i16 %r4, i32 4
+ %r05 = insertelement <8 x i16> %r04, i16 %r5, i32 5
+ %r06 = insertelement <8 x i16> %r05, i16 %r6, i32 6
+ %r07 = insertelement <8 x i16> %r06, i16 %r7, i32 7
+ ret <8 x i16> %r07
+}
+
+;
+; 256-bit vectors
+;
+
+define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @test_v4f64(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT: [[TMP3:%.*]] = fadd <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT: [[TMP6:%.*]] = fadd <2 x double> [[TMP4]], [[TMP5]]
+; SSE-NEXT: [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT: ret <4 x double> [[R03]]
+;
+; SLM-LABEL: @test_v4f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <4 x double> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <4 x double> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <4 x double> [[A]], i32 3
+; SLM-NEXT: [[B0:%.*]] = extractelement <4 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <4 x double> [[B]], i32 1
+; SLM-NEXT: [[B2:%.*]] = extractelement <4 x double> [[B]], i32 2
+; SLM-NEXT: [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
+; SLM-NEXT: [[R0:%.*]] = fadd double [[A0]], [[A1]]
+; SLM-NEXT: [[R1:%.*]] = fadd double [[B0]], [[B1]]
+; SLM-NEXT: [[R2:%.*]] = fadd double [[A2]], [[A3]]
+; SLM-NEXT: [[R3:%.*]] = fadd double [[B2]], [[B3]]
+; SLM-NEXT: [[R00:%.*]] = insertelement <4 x double> poison, double [[R0]], i32 0
+; SLM-NEXT: [[R01:%.*]] = insertelement <4 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT: [[R02:%.*]] = insertelement <4 x double> [[R01]], double [[R2]], i32 2
+; SLM-NEXT: [[R03:%.*]] = insertelement <4 x double> [[R02]], double [[R3]], i32 3
+; SLM-NEXT: ret <4 x double> [[R03]]
+;
+; AVX-LABEL: @test_v4f64(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT: [[TMP3:%.*]] = fadd <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <4 x double> [[TMP3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %r0 = fadd double %a0, %a1
+ %r1 = fadd double %b0, %b1
+ %r2 = fadd double %a2, %a3
+ %r3 = fadd double %b2, %b3
+ %r00 = insertelement <4 x double> poison, double %r0, i32 0
+ %r01 = insertelement <4 x double> %r00, double %r1, i32 1
+ %r02 = insertelement <4 x double> %r01, double %r2, i32 2
+ %r03 = insertelement <4 x double> %r02, double %r3, i32 3
+ ret <4 x double> %r03
+}
+
+define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @test_v8f32(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT: [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; SSE-NEXT: ret <8 x float> [[TMP3]]
+;
+; SLM-LABEL: @test_v8f32(
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT: [[TMP6:%.*]] = fadd <4 x float> [[TMP4]], [[TMP5]]
+; SLM-NEXT: [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT: ret <8 x float> [[R07]]
+;
+; AVX-LABEL: @test_v8f32(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT: [[TMP3:%.*]] = fadd <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <8 x float> [[TMP3]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %r0 = fadd float %a0, %a1
+ %r1 = fadd float %a2, %a3
+ %r2 = fadd float %b0, %b1
+ %r3 = fadd float %b2, %b3
+ %r4 = fadd float %a4, %a5
+ %r5 = fadd float %a6, %a7
+ %r6 = fadd float %b4, %b5
+ %r7 = fadd float %b6, %b7
+ %r00 = insertelement <8 x float> poison, float %r0, i32 0
+ %r01 = insertelement <8 x float> %r00, float %r1, i32 1
+ %r02 = insertelement <8 x float> %r01, float %r2, i32 2
+ %r03 = insertelement <8 x float> %r02, float %r3, i32 3
+ %r04 = insertelement <8 x float> %r03, float %r4, i32 4
+ %r05 = insertelement <8 x float> %r04, float %r5, i32 5
+ %r06 = insertelement <8 x float> %r05, float %r6, i32 6
+ %r07 = insertelement <8 x float> %r06, float %r7, i32 7
+ ret <8 x float> %r07
+}
+
+define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @test_v4i64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %a0 = extractelement <4 x i64> %a, i32 0
+ %a1 = extractelement <4 x i64> %a, i32 1
+ %a2 = extractelement <4 x i64> %a, i32 2
+ %a3 = extractelement <4 x i64> %a, i32 3
+ %b0 = extractelement <4 x i64> %b, i32 0
+ %b1 = extractelement <4 x i64> %b, i32 1
+ %b2 = extractelement <4 x i64> %b, i32 2
+ %b3 = extractelement <4 x i64> %b, i32 3
+ %r0 = add i64 %a0, %a1
+ %r1 = add i64 %b0, %b1
+ %r2 = add i64 %a2, %a3
+ %r3 = add i64 %b2, %b3
+ %r00 = insertelement <4 x i64> poison, i64 %r0, i32 0
+ %r01 = insertelement <4 x i64> %r00, i64 %r1, i32 1
+ %r02 = insertelement <4 x i64> %r01, i64 %r2, i32 2
+ %r03 = insertelement <4 x i64> %r02, i64 %r3, i32 3
+ ret <4 x i64> %r03
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: @test_v8i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %b0 = extractelement <8 x i32> %b, i32 0
+ %b1 = extractelement <8 x i32> %b, i32 1
+ %b2 = extractelement <8 x i32> %b, i32 2
+ %b3 = extractelement <8 x i32> %b, i32 3
+ %b4 = extractelement <8 x i32> %b, i32 4
+ %b5 = extractelement <8 x i32> %b, i32 5
+ %b6 = extractelement <8 x i32> %b, i32 6
+ %b7 = extractelement <8 x i32> %b, i32 7
+ %r0 = add i32 %a0, %a1
+ %r1 = add i32 %a2, %a3
+ %r2 = add i32 %b0, %b1
+ %r3 = add i32 %b2, %b3
+ %r4 = add i32 %a4, %a5
+ %r5 = add i32 %a6, %a7
+ %r6 = add i32 %b4, %b5
+ %r7 = add i32 %b6, %b7
+ %r00 = insertelement <8 x i32> poison, i32 %r0, i32 0
+ %r01 = insertelement <8 x i32> %r00, i32 %r1, i32 1
+ %r02 = insertelement <8 x i32> %r01, i32 %r2, i32 2
+ %r03 = insertelement <8 x i32> %r02, i32 %r3, i32 3
+ %r04 = insertelement <8 x i32> %r03, i32 %r4, i32 4
+ %r05 = insertelement <8 x i32> %r04, i32 %r5, i32 5
+ %r06 = insertelement <8 x i32> %r05, i32 %r6, i32 6
+ %r07 = insertelement <8 x i32> %r06, i32 %r7, i32 7
+ ret <8 x i32> %r07
+}
+
+define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: @test_v16i16(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
+; SSE-NEXT: [[TMP3:%.*]] = add <8 x i16> [[TMP1]], [[TMP2]]
+; SSE-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT: [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SSE-NEXT: [[TMP6:%.*]] = add <8 x i16> [[TMP4]], [[TMP5]]
+; SSE-NEXT: [[RV15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT: ret <16 x i16> [[RV15]]
+;
+; SLM-LABEL: @test_v16i16(
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SLM-NEXT: [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; SLM-NEXT: ret <16 x i16> [[TMP3]]
+;
+; AVX-LABEL: @test_v16i16(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX-NEXT: [[TMP3:%.*]] = add <16 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %a0 = extractelement <16 x i16> %a, i32 0
+ %a1 = extractelement <16 x i16> %a, i32 1
+ %a2 = extractelement <16 x i16> %a, i32 2
+ %a3 = extractelement <16 x i16> %a, i32 3
+ %a4 = extractelement <16 x i16> %a, i32 4
+ %a5 = extractelement <16 x i16> %a, i32 5
+ %a6 = extractelement <16 x i16> %a, i32 6
+ %a7 = extractelement <16 x i16> %a, i32 7
+ %a8 = extractelement <16 x i16> %a, i32 8
+ %a9 = extractelement <16 x i16> %a, i32 9
+ %a10 = extractelement <16 x i16> %a, i32 10
+ %a11 = extractelement <16 x i16> %a, i32 11
+ %a12 = extractelement <16 x i16> %a, i32 12
+ %a13 = extractelement <16 x i16> %a, i32 13
+ %a14 = extractelement <16 x i16> %a, i32 14
+ %a15 = extractelement <16 x i16> %a, i32 15
+ %b0 = extractelement <16 x i16> %b, i32 0
+ %b1 = extractelement <16 x i16> %b, i32 1
+ %b2 = extractelement <16 x i16> %b, i32 2
+ %b3 = extractelement <16 x i16> %b, i32 3
+ %b4 = extractelement <16 x i16> %b, i32 4
+ %b5 = extractelement <16 x i16> %b, i32 5
+ %b6 = extractelement <16 x i16> %b, i32 6
+ %b7 = extractelement <16 x i16> %b, i32 7
+ %b8 = extractelement <16 x i16> %b, i32 8
+ %b9 = extractelement <16 x i16> %b, i32 9
+ %b10 = extractelement <16 x i16> %b, i32 10
+ %b11 = extractelement <16 x i16> %b, i32 11
+ %b12 = extractelement <16 x i16> %b, i32 12
+ %b13 = extractelement <16 x i16> %b, i32 13
+ %b14 = extractelement <16 x i16> %b, i32 14
+ %b15 = extractelement <16 x i16> %b, i32 15
+ %r0 = add i16 %a0 , %a1
+ %r1 = add i16 %a2 , %a3
+ %r2 = add i16 %a4 , %a5
+ %r3 = add i16 %a6 , %a7
+ %r4 = add i16 %b0 , %b1
+ %r5 = add i16 %b2 , %b3
+ %r6 = add i16 %b4 , %b5
+ %r7 = add i16 %b6 , %b7
+ %r8 = add i16 %a8 , %a9
+ %r9 = add i16 %a10, %a11
+ %r10 = add i16 %a12, %a13
+ %r11 = add i16 %a14, %a15
+ %r12 = add i16 %b8 , %b9
+ %r13 = add i16 %b10, %b11
+ %r14 = add i16 %b12, %b13
+ %r15 = add i16 %b14, %b15
+ %rv0 = insertelement <16 x i16> poison, i16 %r0 , i32 0
+ %rv1 = insertelement <16 x i16> %rv0 , i16 %r1 , i32 1
+ %rv2 = insertelement <16 x i16> %rv1 , i16 %r2 , i32 2
+ %rv3 = insertelement <16 x i16> %rv2 , i16 %r3 , i32 3
+ %rv4 = insertelement <16 x i16> %rv3 , i16 %r4 , i32 4
+ %rv5 = insertelement <16 x i16> %rv4 , i16 %r5 , i32 5
+ %rv6 = insertelement <16 x i16> %rv5 , i16 %r6 , i32 6
+ %rv7 = insertelement <16 x i16> %rv6 , i16 %r7 , i32 7
+ %rv8 = insertelement <16 x i16> %rv7 , i16 %r8 , i32 8
+ %rv9 = insertelement <16 x i16> %rv8 , i16 %r9 , i32 9
+ %rv10 = insertelement <16 x i16> %rv9 , i16 %r10, i32 10
+ %rv11 = insertelement <16 x i16> %rv10, i16 %r11, i32 11
+ %rv12 = insertelement <16 x i16> %rv11, i16 %r12, i32 12
+ %rv13 = insertelement <16 x i16> %rv12, i16 %r13, i32 13
+ %rv14 = insertelement <16 x i16> %rv13, i16 %r14, i32 14
+ %rv15 = insertelement <16 x i16> %rv14, i16 %r15, i32 15
+ ret <16 x i16> %rv15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
new file mode 100644
index 000000000000..51711dc352fa
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/hsub-inseltpoison.ll
@@ -0,0 +1,433 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -basic-aa -slp-vectorizer -instcombine -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+;
+; 128-bit vectors
+;
+
+define <2 x double> @test_v2f64(<2 x double> %a, <2 x double> %b) {
+; SSE-LABEL: @test_v2f64(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; SSE-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT: ret <2 x double> [[TMP3]]
+;
+; SLM-LABEL: @test_v2f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <2 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <2 x double> [[A]], i32 1
+; SLM-NEXT: [[B0:%.*]] = extractelement <2 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <2 x double> [[B]], i32 1
+; SLM-NEXT: [[R0:%.*]] = fsub double [[A0]], [[A1]]
+; SLM-NEXT: [[R1:%.*]] = fsub double [[B0]], [[B1]]
+; SLM-NEXT: [[R00:%.*]] = insertelement <2 x double> poison, double [[R0]], i32 0
+; SLM-NEXT: [[R01:%.*]] = insertelement <2 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT: ret <2 x double> [[R01]]
+;
+; AVX-LABEL: @test_v2f64(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[A:%.*]], <2 x double> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <2 x double> [[A]], <2 x double> [[B]], <2 x i32> <i32 1, i32 3>
+; AVX-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <2 x double> [[TMP3]]
+;
+ %a0 = extractelement <2 x double> %a, i32 0
+ %a1 = extractelement <2 x double> %a, i32 1
+ %b0 = extractelement <2 x double> %b, i32 0
+ %b1 = extractelement <2 x double> %b, i32 1
+ %r0 = fsub double %a0, %a1
+ %r1 = fsub double %b0, %b1
+ %r00 = insertelement <2 x double> poison, double %r0, i32 0
+ %r01 = insertelement <2 x double> %r00, double %r1, i32 1
+ ret <2 x double> %r01
+}
+
+define <4 x float> @test_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @test_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x float> [[A]], <4 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x float> [[TMP3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %r0 = fsub float %a0, %a1
+ %r1 = fsub float %a2, %a3
+ %r2 = fsub float %b0, %b1
+ %r3 = fsub float %b2, %b3
+ %r00 = insertelement <4 x float> poison, float %r0, i32 0
+ %r01 = insertelement <4 x float> %r00, float %r1, i32 1
+ %r02 = insertelement <4 x float> %r01, float %r2, i32 2
+ %r03 = insertelement <4 x float> %r02, float %r3, i32 3
+ ret <4 x float> %r03
+}
+
+define <2 x i64> @test_v2i64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: @test_v2i64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x i64> [[A:%.*]], <2 x i64> [[B:%.*]], <2 x i32> <i32 0, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i64> [[A]], <2 x i64> [[B]], <2 x i32> <i32 1, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <2 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <2 x i64> [[TMP3]]
+;
+ %a0 = extractelement <2 x i64> %a, i32 0
+ %a1 = extractelement <2 x i64> %a, i32 1
+ %b0 = extractelement <2 x i64> %b, i32 0
+ %b1 = extractelement <2 x i64> %b, i32 1
+ %r0 = sub i64 %a0, %a1
+ %r1 = sub i64 %b0, %b1
+ %r00 = insertelement <2 x i64> poison, i64 %r0, i32 0
+ %r01 = insertelement <2 x i64> %r00, i64 %r1, i32 1
+ ret <2 x i64> %r01
+}
+
+define <4 x i32> @test_v4i32(<4 x i32> %a, <4 x i32> %b) {
+; CHECK-LABEL: @test_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[A]], <4 x i32> [[B]], <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <4 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i32> [[TMP3]]
+;
+ %a0 = extractelement <4 x i32> %a, i32 0
+ %a1 = extractelement <4 x i32> %a, i32 1
+ %a2 = extractelement <4 x i32> %a, i32 2
+ %a3 = extractelement <4 x i32> %a, i32 3
+ %b0 = extractelement <4 x i32> %b, i32 0
+ %b1 = extractelement <4 x i32> %b, i32 1
+ %b2 = extractelement <4 x i32> %b, i32 2
+ %b3 = extractelement <4 x i32> %b, i32 3
+ %r0 = sub i32 %a0, %a1
+ %r1 = sub i32 %a2, %a3
+ %r2 = sub i32 %b0, %b1
+ %r3 = sub i32 %b2, %b3
+ %r00 = insertelement <4 x i32> poison, i32 %r0, i32 0
+ %r01 = insertelement <4 x i32> %r00, i32 %r1, i32 1
+ %r02 = insertelement <4 x i32> %r01, i32 %r2, i32 2
+ %r03 = insertelement <4 x i32> %r02, i32 %r3, i32 3
+ ret <4 x i32> %r03
+}
+
+define <8 x i16> @test_v8i16(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: @test_v8i16(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i16> [[A]], <8 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i16> [[TMP3]]
+;
+ %a0 = extractelement <8 x i16> %a, i32 0
+ %a1 = extractelement <8 x i16> %a, i32 1
+ %a2 = extractelement <8 x i16> %a, i32 2
+ %a3 = extractelement <8 x i16> %a, i32 3
+ %a4 = extractelement <8 x i16> %a, i32 4
+ %a5 = extractelement <8 x i16> %a, i32 5
+ %a6 = extractelement <8 x i16> %a, i32 6
+ %a7 = extractelement <8 x i16> %a, i32 7
+ %b0 = extractelement <8 x i16> %b, i32 0
+ %b1 = extractelement <8 x i16> %b, i32 1
+ %b2 = extractelement <8 x i16> %b, i32 2
+ %b3 = extractelement <8 x i16> %b, i32 3
+ %b4 = extractelement <8 x i16> %b, i32 4
+ %b5 = extractelement <8 x i16> %b, i32 5
+ %b6 = extractelement <8 x i16> %b, i32 6
+ %b7 = extractelement <8 x i16> %b, i32 7
+ %r0 = sub i16 %a0, %a1
+ %r1 = sub i16 %a2, %a3
+ %r2 = sub i16 %a4, %a5
+ %r3 = sub i16 %a6, %a7
+ %r4 = sub i16 %b0, %b1
+ %r5 = sub i16 %b2, %b3
+ %r6 = sub i16 %b4, %b5
+ %r7 = sub i16 %b6, %b7
+ %r00 = insertelement <8 x i16> poison, i16 %r0, i32 0
+ %r01 = insertelement <8 x i16> %r00, i16 %r1, i32 1
+ %r02 = insertelement <8 x i16> %r01, i16 %r2, i32 2
+ %r03 = insertelement <8 x i16> %r02, i16 %r3, i32 3
+ %r04 = insertelement <8 x i16> %r03, i16 %r4, i32 4
+ %r05 = insertelement <8 x i16> %r04, i16 %r5, i32 5
+ %r06 = insertelement <8 x i16> %r05, i16 %r6, i32 6
+ %r07 = insertelement <8 x i16> %r06, i16 %r7, i32 7
+ ret <8 x i16> %r07
+}
+
+;
+; 256-bit vectors
+;
+
+define <4 x double> @test_v4f64(<4 x double> %a, <4 x double> %b) {
+; SSE-LABEL: @test_v4f64(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <2 x i32> <i32 0, i32 4>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 1, i32 5>
+; SSE-NEXT: [[TMP3:%.*]] = fsub <2 x double> [[TMP1]], [[TMP2]]
+; SSE-NEXT: [[TMP4:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 2, i32 6>
+; SSE-NEXT: [[TMP5:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <2 x i32> <i32 3, i32 7>
+; SSE-NEXT: [[TMP6:%.*]] = fsub <2 x double> [[TMP4]], [[TMP5]]
+; SSE-NEXT: [[R03:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE-NEXT: ret <4 x double> [[R03]]
+;
+; SLM-LABEL: @test_v4f64(
+; SLM-NEXT: [[A0:%.*]] = extractelement <4 x double> [[A:%.*]], i32 0
+; SLM-NEXT: [[A1:%.*]] = extractelement <4 x double> [[A]], i32 1
+; SLM-NEXT: [[A2:%.*]] = extractelement <4 x double> [[A]], i32 2
+; SLM-NEXT: [[A3:%.*]] = extractelement <4 x double> [[A]], i32 3
+; SLM-NEXT: [[B0:%.*]] = extractelement <4 x double> [[B:%.*]], i32 0
+; SLM-NEXT: [[B1:%.*]] = extractelement <4 x double> [[B]], i32 1
+; SLM-NEXT: [[B2:%.*]] = extractelement <4 x double> [[B]], i32 2
+; SLM-NEXT: [[B3:%.*]] = extractelement <4 x double> [[B]], i32 3
+; SLM-NEXT: [[R0:%.*]] = fsub double [[A0]], [[A1]]
+; SLM-NEXT: [[R1:%.*]] = fsub double [[B0]], [[B1]]
+; SLM-NEXT: [[R2:%.*]] = fsub double [[A2]], [[A3]]
+; SLM-NEXT: [[R3:%.*]] = fsub double [[B2]], [[B3]]
+; SLM-NEXT: [[R00:%.*]] = insertelement <4 x double> poison, double [[R0]], i32 0
+; SLM-NEXT: [[R01:%.*]] = insertelement <4 x double> [[R00]], double [[R1]], i32 1
+; SLM-NEXT: [[R02:%.*]] = insertelement <4 x double> [[R01]], double [[R2]], i32 2
+; SLM-NEXT: [[R03:%.*]] = insertelement <4 x double> [[R02]], double [[R3]], i32 3
+; SLM-NEXT: ret <4 x double> [[R03]]
+;
+; AVX-LABEL: @test_v4f64(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <4 x double> [[A:%.*]], <4 x double> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <4 x double> [[A]], <4 x double> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; AVX-NEXT: [[TMP3:%.*]] = fsub <4 x double> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <4 x double> [[TMP3]]
+;
+ %a0 = extractelement <4 x double> %a, i32 0
+ %a1 = extractelement <4 x double> %a, i32 1
+ %a2 = extractelement <4 x double> %a, i32 2
+ %a3 = extractelement <4 x double> %a, i32 3
+ %b0 = extractelement <4 x double> %b, i32 0
+ %b1 = extractelement <4 x double> %b, i32 1
+ %b2 = extractelement <4 x double> %b, i32 2
+ %b3 = extractelement <4 x double> %b, i32 3
+ %r0 = fsub double %a0, %a1
+ %r1 = fsub double %b0, %b1
+ %r2 = fsub double %a2, %a3
+ %r3 = fsub double %b2, %b3
+ %r00 = insertelement <4 x double> poison, double %r0, i32 0
+ %r01 = insertelement <4 x double> %r00, double %r1, i32 1
+ %r02 = insertelement <4 x double> %r01, double %r2, i32 2
+ %r03 = insertelement <4 x double> %r02, double %r3, i32 3
+ ret <4 x double> %r03
+}
+
+define <8 x float> @test_v8f32(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: @test_v8f32(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; SSE-NEXT: [[TMP3:%.*]] = fsub <8 x float> [[TMP1]], [[TMP2]]
+; SSE-NEXT: ret <8 x float> [[TMP3]]
+;
+; SLM-LABEL: @test_v8f32(
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <4 x i32> <i32 0, i32 2, i32 8, i32 10>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 1, i32 3, i32 9, i32 11>
+; SLM-NEXT: [[TMP3:%.*]] = fsub <4 x float> [[TMP1]], [[TMP2]]
+; SLM-NEXT: [[TMP4:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 4, i32 6, i32 12, i32 14>
+; SLM-NEXT: [[TMP5:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <4 x i32> <i32 5, i32 7, i32 13, i32 15>
+; SLM-NEXT: [[TMP6:%.*]] = fsub <4 x float> [[TMP4]], [[TMP5]]
+; SLM-NEXT: [[R07:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SLM-NEXT: ret <8 x float> [[R07]]
+;
+; AVX-LABEL: @test_v8f32(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <8 x float> [[A:%.*]], <8 x float> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <8 x float> [[A]], <8 x float> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; AVX-NEXT: [[TMP3:%.*]] = fsub <8 x float> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <8 x float> [[TMP3]]
+;
+ %a0 = extractelement <8 x float> %a, i32 0
+ %a1 = extractelement <8 x float> %a, i32 1
+ %a2 = extractelement <8 x float> %a, i32 2
+ %a3 = extractelement <8 x float> %a, i32 3
+ %a4 = extractelement <8 x float> %a, i32 4
+ %a5 = extractelement <8 x float> %a, i32 5
+ %a6 = extractelement <8 x float> %a, i32 6
+ %a7 = extractelement <8 x float> %a, i32 7
+ %b0 = extractelement <8 x float> %b, i32 0
+ %b1 = extractelement <8 x float> %b, i32 1
+ %b2 = extractelement <8 x float> %b, i32 2
+ %b3 = extractelement <8 x float> %b, i32 3
+ %b4 = extractelement <8 x float> %b, i32 4
+ %b5 = extractelement <8 x float> %b, i32 5
+ %b6 = extractelement <8 x float> %b, i32 6
+ %b7 = extractelement <8 x float> %b, i32 7
+ %r0 = fsub float %a0, %a1
+ %r1 = fsub float %a2, %a3
+ %r2 = fsub float %b0, %b1
+ %r3 = fsub float %b2, %b3
+ %r4 = fsub float %a4, %a5
+ %r5 = fsub float %a6, %a7
+ %r6 = fsub float %b4, %b5
+ %r7 = fsub float %b6, %b7
+ %r00 = insertelement <8 x float> poison, float %r0, i32 0
+ %r01 = insertelement <8 x float> %r00, float %r1, i32 1
+ %r02 = insertelement <8 x float> %r01, float %r2, i32 2
+ %r03 = insertelement <8 x float> %r02, float %r3, i32 3
+ %r04 = insertelement <8 x float> %r03, float %r4, i32 4
+ %r05 = insertelement <8 x float> %r04, float %r5, i32 5
+ %r06 = insertelement <8 x float> %r05, float %r6, i32 6
+ %r07 = insertelement <8 x float> %r06, float %r7, i32 7
+ ret <8 x float> %r07
+}
+
+define <4 x i64> @test_v4i64(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: @test_v4i64(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x i64> [[A:%.*]], <4 x i64> [[B:%.*]], <4 x i32> <i32 0, i32 4, i32 2, i32 6>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[A]], <4 x i64> [[B]], <4 x i32> <i32 1, i32 5, i32 3, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <4 x i64> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <4 x i64> [[TMP3]]
+;
+ %a0 = extractelement <4 x i64> %a, i32 0
+ %a1 = extractelement <4 x i64> %a, i32 1
+ %a2 = extractelement <4 x i64> %a, i32 2
+ %a3 = extractelement <4 x i64> %a, i32 3
+ %b0 = extractelement <4 x i64> %b, i32 0
+ %b1 = extractelement <4 x i64> %b, i32 1
+ %b2 = extractelement <4 x i64> %b, i32 2
+ %b3 = extractelement <4 x i64> %b, i32 3
+ %r0 = sub i64 %a0, %a1
+ %r1 = sub i64 %b0, %b1
+ %r2 = sub i64 %a2, %a3
+ %r3 = sub i64 %b2, %b3
+ %r00 = insertelement <4 x i64> poison, i64 %r0, i32 0
+ %r01 = insertelement <4 x i64> %r00, i64 %r1, i32 1
+ %r02 = insertelement <4 x i64> %r01, i64 %r2, i32 2
+ %r03 = insertelement <4 x i64> %r02, i64 %r3, i32 3
+ ret <4 x i64> %r03
+}
+
+define <8 x i32> @test_v8i32(<8 x i32> %a, <8 x i32> %b) {
+; CHECK-LABEL: @test_v8i32(
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A:%.*]], <8 x i32> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 8, i32 10, i32 4, i32 6, i32 12, i32 14>
+; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> [[B]], <8 x i32> <i32 1, i32 3, i32 9, i32 11, i32 5, i32 7, i32 13, i32 15>
+; CHECK-NEXT: [[TMP3:%.*]] = sub <8 x i32> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: ret <8 x i32> [[TMP3]]
+;
+ %a0 = extractelement <8 x i32> %a, i32 0
+ %a1 = extractelement <8 x i32> %a, i32 1
+ %a2 = extractelement <8 x i32> %a, i32 2
+ %a3 = extractelement <8 x i32> %a, i32 3
+ %a4 = extractelement <8 x i32> %a, i32 4
+ %a5 = extractelement <8 x i32> %a, i32 5
+ %a6 = extractelement <8 x i32> %a, i32 6
+ %a7 = extractelement <8 x i32> %a, i32 7
+ %b0 = extractelement <8 x i32> %b, i32 0
+ %b1 = extractelement <8 x i32> %b, i32 1
+ %b2 = extractelement <8 x i32> %b, i32 2
+ %b3 = extractelement <8 x i32> %b, i32 3
+ %b4 = extractelement <8 x i32> %b, i32 4
+ %b5 = extractelement <8 x i32> %b, i32 5
+ %b6 = extractelement <8 x i32> %b, i32 6
+ %b7 = extractelement <8 x i32> %b, i32 7
+ %r0 = sub i32 %a0, %a1
+ %r1 = sub i32 %a2, %a3
+ %r2 = sub i32 %b0, %b1
+ %r3 = sub i32 %b2, %b3
+ %r4 = sub i32 %a4, %a5
+ %r5 = sub i32 %a6, %a7
+ %r6 = sub i32 %b4, %b5
+ %r7 = sub i32 %b6, %b7
+ %r00 = insertelement <8 x i32> poison, i32 %r0, i32 0
+ %r01 = insertelement <8 x i32> %r00, i32 %r1, i32 1
+ %r02 = insertelement <8 x i32> %r01, i32 %r2, i32 2
+ %r03 = insertelement <8 x i32> %r02, i32 %r3, i32 3
+ %r04 = insertelement <8 x i32> %r03, i32 %r4, i32 4
+ %r05 = insertelement <8 x i32> %r04, i32 %r5, i32 5
+ %r06 = insertelement <8 x i32> %r05, i32 %r6, i32 6
+ %r07 = insertelement <8 x i32> %r06, i32 %r7, i32 7
+ ret <8 x i32> %r07
+}
+
+define <16 x i16> @test_v16i16(<16 x i16> %a, <16 x i16> %b) {
+; SSE-LABEL: @test_v16i16(
+; SSE-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22>
+; SSE-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23>
+; SSE-NEXT: [[TMP3:%.*]] = sub <8 x i16> [[TMP1]], [[TMP2]]
+; SSE-NEXT: [[TMP4:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SSE-NEXT: [[TMP5:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <8 x i32> <i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SSE-NEXT: [[TMP6:%.*]] = sub <8 x i16> [[TMP4]], [[TMP5]]
+; SSE-NEXT: [[RV15:%.*]] = shufflevector <8 x i16> [[TMP3]], <8 x i16> [[TMP6]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE-NEXT: ret <16 x i16> [[RV15]]
+;
+; SLM-LABEL: @test_v16i16(
+; SLM-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; SLM-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; SLM-NEXT: [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; SLM-NEXT: ret <16 x i16> [[TMP3]]
+;
+; AVX-LABEL: @test_v16i16(
+; AVX-NEXT: [[TMP1:%.*]] = shufflevector <16 x i16> [[A:%.*]], <16 x i16> [[B:%.*]], <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 16, i32 18, i32 20, i32 22, i32 8, i32 10, i32 12, i32 14, i32 24, i32 26, i32 28, i32 30>
+; AVX-NEXT: [[TMP2:%.*]] = shufflevector <16 x i16> [[A]], <16 x i16> [[B]], <16 x i32> <i32 1, i32 3, i32 5, i32 7, i32 17, i32 19, i32 21, i32 23, i32 9, i32 11, i32 13, i32 15, i32 25, i32 27, i32 29, i32 31>
+; AVX-NEXT: [[TMP3:%.*]] = sub <16 x i16> [[TMP1]], [[TMP2]]
+; AVX-NEXT: ret <16 x i16> [[TMP3]]
+;
+ %a0 = extractelement <16 x i16> %a, i32 0
+ %a1 = extractelement <16 x i16> %a, i32 1
+ %a2 = extractelement <16 x i16> %a, i32 2
+ %a3 = extractelement <16 x i16> %a, i32 3
+ %a4 = extractelement <16 x i16> %a, i32 4
+ %a5 = extractelement <16 x i16> %a, i32 5
+ %a6 = extractelement <16 x i16> %a, i32 6
+ %a7 = extractelement <16 x i16> %a, i32 7
+ %a8 = extractelement <16 x i16> %a, i32 8
+ %a9 = extractelement <16 x i16> %a, i32 9
+ %a10 = extractelement <16 x i16> %a, i32 10
+ %a11 = extractelement <16 x i16> %a, i32 11
+ %a12 = extractelement <16 x i16> %a, i32 12
+ %a13 = extractelement <16 x i16> %a, i32 13
+ %a14 = extractelement <16 x i16> %a, i32 14
+ %a15 = extractelement <16 x i16> %a, i32 15
+ %b0 = extractelement <16 x i16> %b, i32 0
+ %b1 = extractelement <16 x i16> %b, i32 1
+ %b2 = extractelement <16 x i16> %b, i32 2
+ %b3 = extractelement <16 x i16> %b, i32 3
+ %b4 = extractelement <16 x i16> %b, i32 4
+ %b5 = extractelement <16 x i16> %b, i32 5
+ %b6 = extractelement <16 x i16> %b, i32 6
+ %b7 = extractelement <16 x i16> %b, i32 7
+ %b8 = extractelement <16 x i16> %b, i32 8
+ %b9 = extractelement <16 x i16> %b, i32 9
+ %b10 = extractelement <16 x i16> %b, i32 10
+ %b11 = extractelement <16 x i16> %b, i32 11
+ %b12 = extractelement <16 x i16> %b, i32 12
+ %b13 = extractelement <16 x i16> %b, i32 13
+ %b14 = extractelement <16 x i16> %b, i32 14
+ %b15 = extractelement <16 x i16> %b, i32 15
+ %r0 = sub i16 %a0 , %a1
+ %r1 = sub i16 %a2 , %a3
+ %r2 = sub i16 %a4 , %a5
+ %r3 = sub i16 %a6 , %a7
+ %r4 = sub i16 %b0 , %b1
+ %r5 = sub i16 %b2 , %b3
+ %r6 = sub i16 %b4 , %b5
+ %r7 = sub i16 %b6 , %b7
+ %r8 = sub i16 %a8 , %a9
+ %r9 = sub i16 %a10, %a11
+ %r10 = sub i16 %a12, %a13
+ %r11 = sub i16 %a14, %a15
+ %r12 = sub i16 %b8 , %b9
+ %r13 = sub i16 %b10, %b11
+ %r14 = sub i16 %b12, %b13
+ %r15 = sub i16 %b14, %b15
+ %rv0 = insertelement <16 x i16> poison, i16 %r0 , i32 0
+ %rv1 = insertelement <16 x i16> %rv0 , i16 %r1 , i32 1
+ %rv2 = insertelement <16 x i16> %rv1 , i16 %r2 , i32 2
+ %rv3 = insertelement <16 x i16> %rv2 , i16 %r3 , i32 3
+ %rv4 = insertelement <16 x i16> %rv3 , i16 %r4 , i32 4
+ %rv5 = insertelement <16 x i16> %rv4 , i16 %r5 , i32 5
+ %rv6 = insertelement <16 x i16> %rv5 , i16 %r6 , i32 6
+ %rv7 = insertelement <16 x i16> %rv6 , i16 %r7 , i32 7
+ %rv8 = insertelement <16 x i16> %rv7 , i16 %r8 , i32 8
+ %rv9 = insertelement <16 x i16> %rv8 , i16 %r9 , i32 9
+ %rv10 = insertelement <16 x i16> %rv9 , i16 %r10, i32 10
+ %rv11 = insertelement <16 x i16> %rv10, i16 %r11, i32 11
+ %rv12 = insertelement <16 x i16> %rv11, i16 %r12, i32 12
+ %rv13 = insertelement <16 x i16> %rv12, i16 %r13, i32 13
+ %rv14 = insertelement <16 x i16> %rv13, i16 %r14, i32 14
+ %rv15 = insertelement <16 x i16> %rv14, i16 %r15, i32 15
+ ret <16 x i16> %rv15
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
new file mode 100644
index 000000000000..003db8629786
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll
@@ -0,0 +1,540 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -slp-vectorizer -slp-threshold=-10000 < %s | FileCheck %s
+; RUN: opt -S -slp-vectorizer -slp-threshold=0 < %s | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+define <4 x float> @simple_select(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT: ret <4 x float> [[RD]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %cmp2 = icmp ne i32 %c2, 0
+ %cmp3 = icmp ne i32 %c3, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %s2 = select i1 %cmp2, float %a2, float %b2
+ %s3 = select i1 %cmp3, float %a3, float %b3
+ %ra = insertelement <4 x float> poison, float %s0, i32 0
+ %rb = insertelement <4 x float> %ra, float %s1, i32 1
+ %rc = insertelement <4 x float> %rb, float %s2, i32 2
+ %rd = insertelement <4 x float> %rc, float %s3, i32 3
+ ret <4 x float> %rd
+}
+
+declare void @llvm.assume(i1) nounwind
+
+; This entire tree is ephemeral, don't vectorize any of it.
+define <4 x float> @simple_select_eph(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_eph(
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; CHECK-NEXT: [[CMP0:%.*]] = icmp ne i32 [[C0]], 0
+; CHECK-NEXT: [[CMP1:%.*]] = icmp ne i32 [[C1]], 0
+; CHECK-NEXT: [[CMP2:%.*]] = icmp ne i32 [[C2]], 0
+; CHECK-NEXT: [[CMP3:%.*]] = icmp ne i32 [[C3]], 0
+; CHECK-NEXT: [[S0:%.*]] = select i1 [[CMP0]], float [[A0]], float [[B0]]
+; CHECK-NEXT: [[S1:%.*]] = select i1 [[CMP1]], float [[A1]], float [[B1]]
+; CHECK-NEXT: [[S2:%.*]] = select i1 [[CMP2]], float [[A2]], float [[B2]]
+; CHECK-NEXT: [[S3:%.*]] = select i1 [[CMP3]], float [[A3]], float [[B3]]
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[S0]], i32 0
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[S1]], i32 1
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[S2]], i32 2
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[S3]], i32 3
+; CHECK-NEXT: [[Q0:%.*]] = extractelement <4 x float> [[RD]], i32 0
+; CHECK-NEXT: [[Q1:%.*]] = extractelement <4 x float> [[RD]], i32 1
+; CHECK-NEXT: [[Q2:%.*]] = extractelement <4 x float> [[RD]], i32 2
+; CHECK-NEXT: [[Q3:%.*]] = extractelement <4 x float> [[RD]], i32 3
+; CHECK-NEXT: [[Q4:%.*]] = fadd float [[Q0]], [[Q1]]
+; CHECK-NEXT: [[Q5:%.*]] = fadd float [[Q2]], [[Q3]]
+; CHECK-NEXT: [[Q6:%.*]] = fadd float [[Q4]], [[Q5]]
+; CHECK-NEXT: [[QI:%.*]] = fcmp olt float [[Q6]], [[Q5]]
+; CHECK-NEXT: call void @llvm.assume(i1 [[QI]])
+; CHECK-NEXT: ret <4 x float> undef
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %cmp2 = icmp ne i32 %c2, 0
+ %cmp3 = icmp ne i32 %c3, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %s2 = select i1 %cmp2, float %a2, float %b2
+ %s3 = select i1 %cmp3, float %a3, float %b3
+ %ra = insertelement <4 x float> poison, float %s0, i32 0
+ %rb = insertelement <4 x float> %ra, float %s1, i32 1
+ %rc = insertelement <4 x float> %rb, float %s2, i32 2
+ %rd = insertelement <4 x float> %rc, float %s3, i32 3
+ %q0 = extractelement <4 x float> %rd, i32 0
+ %q1 = extractelement <4 x float> %rd, i32 1
+ %q2 = extractelement <4 x float> %rd, i32 2
+ %q3 = extractelement <4 x float> %rd, i32 3
+ %q4 = fadd float %q0, %q1
+ %q5 = fadd float %q2, %q3
+ %q6 = fadd float %q4, %q5
+ %qi = fcmp olt float %q6, %q5
+ call void @llvm.assume(i1 %qi)
+ ret <4 x float> undef
+}
+
+; Insert in an order
diff erent from the vector indices to make sure it
+; doesn't matter
+define <4 x float> @simple_select_insert_out_of_order(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_insert_out_of_order(
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[C:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 1, i32 0, i32 3>
+; CHECK-NEXT: [[SHUFFLE1:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 2, i32 1, i32 0, i32 3>
+; CHECK-NEXT: [[SHUFFLE2:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <4 x i32> <i32 2, i32 1, i32 0, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[SHUFFLE]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[SHUFFLE1]], <4 x float> [[SHUFFLE2]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT: ret <4 x float> [[RD]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %cmp2 = icmp ne i32 %c2, 0
+ %cmp3 = icmp ne i32 %c3, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %s2 = select i1 %cmp2, float %a2, float %b2
+ %s3 = select i1 %cmp3, float %a3, float %b3
+ %ra = insertelement <4 x float> poison, float %s0, i32 2
+ %rb = insertelement <4 x float> %ra, float %s1, i32 1
+ %rc = insertelement <4 x float> %rb, float %s2, i32 0
+ %rd = insertelement <4 x float> %rc, float %s3, i32 3
+ ret <4 x float> %rd
+}
+
+declare void @v4f32_user(<4 x float>) #0
+declare void @f32_user(float) #0
+
+; Multiple users of the final constructed vector
+define <4 x float> @simple_select_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_users(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[A:%.*]], <4 x float> [[B:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x float> [[RB]], float [[TMP5]], i32 2
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP6]], i32 3
+; CHECK-NEXT: call void @v4f32_user(<4 x float> [[RD]]) [[ATTR0:#.*]]
+; CHECK-NEXT: ret <4 x float> [[RD]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %cmp2 = icmp ne i32 %c2, 0
+ %cmp3 = icmp ne i32 %c3, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %s2 = select i1 %cmp2, float %a2, float %b2
+ %s3 = select i1 %cmp3, float %a3, float %b3
+ %ra = insertelement <4 x float> poison, float %s0, i32 0
+ %rb = insertelement <4 x float> %ra, float %s1, i32 1
+ %rc = insertelement <4 x float> %rb, float %s2, i32 2
+ %rd = insertelement <4 x float> %rc, float %s3, i32 3
+ call void @v4f32_user(<4 x float> %rd) #0
+ ret <4 x float> %rd
+}
+
+; Unused insertelement
+define <4 x float> @simple_select_no_users(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_no_users(
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT: [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; CHECK-NEXT: [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
+; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x float> [[B]], i32 2
+; CHECK-NEXT: [[B3:%.*]] = extractelement <4 x float> [[B]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> undef, i32 [[C0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> undef, i32 [[C2]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[C3]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <2 x i32> [[TMP5]], zeroinitializer
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> undef, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> undef, float [[B0]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[B1]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP8]], <2 x float> [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x float> undef, float [[A2]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x float> [[TMP12]], float [[A3]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <2 x float> undef, float [[B2]], i32 0
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[B3]], i32 1
+; CHECK-NEXT: [[TMP16:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP13]], <2 x float> [[TMP15]]
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP17]], i32 0
+; CHECK-NEXT: [[TMP18:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP18]], i32 1
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x float> [[TMP16]], i32 0
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x float> poison, float [[TMP19]], i32 2
+; CHECK-NEXT: [[TMP20:%.*]] = extractelement <2 x float> [[TMP16]], i32 1
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x float> [[RC]], float [[TMP20]], i32 3
+; CHECK-NEXT: ret <4 x float> [[RD]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %cmp2 = icmp ne i32 %c2, 0
+ %cmp3 = icmp ne i32 %c3, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %s2 = select i1 %cmp2, float %a2, float %b2
+ %s3 = select i1 %cmp3, float %a3, float %b3
+ %ra = insertelement <4 x float> poison, float %s0, i32 0
+ %rb = insertelement <4 x float> %ra, float %s1, i32 1
+ %rc = insertelement <4 x float> poison, float %s2, i32 2
+ %rd = insertelement <4 x float> %rc, float %s3, i32 3
+ ret <4 x float> %rd
+}
+
+; Make sure infinite loop doesn't happen which I ran into when trying
+; to do this backwards this backwards
+define <4 x i32> @reconstruct(<4 x i32> %c) #0 {
+; CHECK-LABEL: @reconstruct(
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT: [[C2:%.*]] = extractelement <4 x i32> [[C]], i32 2
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x i32> [[C]], i32 3
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x i32> poison, i32 [[C0]], i32 0
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x i32> [[RA]], i32 [[C1]], i32 1
+; CHECK-NEXT: [[RC:%.*]] = insertelement <4 x i32> [[RB]], i32 [[C2]], i32 2
+; CHECK-NEXT: [[RD:%.*]] = insertelement <4 x i32> [[RC]], i32 [[C3]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[RD]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %c2 = extractelement <4 x i32> %c, i32 2
+ %c3 = extractelement <4 x i32> %c, i32 3
+ %ra = insertelement <4 x i32> poison, i32 %c0, i32 0
+ %rb = insertelement <4 x i32> %ra, i32 %c1, i32 1
+ %rc = insertelement <4 x i32> %rb, i32 %c2, i32 2
+ %rd = insertelement <4 x i32> %rc, i32 %c3, i32 3
+ ret <4 x i32> %rd
+}
+
+define <2 x float> @simple_select_v2(<2 x float> %a, <2 x float> %b, <2 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_v2(
+; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <2 x i32> [[C:%.*]], zeroinitializer
+; CHECK-NEXT: [[TMP2:%.*]] = select <2 x i1> [[TMP1]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[RA:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <2 x float> [[RA]], float [[TMP4]], i32 1
+; CHECK-NEXT: ret <2 x float> [[RB]]
+;
+ %c0 = extractelement <2 x i32> %c, i32 0
+ %c1 = extractelement <2 x i32> %c, i32 1
+ %a0 = extractelement <2 x float> %a, i32 0
+ %a1 = extractelement <2 x float> %a, i32 1
+ %b0 = extractelement <2 x float> %b, i32 0
+ %b1 = extractelement <2 x float> %b, i32 1
+ %cmp0 = icmp ne i32 %c0, 0
+ %cmp1 = icmp ne i32 %c1, 0
+ %s0 = select i1 %cmp0, float %a0, float %b0
+ %s1 = select i1 %cmp1, float %a1, float %b1
+ %ra = insertelement <2 x float> poison, float %s0, i32 0
+ %rb = insertelement <2 x float> %ra, float %s1, i32 1
+ ret <2 x float> %rb
+}
+
+; Make sure when we construct partial vectors, we don't keep
+; re-visiting the insertelement chains starting with undef
+; (low cost threshold needed to force this to happen)
+define <4 x float> @simple_select_partial_vector(<4 x float> %a, <4 x float> %b, <4 x i32> %c) #0 {
+; CHECK-LABEL: @simple_select_partial_vector(
+; CHECK-NEXT: [[C0:%.*]] = extractelement <4 x i32> [[C:%.*]], i32 0
+; CHECK-NEXT: [[C1:%.*]] = extractelement <4 x i32> [[C]], i32 1
+; CHECK-NEXT: [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
+; CHECK-NEXT: [[B0:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT: [[B1:%.*]] = extractelement <4 x float> [[B]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[C0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> [[TMP1]], i32 [[C1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <2 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[A0]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[A1]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> poison, float [[B0]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[B1]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP3]], <2 x float> [[TMP5]], <2 x float> [[TMP7]]
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; CHECK-NEXT: [[RA:%.*]] = insertelement <4 x float> poison, float [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; CHECK-NEXT: [[RB:%.*]] = insertelement <4 x float> [[RA]], float [[TMP10]], i32 1
+; CHECK-NEXT: ret <4 x float> [[RB]]
+;
+ %c0 = extractelement <4 x i32> %c, i32 0
+ %c1 = extractelement <4 x i32> %c, i32 1
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %1 = insertelement <2 x i32> poison, i32 %c0, i32 0
+ %2 = insertelement <2 x i32> %1, i32 %c1, i32 1
+ %3 = icmp ne <2 x i32> %2, zeroinitializer
+ %4 = insertelement <2 x float> poison, float %a0, i32 0
+ %5 = insertelement <2 x float> %4, float %a1, i32 1
+ %6 = insertelement <2 x float> poison, float %b0, i32 0
+ %7 = insertelement <2 x float> %6, float %b1, i32 1
+ %8 = select <2 x i1> %3, <2 x float> %5, <2 x float> %7
+ %9 = extractelement <2 x float> %8, i32 0
+ %ra = insertelement <4 x float> poison, float %9, i32 0
+ %10 = extractelement <2 x float> %8, i32 1
+ %rb = insertelement <4 x float> %ra, float %10, i32 1
+ ret <4 x float> %rb
+}
+
+; Make sure that vectorization happens even if insertelements operations
+; must be rescheduled. The case here is from compiling Julia.
+define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @reschedule_extract(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %b0 = extractelement <4 x float> %b, i32 0
+ %c0 = fadd float %a0, %b0
+ %v0 = insertelement <4 x float> poison, float %c0, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %b1 = extractelement <4 x float> %b, i32 1
+ %c1 = fadd float %a1, %b1
+ %v1 = insertelement <4 x float> %v0, float %c1, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %b2 = extractelement <4 x float> %b, i32 2
+ %c2 = fadd float %a2, %b2
+ %v2 = insertelement <4 x float> %v1, float %c2, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c3 = fadd float %a3, %b3
+ %v3 = insertelement <4 x float> %v2, float %c3, i32 3
+ ret <4 x float> %v3
+}
+
+; Check that cost model for vectorization takes credit for
+; instructions that are erased.
+define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @take_credit(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[V0:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> [[V0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %b0 = extractelement <4 x float> %b, i32 0
+ %c0 = fadd float %a0, %b0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %b1 = extractelement <4 x float> %b, i32 1
+ %c1 = fadd float %a1, %b1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %b2 = extractelement <4 x float> %b, i32 2
+ %c2 = fadd float %a2, %b2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %b3 = extractelement <4 x float> %b, i32 3
+ %c3 = fadd float %a3, %b3
+ %v0 = insertelement <4 x float> poison, float %c0, i32 0
+ %v1 = insertelement <4 x float> %v0, float %c1, i32 1
+ %v2 = insertelement <4 x float> %v1, float %c2, i32 2
+ %v3 = insertelement <4 x float> %v2, float %c3, i32 3
+ ret <4 x float> %v3
+}
+
+; Make sure we handle multiple trees that feed one build vector correctly.
+define <4 x double> @multi_tree(double %w, double %x, double %y, double %z) {
+; CHECK-LABEL: @multi_tree(
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> undef, double [[Z:%.*]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[Y:%.*]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[X:%.*]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[W:%.*]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = fadd <4 x double> [[TMP4]], <double 3.000000e+00, double 2.000000e+00, double 1.000000e+00, double 0.000000e+00>
+; CHECK-NEXT: [[TMP6:%.*]] = fmul <4 x double> [[TMP5]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x double> [[TMP6]], i32 3
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x double> poison, double [[TMP7]], i32 3
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x double> [[TMP6]], i32 2
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x double> [[I1]], double [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x double> [[TMP6]], i32 1
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x double> [[I2]], double [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x double> [[TMP6]], i32 0
+; CHECK-NEXT: [[I4:%.*]] = insertelement <4 x double> [[I3]], double [[TMP10]], i32 0
+; CHECK-NEXT: ret <4 x double> [[I4]]
+;
+ %t0 = fadd double %w , 0.000000e+00
+ %t1 = fadd double %x , 1.000000e+00
+ %t2 = fadd double %y , 2.000000e+00
+ %t3 = fadd double %z , 3.000000e+00
+ %t4 = fmul double %t0, 1.000000e+00
+ %i1 = insertelement <4 x double> poison, double %t4, i32 3
+ %t5 = fmul double %t1, 1.000000e+00
+ %i2 = insertelement <4 x double> %i1, double %t5, i32 2
+ %t6 = fmul double %t2, 1.000000e+00
+ %i3 = insertelement <4 x double> %i2, double %t6, i32 1
+ %t7 = fmul double %t3, 1.000000e+00
+ %i4 = insertelement <4 x double> %i3, double %t7, i32 0
+ ret <4 x double> %i4
+}
+
+define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 {
+; CHECK-LABEL: @_vadd256(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINIT_I:%.*]] = insertelement <8 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINIT1_I:%.*]] = insertelement <8 x float> [[VECINIT_I]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINIT2_I:%.*]] = insertelement <8 x float> [[VECINIT1_I]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINIT3_I:%.*]] = insertelement <8 x float> [[VECINIT2_I]], float [[TMP5]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
+; CHECK-NEXT: [[VECINIT4_I:%.*]] = insertelement <8 x float> [[VECINIT3_I]], float [[TMP6]], i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
+; CHECK-NEXT: [[VECINIT5_I:%.*]] = insertelement <8 x float> [[VECINIT4_I]], float [[TMP7]], i32 5
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
+; CHECK-NEXT: [[VECINIT6_I:%.*]] = insertelement <8 x float> [[VECINIT5_I]], float [[TMP8]], i32 6
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[VECINIT7_I:%.*]] = insertelement <8 x float> [[VECINIT6_I]], float [[TMP9]], i32 7
+; CHECK-NEXT: ret <8 x float> [[VECINIT7_I]]
+;
+ %vecext = extractelement <8 x float> %a, i32 0
+ %vecext1 = extractelement <8 x float> %b, i32 0
+ %add = fadd float %vecext, %vecext1
+ %vecext2 = extractelement <8 x float> %a, i32 1
+ %vecext3 = extractelement <8 x float> %b, i32 1
+ %add4 = fadd float %vecext2, %vecext3
+ %vecext5 = extractelement <8 x float> %a, i32 2
+ %vecext6 = extractelement <8 x float> %b, i32 2
+ %add7 = fadd float %vecext5, %vecext6
+ %vecext8 = extractelement <8 x float> %a, i32 3
+ %vecext9 = extractelement <8 x float> %b, i32 3
+ %add10 = fadd float %vecext8, %vecext9
+ %vecext11 = extractelement <8 x float> %a, i32 4
+ %vecext12 = extractelement <8 x float> %b, i32 4
+ %add13 = fadd float %vecext11, %vecext12
+ %vecext14 = extractelement <8 x float> %a, i32 5
+ %vecext15 = extractelement <8 x float> %b, i32 5
+ %add16 = fadd float %vecext14, %vecext15
+ %vecext17 = extractelement <8 x float> %a, i32 6
+ %vecext18 = extractelement <8 x float> %b, i32 6
+ %add19 = fadd float %vecext17, %vecext18
+ %vecext20 = extractelement <8 x float> %a, i32 7
+ %vecext21 = extractelement <8 x float> %b, i32 7
+ %add22 = fadd float %vecext20, %vecext21
+ %vecinit.i = insertelement <8 x float> poison, float %add, i32 0
+ %vecinit1.i = insertelement <8 x float> %vecinit.i, float %add4, i32 1
+ %vecinit2.i = insertelement <8 x float> %vecinit1.i, float %add7, i32 2
+ %vecinit3.i = insertelement <8 x float> %vecinit2.i, float %add10, i32 3
+ %vecinit4.i = insertelement <8 x float> %vecinit3.i, float %add13, i32 4
+ %vecinit5.i = insertelement <8 x float> %vecinit4.i, float %add16, i32 5
+ %vecinit6.i = insertelement <8 x float> %vecinit5.i, float %add19, i32 6
+ %vecinit7.i = insertelement <8 x float> %vecinit6.i, float %add22, i32 7
+ ret <8 x float> %vecinit7.i
+}
+
+attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
new file mode 100644
index 000000000000..c84415642130
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/load-merge-inseltpoison.ll
@@ -0,0 +1,208 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-vectorize-hor -slp-vectorize-hor-store -S < %s -mtriple=x86_64-apple-macosx -mcpu=haswell | FileCheck %s
+
+;unsigned load_le32(unsigned char *data) {
+; unsigned le32 = (data[0]<<0) | (data[1]<<8) | (data[2]<<16) | (data[3]<<24);
+; return le32;
+;}
+
+define i32 @_Z9load_le32Ph(i8* nocapture readonly %data) {
+; CHECK-LABEL: @_Z9load_le32Ph(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load i8, i8* [[DATA:%.*]], align 1
+; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP0]] to i32
+; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load i8, i8* [[ARRAYIDX1]], align 1
+; CHECK-NEXT: [[CONV2:%.*]] = zext i8 [[TMP1]] to i32
+; CHECK-NEXT: [[SHL3:%.*]] = shl nuw nsw i32 [[CONV2]], 8
+; CHECK-NEXT: [[OR:%.*]] = or i32 [[SHL3]], [[CONV]]
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 2
+; CHECK-NEXT: [[TMP2:%.*]] = load i8, i8* [[ARRAYIDX4]], align 1
+; CHECK-NEXT: [[CONV5:%.*]] = zext i8 [[TMP2]] to i32
+; CHECK-NEXT: [[SHL6:%.*]] = shl nuw nsw i32 [[CONV5]], 16
+; CHECK-NEXT: [[OR7:%.*]] = or i32 [[OR]], [[SHL6]]
+; CHECK-NEXT: [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, i8* [[DATA]], i64 3
+; CHECK-NEXT: [[TMP3:%.*]] = load i8, i8* [[ARRAYIDX8]], align 1
+; CHECK-NEXT: [[CONV9:%.*]] = zext i8 [[TMP3]] to i32
+; CHECK-NEXT: [[SHL10:%.*]] = shl nuw i32 [[CONV9]], 24
+; CHECK-NEXT: [[OR11:%.*]] = or i32 [[OR7]], [[SHL10]]
+; CHECK-NEXT: ret i32 [[OR11]]
+;
+entry:
+ %0 = load i8, i8* %data, align 1
+ %conv = zext i8 %0 to i32
+ %arrayidx1 = getelementptr inbounds i8, i8* %data, i64 1
+ %1 = load i8, i8* %arrayidx1, align 1
+ %conv2 = zext i8 %1 to i32
+ %shl3 = shl nuw nsw i32 %conv2, 8
+ %or = or i32 %shl3, %conv
+ %arrayidx4 = getelementptr inbounds i8, i8* %data, i64 2
+ %2 = load i8, i8* %arrayidx4, align 1
+ %conv5 = zext i8 %2 to i32
+ %shl6 = shl nuw nsw i32 %conv5, 16
+ %or7 = or i32 %or, %shl6
+ %arrayidx8 = getelementptr inbounds i8, i8* %data, i64 3
+ %3 = load i8, i8* %arrayidx8, align 1
+ %conv9 = zext i8 %3 to i32
+ %shl10 = shl nuw i32 %conv9, 24
+ %or11 = or i32 %or7, %shl10
+ ret i32 %or11
+}
+
+define <4 x float> @PR16739_byref(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byref(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X:%.*]], i64 0, i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 2
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <2 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[X2:%.*]] = load float, float* [[GEP2]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[TMP4]], i32 1
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[X2]], i32 2
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[X2]], i32 3
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %gep0 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 0
+ %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 1
+ %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 2
+ %x0 = load float, float* %gep0
+ %x1 = load float, float* %gep1
+ %x2 = load float, float* %gep2
+ %i0 = insertelement <4 x float> poison, float %x0, i32 0
+ %i1 = insertelement <4 x float> %i0, float %x1, i32 1
+ %i2 = insertelement <4 x float> %i1, float %x2, i32 2
+ %i3 = insertelement <4 x float> %i2, float %x2, i32 3
+ ret <4 x float> %i3
+}
+
+define <4 x float> @PR16739_byref_alt(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byref_alt(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X:%.*]], i64 0, i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <2 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> undef, <4 x i32> <i32 0, i32 0, i32 1, i32 1>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 0
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[TMP3]], i32 0
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> [[I0]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[SHUFFLE]], i32 2
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x float> [[I1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[I3:%.*]] = insertelement <4 x float> [[I2]], float [[TMP4]], i32 3
+; CHECK-NEXT: ret <4 x float> [[I3]]
+;
+ %gep0 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 0
+ %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 1
+ %x0 = load float, float* %gep0
+ %x1 = load float, float* %gep1
+ %i0 = insertelement <4 x float> poison, float %x0, i32 0
+ %i1 = insertelement <4 x float> %i0, float %x0, i32 1
+ %i2 = insertelement <4 x float> %i1, float %x1, i32 2
+ %i3 = insertelement <4 x float> %i2, float %x1, i32 3
+ ret <4 x float> %i3
+}
+
+define <4 x float> @PR16739_byval(<4 x float>* nocapture readonly dereferenceable(16) %x) {
+; CHECK-LABEL: @PR16739_byval(
+; CHECK-NEXT: [[T0:%.*]] = bitcast <4 x float>* [[X:%.*]] to i64*
+; CHECK-NEXT: [[T1:%.*]] = load i64, i64* [[T0]], align 16
+; CHECK-NEXT: [[T2:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[X]], i64 0, i64 2
+; CHECK-NEXT: [[T3:%.*]] = bitcast float* [[T2]] to i64*
+; CHECK-NEXT: [[T4:%.*]] = load i64, i64* [[T3]], align 8
+; CHECK-NEXT: [[T5:%.*]] = trunc i64 [[T1]] to i32
+; CHECK-NEXT: [[T6:%.*]] = bitcast i32 [[T5]] to float
+; CHECK-NEXT: [[T7:%.*]] = insertelement <4 x float> poison, float [[T6]], i32 0
+; CHECK-NEXT: [[T8:%.*]] = lshr i64 [[T1]], 32
+; CHECK-NEXT: [[T9:%.*]] = trunc i64 [[T8]] to i32
+; CHECK-NEXT: [[T10:%.*]] = bitcast i32 [[T9]] to float
+; CHECK-NEXT: [[T11:%.*]] = insertelement <4 x float> [[T7]], float [[T10]], i32 1
+; CHECK-NEXT: [[T12:%.*]] = trunc i64 [[T4]] to i32
+; CHECK-NEXT: [[T13:%.*]] = bitcast i32 [[T12]] to float
+; CHECK-NEXT: [[T14:%.*]] = insertelement <4 x float> [[T11]], float [[T13]], i32 2
+; CHECK-NEXT: [[T15:%.*]] = insertelement <4 x float> [[T14]], float [[T13]], i32 3
+; CHECK-NEXT: ret <4 x float> [[T15]]
+;
+ %t0 = bitcast <4 x float>* %x to i64*
+ %t1 = load i64, i64* %t0, align 16
+ %t2 = getelementptr inbounds <4 x float>, <4 x float>* %x, i64 0, i64 2
+ %t3 = bitcast float* %t2 to i64*
+ %t4 = load i64, i64* %t3, align 8
+ %t5 = trunc i64 %t1 to i32
+ %t6 = bitcast i32 %t5 to float
+ %t7 = insertelement <4 x float> poison, float %t6, i32 0
+ %t8 = lshr i64 %t1, 32
+ %t9 = trunc i64 %t8 to i32
+ %t10 = bitcast i32 %t9 to float
+ %t11 = insertelement <4 x float> %t7, float %t10, i32 1
+ %t12 = trunc i64 %t4 to i32
+ %t13 = bitcast i32 %t12 to float
+ %t14 = insertelement <4 x float> %t11, float %t13, i32 2
+ %t15 = insertelement <4 x float> %t14, float %t13, i32 3
+ ret <4 x float> %t15
+}
+
+define void @PR43578_prefer128(i32* %r, i64* %p, i64* %q) #0 {
+; CHECK-LABEL: @PR43578_prefer128(
+; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds i64, i64* [[P:%.*]], i64 0
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 2
+; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 3
+; CHECK-NEXT: [[Q0:%.*]] = getelementptr inbounds i64, i64* [[Q:%.*]], i64 0
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 1
+; CHECK-NEXT: [[Q2:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 2
+; CHECK-NEXT: [[Q3:%.*]] = getelementptr inbounds i64, i64* [[Q]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i64* [[P0]] to <2 x i64>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* [[TMP1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i64* [[P2]] to <2 x i64>*
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* [[TMP3]], align 2
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i64* [[Q0]] to <2 x i64>*
+; CHECK-NEXT: [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* [[TMP5]], align 2
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i64* [[Q2]] to <2 x i64>*
+; CHECK-NEXT: [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* [[TMP7]], align 2
+; CHECK-NEXT: [[TMP9:%.*]] = sub nsw <2 x i64> [[TMP2]], [[TMP6]]
+; CHECK-NEXT: [[TMP10:%.*]] = sub nsw <2 x i64> [[TMP4]], [[TMP8]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i64> [[TMP9]], i32 0
+; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, i32* [[R:%.*]], i64 [[TMP11]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i64> [[TMP9]], i32 1
+; CHECK-NEXT: [[G1:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP12]]
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x i64> [[TMP10]], i32 0
+; CHECK-NEXT: [[G2:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP13]]
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x i64> [[TMP10]], i32 1
+; CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds i32, i32* [[R]], i64 [[TMP14]]
+; CHECK-NEXT: ret void
+;
+ %p0 = getelementptr inbounds i64, i64* %p, i64 0
+ %p1 = getelementptr inbounds i64, i64* %p, i64 1
+ %p2 = getelementptr inbounds i64, i64* %p, i64 2
+ %p3 = getelementptr inbounds i64, i64* %p, i64 3
+
+ %q0 = getelementptr inbounds i64, i64* %q, i64 0
+ %q1 = getelementptr inbounds i64, i64* %q, i64 1
+ %q2 = getelementptr inbounds i64, i64* %q, i64 2
+ %q3 = getelementptr inbounds i64, i64* %q, i64 3
+
+ %x0 = load i64, i64* %p0, align 2
+ %x1 = load i64, i64* %p1, align 2
+ %x2 = load i64, i64* %p2, align 2
+ %x3 = load i64, i64* %p3, align 2
+
+ %y0 = load i64, i64* %q0, align 2
+ %y1 = load i64, i64* %q1, align 2
+ %y2 = load i64, i64* %q2, align 2
+ %y3 = load i64, i64* %q3, align 2
+
+ %sub0 = sub nsw i64 %x0, %y0
+ %sub1 = sub nsw i64 %x1, %y1
+ %sub2 = sub nsw i64 %x2, %y2
+ %sub3 = sub nsw i64 %x3, %y3
+
+ %g0 = getelementptr inbounds i32, i32* %r, i64 %sub0
+ %g1 = getelementptr inbounds i32, i32* %r, i64 %sub1
+ %g2 = getelementptr inbounds i32, i32* %r, i64 %sub2
+ %g3 = getelementptr inbounds i32, i32* %r, i64 %sub3
+ ret void
+}
+
+attributes #0 = { "prefer-vector-width"="128" }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr31599-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr31599-inseltpoison.ll
new file mode 100644
index 000000000000..448480d23b41
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr31599-inseltpoison.ll
@@ -0,0 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define <2 x float> @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[SOURCE:%.*]] = insertelement <2 x float> poison, float undef, i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = fsub <2 x float> [[SOURCE]], [[SOURCE]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[RES2:%.*]] = insertelement <2 x float> [[RES1]], float [[TMP2]], i32 1
+; CHECK-NEXT: ret <2 x float> [[RES2]]
+;
+entry:
+ %source = insertelement <2 x float> poison, float undef, i32 0
+ %e0 = extractelement <2 x float> %source, i32 0
+ %e0.dup = extractelement <2 x float> %source, i32 0
+ %sub1 = fsub float %e0, %e0.dup
+ %e1 = extractelement <2 x float> %source, i32 1
+ %e1.dup = extractelement <2 x float> %source, i32 1
+ %sub2 = fsub float %e1, %e1.dup
+ %res1 = insertelement <2 x float> poison, float %sub1, i32 0
+ %res2 = insertelement <2 x float> %res1, float %sub2, i32 1
+ ret <2 x float> %res2
+}
+
+!llvm.ident = !{!0, !0}
+
+!0 = !{!"clang version 4.0.0 "}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
new file mode 100644
index 000000000000..a136a6ead561
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr42022-inseltpoison.ll
@@ -0,0 +1,278 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; See https://reviews.llvm.org/D70068 and https://reviews.llvm.org/D70587 for context
+
+; Checks that vector insertvalues into the struct become SLP seeds.
+define { <2 x float>, <2 x float> } @StructOfVectors(float *%Ptr) {
+; CHECK-LABEL: @StructOfVectors(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[VECIN0:%.*]] = insertelement <2 x float> poison, float [[TMP4]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[VECIN1:%.*]] = insertelement <2 x float> [[VECIN0]], float [[TMP5]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT: [[VECIN2:%.*]] = insertelement <2 x float> poison, float [[TMP6]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT: [[VECIN3:%.*]] = insertelement <2 x float> [[VECIN2]], float [[TMP7]], i64 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue { <2 x float>, <2 x float> } undef, <2 x float> [[VECIN1]], 0
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue { <2 x float>, <2 x float> } [[RET0]], <2 x float> [[VECIN3]], 1
+; CHECK-NEXT: ret { <2 x float>, <2 x float> } [[RET1]]
+;
+ %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+ %L0 = load float, float * %GEP0
+ %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+ %L1 = load float, float * %GEP1
+ %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+ %L2 = load float, float * %GEP2
+ %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+ %L3 = load float, float * %GEP3
+
+ %Fadd0 = fadd fast float %L0, 1.1e+01
+ %Fadd1 = fadd fast float %L1, 1.2e+01
+ %Fadd2 = fadd fast float %L2, 1.3e+01
+ %Fadd3 = fadd fast float %L3, 1.4e+01
+
+ %VecIn0 = insertelement <2 x float> poison, float %Fadd0, i64 0
+ %VecIn1 = insertelement <2 x float> %VecIn0, float %Fadd1, i64 1
+
+ %VecIn2 = insertelement <2 x float> poison, float %Fadd2, i64 0
+ %VecIn3 = insertelement <2 x float> %VecIn2, float %Fadd3, i64 1
+
+ %Ret0 = insertvalue {<2 x float>, <2 x float>} undef, <2 x float> %VecIn1, 0
+ %Ret1 = insertvalue {<2 x float>, <2 x float>} %Ret0, <2 x float> %VecIn3, 1
+ ret {<2 x float>, <2 x float>} %Ret1
+}
+
+%StructTy = type { float, float}
+
+define [2 x %StructTy] @ArrayOfStruct(float *%Ptr) {
+; CHECK-LABEL: @ArrayOfStruct(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[TMP4]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN0]], float [[TMP5]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN2]], float [[TMP7]], 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue [2 x %StructTy] undef, [[STRUCTTY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue [2 x %StructTy] [[RET0]], [[STRUCTTY]] [[STRUCTIN3]], 1
+; CHECK-NEXT: ret [2 x %StructTy] [[RET1]]
+;
+ %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+ %L0 = load float, float * %GEP0
+ %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+ %L1 = load float, float * %GEP1
+ %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+ %L2 = load float, float * %GEP2
+ %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+ %L3 = load float, float * %GEP3
+
+ %Fadd0 = fadd fast float %L0, 1.1e+01
+ %Fadd1 = fadd fast float %L1, 1.2e+01
+ %Fadd2 = fadd fast float %L2, 1.3e+01
+ %Fadd3 = fadd fast float %L3, 1.4e+01
+
+ %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+ %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+ %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0
+ %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1
+
+ %Ret0 = insertvalue [2 x %StructTy] undef, %StructTy %StructIn1, 0
+ %Ret1 = insertvalue [2 x %StructTy] %Ret0, %StructTy %StructIn3, 1
+ ret [2 x %StructTy] %Ret1
+}
+
+define {%StructTy, %StructTy} @StructOfStruct(float *%Ptr) {
+; CHECK-LABEL: @StructOfStruct(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[GEP0]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <4 x float> [[TMP2]], <float 1.100000e+01, float 1.200000e+01, float 1.300000e+01, float 1.400000e+01>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP3]], i32 0
+; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[TMP4]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP3]], i32 1
+; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN0]], float [[TMP5]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <4 x float> [[TMP3]], i32 2
+; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCTTY]] undef, float [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x float> [[TMP3]], i32 3
+; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN2]], float [[TMP7]], 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } undef, [[STRUCTTY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], [[STRUCTTY]] } [[RET0]], [[STRUCTTY]] [[STRUCTIN3]], 1
+; CHECK-NEXT: ret { [[STRUCTTY]], [[STRUCTTY]] } [[RET1]]
+;
+ %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+ %L0 = load float, float * %GEP0
+ %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+ %L1 = load float, float * %GEP1
+ %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+ %L2 = load float, float * %GEP2
+ %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+ %L3 = load float, float * %GEP3
+
+ %Fadd0 = fadd fast float %L0, 1.1e+01
+ %Fadd1 = fadd fast float %L1, 1.2e+01
+ %Fadd2 = fadd fast float %L2, 1.3e+01
+ %Fadd3 = fadd fast float %L3, 1.4e+01
+
+ %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+ %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+ %StructIn2 = insertvalue %StructTy undef, float %Fadd2, 0
+ %StructIn3 = insertvalue %StructTy %StructIn2, float %Fadd3, 1
+
+ %Ret0 = insertvalue {%StructTy, %StructTy} undef, %StructTy %StructIn1, 0
+ %Ret1 = insertvalue {%StructTy, %StructTy} %Ret0, %StructTy %StructIn3, 1
+ ret {%StructTy, %StructTy} %Ret1
+}
+
+define {%StructTy, float, float} @NonHomogeneousStruct(float *%Ptr) {
+; CHECK-LABEL: @NonHomogeneousStruct(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds float, float* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[L0:%.*]] = load float, float* [[GEP0]], align 4
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 1
+; CHECK-NEXT: [[L1:%.*]] = load float, float* [[GEP1]], align 4
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 2
+; CHECK-NEXT: [[L2:%.*]] = load float, float* [[GEP2]], align 4
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, float* [[PTR]], i64 3
+; CHECK-NEXT: [[L3:%.*]] = load float, float* [[GEP3]], align 4
+; CHECK-NEXT: [[FADD0:%.*]] = fadd fast float [[L0]], 1.100000e+01
+; CHECK-NEXT: [[FADD1:%.*]] = fadd fast float [[L1]], 1.200000e+01
+; CHECK-NEXT: [[FADD2:%.*]] = fadd fast float [[L2]], 1.300000e+01
+; CHECK-NEXT: [[FADD3:%.*]] = fadd fast float [[L3]], 1.400000e+01
+; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCTTY:%.*]] undef, float [[FADD0]], 0
+; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCTTY]] [[STRUCTIN0]], float [[FADD1]], 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCTTY]], float, float } undef, [[STRUCTTY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET0]], float [[FADD2]], 1
+; CHECK-NEXT: [[RET2:%.*]] = insertvalue { [[STRUCTTY]], float, float } [[RET1]], float [[FADD3]], 2
+; CHECK-NEXT: ret { [[STRUCTTY]], float, float } [[RET2]]
+;
+ %GEP0 = getelementptr inbounds float, float* %Ptr, i64 0
+ %L0 = load float, float * %GEP0
+ %GEP1 = getelementptr inbounds float, float* %Ptr, i64 1
+ %L1 = load float, float * %GEP1
+ %GEP2 = getelementptr inbounds float, float* %Ptr, i64 2
+ %L2 = load float, float * %GEP2
+ %GEP3 = getelementptr inbounds float, float* %Ptr, i64 3
+ %L3 = load float, float * %GEP3
+
+ %Fadd0 = fadd fast float %L0, 1.1e+01
+ %Fadd1 = fadd fast float %L1, 1.2e+01
+ %Fadd2 = fadd fast float %L2, 1.3e+01
+ %Fadd3 = fadd fast float %L3, 1.4e+01
+
+ %StructIn0 = insertvalue %StructTy undef, float %Fadd0, 0
+ %StructIn1 = insertvalue %StructTy %StructIn0, float %Fadd1, 1
+
+ %Ret0 = insertvalue {%StructTy, float, float} undef, %StructTy %StructIn1, 0
+ %Ret1 = insertvalue {%StructTy, float, float} %Ret0, float %Fadd2, 1
+ %Ret2 = insertvalue {%StructTy, float, float} %Ret1, float %Fadd3, 2
+ ret {%StructTy, float, float} %Ret2
+}
+
+%Struct1Ty = type { i16, i16 }
+%Struct2Ty = type { %Struct1Ty, %Struct1Ty}
+
+define {%Struct2Ty, %Struct2Ty} @StructOfStructOfStruct(i16 *%Ptr) {
+; CHECK-LABEL: @StructOfStructOfStruct(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 3
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 4
+; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 5
+; CHECK-NEXT: [[GEP6:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 6
+; CHECK-NEXT: [[GEP7:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 7
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[GEP0]] to <8 x i16>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[TMP2]], <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCT1TY:%.*]] undef, i16 [[TMP4]], 0
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN0]], i16 [[TMP5]], 1
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN2]], i16 [[TMP7]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT: [[STRUCTIN4:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP8]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT: [[STRUCTIN5:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN4]], i16 [[TMP9]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT: [[STRUCTIN6:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP10]], 0
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT: [[STRUCTIN7:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN6]], i16 [[TMP11]], 1
+; CHECK-NEXT: [[STRUCT2IN0:%.*]] = insertvalue [[STRUCT2TY:%.*]] undef, [[STRUCT1TY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[STRUCT2IN1:%.*]] = insertvalue [[STRUCT2TY]] [[STRUCT2IN0]], [[STRUCT1TY]] [[STRUCTIN3]], 1
+; CHECK-NEXT: [[STRUCT2IN2:%.*]] = insertvalue [[STRUCT2TY]] undef, [[STRUCT1TY]] [[STRUCTIN5]], 0
+; CHECK-NEXT: [[STRUCT2IN3:%.*]] = insertvalue [[STRUCT2TY]] [[STRUCT2IN2]], [[STRUCT1TY]] [[STRUCTIN7]], 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } undef, [[STRUCT2TY]] [[STRUCT2IN1]], 0
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET0]], [[STRUCT2TY]] [[STRUCT2IN3]], 1
+; CHECK-NEXT: ret { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET1]]
+;
+ %GEP0 = getelementptr inbounds i16, i16* %Ptr, i64 0
+ %L0 = load i16, i16 * %GEP0
+ %GEP1 = getelementptr inbounds i16, i16* %Ptr, i64 1
+ %L1 = load i16, i16 * %GEP1
+ %GEP2 = getelementptr inbounds i16, i16* %Ptr, i64 2
+ %L2 = load i16, i16 * %GEP2
+ %GEP3 = getelementptr inbounds i16, i16* %Ptr, i64 3
+ %L3 = load i16, i16 * %GEP3
+ %GEP4 = getelementptr inbounds i16, i16* %Ptr, i64 4
+ %L4 = load i16, i16 * %GEP4
+ %GEP5 = getelementptr inbounds i16, i16* %Ptr, i64 5
+ %L5 = load i16, i16 * %GEP5
+ %GEP6 = getelementptr inbounds i16, i16* %Ptr, i64 6
+ %L6 = load i16, i16 * %GEP6
+ %GEP7 = getelementptr inbounds i16, i16* %Ptr, i64 7
+ %L7 = load i16, i16 * %GEP7
+
+ %Fadd0 = add i16 %L0, 1
+ %Fadd1 = add i16 %L1, 2
+ %Fadd2 = add i16 %L2, 3
+ %Fadd3 = add i16 %L3, 4
+ %Fadd4 = add i16 %L4, 5
+ %Fadd5 = add i16 %L5, 6
+ %Fadd6 = add i16 %L6, 7
+ %Fadd7 = add i16 %L7, 8
+
+ %StructIn0 = insertvalue %Struct1Ty undef, i16 %Fadd0, 0
+ %StructIn1 = insertvalue %Struct1Ty %StructIn0, i16 %Fadd1, 1
+
+ %StructIn2 = insertvalue %Struct1Ty undef, i16 %Fadd2, 0
+ %StructIn3 = insertvalue %Struct1Ty %StructIn2, i16 %Fadd3, 1
+
+ %StructIn4 = insertvalue %Struct1Ty undef, i16 %Fadd4, 0
+ %StructIn5 = insertvalue %Struct1Ty %StructIn4, i16 %Fadd5, 1
+
+ %StructIn6 = insertvalue %Struct1Ty undef, i16 %Fadd6, 0
+ %StructIn7 = insertvalue %Struct1Ty %StructIn6, i16 %Fadd7, 1
+
+ %Struct2In0 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn1, 0
+ %Struct2In1 = insertvalue %Struct2Ty %Struct2In0, %Struct1Ty %StructIn3, 1
+
+ %Struct2In2 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn5, 0
+ %Struct2In3 = insertvalue %Struct2Ty %Struct2In2, %Struct1Ty %StructIn7, 1
+
+ %Ret0 = insertvalue {%Struct2Ty, %Struct2Ty} undef, %Struct2Ty %Struct2In1, 0
+ %Ret1 = insertvalue {%Struct2Ty, %Struct2Ty} %Ret0, %Struct2Ty %Struct2In3, 1
+ ret {%Struct2Ty, %Struct2Ty} %Ret1
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr44067-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr44067-inseltpoison.ll
new file mode 100644
index 000000000000..35a296820b3b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr44067-inseltpoison.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; See https://reviews.llvm.org/D83779
+
+define <2 x float> @foo({{float, float}}* %A) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast { { float, float } }* [[A:%.*]] to <2 x float>*
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 2.000000e+00>
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[INS1:%.*]] = insertelement <2 x float> poison, float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[INS0:%.*]] = insertelement <2 x float> [[INS1]], float [[TMP4]], i32 0
+; CHECK-NEXT: ret <2 x float> [[INS0]]
+;
+entry:
+ %0 = bitcast {{float, float}}* %A to <2 x float>*
+ %1 = load <2 x float>, <2 x float>* %0
+ %L0 = extractelement <2 x float> %1, i32 0
+ %L1 = extractelement <2 x float> %1, i32 1
+ %Mul0 = fmul float %L0, 2.000000e+00
+ %Mul1 = fmul float %L1, 2.000000e+00
+ %Ins1 = insertelement <2 x float> poison, float %Mul1, i32 1
+ %Ins0 = insertelement <2 x float> %Ins1, float %Mul0, i32 0
+ ret <2 x float> %Ins0
+}
+
+
+%Struct1Ty = type { i16, i16 }
+%Struct2Ty = type { %Struct1Ty, %Struct1Ty}
+
+define {%Struct2Ty, %Struct2Ty} @StructOfStructOfStruct(i16 *%Ptr) {
+; CHECK-LABEL: @StructOfStructOfStruct(
+; CHECK-NEXT: [[GEP0:%.*]] = getelementptr inbounds i16, i16* [[PTR:%.*]], i64 0
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 1
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 2
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 3
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 4
+; CHECK-NEXT: [[GEP5:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 5
+; CHECK-NEXT: [[GEP6:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 6
+; CHECK-NEXT: [[GEP7:%.*]] = getelementptr inbounds i16, i16* [[PTR]], i64 7
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[GEP0]] to <8 x i16>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT: [[TMP3:%.*]] = add <8 x i16> [[TMP2]], <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; CHECK-NEXT: [[STRUCTIN0:%.*]] = insertvalue [[STRUCT1TY:%.*]] undef, i16 [[TMP4]], 1
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; CHECK-NEXT: [[STRUCTIN1:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN0]], i16 [[TMP5]], 0
+; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; CHECK-NEXT: [[STRUCTIN2:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP6]], 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; CHECK-NEXT: [[STRUCTIN3:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN2]], i16 [[TMP7]], 1
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; CHECK-NEXT: [[STRUCTIN4:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP8]], 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; CHECK-NEXT: [[STRUCTIN5:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN4]], i16 [[TMP9]], 1
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; CHECK-NEXT: [[STRUCTIN6:%.*]] = insertvalue [[STRUCT1TY]] undef, i16 [[TMP10]], 1
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; CHECK-NEXT: [[STRUCTIN7:%.*]] = insertvalue [[STRUCT1TY]] [[STRUCTIN6]], i16 [[TMP11]], 0
+; CHECK-NEXT: [[STRUCT2IN0:%.*]] = insertvalue [[STRUCT2TY:%.*]] undef, [[STRUCT1TY]] [[STRUCTIN1]], 0
+; CHECK-NEXT: [[STRUCT2IN1:%.*]] = insertvalue [[STRUCT2TY]] [[STRUCT2IN0]], [[STRUCT1TY]] [[STRUCTIN3]], 1
+; CHECK-NEXT: [[STRUCT2IN2:%.*]] = insertvalue [[STRUCT2TY]] undef, [[STRUCT1TY]] [[STRUCTIN5]], 0
+; CHECK-NEXT: [[STRUCT2IN3:%.*]] = insertvalue [[STRUCT2TY]] [[STRUCT2IN2]], [[STRUCT1TY]] [[STRUCTIN7]], 1
+; CHECK-NEXT: [[RET0:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } undef, [[STRUCT2TY]] [[STRUCT2IN3]], 1
+; CHECK-NEXT: [[RET1:%.*]] = insertvalue { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET0]], [[STRUCT2TY]] [[STRUCT2IN1]], 0
+; CHECK-NEXT: ret { [[STRUCT2TY]], [[STRUCT2TY]] } [[RET1]]
+;
+ %GEP0 = getelementptr inbounds i16, i16* %Ptr, i64 0
+ %L0 = load i16, i16 * %GEP0
+ %GEP1 = getelementptr inbounds i16, i16* %Ptr, i64 1
+ %L1 = load i16, i16 * %GEP1
+ %GEP2 = getelementptr inbounds i16, i16* %Ptr, i64 2
+ %L2 = load i16, i16 * %GEP2
+ %GEP3 = getelementptr inbounds i16, i16* %Ptr, i64 3
+ %L3 = load i16, i16 * %GEP3
+ %GEP4 = getelementptr inbounds i16, i16* %Ptr, i64 4
+ %L4 = load i16, i16 * %GEP4
+ %GEP5 = getelementptr inbounds i16, i16* %Ptr, i64 5
+ %L5 = load i16, i16 * %GEP5
+ %GEP6 = getelementptr inbounds i16, i16* %Ptr, i64 6
+ %L6 = load i16, i16 * %GEP6
+ %GEP7 = getelementptr inbounds i16, i16* %Ptr, i64 7
+ %L7 = load i16, i16 * %GEP7
+
+ %Fadd0 = add i16 %L0, 1
+ %Fadd1 = add i16 %L1, 2
+ %Fadd2 = add i16 %L2, 3
+ %Fadd3 = add i16 %L3, 4
+ %Fadd4 = add i16 %L4, 5
+ %Fadd5 = add i16 %L5, 6
+ %Fadd6 = add i16 %L6, 7
+ %Fadd7 = add i16 %L7, 8
+
+ %StructIn0 = insertvalue %Struct1Ty undef, i16 %Fadd1, 1
+ %StructIn1 = insertvalue %Struct1Ty %StructIn0, i16 %Fadd0, 0
+
+ %StructIn2 = insertvalue %Struct1Ty undef, i16 %Fadd2, 0
+ %StructIn3 = insertvalue %Struct1Ty %StructIn2, i16 %Fadd3, 1
+
+ %StructIn4 = insertvalue %Struct1Ty undef, i16 %Fadd4, 0
+ %StructIn5 = insertvalue %Struct1Ty %StructIn4, i16 %Fadd5, 1
+
+ %StructIn6 = insertvalue %Struct1Ty undef, i16 %Fadd7, 1
+ %StructIn7 = insertvalue %Struct1Ty %StructIn6, i16 %Fadd6, 0
+
+ %Struct2In0 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn1, 0
+ %Struct2In1 = insertvalue %Struct2Ty %Struct2In0, %Struct1Ty %StructIn3, 1
+
+ %Struct2In2 = insertvalue %Struct2Ty undef, %Struct1Ty %StructIn5, 0
+ %Struct2In3 = insertvalue %Struct2Ty %Struct2In2, %Struct1Ty %StructIn7, 1
+
+ %Ret0 = insertvalue {%Struct2Ty, %Struct2Ty} undef, %Struct2Ty %Struct2In3, 1
+ %Ret1 = insertvalue {%Struct2Ty, %Struct2Ty} %Ret0, %Struct2Ty %Struct2In1, 0
+ ret {%Struct2Ty, %Struct2Ty} %Ret1
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
new file mode 100644
index 000000000000..090a9ea0dd1d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
@@ -0,0 +1,664 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512
+; RUN: opt < %s -slp-vectorizer -instcombine -S -mtriple=x86_64-unknown-linux -mattr=+avx512vl | FileCheck %s --check-prefixes=CHECK,AVX512
+
+define void @gather_load(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) {
+; CHECK-LABEL: @gather_load(
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP1]], align 4, [[TBAA0:!tbaa !.*]]
+; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11
+; CHECK-NEXT: [[TMP6:%.*]] = load i32, i32* [[TMP5]], align 4, [[TBAA0]]
+; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4
+; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]]
+; CHECK-NEXT: [[TMP9:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0]]
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP8]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP9]], i32 3
+; CHECK-NEXT: [[TMP14:%.*]] = add nsw <4 x i32> [[TMP13]], <i32 1, i32 2, i32 3, i32 4>
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>*
+; CHECK-NEXT: store <4 x i32> [[TMP14]], <4 x i32>* [[TMP15]], align 4, [[TBAA0]]
+; CHECK-NEXT: ret void
+;
+ %3 = getelementptr inbounds i32, i32* %1, i64 1
+ %4 = load i32, i32* %1, align 4, !tbaa !2
+ %5 = getelementptr inbounds i32, i32* %0, i64 1
+ %6 = getelementptr inbounds i32, i32* %1, i64 11
+ %7 = load i32, i32* %6, align 4, !tbaa !2
+ %8 = getelementptr inbounds i32, i32* %0, i64 2
+ %9 = getelementptr inbounds i32, i32* %1, i64 4
+ %10 = load i32, i32* %9, align 4, !tbaa !2
+ %11 = getelementptr inbounds i32, i32* %0, i64 3
+ %12 = load i32, i32* %3, align 4, !tbaa !2
+ %13 = insertelement <4 x i32> poison, i32 %4, i32 0
+ %14 = insertelement <4 x i32> %13, i32 %7, i32 1
+ %15 = insertelement <4 x i32> %14, i32 %10, i32 2
+ %16 = insertelement <4 x i32> %15, i32 %12, i32 3
+ %17 = add nsw <4 x i32> %16, <i32 1, i32 2, i32 3, i32 4>
+ %18 = bitcast i32* %0 to <4 x i32>*
+ store <4 x i32> %17, <4 x i32>* %18, align 4, !tbaa !2
+ ret void
+}
+
+define void @gather_load_2(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) {
+; SSE-LABEL: @gather_load_2(
+; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1
+; SSE-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]]
+; SSE-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
+; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; SSE-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10
+; SSE-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
+; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2
+; SSE-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3
+; SSE-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
+; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3
+; SSE-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5
+; SSE-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
+; SSE-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]]
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @gather_load_2(
+; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP1:%.*]], i64 1
+; AVX-NEXT: [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4, [[TBAA0:!tbaa !.*]]
+; AVX-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP4]], 1
+; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; AVX-NEXT: store i32 [[TMP5]], i32* [[TMP0]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 10
+; AVX-NEXT: [[TMP8:%.*]] = load i32, i32* [[TMP7]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP9:%.*]] = add nsw i32 [[TMP8]], 2
+; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2
+; AVX-NEXT: store i32 [[TMP9]], i32* [[TMP6]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 3
+; AVX-NEXT: [[TMP12:%.*]] = load i32, i32* [[TMP11]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP13:%.*]] = add nsw i32 [[TMP12]], 3
+; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3
+; AVX-NEXT: store i32 [[TMP13]], i32* [[TMP10]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 5
+; AVX-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP15]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP17:%.*]] = add nsw i32 [[TMP16]], 4
+; AVX-NEXT: store i32 [[TMP17]], i32* [[TMP14]], align 4, [[TBAA0]]
+; AVX-NEXT: ret void
+;
+; AVX2-LABEL: @gather_load_2(
+; AVX2-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> undef, i32* [[TMP1:%.*]], i32 0
+; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> <i64 1, i64 10, i64 3, i64 5>
+; AVX2-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0:!tbaa !.*]]
+; AVX2-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], <i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>*
+; AVX2-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]]
+; AVX2-NEXT: ret void
+;
+; AVX512-LABEL: @gather_load_2(
+; AVX512-NEXT: [[TMP3:%.*]] = insertelement <4 x i32*> undef, i32* [[TMP1:%.*]], i32 0
+; AVX512-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32*> [[TMP3]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr i32, <4 x i32*> [[TMP4]], <4 x i64> <i64 1, i64 10, i64 3, i64 5>
+; AVX512-NEXT: [[TMP6:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP5]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0:!tbaa !.*]]
+; AVX512-NEXT: [[TMP7:%.*]] = add nsw <4 x i32> [[TMP6]], <i32 1, i32 2, i32 3, i32 4>
+; AVX512-NEXT: [[TMP8:%.*]] = bitcast i32* [[TMP0:%.*]] to <4 x i32>*
+; AVX512-NEXT: store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4, [[TBAA0]]
+; AVX512-NEXT: ret void
+;
+ %3 = getelementptr inbounds i32, i32* %1, i64 1
+ %4 = load i32, i32* %3, align 4, !tbaa !2
+ %5 = add nsw i32 %4, 1
+ %6 = getelementptr inbounds i32, i32* %0, i64 1
+ store i32 %5, i32* %0, align 4, !tbaa !2
+ %7 = getelementptr inbounds i32, i32* %1, i64 10
+ %8 = load i32, i32* %7, align 4, !tbaa !2
+ %9 = add nsw i32 %8, 2
+ %10 = getelementptr inbounds i32, i32* %0, i64 2
+ store i32 %9, i32* %6, align 4, !tbaa !2
+ %11 = getelementptr inbounds i32, i32* %1, i64 3
+ %12 = load i32, i32* %11, align 4, !tbaa !2
+ %13 = add nsw i32 %12, 3
+ %14 = getelementptr inbounds i32, i32* %0, i64 3
+ store i32 %13, i32* %10, align 4, !tbaa !2
+ %15 = getelementptr inbounds i32, i32* %1, i64 5
+ %16 = load i32, i32* %15, align 4, !tbaa !2
+ %17 = add nsw i32 %16, 4
+ store i32 %17, i32* %14, align 4, !tbaa !2
+ ret void
+}
+
+
+define void @gather_load_3(i32* noalias nocapture %0, i32* noalias nocapture readonly %1) {
+; SSE-LABEL: @gather_load_3(
+; SSE-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1
+; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; SSE-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11
+; SSE-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2
+; SSE-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2
+; SSE-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4
+; SSE-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3
+; SSE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3
+; SSE-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15
+; SSE-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4
+; SSE-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
+; SSE-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18
+; SSE-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1
+; SSE-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5
+; SSE-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9
+; SSE-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2
+; SSE-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6
+; SSE-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6
+; SSE-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3
+; SSE-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7
+; SSE-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21
+; SSE-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4
+; SSE-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]]
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @gather_load_3(
+; AVX-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1
+; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; AVX-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 11
+; AVX-NEXT: [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], 2
+; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 2
+; AVX-NEXT: store i32 [[TMP8]], i32* [[TMP5]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 4
+; AVX-NEXT: [[TMP11:%.*]] = load i32, i32* [[TMP10]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP12:%.*]] = add i32 [[TMP11]], 3
+; AVX-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 3
+; AVX-NEXT: store i32 [[TMP12]], i32* [[TMP9]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 15
+; AVX-NEXT: [[TMP15:%.*]] = load i32, i32* [[TMP14]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 4
+; AVX-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 4
+; AVX-NEXT: store i32 [[TMP16]], i32* [[TMP13]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 18
+; AVX-NEXT: [[TMP19:%.*]] = load i32, i32* [[TMP18]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP20:%.*]] = add i32 [[TMP19]], 1
+; AVX-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5
+; AVX-NEXT: store i32 [[TMP20]], i32* [[TMP17]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9
+; AVX-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP22]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP24:%.*]] = add i32 [[TMP23]], 2
+; AVX-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6
+; AVX-NEXT: store i32 [[TMP24]], i32* [[TMP21]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6
+; AVX-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP26]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP28:%.*]] = add i32 [[TMP27]], 3
+; AVX-NEXT: [[TMP29:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7
+; AVX-NEXT: store i32 [[TMP28]], i32* [[TMP25]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21
+; AVX-NEXT: [[TMP31:%.*]] = load i32, i32* [[TMP30]], align 4, [[TBAA0]]
+; AVX-NEXT: [[TMP32:%.*]] = add i32 [[TMP31]], 4
+; AVX-NEXT: store i32 [[TMP32]], i32* [[TMP29]], align 4, [[TBAA0]]
+; AVX-NEXT: ret void
+;
+; AVX2-LABEL: @gather_load_3(
+; AVX2-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; AVX2-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> undef, i32* [[TMP1]], i32 0
+; AVX2-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> <i64 11, i64 4, i64 15, i64 18>
+; AVX2-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0]]
+; AVX2-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], <i32 2, i32 3, i32 4, i32 1>
+; AVX2-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5
+; AVX2-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; AVX2-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9
+; AVX2-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2
+; AVX2-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6
+; AVX2-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6
+; AVX2-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3
+; AVX2-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7
+; AVX2-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21
+; AVX2-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4
+; AVX2-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]]
+; AVX2-NEXT: ret void
+;
+; AVX512-LABEL: @gather_load_3(
+; AVX512-NEXT: [[TMP3:%.*]] = load i32, i32* [[TMP1:%.*]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], 1
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, i32* [[TMP0:%.*]], i64 1
+; AVX512-NEXT: store i32 [[TMP4]], i32* [[TMP0]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP6:%.*]] = insertelement <4 x i32*> undef, i32* [[TMP1]], i32 0
+; AVX512-NEXT: [[TMP7:%.*]] = shufflevector <4 x i32*> [[TMP6]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP8:%.*]] = getelementptr i32, <4 x i32*> [[TMP7]], <4 x i64> <i64 11, i64 4, i64 15, i64 18>
+; AVX512-NEXT: [[TMP9:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP8]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0]]
+; AVX512-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP9]], <i32 2, i32 3, i32 4, i32 1>
+; AVX512-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 5
+; AVX512-NEXT: [[TMP12:%.*]] = bitcast i32* [[TMP5]] to <4 x i32>*
+; AVX512-NEXT: store <4 x i32> [[TMP10]], <4 x i32>* [[TMP12]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 9
+; AVX512-NEXT: [[TMP14:%.*]] = load i32, i32* [[TMP13]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], 2
+; AVX512-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 6
+; AVX512-NEXT: store i32 [[TMP15]], i32* [[TMP11]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 6
+; AVX512-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP17]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP19:%.*]] = add i32 [[TMP18]], 3
+; AVX512-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i64 7
+; AVX512-NEXT: store i32 [[TMP19]], i32* [[TMP16]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, i32* [[TMP1]], i64 21
+; AVX512-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP21]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP23:%.*]] = add i32 [[TMP22]], 4
+; AVX512-NEXT: store i32 [[TMP23]], i32* [[TMP20]], align 4, [[TBAA0]]
+; AVX512-NEXT: ret void
+;
+ %3 = load i32, i32* %1, align 4, !tbaa !2
+ %4 = add i32 %3, 1
+ %5 = getelementptr inbounds i32, i32* %0, i64 1
+ store i32 %4, i32* %0, align 4, !tbaa !2
+ %6 = getelementptr inbounds i32, i32* %1, i64 11
+ %7 = load i32, i32* %6, align 4, !tbaa !2
+ %8 = add i32 %7, 2
+ %9 = getelementptr inbounds i32, i32* %0, i64 2
+ store i32 %8, i32* %5, align 4, !tbaa !2
+ %10 = getelementptr inbounds i32, i32* %1, i64 4
+ %11 = load i32, i32* %10, align 4, !tbaa !2
+ %12 = add i32 %11, 3
+ %13 = getelementptr inbounds i32, i32* %0, i64 3
+ store i32 %12, i32* %9, align 4, !tbaa !2
+ %14 = getelementptr inbounds i32, i32* %1, i64 15
+ %15 = load i32, i32* %14, align 4, !tbaa !2
+ %16 = add i32 %15, 4
+ %17 = getelementptr inbounds i32, i32* %0, i64 4
+ store i32 %16, i32* %13, align 4, !tbaa !2
+ %18 = getelementptr inbounds i32, i32* %1, i64 18
+ %19 = load i32, i32* %18, align 4, !tbaa !2
+ %20 = add i32 %19, 1
+ %21 = getelementptr inbounds i32, i32* %0, i64 5
+ store i32 %20, i32* %17, align 4, !tbaa !2
+ %22 = getelementptr inbounds i32, i32* %1, i64 9
+ %23 = load i32, i32* %22, align 4, !tbaa !2
+ %24 = add i32 %23, 2
+ %25 = getelementptr inbounds i32, i32* %0, i64 6
+ store i32 %24, i32* %21, align 4, !tbaa !2
+ %26 = getelementptr inbounds i32, i32* %1, i64 6
+ %27 = load i32, i32* %26, align 4, !tbaa !2
+ %28 = add i32 %27, 3
+ %29 = getelementptr inbounds i32, i32* %0, i64 7
+ store i32 %28, i32* %25, align 4, !tbaa !2
+ %30 = getelementptr inbounds i32, i32* %1, i64 21
+ %31 = load i32, i32* %30, align 4, !tbaa !2
+ %32 = add i32 %31, 4
+ store i32 %32, i32* %29, align 4, !tbaa !2
+ ret void
+}
+
+define void @gather_load_4(i32* noalias nocapture %t0, i32* noalias nocapture readonly %t1) {
+; SSE-LABEL: @gather_load_4(
+; SSE-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1
+; SSE-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11
+; SSE-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2
+; SSE-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4
+; SSE-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3
+; SSE-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15
+; SSE-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4
+; SSE-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18
+; SSE-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5
+; SSE-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9
+; SSE-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6
+; SSE-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6
+; SSE-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7
+; SSE-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21
+; SSE-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]]
+; SSE-NEXT: [[T4:%.*]] = add i32 [[T3]], 1
+; SSE-NEXT: [[T8:%.*]] = add i32 [[T7]], 2
+; SSE-NEXT: [[T12:%.*]] = add i32 [[T11]], 3
+; SSE-NEXT: [[T16:%.*]] = add i32 [[T15]], 4
+; SSE-NEXT: [[T20:%.*]] = add i32 [[T19]], 1
+; SSE-NEXT: [[T24:%.*]] = add i32 [[T23]], 2
+; SSE-NEXT: [[T28:%.*]] = add i32 [[T27]], 3
+; SSE-NEXT: [[T32:%.*]] = add i32 [[T31]], 4
+; SSE-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]]
+; SSE-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]]
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @gather_load_4(
+; AVX-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1
+; AVX-NEXT: [[T6:%.*]] = getelementptr inbounds i32, i32* [[T1:%.*]], i64 11
+; AVX-NEXT: [[T9:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 2
+; AVX-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 4
+; AVX-NEXT: [[T13:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 3
+; AVX-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 15
+; AVX-NEXT: [[T17:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 4
+; AVX-NEXT: [[T18:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 18
+; AVX-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5
+; AVX-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9
+; AVX-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6
+; AVX-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6
+; AVX-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7
+; AVX-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21
+; AVX-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T7:%.*]] = load i32, i32* [[T6]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T19:%.*]] = load i32, i32* [[T18]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]]
+; AVX-NEXT: [[T4:%.*]] = add i32 [[T3]], 1
+; AVX-NEXT: [[T8:%.*]] = add i32 [[T7]], 2
+; AVX-NEXT: [[T12:%.*]] = add i32 [[T11]], 3
+; AVX-NEXT: [[T16:%.*]] = add i32 [[T15]], 4
+; AVX-NEXT: [[T20:%.*]] = add i32 [[T19]], 1
+; AVX-NEXT: [[T24:%.*]] = add i32 [[T23]], 2
+; AVX-NEXT: [[T28:%.*]] = add i32 [[T27]], 3
+; AVX-NEXT: [[T32:%.*]] = add i32 [[T31]], 4
+; AVX-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T8]], i32* [[T5]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T12]], i32* [[T9]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T16]], i32* [[T13]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T20]], i32* [[T17]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]]
+; AVX-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]]
+; AVX-NEXT: ret void
+;
+; AVX2-LABEL: @gather_load_4(
+; AVX2-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1
+; AVX2-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> undef, i32* [[T1:%.*]], i32 0
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> <i64 11, i64 4, i64 15, i64 18>
+; AVX2-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5
+; AVX2-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9
+; AVX2-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6
+; AVX2-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6
+; AVX2-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7
+; AVX2-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21
+; AVX2-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0]]
+; AVX2-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[T4:%.*]] = add i32 [[T3]], 1
+; AVX2-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], <i32 2, i32 3, i32 4, i32 1>
+; AVX2-NEXT: [[T24:%.*]] = add i32 [[T23]], 2
+; AVX2-NEXT: [[T28:%.*]] = add i32 [[T27]], 3
+; AVX2-NEXT: [[T32:%.*]] = add i32 [[T31]], 4
+; AVX2-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]]
+; AVX2-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>*
+; AVX2-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]]
+; AVX2-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]]
+; AVX2-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]]
+; AVX2-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]]
+; AVX2-NEXT: ret void
+;
+; AVX512-LABEL: @gather_load_4(
+; AVX512-NEXT: [[T5:%.*]] = getelementptr inbounds i32, i32* [[T0:%.*]], i64 1
+; AVX512-NEXT: [[TMP1:%.*]] = insertelement <4 x i32*> undef, i32* [[T1:%.*]], i32 0
+; AVX512-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32*> [[TMP1]], <4 x i32*> undef, <4 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP3:%.*]] = getelementptr i32, <4 x i32*> [[TMP2]], <4 x i64> <i64 11, i64 4, i64 15, i64 18>
+; AVX512-NEXT: [[T21:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 5
+; AVX512-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 9
+; AVX512-NEXT: [[T25:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 6
+; AVX512-NEXT: [[T26:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 6
+; AVX512-NEXT: [[T29:%.*]] = getelementptr inbounds i32, i32* [[T0]], i64 7
+; AVX512-NEXT: [[T30:%.*]] = getelementptr inbounds i32, i32* [[T1]], i64 21
+; AVX512-NEXT: [[T3:%.*]] = load i32, i32* [[T1]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> [[TMP3]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), [[TBAA0]]
+; AVX512-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[T27:%.*]] = load i32, i32* [[T26]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[T31:%.*]] = load i32, i32* [[T30]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[T4:%.*]] = add i32 [[T3]], 1
+; AVX512-NEXT: [[TMP5:%.*]] = add <4 x i32> [[TMP4]], <i32 2, i32 3, i32 4, i32 1>
+; AVX512-NEXT: [[T24:%.*]] = add i32 [[T23]], 2
+; AVX512-NEXT: [[T28:%.*]] = add i32 [[T27]], 3
+; AVX512-NEXT: [[T32:%.*]] = add i32 [[T31]], 4
+; AVX512-NEXT: store i32 [[T4]], i32* [[T0]], align 4, [[TBAA0]]
+; AVX512-NEXT: [[TMP6:%.*]] = bitcast i32* [[T5]] to <4 x i32>*
+; AVX512-NEXT: store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4, [[TBAA0]]
+; AVX512-NEXT: store i32 [[T24]], i32* [[T21]], align 4, [[TBAA0]]
+; AVX512-NEXT: store i32 [[T28]], i32* [[T25]], align 4, [[TBAA0]]
+; AVX512-NEXT: store i32 [[T32]], i32* [[T29]], align 4, [[TBAA0]]
+; AVX512-NEXT: ret void
+;
+ %t5 = getelementptr inbounds i32, i32* %t0, i64 1
+ %t6 = getelementptr inbounds i32, i32* %t1, i64 11
+ %t9 = getelementptr inbounds i32, i32* %t0, i64 2
+ %t10 = getelementptr inbounds i32, i32* %t1, i64 4
+ %t13 = getelementptr inbounds i32, i32* %t0, i64 3
+ %t14 = getelementptr inbounds i32, i32* %t1, i64 15
+ %t17 = getelementptr inbounds i32, i32* %t0, i64 4
+ %t18 = getelementptr inbounds i32, i32* %t1, i64 18
+ %t21 = getelementptr inbounds i32, i32* %t0, i64 5
+ %t22 = getelementptr inbounds i32, i32* %t1, i64 9
+ %t25 = getelementptr inbounds i32, i32* %t0, i64 6
+ %t26 = getelementptr inbounds i32, i32* %t1, i64 6
+ %t29 = getelementptr inbounds i32, i32* %t0, i64 7
+ %t30 = getelementptr inbounds i32, i32* %t1, i64 21
+
+ %t3 = load i32, i32* %t1, align 4, !tbaa !2
+ %t7 = load i32, i32* %t6, align 4, !tbaa !2
+ %t11 = load i32, i32* %t10, align 4, !tbaa !2
+ %t15 = load i32, i32* %t14, align 4, !tbaa !2
+ %t19 = load i32, i32* %t18, align 4, !tbaa !2
+ %t23 = load i32, i32* %t22, align 4, !tbaa !2
+ %t27 = load i32, i32* %t26, align 4, !tbaa !2
+ %t31 = load i32, i32* %t30, align 4, !tbaa !2
+
+ %t4 = add i32 %t3, 1
+ %t8 = add i32 %t7, 2
+ %t12 = add i32 %t11, 3
+ %t16 = add i32 %t15, 4
+ %t20 = add i32 %t19, 1
+ %t24 = add i32 %t23, 2
+ %t28 = add i32 %t27, 3
+ %t32 = add i32 %t31, 4
+
+ store i32 %t4, i32* %t0, align 4, !tbaa !2
+ store i32 %t8, i32* %t5, align 4, !tbaa !2
+ store i32 %t12, i32* %t9, align 4, !tbaa !2
+ store i32 %t16, i32* %t13, align 4, !tbaa !2
+ store i32 %t20, i32* %t17, align 4, !tbaa !2
+ store i32 %t24, i32* %t21, align 4, !tbaa !2
+ store i32 %t28, i32* %t25, align 4, !tbaa !2
+ store i32 %t32, i32* %t29, align 4, !tbaa !2
+
+ ret void
+}
+
+
+define void @gather_load_div(float* noalias nocapture %0, float* noalias nocapture readonly %1) {
+; SSE-LABEL: @gather_load_div(
+; SSE-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1:%.*]], i64 10
+; SSE-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 3
+; SSE-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 14
+; SSE-NEXT: [[TMP6:%.*]] = insertelement <4 x float*> undef, float* [[TMP1]], i32 0
+; SSE-NEXT: [[TMP7:%.*]] = insertelement <4 x float*> [[TMP6]], float* [[TMP3]], i32 1
+; SSE-NEXT: [[TMP8:%.*]] = insertelement <4 x float*> [[TMP7]], float* [[TMP4]], i32 2
+; SSE-NEXT: [[TMP9:%.*]] = insertelement <4 x float*> [[TMP8]], float* [[TMP5]], i32 3
+; SSE-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP9]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef), [[TBAA0]]
+; SSE-NEXT: [[TMP11:%.*]] = shufflevector <4 x float*> [[TMP6]], <4 x float*> undef, <4 x i32> zeroinitializer
+; SSE-NEXT: [[TMP12:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> <i64 4, i64 13, i64 11, i64 44>
+; SSE-NEXT: [[TMP13:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP12]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef), [[TBAA0]]
+; SSE-NEXT: [[TMP14:%.*]] = fdiv <4 x float> [[TMP10]], [[TMP13]]
+; SSE-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, float* [[TMP0:%.*]], i64 4
+; SSE-NEXT: [[TMP16:%.*]] = bitcast float* [[TMP0]] to <4 x float>*
+; SSE-NEXT: store <4 x float> [[TMP14]], <4 x float>* [[TMP16]], align 4, [[TBAA0]]
+; SSE-NEXT: [[TMP17:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> <i64 17, i64 8, i64 5, i64 20>
+; SSE-NEXT: [[TMP18:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP17]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef), [[TBAA0]]
+; SSE-NEXT: [[TMP19:%.*]] = getelementptr float, <4 x float*> [[TMP11]], <4 x i64> <i64 33, i64 30, i64 27, i64 23>
+; SSE-NEXT: [[TMP20:%.*]] = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> [[TMP19]], i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef), [[TBAA0]]
+; SSE-NEXT: [[TMP21:%.*]] = fdiv <4 x float> [[TMP18]], [[TMP20]]
+; SSE-NEXT: [[TMP22:%.*]] = bitcast float* [[TMP15]] to <4 x float>*
+; SSE-NEXT: store <4 x float> [[TMP21]], <4 x float>* [[TMP22]], align 4, [[TBAA0]]
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @gather_load_div(
+; AVX-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1:%.*]], i64 10
+; AVX-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 3
+; AVX-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 14
+; AVX-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 17
+; AVX-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 8
+; AVX-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 5
+; AVX-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 20
+; AVX-NEXT: [[TMP10:%.*]] = insertelement <8 x float*> undef, float* [[TMP1]], i32 0
+; AVX-NEXT: [[TMP11:%.*]] = insertelement <8 x float*> [[TMP10]], float* [[TMP3]], i32 1
+; AVX-NEXT: [[TMP12:%.*]] = insertelement <8 x float*> [[TMP11]], float* [[TMP4]], i32 2
+; AVX-NEXT: [[TMP13:%.*]] = insertelement <8 x float*> [[TMP12]], float* [[TMP5]], i32 3
+; AVX-NEXT: [[TMP14:%.*]] = insertelement <8 x float*> [[TMP13]], float* [[TMP6]], i32 4
+; AVX-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5
+; AVX-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6
+; AVX-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7
+; AVX-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer
+; AVX-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> <i64 4, i64 13, i64 11, i64 44, i64 33, i64 30, i64 27, i64 23>
+; AVX-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]]
+; AVX-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>*
+; AVX-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]]
+; AVX-NEXT: ret void
+;
+; AVX2-LABEL: @gather_load_div(
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1:%.*]], i64 10
+; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 3
+; AVX2-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 14
+; AVX2-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 17
+; AVX2-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 8
+; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 5
+; AVX2-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 20
+; AVX2-NEXT: [[TMP10:%.*]] = insertelement <8 x float*> undef, float* [[TMP1]], i32 0
+; AVX2-NEXT: [[TMP11:%.*]] = insertelement <8 x float*> [[TMP10]], float* [[TMP3]], i32 1
+; AVX2-NEXT: [[TMP12:%.*]] = insertelement <8 x float*> [[TMP11]], float* [[TMP4]], i32 2
+; AVX2-NEXT: [[TMP13:%.*]] = insertelement <8 x float*> [[TMP12]], float* [[TMP5]], i32 3
+; AVX2-NEXT: [[TMP14:%.*]] = insertelement <8 x float*> [[TMP13]], float* [[TMP6]], i32 4
+; AVX2-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5
+; AVX2-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6
+; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7
+; AVX2-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX2-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer
+; AVX2-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> <i64 4, i64 13, i64 11, i64 44, i64 33, i64 30, i64 27, i64 23>
+; AVX2-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX2-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]]
+; AVX2-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>*
+; AVX2-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]]
+; AVX2-NEXT: ret void
+;
+; AVX512-LABEL: @gather_load_div(
+; AVX512-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, float* [[TMP1:%.*]], i64 10
+; AVX512-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 3
+; AVX512-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 14
+; AVX512-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 17
+; AVX512-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 8
+; AVX512-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 5
+; AVX512-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 20
+; AVX512-NEXT: [[TMP10:%.*]] = insertelement <8 x float*> undef, float* [[TMP1]], i32 0
+; AVX512-NEXT: [[TMP11:%.*]] = insertelement <8 x float*> [[TMP10]], float* [[TMP3]], i32 1
+; AVX512-NEXT: [[TMP12:%.*]] = insertelement <8 x float*> [[TMP11]], float* [[TMP4]], i32 2
+; AVX512-NEXT: [[TMP13:%.*]] = insertelement <8 x float*> [[TMP12]], float* [[TMP5]], i32 3
+; AVX512-NEXT: [[TMP14:%.*]] = insertelement <8 x float*> [[TMP13]], float* [[TMP6]], i32 4
+; AVX512-NEXT: [[TMP15:%.*]] = insertelement <8 x float*> [[TMP14]], float* [[TMP7]], i32 5
+; AVX512-NEXT: [[TMP16:%.*]] = insertelement <8 x float*> [[TMP15]], float* [[TMP8]], i32 6
+; AVX512-NEXT: [[TMP17:%.*]] = insertelement <8 x float*> [[TMP16]], float* [[TMP9]], i32 7
+; AVX512-NEXT: [[TMP18:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP17]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX512-NEXT: [[TMP19:%.*]] = shufflevector <8 x float*> [[TMP10]], <8 x float*> undef, <8 x i32> zeroinitializer
+; AVX512-NEXT: [[TMP20:%.*]] = getelementptr float, <8 x float*> [[TMP19]], <8 x i64> <i64 4, i64 13, i64 11, i64 44, i64 33, i64 30, i64 27, i64 23>
+; AVX512-NEXT: [[TMP21:%.*]] = call <8 x float> @llvm.masked.gather.v8f32.v8p0f32(<8 x float*> [[TMP20]], i32 4, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x float> undef), [[TBAA0]]
+; AVX512-NEXT: [[TMP22:%.*]] = fdiv <8 x float> [[TMP18]], [[TMP21]]
+; AVX512-NEXT: [[TMP23:%.*]] = bitcast float* [[TMP0:%.*]] to <8 x float>*
+; AVX512-NEXT: store <8 x float> [[TMP22]], <8 x float>* [[TMP23]], align 4, [[TBAA0]]
+; AVX512-NEXT: ret void
+;
+ %3 = load float, float* %1, align 4, !tbaa !2
+ %4 = getelementptr inbounds float, float* %1, i64 4
+ %5 = load float, float* %4, align 4, !tbaa !2
+ %6 = fdiv float %3, %5
+ %7 = getelementptr inbounds float, float* %0, i64 1
+ store float %6, float* %0, align 4, !tbaa !2
+ %8 = getelementptr inbounds float, float* %1, i64 10
+ %9 = load float, float* %8, align 4, !tbaa !2
+ %10 = getelementptr inbounds float, float* %1, i64 13
+ %11 = load float, float* %10, align 4, !tbaa !2
+ %12 = fdiv float %9, %11
+ %13 = getelementptr inbounds float, float* %0, i64 2
+ store float %12, float* %7, align 4, !tbaa !2
+ %14 = getelementptr inbounds float, float* %1, i64 3
+ %15 = load float, float* %14, align 4, !tbaa !2
+ %16 = getelementptr inbounds float, float* %1, i64 11
+ %17 = load float, float* %16, align 4, !tbaa !2
+ %18 = fdiv float %15, %17
+ %19 = getelementptr inbounds float, float* %0, i64 3
+ store float %18, float* %13, align 4, !tbaa !2
+ %20 = getelementptr inbounds float, float* %1, i64 14
+ %21 = load float, float* %20, align 4, !tbaa !2
+ %22 = getelementptr inbounds float, float* %1, i64 44
+ %23 = load float, float* %22, align 4, !tbaa !2
+ %24 = fdiv float %21, %23
+ %25 = getelementptr inbounds float, float* %0, i64 4
+ store float %24, float* %19, align 4, !tbaa !2
+ %26 = getelementptr inbounds float, float* %1, i64 17
+ %27 = load float, float* %26, align 4, !tbaa !2
+ %28 = getelementptr inbounds float, float* %1, i64 33
+ %29 = load float, float* %28, align 4, !tbaa !2
+ %30 = fdiv float %27, %29
+ %31 = getelementptr inbounds float, float* %0, i64 5
+ store float %30, float* %25, align 4, !tbaa !2
+ %32 = getelementptr inbounds float, float* %1, i64 8
+ %33 = load float, float* %32, align 4, !tbaa !2
+ %34 = getelementptr inbounds float, float* %1, i64 30
+ %35 = load float, float* %34, align 4, !tbaa !2
+ %36 = fdiv float %33, %35
+ %37 = getelementptr inbounds float, float* %0, i64 6
+ store float %36, float* %31, align 4, !tbaa !2
+ %38 = getelementptr inbounds float, float* %1, i64 5
+ %39 = load float, float* %38, align 4, !tbaa !2
+ %40 = getelementptr inbounds float, float* %1, i64 27
+ %41 = load float, float* %40, align 4, !tbaa !2
+ %42 = fdiv float %39, %41
+ %43 = getelementptr inbounds float, float* %0, i64 7
+ store float %42, float* %37, align 4, !tbaa !2
+ %44 = getelementptr inbounds float, float* %1, i64 20
+ %45 = load float, float* %44, align 4, !tbaa !2
+ %46 = getelementptr inbounds float, float* %1, i64 23
+ %47 = load float, float* %46, align 4, !tbaa !2
+ %48 = fdiv float %45, %47
+ store float %48, float* %43, align 4, !tbaa !2
+ ret void
+}
+
+!2 = !{!3, !3, i64 0}
+!3 = !{!"short", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll
new file mode 100644
index 000000000000..d1690f4b3ac9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sext-inseltpoison.ll
@@ -0,0 +1,1039 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=SSE,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+
+;
+; vXi8
+;
+
+define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
+; SSE-LABEL: @loadext_2i8_to_2i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i8_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %x0 = sext i8 %i0 to i64
+ %x1 = sext i8 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = sext i8 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = sext i8 [[I3]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i32> [[V3]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %x0 = sext i8 %i0 to i32
+ %x1 = sext i8 %i1 to i32
+ %x2 = sext i8 %i2 to i32
+ %x3 = sext i8 %i3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
+; SSE-LABEL: @loadext_4i8_to_4i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SSE-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SSE-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i64
+; SSE-NEXT: [[X2:%.*]] = sext i8 [[I2]] to i64
+; SSE-NEXT: [[X3:%.*]] = sext i8 [[I3]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %x0 = sext i8 %i0 to i64
+ %x1 = sext i8 %i1 to i64
+ %x2 = sext i8 %i2 to i64
+ %x3 = sext i8 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
+
+define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
+; SSE2-LABEL: @loadext_8i8_to_8i16(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i16> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i16(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i16
+; SLM-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i16
+; SLM-NEXT: [[X2:%.*]] = sext i8 [[I2]] to i16
+; SLM-NEXT: [[X3:%.*]] = sext i8 [[I3]] to i16
+; SLM-NEXT: [[X4:%.*]] = sext i8 [[I4]] to i16
+; SLM-NEXT: [[X5:%.*]] = sext i8 [[I5]] to i16
+; SLM-NEXT: [[X6:%.*]] = sext i8 [[I6]] to i16
+; SLM-NEXT: [[X7:%.*]] = sext i8 [[I7]] to i16
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i16> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i16(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i16> [[V7]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %x0 = sext i8 %i0 to i16
+ %x1 = sext i8 %i1 to i16
+ %x2 = sext i8 %i2 to i16
+ %x3 = sext i8 %i3 to i16
+ %x4 = sext i8 %i4 to i16
+ %x5 = sext i8 %i5 to i16
+ %x6 = sext i8 %i6 to i16
+ %x7 = sext i8 %i7 to i16
+ %v0 = insertelement <8 x i16> poison, i16 %x0, i32 0
+ %v1 = insertelement <8 x i16> %v0, i16 %x1, i32 1
+ %v2 = insertelement <8 x i16> %v1, i16 %x2, i32 2
+ %v3 = insertelement <8 x i16> %v2, i16 %x3, i32 3
+ %v4 = insertelement <8 x i16> %v3, i16 %x4, i32 4
+ %v5 = insertelement <8 x i16> %v4, i16 %x5, i32 5
+ %v6 = insertelement <8 x i16> %v5, i16 %x6, i32 6
+ %v7 = insertelement <8 x i16> %v6, i16 %x7, i32 7
+ ret <8 x i16> %v7
+}
+
+define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
+; SSE2-LABEL: @loadext_8i8_to_8i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = sext i8 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = sext i8 [[I3]] to i32
+; SLM-NEXT: [[X4:%.*]] = sext i8 [[I4]] to i32
+; SLM-NEXT: [[X5:%.*]] = sext i8 [[I5]] to i32
+; SLM-NEXT: [[X6:%.*]] = sext i8 [[I6]] to i32
+; SLM-NEXT: [[X7:%.*]] = sext i8 [[I7]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i32> [[V7]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %x0 = sext i8 %i0 to i32
+ %x1 = sext i8 %i1 to i32
+ %x2 = sext i8 %i2 to i32
+ %x3 = sext i8 %i3 to i32
+ %x4 = sext i8 %i4 to i32
+ %x5 = sext i8 %i5 to i32
+ %x6 = sext i8 %i6 to i32
+ %x7 = sext i8 %i7 to i32
+ %v0 = insertelement <8 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <8 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <8 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <8 x i32> %v2, i32 %x3, i32 3
+ %v4 = insertelement <8 x i32> %v3, i32 %x4, i32 4
+ %v5 = insertelement <8 x i32> %v4, i32 %x5, i32 5
+ %v6 = insertelement <8 x i32> %v5, i32 %x6, i32 6
+ %v7 = insertelement <8 x i32> %v6, i32 %x7, i32 7
+ ret <8 x i32> %v7
+}
+
+define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
+; SSE2-LABEL: @loadext_16i8_to_16i16(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SSE2-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SSE2-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SSE2-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SSE2-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SSE2-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SSE2-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SSE2-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT: [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; SSE2-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; SSE2-NEXT: [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; SSE2-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; SSE2-NEXT: [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; SSE2-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; SSE2-NEXT: [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; SSE2-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; SSE2-NEXT: [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; SSE2-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; SSE2-NEXT: [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; SSE2-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; SSE2-NEXT: [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; SSE2-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; SSE2-NEXT: [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; SSE2-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; SSE2-NEXT: ret <16 x i16> [[V15]]
+;
+; SLM-LABEL: @loadext_16i8_to_16i16(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SLM-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SLM-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SLM-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SLM-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SLM-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SLM-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SLM-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[I8:%.*]] = load i8, i8* [[P8]], align 1
+; SLM-NEXT: [[I9:%.*]] = load i8, i8* [[P9]], align 1
+; SLM-NEXT: [[I10:%.*]] = load i8, i8* [[P10]], align 1
+; SLM-NEXT: [[I11:%.*]] = load i8, i8* [[P11]], align 1
+; SLM-NEXT: [[I12:%.*]] = load i8, i8* [[P12]], align 1
+; SLM-NEXT: [[I13:%.*]] = load i8, i8* [[P13]], align 1
+; SLM-NEXT: [[I14:%.*]] = load i8, i8* [[P14]], align 1
+; SLM-NEXT: [[I15:%.*]] = load i8, i8* [[P15]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i8 [[I0]] to i16
+; SLM-NEXT: [[X1:%.*]] = sext i8 [[I1]] to i16
+; SLM-NEXT: [[X2:%.*]] = sext i8 [[I2]] to i16
+; SLM-NEXT: [[X3:%.*]] = sext i8 [[I3]] to i16
+; SLM-NEXT: [[X4:%.*]] = sext i8 [[I4]] to i16
+; SLM-NEXT: [[X5:%.*]] = sext i8 [[I5]] to i16
+; SLM-NEXT: [[X6:%.*]] = sext i8 [[I6]] to i16
+; SLM-NEXT: [[X7:%.*]] = sext i8 [[I7]] to i16
+; SLM-NEXT: [[X8:%.*]] = sext i8 [[I8]] to i16
+; SLM-NEXT: [[X9:%.*]] = sext i8 [[I9]] to i16
+; SLM-NEXT: [[X10:%.*]] = sext i8 [[I10]] to i16
+; SLM-NEXT: [[X11:%.*]] = sext i8 [[I11]] to i16
+; SLM-NEXT: [[X12:%.*]] = sext i8 [[I12]] to i16
+; SLM-NEXT: [[X13:%.*]] = sext i8 [[I13]] to i16
+; SLM-NEXT: [[X14:%.*]] = sext i8 [[I14]] to i16
+; SLM-NEXT: [[X15:%.*]] = sext i8 [[I15]] to i16
+; SLM-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[X8]], i32 8
+; SLM-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[X9]], i32 9
+; SLM-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[X10]], i32 10
+; SLM-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[X11]], i32 11
+; SLM-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[X12]], i32 12
+; SLM-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[X13]], i32 13
+; SLM-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[X14]], i32 14
+; SLM-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[X15]], i32 15
+; SLM-NEXT: ret <16 x i16> [[V15]]
+;
+; AVX-LABEL: @loadext_16i8_to_16i16(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; AVX-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; AVX-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; AVX-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; AVX-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; AVX-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; AVX-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; AVX-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <16 x i8> [[TMP2]] to <16 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT: [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; AVX-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; AVX-NEXT: [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; AVX-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; AVX-NEXT: [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; AVX-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; AVX-NEXT: [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; AVX-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; AVX-NEXT: [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; AVX-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; AVX-NEXT: [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; AVX-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; AVX-NEXT: [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; AVX-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; AVX-NEXT: [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; AVX-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; AVX-NEXT: ret <16 x i16> [[V15]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %p8 = getelementptr inbounds i8, i8* %p0, i64 8
+ %p9 = getelementptr inbounds i8, i8* %p0, i64 9
+ %p10 = getelementptr inbounds i8, i8* %p0, i64 10
+ %p11 = getelementptr inbounds i8, i8* %p0, i64 11
+ %p12 = getelementptr inbounds i8, i8* %p0, i64 12
+ %p13 = getelementptr inbounds i8, i8* %p0, i64 13
+ %p14 = getelementptr inbounds i8, i8* %p0, i64 14
+ %p15 = getelementptr inbounds i8, i8* %p0, i64 15
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %i8 = load i8, i8* %p8, align 1
+ %i9 = load i8, i8* %p9, align 1
+ %i10 = load i8, i8* %p10, align 1
+ %i11 = load i8, i8* %p11, align 1
+ %i12 = load i8, i8* %p12, align 1
+ %i13 = load i8, i8* %p13, align 1
+ %i14 = load i8, i8* %p14, align 1
+ %i15 = load i8, i8* %p15, align 1
+ %x0 = sext i8 %i0 to i16
+ %x1 = sext i8 %i1 to i16
+ %x2 = sext i8 %i2 to i16
+ %x3 = sext i8 %i3 to i16
+ %x4 = sext i8 %i4 to i16
+ %x5 = sext i8 %i5 to i16
+ %x6 = sext i8 %i6 to i16
+ %x7 = sext i8 %i7 to i16
+ %x8 = sext i8 %i8 to i16
+ %x9 = sext i8 %i9 to i16
+ %x10 = sext i8 %i10 to i16
+ %x11 = sext i8 %i11 to i16
+ %x12 = sext i8 %i12 to i16
+ %x13 = sext i8 %i13 to i16
+ %x14 = sext i8 %i14 to i16
+ %x15 = sext i8 %i15 to i16
+ %v0 = insertelement <16 x i16> poison, i16 %x0, i32 0
+ %v1 = insertelement <16 x i16> %v0, i16 %x1, i32 1
+ %v2 = insertelement <16 x i16> %v1, i16 %x2, i32 2
+ %v3 = insertelement <16 x i16> %v2, i16 %x3, i32 3
+ %v4 = insertelement <16 x i16> %v3, i16 %x4, i32 4
+ %v5 = insertelement <16 x i16> %v4, i16 %x5, i32 5
+ %v6 = insertelement <16 x i16> %v5, i16 %x6, i32 6
+ %v7 = insertelement <16 x i16> %v6, i16 %x7, i32 7
+ %v8 = insertelement <16 x i16> %v7, i16 %x8, i32 8
+ %v9 = insertelement <16 x i16> %v8, i16 %x9, i32 9
+ %v10 = insertelement <16 x i16> %v9, i16 %x10, i32 10
+ %v11 = insertelement <16 x i16> %v10, i16 %x11, i32 11
+ %v12 = insertelement <16 x i16> %v11, i16 %x12, i32 12
+ %v13 = insertelement <16 x i16> %v12, i16 %x13, i32 13
+ %v14 = insertelement <16 x i16> %v13, i16 %x14, i32 14
+ %v15 = insertelement <16 x i16> %v14, i16 %x15, i32 15
+ ret <16 x i16> %v15
+}
+
+;
+; vXi16
+;
+
+define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
+; SSE-LABEL: @loadext_2i16_to_2i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i16_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %x0 = sext i16 %i0 to i64
+ %x1 = sext i16 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
+; SSE2-LABEL: @loadext_4i16_to_4i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i16 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = sext i16 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = sext i16 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = sext i16 [[I3]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i32> [[V3]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %x0 = sext i16 %i0 to i32
+ %x1 = sext i16 %i1 to i32
+ %x2 = sext i16 %i2 to i32
+ %x3 = sext i16 %i3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
+; SSE-LABEL: @loadext_4i16_to_4i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SSE-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SSE-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i16 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i16 [[I1]] to i64
+; SSE-NEXT: [[X2:%.*]] = sext i16 [[I2]] to i64
+; SSE-NEXT: [[X3:%.*]] = sext i16 [[I3]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %x0 = sext i16 %i0 to i64
+ %x1 = sext i16 %i1 to i64
+ %x2 = sext i16 %i2 to i64
+ %x3 = sext i16 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
+
+define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
+; SSE2-LABEL: @loadext_8i16_to_8i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i16_to_8i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i16, i16* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i16, i16* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i16, i16* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i16, i16* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = sext i16 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = sext i16 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = sext i16 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = sext i16 [[I3]] to i32
+; SLM-NEXT: [[X4:%.*]] = sext i16 [[I4]] to i32
+; SLM-NEXT: [[X5:%.*]] = sext i16 [[I5]] to i32
+; SLM-NEXT: [[X6:%.*]] = sext i16 [[I6]] to i32
+; SLM-NEXT: [[X7:%.*]] = sext i16 [[I7]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i16_to_8i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <8 x i16> [[TMP2]] to <8 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i32> [[V7]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %p4 = getelementptr inbounds i16, i16* %p0, i64 4
+ %p5 = getelementptr inbounds i16, i16* %p0, i64 5
+ %p6 = getelementptr inbounds i16, i16* %p0, i64 6
+ %p7 = getelementptr inbounds i16, i16* %p0, i64 7
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %i4 = load i16, i16* %p4, align 1
+ %i5 = load i16, i16* %p5, align 1
+ %i6 = load i16, i16* %p6, align 1
+ %i7 = load i16, i16* %p7, align 1
+ %x0 = sext i16 %i0 to i32
+ %x1 = sext i16 %i1 to i32
+ %x2 = sext i16 %i2 to i32
+ %x3 = sext i16 %i3 to i32
+ %x4 = sext i16 %i4 to i32
+ %x5 = sext i16 %i5 to i32
+ %x6 = sext i16 %i6 to i32
+ %x7 = sext i16 %i7 to i32
+ %v0 = insertelement <8 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <8 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <8 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <8 x i32> %v2, i32 %x3, i32 3
+ %v4 = insertelement <8 x i32> %v3, i32 %x4, i32 4
+ %v5 = insertelement <8 x i32> %v4, i32 %x5, i32 5
+ %v6 = insertelement <8 x i32> %v5, i32 %x6, i32 6
+ %v7 = insertelement <8 x i32> %v6, i32 %x7, i32 7
+ ret <8 x i32> %v7
+}
+
+;
+; vXi32
+;
+
+define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
+; SSE-LABEL: @loadext_2i32_to_2i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT: [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i32_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+ %i0 = load i32, i32* %p0, align 1
+ %i1 = load i32, i32* %p1, align 1
+ %x0 = sext i32 %i0 to i64
+ %x1 = sext i32 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
+; SSE-LABEL: @loadext_4i32_to_4i64(
+; SSE-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE-NEXT: [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE-NEXT: [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE-NEXT: [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SSE-NEXT: [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SSE-NEXT: [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SSE-NEXT: [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SSE-NEXT: [[X0:%.*]] = sext i32 [[I0]] to i64
+; SSE-NEXT: [[X1:%.*]] = sext i32 [[I1]] to i64
+; SSE-NEXT: [[X2:%.*]] = sext i32 [[I2]] to i64
+; SSE-NEXT: [[X3:%.*]] = sext i32 [[I3]] to i64
+; SSE-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SSE-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SSE-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SSE-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i32_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+ %p2 = getelementptr inbounds i32, i32* %p0, i64 2
+ %p3 = getelementptr inbounds i32, i32* %p0, i64 3
+ %i0 = load i32, i32* %p0, align 1
+ %i1 = load i32, i32* %p1, align 1
+ %i2 = load i32, i32* %p2, align 1
+ %i3 = load i32, i32* %p3, align 1
+ %x0 = sext i32 %i0 to i64
+ %x1 = sext i32 %i1 to i64
+ %x2 = sext i32 %i2 to i64
+ %x3 = sext i32 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sign-extend-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/sign-extend-inseltpoison.ll
new file mode 100644
index 000000000000..296486b49e14
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sign-extend-inseltpoison.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -o - -mtriple=x86_64-apple-macosx10.10.0 -mcpu=core2 | FileCheck %s
+
+define <4 x i32> @sign_extend_v_v(<4 x i16> %lhs) {
+; CHECK-LABEL: @sign_extend_v_v(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = sext <4 x i16> [[LHS:%.*]] to <4 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
+; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
+; CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <4 x i32> [[VECINIT]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
+; CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <4 x i32> [[VECINIT3]], i32 [[TMP3]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; CHECK-NEXT: [[VECINIT9:%.*]] = insertelement <4 x i32> [[VECINIT6]], i32 [[TMP4]], i32 3
+; CHECK-NEXT: ret <4 x i32> [[VECINIT9]]
+;
+entry:
+ %vecext = extractelement <4 x i16> %lhs, i32 0
+ %conv = sext i16 %vecext to i32
+ %vecinit = insertelement <4 x i32> poison, i32 %conv, i32 0
+ %vecext1 = extractelement <4 x i16> %lhs, i32 1
+ %conv2 = sext i16 %vecext1 to i32
+ %vecinit3 = insertelement <4 x i32> %vecinit, i32 %conv2, i32 1
+ %vecext4 = extractelement <4 x i16> %lhs, i32 2
+ %conv5 = sext i16 %vecext4 to i32
+ %vecinit6 = insertelement <4 x i32> %vecinit3, i32 %conv5, i32 2
+ %vecext7 = extractelement <4 x i16> %lhs, i32 3
+ %conv8 = sext i16 %vecext7 to i32
+ %vecinit9 = insertelement <4 x i32> %vecinit6, i32 %conv8, i32 3
+ ret <4 x i32> %vecinit9
+}
+
+define <4 x i16> @truncate_v_v(<4 x i32> %lhs) {
+; CHECK-LABEL: @truncate_v_v(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = trunc <4 x i32> [[LHS:%.*]] to <4 x i16>
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <4 x i16> [[TMP0]], i32 0
+; CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x i16> poison, i16 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i16> [[TMP0]], i32 1
+; CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <4 x i16> [[VECINIT]], i16 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i16> [[TMP0]], i32 2
+; CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <4 x i16> [[VECINIT3]], i16 [[TMP3]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i16> [[TMP0]], i32 3
+; CHECK-NEXT: [[VECINIT9:%.*]] = insertelement <4 x i16> [[VECINIT6]], i16 [[TMP4]], i32 3
+; CHECK-NEXT: ret <4 x i16> [[VECINIT9]]
+;
+entry:
+ %vecext = extractelement <4 x i32> %lhs, i32 0
+ %conv = trunc i32 %vecext to i16
+ %vecinit = insertelement <4 x i16> poison, i16 %conv, i32 0
+ %vecext1 = extractelement <4 x i32> %lhs, i32 1
+ %conv2 = trunc i32 %vecext1 to i16
+ %vecinit3 = insertelement <4 x i16> %vecinit, i16 %conv2, i32 1
+ %vecext4 = extractelement <4 x i32> %lhs, i32 2
+ %conv5 = trunc i32 %vecext4 to i16
+ %vecinit6 = insertelement <4 x i16> %vecinit3, i16 %conv5, i32 2
+ %vecext7 = extractelement <4 x i32> %lhs, i32 3
+ %conv8 = trunc i32 %vecext7 to i16
+ %vecinit9 = insertelement <4 x i16> %vecinit6, i16 %conv8, i32 3
+ ret <4 x i16> %vecinit9
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
new file mode 100644
index 000000000000..7718bcd727a3
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/sitofp-inseltpoison.ll
@@ -0,0 +1,1331 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=bdver1 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256NODQ
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=-prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skylake-avx512 -mattr=+prefer-256-bit -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX256 --check-prefix=AVX256DQ
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+ at src64 = common global [8 x i64] zeroinitializer, align 64
+ at src32 = common global [16 x i32] zeroinitializer, align 64
+ at src16 = common global [32 x i16] zeroinitializer, align 64
+ at src8 = common global [64 x i8] zeroinitializer, align 64
+
+ at dst64 = common global [8 x double] zeroinitializer, align 64
+ at dst32 = common global [16 x float] zeroinitializer, align 64
+
+;
+; SITOFP to vXf64
+;
+
+define void @sitofp_2i64_2f64() #0 {
+; SSE-LABEL: @sitofp_2i64_2f64(
+; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @sitofp_2i64_2f64(
+; AVX256NODQ-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_2i64_2f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x double>
+; AVX512-NEXT: store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @sitofp_2i64_2f64(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX256DQ-NEXT: [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x double>
+; AVX256DQ-NEXT: store <2 x double> [[TMP2]], <2 x double>* bitcast ([8 x double]* @dst64 to <2 x double>*), align 64
+; AVX256DQ-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %cvt0 = sitofp i64 %ld0 to double
+ %cvt1 = sitofp i64 %ld1 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ ret void
+}
+
+define void @sitofp_4i64_4f64() #0 {
+; SSE-LABEL: @sitofp_4i64_4f64(
+; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @sitofp_4i64_4f64(
+; AVX256NODQ-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; AVX256NODQ-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_4i64_4f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX512-NEXT: store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @sitofp_4i64_4f64(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT: [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX256DQ-NEXT: store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256DQ-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+ %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+ %cvt0 = sitofp i64 %ld0 to double
+ %cvt1 = sitofp i64 %ld1 to double
+ %cvt2 = sitofp i64 %ld2 to double
+ %cvt3 = sitofp i64 %ld3 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ ret void
+}
+
+define void @sitofp_8i64_8f64() #0 {
+; SSE-LABEL: @sitofp_8i64_8f64(
+; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT: [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; SSE-NEXT: [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT: [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; SSE-NEXT: [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i64 [[LD4]] to double
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i64 [[LD5]] to double
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i64 [[LD6]] to double
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i64 [[LD7]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT: store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT: store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @sitofp_8i64_8f64(
+; AVX256NODQ-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT: [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT: [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to double
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to double
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to double
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to double
+; AVX256NODQ-NEXT: [[CVT4:%.*]] = sitofp i64 [[LD4]] to double
+; AVX256NODQ-NEXT: [[CVT5:%.*]] = sitofp i64 [[LD5]] to double
+; AVX256NODQ-NEXT: [[CVT6:%.*]] = sitofp i64 [[LD6]] to double
+; AVX256NODQ-NEXT: [[CVT7:%.*]] = sitofp i64 [[LD7]] to double
+; AVX256NODQ-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT: store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT: store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_8i64_8f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x double>
+; AVX512-NEXT: store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @sitofp_8i64_8f64(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT: [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4) to <4 x i64>*), align 32
+; AVX256DQ-NEXT: [[TMP3:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x double>
+; AVX256DQ-NEXT: [[TMP4:%.*]] = sitofp <4 x i64> [[TMP2]] to <4 x double>
+; AVX256DQ-NEXT: store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256DQ-NEXT: store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256DQ-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+ %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+ %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+ %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+ %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+ %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+ %cvt0 = sitofp i64 %ld0 to double
+ %cvt1 = sitofp i64 %ld1 to double
+ %cvt2 = sitofp i64 %ld2 to double
+ %cvt3 = sitofp i64 %ld3 to double
+ %cvt4 = sitofp i64 %ld4 to double
+ %cvt5 = sitofp i64 %ld5 to double
+ %cvt6 = sitofp i64 %ld6 to double
+ %cvt7 = sitofp i64 %ld7 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+ store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+ store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+ store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+define void @sitofp_2i32_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i32_2f64(
+; CHECK-NEXT: [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; CHECK-NEXT: [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; CHECK-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+ %cvt0 = sitofp i32 %ld0 to double
+ %cvt1 = sitofp i32 %ld1 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ ret void
+}
+
+define void @sitofp_4i32_4f64() #0 {
+; SSE-LABEL: @sitofp_4i32_4f64(
+; SSE-NEXT: [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT: [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT: [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i32 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i32 [[LD3]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_4i32_4f64(
+; AVX-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX-NEXT: store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+ %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+ %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+ %cvt0 = sitofp i32 %ld0 to double
+ %cvt1 = sitofp i32 %ld1 to double
+ %cvt2 = sitofp i32 %ld2 to double
+ %cvt3 = sitofp i32 %ld3 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ ret void
+}
+
+define void @sitofp_8i32_8f64() #0 {
+; SSE-LABEL: @sitofp_8i32_8f64(
+; SSE-NEXT: [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT: [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+; SSE-NEXT: [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT: [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+; SSE-NEXT: [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT: [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+; SSE-NEXT: [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i32 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i32 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i32 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i32 [[LD3]] to double
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i32 [[LD4]] to double
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i32 [[LD5]] to double
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i32 [[LD6]] to double
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i32 [[LD7]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT: store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT: store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_8i32_8f64(
+; AVX256-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x double>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x double>
+; AVX256-NEXT: store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT: store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_8i32_8f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x double>
+; AVX512-NEXT: store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+ %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+ %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+ %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+ %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+ %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+ %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+ %cvt0 = sitofp i32 %ld0 to double
+ %cvt1 = sitofp i32 %ld1 to double
+ %cvt2 = sitofp i32 %ld2 to double
+ %cvt3 = sitofp i32 %ld3 to double
+ %cvt4 = sitofp i32 %ld4 to double
+ %cvt5 = sitofp i32 %ld5 to double
+ %cvt6 = sitofp i32 %ld6 to double
+ %cvt7 = sitofp i32 %ld7 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+ store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+ store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+ store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+define void @sitofp_2i16_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i16_2f64(
+; CHECK-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; CHECK-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; CHECK-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+ %cvt0 = sitofp i16 %ld0 to double
+ %cvt1 = sitofp i16 %ld1 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ ret void
+}
+
+define void @sitofp_4i16_4f64() #0 {
+; SSE-LABEL: @sitofp_4i16_4f64(
+; SSE-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i16 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i16 [[LD3]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_4i16_4f64(
+; AVX-NEXT: [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX-NEXT: store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+ %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+ %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+ %cvt0 = sitofp i16 %ld0 to double
+ %cvt1 = sitofp i16 %ld1 to double
+ %cvt2 = sitofp i16 %ld2 to double
+ %cvt3 = sitofp i16 %ld3 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ ret void
+}
+
+define void @sitofp_8i16_8f64() #0 {
+; SSE-LABEL: @sitofp_8i16_8f64(
+; SSE-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT: [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT: [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT: [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT: [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i16 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i16 [[LD3]] to double
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i16 [[LD4]] to double
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i16 [[LD5]] to double
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i16 [[LD6]] to double
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i16 [[LD7]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT: store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT: store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_8i16_8f64(
+; AVX256-NEXT: [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4) to <4 x i16>*), align 8
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x double>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <4 x i16> [[TMP2]] to <4 x double>
+; AVX256-NEXT: store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT: store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_8i16_8f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x double>
+; AVX512-NEXT: store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+ %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+ %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+ %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+ %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+ %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+ %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+ %cvt0 = sitofp i16 %ld0 to double
+ %cvt1 = sitofp i16 %ld1 to double
+ %cvt2 = sitofp i16 %ld2 to double
+ %cvt3 = sitofp i16 %ld3 to double
+ %cvt4 = sitofp i16 %ld4 to double
+ %cvt5 = sitofp i16 %ld5 to double
+ %cvt6 = sitofp i16 %ld6 to double
+ %cvt7 = sitofp i16 %ld7 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+ store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+ store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+ store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+define void @sitofp_2i8_2f64() #0 {
+; CHECK-LABEL: @sitofp_2i8_2f64(
+; CHECK-NEXT: [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; CHECK-NEXT: [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; CHECK-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; CHECK-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+ %cvt0 = sitofp i8 %ld0 to double
+ %cvt1 = sitofp i8 %ld1 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ ret void
+}
+
+define void @sitofp_4i8_4f64() #0 {
+; SSE-LABEL: @sitofp_4i8_4f64(
+; SSE-NEXT: [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT: [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT: [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i8 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i8 [[LD3]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_4i8_4f64(
+; AVX-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX-NEXT: store <4 x double> [[TMP2]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+ %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+ %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+ %cvt0 = sitofp i8 %ld0 to double
+ %cvt1 = sitofp i8 %ld1 to double
+ %cvt2 = sitofp i8 %ld2 to double
+ %cvt3 = sitofp i8 %ld3 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ ret void
+}
+
+define void @sitofp_8i8_8f64() #0 {
+; SSE-LABEL: @sitofp_8i8_8f64(
+; SSE-NEXT: [[LD0:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+; SSE-NEXT: [[LD2:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+; SSE-NEXT: [[LD3:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+; SSE-NEXT: [[LD4:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+; SSE-NEXT: [[LD5:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+; SSE-NEXT: [[LD6:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+; SSE-NEXT: [[LD7:%.*]] = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i8 [[LD0]] to double
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i8 [[LD1]] to double
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i8 [[LD2]] to double
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i8 [[LD3]] to double
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i8 [[LD4]] to double
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i8 [[LD5]] to double
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i8 [[LD6]] to double
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i8 [[LD7]] to double
+; SSE-NEXT: store double [[CVT0]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+; SSE-NEXT: store double [[CVT1]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+; SSE-NEXT: store double [[CVT2]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+; SSE-NEXT: store double [[CVT3]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+; SSE-NEXT: store double [[CVT4]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+; SSE-NEXT: store double [[CVT5]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+; SSE-NEXT: store double [[CVT6]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+; SSE-NEXT: store double [[CVT7]], double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_8i8_8f64(
+; AVX256-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x double>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x double>
+; AVX256-NEXT: store <4 x double> [[TMP3]], <4 x double>* bitcast ([8 x double]* @dst64 to <4 x double>*), align 64
+; AVX256-NEXT: store <4 x double> [[TMP4]], <4 x double>* bitcast (double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4) to <4 x double>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_8i8_8f64(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x double>
+; AVX512-NEXT: store <8 x double> [[TMP2]], <8 x double>* bitcast ([8 x double]* @dst64 to <8 x double>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+ %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+ %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+ %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+ %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+ %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+ %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+ %cvt0 = sitofp i8 %ld0 to double
+ %cvt1 = sitofp i8 %ld1 to double
+ %cvt2 = sitofp i8 %ld2 to double
+ %cvt3 = sitofp i8 %ld3 to double
+ %cvt4 = sitofp i8 %ld4 to double
+ %cvt5 = sitofp i8 %ld5 to double
+ %cvt6 = sitofp i8 %ld6 to double
+ %cvt7 = sitofp i8 %ld7 to double
+ store double %cvt0, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 0), align 64
+ store double %cvt1, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 1), align 8
+ store double %cvt2, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 2), align 16
+ store double %cvt3, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 3), align 8
+ store double %cvt4, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 4), align 32
+ store double %cvt5, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 5), align 8
+ store double %cvt6, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 6), align 16
+ store double %cvt7, double* getelementptr inbounds ([8 x double], [8 x double]* @dst64, i32 0, i64 7), align 8
+ ret void
+}
+
+;
+; SITOFP to vXf32
+;
+
+define void @sitofp_2i64_2f32() #0 {
+; CHECK-LABEL: @sitofp_2i64_2f32(
+; CHECK-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; CHECK-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; CHECK-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; CHECK-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %cvt0 = sitofp i64 %ld0 to float
+ %cvt1 = sitofp i64 %ld1 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ ret void
+}
+
+define void @sitofp_4i64_4f32() #0 {
+; SSE-LABEL: @sitofp_4i64_4f32(
+; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; SSE-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @sitofp_4i64_4f32(
+; AVX256NODQ-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_4i64_4f32(
+; AVX512-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX512-NEXT: store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @sitofp_4i64_4f32(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @src64 to <4 x i64>*), align 64
+; AVX256DQ-NEXT: [[TMP2:%.*]] = sitofp <4 x i64> [[TMP1]] to <4 x float>
+; AVX256DQ-NEXT: store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX256DQ-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+ %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+ %cvt0 = sitofp i64 %ld0 to float
+ %cvt1 = sitofp i64 %ld1 to float
+ %cvt2 = sitofp i64 %ld2 to float
+ %cvt3 = sitofp i64 %ld3 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ ret void
+}
+
+define void @sitofp_8i64_8f32() #0 {
+; SSE-LABEL: @sitofp_8i64_8f32(
+; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; SSE-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT: [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; SSE-NEXT: [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT: [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; SSE-NEXT: [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i64 [[LD4]] to float
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i64 [[LD5]] to float
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i64 [[LD6]] to float
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i64 [[LD7]] to float
+; SSE-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT: store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT: store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT: store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT: store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT: ret void
+;
+; AVX256NODQ-LABEL: @sitofp_8i64_8f32(
+; AVX256NODQ-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+; AVX256NODQ-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT: [[LD4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+; AVX256NODQ-NEXT: [[LD5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT: [[LD6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+; AVX256NODQ-NEXT: [[LD7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT: [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT: [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT: [[CVT2:%.*]] = sitofp i64 [[LD2]] to float
+; AVX256NODQ-NEXT: [[CVT3:%.*]] = sitofp i64 [[LD3]] to float
+; AVX256NODQ-NEXT: [[CVT4:%.*]] = sitofp i64 [[LD4]] to float
+; AVX256NODQ-NEXT: [[CVT5:%.*]] = sitofp i64 [[LD5]] to float
+; AVX256NODQ-NEXT: [[CVT6:%.*]] = sitofp i64 [[LD6]] to float
+; AVX256NODQ-NEXT: [[CVT7:%.*]] = sitofp i64 [[LD7]] to float
+; AVX256NODQ-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; AVX256NODQ-NEXT: store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; AVX256NODQ-NEXT: store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; AVX256NODQ-NEXT: store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT: store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; AVX256NODQ-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_8i64_8f32(
+; AVX512-NEXT: [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX512-NEXT: store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX512-NEXT: ret void
+;
+; AVX256DQ-LABEL: @sitofp_8i64_8f32(
+; AVX256DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @src64 to <8 x i64>*), align 64
+; AVX256DQ-NEXT: [[TMP2:%.*]] = sitofp <8 x i64> [[TMP1]] to <8 x float>
+; AVX256DQ-NEXT: store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256DQ-NEXT: ret void
+;
+ %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+ %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+ %ld2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 2), align 16
+ %ld3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 3), align 8
+ %ld4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 4), align 32
+ %ld5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 5), align 8
+ %ld6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 6), align 16
+ %ld7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 7), align 8
+ %cvt0 = sitofp i64 %ld0 to float
+ %cvt1 = sitofp i64 %ld1 to float
+ %cvt2 = sitofp i64 %ld2 to float
+ %cvt3 = sitofp i64 %ld3 to float
+ %cvt4 = sitofp i64 %ld4 to float
+ %cvt5 = sitofp i64 %ld5 to float
+ %cvt6 = sitofp i64 %ld6 to float
+ %cvt7 = sitofp i64 %ld7 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+ store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+ store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+ store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @sitofp_4i32_4f32() #0 {
+; CHECK-LABEL: @sitofp_4i32_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+ %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+ %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+ %cvt0 = sitofp i32 %ld0 to float
+ %cvt1 = sitofp i32 %ld1 to float
+ %cvt2 = sitofp i32 %ld2 to float
+ %cvt3 = sitofp i32 %ld3 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ ret void
+}
+
+define void @sitofp_8i32_8f32() #0 {
+; SSE-LABEL: @sitofp_8i32_8f32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT: [[TMP3:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT: [[TMP4:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT: store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT: store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_8i32_8f32(
+; AVX-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX-NEXT: store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1), align 4
+ %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2), align 8
+ %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3), align 4
+ %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4), align 16
+ %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5), align 4
+ %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6), align 8
+ %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7), align 4
+ %cvt0 = sitofp i32 %ld0 to float
+ %cvt1 = sitofp i32 %ld1 to float
+ %cvt2 = sitofp i32 %ld2 to float
+ %cvt3 = sitofp i32 %ld3 to float
+ %cvt4 = sitofp i32 %ld4 to float
+ %cvt5 = sitofp i32 %ld5 to float
+ %cvt6 = sitofp i32 %ld6 to float
+ %cvt7 = sitofp i32 %ld7 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+ store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+ store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+ store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @sitofp_16i32_16f32() #0 {
+; SSE-LABEL: @sitofp_16i32_16f32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @src32 to <4 x i32>*), align 64
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 16
+; SSE-NEXT: [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <4 x i32>*), align 32
+; SSE-NEXT: [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12) to <4 x i32>*), align 16
+; SSE-NEXT: [[TMP5:%.*]] = sitofp <4 x i32> [[TMP1]] to <4 x float>
+; SSE-NEXT: [[TMP6:%.*]] = sitofp <4 x i32> [[TMP2]] to <4 x float>
+; SSE-NEXT: [[TMP7:%.*]] = sitofp <4 x i32> [[TMP3]] to <4 x float>
+; SSE-NEXT: [[TMP8:%.*]] = sitofp <4 x i32> [[TMP4]] to <4 x float>
+; SSE-NEXT: store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT: store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT: store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT: store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_16i32_16f32(
+; AVX256-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @src32 to <8 x i32>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8) to <8 x i32>*), align 32
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <8 x i32> [[TMP1]] to <8 x float>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <8 x i32> [[TMP2]] to <8 x float>
+; AVX256-NEXT: store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT: store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_16i32_16f32(
+; AVX512-NEXT: [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @src32 to <16 x i32>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <16 x i32> [[TMP1]] to <16 x float>
+; AVX512-NEXT: store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 0 ), align 64
+ %ld1 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 1 ), align 4
+ %ld2 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 2 ), align 8
+ %ld3 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 3 ), align 4
+ %ld4 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 4 ), align 16
+ %ld5 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 5 ), align 4
+ %ld6 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 6 ), align 8
+ %ld7 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 7 ), align 4
+ %ld8 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 8 ), align 32
+ %ld9 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 9 ), align 4
+ %ld10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 10), align 8
+ %ld11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 11), align 4
+ %ld12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 12), align 16
+ %ld13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 13), align 4
+ %ld14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 14), align 8
+ %ld15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @src32, i32 0, i64 15), align 4
+ %cvt0 = sitofp i32 %ld0 to float
+ %cvt1 = sitofp i32 %ld1 to float
+ %cvt2 = sitofp i32 %ld2 to float
+ %cvt3 = sitofp i32 %ld3 to float
+ %cvt4 = sitofp i32 %ld4 to float
+ %cvt5 = sitofp i32 %ld5 to float
+ %cvt6 = sitofp i32 %ld6 to float
+ %cvt7 = sitofp i32 %ld7 to float
+ %cvt8 = sitofp i32 %ld8 to float
+ %cvt9 = sitofp i32 %ld9 to float
+ %cvt10 = sitofp i32 %ld10 to float
+ %cvt11 = sitofp i32 %ld11 to float
+ %cvt12 = sitofp i32 %ld12 to float
+ %cvt13 = sitofp i32 %ld13 to float
+ %cvt14 = sitofp i32 %ld14 to float
+ %cvt15 = sitofp i32 %ld15 to float
+ store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+ store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+ store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+ store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+ store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+ store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+ store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+ store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+ store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+ store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+ store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+ store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+ store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+ store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+ store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+ store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+ ret void
+}
+
+define void @sitofp_4i16_4f32() #0 {
+; SSE-LABEL: @sitofp_4i16_4f32(
+; SSE-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_4i16_4f32(
+; AVX-NEXT: [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @src16 to <4 x i16>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <4 x i16> [[TMP1]] to <4 x float>
+; AVX-NEXT: store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+ %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+ %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+ %cvt0 = sitofp i16 %ld0 to float
+ %cvt1 = sitofp i16 %ld1 to float
+ %cvt2 = sitofp i16 %ld2 to float
+ %cvt3 = sitofp i16 %ld3 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ ret void
+}
+
+define void @sitofp_8i16_8f32() #0 {
+; SSE-LABEL: @sitofp_8i16_8f32(
+; SSE-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT: [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT: [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT: [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT: [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i16 [[LD4]] to float
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i16 [[LD5]] to float
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i16 [[LD6]] to float
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i16 [[LD7]] to float
+; SSE-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT: store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT: store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT: store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT: store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_8i16_8f32(
+; AVX-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX-NEXT: store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+ %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+ %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+ %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+ %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+ %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+ %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+ %cvt0 = sitofp i16 %ld0 to float
+ %cvt1 = sitofp i16 %ld1 to float
+ %cvt2 = sitofp i16 %ld2 to float
+ %cvt3 = sitofp i16 %ld3 to float
+ %cvt4 = sitofp i16 %ld4 to float
+ %cvt5 = sitofp i16 %ld5 to float
+ %cvt6 = sitofp i16 %ld6 to float
+ %cvt7 = sitofp i16 %ld7 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+ store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+ store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+ store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @sitofp_16i16_16f32() #0 {
+; SSE-LABEL: @sitofp_16i16_16f32(
+; SSE-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0), align 64
+; SSE-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1), align 2
+; SSE-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2), align 4
+; SSE-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3), align 2
+; SSE-NEXT: [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4), align 8
+; SSE-NEXT: [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5), align 2
+; SSE-NEXT: [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6), align 4
+; SSE-NEXT: [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7), align 2
+; SSE-NEXT: [[LD8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8), align 16
+; SSE-NEXT: [[LD9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9), align 2
+; SSE-NEXT: [[LD10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+; SSE-NEXT: [[LD11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+; SSE-NEXT: [[LD12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+; SSE-NEXT: [[LD13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+; SSE-NEXT: [[LD14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+; SSE-NEXT: [[LD15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+; SSE-NEXT: [[CVT0:%.*]] = sitofp i16 [[LD0]] to float
+; SSE-NEXT: [[CVT1:%.*]] = sitofp i16 [[LD1]] to float
+; SSE-NEXT: [[CVT2:%.*]] = sitofp i16 [[LD2]] to float
+; SSE-NEXT: [[CVT3:%.*]] = sitofp i16 [[LD3]] to float
+; SSE-NEXT: [[CVT4:%.*]] = sitofp i16 [[LD4]] to float
+; SSE-NEXT: [[CVT5:%.*]] = sitofp i16 [[LD5]] to float
+; SSE-NEXT: [[CVT6:%.*]] = sitofp i16 [[LD6]] to float
+; SSE-NEXT: [[CVT7:%.*]] = sitofp i16 [[LD7]] to float
+; SSE-NEXT: [[CVT8:%.*]] = sitofp i16 [[LD8]] to float
+; SSE-NEXT: [[CVT9:%.*]] = sitofp i16 [[LD9]] to float
+; SSE-NEXT: [[CVT10:%.*]] = sitofp i16 [[LD10]] to float
+; SSE-NEXT: [[CVT11:%.*]] = sitofp i16 [[LD11]] to float
+; SSE-NEXT: [[CVT12:%.*]] = sitofp i16 [[LD12]] to float
+; SSE-NEXT: [[CVT13:%.*]] = sitofp i16 [[LD13]] to float
+; SSE-NEXT: [[CVT14:%.*]] = sitofp i16 [[LD14]] to float
+; SSE-NEXT: [[CVT15:%.*]] = sitofp i16 [[LD15]] to float
+; SSE-NEXT: store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT: store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT: store float [[CVT2]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+; SSE-NEXT: store float [[CVT3]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+; SSE-NEXT: store float [[CVT4]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+; SSE-NEXT: store float [[CVT5]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+; SSE-NEXT: store float [[CVT6]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+; SSE-NEXT: store float [[CVT7]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+; SSE-NEXT: store float [[CVT8]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8), align 32
+; SSE-NEXT: store float [[CVT9]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9), align 4
+; SSE-NEXT: store float [[CVT10]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+; SSE-NEXT: store float [[CVT11]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+; SSE-NEXT: store float [[CVT12]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+; SSE-NEXT: store float [[CVT13]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+; SSE-NEXT: store float [[CVT14]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+; SSE-NEXT: store float [[CVT15]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_16i16_16f32(
+; AVX256-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @src16 to <8 x i16>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8) to <8 x i16>*), align 16
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <8 x i16> [[TMP1]] to <8 x float>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <8 x i16> [[TMP2]] to <8 x float>
+; AVX256-NEXT: store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT: store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_16i16_16f32(
+; AVX512-NEXT: [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @src16 to <16 x i16>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <16 x i16> [[TMP1]] to <16 x float>
+; AVX512-NEXT: store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 0 ), align 64
+ %ld1 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 1 ), align 2
+ %ld2 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 2 ), align 4
+ %ld3 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 3 ), align 2
+ %ld4 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 4 ), align 8
+ %ld5 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 5 ), align 2
+ %ld6 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 6 ), align 4
+ %ld7 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 7 ), align 2
+ %ld8 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 8 ), align 16
+ %ld9 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 9 ), align 2
+ %ld10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 10), align 4
+ %ld11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 11), align 2
+ %ld12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 12), align 8
+ %ld13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 13), align 2
+ %ld14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 14), align 4
+ %ld15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @src16, i32 0, i64 15), align 2
+ %cvt0 = sitofp i16 %ld0 to float
+ %cvt1 = sitofp i16 %ld1 to float
+ %cvt2 = sitofp i16 %ld2 to float
+ %cvt3 = sitofp i16 %ld3 to float
+ %cvt4 = sitofp i16 %ld4 to float
+ %cvt5 = sitofp i16 %ld5 to float
+ %cvt6 = sitofp i16 %ld6 to float
+ %cvt7 = sitofp i16 %ld7 to float
+ %cvt8 = sitofp i16 %ld8 to float
+ %cvt9 = sitofp i16 %ld9 to float
+ %cvt10 = sitofp i16 %ld10 to float
+ %cvt11 = sitofp i16 %ld11 to float
+ %cvt12 = sitofp i16 %ld12 to float
+ %cvt13 = sitofp i16 %ld13 to float
+ %cvt14 = sitofp i16 %ld14 to float
+ %cvt15 = sitofp i16 %ld15 to float
+ store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+ store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+ store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+ store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+ store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+ store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+ store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+ store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+ store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+ store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+ store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+ store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+ store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+ store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+ store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+ store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+ ret void
+}
+
+define void @sitofp_4i8_4f32() #0 {
+; CHECK-LABEL: @sitofp_4i8_4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; CHECK-NEXT: [[TMP2:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; CHECK-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+ %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+ %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+ %cvt0 = sitofp i8 %ld0 to float
+ %cvt1 = sitofp i8 %ld1 to float
+ %cvt2 = sitofp i8 %ld2 to float
+ %cvt3 = sitofp i8 %ld3 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ ret void
+}
+
+define void @sitofp_8i8_8f32() #0 {
+; SSE-LABEL: @sitofp_8i8_8f32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT: [[TMP3:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT: [[TMP4:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT: store <4 x float> [[TMP3]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT: store <4 x float> [[TMP4]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT: ret void
+;
+; AVX-LABEL: @sitofp_8i8_8f32(
+; AVX-NEXT: [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX-NEXT: [[TMP2:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX-NEXT: store <8 x float> [[TMP2]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1), align 1
+ %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2), align 2
+ %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3), align 1
+ %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4), align 4
+ %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5), align 1
+ %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6), align 2
+ %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7), align 1
+ %cvt0 = sitofp i8 %ld0 to float
+ %cvt1 = sitofp i8 %ld1 to float
+ %cvt2 = sitofp i8 %ld2 to float
+ %cvt3 = sitofp i8 %ld3 to float
+ %cvt4 = sitofp i8 %ld4 to float
+ %cvt5 = sitofp i8 %ld5 to float
+ %cvt6 = sitofp i8 %ld6 to float
+ %cvt7 = sitofp i8 %ld7 to float
+ store float %cvt0, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+ store float %cvt1, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+ store float %cvt2, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2), align 8
+ store float %cvt3, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3), align 4
+ store float %cvt4, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4), align 16
+ store float %cvt5, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5), align 4
+ store float %cvt6, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6), align 8
+ store float %cvt7, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7), align 4
+ ret void
+}
+
+define void @sitofp_16i8_16f32() #0 {
+; SSE-LABEL: @sitofp_16i8_16f32(
+; SSE-NEXT: [[TMP1:%.*]] = load <4 x i8>, <4 x i8>* bitcast ([64 x i8]* @src8 to <4 x i8>*), align 64
+; SSE-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4) to <4 x i8>*), align 4
+; SSE-NEXT: [[TMP3:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <4 x i8>*), align 8
+; SSE-NEXT: [[TMP4:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12) to <4 x i8>*), align 4
+; SSE-NEXT: [[TMP5:%.*]] = sitofp <4 x i8> [[TMP1]] to <4 x float>
+; SSE-NEXT: [[TMP6:%.*]] = sitofp <4 x i8> [[TMP2]] to <4 x float>
+; SSE-NEXT: [[TMP7:%.*]] = sitofp <4 x i8> [[TMP3]] to <4 x float>
+; SSE-NEXT: [[TMP8:%.*]] = sitofp <4 x i8> [[TMP4]] to <4 x float>
+; SSE-NEXT: store <4 x float> [[TMP5]], <4 x float>* bitcast ([16 x float]* @dst32 to <4 x float>*), align 64
+; SSE-NEXT: store <4 x float> [[TMP6]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4) to <4 x float>*), align 16
+; SSE-NEXT: store <4 x float> [[TMP7]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <4 x float>*), align 32
+; SSE-NEXT: store <4 x float> [[TMP8]], <4 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12) to <4 x float>*), align 16
+; SSE-NEXT: ret void
+;
+; AVX256-LABEL: @sitofp_16i8_16f32(
+; AVX256-NEXT: [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* bitcast ([64 x i8]* @src8 to <8 x i8>*), align 64
+; AVX256-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8) to <8 x i8>*), align 8
+; AVX256-NEXT: [[TMP3:%.*]] = sitofp <8 x i8> [[TMP1]] to <8 x float>
+; AVX256-NEXT: [[TMP4:%.*]] = sitofp <8 x i8> [[TMP2]] to <8 x float>
+; AVX256-NEXT: store <8 x float> [[TMP3]], <8 x float>* bitcast ([16 x float]* @dst32 to <8 x float>*), align 64
+; AVX256-NEXT: store <8 x float> [[TMP4]], <8 x float>* bitcast (float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8) to <8 x float>*), align 32
+; AVX256-NEXT: ret void
+;
+; AVX512-LABEL: @sitofp_16i8_16f32(
+; AVX512-NEXT: [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @src8 to <16 x i8>*), align 64
+; AVX512-NEXT: [[TMP2:%.*]] = sitofp <16 x i8> [[TMP1]] to <16 x float>
+; AVX512-NEXT: store <16 x float> [[TMP2]], <16 x float>* bitcast ([16 x float]* @dst32 to <16 x float>*), align 64
+; AVX512-NEXT: ret void
+;
+ %ld0 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 0 ), align 64
+ %ld1 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 1 ), align 1
+ %ld2 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 2 ), align 2
+ %ld3 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 3 ), align 1
+ %ld4 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 4 ), align 4
+ %ld5 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 5 ), align 1
+ %ld6 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 6 ), align 2
+ %ld7 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 7 ), align 1
+ %ld8 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 8 ), align 8
+ %ld9 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 9 ), align 1
+ %ld10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 10), align 2
+ %ld11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 11), align 1
+ %ld12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 12), align 4
+ %ld13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 13), align 1
+ %ld14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 14), align 2
+ %ld15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @src8, i32 0, i64 15), align 1
+ %cvt0 = sitofp i8 %ld0 to float
+ %cvt1 = sitofp i8 %ld1 to float
+ %cvt2 = sitofp i8 %ld2 to float
+ %cvt3 = sitofp i8 %ld3 to float
+ %cvt4 = sitofp i8 %ld4 to float
+ %cvt5 = sitofp i8 %ld5 to float
+ %cvt6 = sitofp i8 %ld6 to float
+ %cvt7 = sitofp i8 %ld7 to float
+ %cvt8 = sitofp i8 %ld8 to float
+ %cvt9 = sitofp i8 %ld9 to float
+ %cvt10 = sitofp i8 %ld10 to float
+ %cvt11 = sitofp i8 %ld11 to float
+ %cvt12 = sitofp i8 %ld12 to float
+ %cvt13 = sitofp i8 %ld13 to float
+ %cvt14 = sitofp i8 %ld14 to float
+ %cvt15 = sitofp i8 %ld15 to float
+ store float %cvt0 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0 ), align 64
+ store float %cvt1 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1 ), align 4
+ store float %cvt2 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 2 ), align 8
+ store float %cvt3 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 3 ), align 4
+ store float %cvt4 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 4 ), align 16
+ store float %cvt5 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 5 ), align 4
+ store float %cvt6 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 6 ), align 8
+ store float %cvt7 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 7 ), align 4
+ store float %cvt8 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 8 ), align 32
+ store float %cvt9 , float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 9 ), align 4
+ store float %cvt10, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 10), align 8
+ store float %cvt11, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 11), align 4
+ store float %cvt12, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 12), align 16
+ store float %cvt13, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 13), align 4
+ store float %cvt14, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 14), align 8
+ store float %cvt15, float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 15), align 4
+ ret void
+}
+
+;
+; SITOFP BUILDVECTOR
+;
+
+define <4 x double> @sitofp_4xi32_4f64(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-LABEL: @sitofp_4xi32_4f64(
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i32 [[A0:%.*]] to double
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i32 [[A1:%.*]] to double
+; CHECK-NEXT: [[CVT2:%.*]] = sitofp i32 [[A2:%.*]] to double
+; CHECK-NEXT: [[CVT3:%.*]] = sitofp i32 [[A3:%.*]] to double
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <4 x double> poison, double [[CVT0]], i32 0
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <4 x double> [[RES0]], double [[CVT1]], i32 1
+; CHECK-NEXT: [[RES2:%.*]] = insertelement <4 x double> [[RES1]], double [[CVT2]], i32 2
+; CHECK-NEXT: [[RES3:%.*]] = insertelement <4 x double> [[RES2]], double [[CVT3]], i32 3
+; CHECK-NEXT: ret <4 x double> [[RES3]]
+;
+ %cvt0 = sitofp i32 %a0 to double
+ %cvt1 = sitofp i32 %a1 to double
+ %cvt2 = sitofp i32 %a2 to double
+ %cvt3 = sitofp i32 %a3 to double
+ %res0 = insertelement <4 x double> poison, double %cvt0, i32 0
+ %res1 = insertelement <4 x double> %res0, double %cvt1, i32 1
+ %res2 = insertelement <4 x double> %res1, double %cvt2, i32 2
+ %res3 = insertelement <4 x double> %res2, double %cvt3, i32 3
+ ret <4 x double> %res3
+}
+
+define <4 x float> @sitofp_4xi32_4f32(i32 %a0, i32 %a1, i32 %a2, i32 %a3) #0 {
+; CHECK-LABEL: @sitofp_4xi32_4f32(
+; CHECK-NEXT: [[CVT0:%.*]] = sitofp i32 [[A0:%.*]] to float
+; CHECK-NEXT: [[CVT1:%.*]] = sitofp i32 [[A1:%.*]] to float
+; CHECK-NEXT: [[CVT2:%.*]] = sitofp i32 [[A2:%.*]] to float
+; CHECK-NEXT: [[CVT3:%.*]] = sitofp i32 [[A3:%.*]] to float
+; CHECK-NEXT: [[RES0:%.*]] = insertelement <4 x float> poison, float [[CVT0]], i32 0
+; CHECK-NEXT: [[RES1:%.*]] = insertelement <4 x float> [[RES0]], float [[CVT1]], i32 1
+; CHECK-NEXT: [[RES2:%.*]] = insertelement <4 x float> [[RES1]], float [[CVT2]], i32 2
+; CHECK-NEXT: [[RES3:%.*]] = insertelement <4 x float> [[RES2]], float [[CVT3]], i32 3
+; CHECK-NEXT: ret <4 x float> [[RES3]]
+;
+ %cvt0 = sitofp i32 %a0 to float
+ %cvt1 = sitofp i32 %a1 to float
+ %cvt2 = sitofp i32 %a2 to float
+ %cvt3 = sitofp i32 %a3 to float
+ %res0 = insertelement <4 x float> poison, float %cvt0, i32 0
+ %res1 = insertelement <4 x float> %res0, float %cvt1, i32 1
+ %res2 = insertelement <4 x float> %res1, float %cvt2, i32 2
+ %res3 = insertelement <4 x float> %res2, float %cvt3, i32 3
+ ret <4 x float> %res3
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
new file mode 100644
index 000000000000..8f815afc9bfe
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/value-bug-inseltpoison.ll
@@ -0,0 +1,108 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer < %s -S -mtriple="x86_64-grtev3-linux-gnu" -mcpu=corei7-avx | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; We used to crash on this example because we were building a constant
+; expression during vectorization and the vectorizer expects instructions
+; as elements of the vectorized tree.
+; PR19621
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: bb279:
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x float> undef, float undef, i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> [[TMP0]], float undef, i32 1
+; CHECK-NEXT: br label [[BB283:%.*]]
+; CHECK: bb283:
+; CHECK-NEXT: [[TMP2:%.*]] = phi <2 x float> [ undef, [[BB279:%.*]] ], [ [[TMP13:%.*]], [[EXIT:%.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x float> [ undef, [[BB279]] ], [ [[TMP1]], [[EXIT]] ]
+; CHECK-NEXT: br label [[BB284:%.*]]
+; CHECK: bb284:
+; CHECK-NEXT: [[TMP4:%.*]] = fpext <2 x float> [[TMP2]] to <2 x double>
+; CHECK-NEXT: [[TMP5:%.*]] = fsub <2 x double> [[TMP4]], undef
+; CHECK-NEXT: [[TMP6:%.*]] = fsub <2 x double> [[TMP5]], undef
+; CHECK-NEXT: br label [[BB21_I:%.*]]
+; CHECK: bb21.i:
+; CHECK-NEXT: br i1 undef, label [[BB22_I:%.*]], label [[EXIT]]
+; CHECK: bb22.i:
+; CHECK-NEXT: [[TMP7:%.*]] = fadd <2 x double> undef, [[TMP6]]
+; CHECK-NEXT: br label [[BB32_I:%.*]]
+; CHECK: bb32.i:
+; CHECK-NEXT: [[TMP8:%.*]] = phi <2 x double> [ [[TMP7]], [[BB22_I]] ], [ zeroinitializer, [[BB32_I]] ]
+; CHECK-NEXT: br i1 undef, label [[BB32_I]], label [[BB21_I]]
+; CHECK: exit:
+; CHECK-NEXT: [[TMP9:%.*]] = fpext <2 x float> [[TMP3]] to <2 x double>
+; CHECK-NEXT: [[TMP10:%.*]] = fmul <2 x double> [[TMP9]], <double undef, double 0.000000e+00>
+; CHECK-NEXT: [[TMP11:%.*]] = fadd <2 x double> undef, [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fadd <2 x double> [[TMP11]], undef
+; CHECK-NEXT: [[TMP13]] = fptrunc <2 x double> [[TMP12]] to <2 x float>
+; CHECK-NEXT: br label [[BB283]]
+;
+bb279:
+ br label %bb283
+
+bb283:
+ %Av.sroa.8.0 = phi float [ undef, %bb279 ], [ %tmp315, %exit ]
+ %Av.sroa.5.0 = phi float [ undef, %bb279 ], [ %tmp319, %exit ]
+ %Av.sroa.3.0 = phi float [ undef, %bb279 ], [ %tmp307, %exit ]
+ %Av.sroa.0.0 = phi float [ undef, %bb279 ], [ %tmp317, %exit ]
+ br label %bb284
+
+bb284:
+ %tmp7.i = fpext float %Av.sroa.3.0 to double
+ %tmp8.i = fsub double %tmp7.i, undef
+ %tmp9.i = fsub double %tmp8.i, undef
+ %tmp17.i = fpext float %Av.sroa.8.0 to double
+ %tmp19.i = fsub double %tmp17.i, undef
+ %tmp20.i = fsub double %tmp19.i, undef
+ br label %bb21.i
+
+bb21.i:
+ br i1 undef, label %bb22.i, label %exit
+
+bb22.i:
+ %tmp24.i = fadd double undef, %tmp9.i
+ %tmp26.i = fadd double undef, %tmp20.i
+ br label %bb32.i
+
+bb32.i:
+ %xs.0.i = phi double [ %tmp24.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
+ %ys.0.i = phi double [ %tmp26.i, %bb22.i ], [ 0.000000e+00, %bb32.i ]
+ br i1 undef, label %bb32.i, label %bb21.i
+
+exit:
+ %tmp303 = fpext float %Av.sroa.0.0 to double
+ %tmp304 = fmul double %tmp303, undef
+ %tmp305 = fadd double undef, %tmp304
+ %tmp306 = fadd double %tmp305, undef
+ %tmp307 = fptrunc double %tmp306 to float
+ %tmp311 = fpext float %Av.sroa.5.0 to double
+ %tmp312 = fmul double %tmp311, 0.000000e+00
+ %tmp313 = fadd double undef, %tmp312
+ %tmp314 = fadd double %tmp313, undef
+ %tmp315 = fptrunc double %tmp314 to float
+ %tmp317 = fptrunc double undef to float
+ %tmp319 = fptrunc double undef to float
+ br label %bb283
+}
+
+; Make sure that we probably handle constant folded vectorized trees. The
+; vectorizer starts at the type (%t2, %t3) and wil constant fold the tree.
+; The code that handles insertelement instructions must handle this.
+define <4 x double> @constant_folding() {
+; CHECK-LABEL: @constant_folding(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x double> poison, double 1.000000e+00, i32 1
+; CHECK-NEXT: [[I2:%.*]] = insertelement <4 x double> [[I1]], double 2.000000e+00, i32 0
+; CHECK-NEXT: ret <4 x double> [[I2]]
+;
+entry:
+ %t0 = fadd double 1.000000e+00 , 0.000000e+00
+ %t1 = fadd double 1.000000e+00 , 1.000000e+00
+ %t2 = fmul double %t0, 1.000000e+00
+ %i1 = insertelement <4 x double> poison, double %t2, i32 1
+ %t3 = fmul double %t1, 1.000000e+00
+ %i2 = insertelement <4 x double> %i1, double %t3, i32 0
+ ret <4 x double> %i2
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
new file mode 100644
index 000000000000..7f2243e8d6c3
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vec_list_bias-inseltpoison.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -S | FileCheck %s
+
+; Check no vectorization triggered with any portion of
+; insertelement <8 x i32> instructions that build entire vector.
+; Vectorization triggered by cost bias caused by subtracting
+; the cost of entire "aggregate build" sequence while
+; building vectorizable tree from only a portion of it.
+
+define void @test(i32* nocapture %t2) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT: [[T3:%.*]] = load i32, i32* [[T2:%.*]], align 4
+; CHECK-NEXT: [[T4:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 7
+; CHECK-NEXT: [[T5:%.*]] = load i32, i32* [[T4]], align 4
+; CHECK-NEXT: [[T8:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 1
+; CHECK-NEXT: [[T9:%.*]] = load i32, i32* [[T8]], align 4
+; CHECK-NEXT: [[T10:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 6
+; CHECK-NEXT: [[T11:%.*]] = load i32, i32* [[T10]], align 4
+; CHECK-NEXT: [[T14:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 2
+; CHECK-NEXT: [[T15:%.*]] = load i32, i32* [[T14]], align 4
+; CHECK-NEXT: [[T16:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 5
+; CHECK-NEXT: [[T17:%.*]] = load i32, i32* [[T16]], align 4
+; CHECK-NEXT: [[T20:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 3
+; CHECK-NEXT: [[T21:%.*]] = load i32, i32* [[T20]], align 4
+; CHECK-NEXT: [[T22:%.*]] = getelementptr inbounds i32, i32* [[T2]], i64 4
+; CHECK-NEXT: [[T23:%.*]] = load i32, i32* [[T22]], align 4
+; CHECK-NEXT: [[T24:%.*]] = add nsw i32 [[T23]], [[T21]]
+; CHECK-NEXT: [[T25:%.*]] = sub nsw i32 [[T21]], [[T23]]
+; CHECK-NEXT: [[T27:%.*]] = sub nsw i32 [[T3]], [[T24]]
+; CHECK-NEXT: [[T28:%.*]] = add nsw i32 [[T15]], [[T9]]
+; CHECK-NEXT: [[T29:%.*]] = sub nsw i32 [[T9]], [[T15]]
+; CHECK-NEXT: [[T30:%.*]] = add nsw i32 [[T27]], [[T29]]
+; CHECK-NEXT: [[T31:%.*]] = mul nsw i32 [[T30]], 4433
+; CHECK-NEXT: [[T32:%.*]] = mul nsw i32 [[T27]], 6270
+; CHECK-NEXT: [[T34:%.*]] = mul nsw i32 [[T29]], -15137
+; CHECK-NEXT: [[T37:%.*]] = add nsw i32 [[T25]], [[T11]]
+; CHECK-NEXT: [[T38:%.*]] = add nsw i32 [[T17]], [[T5]]
+; CHECK-NEXT: [[T39:%.*]] = add nsw i32 [[T37]], [[T38]]
+; CHECK-NEXT: [[T40:%.*]] = mul nsw i32 [[T39]], 9633
+; CHECK-NEXT: [[T41:%.*]] = mul nsw i32 [[T25]], 2446
+; CHECK-NEXT: [[T42:%.*]] = mul nsw i32 [[T17]], 16819
+; CHECK-NEXT: [[T47:%.*]] = mul nsw i32 [[T37]], -16069
+; CHECK-NEXT: [[T48:%.*]] = mul nsw i32 [[T38]], -3196
+; CHECK-NEXT: [[T49:%.*]] = add nsw i32 [[T40]], [[T47]]
+; CHECK-NEXT: [[T50:%.*]] = add nsw i32 [[T40]], [[T48]]
+; CHECK-NEXT: [[T65:%.*]] = insertelement <8 x i32> poison, i32 [[T28]], i32 0
+; CHECK-NEXT: [[T66:%.*]] = insertelement <8 x i32> [[T65]], i32 [[T50]], i32 1
+; CHECK-NEXT: [[T67:%.*]] = insertelement <8 x i32> [[T66]], i32 [[T32]], i32 2
+; CHECK-NEXT: [[T68:%.*]] = insertelement <8 x i32> [[T67]], i32 [[T49]], i32 3
+; CHECK-NEXT: [[T69:%.*]] = insertelement <8 x i32> [[T68]], i32 [[T28]], i32 4
+; CHECK-NEXT: [[T70:%.*]] = insertelement <8 x i32> [[T69]], i32 [[T50]], i32 5
+; CHECK-NEXT: [[T71:%.*]] = insertelement <8 x i32> [[T70]], i32 [[T34]], i32 6
+; CHECK-NEXT: [[T72:%.*]] = insertelement <8 x i32> [[T71]], i32 [[T49]], i32 7
+; CHECK-NEXT: [[T76:%.*]] = shl <8 x i32> [[T72]], <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; CHECK-NEXT: [[T79:%.*]] = bitcast i32* [[T2]] to <8 x i32>*
+; CHECK-NEXT: store <8 x i32> [[T76]], <8 x i32>* [[T79]], align 4
+; CHECK-NEXT: ret void
+;
+ %t3 = load i32, i32* %t2, align 4
+ %t4 = getelementptr inbounds i32, i32* %t2, i64 7
+ %t5 = load i32, i32* %t4, align 4
+ %t8 = getelementptr inbounds i32, i32* %t2, i64 1
+ %t9 = load i32, i32* %t8, align 4
+ %t10 = getelementptr inbounds i32, i32* %t2, i64 6
+ %t11 = load i32, i32* %t10, align 4
+ %t14 = getelementptr inbounds i32, i32* %t2, i64 2
+ %t15 = load i32, i32* %t14, align 4
+ %t16 = getelementptr inbounds i32, i32* %t2, i64 5
+ %t17 = load i32, i32* %t16, align 4
+ %t20 = getelementptr inbounds i32, i32* %t2, i64 3
+ %t21 = load i32, i32* %t20, align 4
+ %t22 = getelementptr inbounds i32, i32* %t2, i64 4
+ %t23 = load i32, i32* %t22, align 4
+ %t24 = add nsw i32 %t23, %t21
+ %t25 = sub nsw i32 %t21, %t23
+ %t27 = sub nsw i32 %t3, %t24
+ %t28 = add nsw i32 %t15, %t9
+ %t29 = sub nsw i32 %t9, %t15
+ %t30 = add nsw i32 %t27, %t29
+ %t31 = mul nsw i32 %t30, 4433
+ %t32 = mul nsw i32 %t27, 6270
+ %t34 = mul nsw i32 %t29, -15137
+ %t37 = add nsw i32 %t25, %t11
+ %t38 = add nsw i32 %t17, %t5
+ %t39 = add nsw i32 %t37, %t38
+ %t40 = mul nsw i32 %t39, 9633
+ %t41 = mul nsw i32 %t25, 2446
+ %t42 = mul nsw i32 %t17, 16819
+ %t47 = mul nsw i32 %t37, -16069
+ %t48 = mul nsw i32 %t38, -3196
+ %t49 = add nsw i32 %t40, %t47
+ %t50 = add nsw i32 %t40, %t48
+ %t65 = insertelement <8 x i32> poison, i32 %t28, i32 0
+ %t66 = insertelement <8 x i32> %t65, i32 %t50, i32 1
+ %t67 = insertelement <8 x i32> %t66, i32 %t32, i32 2
+ %t68 = insertelement <8 x i32> %t67, i32 %t49, i32 3
+ %t69 = insertelement <8 x i32> %t68, i32 %t28, i32 4
+ %t70 = insertelement <8 x i32> %t69, i32 %t50, i32 5
+ %t71 = insertelement <8 x i32> %t70, i32 %t34, i32 6
+ %t72 = insertelement <8 x i32> %t71, i32 %t49, i32 7
+ %t76 = shl <8 x i32> %t72, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %t79 = bitcast i32* %t2 to <8 x i32>*
+ store <8 x i32> %t76, <8 x i32>* %t79, align 4
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll
new file mode 100644
index 000000000000..c16a988565ba
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/zext-inseltpoison.ll
@@ -0,0 +1,1123 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+avx512bw -basic-aa -slp-vectorizer -S | FileCheck %s --check-prefixes=AVX
+
+;
+; vXi8
+;
+
+define <2 x i64> @loadext_2i8_to_2i64(i8* %p0) {
+; SSE2-LABEL: @loadext_2i8_to_2i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i8_to_2i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i8_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <2 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <2 x i8> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %x0 = zext i8 %i0 to i64
+ %x1 = zext i8 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i8_to_4i32(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = zext i8 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = zext i8 [[I3]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i32> [[V3]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %x0 = zext i8 %i0 to i32
+ %x1 = zext i8 %i1 to i32
+ %x2 = zext i8 %i2 to i32
+ %x3 = zext i8 %i3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i8_to_4i64(i8* %p0) {
+; SSE2-LABEL: @loadext_4i8_to_4i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i8_to_4i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i64
+; SLM-NEXT: [[X2:%.*]] = zext i8 [[I2]] to i64
+; SLM-NEXT: [[X3:%.*]] = zext i8 [[I3]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i8_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <4 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %x0 = zext i8 %i0 to i64
+ %x1 = zext i8 %i1 to i64
+ %x2 = zext i8 %i2 to i64
+ %x3 = zext i8 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
+
+define <8 x i16> @loadext_8i8_to_8i16(i8* %p0) {
+; SSE2-LABEL: @loadext_8i8_to_8i16(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i16> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i16(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i16
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i16
+; SLM-NEXT: [[X2:%.*]] = zext i8 [[I2]] to i16
+; SLM-NEXT: [[X3:%.*]] = zext i8 [[I3]] to i16
+; SLM-NEXT: [[X4:%.*]] = zext i8 [[I4]] to i16
+; SLM-NEXT: [[X5:%.*]] = zext i8 [[I5]] to i16
+; SLM-NEXT: [[X6:%.*]] = zext i8 [[I6]] to i16
+; SLM-NEXT: [[X7:%.*]] = zext i8 [[I7]] to i16
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i16> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i16(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i16> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i16> poison, i16 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i16> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i16> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i16> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i16> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i16> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i16> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i16> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i16> [[V7]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %x0 = zext i8 %i0 to i16
+ %x1 = zext i8 %i1 to i16
+ %x2 = zext i8 %i2 to i16
+ %x3 = zext i8 %i3 to i16
+ %x4 = zext i8 %i4 to i16
+ %x5 = zext i8 %i5 to i16
+ %x6 = zext i8 %i6 to i16
+ %x7 = zext i8 %i7 to i16
+ %v0 = insertelement <8 x i16> poison, i16 %x0, i32 0
+ %v1 = insertelement <8 x i16> %v0, i16 %x1, i32 1
+ %v2 = insertelement <8 x i16> %v1, i16 %x2, i32 2
+ %v3 = insertelement <8 x i16> %v2, i16 %x3, i32 3
+ %v4 = insertelement <8 x i16> %v3, i16 %x4, i32 4
+ %v5 = insertelement <8 x i16> %v4, i16 %x5, i32 5
+ %v6 = insertelement <8 x i16> %v5, i16 %x6, i32 6
+ %v7 = insertelement <8 x i16> %v6, i16 %x7, i32 7
+ ret <8 x i16> %v7
+}
+
+define <8 x i32> @loadext_8i8_to_8i32(i8* %p0) {
+; SSE2-LABEL: @loadext_8i8_to_8i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i8_to_8i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = zext i8 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = zext i8 [[I3]] to i32
+; SLM-NEXT: [[X4:%.*]] = zext i8 [[I4]] to i32
+; SLM-NEXT: [[X5:%.*]] = zext i8 [[I5]] to i32
+; SLM-NEXT: [[X6:%.*]] = zext i8 [[I6]] to i32
+; SLM-NEXT: [[X7:%.*]] = zext i8 [[I7]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i8_to_8i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <8 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i32> [[V7]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %x0 = zext i8 %i0 to i32
+ %x1 = zext i8 %i1 to i32
+ %x2 = zext i8 %i2 to i32
+ %x3 = zext i8 %i3 to i32
+ %x4 = zext i8 %i4 to i32
+ %x5 = zext i8 %i5 to i32
+ %x6 = zext i8 %i6 to i32
+ %x7 = zext i8 %i7 to i32
+ %v0 = insertelement <8 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <8 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <8 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <8 x i32> %v2, i32 %x3, i32 3
+ %v4 = insertelement <8 x i32> %v3, i32 %x4, i32 4
+ %v5 = insertelement <8 x i32> %v4, i32 %x5, i32 5
+ %v6 = insertelement <8 x i32> %v5, i32 %x6, i32 6
+ %v7 = insertelement <8 x i32> %v6, i32 %x7, i32 7
+ ret <8 x i32> %v7
+}
+
+define <16 x i16> @loadext_16i8_to_16i16(i8* %p0) {
+; SSE2-LABEL: @loadext_16i8_to_16i16(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SSE2-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SSE2-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SSE2-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SSE2-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SSE2-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SSE2-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SSE2-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SSE2-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; SSE2-NEXT: [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; SSE2-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; SSE2-NEXT: [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; SSE2-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; SSE2-NEXT: [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; SSE2-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; SSE2-NEXT: [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; SSE2-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; SSE2-NEXT: [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; SSE2-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; SSE2-NEXT: [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; SSE2-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; SSE2-NEXT: [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; SSE2-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; SSE2-NEXT: [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; SSE2-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; SSE2-NEXT: ret <16 x i16> [[V15]]
+;
+; SLM-LABEL: @loadext_16i8_to_16i16(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; SLM-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; SLM-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; SLM-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; SLM-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; SLM-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; SLM-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; SLM-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; SLM-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; SLM-NEXT: [[I0:%.*]] = load i8, i8* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i8, i8* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i8, i8* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i8, i8* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i8, i8* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i8, i8* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i8, i8* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i8, i8* [[P7]], align 1
+; SLM-NEXT: [[I8:%.*]] = load i8, i8* [[P8]], align 1
+; SLM-NEXT: [[I9:%.*]] = load i8, i8* [[P9]], align 1
+; SLM-NEXT: [[I10:%.*]] = load i8, i8* [[P10]], align 1
+; SLM-NEXT: [[I11:%.*]] = load i8, i8* [[P11]], align 1
+; SLM-NEXT: [[I12:%.*]] = load i8, i8* [[P12]], align 1
+; SLM-NEXT: [[I13:%.*]] = load i8, i8* [[P13]], align 1
+; SLM-NEXT: [[I14:%.*]] = load i8, i8* [[P14]], align 1
+; SLM-NEXT: [[I15:%.*]] = load i8, i8* [[P15]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i8 [[I0]] to i16
+; SLM-NEXT: [[X1:%.*]] = zext i8 [[I1]] to i16
+; SLM-NEXT: [[X2:%.*]] = zext i8 [[I2]] to i16
+; SLM-NEXT: [[X3:%.*]] = zext i8 [[I3]] to i16
+; SLM-NEXT: [[X4:%.*]] = zext i8 [[I4]] to i16
+; SLM-NEXT: [[X5:%.*]] = zext i8 [[I5]] to i16
+; SLM-NEXT: [[X6:%.*]] = zext i8 [[I6]] to i16
+; SLM-NEXT: [[X7:%.*]] = zext i8 [[I7]] to i16
+; SLM-NEXT: [[X8:%.*]] = zext i8 [[I8]] to i16
+; SLM-NEXT: [[X9:%.*]] = zext i8 [[I9]] to i16
+; SLM-NEXT: [[X10:%.*]] = zext i8 [[I10]] to i16
+; SLM-NEXT: [[X11:%.*]] = zext i8 [[I11]] to i16
+; SLM-NEXT: [[X12:%.*]] = zext i8 [[I12]] to i16
+; SLM-NEXT: [[X13:%.*]] = zext i8 [[I13]] to i16
+; SLM-NEXT: [[X14:%.*]] = zext i8 [[I14]] to i16
+; SLM-NEXT: [[X15:%.*]] = zext i8 [[I15]] to i16
+; SLM-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[X7]], i32 7
+; SLM-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[X8]], i32 8
+; SLM-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[X9]], i32 9
+; SLM-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[X10]], i32 10
+; SLM-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[X11]], i32 11
+; SLM-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[X12]], i32 12
+; SLM-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[X13]], i32 13
+; SLM-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[X14]], i32 14
+; SLM-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[X15]], i32 15
+; SLM-NEXT: ret <16 x i16> [[V15]]
+;
+; AVX-LABEL: @loadext_16i8_to_16i16(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i8, i8* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 7
+; AVX-NEXT: [[P8:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 8
+; AVX-NEXT: [[P9:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 9
+; AVX-NEXT: [[P10:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 10
+; AVX-NEXT: [[P11:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 11
+; AVX-NEXT: [[P12:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 12
+; AVX-NEXT: [[P13:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 13
+; AVX-NEXT: [[P14:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 14
+; AVX-NEXT: [[P15:%.*]] = getelementptr inbounds i8, i8* [[P0]], i64 15
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i8* [[P0]] to <16 x i8>*
+; AVX-NEXT: [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <16 x i8> [[TMP2]] to <16 x i16>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <16 x i16> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <16 x i16> poison, i16 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <16 x i16> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <16 x i16> [[V0]], i16 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <16 x i16> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <16 x i16> [[V1]], i16 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <16 x i16> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <16 x i16> [[V2]], i16 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <16 x i16> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <16 x i16> [[V3]], i16 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <16 x i16> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <16 x i16> [[V4]], i16 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <16 x i16> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <16 x i16> [[V5]], i16 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <16 x i16> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <16 x i16> [[V6]], i16 [[TMP11]], i32 7
+; AVX-NEXT: [[TMP12:%.*]] = extractelement <16 x i16> [[TMP3]], i32 8
+; AVX-NEXT: [[V8:%.*]] = insertelement <16 x i16> [[V7]], i16 [[TMP12]], i32 8
+; AVX-NEXT: [[TMP13:%.*]] = extractelement <16 x i16> [[TMP3]], i32 9
+; AVX-NEXT: [[V9:%.*]] = insertelement <16 x i16> [[V8]], i16 [[TMP13]], i32 9
+; AVX-NEXT: [[TMP14:%.*]] = extractelement <16 x i16> [[TMP3]], i32 10
+; AVX-NEXT: [[V10:%.*]] = insertelement <16 x i16> [[V9]], i16 [[TMP14]], i32 10
+; AVX-NEXT: [[TMP15:%.*]] = extractelement <16 x i16> [[TMP3]], i32 11
+; AVX-NEXT: [[V11:%.*]] = insertelement <16 x i16> [[V10]], i16 [[TMP15]], i32 11
+; AVX-NEXT: [[TMP16:%.*]] = extractelement <16 x i16> [[TMP3]], i32 12
+; AVX-NEXT: [[V12:%.*]] = insertelement <16 x i16> [[V11]], i16 [[TMP16]], i32 12
+; AVX-NEXT: [[TMP17:%.*]] = extractelement <16 x i16> [[TMP3]], i32 13
+; AVX-NEXT: [[V13:%.*]] = insertelement <16 x i16> [[V12]], i16 [[TMP17]], i32 13
+; AVX-NEXT: [[TMP18:%.*]] = extractelement <16 x i16> [[TMP3]], i32 14
+; AVX-NEXT: [[V14:%.*]] = insertelement <16 x i16> [[V13]], i16 [[TMP18]], i32 14
+; AVX-NEXT: [[TMP19:%.*]] = extractelement <16 x i16> [[TMP3]], i32 15
+; AVX-NEXT: [[V15:%.*]] = insertelement <16 x i16> [[V14]], i16 [[TMP19]], i32 15
+; AVX-NEXT: ret <16 x i16> [[V15]]
+;
+ %p1 = getelementptr inbounds i8, i8* %p0, i64 1
+ %p2 = getelementptr inbounds i8, i8* %p0, i64 2
+ %p3 = getelementptr inbounds i8, i8* %p0, i64 3
+ %p4 = getelementptr inbounds i8, i8* %p0, i64 4
+ %p5 = getelementptr inbounds i8, i8* %p0, i64 5
+ %p6 = getelementptr inbounds i8, i8* %p0, i64 6
+ %p7 = getelementptr inbounds i8, i8* %p0, i64 7
+ %p8 = getelementptr inbounds i8, i8* %p0, i64 8
+ %p9 = getelementptr inbounds i8, i8* %p0, i64 9
+ %p10 = getelementptr inbounds i8, i8* %p0, i64 10
+ %p11 = getelementptr inbounds i8, i8* %p0, i64 11
+ %p12 = getelementptr inbounds i8, i8* %p0, i64 12
+ %p13 = getelementptr inbounds i8, i8* %p0, i64 13
+ %p14 = getelementptr inbounds i8, i8* %p0, i64 14
+ %p15 = getelementptr inbounds i8, i8* %p0, i64 15
+ %i0 = load i8, i8* %p0, align 1
+ %i1 = load i8, i8* %p1, align 1
+ %i2 = load i8, i8* %p2, align 1
+ %i3 = load i8, i8* %p3, align 1
+ %i4 = load i8, i8* %p4, align 1
+ %i5 = load i8, i8* %p5, align 1
+ %i6 = load i8, i8* %p6, align 1
+ %i7 = load i8, i8* %p7, align 1
+ %i8 = load i8, i8* %p8, align 1
+ %i9 = load i8, i8* %p9, align 1
+ %i10 = load i8, i8* %p10, align 1
+ %i11 = load i8, i8* %p11, align 1
+ %i12 = load i8, i8* %p12, align 1
+ %i13 = load i8, i8* %p13, align 1
+ %i14 = load i8, i8* %p14, align 1
+ %i15 = load i8, i8* %p15, align 1
+ %x0 = zext i8 %i0 to i16
+ %x1 = zext i8 %i1 to i16
+ %x2 = zext i8 %i2 to i16
+ %x3 = zext i8 %i3 to i16
+ %x4 = zext i8 %i4 to i16
+ %x5 = zext i8 %i5 to i16
+ %x6 = zext i8 %i6 to i16
+ %x7 = zext i8 %i7 to i16
+ %x8 = zext i8 %i8 to i16
+ %x9 = zext i8 %i9 to i16
+ %x10 = zext i8 %i10 to i16
+ %x11 = zext i8 %i11 to i16
+ %x12 = zext i8 %i12 to i16
+ %x13 = zext i8 %i13 to i16
+ %x14 = zext i8 %i14 to i16
+ %x15 = zext i8 %i15 to i16
+ %v0 = insertelement <16 x i16> poison, i16 %x0, i32 0
+ %v1 = insertelement <16 x i16> %v0, i16 %x1, i32 1
+ %v2 = insertelement <16 x i16> %v1, i16 %x2, i32 2
+ %v3 = insertelement <16 x i16> %v2, i16 %x3, i32 3
+ %v4 = insertelement <16 x i16> %v3, i16 %x4, i32 4
+ %v5 = insertelement <16 x i16> %v4, i16 %x5, i32 5
+ %v6 = insertelement <16 x i16> %v5, i16 %x6, i32 6
+ %v7 = insertelement <16 x i16> %v6, i16 %x7, i32 7
+ %v8 = insertelement <16 x i16> %v7, i16 %x8, i32 8
+ %v9 = insertelement <16 x i16> %v8, i16 %x9, i32 9
+ %v10 = insertelement <16 x i16> %v9, i16 %x10, i32 10
+ %v11 = insertelement <16 x i16> %v10, i16 %x11, i32 11
+ %v12 = insertelement <16 x i16> %v11, i16 %x12, i32 12
+ %v13 = insertelement <16 x i16> %v12, i16 %x13, i32 13
+ %v14 = insertelement <16 x i16> %v13, i16 %x14, i32 14
+ %v15 = insertelement <16 x i16> %v14, i16 %x15, i32 15
+ ret <16 x i16> %v15
+}
+
+;
+; vXi16
+;
+
+define <2 x i64> @loadext_2i16_to_2i64(i16* %p0) {
+; SSE2-LABEL: @loadext_2i16_to_2i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i16_to_2i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i16 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i16 [[I1]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i16_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <2 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i16>, <2 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <2 x i16> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %x0 = zext i16 %i0 to i64
+ %x1 = zext i16 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i32> @loadext_4i16_to_4i32(i16* %p0) {
+; SSE2-LABEL: @loadext_4i16_to_4i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i32> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i16 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = zext i16 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = zext i16 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = zext i16 [[I3]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i32> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i32> [[V3]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %x0 = zext i16 %i0 to i32
+ %x1 = zext i16 %i1 to i32
+ %x2 = zext i16 %i2 to i32
+ %x3 = zext i16 %i3 to i32
+ %v0 = insertelement <4 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <4 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <4 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <4 x i32> %v2, i32 %x3, i32 3
+ ret <4 x i32> %v3
+}
+
+define <4 x i64> @loadext_4i16_to_4i64(i16* %p0) {
+; SSE2-LABEL: @loadext_4i16_to_4i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i16_to_4i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i16 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i16 [[I1]] to i64
+; SLM-NEXT: [[X2:%.*]] = zext i16 [[I2]] to i64
+; SLM-NEXT: [[X3:%.*]] = zext i16 [[I3]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i16_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <4 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <4 x i16> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %x0 = zext i16 %i0 to i64
+ %x1 = zext i16 %i1 to i64
+ %x2 = zext i16 %i2 to i64
+ %x3 = zext i16 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
+
+define <8 x i32> @loadext_8i16_to_8i32(i16* %p0) {
+; SSE2-LABEL: @loadext_8i16_to_8i32(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SSE2-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SSE2-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SSE2-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SSE2-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; SSE2-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; SSE2-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; SSE2-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; SSE2-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; SSE2-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; SSE2-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; SSE2-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; SSE2-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; SSE2-NEXT: ret <8 x i32> [[V7]]
+;
+; SLM-LABEL: @loadext_8i16_to_8i32(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; SLM-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; SLM-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; SLM-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; SLM-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; SLM-NEXT: [[I0:%.*]] = load i16, i16* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i16, i16* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i16, i16* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i16, i16* [[P3]], align 1
+; SLM-NEXT: [[I4:%.*]] = load i16, i16* [[P4]], align 1
+; SLM-NEXT: [[I5:%.*]] = load i16, i16* [[P5]], align 1
+; SLM-NEXT: [[I6:%.*]] = load i16, i16* [[P6]], align 1
+; SLM-NEXT: [[I7:%.*]] = load i16, i16* [[P7]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i16 [[I0]] to i32
+; SLM-NEXT: [[X1:%.*]] = zext i16 [[I1]] to i32
+; SLM-NEXT: [[X2:%.*]] = zext i16 [[I2]] to i32
+; SLM-NEXT: [[X3:%.*]] = zext i16 [[I3]] to i32
+; SLM-NEXT: [[X4:%.*]] = zext i16 [[I4]] to i32
+; SLM-NEXT: [[X5:%.*]] = zext i16 [[I5]] to i32
+; SLM-NEXT: [[X6:%.*]] = zext i16 [[I6]] to i32
+; SLM-NEXT: [[X7:%.*]] = zext i16 [[I7]] to i32
+; SLM-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[X3]], i32 3
+; SLM-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[X4]], i32 4
+; SLM-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[X5]], i32 5
+; SLM-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[X6]], i32 6
+; SLM-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[X7]], i32 7
+; SLM-NEXT: ret <8 x i32> [[V7]]
+;
+; AVX-LABEL: @loadext_8i16_to_8i32(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i16, i16* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 3
+; AVX-NEXT: [[P4:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 4
+; AVX-NEXT: [[P5:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 5
+; AVX-NEXT: [[P6:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 6
+; AVX-NEXT: [[P7:%.*]] = getelementptr inbounds i16, i16* [[P0]], i64 7
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i16* [[P0]] to <8 x i16>*
+; AVX-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <8 x i16> [[TMP2]] to <8 x i32>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <8 x i32> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <8 x i32> [[V0]], i32 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <8 x i32> [[V1]], i32 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <8 x i32> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <8 x i32> [[V2]], i32 [[TMP7]], i32 3
+; AVX-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[TMP3]], i32 4
+; AVX-NEXT: [[V4:%.*]] = insertelement <8 x i32> [[V3]], i32 [[TMP8]], i32 4
+; AVX-NEXT: [[TMP9:%.*]] = extractelement <8 x i32> [[TMP3]], i32 5
+; AVX-NEXT: [[V5:%.*]] = insertelement <8 x i32> [[V4]], i32 [[TMP9]], i32 5
+; AVX-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[TMP3]], i32 6
+; AVX-NEXT: [[V6:%.*]] = insertelement <8 x i32> [[V5]], i32 [[TMP10]], i32 6
+; AVX-NEXT: [[TMP11:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
+; AVX-NEXT: [[V7:%.*]] = insertelement <8 x i32> [[V6]], i32 [[TMP11]], i32 7
+; AVX-NEXT: ret <8 x i32> [[V7]]
+;
+ %p1 = getelementptr inbounds i16, i16* %p0, i64 1
+ %p2 = getelementptr inbounds i16, i16* %p0, i64 2
+ %p3 = getelementptr inbounds i16, i16* %p0, i64 3
+ %p4 = getelementptr inbounds i16, i16* %p0, i64 4
+ %p5 = getelementptr inbounds i16, i16* %p0, i64 5
+ %p6 = getelementptr inbounds i16, i16* %p0, i64 6
+ %p7 = getelementptr inbounds i16, i16* %p0, i64 7
+ %i0 = load i16, i16* %p0, align 1
+ %i1 = load i16, i16* %p1, align 1
+ %i2 = load i16, i16* %p2, align 1
+ %i3 = load i16, i16* %p3, align 1
+ %i4 = load i16, i16* %p4, align 1
+ %i5 = load i16, i16* %p5, align 1
+ %i6 = load i16, i16* %p6, align 1
+ %i7 = load i16, i16* %p7, align 1
+ %x0 = zext i16 %i0 to i32
+ %x1 = zext i16 %i1 to i32
+ %x2 = zext i16 %i2 to i32
+ %x3 = zext i16 %i3 to i32
+ %x4 = zext i16 %i4 to i32
+ %x5 = zext i16 %i5 to i32
+ %x6 = zext i16 %i6 to i32
+ %x7 = zext i16 %i7 to i32
+ %v0 = insertelement <8 x i32> poison, i32 %x0, i32 0
+ %v1 = insertelement <8 x i32> %v0, i32 %x1, i32 1
+ %v2 = insertelement <8 x i32> %v1, i32 %x2, i32 2
+ %v3 = insertelement <8 x i32> %v2, i32 %x3, i32 3
+ %v4 = insertelement <8 x i32> %v3, i32 %x4, i32 4
+ %v5 = insertelement <8 x i32> %v4, i32 %x5, i32 5
+ %v6 = insertelement <8 x i32> %v5, i32 %x6, i32 6
+ %v7 = insertelement <8 x i32> %v6, i32 %x7, i32 7
+ ret <8 x i32> %v7
+}
+
+;
+; vXi32
+;
+
+define <2 x i64> @loadext_2i32_to_2i64(i32* %p0) {
+; SSE2-LABEL: @loadext_2i32_to_2i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: ret <2 x i64> [[V1]]
+;
+; SLM-LABEL: @loadext_2i32_to_2i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT: [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i32 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i32 [[I1]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: ret <2 x i64> [[V1]]
+;
+; AVX-LABEL: @loadext_2i32_to_2i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; AVX-NEXT: [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <2 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: ret <2 x i64> [[V1]]
+;
+ %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+ %i0 = load i32, i32* %p0, align 1
+ %i1 = load i32, i32* %p1, align 1
+ %x0 = zext i32 %i0 to i64
+ %x1 = zext i32 %i1 to i64
+ %v0 = insertelement <2 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <2 x i64> %v0, i64 %x1, i32 1
+ ret <2 x i64> %v1
+}
+
+define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
+; SSE2-LABEL: @loadext_4i32_to_4i64(
+; SSE2-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SSE2-NEXT: [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SSE2-NEXT: [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SSE2-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; SSE2-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; SSE2-NEXT: [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; SSE2-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; SSE2-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; SSE2-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; SSE2-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; SSE2-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; SSE2-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; SSE2-NEXT: ret <4 x i64> [[V3]]
+;
+; SLM-LABEL: @loadext_4i32_to_4i64(
+; SLM-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; SLM-NEXT: [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; SLM-NEXT: [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; SLM-NEXT: [[I0:%.*]] = load i32, i32* [[P0]], align 1
+; SLM-NEXT: [[I1:%.*]] = load i32, i32* [[P1]], align 1
+; SLM-NEXT: [[I2:%.*]] = load i32, i32* [[P2]], align 1
+; SLM-NEXT: [[I3:%.*]] = load i32, i32* [[P3]], align 1
+; SLM-NEXT: [[X0:%.*]] = zext i32 [[I0]] to i64
+; SLM-NEXT: [[X1:%.*]] = zext i32 [[I1]] to i64
+; SLM-NEXT: [[X2:%.*]] = zext i32 [[I2]] to i64
+; SLM-NEXT: [[X3:%.*]] = zext i32 [[I3]] to i64
+; SLM-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[X0]], i32 0
+; SLM-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
+; SLM-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
+; SLM-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SLM-NEXT: ret <4 x i64> [[V3]]
+;
+; AVX-LABEL: @loadext_4i32_to_4i64(
+; AVX-NEXT: [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
+; AVX-NEXT: [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
+; AVX-NEXT: [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
+; AVX-NEXT: [[TMP1:%.*]] = bitcast i32* [[P0]] to <4 x i32>*
+; AVX-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; AVX-NEXT: [[TMP3:%.*]] = zext <4 x i32> [[TMP2]] to <4 x i64>
+; AVX-NEXT: [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
+; AVX-NEXT: [[V0:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i32 0
+; AVX-NEXT: [[TMP5:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
+; AVX-NEXT: [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
+; AVX-NEXT: [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
+; AVX-NEXT: [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP6]], i32 2
+; AVX-NEXT: [[TMP7:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
+; AVX-NEXT: [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP7]], i32 3
+; AVX-NEXT: ret <4 x i64> [[V3]]
+;
+ %p1 = getelementptr inbounds i32, i32* %p0, i64 1
+ %p2 = getelementptr inbounds i32, i32* %p0, i64 2
+ %p3 = getelementptr inbounds i32, i32* %p0, i64 3
+ %i0 = load i32, i32* %p0, align 1
+ %i1 = load i32, i32* %p1, align 1
+ %i2 = load i32, i32* %p2, align 1
+ %i3 = load i32, i32* %p3, align 1
+ %x0 = zext i32 %i0 to i64
+ %x1 = zext i32 %i1 to i64
+ %x2 = zext i32 %i2 to i64
+ %x3 = zext i32 %i3 to i64
+ %v0 = insertelement <4 x i64> poison, i64 %x0, i32 0
+ %v1 = insertelement <4 x i64> %v0, i64 %x1, i32 1
+ %v2 = insertelement <4 x i64> %v1, i64 %x2, i32 2
+ %v3 = insertelement <4 x i64> %v2, i64 %x3, i32 3
+ ret <4 x i64> %v3
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/vectorizable-functions-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/vectorizable-functions-inseltpoison.ll
new file mode 100644
index 000000000000..b17f202fa55c
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/vectorizable-functions-inseltpoison.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S %s | FileCheck %s
+
+declare float @memread(float) readonly #0
+declare <4 x float> @vmemread(<4 x float>)
+
+define <4 x float> @memread_4x(<4 x float>* %a) {
+; CHECK-LABEL: @memread_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = call fast <4 x float> @vmemread(<4 x float> [[TMP0]])
+; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP5]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @memread(float %vecext) #0
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @memread(float %vecext.1) #0
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @memread(float %vecext.2) #0
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @memread(float %vecext.3) #0
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+declare float @memwrite(float) #1
+declare <4 x float> @vmemwrite(<4 x float>)
+
+define <4 x float> @memwrite_4x(<4 x float>* %a) {
+; CHECK-LABEL: @memwrite_4x(
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, <4 x float>* [[A:%.*]], align 16
+; CHECK-NEXT: [[VECEXT:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = tail call fast float @memwrite(float [[VECEXT]]) [[ATTR1:#.*]]
+; CHECK-NEXT: [[VECINS:%.*]] = insertelement <4 x float> poison, float [[TMP1]], i32 0
+; CHECK-NEXT: [[VECEXT_1:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = tail call fast float @memwrite(float [[VECEXT_1]]) [[ATTR1]]
+; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <4 x float> [[VECINS]], float [[TMP2]], i32 1
+; CHECK-NEXT: [[VECEXT_2:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = tail call fast float @memwrite(float [[VECEXT_2]]) [[ATTR1]]
+; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <4 x float> [[VECINS_1]], float [[TMP3]], i32 2
+; CHECK-NEXT: [[VECEXT_3:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = tail call fast float @memwrite(float [[VECEXT_3]]) [[ATTR1]]
+; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <4 x float> [[VECINS_2]], float [[TMP4]], i32 3
+; CHECK-NEXT: ret <4 x float> [[VECINS_3]]
+;
+entry:
+ %0 = load <4 x float>, <4 x float>* %a, align 16
+ %vecext = extractelement <4 x float> %0, i32 0
+ %1 = tail call fast float @memwrite(float %vecext) #1
+ %vecins = insertelement <4 x float> poison, float %1, i32 0
+ %vecext.1 = extractelement <4 x float> %0, i32 1
+ %2 = tail call fast float @memwrite(float %vecext.1) #1
+ %vecins.1 = insertelement <4 x float> %vecins, float %2, i32 1
+ %vecext.2 = extractelement <4 x float> %0, i32 2
+ %3 = tail call fast float @memwrite(float %vecext.2) #1
+ %vecins.2 = insertelement <4 x float> %vecins.1, float %3, i32 2
+ %vecext.3 = extractelement <4 x float> %0, i32 3
+ %4 = tail call fast float @memwrite(float %vecext.3) #1
+ %vecins.3 = insertelement <4 x float> %vecins.2, float %4, i32 3
+ ret <4 x float> %vecins.3
+}
+
+attributes #0 = { "vector-function-abi-variant"="_ZGV_LLVM_N4v_memread(vmemread)" }
+attributes #1 = { "vector-function-abi-variant"="_ZGV_LLVM_N4v_memwrite(vmemwrite)" }
diff --git a/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
new file mode 100644
index 000000000000..26fb97278d72
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/basic-inseltpoison.ll
@@ -0,0 +1,561 @@
+; RUN: opt %s -scalarizer -scalarize-load-store -dce -S | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer,dce)' -scalarize-load-store -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+declare <4 x float> @ext(<4 x float>)
+ at g = global <4 x float> zeroinitializer
+
+define void @f1(<4 x float> %init, <4 x float> *%base, i32 %count) {
+; CHECK-LABEL: @f1(
+; CHECK: entry:
+; CHECK: %init.i0 = extractelement <4 x float> %init, i32 0
+; CHECK: %init.i1 = extractelement <4 x float> %init, i32 1
+; CHECK: %init.i2 = extractelement <4 x float> %init, i32 2
+; CHECK: %init.i3 = extractelement <4 x float> %init, i32 3
+; CHECK: br label %loop
+; CHECK: loop:
+; CHECK: %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+; CHECK: %acc.i0 = phi float [ %init.i0, %entry ], [ %sel.i0, %loop ]
+; CHECK: %acc.i1 = phi float [ %init.i1, %entry ], [ %sel.i1, %loop ]
+; CHECK: %acc.i2 = phi float [ %init.i2, %entry ], [ %sel.i2, %loop ]
+; CHECK: %acc.i3 = phi float [ %init.i3, %entry ], [ %sel.i3, %loop ]
+; CHECK: %nexti = sub i32 %i, 1
+; CHECK: %ptr = getelementptr <4 x float>, <4 x float>* %base, i32 %i
+; CHECK: %ptr.i0 = bitcast <4 x float>* %ptr to float*
+; CHECK: %val.i0 = load float, float* %ptr.i0, align 16
+; CHECK: %ptr.i1 = getelementptr float, float* %ptr.i0, i32 1
+; CHECK: %val.i1 = load float, float* %ptr.i1, align 4
+; CHECK: %ptr.i2 = getelementptr float, float* %ptr.i0, i32 2
+; CHECK: %val.i2 = load float, float* %ptr.i2, align 8
+; CHECK: %ptr.i3 = getelementptr float, float* %ptr.i0, i32 3
+; CHECK: %val.i3 = load float, float* %ptr.i3, align 4
+; CHECK: %add.i0 = fadd float %val.i0, %val.i2
+; CHECK: %add.i1 = fadd float %val.i1, %val.i3
+; CHECK: %add.i2 = fadd float %acc.i0, %acc.i2
+; CHECK: %add.i3 = fadd float %acc.i1, %acc.i3
+; CHECK: %add.upto0 = insertelement <4 x float> undef, float %add.i0, i32 0
+; CHECK: %add.upto1 = insertelement <4 x float> %add.upto0, float %add.i1, i32 1
+; CHECK: %add.upto2 = insertelement <4 x float> %add.upto1, float %add.i2, i32 2
+; CHECK: %add = insertelement <4 x float> %add.upto2, float %add.i3, i32 3
+; CHECK: %call = call <4 x float> @ext(<4 x float> %add)
+; CHECK: %call.i0 = extractelement <4 x float> %call, i32 0
+; CHECK: %cmp.i0 = fcmp ogt float %call.i0, 1.0
+; CHECK: %call.i1 = extractelement <4 x float> %call, i32 1
+; CHECK: %cmp.i1 = fcmp ogt float %call.i1, 2.0
+; CHECK: %call.i2 = extractelement <4 x float> %call, i32 2
+; CHECK: %cmp.i2 = fcmp ogt float %call.i2, 3.0
+; CHECK: %call.i3 = extractelement <4 x float> %call, i32 3
+; CHECK: %cmp.i3 = fcmp ogt float %call.i3, 4.0
+; CHECK: %sel.i0 = select i1 %cmp.i0, float %call.i0, float 5.0
+; CHECK: %sel.i1 = select i1 %cmp.i1, float %call.i1, float 6.0
+; CHECK: %sel.i2 = select i1 %cmp.i2, float %call.i2, float 7.0
+; CHECK: %sel.i3 = select i1 %cmp.i3, float %call.i3, float 8.0
+; CHECK: store float %sel.i0, float* %ptr.i0
+; CHECK: store float %sel.i1, float* %ptr.i1
+; CHECK: store float %sel.i2, float* %ptr.i2
+; CHECK: store float %sel.i3, float* %ptr.i3
+; CHECK: %test = icmp eq i32 %nexti, 0
+; CHECK: br i1 %test, label %loop, label %exit
+; CHECK: exit:
+; CHECK: ret void
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+ %acc = phi <4 x float> [ %init, %entry ], [ %sel, %loop ]
+ %nexti = sub i32 %i, 1
+
+ %ptr = getelementptr <4 x float>, <4 x float> *%base, i32 %i
+ %val = load <4 x float> , <4 x float> *%ptr
+ %dval = bitcast <4 x float> %val to <2 x double>
+ %dacc = bitcast <4 x float> %acc to <2 x double>
+ %shuffle1 = shufflevector <2 x double> %dval, <2 x double> %dacc,
+ <2 x i32> <i32 0, i32 2>
+ %shuffle2 = shufflevector <2 x double> %dval, <2 x double> %dacc,
+ <2 x i32> <i32 1, i32 3>
+ %f1 = bitcast <2 x double> %shuffle1 to <4 x float>
+ %f2 = bitcast <2 x double> %shuffle2 to <4 x float>
+ %add = fadd <4 x float> %f1, %f2
+ %call = call <4 x float> @ext(<4 x float> %add)
+ %cmp = fcmp ogt <4 x float> %call,
+ <float 1.0, float 2.0, float 3.0, float 4.0>
+ %sel = select <4 x i1> %cmp, <4 x float> %call,
+ <4 x float> <float 5.0, float 6.0, float 7.0, float 8.0>
+ store <4 x float> %sel, <4 x float> *%ptr
+
+ %test = icmp eq i32 %nexti, 0
+ br i1 %test, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+define void @f2(<4 x i32> %init, <4 x i8> *%base, i32 %count) {
+; CHECK-LABEL: define void @f2(<4 x i32> %init, <4 x i8>* %base, i32 %count) {
+; CHECK: entry:
+; CHECK: %init.i0 = extractelement <4 x i32> %init, i32 0
+; CHECK: %init.i1 = extractelement <4 x i32> %init, i32 1
+; CHECK: %init.i2 = extractelement <4 x i32> %init, i32 2
+; CHECK: %init.i3 = extractelement <4 x i32> %init, i32 3
+; CHECK: br label %loop
+; CHECK: loop:
+; CHECK: %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+; CHECK: %acc.i0 = phi i32 [ %init.i0, %entry ], [ %sel.i0, %loop ]
+; CHECK: %acc.i1 = phi i32 [ %init.i1, %entry ], [ %sel.i1, %loop ]
+; CHECK: %acc.i2 = phi i32 [ %init.i2, %entry ], [ %sel.i2, %loop ]
+; CHECK: %acc.i3 = phi i32 [ %init.i3, %entry ], [ %sel.i3, %loop ]
+; CHECK: %nexti = sub i32 %i, 1
+; CHECK: %ptr = getelementptr <4 x i8>, <4 x i8>* %base, i32 %i
+; CHECK: %ptr.i0 = bitcast <4 x i8>* %ptr to i8*
+; CHECK: %val.i0 = load i8, i8* %ptr.i0, align 4
+; CHECK: %ptr.i1 = getelementptr i8, i8* %ptr.i0, i32 1
+; CHECK: %val.i1 = load i8, i8* %ptr.i1, align 1
+; CHECK: %ptr.i2 = getelementptr i8, i8* %ptr.i0, i32 2
+; CHECK: %val.i2 = load i8, i8* %ptr.i2, align 2
+; CHECK: %ptr.i3 = getelementptr i8, i8* %ptr.i0, i32 3
+; CHECK: %val.i3 = load i8, i8* %ptr.i3, align 1
+; CHECK: %ext.i0 = sext i8 %val.i0 to i32
+; CHECK: %ext.i1 = sext i8 %val.i1 to i32
+; CHECK: %ext.i2 = sext i8 %val.i2 to i32
+; CHECK: %ext.i3 = sext i8 %val.i3 to i32
+; CHECK: %add.i0 = add i32 %ext.i0, %acc.i0
+; CHECK: %add.i1 = add i32 %ext.i1, %acc.i1
+; CHECK: %add.i2 = add i32 %ext.i2, %acc.i2
+; CHECK: %add.i3 = add i32 %ext.i3, %acc.i3
+; CHECK: %cmp.i0 = icmp slt i32 %add.i0, -10
+; CHECK: %cmp.i1 = icmp slt i32 %add.i1, -11
+; CHECK: %cmp.i2 = icmp slt i32 %add.i2, -12
+; CHECK: %cmp.i3 = icmp slt i32 %add.i3, -13
+; CHECK: %sel.i0 = select i1 %cmp.i0, i32 %add.i0, i32 %i
+; CHECK: %sel.i1 = select i1 %cmp.i1, i32 %add.i1, i32 %i
+; CHECK: %sel.i2 = select i1 %cmp.i2, i32 %add.i2, i32 %i
+; CHECK: %sel.i3 = select i1 %cmp.i3, i32 %add.i3, i32 %i
+; CHECK: %trunc.i0 = trunc i32 %sel.i0 to i8
+; CHECK: %trunc.i1 = trunc i32 %sel.i1 to i8
+; CHECK: %trunc.i2 = trunc i32 %sel.i2 to i8
+; CHECK: %trunc.i3 = trunc i32 %sel.i3 to i8
+; CHECK: store i8 %trunc.i0, i8* %ptr.i0, align 4
+; CHECK: store i8 %trunc.i1, i8* %ptr.i1, align 1
+; CHECK: store i8 %trunc.i2, i8* %ptr.i2, align 2
+; CHECK: store i8 %trunc.i3, i8* %ptr.i3, align 1
+; CHECK: %test = icmp eq i32 %nexti, 0
+; CHECK: br i1 %test, label %loop, label %exit
+; CHECK: exit:
+; CHECK: ret void
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+ %acc = phi <4 x i32> [ %init, %entry ], [ %sel, %loop ]
+ %nexti = sub i32 %i, 1
+
+ %ptr = getelementptr <4 x i8>, <4 x i8> *%base, i32 %i
+ %val = load <4 x i8> , <4 x i8> *%ptr
+ %ext = sext <4 x i8> %val to <4 x i32>
+ %add = add <4 x i32> %ext, %acc
+ %cmp = icmp slt <4 x i32> %add, <i32 -10, i32 -11, i32 -12, i32 -13>
+ %single = insertelement <4 x i32> poison, i32 %i, i32 0
+ %limit = shufflevector <4 x i32> %single, <4 x i32> undef,
+ <4 x i32> zeroinitializer
+ %sel = select <4 x i1> %cmp, <4 x i32> %add, <4 x i32> %limit
+ %trunc = trunc <4 x i32> %sel to <4 x i8>
+ store <4 x i8> %trunc, <4 x i8> *%ptr
+
+ %test = icmp eq i32 %nexti, 0
+ br i1 %test, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Check that !tbaa information is preserved.
+define void @f3(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f3(
+; CHECK: %val.i0 = load i32, i32* %src.i0, align 16, !tbaa ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %src.i1, align 4, !tbaa ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %src.i2, align 8, !tbaa ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %src.i3, align 4, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %dst.i0, align 16, !tbaa ![[TAG:[0-9]*]]
+; CHECK: store i32 %add.i1, i32* %dst.i1, align 4, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %dst.i2, align 8, !tbaa ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %dst.i3, align 4, !tbaa ![[TAG]]
+; CHECK: ret void
+ %val = load <4 x i32> , <4 x i32> *%src, !tbaa !1
+ %add = add <4 x i32> %val, %val
+ store <4 x i32> %add, <4 x i32> *%dst, !tbaa !2
+ ret void
+}
+
+; Check that !tbaa.struct information is preserved.
+define void @f4(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f4(
+; CHECK: %val.i0 = load i32, i32* %src.i0, align 16, !tbaa.struct ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %src.i1, align 4, !tbaa.struct ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %src.i2, align 8, !tbaa.struct ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %src.i3, align 4, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %dst.i0, align 16, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i1, i32* %dst.i1, align 4, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %dst.i2, align 8, !tbaa.struct ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %dst.i3, align 4, !tbaa.struct ![[TAG]]
+; CHECK: ret void
+ %val = load <4 x i32> , <4 x i32> *%src, !tbaa.struct !5
+ %add = add <4 x i32> %val, %val
+ store <4 x i32> %add, <4 x i32> *%dst, !tbaa.struct !5
+ ret void
+}
+
+; Check that llvm.access.group information is preserved.
+define void @f5(i32 %count, <4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f5(
+; CHECK: %val.i0 = load i32, i32* %this_src.i0, align 16, !llvm.access.group ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %this_src.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %this_src.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %this_src.i3, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %this_dst.i0, align 16, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i1, i32* %this_dst.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %this_dst.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %this_dst.i3, align 4, !llvm.access.group ![[TAG]]
+; CHECK: ret void
+entry:
+ br label %loop
+
+loop:
+ %index = phi i32 [ 0, %entry ], [ %next_index, %loop ]
+ %this_src = getelementptr <4 x i32>, <4 x i32> *%src, i32 %index
+ %this_dst = getelementptr <4 x i32>, <4 x i32> *%dst, i32 %index
+ %val = load <4 x i32> , <4 x i32> *%this_src, !llvm.access.group !13
+ %add = add <4 x i32> %val, %val
+ store <4 x i32> %add, <4 x i32> *%this_dst, !llvm.access.group !13
+ %next_index = add i32 %index, -1
+ %continue = icmp ne i32 %next_index, %count
+ br i1 %continue, label %loop, label %end, !llvm.loop !3
+
+end:
+ ret void
+}
+
+; Check that fpmath information is preserved.
+define <4 x float> @f6(<4 x float> %x) {
+; CHECK-LABEL: @f6(
+; CHECK: %x.i0 = extractelement <4 x float> %x, i32 0
+; CHECK: %res.i0 = fadd float %x.i0, 1.0{{[e+0]*}}, !fpmath ![[TAG:[0-9]*]]
+; CHECK: %x.i1 = extractelement <4 x float> %x, i32 1
+; CHECK: %res.i1 = fadd float %x.i1, 2.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %x.i2 = extractelement <4 x float> %x, i32 2
+; CHECK: %res.i2 = fadd float %x.i2, 3.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %x.i3 = extractelement <4 x float> %x, i32 3
+; CHECK: %res.i3 = fadd float %x.i3, 4.0{{[e+0]*}}, !fpmath ![[TAG]]
+; CHECK: %res.upto0 = insertelement <4 x float> undef, float %res.i0, i32 0
+; CHECK: %res.upto1 = insertelement <4 x float> %res.upto0, float %res.i1, i32 1
+; CHECK: %res.upto2 = insertelement <4 x float> %res.upto1, float %res.i2, i32 2
+; CHECK: %res = insertelement <4 x float> %res.upto2, float %res.i3, i32 3
+; CHECK: ret <4 x float> %res
+ %res = fadd <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 4.0>,
+ !fpmath !4
+ ret <4 x float> %res
+}
+
+; Check that random metadata isn't kept.
+define void @f7(<4 x i32> *%src, <4 x i32> *%dst) {
+; CHECK-LABEL: @f7(
+; CHECK-NOT: !foo
+; CHECK: ret void
+ %val = load <4 x i32> , <4 x i32> *%src, !foo !5
+ %add = add <4 x i32> %val, %val
+ store <4 x i32> %add, <4 x i32> *%dst, !foo !5
+ ret void
+}
+
+; Test GEP with vectors.
+define void @f8(<4 x float *> *%dest, <4 x float *> %ptr0, <4 x i32> %i0,
+ float *%other) {
+; CHECK-LABEL: @f8(
+; CHECK: %dest.i0 = bitcast <4 x float*>* %dest to float**
+; CHECK: %dest.i1 = getelementptr float*, float** %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float*, float** %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float*, float** %dest.i0, i32 3
+; CHECK: %ptr0.i0 = extractelement <4 x float*> %ptr0, i32 0
+; CHECK: %ptr0.i2 = extractelement <4 x float*> %ptr0, i32 2
+; CHECK: %ptr0.i3 = extractelement <4 x float*> %ptr0, i32 3
+; CHECK: %i0.i1 = extractelement <4 x i32> %i0, i32 1
+; CHECK: %i0.i3 = extractelement <4 x i32> %i0, i32 3
+; CHECK: %val.i0 = getelementptr float, float* %ptr0.i0, i32 100
+; CHECK: %val.i1 = getelementptr float, float* %other, i32 %i0.i1
+; CHECK: %val.i2 = getelementptr float, float* %ptr0.i2, i32 100
+; CHECK: %val.i3 = getelementptr float, float* %ptr0.i3, i32 %i0.i3
+; CHECK: store float* %val.i0, float** %dest.i0, align 32
+; CHECK: store float* %val.i1, float** %dest.i1, align 8
+; CHECK: store float* %val.i2, float** %dest.i2, align 16
+; CHECK: store float* %val.i3, float** %dest.i3, align 8
+; CHECK: ret void
+ %i1 = insertelement <4 x i32> %i0, i32 100, i32 0
+ %i2 = insertelement <4 x i32> %i1, i32 100, i32 2
+ %ptr1 = insertelement <4 x float *> %ptr0, float *%other, i32 1
+ %val = getelementptr float, <4 x float *> %ptr1, <4 x i32> %i2
+ store <4 x float *> %val, <4 x float *> *%dest
+ ret void
+}
+
+; Test the handling of unaligned loads.
+define void @f9(<4 x float> *%dest, <4 x float> *%src) {
+; CHECK: @f9(
+; CHECK: %dest.i0 = bitcast <4 x float>* %dest to float*
+; CHECK: %dest.i1 = getelementptr float, float* %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float, float* %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float, float* %dest.i0, i32 3
+; CHECK: %src.i0 = bitcast <4 x float>* %src to float*
+; CHECK: %val.i0 = load float, float* %src.i0, align 4
+; CHECK: %src.i1 = getelementptr float, float* %src.i0, i32 1
+; CHECK: %val.i1 = load float, float* %src.i1, align 4
+; CHECK: %src.i2 = getelementptr float, float* %src.i0, i32 2
+; CHECK: %val.i2 = load float, float* %src.i2, align 4
+; CHECK: %src.i3 = getelementptr float, float* %src.i0, i32 3
+; CHECK: %val.i3 = load float, float* %src.i3, align 4
+; CHECK: store float %val.i0, float* %dest.i0, align 8
+; CHECK: store float %val.i1, float* %dest.i1, align 4
+; CHECK: store float %val.i2, float* %dest.i2, align 8
+; CHECK: store float %val.i3, float* %dest.i3, align 4
+; CHECK: ret void
+ %val = load <4 x float> , <4 x float> *%src, align 4
+ store <4 x float> %val, <4 x float> *%dest, align 8
+ ret void
+}
+
+; ...and again with subelement alignment.
+define void @f10(<4 x float> *%dest, <4 x float> *%src) {
+; CHECK: @f10(
+; CHECK: %dest.i0 = bitcast <4 x float>* %dest to float*
+; CHECK: %dest.i1 = getelementptr float, float* %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float, float* %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float, float* %dest.i0, i32 3
+; CHECK: %src.i0 = bitcast <4 x float>* %src to float*
+; CHECK: %val.i0 = load float, float* %src.i0, align 1
+; CHECK: %src.i1 = getelementptr float, float* %src.i0, i32 1
+; CHECK: %val.i1 = load float, float* %src.i1, align 1
+; CHECK: %src.i2 = getelementptr float, float* %src.i0, i32 2
+; CHECK: %val.i2 = load float, float* %src.i2, align 1
+; CHECK: %src.i3 = getelementptr float, float* %src.i0, i32 3
+; CHECK: %val.i3 = load float, float* %src.i3, align 1
+; CHECK: store float %val.i0, float* %dest.i0, align 2
+; CHECK: store float %val.i1, float* %dest.i1, align 2
+; CHECK: store float %val.i2, float* %dest.i2, align 2
+; CHECK: store float %val.i3, float* %dest.i3, align 2
+; CHECK: ret void
+ %val = load <4 x float> , <4 x float> *%src, align 1
+ store <4 x float> %val, <4 x float> *%dest, align 2
+ ret void
+}
+
+; Test that sub-byte loads aren't scalarized.
+define void @f11(<32 x i1> *%dest, <32 x i1> *%src0) {
+; CHECK: @f11(
+; CHECK: %val0 = load <32 x i1>, <32 x i1>* %src0
+; CHECK: %val1 = load <32 x i1>, <32 x i1>* %src1
+; CHECK: store <32 x i1> %and, <32 x i1>* %dest
+; CHECK: ret void
+ %src1 = getelementptr <32 x i1>, <32 x i1> *%src0, i32 1
+ %val0 = load <32 x i1> , <32 x i1> *%src0
+ %val1 = load <32 x i1> , <32 x i1> *%src1
+ %and = and <32 x i1> %val0, %val1
+ store <32 x i1> %and, <32 x i1> *%dest
+ ret void
+}
+
+; Test vector GEPs with more than one index.
+define void @f13(<4 x float *> *%dest, <4 x [4 x float] *> %ptr, <4 x i32> %i,
+ float *%other) {
+; CHECK-LABEL: @f13(
+; CHECK: %dest.i0 = bitcast <4 x float*>* %dest to float**
+; CHECK: %dest.i1 = getelementptr float*, float** %dest.i0, i32 1
+; CHECK: %dest.i2 = getelementptr float*, float** %dest.i0, i32 2
+; CHECK: %dest.i3 = getelementptr float*, float** %dest.i0, i32 3
+; CHECK: %i.i0 = extractelement <4 x i32> %i, i32 0
+; CHECK: %ptr.i0 = extractelement <4 x [4 x float]*> %ptr, i32 0
+; CHECK: %val.i0 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i0, i32 0, i32 %i.i0
+; CHECK: %i.i1 = extractelement <4 x i32> %i, i32 1
+; CHECK: %ptr.i1 = extractelement <4 x [4 x float]*> %ptr, i32 1
+; CHECK: %val.i1 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i1, i32 1, i32 %i.i1
+; CHECK: %i.i2 = extractelement <4 x i32> %i, i32 2
+; CHECK: %ptr.i2 = extractelement <4 x [4 x float]*> %ptr, i32 2
+; CHECK: %val.i2 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i2, i32 2, i32 %i.i2
+; CHECK: %i.i3 = extractelement <4 x i32> %i, i32 3
+; CHECK: %ptr.i3 = extractelement <4 x [4 x float]*> %ptr, i32 3
+; CHECK: %val.i3 = getelementptr inbounds [4 x float], [4 x float]* %ptr.i3, i32 3, i32 %i.i3
+; CHECK: store float* %val.i0, float** %dest.i0, align 32
+; CHECK: store float* %val.i1, float** %dest.i1, align 8
+; CHECK: store float* %val.i2, float** %dest.i2, align 16
+; CHECK: store float* %val.i3, float** %dest.i3, align 8
+; CHECK: ret void
+ %val = getelementptr inbounds [4 x float], <4 x [4 x float] *> %ptr,
+ <4 x i32> <i32 0, i32 1, i32 2, i32 3>,
+ <4 x i32> %i
+ store <4 x float *> %val, <4 x float *> *%dest
+ ret void
+}
+
+; Test combinations of vector and non-vector PHIs.
+define <4 x float> @f14(<4 x float> %acc, i32 %count) {
+; CHECK-LABEL: @f14(
+; CHECK: %this_acc.i0 = phi float [ %acc.i0, %entry ], [ %next_acc.i0, %loop ]
+; CHECK: %this_acc.i1 = phi float [ %acc.i1, %entry ], [ %next_acc.i1, %loop ]
+; CHECK: %this_acc.i2 = phi float [ %acc.i2, %entry ], [ %next_acc.i2, %loop ]
+; CHECK: %this_acc.i3 = phi float [ %acc.i3, %entry ], [ %next_acc.i3, %loop ]
+; CHECK: %this_count = phi i32 [ %count, %entry ], [ %next_count, %loop ]
+; CHECK: %this_acc.upto0 = insertelement <4 x float> undef, float %this_acc.i0, i32 0
+; CHECK: %this_acc.upto1 = insertelement <4 x float> %this_acc.upto0, float %this_acc.i1, i32 1
+; CHECK: %this_acc.upto2 = insertelement <4 x float> %this_acc.upto1, float %this_acc.i2, i32 2
+; CHECK: %this_acc = insertelement <4 x float> %this_acc.upto2, float %this_acc.i3, i32 3
+; CHECK: ret <4 x float> %next_acc
+entry:
+ br label %loop
+
+loop:
+ %this_acc = phi <4 x float> [ %acc, %entry ], [ %next_acc, %loop ]
+ %this_count = phi i32 [ %count, %entry ], [ %next_count, %loop ]
+ %foo = call <4 x float> @ext(<4 x float> %this_acc)
+ %next_acc = fadd <4 x float> %this_acc, %foo
+ %next_count = sub i32 %this_count, 1
+ %cmp = icmp eq i32 %next_count, 0
+ br i1 %cmp, label %loop, label %exit
+
+exit:
+ ret <4 x float> %next_acc
+}
+
+; Test unary operator scalarization.
+define void @f15(<4 x float> %init, <4 x float> *%base, i32 %count) {
+; CHECK-LABEL: @f15(
+; CHECK: %ptr = getelementptr <4 x float>, <4 x float>* %base, i32 %i
+; CHECK: %ptr.i0 = bitcast <4 x float>* %ptr to float*
+; CHECK: %val.i0 = load float, float* %ptr.i0, align 16
+; CHECK: %ptr.i1 = getelementptr float, float* %ptr.i0, i32 1
+; CHECK: %val.i1 = load float, float* %ptr.i1, align 4
+; CHECK: %ptr.i2 = getelementptr float, float* %ptr.i0, i32 2
+; CHECK: %val.i2 = load float, float* %ptr.i2, align 8
+; CHECK: %ptr.i3 = getelementptr float, float* %ptr.i0, i32 3
+; CHECK: %val.i3 = load float, float* %ptr.i3, align 4
+; CHECK: %neg.i0 = fneg float %val.i0
+; CHECK: %neg.i1 = fneg float %val.i1
+; CHECK: %neg.i2 = fneg float %val.i2
+; CHECK: %neg.i3 = fneg float %val.i3
+; CHECK: %neg.upto0 = insertelement <4 x float> undef, float %neg.i0, i32 0
+; CHECK: %neg.upto1 = insertelement <4 x float> %neg.upto0, float %neg.i1, i32 1
+; CHECK: %neg.upto2 = insertelement <4 x float> %neg.upto1, float %neg.i2, i32 2
+; CHECK: %neg = insertelement <4 x float> %neg.upto2, float %neg.i3, i32 3
+; CHECK: %call = call <4 x float> @ext(<4 x float> %neg)
+; CHECK: %call.i0 = extractelement <4 x float> %call, i32 0
+; CHECK: %cmp.i0 = fcmp ogt float %call.i0, 1.000000e+00
+; CHECK: %call.i1 = extractelement <4 x float> %call, i32 1
+; CHECK: %cmp.i1 = fcmp ogt float %call.i1, 2.000000e+00
+; CHECK: %call.i2 = extractelement <4 x float> %call, i32 2
+; CHECK: %cmp.i2 = fcmp ogt float %call.i2, 3.000000e+00
+; CHECK: %call.i3 = extractelement <4 x float> %call, i32 3
+; CHECK: %cmp.i3 = fcmp ogt float %call.i3, 4.000000e+00
+; CHECK: %sel.i0 = select i1 %cmp.i0, float %call.i0, float 5.000000e+00
+; CHECK: %sel.i1 = select i1 %cmp.i1, float %call.i1, float 6.000000e+00
+; CHECK: %sel.i2 = select i1 %cmp.i2, float %call.i2, float 7.000000e+00
+; CHECK: %sel.i3 = select i1 %cmp.i3, float %call.i3, float 8.000000e+00
+; CHECK: store float %sel.i0, float* %ptr.i0, align 16
+; CHECK: store float %sel.i1, float* %ptr.i1, align 4
+; CHECK: store float %sel.i2, float* %ptr.i2, align 8
+; CHECK: store float %sel.i3, float* %ptr.i3, align 4
+entry:
+ br label %loop
+
+loop:
+ %i = phi i32 [ %count, %entry ], [ %nexti, %loop ]
+ %acc = phi <4 x float> [ %init, %entry ], [ %sel, %loop ]
+ %nexti = sub i32 %i, 1
+
+ %ptr = getelementptr <4 x float>, <4 x float> *%base, i32 %i
+ %val = load <4 x float> , <4 x float> *%ptr
+ %neg = fneg <4 x float> %val
+ %call = call <4 x float> @ext(<4 x float> %neg)
+ %cmp = fcmp ogt <4 x float> %call,
+ <float 1.0, float 2.0, float 3.0, float 4.0>
+ %sel = select <4 x i1> %cmp, <4 x float> %call,
+ <4 x float> <float 5.0, float 6.0, float 7.0, float 8.0>
+ store <4 x float> %sel, <4 x float> *%ptr
+
+ %test = icmp eq i32 %nexti, 0
+ br i1 %test, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; Check that IR flags are preserved.
+define <2 x i32> @f16(<2 x i32> %i, <2 x i32> %j) {
+; CHECK-LABEL: @f16(
+; CHECK: %res.i0 = add nuw nsw i32
+; CHECK: %res.i1 = add nuw nsw i32
+ %res = add nuw nsw <2 x i32> %i, %j
+ ret <2 x i32> %res
+}
+define <2 x i32> @f17(<2 x i32> %i, <2 x i32> %j) {
+; CHECK-LABEL: @f17(
+; CHECK: %res.i0 = sdiv exact i32
+; CHECK: %res.i1 = sdiv exact i32
+ %res = sdiv exact <2 x i32> %i, %j
+ ret <2 x i32> %res
+}
+define <2 x float> @f18(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @f18(
+; CHECK: %res.i0 = fadd fast float
+; CHECK: %res.i1 = fadd fast float
+ %res = fadd fast <2 x float> %x, %y
+ ret <2 x float> %res
+}
+define <2 x float> @f19(<2 x float> %x) {
+; CHECK-LABEL: @f19(
+; CHECK: %res.i0 = fneg fast float
+; CHECK: %res.i1 = fneg fast float
+ %res = fneg fast <2 x float> %x
+ ret <2 x float> %res
+}
+define <2 x i1> @f20(<2 x float> %x, <2 x float> %y) {
+; CHECK-LABEL: @f20(
+; CHECK: %res.i0 = fcmp fast ogt float
+; CHECK: %res.i1 = fcmp fast ogt float
+ %res = fcmp fast ogt <2 x float> %x, %y
+ ret <2 x i1> %res
+}
+declare <2 x float> @llvm.sqrt.v2f32(<2 x float>)
+define <2 x float> @f21(<2 x float> %x) {
+; CHECK-LABEL: @f21(
+; CHECK: %res.i0 = call fast float @llvm.sqrt.f32
+; CHECK: %res.i1 = call fast float @llvm.sqrt.f32
+ %res = call fast <2 x float> @llvm.sqrt.v2f32(<2 x float> %x)
+ ret <2 x float> %res
+}
+declare <2 x float> @llvm.fma.v2f32(<2 x float>, <2 x float>, <2 x float>)
+define <2 x float> @f22(<2 x float> %x, <2 x float> %y, <2 x float> %z) {
+; CHECK-LABEL: @f22(
+; CHECK: %res.i0 = call fast float @llvm.fma.f32
+; CHECK: %res.i1 = call fast float @llvm.fma.f32
+ %res = call fast <2 x float> @llvm.fma.v2f32(<2 x float> %x, <2 x float> %y, <2 x float> %z)
+ ret <2 x float> %res
+}
+
+; See https://reviews.llvm.org/D83101#2133062
+define <2 x i32> @f23_crash(<2 x i32> %srcvec, i32 %v1) {
+; CHECK-LABEL: @f23_crash(
+; CHECK: %v0 = extractelement <2 x i32> %srcvec, i32 0
+; CHECK: %t1.upto0 = insertelement <2 x i32> undef, i32 %v0, i32 0
+; CHECK: %t1 = insertelement <2 x i32> %t1.upto0, i32 %v1, i32 1
+; CHECK: ret <2 x i32> %t1
+ %v0 = extractelement <2 x i32> %srcvec, i32 0
+ %t0 = insertelement <2 x i32> poison, i32 %v0, i32 0
+ %t1 = insertelement <2 x i32> %t0, i32 %v1, i32 1
+ ret <2 x i32> %t1
+}
+
+!0 = !{ !"root" }
+!1 = !{ !"set1", !0 }
+!2 = !{ !"set2", !0 }
+!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
+!4 = !{ float 4.0 }
+!5 = !{ i64 0, i64 8, null }
+!13 = distinct !{}
diff --git a/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
new file mode 100644
index 000000000000..718018b28523
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/dbgloc-bug-inseltpoison.ll
@@ -0,0 +1,44 @@
+; RUN: opt -S -march=x86 -scalarizer %s | FileCheck %s
+; RUN: opt -S -march=x86 -passes='function(scalarizer)' %s | FileCheck %s
+
+; Reproducer for pr27938
+; https://llvm.org/bugs/show_bug.cgi?id=27938
+
+define i16 @f1() !dbg !5 {
+ ret i16 undef, !dbg !9
+}
+
+define void @f2() !dbg !10 {
+bb1:
+ %_tmp7 = tail call i16 @f1(), !dbg !13
+; CHECK: call i16 @f1(), !dbg !13
+ %broadcast.splatinsert5 = insertelement <4 x i16> poison, i16 %_tmp7, i32 0
+ %broadcast.splat6 = shufflevector <4 x i16> %broadcast.splatinsert5, <4 x i16> undef, <4 x i32> zeroinitializer
+ br label %vector.body
+
+vector.body:
+ br i1 undef, label %middle.block, label %vector.body
+
+middle.block:
+ ret void, !dbg !15
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, retainedTypes: !2)
+!1 = !DIFile(filename: "dbgloc-bug.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 9, type: !6, isLocal: false, isDefinition: true, scopeLine: 10, isOptimized: true, unit: !0, retainedNodes: !2)
+!6 = !DISubroutineType(types: !7)
+!7 = !{!8}
+!8 = !DIBasicType(name: "short", size: 16, align: 16, encoding: DW_ATE_signed)
+!9 = !DILocation(line: 11, column: 5, scope: !5)
+!10 = distinct !DISubprogram(name: "f2", scope: !1, file: !1, line: 14, type: !11, isLocal: false, isDefinition: true, scopeLine: 15, isOptimized: true, unit: !0, retainedNodes: !2)
+!11 = !DISubroutineType(types: !12)
+!12 = !{null}
+!13 = !DILocation(line: 24, column: 9, scope: !14)
+!14 = !DILexicalBlock(scope: !10, file: !1, line: 17, column: 5)
+!15 = !DILocation(line: 28, column: 1, scope: !10)
diff --git a/llvm/test/Transforms/Scalarizer/order-bug-inseltpoison.ll b/llvm/test/Transforms/Scalarizer/order-bug-inseltpoison.ll
new file mode 100644
index 000000000000..fd7e009166ef
--- /dev/null
+++ b/llvm/test/Transforms/Scalarizer/order-bug-inseltpoison.ll
@@ -0,0 +1,24 @@
+; RUN: opt %s -scalarizer -S -o - | FileCheck %s
+; RUN: opt %s -passes='function(scalarizer)' -S -o - | FileCheck %s
+
+; This input caused the scalarizer to replace & erase gathered results when
+; future gathered results depended on them being alive
+
+define dllexport spir_func <4 x i32> @main(float %a) {
+entry:
+ %i = insertelement <4 x float> poison, float %a, i32 0
+ br label %z
+
+y:
+; CHECK: %f.upto0 = insertelement <4 x i32> undef, i32 %b.i0, i32 0
+; CHECK: %f.upto1 = insertelement <4 x i32> %f.upto0, i32 %b.i0, i32 1
+; CHECK: %f.upto2 = insertelement <4 x i32> %f.upto1, i32 %b.i0, i32 2
+; CHECK: %f = insertelement <4 x i32> %f.upto2, i32 %b.i0, i32 3
+ %f = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
+ ret <4 x i32> %f
+
+z:
+; CHECK: %b.i0 = bitcast float %a to i32
+ %b = bitcast <4 x float> %i to <4 x i32>
+ br label %y
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/ARM/speculate-vector-ops-inseltpoison.ll b/llvm/test/Transforms/SimplifyCFG/ARM/speculate-vector-ops-inseltpoison.ll
new file mode 100644
index 000000000000..b50abd6ad182
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/ARM/speculate-vector-ops-inseltpoison.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -simplifycfg -simplifycfg-require-and-preserve-domtree=1 -mtriple=thumbv8.1m.main -mattr=+mve -S %s -o - | FileCheck %s --check-prefix=CHECK-MVE
+; RUN: opt -simplifycfg -simplifycfg-require-and-preserve-domtree=1 -mtriple=thumbv8.1m.main -S %s -o - | FileCheck %s --check-prefix=CHECK-NOMVE
+
+define i32 @speculate_vector_extract(i32 %d, <4 x i32> %v) {
+; CHECK-MVE-LABEL: @speculate_vector_extract(
+; CHECK-MVE-NEXT: entry:
+; CHECK-MVE-NEXT: [[CONV:%.*]] = insertelement <4 x i32> poison, i32 [[D:%.*]], i32 0
+; CHECK-MVE-NEXT: [[CONV2:%.*]] = insertelement <4 x i32> [[CONV]], i32 [[D]], i32 1
+; CHECK-MVE-NEXT: [[CONV3:%.*]] = insertelement <4 x i32> [[CONV2]], i32 [[D]], i32 2
+; CHECK-MVE-NEXT: [[CONV4:%.*]] = insertelement <4 x i32> [[CONV3]], i32 [[D]], i32 3
+; CHECK-MVE-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[CONV4]], <i32 0, i32 -1, i32 -2, i32 -3>
+; CHECK-MVE-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[TMP6]], zeroinitializer
+; CHECK-MVE-NEXT: [[CMP_EXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-MVE-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 0
+; CHECK-MVE-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-MVE-NEXT: br i1 [[TOBOOL]], label [[COND_ELSE:%.*]], label [[COND_THEN:%.*]]
+; CHECK-MVE: cond.then:
+; CHECK-MVE-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[V:%.*]], i32 0
+; CHECK-MVE-NEXT: br label [[COND_END:%.*]]
+; CHECK-MVE: cond.else:
+; CHECK-MVE-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[V]], i32 3
+; CHECK-MVE-NEXT: br label [[COND_END]]
+; CHECK-MVE: cond.end:
+; CHECK-MVE-NEXT: [[COND:%.*]] = phi i32 [ [[TMP10]], [[COND_THEN]] ], [ [[TMP12]], [[COND_ELSE]] ]
+; CHECK-MVE-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 1
+; CHECK-MVE-NEXT: [[TOBOOL15:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-MVE-NEXT: [[TMP20:%.*]] = extractelement <4 x i32> [[V]], i32 1
+; CHECK-MVE-NEXT: [[COND22:%.*]] = select i1 [[TOBOOL15]], i32 [[COND]], i32 [[TMP20]]
+; CHECK-MVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 2
+; CHECK-MVE-NEXT: [[TOBOOL25:%.*]] = icmp eq i32 [[TMP24]], 0
+; CHECK-MVE-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[V]], i32 2
+; CHECK-MVE-NEXT: [[COND32:%.*]] = select i1 [[TOBOOL25]], i32 [[COND22]], i32 [[TMP30]]
+; CHECK-MVE-NEXT: ret i32 [[COND32]]
+;
+; CHECK-NOMVE-LABEL: @speculate_vector_extract(
+; CHECK-NOMVE-NEXT: entry:
+; CHECK-NOMVE-NEXT: [[CONV:%.*]] = insertelement <4 x i32> poison, i32 [[D:%.*]], i32 0
+; CHECK-NOMVE-NEXT: [[CONV2:%.*]] = insertelement <4 x i32> [[CONV]], i32 [[D]], i32 1
+; CHECK-NOMVE-NEXT: [[CONV3:%.*]] = insertelement <4 x i32> [[CONV2]], i32 [[D]], i32 2
+; CHECK-NOMVE-NEXT: [[CONV4:%.*]] = insertelement <4 x i32> [[CONV3]], i32 [[D]], i32 3
+; CHECK-NOMVE-NEXT: [[TMP6:%.*]] = add nsw <4 x i32> [[CONV4]], <i32 0, i32 -1, i32 -2, i32 -3>
+; CHECK-NOMVE-NEXT: [[CMP:%.*]] = icmp eq <4 x i32> [[TMP6]], zeroinitializer
+; CHECK-NOMVE-NEXT: [[CMP_EXT:%.*]] = sext <4 x i1> [[CMP]] to <4 x i32>
+; CHECK-NOMVE-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 0
+; CHECK-NOMVE-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP8]], 0
+; CHECK-NOMVE-NEXT: [[TMP10:%.*]] = extractelement <4 x i32> [[V:%.*]], i32 0
+; CHECK-NOMVE-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[V]], i32 3
+; CHECK-NOMVE-NEXT: [[COND:%.*]] = select i1 [[TOBOOL]], i32 [[TMP12]], i32 [[TMP10]]
+; CHECK-NOMVE-NEXT: [[TMP14:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 1
+; CHECK-NOMVE-NEXT: [[TOBOOL15:%.*]] = icmp eq i32 [[TMP14]], 0
+; CHECK-NOMVE-NEXT: [[TMP20:%.*]] = extractelement <4 x i32> [[V]], i32 1
+; CHECK-NOMVE-NEXT: [[COND22:%.*]] = select i1 [[TOBOOL15]], i32 [[COND]], i32 [[TMP20]]
+; CHECK-NOMVE-NEXT: [[TMP24:%.*]] = extractelement <4 x i32> [[CMP_EXT]], i32 2
+; CHECK-NOMVE-NEXT: [[TOBOOL25:%.*]] = icmp eq i32 [[TMP24]], 0
+; CHECK-NOMVE-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[V]], i32 2
+; CHECK-NOMVE-NEXT: [[COND32:%.*]] = select i1 [[TOBOOL25]], i32 [[COND22]], i32 [[TMP30]]
+; CHECK-NOMVE-NEXT: ret i32 [[COND32]]
+;
+entry:
+ %conv = insertelement <4 x i32> poison, i32 %d, i32 0
+ %conv2 = insertelement <4 x i32> %conv, i32 %d, i32 1
+ %conv3 = insertelement <4 x i32> %conv2, i32 %d, i32 2
+ %conv4 = insertelement <4 x i32> %conv3, i32 %d, i32 3
+ %tmp6 = add nsw <4 x i32> %conv4, <i32 0, i32 -1, i32 -2, i32 -3>
+ %cmp = icmp eq <4 x i32> %tmp6, zeroinitializer
+ %cmp.ext = sext <4 x i1> %cmp to <4 x i32>
+ %tmp8 = extractelement <4 x i32> %cmp.ext, i32 0
+ %tobool = icmp eq i32 %tmp8, 0
+ br i1 %tobool, label %cond.else, label %cond.then
+
+return: ; preds = %cond.end28
+ ret i32 %cond32
+
+cond.then: ; preds = %entry
+ %tmp10 = extractelement <4 x i32> %v, i32 0
+ br label %cond.end
+
+cond.else: ; preds = %entry
+ %tmp12 = extractelement <4 x i32> %v, i32 3
+ br label %cond.end
+
+cond.end: ; preds = %cond.else, %cond.then
+ %cond = phi i32 [ %tmp10, %cond.then ], [ %tmp12, %cond.else ]
+ %tmp14 = extractelement <4 x i32> %cmp.ext, i32 1
+ %tobool15 = icmp eq i32 %tmp14, 0
+ br i1 %tobool15, label %cond.else17, label %cond.then16
+
+cond.then16: ; preds = %cond.end
+ %tmp20 = extractelement <4 x i32> %v, i32 1
+ br label %cond.end18
+
+cond.else17: ; preds = %cond.end
+ br label %cond.end18
+
+cond.end18: ; preds = %cond.else17, %cond.then16
+ %cond22 = phi i32 [ %tmp20, %cond.then16 ], [ %cond, %cond.else17 ]
+ %tmp24 = extractelement <4 x i32> %cmp.ext, i32 2
+ %tobool25 = icmp eq i32 %tmp24, 0
+ br i1 %tobool25, label %cond.else27, label %cond.then26
+
+cond.then26: ; preds = %cond.end18
+ %tmp30 = extractelement <4 x i32> %v, i32 2
+ br label %cond.end28
+
+cond.else27: ; preds = %cond.end18
+ br label %cond.end28
+
+cond.end28: ; preds = %cond.else27, %cond.then26
+ %cond32 = phi i32 [ %tmp30, %cond.then26 ], [ %cond22, %cond.else27 ]
+ br label %return
+}
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops-inseltpoison.ll b/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops-inseltpoison.ll
new file mode 100644
index 000000000000..7a694481f149
--- /dev/null
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-vector-ops-inseltpoison.ll
@@ -0,0 +1,60 @@
+; RUN: opt -S -simplifycfg -simplifycfg-require-and-preserve-domtree=1 < %s | FileCheck %s
+
+define i32 @speculate_vector_extract(i32 %d, <4 x i32> %v) #0 {
+; CHECK-LABEL: @speculate_vector_extract(
+; CHECK-NOT: br
+entry:
+ %conv = insertelement <4 x i32> poison, i32 %d, i32 0
+ %conv2 = insertelement <4 x i32> %conv, i32 %d, i32 1
+ %conv3 = insertelement <4 x i32> %conv2, i32 %d, i32 2
+ %conv4 = insertelement <4 x i32> %conv3, i32 %d, i32 3
+ %tmp6 = add nsw <4 x i32> %conv4, <i32 0, i32 -1, i32 -2, i32 -3>
+ %cmp = icmp eq <4 x i32> %tmp6, zeroinitializer
+ %cmp.ext = sext <4 x i1> %cmp to <4 x i32>
+ %tmp8 = extractelement <4 x i32> %cmp.ext, i32 0
+ %tobool = icmp eq i32 %tmp8, 0
+ br i1 %tobool, label %cond.else, label %cond.then
+
+return: ; preds = %cond.end28
+ ret i32 %cond32
+
+cond.then: ; preds = %entry
+ %tmp10 = extractelement <4 x i32> %v, i32 0
+ br label %cond.end
+
+cond.else: ; preds = %entry
+ %tmp12 = extractelement <4 x i32> %v, i32 3
+ br label %cond.end
+
+cond.end: ; preds = %cond.else, %cond.then
+ %cond = phi i32 [ %tmp10, %cond.then ], [ %tmp12, %cond.else ]
+ %tmp14 = extractelement <4 x i32> %cmp.ext, i32 1
+ %tobool15 = icmp eq i32 %tmp14, 0
+ br i1 %tobool15, label %cond.else17, label %cond.then16
+
+cond.then16: ; preds = %cond.end
+ %tmp20 = extractelement <4 x i32> %v, i32 1
+ br label %cond.end18
+
+cond.else17: ; preds = %cond.end
+ br label %cond.end18
+
+cond.end18: ; preds = %cond.else17, %cond.then16
+ %cond22 = phi i32 [ %tmp20, %cond.then16 ], [ %cond, %cond.else17 ]
+ %tmp24 = extractelement <4 x i32> %cmp.ext, i32 2
+ %tobool25 = icmp eq i32 %tmp24, 0
+ br i1 %tobool25, label %cond.else27, label %cond.then26
+
+cond.then26: ; preds = %cond.end18
+ %tmp30 = extractelement <4 x i32> %v, i32 2
+ br label %cond.end28
+
+cond.else27: ; preds = %cond.end18
+ br label %cond.end28
+
+cond.end28: ; preds = %cond.else27, %cond.then26
+ %cond32 = phi i32 [ %tmp30, %cond.then26 ], [ %cond22, %cond.else27 ]
+ br label %return
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/SpeculativeExecution/spec-other-inseltpoison.ll b/llvm/test/Transforms/SpeculativeExecution/spec-other-inseltpoison.ll
new file mode 100644
index 000000000000..c2eb266abbb0
--- /dev/null
+++ b/llvm/test/Transforms/SpeculativeExecution/spec-other-inseltpoison.ll
@@ -0,0 +1,88 @@
+; RUN: opt < %s -S -speculative-execution \
+; RUN: -spec-exec-max-speculation-cost 4 -spec-exec-max-not-hoisted 3 \
+; RUN: | FileCheck %s
+
+; CHECK-LABEL: @ifThen_shuffle(
+; CHECK: shufflevector
+; CHECK: br i1 true
+define void @ifThen_shuffle() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = shufflevector <2 x float> undef, <2 x float> undef, <2 x i32> zeroinitializer
+ br label %b
+
+b:
+ ret void
+}
+
+; CHECK-LABEL: @ifThen_extract(
+; CHECK: extractelement
+; CHECK: br i1 true
+define void @ifThen_extract() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = extractelement <2 x float> undef, i32 1
+ br label %b
+
+b:
+ ret void
+}
+
+
+; CHECK-LABEL: @ifThen_insert(
+; CHECK: insertelement
+; CHECK: br i1 true
+define void @ifThen_insert() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = insertelement <2 x float> poison, float undef, i32 1
+ br label %b
+
+b:
+ ret void
+}
+
+; CHECK-LABEL: @ifThen_extractvalue(
+; CHECK: extractvalue
+; CHECK: br i1 true
+define void @ifThen_extractvalue() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = extractvalue { i32, i32 } undef, 0
+ br label %b
+
+b:
+ ret void
+}
+
+; CHECK-LABEL: @ifThen_insertvalue(
+; CHECK: insertvalue
+; CHECK: br i1 true
+define void @ifThen_insertvalue() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = insertvalue { i32, i32 } undef, i32 undef, 0
+ br label %b
+
+b:
+ ret void
+}
+
+; CHECK-LABEL: @ifThen_freeze(
+; CHECK: freeze
+; CHECK: br i1 true
+define void @ifThen_freeze() {
+ br i1 true, label %a, label %b
+
+a:
+ %x = freeze i32 undef
+ br label %b
+
+b:
+ ret void
+}
diff --git a/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop-inseltpoison.ll b/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop-inseltpoison.ll
new file mode 100644
index 000000000000..392dfd4620ee
--- /dev/null
+++ b/llvm/test/Transforms/StructurizeCFG/rebuild-ssa-infinite-loop-inseltpoison.ll
@@ -0,0 +1,53 @@
+; RUN: opt -o /dev/null -structurizecfg %s
+
+; The following function caused an infinite loop inside the structurizer's
+; rebuildSSA routine, where we were iterating over an instruction's uses while
+; modifying the use list, without taking care to do this safely.
+
+target triple = "amdgcn--"
+
+define amdgpu_vs void @wrapper(i32 inreg %arg, i32 %arg1) {
+main_body:
+ %tmp = add i32 %arg1, %arg
+ %tmp2 = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> undef, i32 %tmp, i32 0, i32 0, i32 0)
+ %tmp3 = extractelement <4 x float> %tmp2, i32 1
+ %tmp4 = fptosi float %tmp3 to i32
+ %tmp5 = insertelement <2 x i32> poison, i32 %tmp4, i32 1
+ br label %loop11.i
+
+loop11.i: ; preds = %endif46.i, %main_body
+ %tmp6 = phi i32 [ 0, %main_body ], [ %tmp14, %endif46.i ]
+ %tmp7 = icmp sgt i32 %tmp6, 999
+ br i1 %tmp7, label %main.exit, label %if16.i
+
+if16.i: ; preds = %loop11.i
+ %tmp8 = call <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32> %tmp5, <8 x i32> undef, i32 15, i1 true, i1 false, i1 false, i1 false)
+ %tmp9 = extractelement <4 x float> %tmp8, i32 0
+ %tmp10 = fcmp ult float 0.000000e+00, %tmp9
+ br i1 %tmp10, label %if28.i, label %endif46.i
+
+if28.i: ; preds = %if16.i
+ %tmp11 = bitcast float %tmp9 to i32
+ %tmp12 = shl i32 %tmp11, 16
+ %tmp13 = bitcast i32 %tmp12 to float
+ br label %main.exit
+
+endif46.i: ; preds = %if16.i
+ %tmp14 = add i32 %tmp6, 1
+ br label %loop11.i
+
+main.exit: ; preds = %if28.i, %loop11.i
+ %tmp15 = phi float [ %tmp13, %if28.i ], [ 0x36F0800000000000, %loop11.i ]
+ call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %tmp15, float 0.000000e+00, float 0.000000e+00, float 0x36A0000000000000, i1 false, i1 false) #0
+ ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
+
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32 immarg) #2
+declare <4 x float> @llvm.amdgcn.image.load.v4f32.v2i32.v8i32(<2 x i32>, <8 x i32>, i32, i1, i1, i1, i1) #2
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readonly }
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
new file mode 100644
index 000000000000..51d5f71e5536
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/as-transition-inseltpoison.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=amdgcn-amd-amdhsa | FileCheck %s --check-prefixes=CHECK
+
+; ModuleID = 'load-as-transition.ll'
+target datalayout = "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5-ni:7"
+target triple = "amdgcn-amd-amdhsa"
+
+%struct.hoge = type { float }
+
+define protected amdgpu_kernel void @load_from_other_as(<4 x float>* nocapture nonnull %resultptr) local_unnamed_addr #0 {
+; CHECK-LABEL: @load_from_other_as(
+; CHECK-NEXT: bb:
+; CHECK-NEXT: [[A:%.*]] = alloca [[STRUCT_HOGE:%.*]], align 4, addrspace(5)
+; CHECK-NEXT: [[B:%.*]] = addrspacecast [[STRUCT_HOGE]] addrspace(5)* [[A]] to %struct.hoge*
+; CHECK-NEXT: [[C:%.*]] = getelementptr inbounds [[STRUCT_HOGE]], %struct.hoge* [[B]], i64 0, i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[C]] to <1 x float>*
+; CHECK-NEXT: [[TMP1:%.*]] = load <1 x float>, <1 x float>* [[TMP0]], align 4
+; CHECK-NEXT: [[E:%.*]] = shufflevector <1 x float> [[TMP1]], <1 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: store <4 x float> [[E]], <4 x float>* [[RESULTPTR:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+bb:
+ %a = alloca %struct.hoge, align 4, addrspace(5)
+ %b = addrspacecast %struct.hoge addrspace(5)* %a to %struct.hoge*
+ %c = getelementptr inbounds %struct.hoge, %struct.hoge* %b, i64 0, i32 0
+ %d = load float, float* %c, align 4
+ %e = insertelement <4 x float> poison, float %d, i32 0
+ store <4 x float> %e, <4 x float>* %resultptr, align 16
+ ret void
+}
+
+attributes #0 = { "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 12.0.0"}
diff --git a/llvm/test/Transforms/VectorCombine/Hexagon/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/Hexagon/load-inseltpoison.ll
new file mode 100644
index 000000000000..da19dcf53fcc
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/Hexagon/load-inseltpoison.ll
@@ -0,0 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=hexagon-- | FileCheck %s --check-prefixes=CHECK
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+; This would crash because TTI returns "0" for vector length.
+
+define <4 x float> @load_f32_insert_v4f32(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v4f32(
+; CHECK-NEXT: [[S:%.*]] = load float, float* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> poison, float [[S]], i32 0
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
new file mode 100644
index 000000000000..c182b35f5539
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/extract-binop-inseltpoison.ll
@@ -0,0 +1,575 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+declare void @use_i8(i8)
+declare void @use_f32(float)
+
+; Eliminating extract is profitable.
+
+define i8 @ext0_ext0_add(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext0_ext0_add(
+; CHECK-NEXT: [[TMP1:%.*]] = add <16 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ %e1 = extractelement <16 x i8> %y, i32 0
+ %r = add i8 %e0, %e1
+ ret i8 %r
+}
+
+; Eliminating extract is still profitable. Flags propagate.
+
+define i8 @ext1_ext1_add_flags(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext1_ext1_add_flags(
+; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw <16 x i8> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 1
+ %e1 = extractelement <16 x i8> %y, i32 1
+ %r = add nsw nuw i8 %e0, %e1
+ ret i8 %r
+}
+
+; Negative test - eliminating extract is profitable, but vector shift is expensive.
+
+define i8 @ext1_ext1_shl(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext1_ext1_shl(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 1
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[R:%.*]] = shl i8 [[E0]], [[E1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 1
+ %e1 = extractelement <16 x i8> %y, i32 1
+ %r = shl i8 %e0, %e1
+ ret i8 %r
+}
+
+; Negative test - eliminating extract is profitable, but vector multiply is expensive.
+
+define i8 @ext13_ext13_mul(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext13_ext13_mul(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 13
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 13
+; CHECK-NEXT: [[R:%.*]] = mul i8 [[E0]], [[E1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 13
+ %e1 = extractelement <16 x i8> %y, i32 13
+ %r = mul i8 %e0, %e1
+ ret i8 %r
+}
+
+; Negative test - cost is irrelevant because sdiv has potential UB.
+
+define i8 @ext0_ext0_sdiv(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext0_ext0_sdiv(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = sdiv i8 [[E0]], [[E1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ %e1 = extractelement <16 x i8> %y, i32 0
+ %r = sdiv i8 %e0, %e1
+ ret i8 %r
+}
+
+; Extracts are free and vector op has same cost as scalar, but we
+; speculatively transform to vector to create more optimization
+; opportunities..
+
+define double @ext0_ext0_fadd(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @ext0_ext0_fadd(
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; CHECK-NEXT: ret double [[R]]
+;
+ %e0 = extractelement <2 x double> %x, i32 0
+ %e1 = extractelement <2 x double> %y, i32 0
+ %r = fadd double %e0, %e1
+ ret double %r
+}
+
+; Eliminating extract is profitable. Flags propagate.
+
+define double @ext1_ext1_fsub(<2 x double> %x, <2 x double> %y) {
+; CHECK-LABEL: @ext1_ext1_fsub(
+; CHECK-NEXT: [[TMP1:%.*]] = fsub fast <2 x double> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: ret double [[R]]
+;
+ %e0 = extractelement <2 x double> %x, i32 1
+ %e1 = extractelement <2 x double> %y, i32 1
+ %r = fsub fast double %e0, %e1
+ ret double %r
+}
+
+; Negative test - type mismatch.
+
+define double @ext1_ext1_fadd_
diff erent_types(<2 x double> %x, <4 x double> %y) {
+; CHECK-LABEL: @ext1_ext1_fadd_
diff erent_types(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <2 x double> [[X:%.*]], i32 1
+; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x double> [[Y:%.*]], i32 1
+; CHECK-NEXT: [[R:%.*]] = fadd fast double [[E0]], [[E1]]
+; CHECK-NEXT: ret double [[R]]
+;
+ %e0 = extractelement <2 x double> %x, i32 1
+ %e1 = extractelement <4 x double> %y, i32 1
+ %r = fadd fast double %e0, %e1
+ ret double %r
+}
+
+; Disguised same vector operand; scalar code is not cheaper (with default
+; x86 target), so aggressively form vector binop.
+
+define i32 @ext1_ext1_add_same_vec(<4 x i32> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %e0 = extractelement <4 x i32> %x, i32 1
+ %e1 = extractelement <4 x i32> %x, i32 1
+ %r = add i32 %e0, %e1
+ ret i32 %r
+}
+
+; Functionally equivalent to above test; should transform as above.
+
+define i32 @ext1_ext1_add_same_vec_cse(<4 x i32> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec_cse(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %e0 = extractelement <4 x i32> %x, i32 1
+ %r = add i32 %e0, %e0
+ ret i32 %r
+}
+
+; Don't assert if extract indices have
diff erent types.
+
+define i32 @ext1_ext1_add_same_vec_
diff _idx_ty(<4 x i32> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec_
diff _idx_ty(
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X:%.*]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %e0 = extractelement <4 x i32> %x, i32 1
+ %e1 = extractelement <4 x i32> %x, i64 1
+ %r = add i32 %e0, %e1
+ ret i32 %r
+}
+
+; Negative test - same vector operand; scalar code is cheaper than general case
+; and vector code would be more expensive still.
+
+define i8 @ext1_ext1_add_same_vec_extra_use0(<16 x i8> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec_extra_use0(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use_i8(i8 [[E0]])
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[X]], i32 0
+; CHECK-NEXT: [[R:%.*]] = add i8 [[E0]], [[E1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ call void @use_i8(i8 %e0)
+ %e1 = extractelement <16 x i8> %x, i32 0
+ %r = add i8 %e0, %e1
+ ret i8 %r
+}
+
+; Negative test - same vector operand; scalar code is cheaper than general case
+; and vector code would be more expensive still.
+
+define i8 @ext1_ext1_add_same_vec_extra_use1(<16 x i8> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec_extra_use1(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[X]], i32 0
+; CHECK-NEXT: call void @use_i8(i8 [[E1]])
+; CHECK-NEXT: [[R:%.*]] = add i8 [[E0]], [[E1]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ %e1 = extractelement <16 x i8> %x, i32 0
+ call void @use_i8(i8 %e1)
+ %r = add i8 %e0, %e1
+ ret i8 %r
+}
+
+; Negative test - same vector operand; scalar code is cheaper than general case
+; and vector code would be more expensive still.
+
+define i8 @ext1_ext1_add_same_vec_cse_extra_use(<16 x i8> %x) {
+; CHECK-LABEL: @ext1_ext1_add_same_vec_cse_extra_use(
+; CHECK-NEXT: [[E:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use_i8(i8 [[E]])
+; CHECK-NEXT: [[R:%.*]] = add i8 [[E]], [[E]]
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e = extractelement <16 x i8> %x, i32 0
+ call void @use_i8(i8 %e)
+ %r = add i8 %e, %e
+ ret i8 %r
+}
+
+; Vector code costs the same as scalar, so aggressively form vector op.
+
+define i8 @ext1_ext1_add_uses1(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext1_ext1_add_uses1(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use_i8(i8 [[E0]])
+; CHECK-NEXT: [[TMP1:%.*]] = add <16 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ call void @use_i8(i8 %e0)
+ %e1 = extractelement <16 x i8> %y, i32 0
+ %r = add i8 %e0, %e1
+ ret i8 %r
+}
+
+; Vector code costs the same as scalar, so aggressively form vector op.
+
+define i8 @ext1_ext1_add_uses2(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: @ext1_ext1_add_uses2(
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
+; CHECK-NEXT: call void @use_i8(i8 [[E1]])
+; CHECK-NEXT: [[TMP1:%.*]] = add <16 x i8> [[X:%.*]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ %e1 = extractelement <16 x i8> %y, i32 0
+ call void @use_i8(i8 %e1)
+ %r = add i8 %e0, %e1
+ ret i8 %r
+}
+
+define i8 @ext0_ext1_add(<16 x i8> %x, <16 x i8> %y) {
+; SSE-LABEL: @ext0_ext1_add(
+; SSE-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 0
+; SSE-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 1
+; SSE-NEXT: [[R:%.*]] = add nuw i8 [[E0]], [[E1]]
+; SSE-NEXT: ret i8 [[R]]
+;
+; AVX-LABEL: @ext0_ext1_add(
+; AVX-NEXT: [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT: [[TMP1:%.*]] = add nuw <16 x i8> [[X:%.*]], [[SHIFT]]
+; AVX-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
+; AVX-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 0
+ %e1 = extractelement <16 x i8> %y, i32 1
+ %r = add nuw i8 %e0, %e1
+ ret i8 %r
+}
+
+define i8 @ext5_ext0_add(<16 x i8> %x, <16 x i8> %y) {
+; SSE-LABEL: @ext5_ext0_add(
+; SSE-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 5
+; SSE-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 0
+; SSE-NEXT: [[R:%.*]] = sub nsw i8 [[E0]], [[E1]]
+; SSE-NEXT: ret i8 [[R]]
+;
+; AVX-LABEL: @ext5_ext0_add(
+; AVX-NEXT: [[SHIFT:%.*]] = shufflevector <16 x i8> [[X:%.*]], <16 x i8> undef, <16 x i32> <i32 5, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT: [[TMP1:%.*]] = sub nsw <16 x i8> [[SHIFT]], [[Y:%.*]]
+; AVX-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i64 0
+; AVX-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 5
+ %e1 = extractelement <16 x i8> %y, i32 0
+ %r = sub nsw i8 %e0, %e1
+ ret i8 %r
+}
+
+define i8 @ext1_ext6_add(<16 x i8> %x, <16 x i8> %y) {
+; SSE-LABEL: @ext1_ext6_add(
+; SSE-NEXT: [[E0:%.*]] = extractelement <16 x i8> [[X:%.*]], i32 1
+; SSE-NEXT: [[E1:%.*]] = extractelement <16 x i8> [[Y:%.*]], i32 6
+; SSE-NEXT: [[R:%.*]] = and i8 [[E0]], [[E1]]
+; SSE-NEXT: ret i8 [[R]]
+;
+; AVX-LABEL: @ext1_ext6_add(
+; AVX-NEXT: [[SHIFT:%.*]] = shufflevector <16 x i8> [[Y:%.*]], <16 x i8> undef, <16 x i32> <i32 undef, i32 6, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT: [[TMP1:%.*]] = and <16 x i8> [[X:%.*]], [[SHIFT]]
+; AVX-NEXT: [[R:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
+; AVX-NEXT: ret i8 [[R]]
+;
+ %e0 = extractelement <16 x i8> %x, i32 1
+ %e1 = extractelement <16 x i8> %y, i32 6
+ %r = and i8 %e0, %e1
+ ret i8 %r
+}
+
+define float @ext1_ext0_fmul(<4 x float> %x) {
+; CHECK-LABEL: @ext1_ext0_fmul(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x float> [[X:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = fmul <4 x float> [[SHIFT]], [[X]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[TMP1]], i64 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <4 x float> %x, i32 1
+ %e1 = extractelement <4 x float> %x, i32 0
+ %r = fmul float %e0, %e1
+ ret float %r
+}
+
+define float @ext0_ext3_fmul_extra_use1(<4 x float> %x) {
+; CHECK-LABEL: @ext0_ext3_fmul_extra_use1(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use_f32(float [[E0]])
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x float> [[X]], <4 x float> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = fmul nnan <4 x float> [[X]], [[SHIFT]]
+; CHECK-NEXT: [[R:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
+; CHECK-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <4 x float> %x, i32 0
+ call void @use_f32(float %e0)
+ %e1 = extractelement <4 x float> %x, i32 3
+ %r = fmul nnan float %e0, %e1
+ ret float %r
+}
+
+define float @ext0_ext3_fmul_extra_use2(<4 x float> %x) {
+; CHECK-LABEL: @ext0_ext3_fmul_extra_use2(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x float> [[X]], i32 3
+; CHECK-NEXT: call void @use_f32(float [[E1]])
+; CHECK-NEXT: [[R:%.*]] = fmul ninf nsz float [[E0]], [[E1]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <4 x float> %x, i32 0
+ %e1 = extractelement <4 x float> %x, i32 3
+ call void @use_f32(float %e1)
+ %r = fmul ninf nsz float %e0, %e1
+ ret float %r
+}
+
+define float @ext0_ext4_fmul_v8f32(<8 x float> %x) {
+; SSE-LABEL: @ext0_ext4_fmul_v8f32(
+; SSE-NEXT: [[E0:%.*]] = extractelement <8 x float> [[X:%.*]], i32 0
+; SSE-NEXT: [[E1:%.*]] = extractelement <8 x float> [[X]], i32 4
+; SSE-NEXT: [[R:%.*]] = fadd float [[E0]], [[E1]]
+; SSE-NEXT: ret float [[R]]
+;
+; AVX-LABEL: @ext0_ext4_fmul_v8f32(
+; AVX-NEXT: [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 4, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[X]], [[SHIFT]]
+; AVX-NEXT: [[R:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
+; AVX-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <8 x float> %x, i32 0
+ %e1 = extractelement <8 x float> %x, i32 4
+ %r = fadd float %e0, %e1
+ ret float %r
+}
+
+define float @ext7_ext4_fmul_v8f32(<8 x float> %x) {
+; SSE-LABEL: @ext7_ext4_fmul_v8f32(
+; SSE-NEXT: [[E0:%.*]] = extractelement <8 x float> [[X:%.*]], i32 7
+; SSE-NEXT: [[E1:%.*]] = extractelement <8 x float> [[X]], i32 4
+; SSE-NEXT: [[R:%.*]] = fadd float [[E0]], [[E1]]
+; SSE-NEXT: ret float [[R]]
+;
+; AVX-LABEL: @ext7_ext4_fmul_v8f32(
+; AVX-NEXT: [[SHIFT:%.*]] = shufflevector <8 x float> [[X:%.*]], <8 x float> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 7, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[SHIFT]], [[X]]
+; AVX-NEXT: [[R:%.*]] = extractelement <8 x float> [[TMP1]], i64 4
+; AVX-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <8 x float> %x, i32 7
+ %e1 = extractelement <8 x float> %x, i32 4
+ %r = fadd float %e0, %e1
+ ret float %r
+}
+
+define float @ext0_ext8_fmul_v16f32(<16 x float> %x) {
+; CHECK-LABEL: @ext0_ext8_fmul_v16f32(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x float> [[X:%.*]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x float> [[X]], i32 8
+; CHECK-NEXT: [[R:%.*]] = fadd float [[E0]], [[E1]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <16 x float> %x, i32 0
+ %e1 = extractelement <16 x float> %x, i32 8
+ %r = fadd float %e0, %e1
+ ret float %r
+}
+
+define float @ext14_ext15_fmul_v16f32(<16 x float> %x) {
+; CHECK-LABEL: @ext14_ext15_fmul_v16f32(
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x float> [[X:%.*]], i32 14
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x float> [[X]], i32 15
+; CHECK-NEXT: [[R:%.*]] = fadd float [[E0]], [[E1]]
+; CHECK-NEXT: ret float [[R]]
+;
+ %e0 = extractelement <16 x float> %x, i32 14
+ %e1 = extractelement <16 x float> %x, i32 15
+ %r = fadd float %e0, %e1
+ ret float %r
+}
+
+define <4 x float> @ins_bo_ext_ext(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @ins_bo_ext_ext(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[SHIFT]], [[A]]
+; CHECK-NEXT: [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i64 3
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %a23 = fadd float %a2, %a3
+ %v3 = insertelement <4 x float> %b, float %a23, i32 3
+ ret <4 x float> %v3
+}
+
+; TODO: This is conservatively left to extract from the lower index value,
+; but it is likely that extracting from index 3 is the better option.
+
+define <4 x float> @ins_bo_ext_ext_uses(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @ins_bo_ext_ext_uses(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
+; CHECK-NEXT: [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: call void @use_f32(float [[A23]])
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[B:%.*]], float [[A23]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+ %a23 = fadd float %a2, %a3
+ call void @use_f32(float %a23)
+ %v3 = insertelement <4 x float> %b, float %a23, i32 3
+ ret <4 x float> %v3
+}
+
+define <4 x float> @PR34724(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: @PR34724(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x float> [[A:%.*]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 3, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A]], [[SHIFT]]
+; CHECK-NEXT: [[A23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
+; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x float> [[B:%.*]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[B]], [[SHIFT1]]
+; CHECK-NEXT: [[B01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
+; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x float> [[B]], <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = fadd <4 x float> [[SHIFT2]], [[B]]
+; CHECK-NEXT: [[B23:%.*]] = extractelement <4 x float> [[TMP3]], i64 3
+; CHECK-NEXT: [[V1:%.*]] = insertelement <4 x float> poison, float [[A23]], i32 1
+; CHECK-NEXT: [[V2:%.*]] = insertelement <4 x float> [[V1]], float [[B01]], i32 2
+; CHECK-NEXT: [[V3:%.*]] = insertelement <4 x float> [[V2]], float [[B23]], i32 3
+; CHECK-NEXT: ret <4 x float> [[V3]]
+;
+ %a0 = extractelement <4 x float> %a, i32 0
+ %a1 = extractelement <4 x float> %a, i32 1
+ %a2 = extractelement <4 x float> %a, i32 2
+ %a3 = extractelement <4 x float> %a, i32 3
+
+ %b0 = extractelement <4 x float> %b, i32 0
+ %b1 = extractelement <4 x float> %b, i32 1
+ %b2 = extractelement <4 x float> %b, i32 2
+ %b3 = extractelement <4 x float> %b, i32 3
+
+ %a23 = fadd float %a2, %a3
+ %b01 = fadd float %b0, %b1
+ %b23 = fadd float %b2, %b3
+
+ %v1 = insertelement <4 x float> poison, float %a23, i32 1
+ %v2 = insertelement <4 x float> %v1, float %b01, i32 2
+ %v3 = insertelement <4 x float> %v2, float %b23, i32 3
+ ret <4 x float> %v3
+}
+
+define i32 @ext_ext_or_reduction_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @ext_ext_or_reduction_v4i32(
+; CHECK-NEXT: [[Z:%.*]] = and <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = or <4 x i32> [[Z]], [[SHIFT]]
+; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = or <4 x i32> [[TMP1]], [[SHIFT1]]
+; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[Z]], <4 x i32> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP3:%.*]] = or <4 x i32> [[SHIFT2]], [[TMP2]]
+; CHECK-NEXT: [[Z0123:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: ret i32 [[Z0123]]
+;
+ %z = and <4 x i32> %x, %y
+ %z0 = extractelement <4 x i32> %z, i32 0
+ %z1 = extractelement <4 x i32> %z, i32 1
+ %z01 = or i32 %z0, %z1
+ %z2 = extractelement <4 x i32> %z, i32 2
+ %z012 = or i32 %z01, %z2
+ %z3 = extractelement <4 x i32> %z, i32 3
+ %z0123 = or i32 %z3, %z012
+ ret i32 %z0123
+}
+
+define i32 @ext_ext_partial_add_reduction_v4i32(<4 x i32> %x) {
+; CHECK-LABEL: @ext_ext_partial_add_reduction_v4i32(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[X]]
+; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[X]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
+; CHECK-NEXT: [[X210:%.*]] = extractelement <4 x i32> [[TMP2]], i64 0
+; CHECK-NEXT: ret i32 [[X210]]
+;
+ %x0 = extractelement <4 x i32> %x, i32 0
+ %x1 = extractelement <4 x i32> %x, i32 1
+ %x10 = add i32 %x1, %x0
+ %x2 = extractelement <4 x i32> %x, i32 2
+ %x210 = add i32 %x2, %x10
+ ret i32 %x210
+}
+
+define i32 @ext_ext_partial_add_reduction_and_extra_add_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @ext_ext_partial_add_reduction_and_extra_add_v4i32(
+; CHECK-NEXT: [[SHIFT:%.*]] = shufflevector <4 x i32> [[Y:%.*]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[SHIFT]], [[Y]]
+; CHECK-NEXT: [[SHIFT1:%.*]] = shufflevector <4 x i32> [[Y]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[SHIFT1]], [[TMP1]]
+; CHECK-NEXT: [[SHIFT2:%.*]] = shufflevector <4 x i32> [[X:%.*]], <4 x i32> undef, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: [[TMP3:%.*]] = add <4 x i32> [[SHIFT2]], [[TMP2]]
+; CHECK-NEXT: [[X2Y210:%.*]] = extractelement <4 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: ret i32 [[X2Y210]]
+;
+ %y0 = extractelement <4 x i32> %y, i32 0
+ %y1 = extractelement <4 x i32> %y, i32 1
+ %y10 = add i32 %y1, %y0
+ %y2 = extractelement <4 x i32> %y, i32 2
+ %y210 = add i32 %y2, %y10
+ %x2 = extractelement <4 x i32> %x, i32 2
+ %x2y210 = add i32 %x2, %y210
+ ret i32 %x2y210
+}
+
+define i32 @constant_fold_crash(<4 x i32> %x) {
+; CHECK-LABEL: @constant_fold_crash(
+; CHECK-NEXT: [[A:%.*]] = extractelement <4 x i32> <i32 16, i32 17, i32 18, i32 19>, i32 1
+; CHECK-NEXT: [[B:%.*]] = extractelement <4 x i32> [[X:%.*]], i32 0
+; CHECK-NEXT: [[C:%.*]] = add i32 [[A]], [[B]]
+; CHECK-NEXT: ret i32 [[C]]
+;
+ %a = extractelement <4 x i32> <i32 16, i32 17, i32 18, i32 19>, i32 1
+ %b = extractelement <4 x i32> %x, i32 0
+ %c = add i32 %a, %b
+ ret i32 %c
+}
+
+define float @constant_fold_crash_commute(<4 x float> %x) {
+; CHECK-LABEL: @constant_fold_crash_commute(
+; CHECK-NEXT: [[A:%.*]] = extractelement <4 x float> <float 1.600000e+01, float 1.700000e+01, float 1.800000e+01, float 1.900000e+01>, i32 3
+; CHECK-NEXT: [[B:%.*]] = extractelement <4 x float> [[X:%.*]], i32 1
+; CHECK-NEXT: [[C:%.*]] = fadd float [[B]], [[A]]
+; CHECK-NEXT: ret float [[C]]
+;
+ %a = extractelement <4 x float> <float 16.0, float 17.0, float 18.0, float 19.0>, i32 3
+ %b = extractelement <4 x float> %x, i32 1
+ %c = fadd float %b, %a
+ ret float %c
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/insert-binop-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/insert-binop-inseltpoison.ll
new file mode 100644
index 000000000000..8a6b1e98c968
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/insert-binop-inseltpoison.ll
@@ -0,0 +1,234 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+declare void @use(<4 x i32>)
+declare void @usef(<4 x float>)
+
+; Eliminating an insert is profitable.
+
+define <16 x i8> @ins0_ins0_add(i8 %x, i8 %y) {
+; CHECK-LABEL: @ins0_ins0_add(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = add i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <16 x i8> poison, i8 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <16 x i8> [[R]]
+;
+ %i0 = insertelement <16 x i8> poison, i8 %x, i32 0
+ %i1 = insertelement <16 x i8> poison, i8 %y, i32 0
+ %r = add <16 x i8> %i0, %i1
+ ret <16 x i8> %r
+}
+
+; Eliminating an insert is still profitable. Flags propagate. Mismatch types on index is ok.
+
+define <8 x i16> @ins0_ins0_sub_flags(i16 %x, i16 %y) {
+; CHECK-LABEL: @ins0_ins0_sub_flags(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = sub nuw nsw i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[R_SCALAR]], i64 5
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %i0 = insertelement <8 x i16> poison, i16 %x, i8 5
+ %i1 = insertelement <8 x i16> poison, i16 %y, i32 5
+ %r = sub nsw nuw <8 x i16> %i0, %i1
+ ret <8 x i16> %r
+}
+
+; The new vector constant is calculated by constant folding.
+; This is conservatively created as zero rather than undef for 'undef ^ undef'.
+
+define <2 x i64> @ins1_ins1_xor(i64 %x, i64 %y) {
+; CHECK-LABEL: @ins1_ins1_xor(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = xor i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i64> poison, i64 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %i0 = insertelement <2 x i64> poison, i64 %x, i64 1
+ %i1 = insertelement <2 x i64> poison, i64 %y, i32 1
+ %r = xor <2 x i64> %i0, %i1
+ ret <2 x i64> %r
+}
+
+define <2 x i64> @ins1_ins1_iterate(i64 %w, i64 %x, i64 %y, i64 %z) {
+; CHECK-LABEL: @ins1_ins1_iterate(
+; CHECK-NEXT: [[S0_SCALAR:%.*]] = sub i64 [[W:%.*]], [[X:%.*]]
+; CHECK-NEXT: [[S1_SCALAR:%.*]] = or i64 [[S0_SCALAR]], [[Y:%.*]]
+; CHECK-NEXT: [[S2_SCALAR:%.*]] = shl i64 [[Z:%.*]], [[S1_SCALAR]]
+; CHECK-NEXT: [[S2:%.*]] = insertelement <2 x i64> poison, i64 [[S2_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[S2]]
+;
+ %i0 = insertelement <2 x i64> poison, i64 %w, i64 1
+ %i1 = insertelement <2 x i64> poison, i64 %x, i32 1
+ %s0 = sub <2 x i64> %i0, %i1
+ %i2 = insertelement <2 x i64> poison, i64 %y, i32 1
+ %s1 = or <2 x i64> %s0, %i2
+ %i3 = insertelement <2 x i64> poison, i64 %z, i32 1
+ %s2 = shl <2 x i64> %i3, %s1
+ ret <2 x i64> %s2
+}
+
+; The inserts are free, but it's still better to scalarize.
+
+define <2 x double> @ins0_ins0_fadd(double %x, double %y) {
+; CHECK-LABEL: @ins0_ins0_fadd(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fadd reassoc nsz double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x double> poison, double [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[R]]
+;
+ %i0 = insertelement <2 x double> poison, double %x, i32 0
+ %i1 = insertelement <2 x double> poison, double %y, i32 0
+ %r = fadd reassoc nsz <2 x double> %i0, %i1
+ ret <2 x double> %r
+}
+
+; Negative test - mismatched indexes (but could fold this).
+
+define <16 x i8> @ins1_ins0_add(i8 %x, i8 %y) {
+; CHECK-LABEL: @ins1_ins0_add(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <16 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = insertelement <16 x i8> poison, i8 [[Y:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = add <16 x i8> [[I0]], [[I1]]
+; CHECK-NEXT: ret <16 x i8> [[R]]
+;
+ %i0 = insertelement <16 x i8> poison, i8 %x, i32 1
+ %i1 = insertelement <16 x i8> poison, i8 %y, i32 0
+ %r = add <16 x i8> %i0, %i1
+ ret <16 x i8> %r
+}
+
+; Base vector does not have to be undef.
+
+define <4 x i32> @ins0_ins0_mul(i32 %x, i32 %y) {
+; CHECK-LABEL: @ins0_ins0_mul(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i32> poison, i32 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %i0 = insertelement <4 x i32> zeroinitializer, i32 %x, i32 0
+ %i1 = insertelement <4 x i32> poison, i32 %y, i32 0
+ %r = mul <4 x i32> %i0, %i1
+ ret <4 x i32> %r
+}
+
+; It is safe to scalarize any binop (no extra UB/poison danger).
+
+define <2 x i64> @ins1_ins1_sdiv(i64 %x, i64 %y) {
+; CHECK-LABEL: @ins1_ins1_sdiv(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = sdiv i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i64> <i64 -6, i64 0>, i64 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %i0 = insertelement <2 x i64> <i64 42, i64 -42>, i64 %x, i64 1
+ %i1 = insertelement <2 x i64> <i64 -7, i64 128>, i64 %y, i32 1
+ %r = sdiv <2 x i64> %i0, %i1
+ ret <2 x i64> %r
+}
+
+; Constant folding deals with undef per element - the entire value does not become undef.
+
+define <2 x i64> @ins1_ins1_udiv(i64 %x, i64 %y) {
+; CHECK-LABEL: @ins1_ins1_udiv(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = udiv i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i64> <i64 6, i64 poison>, i64 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %i0 = insertelement <2 x i64> <i64 42, i64 undef>, i64 %x, i32 1
+ %i1 = insertelement <2 x i64> <i64 7, i64 undef>, i64 %y, i32 1
+ %r = udiv <2 x i64> %i0, %i1
+ ret <2 x i64> %r
+}
+
+; This could be simplified -- creates immediate UB without the transform because
+; divisor has an undef element -- but that is hidden after the transform.
+
+define <2 x i64> @ins1_ins1_urem(i64 %x, i64 %y) {
+; CHECK-LABEL: @ins1_ins1_urem(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = urem i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i64> <i64 poison, i64 0>, i64 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[R]]
+;
+ %i0 = insertelement <2 x i64> <i64 42, i64 undef>, i64 %x, i64 1
+ %i1 = insertelement <2 x i64> <i64 undef, i64 128>, i64 %y, i32 1
+ %r = urem <2 x i64> %i0, %i1
+ ret <2 x i64> %r
+}
+
+; Extra use is accounted for in cost calculation.
+
+define <4 x i32> @ins0_ins0_xor(i32 %x, i32 %y) {
+; CHECK-LABEL: @ins0_ins0_xor(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use(<4 x i32> [[I0]])
+; CHECK-NEXT: [[R_SCALAR:%.*]] = xor i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i32> poison, i32 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %i0 = insertelement <4 x i32> poison, i32 %x, i32 0
+ call void @use(<4 x i32> %i0)
+ %i1 = insertelement <4 x i32> poison, i32 %y, i32 0
+ %r = xor <4 x i32> %i0, %i1
+ ret <4 x i32> %r
+}
+
+; Extra use is accounted for in cost calculation.
+
+define <4 x float> @ins1_ins1_fmul(float %x, float %y) {
+; CHECK-LABEL: @ins1_ins1_fmul(
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 1
+; CHECK-NEXT: call void @usef(<4 x float> [[I1]])
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fmul float [[X:%.*]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> poison, float [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %i0 = insertelement <4 x float> poison, float %x, i32 1
+ %i1 = insertelement <4 x float> poison, float %y, i32 1
+ call void @usef(<4 x float> %i1)
+ %r = fmul <4 x float> %i0, %i1
+ ret <4 x float> %r
+}
+
+; If the scalar binop is not cheaper than the vector binop, extra uses can prevent the transform.
+
+define <4 x float> @ins2_ins2_fsub(float %x, float %y) {
+; CHECK-LABEL: @ins2_ins2_fsub(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 2
+; CHECK-NEXT: call void @usef(<4 x float> [[I0]])
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 2
+; CHECK-NEXT: call void @usef(<4 x float> [[I1]])
+; CHECK-NEXT: [[R:%.*]] = fsub <4 x float> [[I0]], [[I1]]
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %i0 = insertelement <4 x float> poison, float %x, i32 2
+ call void @usef(<4 x float> %i0)
+ %i1 = insertelement <4 x float> poison, float %y, i32 2
+ call void @usef(<4 x float> %i1)
+ %r = fsub <4 x float> %i0, %i1
+ ret <4 x float> %r
+}
+
+; It may be worth scalarizing an expensive binop even if both inserts have extra uses.
+
+define <4 x float> @ins3_ins3_fdiv(float %x, float %y) {
+; SSE-LABEL: @ins3_ins3_fdiv(
+; SSE-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 3
+; SSE-NEXT: call void @usef(<4 x float> [[I0]])
+; SSE-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 3
+; SSE-NEXT: call void @usef(<4 x float> [[I1]])
+; SSE-NEXT: [[R_SCALAR:%.*]] = fdiv float [[X]], [[Y]]
+; SSE-NEXT: [[R:%.*]] = insertelement <4 x float> poison, float [[R_SCALAR]], i64 3
+; SSE-NEXT: ret <4 x float> [[R]]
+;
+; AVX-LABEL: @ins3_ins3_fdiv(
+; AVX-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 3
+; AVX-NEXT: call void @usef(<4 x float> [[I0]])
+; AVX-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 3
+; AVX-NEXT: call void @usef(<4 x float> [[I1]])
+; AVX-NEXT: [[R:%.*]] = fdiv <4 x float> [[I0]], [[I1]]
+; AVX-NEXT: ret <4 x float> [[R]]
+;
+ %i0 = insertelement <4 x float> poison, float %x, i32 3
+ call void @usef(<4 x float> %i0)
+ %i1 = insertelement <4 x float> poison, float %y, i32 3
+ call void @usef(<4 x float> %i1)
+ %r = fdiv <4 x float> %i0, %i1
+ ret <4 x float> %r
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/insert-binop-with-constant-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/insert-binop-with-constant-inseltpoison.ll
new file mode 100644
index 000000000000..9f41db0c9e32
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/insert-binop-with-constant-inseltpoison.ll
@@ -0,0 +1,728 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=SSE2 | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=AVX2 | FileCheck %s --check-prefixes=CHECK,AVX
+
+define <2 x i64> @add_constant(i64 %x) {
+; CHECK-LABEL: @add_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = add i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = add <2 x i64> %ins, <i64 42, i64 undef>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @add_constant_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @add_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = add i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = add <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @add_constant_load(i64* %p) {
+; CHECK-LABEL: @add_constant_load(
+; CHECK-NEXT: [[LD:%.*]] = load i64, i64* [[P:%.*]], align 4
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i64> poison, i64 [[LD]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = add <2 x i64> [[INS]], <i64 42, i64 -42>
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ld = load i64, i64* %p
+ %ins = insertelement <2 x i64> poison, i64 %ld, i32 0
+ %bo = add <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i64> %bo
+}
+
+; IR flags are not required, but they should propagate.
+
+define <4 x i32> @sub_constant_op0(i32 %x) {
+; CHECK-LABEL: @sub_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sub nuw nsw i32 -42, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <4 x i32> poison, i32 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x i32> [[BO]]
+;
+ %ins = insertelement <4 x i32> poison, i32 %x, i32 1
+ %bo = sub nsw nuw <4 x i32> <i32 undef, i32 -42, i32 undef, i32 undef>, %ins
+ ret <4 x i32> %bo
+}
+
+define <4 x i32> @sub_constant_op0_not_undef_lane(i32 %x) {
+; CHECK-LABEL: @sub_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sub nuw i32 42, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <4 x i32> poison, i32 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x i32> [[BO]]
+;
+ %ins = insertelement <4 x i32> poison, i32 %x, i32 1
+ %bo = sub nuw <4 x i32> <i32 1, i32 42, i32 42, i32 -42>, %ins
+ ret <4 x i32> %bo
+}
+
+define <8 x i16> @sub_constant_op1(i16 %x) {
+; CHECK-LABEL: @sub_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sub nuw i16 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <8 x i16> poison, i16 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[BO]]
+;
+ %ins = insertelement <8 x i16> poison, i16 %x, i32 0
+ %bo = sub nuw <8 x i16> %ins, <i16 42, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef, i16 undef>
+ ret <8 x i16> %bo
+}
+
+define <8 x i16> @sub_constant_op1_not_undef_lane(i16 %x) {
+; CHECK-LABEL: @sub_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sub nuw i16 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <8 x i16> poison, i16 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[BO]]
+;
+ %ins = insertelement <8 x i16> poison, i16 %x, i32 0
+ %bo = sub nuw <8 x i16> %ins, <i16 42, i16 -42, i16 0, i16 1, i16 -2, i16 3, i16 -4, i16 5>
+ ret <8 x i16> %bo
+}
+
+define <16 x i8> @mul_constant(i8 %x) {
+; CHECK-LABEL: @mul_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = mul i8 [[X:%.*]], -42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <16 x i8> poison, i8 [[BO_SCALAR]], i64 2
+; CHECK-NEXT: ret <16 x i8> [[BO]]
+;
+ %ins = insertelement <16 x i8> poison, i8 %x, i32 2
+ %bo = mul <16 x i8> %ins, <i8 undef, i8 undef, i8 -42, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+ ret <16 x i8> %bo
+}
+
+define <3 x i64> @mul_constant_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @mul_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = mul i64 [[X:%.*]], -42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <3 x i64> poison, i64 [[BO_SCALAR]], i64 2
+; CHECK-NEXT: ret <3 x i64> [[BO]]
+;
+ %ins = insertelement <3 x i64> poison, i64 %x, i32 2
+ %bo = mul <3 x i64> %ins, <i64 42, i64 undef, i64 -42>
+ ret <3 x i64> %bo
+}
+
+define <16 x i8> @mul_constant_multiuse(i8 %a0, <16 x i8> %a1) {
+; SSE-LABEL: @mul_constant_multiuse(
+; SSE-NEXT: [[INS:%.*]] = insertelement <16 x i8> <i8 undef, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, i8 [[A0:%.*]], i32 0
+; SSE-NEXT: [[MUL:%.*]] = mul <16 x i8> [[INS]], <i8 3, i8 7, i8 9, i8 11, i8 13, i8 15, i8 17, i8 19, i8 21, i8 23, i8 25, i8 27, i8 29, i8 31, i8 33, i8 35>
+; SSE-NEXT: [[AND:%.*]] = and <16 x i8> [[INS]], [[A1:%.*]]
+; SSE-NEXT: [[XOR:%.*]] = xor <16 x i8> [[AND]], [[MUL]]
+; SSE-NEXT: ret <16 x i8> [[XOR]]
+;
+; AVX-LABEL: @mul_constant_multiuse(
+; AVX-NEXT: [[INS:%.*]] = insertelement <16 x i8> <i8 undef, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, i8 [[A0:%.*]], i32 0
+; AVX-NEXT: [[MUL_SCALAR:%.*]] = mul i8 [[A0]], 3
+; AVX-NEXT: [[MUL:%.*]] = insertelement <16 x i8> <i8 undef, i8 7, i8 18, i8 33, i8 52, i8 75, i8 102, i8 -123, i8 -88, i8 -49, i8 -6, i8 41, i8 92, i8 -109, i8 -50, i8 13>, i8 [[MUL_SCALAR]], i64 0
+; AVX-NEXT: [[AND:%.*]] = and <16 x i8> [[INS]], [[A1:%.*]]
+; AVX-NEXT: [[XOR:%.*]] = xor <16 x i8> [[AND]], [[MUL]]
+; AVX-NEXT: ret <16 x i8> [[XOR]]
+;
+ %ins = insertelement <16 x i8> <i8 undef, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, i8 %a0, i32 0
+ %mul = mul <16 x i8> %ins, <i8 3, i8 7, i8 9, i8 11, i8 13, i8 15, i8 17, i8 19, i8 21, i8 23, i8 25, i8 27, i8 29, i8 31, i8 33, i8 35>
+ %and = and <16 x i8> %ins, %a1
+ %xor = xor <16 x i8> %and, %mul
+ ret <16 x i8> %xor
+}
+
+define <2 x i64> @shl_constant_op0(i64 %x) {
+; CHECK-LABEL: @shl_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = shl i64 2, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = shl <2 x i64> <i64 undef, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @shl_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @shl_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = shl i64 2, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = shl <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @shl_constant_op0_load(i64* %p) {
+; CHECK-LABEL: @shl_constant_op0_load(
+; CHECK-NEXT: [[LD:%.*]] = load i64, i64* [[P:%.*]], align 4
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i64> poison, i64 [[LD]], i32 1
+; CHECK-NEXT: [[BO:%.*]] = shl <2 x i64> <i64 undef, i64 2>, [[INS]]
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ld = load i64, i64* %p
+ %ins = insertelement <2 x i64> poison, i64 %ld, i32 1
+ %bo = shl <2 x i64> <i64 undef, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <4 x i32> @shl_constant_op0_multiuse(i32 %a0, <4 x i32> %a1) {
+; SSE-LABEL: @shl_constant_op0_multiuse(
+; SSE-NEXT: [[INS:%.*]] = insertelement <4 x i32> <i32 undef, i32 1, i32 2, i32 3>, i32 [[A0:%.*]], i32 0
+; SSE-NEXT: [[MUL_SCALAR:%.*]] = shl i32 [[A0]], 3
+; SSE-NEXT: [[MUL:%.*]] = insertelement <4 x i32> <i32 0, i32 16, i32 64, i32 192>, i32 [[MUL_SCALAR]], i64 0
+; SSE-NEXT: [[AND:%.*]] = and <4 x i32> [[INS]], [[A1:%.*]]
+; SSE-NEXT: [[XOR:%.*]] = xor <4 x i32> [[AND]], [[MUL]]
+; SSE-NEXT: ret <4 x i32> [[XOR]]
+;
+; AVX-LABEL: @shl_constant_op0_multiuse(
+; AVX-NEXT: [[INS:%.*]] = insertelement <4 x i32> <i32 undef, i32 1, i32 2, i32 3>, i32 [[A0:%.*]], i32 0
+; AVX-NEXT: [[MUL:%.*]] = shl <4 x i32> [[INS]], <i32 3, i32 4, i32 5, i32 6>
+; AVX-NEXT: [[AND:%.*]] = and <4 x i32> [[INS]], [[A1:%.*]]
+; AVX-NEXT: [[XOR:%.*]] = xor <4 x i32> [[AND]], [[MUL]]
+; AVX-NEXT: ret <4 x i32> [[XOR]]
+;
+ %ins = insertelement <4 x i32> <i32 undef, i32 1, i32 2, i32 3>, i32 %a0, i32 0
+ %mul = shl <4 x i32> %ins, <i32 3, i32 4, i32 5, i32 6>
+ %and = and <4 x i32> %ins, %a1
+ %xor = xor <4 x i32> %and, %mul
+ ret <4 x i32> %xor
+}
+
+define <2 x i64> @shl_constant_op1(i64 %x) {
+; CHECK-LABEL: @shl_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = shl nuw i64 [[X:%.*]], 5
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = shl nuw <2 x i64> %ins, <i64 5, i64 undef>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @shl_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @shl_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = shl nuw i64 [[X:%.*]], 5
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = shl nuw <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @shl_constant_op1_load(i64* %p) {
+; CHECK-LABEL: @shl_constant_op1_load(
+; CHECK-NEXT: [[LD:%.*]] = load i64, i64* [[P:%.*]], align 4
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i64> poison, i64 [[LD]], i32 0
+; CHECK-NEXT: [[BO:%.*]] = shl nuw <2 x i64> [[INS]], <i64 5, i64 2>
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ld = load i64, i64* %p
+ %ins = insertelement <2 x i64> poison, i64 %ld, i32 0
+ %bo = shl nuw <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @ashr_constant_op0(i64 %x) {
+; CHECK-LABEL: @ashr_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = ashr exact i64 2, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = ashr exact <2 x i64> <i64 undef, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @ashr_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @ashr_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = ashr exact i64 2, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = ashr exact <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @ashr_constant_op1(i64 %x) {
+; CHECK-LABEL: @ashr_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = ashr i64 [[X:%.*]], 5
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = ashr <2 x i64> %ins, <i64 5, i64 undef>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @ashr_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @ashr_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = ashr i64 [[X:%.*]], 5
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = ashr <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @lshr_constant_op0(i64 %x) {
+; CHECK-LABEL: @lshr_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = lshr i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = lshr <2 x i64> <i64 5, i64 undef>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @lshr_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @lshr_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = lshr i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = lshr <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @lshr_constant_op1(i64 %x) {
+; CHECK-LABEL: @lshr_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = lshr exact i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = lshr exact <2 x i64> %ins, <i64 undef, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @lshr_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @lshr_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = lshr exact i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = lshr exact <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @urem_constant_op0(i64 %x) {
+; CHECK-LABEL: @urem_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = urem i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = urem <2 x i64> <i64 5, i64 undef>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @urem_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @urem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = urem i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = urem <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @urem_constant_op1(i64 %x) {
+; CHECK-LABEL: @urem_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = urem i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = urem <2 x i64> %ins, <i64 undef, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @urem_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @urem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = urem i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = urem <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @srem_constant_op0(i64 %x) {
+; CHECK-LABEL: @srem_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = srem i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = srem <2 x i64> <i64 5, i64 undef>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @srem_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @srem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = srem i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = srem <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @srem_constant_op1(i64 %x) {
+; CHECK-LABEL: @srem_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = srem i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = srem <2 x i64> %ins, <i64 undef, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @srem_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @srem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = srem i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = srem <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @udiv_constant_op0(i64 %x) {
+; CHECK-LABEL: @udiv_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = udiv exact i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = udiv exact <2 x i64> <i64 5, i64 undef>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @udiv_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @udiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = udiv exact i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = udiv exact <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @udiv_constant_op1(i64 %x) {
+; CHECK-LABEL: @udiv_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = udiv i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = udiv <2 x i64> %ins, <i64 undef, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @udiv_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @udiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = udiv i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = udiv <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @sdiv_constant_op0(i64 %x) {
+; CHECK-LABEL: @sdiv_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sdiv i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = sdiv <2 x i64> <i64 5, i64 undef>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @sdiv_constant_op0_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @sdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sdiv i64 5, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = sdiv <2 x i64> <i64 5, i64 2>, %ins
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @sdiv_constant_op1(i64 %x) {
+; CHECK-LABEL: @sdiv_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sdiv exact i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = sdiv exact <2 x i64> %ins, <i64 undef, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @sdiv_constant_op1_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @sdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = sdiv exact i64 [[X:%.*]], 2
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = sdiv exact <2 x i64> %ins, <i64 5, i64 2>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @and_constant(i64 %x) {
+; CHECK-LABEL: @and_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = and i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = and <2 x i64> %ins, <i64 42, i64 undef>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @and_constant_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @and_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = and i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = and <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @or_constant(i64 %x) {
+; CHECK-LABEL: @or_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = or i64 [[X:%.*]], -42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = or <2 x i64> %ins, <i64 undef, i64 -42>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @or_constant_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @or_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = or i64 [[X:%.*]], -42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 1
+ %bo = or <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @xor_constant(i64 %x) {
+; CHECK-LABEL: @xor_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = xor i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = xor <2 x i64> %ins, <i64 42, i64 undef>
+ ret <2 x i64> %bo
+}
+
+define <2 x i64> @xor_constant_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @xor_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = xor i64 [[X:%.*]], 42
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x i64> poison, i64 [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i64> [[BO]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %bo = xor <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i64> %bo
+}
+
+define <2 x double> @fadd_constant(double %x) {
+; CHECK-LABEL: @fadd_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fadd double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fadd <2 x double> %ins, <double 42.0, double undef>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fadd_constant_not_undef_lane(double %x) {
+; CHECK-LABEL: @fadd_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fadd double [[X:%.*]], -4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = fadd <2 x double> %ins, <double 42.0, double -42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fsub_constant_op0(double %x) {
+; CHECK-LABEL: @fsub_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fsub fast double 4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fsub fast <2 x double> <double 42.0, double undef>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fsub_constant_op0_not_undef_lane(double %x) {
+; CHECK-LABEL: @fsub_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fsub nsz double -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = fsub nsz <2 x double> <double 42.0, double -42.0>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fsub_constant_op1(double %x) {
+; CHECK-LABEL: @fsub_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fsub double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = fsub <2 x double> %ins, <double undef, double 42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fsub_constant_op1_not_undef_lane(double %x) {
+; CHECK-LABEL: @fsub_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fsub double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fsub <2 x double> %ins, <double 42.0, double -42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fmul_constant(double %x) {
+; CHECK-LABEL: @fmul_constant(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fmul reassoc double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fmul reassoc <2 x double> %ins, <double 42.0, double undef>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fmul_constant_not_undef_lane(double %x) {
+; CHECK-LABEL: @fmul_constant_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fmul double [[X:%.*]], -4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = fmul <2 x double> %ins, <double 42.0, double -42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fdiv_constant_op0(double %x) {
+; CHECK-LABEL: @fdiv_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fdiv nnan double 4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = fdiv nnan <2 x double> <double undef, double 42.0>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fdiv_constant_op0_not_undef_lane(double %x) {
+; CHECK-LABEL: @fdiv_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fdiv ninf double 4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fdiv ninf <2 x double> <double 42.0, double -42.0>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fdiv_constant_op1(double %x) {
+; CHECK-LABEL: @fdiv_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fdiv double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fdiv <2 x double> %ins, <double 42.0, double undef>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @fdiv_constant_op1_not_undef_lane(double %x) {
+; CHECK-LABEL: @fdiv_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = fdiv double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = fdiv <2 x double> %ins, <double 42.0, double -42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @frem_constant_op0(double %x) {
+; CHECK-LABEL: @frem_constant_op0(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = frem fast double 4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = frem fast <2 x double> <double 42.0, double undef>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @frem_constant_op0_not_undef_lane(double %x) {
+; CHECK-LABEL: @frem_constant_op0_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = frem double -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = frem <2 x double> <double 42.0, double -42.0>, %ins
+ ret <2 x double> %bo
+}
+
+define <2 x double> @frem_constant_op1(double %x) {
+; CHECK-LABEL: @frem_constant_op1(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = frem ninf double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %bo = frem ninf <2 x double> %ins, <double undef, double 42.0>
+ ret <2 x double> %bo
+}
+
+define <2 x double> @frem_constant_op1_not_undef_lane(double %x) {
+; CHECK-LABEL: @frem_constant_op1_not_undef_lane(
+; CHECK-NEXT: [[BO_SCALAR:%.*]] = frem nnan double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[BO:%.*]] = insertelement <2 x double> poison, double [[BO_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x double> [[BO]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %bo = frem nnan <2 x double> %ins, <double 42.0, double -42.0>
+ ret <2 x double> %bo
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
new file mode 100644
index 000000000000..dd0ef45c7f02
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/load-inseltpoison.ll
@@ -0,0 +1,649 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define float @matching_fp_scalar(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_scalar(
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[P:%.*]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %r = load float, float* %p, align 16
+ ret float %r
+}
+
+define float @matching_fp_scalar_volatile(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_scalar_volatile(
+; CHECK-NEXT: [[R:%.*]] = load volatile float, float* [[P:%.*]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %r = load volatile float, float* %p, align 16
+ ret float %r
+}
+
+define double @larger_fp_scalar(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @larger_fp_scalar(
+; CHECK-NEXT: [[BC:%.*]] = bitcast float* [[P:%.*]] to double*
+; CHECK-NEXT: [[R:%.*]] = load double, double* [[BC]], align 16
+; CHECK-NEXT: ret double [[R]]
+;
+ %bc = bitcast float* %p to double*
+ %r = load double, double* %bc, align 16
+ ret double %r
+}
+
+define float @smaller_fp_scalar(double* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @smaller_fp_scalar(
+; CHECK-NEXT: [[BC:%.*]] = bitcast double* [[P:%.*]] to float*
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %bc = bitcast double* %p to float*
+ %r = load float, float* %bc, align 16
+ ret float %r
+}
+
+define float @matching_fp_vector(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_vector(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to float*
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %bc = bitcast <4 x float>* %p to float*
+ %r = load float, float* %bc, align 16
+ ret float %r
+}
+
+define float @matching_fp_vector_gep00(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep00(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 0
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 0
+ %r = load float, float* %gep, align 16
+ ret float %r
+}
+
+define float @matching_fp_vector_gep01(<4 x float>* align 16 dereferenceable(20) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep01(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[GEP]], align 4
+; CHECK-NEXT: ret float [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 1
+ %r = load float, float* %gep, align 4
+ ret float %r
+}
+
+define float @matching_fp_vector_gep01_deref(<4 x float>* align 16 dereferenceable(19) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep01_deref(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[GEP]], align 4
+; CHECK-NEXT: ret float [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 1
+ %r = load float, float* %gep, align 4
+ ret float %r
+}
+
+define float @matching_fp_vector_gep10(<4 x float>* align 16 dereferenceable(32) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep10(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 1, i64 0
+ %r = load float, float* %gep, align 16
+ ret float %r
+}
+
+define float @matching_fp_vector_gep10_deref(<4 x float>* align 16 dereferenceable(31) %p) {
+; CHECK-LABEL: @matching_fp_vector_gep10_deref(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <4 x float>, <4 x float>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[GEP]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 1, i64 0
+ %r = load float, float* %gep, align 16
+ ret float %r
+}
+
+define float @nonmatching_int_vector(<2 x i64>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @nonmatching_int_vector(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <2 x i64>* [[P:%.*]] to float*
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[BC]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %bc = bitcast <2 x i64>* %p to float*
+ %r = load float, float* %bc, align 16
+ ret float %r
+}
+
+define double @less_aligned(double* align 4 dereferenceable(16) %p) {
+; CHECK-LABEL: @less_aligned(
+; CHECK-NEXT: [[R:%.*]] = load double, double* [[P:%.*]], align 4
+; CHECK-NEXT: ret double [[R]]
+;
+ %r = load double, double* %p, align 4
+ ret double %r
+}
+
+define float @matching_fp_scalar_small_deref(float* align 16 dereferenceable(15) %p) {
+; CHECK-LABEL: @matching_fp_scalar_small_deref(
+; CHECK-NEXT: [[R:%.*]] = load float, float* [[P:%.*]], align 16
+; CHECK-NEXT: ret float [[R]]
+;
+ %r = load float, float* %p, align 16
+ ret float %r
+}
+
+define i64 @larger_int_scalar(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @larger_int_scalar(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to i64*
+; CHECK-NEXT: [[R:%.*]] = load i64, i64* [[BC]], align 16
+; CHECK-NEXT: ret i64 [[R]]
+;
+ %bc = bitcast <4 x float>* %p to i64*
+ %r = load i64, i64* %bc, align 16
+ ret i64 %r
+}
+
+define i8 @smaller_int_scalar(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @smaller_int_scalar(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <4 x float>* [[P:%.*]] to i8*
+; CHECK-NEXT: [[R:%.*]] = load i8, i8* [[BC]], align 16
+; CHECK-NEXT: ret i8 [[R]]
+;
+ %bc = bitcast <4 x float>* %p to i8*
+ %r = load i8, i8* %bc, align 16
+ ret i8 %r
+}
+
+define double @larger_fp_scalar_256bit_vec(<8 x float>* align 32 dereferenceable(32) %p) {
+; CHECK-LABEL: @larger_fp_scalar_256bit_vec(
+; CHECK-NEXT: [[BC:%.*]] = bitcast <8 x float>* [[P:%.*]] to double*
+; CHECK-NEXT: [[R:%.*]] = load double, double* [[BC]], align 32
+; CHECK-NEXT: ret double [[R]]
+;
+ %bc = bitcast <8 x float>* %p to double*
+ %r = load double, double* %bc, align 32
+ ret double %r
+}
+
+define <4 x float> @load_f32_insert_v4f32(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+define <4 x float> @casted_load_f32_insert_v4f32(<4 x float>* align 4 dereferenceable(16) %p) {
+; CHECK-LABEL: @casted_load_f32_insert_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %b = bitcast <4 x float>* %p to float*
+ %s = load float, float* %b, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+; Element type does not change cost.
+
+define <4 x i32> @load_i32_insert_v4i32(i32* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_i32_insert_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[P:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %s = load i32, i32* %p, align 4
+ %r = insertelement <4 x i32> poison, i32 %s, i32 0
+ ret <4 x i32> %r
+}
+
+; Pointer type does not change cost.
+
+define <4 x i32> @casted_load_i32_insert_v4i32(<16 x i8>* align 4 dereferenceable(16) %p) {
+; CHECK-LABEL: @casted_load_i32_insert_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8>* [[P:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %b = bitcast <16 x i8>* %p to i32*
+ %s = load i32, i32* %b, align 4
+ %r = insertelement <4 x i32> poison, i32 %s, i32 0
+ ret <4 x i32> %r
+}
+
+; This is canonical form for vector element access.
+
+define <4 x float> @gep00_load_f32_insert_v4f32(<4 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @gep00_load_f32_insert_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[P:%.*]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float>* %p, i64 0, i64 0
+ %s = load float, float* %gep, align 16
+ %r = insertelement <4 x float> poison, float %s, i64 0
+ ret <4 x float> %r
+}
+
+; Should work with addrspace as well.
+
+define <4 x float> @gep00_load_f32_insert_v4f32_addrspace(<4 x float> addrspace(44)* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @gep00_load_f32_insert_v4f32_addrspace(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float> addrspace(44)* [[P:%.*]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %gep = getelementptr inbounds <4 x float>, <4 x float> addrspace(44)* %p, i64 0, i64 0
+ %s = load float, float addrspace(44)* %gep, align 16
+ %r = insertelement <4 x float> poison, float %s, i64 0
+ ret <4 x float> %r
+}
+
+; If there are enough dereferenceable bytes, we can offset the vector load.
+
+define <8 x i16> @gep01_load_i16_insert_v8i16(<8 x i16>* align 16 dereferenceable(18) %p) {
+; CHECK-LABEL: @gep01_load_i16_insert_v8i16(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[GEP]] to <8 x i16>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 2
+; CHECK-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 0, i64 1
+ %s = load i16, i16* %gep, align 2
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Can't safely load the offset vector, but can load+shuffle if it is profitable.
+
+define <8 x i16> @gep01_load_i16_insert_v8i16_deref(<8 x i16>* align 16 dereferenceable(17) %p) {
+; SSE2-LABEL: @gep01_load_i16_insert_v8i16_deref(
+; SSE2-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 0, i64 1
+; SSE2-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 2
+; SSE2-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; SSE2-NEXT: ret <8 x i16> [[R]]
+;
+; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref(
+; AVX2-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[P:%.*]], align 16
+; AVX2-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 0, i64 1
+ %s = load i16, i16* %gep, align 2
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Verify that alignment of the new load is not over-specified.
+
+define <8 x i16> @gep01_load_i16_insert_v8i16_deref_minalign(<8 x i16>* align 2 dereferenceable(16) %p) {
+; SSE2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
+; SSE2-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 0, i64 1
+; SSE2-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 8
+; SSE2-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; SSE2-NEXT: ret <8 x i16> [[R]]
+;
+; AVX2-LABEL: @gep01_load_i16_insert_v8i16_deref_minalign(
+; AVX2-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* [[P:%.*]], align 2
+; AVX2-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 0, i64 1
+ %s = load i16, i16* %gep, align 8
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Negative test - if we are shuffling a load from the base pointer, the address offset
+; must be a multiple of element size.
+; TODO: Could bitcast around this limitation.
+
+define <4 x i32> @gep01_bitcast_load_i32_insert_v4i32(<16 x i8>* align 1 dereferenceable(16) %p) {
+; CHECK-LABEL: @gep01_bitcast_load_i32_insert_v4i32(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <16 x i8>, <16 x i8>* [[P:%.*]], i64 0, i64 1
+; CHECK-NEXT: [[B:%.*]] = bitcast i8* [[GEP]] to i32*
+; CHECK-NEXT: [[S:%.*]] = load i32, i32* [[B]], align 1
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i32> poison, i32 [[S]], i64 0
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %gep = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i64 0, i64 1
+ %b = bitcast i8* %gep to i32*
+ %s = load i32, i32* %b, align 1
+ %r = insertelement <4 x i32> poison, i32 %s, i64 0
+ ret <4 x i32> %r
+}
+
+define <4 x i32> @gep012_bitcast_load_i32_insert_v4i32(<16 x i8>* align 1 dereferenceable(20) %p) {
+; CHECK-LABEL: @gep012_bitcast_load_i32_insert_v4i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <16 x i8>* [[P:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 1
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %gep = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i64 0, i64 12
+ %b = bitcast i8* %gep to i32*
+ %s = load i32, i32* %b, align 1
+ %r = insertelement <4 x i32> poison, i32 %s, i64 0
+ ret <4 x i32> %r
+}
+
+; Negative test - if we are shuffling a load from the base pointer, the address offset
+; must be a multiple of element size and the offset must be low enough to fit in the vector
+; (bitcasting would not help this case).
+
+define <4 x i32> @gep013_bitcast_load_i32_insert_v4i32(<16 x i8>* align 1 dereferenceable(20) %p) {
+; CHECK-LABEL: @gep013_bitcast_load_i32_insert_v4i32(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <16 x i8>, <16 x i8>* [[P:%.*]], i64 0, i64 13
+; CHECK-NEXT: [[B:%.*]] = bitcast i8* [[GEP]] to i32*
+; CHECK-NEXT: [[S:%.*]] = load i32, i32* [[B]], align 1
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i32> poison, i32 [[S]], i64 0
+; CHECK-NEXT: ret <4 x i32> [[R]]
+;
+ %gep = getelementptr inbounds <16 x i8>, <16 x i8>* %p, i64 0, i64 13
+ %b = bitcast i8* %gep to i32*
+ %s = load i32, i32* %b, align 1
+ %r = insertelement <4 x i32> poison, i32 %s, i64 0
+ ret <4 x i32> %r
+}
+
+; If there are enough dereferenceable bytes, we can offset the vector load.
+
+define <8 x i16> @gep10_load_i16_insert_v8i16(<8 x i16>* align 16 dereferenceable(32) %p) {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i16* [[GEP]] to <8 x i16>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Negative test - disable under asan because widened load can cause spurious
+; use-after-poison issues when __asan_poison_memory_region is used.
+
+define <8 x i16> @gep10_load_i16_insert_v8i16_asan(<8 x i16>* align 16 dereferenceable(32) %p) sanitize_address {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16_asan(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 16
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; hwasan and memtag should be similarly suppressed.
+
+define <8 x i16> @gep10_load_i16_insert_v8i16_hwasan(<8 x i16>* align 16 dereferenceable(32) %p) sanitize_hwaddress {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16_hwasan(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 16
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+define <8 x i16> @gep10_load_i16_insert_v8i16_memtag(<8 x i16>* align 16 dereferenceable(32) %p) sanitize_memtag {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16_memtag(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 16
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Negative test - disable under tsan because widened load may overlap bytes
+; being concurrently modified. tsan does not know that some bytes are undef.
+
+define <8 x i16> @gep10_load_i16_insert_v8i16_tsan(<8 x i16>* align 16 dereferenceable(32) %p) sanitize_thread {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16_tsan(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 16
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Negative test - can't safely load the offset vector, but could load+shuffle.
+
+define <8 x i16> @gep10_load_i16_insert_v8i16_deref(<8 x i16>* align 16 dereferenceable(31) %p) {
+; CHECK-LABEL: @gep10_load_i16_insert_v8i16_deref(
+; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds <8 x i16>, <8 x i16>* [[P:%.*]], i64 1, i64 0
+; CHECK-NEXT: [[S:%.*]] = load i16, i16* [[GEP]], align 16
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; CHECK-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <8 x i16>, <8 x i16>* %p, i64 1, i64 0
+ %s = load i16, i16* %gep, align 16
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
+
+; Negative test - do not alter volatile.
+
+define <4 x float> @load_f32_insert_v4f32_volatile(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v4f32_volatile(
+; CHECK-NEXT: [[S:%.*]] = load volatile float, float* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> poison, float [[S]], i32 0
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %s = load volatile float, float* %p, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+; Pointer is not as aligned as load, but that's ok.
+; The new load uses the larger alignment value.
+
+define <4 x float> @load_f32_insert_v4f32_align(float* align 1 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v4f32_align(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+; Negative test - not enough bytes.
+
+define <4 x float> @load_f32_insert_v4f32_deref(float* align 4 dereferenceable(15) %p) {
+; CHECK-LABEL: @load_f32_insert_v4f32_deref(
+; CHECK-NEXT: [[S:%.*]] = load float, float* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x float> poison, float [[S]], i32 0
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+define <8 x i32> @load_i32_insert_v8i32(i32* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_i32_insert_v8i32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32* [[P:%.*]] to <4 x i32>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <8 x i32> [[R]]
+;
+ %s = load i32, i32* %p, align 4
+ %r = insertelement <8 x i32> poison, i32 %s, i32 0
+ ret <8 x i32> %r
+}
+
+define <8 x i32> @casted_load_i32_insert_v8i32(<4 x i32>* align 4 dereferenceable(16) %p) {
+; CHECK-LABEL: @casted_load_i32_insert_v8i32(
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <8 x i32> [[R]]
+;
+ %b = bitcast <4 x i32>* %p to i32*
+ %s = load i32, i32* %b, align 4
+ %r = insertelement <8 x i32> poison, i32 %s, i32 0
+ ret <8 x i32> %r
+}
+
+define <16 x float> @load_f32_insert_v16f32(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v16f32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <16 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <16 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <16 x float> poison, float %s, i32 0
+ ret <16 x float> %r
+}
+
+define <2 x float> @load_f32_insert_v2f32(float* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_f32_insert_v2f32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast float* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <2 x i32> <i32 0, i32 undef>
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <2 x float> poison, float %s, i32 0
+ ret <2 x float> %r
+}
+
+; Negative test - suppress load widening for asan/hwasan/memtag/tsan.
+
+define <2 x float> @load_f32_insert_v2f32_asan(float* align 16 dereferenceable(16) %p) sanitize_address {
+; CHECK-LABEL: @load_f32_insert_v2f32_asan(
+; CHECK-NEXT: [[S:%.*]] = load float, float* [[P:%.*]], align 4
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x float> poison, float [[S]], i32 0
+; CHECK-NEXT: ret <2 x float> [[R]]
+;
+ %s = load float, float* %p, align 4
+ %r = insertelement <2 x float> poison, float %s, i32 0
+ ret <2 x float> %r
+}
+
+declare float* @getscaleptr()
+define void @PR47558_multiple_use_load(<2 x float>* nocapture nonnull %resultptr, <2 x float>* nocapture nonnull readonly %opptr) {
+; CHECK-LABEL: @PR47558_multiple_use_load(
+; CHECK-NEXT: [[SCALEPTR:%.*]] = tail call nonnull align 16 dereferenceable(64) float* @getscaleptr()
+; CHECK-NEXT: [[OP:%.*]] = load <2 x float>, <2 x float>* [[OPPTR:%.*]], align 4
+; CHECK-NEXT: [[SCALE:%.*]] = load float, float* [[SCALEPTR]], align 16
+; CHECK-NEXT: [[T1:%.*]] = insertelement <2 x float> poison, float [[SCALE]], i32 0
+; CHECK-NEXT: [[T2:%.*]] = insertelement <2 x float> [[T1]], float [[SCALE]], i32 1
+; CHECK-NEXT: [[T3:%.*]] = fmul <2 x float> [[OP]], [[T2]]
+; CHECK-NEXT: [[T4:%.*]] = extractelement <2 x float> [[T3]], i32 0
+; CHECK-NEXT: [[RESULT0:%.*]] = insertelement <2 x float> poison, float [[T4]], i32 0
+; CHECK-NEXT: [[T5:%.*]] = extractelement <2 x float> [[T3]], i32 1
+; CHECK-NEXT: [[RESULT1:%.*]] = insertelement <2 x float> [[RESULT0]], float [[T5]], i32 1
+; CHECK-NEXT: store <2 x float> [[RESULT1]], <2 x float>* [[RESULTPTR:%.*]], align 8
+; CHECK-NEXT: ret void
+;
+ %scaleptr = tail call nonnull align 16 dereferenceable(64) float* @getscaleptr()
+ %op = load <2 x float>, <2 x float>* %opptr, align 4
+ %scale = load float, float* %scaleptr, align 16
+ %t1 = insertelement <2 x float> poison, float %scale, i32 0
+ %t2 = insertelement <2 x float> %t1, float %scale, i32 1
+ %t3 = fmul <2 x float> %op, %t2
+ %t4 = extractelement <2 x float> %t3, i32 0
+ %result0 = insertelement <2 x float> poison, float %t4, i32 0
+ %t5 = extractelement <2 x float> %t3, i32 1
+ %result1 = insertelement <2 x float> %result0, float %t5, i32 1
+ store <2 x float> %result1, <2 x float>* %resultptr, align 8
+ ret void
+}
+
+define <4 x float> @load_v2f32_extract_insert_v4f32(<2 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_v2f32_extract_insert_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x float>* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %l = load <2 x float>, <2 x float>* %p, align 4
+ %s = extractelement <2 x float> %l, i32 0
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+define <4 x float> @load_v8f32_extract_insert_v4f32(<8 x float>* align 16 dereferenceable(16) %p) {
+; CHECK-LABEL: @load_v8f32_extract_insert_v4f32(
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x float>* [[P:%.*]] to <4 x float>*
+; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 16
+; CHECK-NEXT: [[R:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %l = load <8 x float>, <8 x float>* %p, align 4
+ %s = extractelement <8 x float> %l, i32 0
+ %r = insertelement <4 x float> poison, float %s, i32 0
+ ret <4 x float> %r
+}
+
+define <8 x i32> @load_v1i32_extract_insert_v8i32_extra_use(<1 x i32>* align 16 dereferenceable(16) %p, <1 x i32>* %store_ptr) {
+; CHECK-LABEL: @load_v1i32_extract_insert_v8i32_extra_use(
+; CHECK-NEXT: [[L:%.*]] = load <1 x i32>, <1 x i32>* [[P:%.*]], align 4
+; CHECK-NEXT: store <1 x i32> [[L]], <1 x i32>* [[STORE_PTR:%.*]], align 4
+; CHECK-NEXT: [[S:%.*]] = extractelement <1 x i32> [[L]], i32 0
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i32> poison, i32 [[S]], i32 0
+; CHECK-NEXT: ret <8 x i32> [[R]]
+;
+ %l = load <1 x i32>, <1 x i32>* %p, align 4
+ store <1 x i32> %l, <1 x i32>* %store_ptr
+ %s = extractelement <1 x i32> %l, i32 0
+ %r = insertelement <8 x i32> poison, i32 %s, i32 0
+ ret <8 x i32> %r
+}
+
+; Can't safely load the offset vector, but can load+shuffle if it is profitable.
+
+define <8 x i16> @gep1_load_v2i16_extract_insert_v8i16(<2 x i16>* align 1 dereferenceable(16) %p) {
+; SSE2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
+; SSE2-NEXT: [[GEP:%.*]] = getelementptr inbounds <2 x i16>, <2 x i16>* [[P:%.*]], i64 1
+; SSE2-NEXT: [[L:%.*]] = load <2 x i16>, <2 x i16>* [[GEP]], align 8
+; SSE2-NEXT: [[S:%.*]] = extractelement <2 x i16> [[L]], i32 0
+; SSE2-NEXT: [[R:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0
+; SSE2-NEXT: ret <8 x i16> [[R]]
+;
+; AVX2-LABEL: @gep1_load_v2i16_extract_insert_v8i16(
+; AVX2-NEXT: [[TMP1:%.*]] = bitcast <2 x i16>* [[P:%.*]] to <8 x i16>*
+; AVX2-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* [[TMP1]], align 4
+; AVX2-NEXT: [[R:%.*]] = shufflevector <8 x i16> [[TMP2]], <8 x i16> undef, <8 x i32> <i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; AVX2-NEXT: ret <8 x i16> [[R]]
+;
+ %gep = getelementptr inbounds <2 x i16>, <2 x i16>* %p, i64 1
+ %l = load <2 x i16>, <2 x i16>* %gep, align 8
+ %s = extractelement <2 x i16> %l, i32 0
+ %r = insertelement <8 x i16> poison, i16 %s, i64 0
+ ret <8 x i16> %r
+}
diff --git a/llvm/test/Transforms/VectorCombine/X86/scalarize-cmp-inseltpoison.ll b/llvm/test/Transforms/VectorCombine/X86/scalarize-cmp-inseltpoison.ll
new file mode 100644
index 000000000000..a9214aa26e76
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/X86/scalarize-cmp-inseltpoison.ll
@@ -0,0 +1,290 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=sse2 | FileCheck %s
+; RUN: opt < %s -vector-combine -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s
+
+declare void @use(<4 x i32>)
+declare void @usef(<4 x float>)
+
+; Eliminating an insert is profitable.
+
+define <16 x i1> @ins0_ins0_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @ins0_ins0_i8(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp eq i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <16 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <16 x i1> [[R]]
+;
+ %i0 = insertelement <16 x i8> poison, i8 %x, i32 0
+ %i1 = insertelement <16 x i8> poison, i8 %y, i32 0
+ %r = icmp eq <16 x i8> %i0, %i1
+ ret <16 x i1> %r
+}
+
+; Eliminating an insert is still profitable. Mismatch types on index is ok.
+
+define <8 x i1> @ins5_ins5_i16(i16 %x, i16 %y) {
+; CHECK-LABEL: @ins5_ins5_i16(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <8 x i1> poison, i1 [[R_SCALAR]], i64 5
+; CHECK-NEXT: ret <8 x i1> [[R]]
+;
+ %i0 = insertelement <8 x i16> poison, i16 %x, i8 5
+ %i1 = insertelement <8 x i16> poison, i16 %y, i32 5
+ %r = icmp sgt <8 x i16> %i0, %i1
+ ret <8 x i1> %r
+}
+
+; The new vector constant is calculated by constant folding.
+
+define <2 x i1> @ins1_ins1_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @ins1_ins1_i64(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp sle i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> <i1 true, i1 false>, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %i0 = insertelement <2 x i64> zeroinitializer, i64 %x, i64 1
+ %i1 = insertelement <2 x i64> <i64 1, i64 -1>, i64 %y, i32 1
+ %r = icmp sle <2 x i64> %i0, %i1
+ ret <2 x i1> %r
+}
+
+; The inserts are free, but it's still better to scalarize.
+
+define <2 x i1> @ins0_ins0_f64(double %x, double %y) {
+; CHECK-LABEL: @ins0_ins0_f64(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp nnan ninf uge double [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %i0 = insertelement <2 x double> poison, double %x, i32 0
+ %i1 = insertelement <2 x double> poison, double %y, i32 0
+ %r = fcmp nnan ninf uge <2 x double> %i0, %i1
+ ret <2 x i1> %r
+}
+
+; Negative test - mismatched indexes (but could fold this).
+
+define <16 x i1> @ins1_ins0_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @ins1_ins0_i8(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <16 x i8> poison, i8 [[X:%.*]], i32 1
+; CHECK-NEXT: [[I1:%.*]] = insertelement <16 x i8> poison, i8 [[Y:%.*]], i32 0
+; CHECK-NEXT: [[R:%.*]] = icmp sle <16 x i8> [[I0]], [[I1]]
+; CHECK-NEXT: ret <16 x i1> [[R]]
+;
+ %i0 = insertelement <16 x i8> poison, i8 %x, i32 1
+ %i1 = insertelement <16 x i8> poison, i8 %y, i32 0
+ %r = icmp sle <16 x i8> %i0, %i1
+ ret <16 x i1> %r
+}
+
+; Base vector does not have to be undef.
+
+define <4 x i1> @ins0_ins0_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @ins0_ins0_i32(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %i0 = insertelement <4 x i32> zeroinitializer, i32 %x, i32 0
+ %i1 = insertelement <4 x i32> poison, i32 %y, i32 0
+ %r = icmp ne <4 x i32> %i0, %i1
+ ret <4 x i1> %r
+}
+
+; Extra use is accounted for in cost calculation.
+
+define <4 x i1> @ins0_ins0_i32_use(i32 %x, i32 %y) {
+; CHECK-LABEL: @ins0_ins0_i32_use(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i32 0
+; CHECK-NEXT: call void @use(<4 x i32> [[I0]])
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp ugt i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %i0 = insertelement <4 x i32> poison, i32 %x, i32 0
+ call void @use(<4 x i32> %i0)
+ %i1 = insertelement <4 x i32> poison, i32 %y, i32 0
+ %r = icmp ugt <4 x i32> %i0, %i1
+ ret <4 x i1> %r
+}
+
+; Extra use is accounted for in cost calculation.
+
+define <4 x i1> @ins1_ins1_f32_use(float %x, float %y) {
+; CHECK-LABEL: @ins1_ins1_f32_use(
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 1
+; CHECK-NEXT: call void @usef(<4 x float> [[I1]])
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp ogt float [[X:%.*]], [[Y]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %i0 = insertelement <4 x float> poison, float %x, i32 1
+ %i1 = insertelement <4 x float> poison, float %y, i32 1
+ call void @usef(<4 x float> %i1)
+ %r = fcmp ogt <4 x float> %i0, %i1
+ ret <4 x i1> %r
+}
+
+; If the scalar cmp is not cheaper than the vector cmp, extra uses can prevent the transform.
+
+define <4 x i1> @ins2_ins2_f32_uses(float %x, float %y) {
+; CHECK-LABEL: @ins2_ins2_f32_uses(
+; CHECK-NEXT: [[I0:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i32 2
+; CHECK-NEXT: call void @usef(<4 x float> [[I0]])
+; CHECK-NEXT: [[I1:%.*]] = insertelement <4 x float> poison, float [[Y:%.*]], i32 2
+; CHECK-NEXT: call void @usef(<4 x float> [[I1]])
+; CHECK-NEXT: [[R:%.*]] = fcmp oeq <4 x float> [[I0]], [[I1]]
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %i0 = insertelement <4 x float> poison, float %x, i32 2
+ call void @usef(<4 x float> %i0)
+ %i1 = insertelement <4 x float> poison, float %y, i32 2
+ call void @usef(<4 x float> %i1)
+ %r = fcmp oeq <4 x float> %i0, %i1
+ ret <4 x i1> %r
+}
+
+define <2 x i1> @constant_op1_i64(i64 %x) {
+; CHECK-LABEL: @constant_op1_i64(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp ne i64 [[X:%.*]], 42
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %r = icmp ne <2 x i64> %ins, <i64 42, i64 undef>
+ ret <2 x i1> %r
+}
+
+define <2 x i1> @constant_op1_i64_not_undef_lane(i64 %x) {
+; CHECK-LABEL: @constant_op1_i64_not_undef_lane(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp sge i64 [[X:%.*]], 42
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ins = insertelement <2 x i64> poison, i64 %x, i32 0
+ %r = icmp sge <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i1> %r
+}
+
+; negative test - load prevents the transform
+
+define <2 x i1> @constant_op1_i64_load(i64* %p) {
+; CHECK-LABEL: @constant_op1_i64_load(
+; CHECK-NEXT: [[LD:%.*]] = load i64, i64* [[P:%.*]], align 4
+; CHECK-NEXT: [[INS:%.*]] = insertelement <2 x i64> poison, i64 [[LD]], i32 0
+; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i64> [[INS]], <i64 42, i64 -42>
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ld = load i64, i64* %p
+ %ins = insertelement <2 x i64> poison, i64 %ld, i32 0
+ %r = icmp eq <2 x i64> %ins, <i64 42, i64 -42>
+ ret <2 x i1> %r
+}
+
+define <4 x i1> @constant_op0_i32(i32 %x) {
+; CHECK-LABEL: @constant_op0_i32(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp ult i32 -42, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %ins = insertelement <4 x i32> poison, i32 %x, i32 1
+ %r = icmp ult <4 x i32> <i32 undef, i32 -42, i32 undef, i32 undef>, %ins
+ ret <4 x i1> %r
+}
+
+define <4 x i1> @constant_op0_i32_not_undef_lane(i32 %x) {
+; CHECK-LABEL: @constant_op0_i32_not_undef_lane(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = icmp ule i32 42, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %ins = insertelement <4 x i32> poison, i32 %x, i32 1
+ %r = icmp ule <4 x i32> <i32 1, i32 42, i32 42, i32 -42>, %ins
+ ret <4 x i1> %r
+}
+
+define <2 x i1> @constant_op0_f64(double %x) {
+; CHECK-LABEL: @constant_op0_f64(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp fast olt double 4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 0
+ %r = fcmp fast olt <2 x double> <double 42.0, double undef>, %ins
+ ret <2 x i1> %r
+}
+
+define <2 x i1> @constant_op0_f64_not_undef_lane(double %x) {
+; CHECK-LABEL: @constant_op0_f64_not_undef_lane(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp nnan ueq double -4.200000e+01, [[X:%.*]]
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %r = fcmp nnan ueq <2 x double> <double 42.0, double -42.0>, %ins
+ ret <2 x i1> %r
+}
+
+define <2 x i1> @constant_op1_f64(double %x) {
+; CHECK-LABEL: @constant_op1_f64(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp one double [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[R:%.*]] = insertelement <2 x i1> poison, i1 [[R_SCALAR]], i64 1
+; CHECK-NEXT: ret <2 x i1> [[R]]
+;
+ %ins = insertelement <2 x double> poison, double %x, i32 1
+ %r = fcmp one <2 x double> %ins, <double undef, double 42.0>
+ ret <2 x i1> %r
+}
+
+define <4 x i1> @constant_op1_f32_not_undef_lane(float %x) {
+; CHECK-LABEL: @constant_op1_f32_not_undef_lane(
+; CHECK-NEXT: [[R_SCALAR:%.*]] = fcmp uge float [[X:%.*]], 4.200000e+01
+; CHECK-NEXT: [[R:%.*]] = insertelement <4 x i1> poison, i1 [[R_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i1> [[R]]
+;
+ %ins = insertelement <4 x float> poison, float %x, i32 0
+ %r = fcmp uge <4 x float> %ins, <float 42.0, float -42.0, float 0.0, float 1.0>
+ ret <4 x i1> %r
+}
+
+; negative test - select prevents the transform
+
+define <4 x float> @vec_select_use1(<4 x float> %x, <4 x float> %y, i32 %a, i32 %b) {
+; CHECK-LABEL: @vec_select_use1(
+; CHECK-NEXT: [[VECA:%.*]] = insertelement <4 x i32> poison, i32 [[A:%.*]], i8 0
+; CHECK-NEXT: [[VECB:%.*]] = insertelement <4 x i32> poison, i32 [[B:%.*]], i8 0
+; CHECK-NEXT: [[COND:%.*]] = icmp eq <4 x i32> [[VECA]], [[VECB]]
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %veca = insertelement <4 x i32> poison, i32 %a, i8 0
+ %vecb = insertelement <4 x i32> poison, i32 %b, i8 0
+ %cond = icmp eq <4 x i32> %veca, %vecb
+ %r = select <4 x i1> %cond, <4 x float> %x, <4 x float> %y
+ ret <4 x float> %r
+}
+
+; negative test - select prevents the transform
+
+define <4 x float> @vec_select_use2(<4 x float> %x, <4 x float> %y, float %a) {
+; CHECK-LABEL: @vec_select_use2(
+; CHECK-NEXT: [[VECA:%.*]] = insertelement <4 x float> poison, float [[A:%.*]], i8 0
+; CHECK-NEXT: [[COND:%.*]] = fcmp oeq <4 x float> [[VECA]], zeroinitializer
+; CHECK-NEXT: [[R:%.*]] = select <4 x i1> [[COND]], <4 x float> [[X:%.*]], <4 x float> [[Y:%.*]]
+; CHECK-NEXT: ret <4 x float> [[R]]
+;
+ %veca = insertelement <4 x float> poison, float %a, i8 0
+ %cond = fcmp oeq <4 x float> %veca, zeroinitializer
+ %r = select <4 x i1> %cond, <4 x float> %x, <4 x float> %y
+ ret <4 x float> %r
+}
+
+define <4 x i1> @vector_of_pointers(i32* %t1) {
+; CHECK-LABEL: @vector_of_pointers(
+; CHECK-NEXT: [[T6_SCALAR:%.*]] = icmp ne i32* [[T1:%.*]], null
+; CHECK-NEXT: [[T6:%.*]] = insertelement <4 x i1> poison, i1 [[T6_SCALAR]], i64 0
+; CHECK-NEXT: ret <4 x i1> [[T6]]
+;
+ %t5 = insertelement <4 x i32*> poison, i32* %t1, i32 0
+ %t6 = icmp ne <4 x i32*> %t5, zeroinitializer
+ ret <4 x i1> %t6
+}
More information about the llvm-branch-commits
mailing list