[llvm] [CostModel][X86] Add variable divisor div/rem costs for scalar and <=i32 vectors (PR #215124)
Adam Scott via llvm-commits
llvm-commits at lists.llvm.org
Sat Aug 15 12:20:40 PDT 2026
https://github.com/as4230 updated https://github.com/llvm/llvm-project/pull/215124
>From 32cdc881d7e3bd77b694f150794cbde5b03fe1e5 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sun, 9 Aug 2026 17:49:49 +0000
Subject: [PATCH 1/3] [CostModel][X86] Add variable divisor div/rem costs
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 206 ++++++
.../CostModel/X86/div-rem-strictfp.ll | 310 +++++++++
llvm/test/Analysis/CostModel/X86/div.ll | 264 ++++++--
llvm/test/Analysis/CostModel/X86/rem.ll | 322 +++++++---
llvm/test/Analysis/CostModel/X86/size-cost.ll | 8 +-
.../X86/cost-conditional-branches.ll | 598 ++----------------
.../LoopVectorize/X86/cost-model.ll | 31 +-
.../X86/pr109581-unused-blend.ll | 41 +-
.../LoopVectorize/X86/x86-predication.ll | 162 +----
.../X86/alternate-int-inseltpoison.ll | 121 +---
.../SLPVectorizer/X86/alternate-int.ll | 121 +---
.../X86/div-possibly-extended-with-poisons.ll | 2 +-
.../X86/multi-nodes-to-shuffle.ll | 4 +-
.../SLPVectorizer/X86/no_alternate_divrem.ll | 60 +-
.../X86/parent-node-split-non-schedulable.ll | 49 +-
15 files changed, 1111 insertions(+), 1188 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2c57855a687a6..fa9e5ee02041d 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -753,6 +753,212 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
if (auto KindCost = Entry->Cost[CostKind])
return LT.first * *KindCost;
+ // rem matches div because the divider returns the remainder for free.
+ static const CostKindTblEntry ScalarVarDivCostTable[] = {
+ { ISD::SDIV, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::UDIV, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::SREM, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::UREM, MVT::i8, { 15, 20, 2, 4 } },
+ { ISD::SDIV, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::UDIV, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::SREM, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::UREM, MVT::i16, { 17, 20, 2, 4 } },
+ { ISD::SDIV, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::UDIV, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::SREM, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::UREM, MVT::i32, { 25, 22, 2, 4 } },
+ { ISD::SDIV, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::UDIV, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::SREM, MVT::i64, { 41, 24, 2, 4 } },
+ { ISD::UREM, MVT::i64, { 41, 24, 2, 4 } },
+ };
+
+ if (!LT.second.isVector() && !Op2Info.isConstant())
+ if (const auto *Entry =
+ CostTableLookup(ScalarVarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ // Variable divisors lower through a float divide. strictfp needs SAE
+ // rounding which is 512-bit only.
+ bool IsStrictFP =
+ CxtI && CxtI->getFunction()->hasFnAttribute(Attribute::StrictFP);
+ bool VarDivToFP =
+ !Op2Info.isConstant() && (!IsStrictFP || ST->useAVX512Regs());
+
+ static const CostKindTblEntry AVX512BWVarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 10 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 10 } },
+ { ISD::UREM, MVT::v16i8, { 10 } },
+ { ISD::SREM, MVT::v16i8, { 10 } },
+ { ISD::UDIV, MVT::v32i8, { 20 } },
+ { ISD::SDIV, MVT::v32i8, { 20 } },
+ { ISD::UREM, MVT::v32i8, { 20 } },
+ { ISD::SREM, MVT::v32i8, { 20 } },
+ { ISD::UDIV, MVT::v64i8, { 40 } },
+ { ISD::SDIV, MVT::v64i8, { 40 } },
+ { ISD::UREM, MVT::v64i8, { 40 } },
+ { ISD::SREM, MVT::v64i8, { 40 } },
+ { ISD::UDIV, MVT::v8i16, { 5 } },
+ { ISD::SDIV, MVT::v8i16, { 5 } },
+ { ISD::UREM, MVT::v8i16, { 5 } },
+ { ISD::SREM, MVT::v8i16, { 5 } },
+ { ISD::UDIV, MVT::v16i16, { 10 } },
+ { ISD::SDIV, MVT::v16i16, { 10 } },
+ { ISD::UREM, MVT::v16i16, { 10 } },
+ { ISD::SREM, MVT::v16i16, { 10 } },
+ { ISD::UDIV, MVT::v32i16, { 20 } },
+ { ISD::SDIV, MVT::v32i16, { 20 } },
+ { ISD::UREM, MVT::v32i16, { 20 } },
+ { ISD::SREM, MVT::v32i16, { 20 } },
+ { ISD::UDIV, MVT::v4i32, { 8 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 8 } },
+ { ISD::UREM, MVT::v4i32, { 8 } },
+ { ISD::SREM, MVT::v4i32, { 8 } },
+ { ISD::UDIV, MVT::v8i32, { 16 } },
+ { ISD::SDIV, MVT::v8i32, { 16 } },
+ { ISD::UREM, MVT::v8i32, { 16 } },
+ { ISD::SREM, MVT::v8i32, { 16 } },
+ { ISD::UDIV, MVT::v16i32, { 32 } },
+ { ISD::SDIV, MVT::v16i32, { 32 } },
+ { ISD::UREM, MVT::v16i32, { 32 } },
+ { ISD::SREM, MVT::v16i32, { 32 } },
+ };
+
+ if (VarDivToFP && ST->hasBWI())
+ if (const auto *Entry =
+ CostTableLookup(AVX512BWVarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX512VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 14 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 14 } },
+ { ISD::UREM, MVT::v16i8, { 14 } },
+ { ISD::SREM, MVT::v16i8, { 14 } },
+ { ISD::UDIV, MVT::v32i8, { 28 } },
+ { ISD::SDIV, MVT::v32i8, { 28 } },
+ { ISD::UREM, MVT::v32i8, { 28 } },
+ { ISD::SREM, MVT::v32i8, { 28 } },
+ { ISD::UDIV, MVT::v64i8, { 56 } },
+ { ISD::SDIV, MVT::v64i8, { 56 } },
+ { ISD::UREM, MVT::v64i8, { 56 } },
+ { ISD::SREM, MVT::v64i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 14 } },
+ { ISD::SDIV, MVT::v8i16, { 14 } },
+ { ISD::UREM, MVT::v8i16, { 14 } },
+ { ISD::SREM, MVT::v8i16, { 14 } },
+ { ISD::UDIV, MVT::v16i16, { 14 } },
+ { ISD::SDIV, MVT::v16i16, { 14 } },
+ { ISD::UREM, MVT::v16i16, { 14 } },
+ { ISD::SREM, MVT::v16i16, { 14 } },
+ { ISD::UDIV, MVT::v32i16, { 28 } },
+ { ISD::SDIV, MVT::v32i16, { 28 } },
+ { ISD::UREM, MVT::v32i16, { 28 } },
+ { ISD::SREM, MVT::v32i16, { 28 } },
+ { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 28 } },
+ { ISD::UREM, MVT::v4i32, { 28 } },
+ { ISD::SREM, MVT::v4i32, { 28 } },
+ { ISD::UDIV, MVT::v8i32, { 28 } },
+ { ISD::SDIV, MVT::v8i32, { 28 } },
+ { ISD::UREM, MVT::v8i32, { 28 } },
+ { ISD::SREM, MVT::v8i32, { 28 } },
+ { ISD::UDIV, MVT::v16i32, { 56 } },
+ { ISD::SDIV, MVT::v16i32, { 56 } },
+ { ISD::UREM, MVT::v16i32, { 56 } },
+ { ISD::SREM, MVT::v16i32, { 56 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX512())
+ if (const auto *Entry =
+ CostTableLookup(AVX512VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry AVX2VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 28 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 28 } },
+ { ISD::UREM, MVT::v16i8, { 28 } },
+ { ISD::SREM, MVT::v16i8, { 28 } },
+ { ISD::UDIV, MVT::v32i8, { 56 } },
+ { ISD::SDIV, MVT::v32i8, { 56 } },
+ { ISD::UREM, MVT::v32i8, { 56 } },
+ { ISD::SREM, MVT::v32i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 14 } },
+ { ISD::SDIV, MVT::v8i16, { 14 } },
+ { ISD::UREM, MVT::v8i16, { 14 } },
+ { ISD::SREM, MVT::v8i16, { 14 } },
+ { ISD::UDIV, MVT::v16i16, { 28 } },
+ { ISD::SDIV, MVT::v16i16, { 28 } },
+ { ISD::UREM, MVT::v16i16, { 28 } },
+ { ISD::SREM, MVT::v16i16, { 28 } },
+ { ISD::UDIV, MVT::v4i32, { 28 } }, // cvt+divpd sequence
+ { ISD::SDIV, MVT::v4i32, { 28 } },
+ { ISD::UREM, MVT::v4i32, { 28 } },
+ { ISD::SREM, MVT::v4i32, { 28 } },
+ { ISD::UDIV, MVT::v8i32, { 56 } },
+ { ISD::SDIV, MVT::v8i32, { 56 } },
+ { ISD::UREM, MVT::v8i32, { 56 } },
+ { ISD::SREM, MVT::v8i32, { 56 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX2())
+ if (const auto *Entry =
+ CostTableLookup(AVX2VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ // No unsigned i32 entries below AVX2, where the u32 to f64 converts are
+ // emulated and the fold stays off.
+ static const CostKindTblEntry AVX1VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 56 } },
+ { ISD::UREM, MVT::v16i8, { 56 } },
+ { ISD::SREM, MVT::v16i8, { 56 } },
+ { ISD::UDIV, MVT::v32i8, { 112 } },
+ { ISD::SDIV, MVT::v32i8, { 112 } },
+ { ISD::UREM, MVT::v32i8, { 112 } },
+ { ISD::SREM, MVT::v32i8, { 112 } },
+ { ISD::UDIV, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v8i16, { 28 } },
+ { ISD::UREM, MVT::v8i16, { 28 } },
+ { ISD::SREM, MVT::v8i16, { 28 } },
+ { ISD::UDIV, MVT::v16i16, { 56 } },
+ { ISD::SDIV, MVT::v16i16, { 56 } },
+ { ISD::UREM, MVT::v16i16, { 56 } },
+ { ISD::SREM, MVT::v16i16, { 56 } },
+ { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
+ { ISD::SREM, MVT::v4i32, { 44 } },
+ { ISD::SDIV, MVT::v8i32, { 88 } },
+ { ISD::SREM, MVT::v8i32, { 88 } },
+ };
+
+ if (VarDivToFP && ST->hasAVX())
+ if (const auto *Entry =
+ CostTableLookup(AVX1VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
+ static const CostKindTblEntry SSE2VarDivCostTable[] = {
+ { ISD::UDIV, MVT::v16i8, { 56 } }, // unpack+cvt+divps sequence
+ { ISD::SDIV, MVT::v16i8, { 56 } },
+ { ISD::UREM, MVT::v16i8, { 56 } },
+ { ISD::SREM, MVT::v16i8, { 56 } },
+ { ISD::UDIV, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v8i16, { 28 } },
+ { ISD::UREM, MVT::v8i16, { 28 } },
+ { ISD::SREM, MVT::v8i16, { 28 } },
+ { ISD::SDIV, MVT::v4i32, { 44 } }, // cvt+divpd sequence
+ { ISD::SREM, MVT::v4i32, { 44 } },
+ };
+
+ if (VarDivToFP && ST->hasSSE2())
+ if (const auto *Entry =
+ CostTableLookup(SSE2VarDivCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
static const CostKindTblEntry AVX512BWUniformCostTable[] = {
{ ISD::SHL, MVT::v16i8, { 3, 5, 5, 7 } }, // psllw + pand.
{ ISD::SRL, MVT::v16i8, { 3,10, 5, 8 } }, // psrlw + pand.
diff --git a/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll b/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll
new file mode 100644
index 0000000000000..9a745c20a7fec
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/div-rem-strictfp.ll
@@ -0,0 +1,310 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=throughput -mattr=+avx512f,+avx512bw,+avx512vl,+prefer-256-bit | FileCheck %s --check-prefix=AVX512-256
+
+define void @vector_div_rem(<32 x i8> %a8, <32 x i8> %b8, <16 x i16> %a16, <16 x i16> %b16, <8 x i32> %a32, <8 x i32> %b32) {
+; SSE2-LABEL: 'vector_div_rem'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 112 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 88 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'vector_div_rem'
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 56 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'vector_div_rem'
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-256-LABEL: 'vector_div_rem'
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'vector_div_rem'
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512VL-LABEL: 'vector_div_rem'
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %sdiv8 = sdiv <32 x i8> %a8, %b8
+ %udiv8 = udiv <32 x i8> %a8, %b8
+ %srem8 = srem <32 x i8> %a8, %b8
+ %urem8 = urem <32 x i8> %a8, %b8
+ %sdiv16 = sdiv <16 x i16> %a16, %b16
+ %udiv16 = udiv <16 x i16> %a16, %b16
+ %srem16 = srem <16 x i16> %a16, %b16
+ %urem16 = urem <16 x i16> %a16, %b16
+ %sdiv32 = sdiv <8 x i32> %a32, %b32
+ %udiv32 = udiv <8 x i32> %a32, %b32
+ %srem32 = srem <8 x i32> %a32, %b32
+ %urem32 = urem <8 x i32> %a32, %b32
+ ret void
+}
+
+define void @vector_div_rem_strictfp(<32 x i8> %a8, <32 x i8> %b8, <16 x i16> %a16, <16 x i16> %b16, <8 x i32> %a32, <8 x i32> %b32) strictfp {
+; SSE2-LABEL: 'vector_div_rem_strictfp'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; SSE2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'vector_div_rem_strictfp'
+; AVX2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 9600 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 5440 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'vector_div_rem_strictfp'
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-256-LABEL: 'vector_div_rem_strictfp'
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'vector_div_rem_strictfp'
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512VL-LABEL: 'vector_div_rem_strictfp'
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %sdiv8 = sdiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %udiv8 = udiv <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %srem8 = srem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %urem8 = urem <32 x i8> %a8, %b8
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %sdiv16 = sdiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv16 = udiv <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %srem16 = srem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %urem16 = urem <16 x i16> %a16, %b16
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %sdiv8 = sdiv <32 x i8> %a8, %b8
+ %udiv8 = udiv <32 x i8> %a8, %b8
+ %srem8 = srem <32 x i8> %a8, %b8
+ %urem8 = urem <32 x i8> %a8, %b8
+ %sdiv16 = sdiv <16 x i16> %a16, %b16
+ %udiv16 = udiv <16 x i16> %a16, %b16
+ %srem16 = srem <16 x i16> %a16, %b16
+ %urem16 = urem <16 x i16> %a16, %b16
+ %sdiv32 = sdiv <8 x i32> %a32, %b32
+ %udiv32 = udiv <8 x i32> %a32, %b32
+ %srem32 = srem <8 x i32> %a32, %b32
+ %urem32 = urem <8 x i32> %a32, %b32
+ ret void
+}
+
+define void @vector_div_rem_strictfp_prefer256(<8 x i32> %a32, <8 x i32> %b32) strictfp "min-legal-vector-width"="0" {
+; SSE2-LABEL: 'vector_div_rem_strictfp_prefer256'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'vector_div_rem_strictfp_prefer256'
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'vector_div_rem_strictfp_prefer256'
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-256-LABEL: 'vector_div_rem_strictfp_prefer256'
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 4000 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'vector_div_rem_strictfp_prefer256'
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512VL-LABEL: 'vector_div_rem_strictfp_prefer256'
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %sdiv32 = sdiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %udiv32 = udiv <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %urem32 = urem <8 x i32> %a32, %b32
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %sdiv32 = sdiv <8 x i32> %a32, %b32
+ %udiv32 = udiv <8 x i32> %a32, %b32
+ %srem32 = srem <8 x i32> %a32, %b32
+ %urem32 = urem <8 x i32> %a32, %b32
+ ret void
+}
+
+define void @vector_div_rem_constant_strictfp(<8 x i32> %a32) strictfp {
+; SSE2-LABEL: 'vector_div_rem_constant_strictfp'
+; SSE2-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 10 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; SSE2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX2-LABEL: 'vector_div_rem_constant_strictfp'
+; AVX2-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; AVX2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'vector_div_rem_constant_strictfp'
+; AVX512-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; AVX512-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; AVX512-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; AVX512-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; AVX512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-256-LABEL: 'vector_div_rem_constant_strictfp'
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; AVX512-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512BW-LABEL: 'vector_div_rem_constant_strictfp'
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512VL-LABEL: 'vector_div_rem_constant_strictfp'
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %srem32 = srem <8 x i32> %a32, splat (i32 7)
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %urem32 = urem <8 x i32> %a32, splat (i32 7)
+; AVX512VL-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
+ %sdiv32 = sdiv <8 x i32> %a32, splat (i32 7)
+ %udiv32 = udiv <8 x i32> %a32, splat (i32 7)
+ %srem32 = srem <8 x i32> %a32, splat (i32 7)
+ %urem32 = urem <8 x i32> %a32, splat (i32 7)
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/div.ll b/llvm/test/Analysis/CostModel/X86/div.ll
index 3880898f2bc7b..5e583d8356dd7 100644
--- a/llvm/test/Analysis/CostModel/X86/div.ll
+++ b/llvm/test/Analysis/CostModel/X86/div.ll
@@ -13,24 +13,119 @@
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,AVX1
define i32 @sdiv() {
-; CHECK-LABEL: 'sdiv'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = sdiv i16 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = sdiv i8 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; SSE-LABEL: 'sdiv'
+; SSE-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX1-LABEL: 'sdiv'
+; AVX1-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:88 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:176 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'sdiv'
+; AVX2-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512F-LABEL: 'sdiv'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512BW-LABEL: 'sdiv'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sdiv'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = sdiv i64 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = sdiv i32 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = sdiv i16 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = sdiv i8 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = sdiv i64 undef, undef
%V2i64 = sdiv <2 x i64> undef, undef
@@ -56,24 +151,119 @@ define i32 @sdiv() {
}
define i32 @udiv() {
-; CHECK-LABEL: 'udiv'
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = udiv i16 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = udiv i8 undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; SSE-LABEL: 'udiv'
+; SSE-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:2000 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:4000 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:8000 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX1-LABEL: 'udiv'
+; AVX1-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:2000 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:4000 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:8000 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX2-LABEL: 'udiv'
+; AVX2-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512F-LABEL: 'udiv'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512BW-LABEL: 'udiv'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = udiv i64 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = udiv i32 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = udiv i16 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = udiv i8 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, undef
%V2i64 = udiv <2 x i64> undef, undef
@@ -1973,3 +2163,5 @@ define i32 @udiv_uniformconstnegpow2() {
ret i32 undef
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/X86/rem.ll b/llvm/test/Analysis/CostModel/X86/rem.ll
index 3e7b5499e04c1..42da8210d27c5 100644
--- a/llvm/test/Analysis/CostModel/X86/rem.ll
+++ b/llvm/test/Analysis/CostModel/X86/rem.ll
@@ -14,61 +14,118 @@
define i32 @srem() {
; SSE-LABEL: 'srem'
-; SSE-NEXT: Cost Model: Found costs of 4 for: %I64 = srem i64 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:88 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:176 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'srem'
-; AVX-NEXT: Cost Model: Found costs of 4 for: %I64 = srem i64 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'srem'
+; AVX1-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:88 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:176 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX512-LABEL: 'srem'
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = srem i64 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX2-LABEL: 'srem'
+; AVX2-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512F-LABEL: 'srem'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512BW-LABEL: 'srem'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'srem'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = srem i64 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = srem i32 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = srem i16 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = srem i8 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = srem i64 undef, undef
%V2i64 = srem <2 x i64> undef, undef
@@ -95,61 +152,118 @@ define i32 @srem() {
define i32 @urem() {
; SSE-LABEL: 'urem'
-; SSE-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
-; SSE-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
-; SSE-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:2000 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:4000 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:8000 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; SSE-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
; SSE-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX-LABEL: 'urem'
-; AVX-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
-; AVX-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
-; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX1-LABEL: 'urem'
+; AVX1-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:2000 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:4000 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:8000 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:224 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX512-LABEL: 'urem'
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:960 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:1280 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:2560 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:1600 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:3200 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:6400 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX2-LABEL: 'urem'
+; AVX2-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:112 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512F-LABEL: 'urem'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512BW-LABEL: 'urem'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:10 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'urem'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:2 Lat:24 SizeLat:4 for: %I64 = urem i64 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1640 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3280 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6560 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:25 CodeSize:2 Lat:22 SizeLat:4 for: %I32 = urem i32 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:2 Lat:20 SizeLat:4 for: %I16 = urem i16 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:2 Lat:20 SizeLat:4 for: %I8 = urem i8 undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, undef
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = urem i64 undef, undef
%V2i64 = urem <2 x i64> undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/size-cost.ll b/llvm/test/Analysis/CostModel/X86/size-cost.ll
index aa905dcec777a..36e7ecd16ceb0 100644
--- a/llvm/test/Analysis/CostModel/X86/size-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/size-cost.ll
@@ -95,7 +95,7 @@ define i64 @mul_i64(i64 %x, i64 %y) {
define i64 @sdiv_i64(i64 %x, i64 %y) {
; CHECK-LABEL: 'sdiv_i64'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %r = sdiv i64 %x, %y
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = sdiv i64 %x, %y
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %r
;
%r = sdiv i64 %x, %y
@@ -106,7 +106,7 @@ define i64 @sdiv_i64(i64 %x, i64 %y) {
define i64 @udiv_i64(i64 %x, i64 %y) {
; CHECK-LABEL: 'udiv_i64'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %r = udiv i64 %x, %y
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = udiv i64 %x, %y
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %r
;
%r = udiv i64 %x, %y
@@ -117,7 +117,7 @@ define i64 @udiv_i64(i64 %x, i64 %y) {
define i64 @srem_i64(i64 %x, i64 %y) {
; CHECK-LABEL: 'srem_i64'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %r = srem i64 %x, %y
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = srem i64 %x, %y
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %r
;
%r = srem i64 %x, %y
@@ -128,7 +128,7 @@ define i64 @srem_i64(i64 %x, i64 %y) {
define i64 @urem_i64(i64 %x, i64 %y) {
; CHECK-LABEL: 'urem_i64'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %r = urem i64 %x, %y
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %r = urem i64 %x, %y
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: ret i64 %r
;
%r = urem i64 %x, %y
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
index 100cd3e815f77..c2f3182baca9e 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-conditional-branches.ll
@@ -185,134 +185,22 @@ define i64 @avx512_cond_load_cost(ptr %src, i32 %a, i64 %b, i32 %c, i32 %d) #1 {
; CHECK: vector.ph:
; CHECK-NEXT: [[TMP1:%.*]] = and i32 [[TMP0]], 7
; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <8 x i32> poison, i32 [[A:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT2]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i32> poison, i32 [[D:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT3]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE28:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE28]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP2:%.*]] = icmp slt <8 x i32> [[VEC_IND]], zeroinitializer
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i1> [[TMP2]], i64 0
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_UREM_IF:%.*]], label [[PRED_UREM_CONTINUE:%.*]]
-; CHECK: pred.urem.if:
-; CHECK-NEXT: [[TMP4:%.*]] = urem i32 [[A:%.*]], [[C]]
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE]]
-; CHECK: pred.urem.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_UREM_IF]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP2]], i64 1
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_UREM_IF1:%.*]], label [[PRED_UREM_CONTINUE2:%.*]]
-; CHECK: pred.urem.if1:
-; CHECK-NEXT: [[TMP8:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP8]], i64 1
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE2]]
-; CHECK: pred.urem.continue2:
-; CHECK-NEXT: [[TMP10:%.*]] = phi <8 x i32> [ [[TMP6]], [[PRED_UREM_CONTINUE]] ], [ [[TMP9]], [[PRED_UREM_IF1]] ]
-; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x i1> [[TMP2]], i64 2
-; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_UREM_IF3:%.*]], label [[PRED_UREM_CONTINUE4:%.*]]
-; CHECK: pred.urem.if3:
-; CHECK-NEXT: [[TMP12:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[TMP12]], i64 2
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE4]]
-; CHECK: pred.urem.continue4:
-; CHECK-NEXT: [[TMP14:%.*]] = phi <8 x i32> [ [[TMP10]], [[PRED_UREM_CONTINUE2]] ], [ [[TMP13]], [[PRED_UREM_IF3]] ]
-; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x i1> [[TMP2]], i64 3
-; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_UREM_IF5:%.*]], label [[PRED_UREM_CONTINUE6:%.*]]
-; CHECK: pred.urem.if5:
-; CHECK-NEXT: [[TMP16:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[TMP16]], i64 3
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE6]]
-; CHECK: pred.urem.continue6:
-; CHECK-NEXT: [[TMP18:%.*]] = phi <8 x i32> [ [[TMP14]], [[PRED_UREM_CONTINUE4]] ], [ [[TMP17]], [[PRED_UREM_IF5]] ]
-; CHECK-NEXT: [[TMP19:%.*]] = extractelement <8 x i1> [[TMP2]], i64 4
-; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_UREM_IF7:%.*]], label [[PRED_UREM_CONTINUE8:%.*]]
-; CHECK: pred.urem.if7:
-; CHECK-NEXT: [[TMP20:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP20]], i64 4
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE8]]
-; CHECK: pred.urem.continue8:
-; CHECK-NEXT: [[TMP22:%.*]] = phi <8 x i32> [ [[TMP18]], [[PRED_UREM_CONTINUE6]] ], [ [[TMP21]], [[PRED_UREM_IF7]] ]
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <8 x i1> [[TMP2]], i64 5
-; CHECK-NEXT: br i1 [[TMP23]], label [[PRED_UREM_IF9:%.*]], label [[PRED_UREM_CONTINUE10:%.*]]
-; CHECK: pred.urem.if9:
-; CHECK-NEXT: [[TMP24:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <8 x i32> [[TMP22]], i32 [[TMP24]], i64 5
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE10]]
-; CHECK: pred.urem.continue10:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <8 x i32> [ [[TMP22]], [[PRED_UREM_CONTINUE8]] ], [ [[TMP25]], [[PRED_UREM_IF9]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[TMP2]], i64 6
-; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_UREM_IF11:%.*]], label [[PRED_UREM_CONTINUE12:%.*]]
-; CHECK: pred.urem.if11:
-; CHECK-NEXT: [[TMP28:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP29:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP28]], i64 6
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE12]]
-; CHECK: pred.urem.continue12:
-; CHECK-NEXT: [[TMP30:%.*]] = phi <8 x i32> [ [[TMP26]], [[PRED_UREM_CONTINUE10]] ], [ [[TMP29]], [[PRED_UREM_IF11]] ]
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <8 x i1> [[TMP2]], i64 7
-; CHECK-NEXT: br i1 [[TMP31]], label [[PRED_UREM_IF13:%.*]], label [[PRED_UREM_CONTINUE14:%.*]]
-; CHECK: pred.urem.if13:
-; CHECK-NEXT: [[TMP32:%.*]] = urem i32 [[A]], [[C]]
-; CHECK-NEXT: [[TMP33:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP32]], i64 7
-; CHECK-NEXT: br label [[PRED_UREM_CONTINUE14]]
-; CHECK: pred.urem.continue14:
-; CHECK-NEXT: [[TMP34:%.*]] = phi <8 x i32> [ [[TMP30]], [[PRED_UREM_CONTINUE12]] ], [ [[TMP33]], [[PRED_UREM_IF13]] ]
+; CHECK-NEXT: [[TMP34:%.*]] = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> [[BROADCAST_SPLAT1]], <8 x i32> [[BROADCAST_SPLAT2]], <8 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP35:%.*]] = sub <8 x i32> zeroinitializer, [[TMP34]]
-; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP36:%.*]] = udiv i32 [[C]], [[D:%.*]]
-; CHECK-NEXT: [[TMP37:%.*]] = insertelement <8 x i32> poison, i32 [[TMP36]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP38:%.*]] = phi <8 x i32> [ poison, [[PRED_UREM_CONTINUE14]] ], [ [[TMP37]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; CHECK: pred.udiv.if15:
-; CHECK-NEXT: [[TMP39:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <8 x i32> [[TMP38]], i32 [[TMP39]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; CHECK: pred.udiv.continue16:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <8 x i32> [ [[TMP38]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP40]], [[PRED_UDIV_IF15]] ]
-; CHECK-NEXT: br i1 [[TMP11]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18:%.*]]
-; CHECK: pred.udiv.if17:
-; CHECK-NEXT: [[TMP42:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP43:%.*]] = insertelement <8 x i32> [[TMP41]], i32 [[TMP42]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE18]]
-; CHECK: pred.udiv.continue18:
-; CHECK-NEXT: [[TMP44:%.*]] = phi <8 x i32> [ [[TMP41]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP43]], [[PRED_UDIV_IF17]] ]
-; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20:%.*]]
-; CHECK: pred.udiv.if19:
-; CHECK-NEXT: [[TMP45:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP46:%.*]] = insertelement <8 x i32> [[TMP44]], i32 [[TMP45]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE20]]
-; CHECK: pred.udiv.continue20:
-; CHECK-NEXT: [[TMP47:%.*]] = phi <8 x i32> [ [[TMP44]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP46]], [[PRED_UDIV_IF19]] ]
-; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_UDIV_IF21:%.*]], label [[PRED_UDIV_CONTINUE22:%.*]]
-; CHECK: pred.udiv.if21:
-; CHECK-NEXT: [[TMP48:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP49:%.*]] = insertelement <8 x i32> [[TMP47]], i32 [[TMP48]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE22]]
-; CHECK: pred.udiv.continue22:
-; CHECK-NEXT: [[TMP50:%.*]] = phi <8 x i32> [ [[TMP47]], [[PRED_UDIV_CONTINUE20]] ], [ [[TMP49]], [[PRED_UDIV_IF21]] ]
-; CHECK-NEXT: br i1 [[TMP23]], label [[PRED_UDIV_IF23:%.*]], label [[PRED_UDIV_CONTINUE24:%.*]]
-; CHECK: pred.udiv.if23:
-; CHECK-NEXT: [[TMP51:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <8 x i32> [[TMP50]], i32 [[TMP51]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE24]]
-; CHECK: pred.udiv.continue24:
-; CHECK-NEXT: [[TMP53:%.*]] = phi <8 x i32> [ [[TMP50]], [[PRED_UDIV_CONTINUE22]] ], [ [[TMP52]], [[PRED_UDIV_IF23]] ]
-; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_UDIV_IF25:%.*]], label [[PRED_UDIV_CONTINUE26:%.*]]
-; CHECK: pred.udiv.if25:
-; CHECK-NEXT: [[TMP54:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP55:%.*]] = insertelement <8 x i32> [[TMP53]], i32 [[TMP54]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE26]]
-; CHECK: pred.udiv.continue26:
-; CHECK-NEXT: [[TMP56:%.*]] = phi <8 x i32> [ [[TMP53]], [[PRED_UDIV_CONTINUE24]] ], [ [[TMP55]], [[PRED_UDIV_IF25]] ]
-; CHECK-NEXT: br i1 [[TMP31]], label [[PRED_UDIV_IF27:%.*]], label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.if27:
-; CHECK-NEXT: [[TMP57:%.*]] = udiv i32 [[C]], [[D]]
-; CHECK-NEXT: [[TMP58:%.*]] = insertelement <8 x i32> [[TMP56]], i32 [[TMP57]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.continue28:
-; CHECK-NEXT: [[TMP59:%.*]] = phi <8 x i32> [ [[TMP56]], [[PRED_UDIV_CONTINUE26]] ], [ [[TMP58]], [[PRED_UDIV_IF27]] ]
+; CHECK-NEXT: [[TMP59:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[BROADCAST_SPLAT2]], <8 x i32> [[BROADCAST_SPLAT4]], <8 x i1> [[TMP2]])
; CHECK-NEXT: [[TMP60:%.*]] = or <8 x i32> [[TMP59]], [[TMP35]]
; CHECK-NEXT: [[TMP61:%.*]] = sext <8 x i32> [[TMP60]] to <8 x i64>
; CHECK-NEXT: [[WIDE_GEP:%.*]] = getelementptr { i64, i64, i64 }, ptr [[SRC:%.*]], <8 x i64> [[TMP61]], i32 2
@@ -730,99 +618,10 @@ exit:
define void @sdiv_by_zero(ptr noalias %src, ptr noalias %dst, i32 %d) #2 {
; CHECK-LABEL: @sdiv_by_zero(
; CHECK-NEXT: bb:
-; CHECK-NEXT: br label [[VECTOR_PH:%.*]]
-; CHECK: vector.ph:
-; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
-; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_SDIV_CONTINUE14:%.*]] ]
-; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
-; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <8 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i64 0
-; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_SDIV_IF:%.*]], label [[PRED_SDIV_CONTINUE:%.*]]
-; CHECK: pred.sdiv.if:
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = sdiv i32 [[TMP3]], 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE]]
-; CHECK: pred.sdiv.continue:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_SDIV_IF]] ]
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i64 1
-; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_SDIV_IF1:%.*]], label [[PRED_SDIV_CONTINUE2:%.*]]
-; CHECK: pred.sdiv.if1:
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 1
-; CHECK-NEXT: [[TMP9:%.*]] = sdiv i32 [[TMP8]], 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP9]], i64 1
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE2]]
-; CHECK: pred.sdiv.continue2:
-; CHECK-NEXT: [[TMP11:%.*]] = phi <8 x i32> [ [[TMP6]], [[PRED_SDIV_CONTINUE]] ], [ [[TMP10]], [[PRED_SDIV_IF1]] ]
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i64 2
-; CHECK-NEXT: br i1 [[TMP12]], label [[PRED_SDIV_IF3:%.*]], label [[PRED_SDIV_CONTINUE4:%.*]]
-; CHECK: pred.sdiv.if3:
-; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 2
-; CHECK-NEXT: [[TMP14:%.*]] = sdiv i32 [[TMP13]], 0
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[TMP14]], i64 2
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE4]]
-; CHECK: pred.sdiv.continue4:
-; CHECK-NEXT: [[TMP16:%.*]] = phi <8 x i32> [ [[TMP11]], [[PRED_SDIV_CONTINUE2]] ], [ [[TMP15]], [[PRED_SDIV_IF3]] ]
-; CHECK-NEXT: [[TMP17:%.*]] = extractelement <8 x i1> [[TMP1]], i64 3
-; CHECK-NEXT: br i1 [[TMP17]], label [[PRED_SDIV_IF5:%.*]], label [[PRED_SDIV_CONTINUE6:%.*]]
-; CHECK: pred.sdiv.if5:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 3
-; CHECK-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP18]], 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP16]], i32 [[TMP19]], i64 3
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE6]]
-; CHECK: pred.sdiv.continue6:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP16]], [[PRED_SDIV_CONTINUE4]] ], [ [[TMP20]], [[PRED_SDIV_IF5]] ]
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <8 x i1> [[TMP1]], i64 4
-; CHECK-NEXT: br i1 [[TMP22]], label [[PRED_SDIV_IF7:%.*]], label [[PRED_SDIV_CONTINUE8:%.*]]
-; CHECK: pred.sdiv.if7:
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 4
-; CHECK-NEXT: [[TMP24:%.*]] = sdiv i32 [[TMP23]], 0
-; CHECK-NEXT: [[TMP25:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP24]], i64 4
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE8]]
-; CHECK: pred.sdiv.continue8:
-; CHECK-NEXT: [[TMP26:%.*]] = phi <8 x i32> [ [[TMP21]], [[PRED_SDIV_CONTINUE6]] ], [ [[TMP25]], [[PRED_SDIV_IF7]] ]
-; CHECK-NEXT: [[TMP27:%.*]] = extractelement <8 x i1> [[TMP1]], i64 5
-; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_SDIV_IF9:%.*]], label [[PRED_SDIV_CONTINUE10:%.*]]
-; CHECK: pred.sdiv.if9:
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 5
-; CHECK-NEXT: [[TMP29:%.*]] = sdiv i32 [[TMP28]], 0
-; CHECK-NEXT: [[TMP30:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP29]], i64 5
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE10]]
-; CHECK: pred.sdiv.continue10:
-; CHECK-NEXT: [[TMP31:%.*]] = phi <8 x i32> [ [[TMP26]], [[PRED_SDIV_CONTINUE8]] ], [ [[TMP30]], [[PRED_SDIV_IF9]] ]
-; CHECK-NEXT: [[TMP32:%.*]] = extractelement <8 x i1> [[TMP1]], i64 6
-; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_SDIV_IF11:%.*]], label [[PRED_SDIV_CONTINUE12:%.*]]
-; CHECK: pred.sdiv.if11:
-; CHECK-NEXT: [[TMP33:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 6
-; CHECK-NEXT: [[TMP34:%.*]] = sdiv i32 [[TMP33]], 0
-; CHECK-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP31]], i32 [[TMP34]], i64 6
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE12]]
-; CHECK: pred.sdiv.continue12:
-; CHECK-NEXT: [[TMP36:%.*]] = phi <8 x i32> [ [[TMP31]], [[PRED_SDIV_CONTINUE10]] ], [ [[TMP35]], [[PRED_SDIV_IF11]] ]
-; CHECK-NEXT: [[TMP37:%.*]] = extractelement <8 x i1> [[TMP1]], i64 7
-; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_SDIV_IF13:%.*]], label [[PRED_SDIV_CONTINUE14]]
-; CHECK: pred.sdiv.if13:
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <8 x i32> [[WIDE_LOAD]], i64 7
-; CHECK-NEXT: [[TMP39:%.*]] = sdiv i32 [[TMP38]], 0
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[TMP39]], i64 7
-; CHECK-NEXT: br label [[PRED_SDIV_CONTINUE14]]
-; CHECK: pred.sdiv.continue14:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <8 x i32> [ [[TMP36]], [[PRED_SDIV_CONTINUE12]] ], [ [[TMP40]], [[PRED_SDIV_IF13]] ]
-; CHECK-NEXT: [[PREDPHI:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> [[TMP41]], <8 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP42:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]]
-; CHECK-NEXT: store <8 x i32> [[PREDPHI]], ptr [[TMP42]], align 4
-; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT: [[TMP43:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT: br i1 [[TMP43]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
-; CHECK: middle.block:
-; CHECK-NEXT: br label [[SCALAR_PH:%.*]]
-; CHECK: scalar.ph:
; CHECK-NEXT: br label [[LOOP_HEADER:%.*]]
; CHECK: loop.header:
-; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 16, [[SCALAR_PH]] ]
-; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[BB:%.*]] ]
+; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[IV]]
; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP_SRC]], align 4
; CHECK-NEXT: [[ICMP:%.*]] = icmp eq i32 [[L]], 0
; CHECK-NEXT: br i1 [[ICMP]], label [[LOOP_LATCH]], label [[THEN:%.*]]
@@ -831,11 +630,11 @@ define void @sdiv_by_zero(ptr noalias %src, ptr noalias %dst, i32 %d) #2 {
; CHECK-NEXT: br label [[LOOP_LATCH]]
; CHECK: loop.latch:
; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[SDIV]], [[THEN]] ], [ 0, [[LOOP_HEADER]] ]
-; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[IV]]
; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_DST]], align 4
; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[IV]], 16
-; CHECK-NEXT: br i1 [[EC]], label [[LOOP_HEADER]], label [[EXIT:%.*]], !llvm.loop [[LOOP11:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label [[LOOP_HEADER]], label [[EXIT:%.*]]
; CHECK: exit:
; CHECK-NEXT: ret void
;
@@ -873,359 +672,52 @@ define i64 @test_predicated_udiv(i32 %d, i1 %c) #2 {
; CHECK: vector.main.loop.iter.check:
; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
-; CHECK-NEXT: [[TMP0:%.*]] = xor i1 [[C:%.*]], true
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <8 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT2]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP0:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT1]], splat (i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[D:%.*]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_UDIV_CONTINUE62:%.*]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <32 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_UDIV_CONTINUE62]] ]
-; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i32> @llvm.usub.sat.v32i32(<32 x i32> [[VEC_IND]], <32 x i32> splat (i32 1))
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF:%.*]], label [[PRED_UDIV_CONTINUE:%.*]]
-; CHECK: pred.udiv.if:
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <32 x i32> [[TMP1]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = udiv i32 [[TMP2]], [[D:%.*]]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <32 x i32> poison, i32 [[TMP3]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE]]
-; CHECK: pred.udiv.continue:
-; CHECK-NEXT: [[TMP5:%.*]] = phi <32 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP4]], [[PRED_UDIV_IF]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF1:%.*]], label [[PRED_UDIV_CONTINUE2:%.*]]
-; CHECK: pred.udiv.if1:
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <32 x i32> [[TMP1]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = udiv i32 [[TMP6]], [[D]]
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <32 x i32> [[TMP5]], i32 [[TMP7]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE2]]
-; CHECK: pred.udiv.continue2:
-; CHECK-NEXT: [[TMP9:%.*]] = phi <32 x i32> [ [[TMP5]], [[PRED_UDIV_CONTINUE]] ], [ [[TMP8]], [[PRED_UDIV_IF1]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF3:%.*]], label [[PRED_UDIV_CONTINUE4:%.*]]
-; CHECK: pred.udiv.if3:
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <32 x i32> [[TMP1]], i64 2
-; CHECK-NEXT: [[TMP11:%.*]] = udiv i32 [[TMP10]], [[D]]
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <32 x i32> [[TMP9]], i32 [[TMP11]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE4]]
-; CHECK: pred.udiv.continue4:
-; CHECK-NEXT: [[TMP13:%.*]] = phi <32 x i32> [ [[TMP9]], [[PRED_UDIV_CONTINUE2]] ], [ [[TMP12]], [[PRED_UDIV_IF3]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF5:%.*]], label [[PRED_UDIV_CONTINUE6:%.*]]
-; CHECK: pred.udiv.if5:
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <32 x i32> [[TMP1]], i64 3
-; CHECK-NEXT: [[TMP15:%.*]] = udiv i32 [[TMP14]], [[D]]
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <32 x i32> [[TMP13]], i32 [[TMP15]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE6]]
-; CHECK: pred.udiv.continue6:
-; CHECK-NEXT: [[TMP17:%.*]] = phi <32 x i32> [ [[TMP13]], [[PRED_UDIV_CONTINUE4]] ], [ [[TMP16]], [[PRED_UDIV_IF5]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF7:%.*]], label [[PRED_UDIV_CONTINUE8:%.*]]
-; CHECK: pred.udiv.if7:
-; CHECK-NEXT: [[TMP18:%.*]] = extractelement <32 x i32> [[TMP1]], i64 4
-; CHECK-NEXT: [[TMP19:%.*]] = udiv i32 [[TMP18]], [[D]]
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <32 x i32> [[TMP17]], i32 [[TMP19]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE8]]
-; CHECK: pred.udiv.continue8:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <32 x i32> [ [[TMP17]], [[PRED_UDIV_CONTINUE6]] ], [ [[TMP20]], [[PRED_UDIV_IF7]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF9:%.*]], label [[PRED_UDIV_CONTINUE10:%.*]]
-; CHECK: pred.udiv.if9:
-; CHECK-NEXT: [[TMP22:%.*]] = extractelement <32 x i32> [[TMP1]], i64 5
-; CHECK-NEXT: [[TMP23:%.*]] = udiv i32 [[TMP22]], [[D]]
-; CHECK-NEXT: [[TMP24:%.*]] = insertelement <32 x i32> [[TMP21]], i32 [[TMP23]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE10]]
-; CHECK: pred.udiv.continue10:
-; CHECK-NEXT: [[TMP25:%.*]] = phi <32 x i32> [ [[TMP21]], [[PRED_UDIV_CONTINUE8]] ], [ [[TMP24]], [[PRED_UDIV_IF9]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF11:%.*]], label [[PRED_UDIV_CONTINUE12:%.*]]
-; CHECK: pred.udiv.if11:
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <32 x i32> [[TMP1]], i64 6
-; CHECK-NEXT: [[TMP27:%.*]] = udiv i32 [[TMP26]], [[D]]
-; CHECK-NEXT: [[TMP28:%.*]] = insertelement <32 x i32> [[TMP25]], i32 [[TMP27]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE12]]
-; CHECK: pred.udiv.continue12:
-; CHECK-NEXT: [[TMP29:%.*]] = phi <32 x i32> [ [[TMP25]], [[PRED_UDIV_CONTINUE10]] ], [ [[TMP28]], [[PRED_UDIV_IF11]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF13:%.*]], label [[PRED_UDIV_CONTINUE14:%.*]]
-; CHECK: pred.udiv.if13:
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <32 x i32> [[TMP1]], i64 7
-; CHECK-NEXT: [[TMP31:%.*]] = udiv i32 [[TMP30]], [[D]]
-; CHECK-NEXT: [[TMP32:%.*]] = insertelement <32 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE14]]
-; CHECK: pred.udiv.continue14:
-; CHECK-NEXT: [[TMP33:%.*]] = phi <32 x i32> [ [[TMP29]], [[PRED_UDIV_CONTINUE12]] ], [ [[TMP32]], [[PRED_UDIV_IF13]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF15:%.*]], label [[PRED_UDIV_CONTINUE16:%.*]]
-; CHECK: pred.udiv.if15:
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <32 x i32> [[TMP1]], i64 8
-; CHECK-NEXT: [[TMP35:%.*]] = udiv i32 [[TMP34]], [[D]]
-; CHECK-NEXT: [[TMP36:%.*]] = insertelement <32 x i32> [[TMP33]], i32 [[TMP35]], i64 8
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE16]]
-; CHECK: pred.udiv.continue16:
-; CHECK-NEXT: [[TMP37:%.*]] = phi <32 x i32> [ [[TMP33]], [[PRED_UDIV_CONTINUE14]] ], [ [[TMP36]], [[PRED_UDIV_IF15]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF17:%.*]], label [[PRED_UDIV_CONTINUE18:%.*]]
-; CHECK: pred.udiv.if17:
-; CHECK-NEXT: [[TMP38:%.*]] = extractelement <32 x i32> [[TMP1]], i64 9
-; CHECK-NEXT: [[TMP39:%.*]] = udiv i32 [[TMP38]], [[D]]
-; CHECK-NEXT: [[TMP40:%.*]] = insertelement <32 x i32> [[TMP37]], i32 [[TMP39]], i64 9
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE18]]
-; CHECK: pred.udiv.continue18:
-; CHECK-NEXT: [[TMP41:%.*]] = phi <32 x i32> [ [[TMP37]], [[PRED_UDIV_CONTINUE16]] ], [ [[TMP40]], [[PRED_UDIV_IF17]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF19:%.*]], label [[PRED_UDIV_CONTINUE20:%.*]]
-; CHECK: pred.udiv.if19:
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <32 x i32> [[TMP1]], i64 10
-; CHECK-NEXT: [[TMP43:%.*]] = udiv i32 [[TMP42]], [[D]]
-; CHECK-NEXT: [[TMP44:%.*]] = insertelement <32 x i32> [[TMP41]], i32 [[TMP43]], i64 10
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE20]]
-; CHECK: pred.udiv.continue20:
-; CHECK-NEXT: [[TMP45:%.*]] = phi <32 x i32> [ [[TMP41]], [[PRED_UDIV_CONTINUE18]] ], [ [[TMP44]], [[PRED_UDIV_IF19]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF21:%.*]], label [[PRED_UDIV_CONTINUE22:%.*]]
-; CHECK: pred.udiv.if21:
-; CHECK-NEXT: [[TMP46:%.*]] = extractelement <32 x i32> [[TMP1]], i64 11
-; CHECK-NEXT: [[TMP47:%.*]] = udiv i32 [[TMP46]], [[D]]
-; CHECK-NEXT: [[TMP48:%.*]] = insertelement <32 x i32> [[TMP45]], i32 [[TMP47]], i64 11
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE22]]
-; CHECK: pred.udiv.continue22:
-; CHECK-NEXT: [[TMP49:%.*]] = phi <32 x i32> [ [[TMP45]], [[PRED_UDIV_CONTINUE20]] ], [ [[TMP48]], [[PRED_UDIV_IF21]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF23:%.*]], label [[PRED_UDIV_CONTINUE24:%.*]]
-; CHECK: pred.udiv.if23:
-; CHECK-NEXT: [[TMP50:%.*]] = extractelement <32 x i32> [[TMP1]], i64 12
-; CHECK-NEXT: [[TMP51:%.*]] = udiv i32 [[TMP50]], [[D]]
-; CHECK-NEXT: [[TMP52:%.*]] = insertelement <32 x i32> [[TMP49]], i32 [[TMP51]], i64 12
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE24]]
-; CHECK: pred.udiv.continue24:
-; CHECK-NEXT: [[TMP53:%.*]] = phi <32 x i32> [ [[TMP49]], [[PRED_UDIV_CONTINUE22]] ], [ [[TMP52]], [[PRED_UDIV_IF23]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF25:%.*]], label [[PRED_UDIV_CONTINUE26:%.*]]
-; CHECK: pred.udiv.if25:
-; CHECK-NEXT: [[TMP54:%.*]] = extractelement <32 x i32> [[TMP1]], i64 13
-; CHECK-NEXT: [[TMP55:%.*]] = udiv i32 [[TMP54]], [[D]]
-; CHECK-NEXT: [[TMP56:%.*]] = insertelement <32 x i32> [[TMP53]], i32 [[TMP55]], i64 13
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE26]]
-; CHECK: pred.udiv.continue26:
-; CHECK-NEXT: [[TMP57:%.*]] = phi <32 x i32> [ [[TMP53]], [[PRED_UDIV_CONTINUE24]] ], [ [[TMP56]], [[PRED_UDIV_IF25]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF27:%.*]], label [[PRED_UDIV_CONTINUE28:%.*]]
-; CHECK: pred.udiv.if27:
-; CHECK-NEXT: [[TMP58:%.*]] = extractelement <32 x i32> [[TMP1]], i64 14
-; CHECK-NEXT: [[TMP59:%.*]] = udiv i32 [[TMP58]], [[D]]
-; CHECK-NEXT: [[TMP60:%.*]] = insertelement <32 x i32> [[TMP57]], i32 [[TMP59]], i64 14
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE28]]
-; CHECK: pred.udiv.continue28:
-; CHECK-NEXT: [[TMP61:%.*]] = phi <32 x i32> [ [[TMP57]], [[PRED_UDIV_CONTINUE26]] ], [ [[TMP60]], [[PRED_UDIV_IF27]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF29:%.*]], label [[PRED_UDIV_CONTINUE30:%.*]]
-; CHECK: pred.udiv.if29:
-; CHECK-NEXT: [[TMP62:%.*]] = extractelement <32 x i32> [[TMP1]], i64 15
-; CHECK-NEXT: [[TMP63:%.*]] = udiv i32 [[TMP62]], [[D]]
-; CHECK-NEXT: [[TMP64:%.*]] = insertelement <32 x i32> [[TMP61]], i32 [[TMP63]], i64 15
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE30]]
-; CHECK: pred.udiv.continue30:
-; CHECK-NEXT: [[TMP65:%.*]] = phi <32 x i32> [ [[TMP61]], [[PRED_UDIV_CONTINUE28]] ], [ [[TMP64]], [[PRED_UDIV_IF29]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF31:%.*]], label [[PRED_UDIV_CONTINUE32:%.*]]
-; CHECK: pred.udiv.if31:
-; CHECK-NEXT: [[TMP66:%.*]] = extractelement <32 x i32> [[TMP1]], i64 16
-; CHECK-NEXT: [[TMP67:%.*]] = udiv i32 [[TMP66]], [[D]]
-; CHECK-NEXT: [[TMP68:%.*]] = insertelement <32 x i32> [[TMP65]], i32 [[TMP67]], i64 16
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE32]]
-; CHECK: pred.udiv.continue32:
-; CHECK-NEXT: [[TMP69:%.*]] = phi <32 x i32> [ [[TMP65]], [[PRED_UDIV_CONTINUE30]] ], [ [[TMP68]], [[PRED_UDIV_IF31]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF33:%.*]], label [[PRED_UDIV_CONTINUE34:%.*]]
-; CHECK: pred.udiv.if33:
-; CHECK-NEXT: [[TMP70:%.*]] = extractelement <32 x i32> [[TMP1]], i64 17
-; CHECK-NEXT: [[TMP71:%.*]] = udiv i32 [[TMP70]], [[D]]
-; CHECK-NEXT: [[TMP72:%.*]] = insertelement <32 x i32> [[TMP69]], i32 [[TMP71]], i64 17
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE34]]
-; CHECK: pred.udiv.continue34:
-; CHECK-NEXT: [[TMP73:%.*]] = phi <32 x i32> [ [[TMP69]], [[PRED_UDIV_CONTINUE32]] ], [ [[TMP72]], [[PRED_UDIV_IF33]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF35:%.*]], label [[PRED_UDIV_CONTINUE36:%.*]]
-; CHECK: pred.udiv.if35:
-; CHECK-NEXT: [[TMP74:%.*]] = extractelement <32 x i32> [[TMP1]], i64 18
-; CHECK-NEXT: [[TMP75:%.*]] = udiv i32 [[TMP74]], [[D]]
-; CHECK-NEXT: [[TMP76:%.*]] = insertelement <32 x i32> [[TMP73]], i32 [[TMP75]], i64 18
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE36]]
-; CHECK: pred.udiv.continue36:
-; CHECK-NEXT: [[TMP77:%.*]] = phi <32 x i32> [ [[TMP73]], [[PRED_UDIV_CONTINUE34]] ], [ [[TMP76]], [[PRED_UDIV_IF35]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF37:%.*]], label [[PRED_UDIV_CONTINUE38:%.*]]
-; CHECK: pred.udiv.if37:
-; CHECK-NEXT: [[TMP78:%.*]] = extractelement <32 x i32> [[TMP1]], i64 19
-; CHECK-NEXT: [[TMP79:%.*]] = udiv i32 [[TMP78]], [[D]]
-; CHECK-NEXT: [[TMP80:%.*]] = insertelement <32 x i32> [[TMP77]], i32 [[TMP79]], i64 19
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE38]]
-; CHECK: pred.udiv.continue38:
-; CHECK-NEXT: [[TMP81:%.*]] = phi <32 x i32> [ [[TMP77]], [[PRED_UDIV_CONTINUE36]] ], [ [[TMP80]], [[PRED_UDIV_IF37]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF39:%.*]], label [[PRED_UDIV_CONTINUE40:%.*]]
-; CHECK: pred.udiv.if39:
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <32 x i32> [[TMP1]], i64 20
-; CHECK-NEXT: [[TMP83:%.*]] = udiv i32 [[TMP82]], [[D]]
-; CHECK-NEXT: [[TMP84:%.*]] = insertelement <32 x i32> [[TMP81]], i32 [[TMP83]], i64 20
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE40]]
-; CHECK: pred.udiv.continue40:
-; CHECK-NEXT: [[TMP85:%.*]] = phi <32 x i32> [ [[TMP81]], [[PRED_UDIV_CONTINUE38]] ], [ [[TMP84]], [[PRED_UDIV_IF39]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF41:%.*]], label [[PRED_UDIV_CONTINUE42:%.*]]
-; CHECK: pred.udiv.if41:
-; CHECK-NEXT: [[TMP86:%.*]] = extractelement <32 x i32> [[TMP1]], i64 21
-; CHECK-NEXT: [[TMP87:%.*]] = udiv i32 [[TMP86]], [[D]]
-; CHECK-NEXT: [[TMP88:%.*]] = insertelement <32 x i32> [[TMP85]], i32 [[TMP87]], i64 21
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE42]]
-; CHECK: pred.udiv.continue42:
-; CHECK-NEXT: [[TMP89:%.*]] = phi <32 x i32> [ [[TMP85]], [[PRED_UDIV_CONTINUE40]] ], [ [[TMP88]], [[PRED_UDIV_IF41]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF43:%.*]], label [[PRED_UDIV_CONTINUE44:%.*]]
-; CHECK: pred.udiv.if43:
-; CHECK-NEXT: [[TMP90:%.*]] = extractelement <32 x i32> [[TMP1]], i64 22
-; CHECK-NEXT: [[TMP91:%.*]] = udiv i32 [[TMP90]], [[D]]
-; CHECK-NEXT: [[TMP92:%.*]] = insertelement <32 x i32> [[TMP89]], i32 [[TMP91]], i64 22
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE44]]
-; CHECK: pred.udiv.continue44:
-; CHECK-NEXT: [[TMP93:%.*]] = phi <32 x i32> [ [[TMP89]], [[PRED_UDIV_CONTINUE42]] ], [ [[TMP92]], [[PRED_UDIV_IF43]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF45:%.*]], label [[PRED_UDIV_CONTINUE46:%.*]]
-; CHECK: pred.udiv.if45:
-; CHECK-NEXT: [[TMP94:%.*]] = extractelement <32 x i32> [[TMP1]], i64 23
-; CHECK-NEXT: [[TMP95:%.*]] = udiv i32 [[TMP94]], [[D]]
-; CHECK-NEXT: [[TMP96:%.*]] = insertelement <32 x i32> [[TMP93]], i32 [[TMP95]], i64 23
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE46]]
-; CHECK: pred.udiv.continue46:
-; CHECK-NEXT: [[TMP97:%.*]] = phi <32 x i32> [ [[TMP93]], [[PRED_UDIV_CONTINUE44]] ], [ [[TMP96]], [[PRED_UDIV_IF45]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF47:%.*]], label [[PRED_UDIV_CONTINUE48:%.*]]
-; CHECK: pred.udiv.if47:
-; CHECK-NEXT: [[TMP98:%.*]] = extractelement <32 x i32> [[TMP1]], i64 24
-; CHECK-NEXT: [[TMP99:%.*]] = udiv i32 [[TMP98]], [[D]]
-; CHECK-NEXT: [[TMP100:%.*]] = insertelement <32 x i32> [[TMP97]], i32 [[TMP99]], i64 24
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE48]]
-; CHECK: pred.udiv.continue48:
-; CHECK-NEXT: [[TMP101:%.*]] = phi <32 x i32> [ [[TMP97]], [[PRED_UDIV_CONTINUE46]] ], [ [[TMP100]], [[PRED_UDIV_IF47]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF49:%.*]], label [[PRED_UDIV_CONTINUE50:%.*]]
-; CHECK: pred.udiv.if49:
-; CHECK-NEXT: [[TMP102:%.*]] = extractelement <32 x i32> [[TMP1]], i64 25
-; CHECK-NEXT: [[TMP103:%.*]] = udiv i32 [[TMP102]], [[D]]
-; CHECK-NEXT: [[TMP104:%.*]] = insertelement <32 x i32> [[TMP101]], i32 [[TMP103]], i64 25
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE50]]
-; CHECK: pred.udiv.continue50:
-; CHECK-NEXT: [[TMP105:%.*]] = phi <32 x i32> [ [[TMP101]], [[PRED_UDIV_CONTINUE48]] ], [ [[TMP104]], [[PRED_UDIV_IF49]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF51:%.*]], label [[PRED_UDIV_CONTINUE52:%.*]]
-; CHECK: pred.udiv.if51:
-; CHECK-NEXT: [[TMP106:%.*]] = extractelement <32 x i32> [[TMP1]], i64 26
-; CHECK-NEXT: [[TMP107:%.*]] = udiv i32 [[TMP106]], [[D]]
-; CHECK-NEXT: [[TMP108:%.*]] = insertelement <32 x i32> [[TMP105]], i32 [[TMP107]], i64 26
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE52]]
-; CHECK: pred.udiv.continue52:
-; CHECK-NEXT: [[TMP109:%.*]] = phi <32 x i32> [ [[TMP105]], [[PRED_UDIV_CONTINUE50]] ], [ [[TMP108]], [[PRED_UDIV_IF51]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF53:%.*]], label [[PRED_UDIV_CONTINUE54:%.*]]
-; CHECK: pred.udiv.if53:
-; CHECK-NEXT: [[TMP110:%.*]] = extractelement <32 x i32> [[TMP1]], i64 27
-; CHECK-NEXT: [[TMP111:%.*]] = udiv i32 [[TMP110]], [[D]]
-; CHECK-NEXT: [[TMP112:%.*]] = insertelement <32 x i32> [[TMP109]], i32 [[TMP111]], i64 27
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE54]]
-; CHECK: pred.udiv.continue54:
-; CHECK-NEXT: [[TMP113:%.*]] = phi <32 x i32> [ [[TMP109]], [[PRED_UDIV_CONTINUE52]] ], [ [[TMP112]], [[PRED_UDIV_IF53]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF55:%.*]], label [[PRED_UDIV_CONTINUE56:%.*]]
-; CHECK: pred.udiv.if55:
-; CHECK-NEXT: [[TMP114:%.*]] = extractelement <32 x i32> [[TMP1]], i64 28
-; CHECK-NEXT: [[TMP115:%.*]] = udiv i32 [[TMP114]], [[D]]
-; CHECK-NEXT: [[TMP116:%.*]] = insertelement <32 x i32> [[TMP113]], i32 [[TMP115]], i64 28
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE56]]
-; CHECK: pred.udiv.continue56:
-; CHECK-NEXT: [[TMP117:%.*]] = phi <32 x i32> [ [[TMP113]], [[PRED_UDIV_CONTINUE54]] ], [ [[TMP116]], [[PRED_UDIV_IF55]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF57:%.*]], label [[PRED_UDIV_CONTINUE58:%.*]]
-; CHECK: pred.udiv.if57:
-; CHECK-NEXT: [[TMP118:%.*]] = extractelement <32 x i32> [[TMP1]], i64 29
-; CHECK-NEXT: [[TMP119:%.*]] = udiv i32 [[TMP118]], [[D]]
-; CHECK-NEXT: [[TMP120:%.*]] = insertelement <32 x i32> [[TMP117]], i32 [[TMP119]], i64 29
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE58]]
-; CHECK: pred.udiv.continue58:
-; CHECK-NEXT: [[TMP121:%.*]] = phi <32 x i32> [ [[TMP117]], [[PRED_UDIV_CONTINUE56]] ], [ [[TMP120]], [[PRED_UDIV_IF57]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF59:%.*]], label [[PRED_UDIV_CONTINUE60:%.*]]
-; CHECK: pred.udiv.if59:
-; CHECK-NEXT: [[TMP122:%.*]] = extractelement <32 x i32> [[TMP1]], i64 30
-; CHECK-NEXT: [[TMP123:%.*]] = udiv i32 [[TMP122]], [[D]]
-; CHECK-NEXT: [[TMP124:%.*]] = insertelement <32 x i32> [[TMP121]], i32 [[TMP123]], i64 30
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE60]]
-; CHECK: pred.udiv.continue60:
-; CHECK-NEXT: [[TMP125:%.*]] = phi <32 x i32> [ [[TMP121]], [[PRED_UDIV_CONTINUE58]] ], [ [[TMP124]], [[PRED_UDIV_IF59]] ]
-; CHECK-NEXT: br i1 [[TMP0]], label [[PRED_UDIV_IF61:%.*]], label [[PRED_UDIV_CONTINUE62]]
-; CHECK: pred.udiv.if61:
-; CHECK-NEXT: [[TMP126:%.*]] = extractelement <32 x i32> [[TMP1]], i64 31
-; CHECK-NEXT: [[TMP127:%.*]] = udiv i32 [[TMP126]], [[D]]
-; CHECK-NEXT: [[TMP128:%.*]] = insertelement <32 x i32> [[TMP125]], i32 [[TMP127]], i64 31
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE62]]
-; CHECK: pred.udiv.continue62:
-; CHECK-NEXT: [[TMP129:%.*]] = phi <32 x i32> [ [[TMP125]], [[PRED_UDIV_CONTINUE60]] ], [ [[TMP128]], [[PRED_UDIV_IF61]] ]
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT: [[STEP_ADD:%.*]] = add nuw <8 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-NEXT: [[STEP_ADD_2:%.*]] = add nuw <8 x i32> [[STEP_ADD]], splat (i32 8)
+; CHECK-NEXT: [[STEP_ADD_3:%.*]] = add nuw <8 x i32> [[STEP_ADD_2]], splat (i32 8)
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 32
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add <32 x i32> [[VEC_IND]], splat (i32 32)
+; CHECK-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[STEP_ADD_3]], splat (i32 8)
; CHECK-NEXT: [[TMP130:%.*]] = icmp eq i32 [[INDEX_NEXT]], 992
-; CHECK-NEXT: br i1 [[TMP130]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP130]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
; CHECK: middle.block:
-; CHECK-NEXT: [[TMP131:%.*]] = zext <32 x i32> [[TMP129]] to <32 x i64>
-; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <32 x i64> zeroinitializer, <32 x i64> [[TMP131]]
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <32 x i64> [[PREDPHI]], i64 31
+; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[STEP_ADD_3]], <8 x i32> splat (i32 1))
+; CHECK-NEXT: [[TMP3:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[BROADCAST_SPLAT2]], <8 x i1> [[TMP0]])
+; CHECK-NEXT: [[TMP4:%.*]] = zext <8 x i32> [[TMP3]] to <8 x i64>
+; CHECK-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i64> zeroinitializer, <8 x i64> [[TMP4]]
+; CHECK-NEXT: [[TMP132:%.*]] = extractelement <8 x i64> [[PREDPHI]], i64 7
; CHECK-NEXT: br i1 false, label [[EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
; CHECK: vec.epilog.iter.check:
-; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF13:![0-9]+]]
+; CHECK-NEXT: br i1 false, label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]], !prof [[PROF11:![0-9]+]]
; CHECK: vec.epilog.ph:
; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i32 [ 992, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
-; CHECK-NEXT: [[TMP133:%.*]] = xor i1 [[C]], true
-; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT4:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT3]], <8 x i1> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP6:%.*]] = xor <8 x i1> [[BROADCAST_SPLAT4]], splat (i1 true)
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[D]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <8 x i32> poison, i32 [[VEC_EPILOG_RESUME_VAL]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT8:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT7]], <8 x i32> poison, <8 x i32> zeroinitializer
+; CHECK-NEXT: [[INDUCTION:%.*]] = add <8 x i32> [[BROADCAST_SPLAT8]], <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
; CHECK: vec.epilog.vector.body:
-; CHECK-NEXT: [[INDEX63:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT81:%.*]], [[PRED_UDIV_CONTINUE80:%.*]] ]
-; CHECK-NEXT: [[VEC_IND64:%.*]] = phi <8 x i32> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT82:%.*]], [[PRED_UDIV_CONTINUE80]] ]
-; CHECK-NEXT: [[TMP134:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[VEC_IND64]], <8 x i32> splat (i32 1))
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF65:%.*]], label [[PRED_UDIV_CONTINUE66:%.*]]
-; CHECK: pred.udiv.if65:
-; CHECK-NEXT: [[TMP135:%.*]] = extractelement <8 x i32> [[TMP134]], i64 0
-; CHECK-NEXT: [[TMP136:%.*]] = udiv i32 [[TMP135]], [[D]]
-; CHECK-NEXT: [[TMP137:%.*]] = insertelement <8 x i32> poison, i32 [[TMP136]], i64 0
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE66]]
-; CHECK: pred.udiv.continue66:
-; CHECK-NEXT: [[TMP138:%.*]] = phi <8 x i32> [ poison, [[VEC_EPILOG_VECTOR_BODY]] ], [ [[TMP137]], [[PRED_UDIV_IF65]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF67:%.*]], label [[PRED_UDIV_CONTINUE68:%.*]]
-; CHECK: pred.udiv.if67:
-; CHECK-NEXT: [[TMP139:%.*]] = extractelement <8 x i32> [[TMP134]], i64 1
-; CHECK-NEXT: [[TMP140:%.*]] = udiv i32 [[TMP139]], [[D]]
-; CHECK-NEXT: [[TMP141:%.*]] = insertelement <8 x i32> [[TMP138]], i32 [[TMP140]], i64 1
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE68]]
-; CHECK: pred.udiv.continue68:
-; CHECK-NEXT: [[TMP142:%.*]] = phi <8 x i32> [ [[TMP138]], [[PRED_UDIV_CONTINUE66]] ], [ [[TMP141]], [[PRED_UDIV_IF67]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF69:%.*]], label [[PRED_UDIV_CONTINUE70:%.*]]
-; CHECK: pred.udiv.if69:
-; CHECK-NEXT: [[TMP143:%.*]] = extractelement <8 x i32> [[TMP134]], i64 2
-; CHECK-NEXT: [[TMP144:%.*]] = udiv i32 [[TMP143]], [[D]]
-; CHECK-NEXT: [[TMP145:%.*]] = insertelement <8 x i32> [[TMP142]], i32 [[TMP144]], i64 2
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE70]]
-; CHECK: pred.udiv.continue70:
-; CHECK-NEXT: [[TMP146:%.*]] = phi <8 x i32> [ [[TMP142]], [[PRED_UDIV_CONTINUE68]] ], [ [[TMP145]], [[PRED_UDIV_IF69]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF71:%.*]], label [[PRED_UDIV_CONTINUE72:%.*]]
-; CHECK: pred.udiv.if71:
-; CHECK-NEXT: [[TMP147:%.*]] = extractelement <8 x i32> [[TMP134]], i64 3
-; CHECK-NEXT: [[TMP148:%.*]] = udiv i32 [[TMP147]], [[D]]
-; CHECK-NEXT: [[TMP149:%.*]] = insertelement <8 x i32> [[TMP146]], i32 [[TMP148]], i64 3
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE72]]
-; CHECK: pred.udiv.continue72:
-; CHECK-NEXT: [[TMP150:%.*]] = phi <8 x i32> [ [[TMP146]], [[PRED_UDIV_CONTINUE70]] ], [ [[TMP149]], [[PRED_UDIV_IF71]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF73:%.*]], label [[PRED_UDIV_CONTINUE74:%.*]]
-; CHECK: pred.udiv.if73:
-; CHECK-NEXT: [[TMP151:%.*]] = extractelement <8 x i32> [[TMP134]], i64 4
-; CHECK-NEXT: [[TMP152:%.*]] = udiv i32 [[TMP151]], [[D]]
-; CHECK-NEXT: [[TMP153:%.*]] = insertelement <8 x i32> [[TMP150]], i32 [[TMP152]], i64 4
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE74]]
-; CHECK: pred.udiv.continue74:
-; CHECK-NEXT: [[TMP154:%.*]] = phi <8 x i32> [ [[TMP150]], [[PRED_UDIV_CONTINUE72]] ], [ [[TMP153]], [[PRED_UDIV_IF73]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF75:%.*]], label [[PRED_UDIV_CONTINUE76:%.*]]
-; CHECK: pred.udiv.if75:
-; CHECK-NEXT: [[TMP155:%.*]] = extractelement <8 x i32> [[TMP134]], i64 5
-; CHECK-NEXT: [[TMP156:%.*]] = udiv i32 [[TMP155]], [[D]]
-; CHECK-NEXT: [[TMP157:%.*]] = insertelement <8 x i32> [[TMP154]], i32 [[TMP156]], i64 5
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE76]]
-; CHECK: pred.udiv.continue76:
-; CHECK-NEXT: [[TMP158:%.*]] = phi <8 x i32> [ [[TMP154]], [[PRED_UDIV_CONTINUE74]] ], [ [[TMP157]], [[PRED_UDIV_IF75]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF77:%.*]], label [[PRED_UDIV_CONTINUE78:%.*]]
-; CHECK: pred.udiv.if77:
-; CHECK-NEXT: [[TMP159:%.*]] = extractelement <8 x i32> [[TMP134]], i64 6
-; CHECK-NEXT: [[TMP160:%.*]] = udiv i32 [[TMP159]], [[D]]
-; CHECK-NEXT: [[TMP161:%.*]] = insertelement <8 x i32> [[TMP158]], i32 [[TMP160]], i64 6
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE78]]
-; CHECK: pred.udiv.continue78:
-; CHECK-NEXT: [[TMP162:%.*]] = phi <8 x i32> [ [[TMP158]], [[PRED_UDIV_CONTINUE76]] ], [ [[TMP161]], [[PRED_UDIV_IF77]] ]
-; CHECK-NEXT: br i1 [[TMP133]], label [[PRED_UDIV_IF79:%.*]], label [[PRED_UDIV_CONTINUE80]]
-; CHECK: pred.udiv.if79:
-; CHECK-NEXT: [[TMP163:%.*]] = extractelement <8 x i32> [[TMP134]], i64 7
-; CHECK-NEXT: [[TMP164:%.*]] = udiv i32 [[TMP163]], [[D]]
-; CHECK-NEXT: [[TMP165:%.*]] = insertelement <8 x i32> [[TMP162]], i32 [[TMP164]], i64 7
-; CHECK-NEXT: br label [[PRED_UDIV_CONTINUE80]]
-; CHECK: pred.udiv.continue80:
-; CHECK-NEXT: [[TMP166:%.*]] = phi <8 x i32> [ [[TMP162]], [[PRED_UDIV_CONTINUE78]] ], [ [[TMP165]], [[PRED_UDIV_IF79]] ]
+; CHECK-NEXT: [[INDEX63:%.*]] = phi i32 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT81:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
+; CHECK-NEXT: [[VEC_IND64:%.*]] = phi <8 x i32> [ [[INDUCTION]], [[VEC_EPILOG_PH]] ], [ [[VEC_IND_NEXT82:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
; CHECK-NEXT: [[INDEX_NEXT81]] = add nuw i32 [[INDEX63]], 8
; CHECK-NEXT: [[VEC_IND_NEXT82]] = add <8 x i32> [[VEC_IND64]], splat (i32 8)
; CHECK-NEXT: [[TMP167:%.*]] = icmp eq i32 [[INDEX_NEXT81]], 1000
-; CHECK-NEXT: br i1 [[TMP167]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP167]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
; CHECK: vec.epilog.middle.block:
+; CHECK-NEXT: [[TMP8:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[VEC_IND64]], <8 x i32> splat (i32 1))
+; CHECK-NEXT: [[TMP166:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[TMP8]], <8 x i32> [[BROADCAST_SPLAT]], <8 x i1> [[TMP6]])
; CHECK-NEXT: [[TMP168:%.*]] = zext <8 x i32> [[TMP166]] to <8 x i64>
; CHECK-NEXT: [[PREDPHI83:%.*]] = select i1 [[C]], <8 x i64> zeroinitializer, <8 x i64> [[TMP168]]
; CHECK-NEXT: [[TMP169:%.*]] = extractelement <8 x i64> [[PREDPHI83]], i64 7
@@ -1245,7 +737,7 @@ define i64 @test_predicated_udiv(i32 %d, i1 %c) #2 {
; CHECK-NEXT: [[MERGE:%.*]] = phi i64 [ [[ZEXT]], [[THEN]] ], [ 0, [[LOOP_HEADER]] ]
; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1
; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV]], 1000
-; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP15:![0-9]+]]
+; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: exit:
; CHECK-NEXT: [[MERGE_LCSSA:%.*]] = phi i64 [ [[MERGE]], [[LOOP_LATCH]] ], [ [[TMP132]], [[MIDDLE_BLOCK]] ], [ [[TMP169]], [[VEC_EPILOG_MIDDLE_BLOCK]] ]
; CHECK-NEXT: ret i64 [[MERGE_LCSSA]]
diff --git a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
index d42c587963964..ae77e48dcef85 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/cost-model.ll
@@ -986,22 +986,27 @@ exit:
define void @replicating_sdiv_operand_profitable_to_scalarize(i32 %x) #3 {
; CHECK-LABEL: define void @replicating_sdiv_operand_profitable_to_scalarize(
; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR4:[0-9]+]] {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
-; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[DIV1:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACCUM:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[DIV1]] = sdiv i32 2, [[X]]
-; CHECK-NEXT: [[SUB:%.*]] = sub i32 [[X]], [[DIV1]]
-; CHECK-NEXT: [[OR1:%.*]] = or i32 [[SUB]], [[IV]]
-; CHECK-NEXT: [[OR2:%.*]] = or i32 [[DIV1]], 3
-; CHECK-NEXT: [[DIV2:%.*]] = sdiv i32 [[DIV1]], [[OR2]]
-; CHECK-NEXT: [[OR3:%.*]] = or i32 [[OR1]], [[DIV2]]
-; CHECK-NEXT: [[NEXT]] = add i32 [[ACCUM]], 3
-; CHECK-NEXT: [[COND:%.*]] = icmp ugt i32 [[ACCUM]], 35
-; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[X]], i64 0
+; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <16 x i32> [[BROADCAST_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: br label %[[EXIT:.*]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[RESULT:%.*]] = phi i32 [ [[OR3]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP0:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> splat (i32 2), <16 x i32> [[BROADCAST_SPLAT]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: br label %[[MIDDLE_BLOCK:.*]]
+; CHECK: [[MIDDLE_BLOCK]]:
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <16 x i32> zeroinitializer, <16 x i32> [[TMP0]], <16 x i32> <i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30>
+; CHECK-NEXT: [[TMP2:%.*]] = sub <16 x i32> [[BROADCAST_SPLAT]], [[TMP0]]
+; CHECK-NEXT: [[TMP3:%.*]] = or <16 x i32> [[TMP2]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = or <16 x i32> [[TMP0]], splat (i32 3)
+; CHECK-NEXT: [[TMP5:%.*]] = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> [[TMP0]], <16 x i32> [[TMP4]], <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP6:%.*]] = or <16 x i32> [[TMP3]], [[TMP5]]
+; CHECK-NEXT: [[FIRST_INACTIVE_LANE:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v16i1(<16 x i1> <i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, i1 false)
+; CHECK-NEXT: [[LAST_ACTIVE_LANE:%.*]] = sub i64 [[FIRST_INACTIVE_LANE]], 1
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <16 x i32> [[TMP6]], i64 [[LAST_ACTIVE_LANE]]
+; CHECK-NEXT: br label %[[EXIT1:.*]]
+; CHECK: [[EXIT1]]:
; CHECK-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
index 0323a6771949d..58b8098e81517 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/pr109581-unused-blend.ll
@@ -14,50 +14,17 @@ define i32 @unused_blend_after_unrolling(ptr %p, i32 %a, i1 %c.1, i16 %x, i16 %y
; CHECK: [[VECTOR_PH]]:
; CHECK-NEXT: [[BROADCAST_SPLATINSERT16:%.*]] = insertelement <4 x i1> poison, i1 [[C]], i64 0
; CHECK-NEXT: [[BROADCAST_SPLAT17:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT16]], <4 x i1> poison, <4 x i32> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = xor i1 [[C_1]], true
; CHECK-NEXT: [[TMP22:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT17]], splat (i1 true)
-; CHECK-NEXT: br label %[[VECTOR_BODY:.*]]
-; CHECK: [[VECTOR_BODY]]:
-; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE17:.*]] ]
+; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17:.*]]
+; CHECK: [[PRED_SDIV_CONTINUE17]]:
+; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
; CHECK-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i1> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP25:%.*]], %[[PRED_SDIV_CONTINUE17]] ]
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; CHECK: [[PRED_SDIV_IF]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; CHECK: [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF4:.*]], label %[[PRED_SDIV_CONTINUE5:.*]]
-; CHECK: [[PRED_SDIV_IF4]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE5]]
-; CHECK: [[PRED_SDIV_CONTINUE5]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF6:.*]], label %[[PRED_SDIV_CONTINUE7:.*]]
-; CHECK: [[PRED_SDIV_IF6]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE7]]
-; CHECK: [[PRED_SDIV_CONTINUE7]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF8:.*]], label %[[PRED_SDIV_CONTINUE9:.*]]
-; CHECK: [[PRED_SDIV_IF8]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE9]]
-; CHECK: [[PRED_SDIV_CONTINUE9]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF10:.*]], label %[[PRED_SDIV_CONTINUE11:.*]]
-; CHECK: [[PRED_SDIV_IF10]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE11]]
-; CHECK: [[PRED_SDIV_CONTINUE11]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF12:.*]], label %[[PRED_SDIV_CONTINUE13:.*]]
-; CHECK: [[PRED_SDIV_IF12]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE13]]
-; CHECK: [[PRED_SDIV_CONTINUE13]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF14:.*]], label %[[PRED_SDIV_CONTINUE15:.*]]
-; CHECK: [[PRED_SDIV_IF14]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE15]]
-; CHECK: [[PRED_SDIV_CONTINUE15]]:
-; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_SDIV_IF16:.*]], label %[[PRED_SDIV_CONTINUE17]]
-; CHECK: [[PRED_SDIV_IF16]]:
-; CHECK-NEXT: br label %[[PRED_SDIV_CONTINUE17]]
-; CHECK: [[PRED_SDIV_CONTINUE17]]:
; CHECK-NEXT: [[TMP24]] = or <4 x i1> [[VEC_PHI]], [[TMP22]]
; CHECK-NEXT: [[TMP25]] = or <4 x i1> [[VEC_PHI3]], [[TMP22]]
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 96
-; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[PRED_SDIV_CONTINUE17]], !llvm.loop [[LOOP0:![0-9]+]]
; CHECK: [[MIDDLE_BLOCK]]:
; CHECK-NEXT: [[BIN_RDX:%.*]] = or <4 x i1> [[TMP25]], [[TMP24]]
; CHECK-NEXT: [[TMP27:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[BIN_RDX]])
diff --git a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
index 8947813a4d670..ea42e31c88e90 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/x86-predication.ll
@@ -60,78 +60,17 @@ define i32 @predicated_sdiv_masked_load(ptr %a, ptr %b, i32 %x, i1 %c) {
; SINK-GATHER: [[VECTOR_PH]]:
; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT]], <8 x i1> poison, <8 x i32> zeroinitializer
+; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
+; SINK-GATHER-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT1]], <8 x i32> poison, <8 x i32> zeroinitializer
; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
; SINK-GATHER: [[VECTOR_BODY]]:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SDIV_CONTINUE14:.*]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[PRED_SDIV_CONTINUE14]] ]
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP35:%.*]], %[[VECTOR_BODY]] ]
; SINK-GATHER-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
; SINK-GATHER-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i32>, ptr [[TMP0]], align 4
; SINK-GATHER-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]]
; SINK-GATHER-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 4 [[TMP1]], <8 x i1> [[BROADCAST_SPLAT]], <8 x i32> poison)
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF]]:
-; SINK-GATHER-NEXT: [[TMP2:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 0
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = sdiv i32 [[TMP2]], [[X]]
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE]]:
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP4]], %[[PRED_SDIV_IF]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 1
-; SINK-GATHER-NEXT: [[TMP7:%.*]] = sdiv i32 [[TMP6]], [[X]]
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[TMP7]], i64 1
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE2]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = phi <8 x i32> [ [[TMP5]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP8]], %[[PRED_SDIV_IF1]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF3:.*]], label %[[PRED_SDIV_CONTINUE4:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 2
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = sdiv i32 [[TMP10]], [[X]]
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[TMP11]], i64 2
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE4]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE4]]:
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = phi <8 x i32> [ [[TMP9]], %[[PRED_SDIV_CONTINUE2]] ], [ [[TMP12]], %[[PRED_SDIV_IF3]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF5:.*]], label %[[PRED_SDIV_CONTINUE6:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 3
-; SINK-GATHER-NEXT: [[TMP15:%.*]] = sdiv i32 [[TMP14]], [[X]]
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[TMP15]], i64 3
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE6]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE6]]:
-; SINK-GATHER-NEXT: [[TMP17:%.*]] = phi <8 x i32> [ [[TMP13]], %[[PRED_SDIV_CONTINUE4]] ], [ [[TMP16]], %[[PRED_SDIV_IF5]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF7:.*]], label %[[PRED_SDIV_CONTINUE8:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 4
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = sdiv i32 [[TMP18]], [[X]]
-; SINK-GATHER-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP19]], i64 4
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE8]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE8]]:
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP17]], %[[PRED_SDIV_CONTINUE6]] ], [ [[TMP20]], %[[PRED_SDIV_IF7]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF9:.*]], label %[[PRED_SDIV_CONTINUE10:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 5
-; SINK-GATHER-NEXT: [[TMP23:%.*]] = sdiv i32 [[TMP22]], [[X]]
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP21]], i32 [[TMP23]], i64 5
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE10]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE10]]:
-; SINK-GATHER-NEXT: [[TMP25:%.*]] = phi <8 x i32> [ [[TMP21]], %[[PRED_SDIV_CONTINUE8]] ], [ [[TMP24]], %[[PRED_SDIV_IF9]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF11:.*]], label %[[PRED_SDIV_CONTINUE12:.*]]
-; SINK-GATHER: [[PRED_SDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 6
-; SINK-GATHER-NEXT: [[TMP27:%.*]] = sdiv i32 [[TMP26]], [[X]]
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = insertelement <8 x i32> [[TMP25]], i32 [[TMP27]], i64 6
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE12]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE12]]:
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = phi <8 x i32> [ [[TMP25]], %[[PRED_SDIV_CONTINUE10]] ], [ [[TMP28]], %[[PRED_SDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_SDIV_IF13:.*]], label %[[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_SDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[WIDE_MASKED_LOAD]], i64 7
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = sdiv i32 [[TMP30]], [[X]]
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = insertelement <8 x i32> [[TMP29]], i32 [[TMP31]], i64 7
-; SINK-GATHER-NEXT: br label %[[PRED_SDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_SDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = phi <8 x i32> [ [[TMP29]], %[[PRED_SDIV_CONTINUE12]] ], [ [[TMP32]], %[[PRED_SDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[TMP33:%.*]] = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> [[WIDE_MASKED_LOAD]], <8 x i32> [[BROADCAST_SPLAT2]], <8 x i1> [[BROADCAST_SPLAT]])
; SINK-GATHER-NEXT: [[TMP34:%.*]] = add nsw <8 x i32> [[TMP33]], [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP34]], <8 x i32> [[WIDE_LOAD]]
; SINK-GATHER-NEXT: [[TMP35]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
@@ -255,94 +194,19 @@ define i32 @scalarize_and_sink_gather(ptr %a, i1 %c, i32 %x, i64 %n) {
; SINK-GATHER: [[VECTOR_PH]]:
; SINK-GATHER-NEXT: [[N_MOD_VF:%.*]] = and i64 [[SMAX]], 7
; SINK-GATHER-NEXT: [[N_VEC:%.*]] = sub i64 [[SMAX]], [[N_MOD_VF]]
+; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <8 x i1> poison, i1 [[C]], i64 0
+; SINK-GATHER-NEXT: [[BROADCAST_SPLAT1:%.*]] = shufflevector <8 x i1> [[BROADCAST_SPLATINSERT1]], <8 x i1> poison, <8 x i32> zeroinitializer
; SINK-GATHER-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[X]], i64 0
; SINK-GATHER-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer
; SINK-GATHER-NEXT: br label %[[VECTOR_BODY:.*]]
; SINK-GATHER: [[VECTOR_BODY]]:
-; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14:.*]] ]
-; SINK-GATHER-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_UDIV_CONTINUE14]] ]
-; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[PRED_UDIV_CONTINUE14]] ]
+; SINK-GATHER-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SINK-GATHER-NEXT: [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SINK-GATHER-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[TMP49:%.*]], %[[VECTOR_BODY]] ]
; SINK-GATHER-NEXT: [[TMP0:%.*]] = mul <8 x i64> [[VEC_IND]], splat (i64 777)
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF:.*]], label %[[PRED_UDIV_CONTINUE:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF]]:
-; SINK-GATHER-NEXT: [[TMP1:%.*]] = extractelement <8 x i64> [[TMP0]], i64 0
-; SINK-GATHER-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP1]]
-; SINK-GATHER-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
-; SINK-GATHER-NEXT: [[TMP4:%.*]] = udiv i32 [[TMP3]], [[X]]
-; SINK-GATHER-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> poison, i32 [[TMP4]], i64 0
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE]]:
-; SINK-GATHER-NEXT: [[TMP6:%.*]] = phi <8 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP5]], %[[PRED_UDIV_IF]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF1:.*]], label %[[PRED_UDIV_CONTINUE2:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF1]]:
-; SINK-GATHER-NEXT: [[TMP7:%.*]] = extractelement <8 x i64> [[TMP0]], i64 1
-; SINK-GATHER-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP7]]
-; SINK-GATHER-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-; SINK-GATHER-NEXT: [[TMP10:%.*]] = udiv i32 [[TMP9]], [[X]]
-; SINK-GATHER-NEXT: [[TMP11:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[TMP10]], i64 1
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE2]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE2]]:
-; SINK-GATHER-NEXT: [[TMP12:%.*]] = phi <8 x i32> [ [[TMP6]], %[[PRED_UDIV_CONTINUE]] ], [ [[TMP11]], %[[PRED_UDIV_IF1]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF3:.*]], label %[[PRED_UDIV_CONTINUE4:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF3]]:
-; SINK-GATHER-NEXT: [[TMP13:%.*]] = extractelement <8 x i64> [[TMP0]], i64 2
-; SINK-GATHER-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP13]]
-; SINK-GATHER-NEXT: [[TMP15:%.*]] = load i32, ptr [[TMP14]], align 4
-; SINK-GATHER-NEXT: [[TMP16:%.*]] = udiv i32 [[TMP15]], [[X]]
-; SINK-GATHER-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[TMP16]], i64 2
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE4]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE4]]:
-; SINK-GATHER-NEXT: [[TMP18:%.*]] = phi <8 x i32> [ [[TMP12]], %[[PRED_UDIV_CONTINUE2]] ], [ [[TMP17]], %[[PRED_UDIV_IF3]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF5:.*]], label %[[PRED_UDIV_CONTINUE6:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF5]]:
-; SINK-GATHER-NEXT: [[TMP19:%.*]] = extractelement <8 x i64> [[TMP0]], i64 3
-; SINK-GATHER-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP19]]
-; SINK-GATHER-NEXT: [[TMP21:%.*]] = load i32, ptr [[TMP20]], align 4
-; SINK-GATHER-NEXT: [[TMP22:%.*]] = udiv i32 [[TMP21]], [[X]]
-; SINK-GATHER-NEXT: [[TMP23:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP22]], i64 3
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE6]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE6]]:
-; SINK-GATHER-NEXT: [[TMP24:%.*]] = phi <8 x i32> [ [[TMP18]], %[[PRED_UDIV_CONTINUE4]] ], [ [[TMP23]], %[[PRED_UDIV_IF5]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF7:.*]], label %[[PRED_UDIV_CONTINUE8:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF7]]:
-; SINK-GATHER-NEXT: [[TMP25:%.*]] = extractelement <8 x i64> [[TMP0]], i64 4
-; SINK-GATHER-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP25]]
-; SINK-GATHER-NEXT: [[TMP27:%.*]] = load i32, ptr [[TMP26]], align 4
-; SINK-GATHER-NEXT: [[TMP28:%.*]] = udiv i32 [[TMP27]], [[X]]
-; SINK-GATHER-NEXT: [[TMP29:%.*]] = insertelement <8 x i32> [[TMP24]], i32 [[TMP28]], i64 4
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE8]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE8]]:
-; SINK-GATHER-NEXT: [[TMP30:%.*]] = phi <8 x i32> [ [[TMP24]], %[[PRED_UDIV_CONTINUE6]] ], [ [[TMP29]], %[[PRED_UDIV_IF7]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF9:.*]], label %[[PRED_UDIV_CONTINUE10:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF9]]:
-; SINK-GATHER-NEXT: [[TMP31:%.*]] = extractelement <8 x i64> [[TMP0]], i64 5
-; SINK-GATHER-NEXT: [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP31]]
-; SINK-GATHER-NEXT: [[TMP33:%.*]] = load i32, ptr [[TMP32]], align 4
-; SINK-GATHER-NEXT: [[TMP34:%.*]] = udiv i32 [[TMP33]], [[X]]
-; SINK-GATHER-NEXT: [[TMP35:%.*]] = insertelement <8 x i32> [[TMP30]], i32 [[TMP34]], i64 5
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE10]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE10]]:
-; SINK-GATHER-NEXT: [[TMP36:%.*]] = phi <8 x i32> [ [[TMP30]], %[[PRED_UDIV_CONTINUE8]] ], [ [[TMP35]], %[[PRED_UDIV_IF9]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF11:.*]], label %[[PRED_UDIV_CONTINUE12:.*]]
-; SINK-GATHER: [[PRED_UDIV_IF11]]:
-; SINK-GATHER-NEXT: [[TMP37:%.*]] = extractelement <8 x i64> [[TMP0]], i64 6
-; SINK-GATHER-NEXT: [[TMP38:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP37]]
-; SINK-GATHER-NEXT: [[TMP39:%.*]] = load i32, ptr [[TMP38]], align 4
-; SINK-GATHER-NEXT: [[TMP40:%.*]] = udiv i32 [[TMP39]], [[X]]
-; SINK-GATHER-NEXT: [[TMP41:%.*]] = insertelement <8 x i32> [[TMP36]], i32 [[TMP40]], i64 6
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE12]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE12]]:
-; SINK-GATHER-NEXT: [[TMP42:%.*]] = phi <8 x i32> [ [[TMP36]], %[[PRED_UDIV_CONTINUE10]] ], [ [[TMP41]], %[[PRED_UDIV_IF11]] ]
-; SINK-GATHER-NEXT: br i1 [[C]], label %[[PRED_UDIV_IF13:.*]], label %[[PRED_UDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_UDIV_IF13]]:
-; SINK-GATHER-NEXT: [[TMP43:%.*]] = extractelement <8 x i64> [[TMP0]], i64 7
-; SINK-GATHER-NEXT: [[TMP44:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP43]]
-; SINK-GATHER-NEXT: [[TMP45:%.*]] = load i32, ptr [[TMP44]], align 4
-; SINK-GATHER-NEXT: [[TMP46:%.*]] = udiv i32 [[TMP45]], [[X]]
-; SINK-GATHER-NEXT: [[TMP47:%.*]] = insertelement <8 x i32> [[TMP42]], i32 [[TMP46]], i64 7
-; SINK-GATHER-NEXT: br label %[[PRED_UDIV_CONTINUE14]]
-; SINK-GATHER: [[PRED_UDIV_CONTINUE14]]:
-; SINK-GATHER-NEXT: [[TMP48:%.*]] = phi <8 x i32> [ [[TMP42]], %[[PRED_UDIV_CONTINUE12]] ], [ [[TMP47]], %[[PRED_UDIV_IF13]] ]
+; SINK-GATHER-NEXT: [[WIDE_GEP:%.*]] = getelementptr inbounds i32, ptr [[A]], <8 x i64> [[TMP0]]
+; SINK-GATHER-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 4 [[WIDE_GEP]], <8 x i1> [[BROADCAST_SPLAT1]], <8 x i32> poison)
+; SINK-GATHER-NEXT: [[TMP48:%.*]] = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> [[WIDE_MASKED_GATHER]], <8 x i32> [[BROADCAST_SPLAT]], <8 x i1> [[BROADCAST_SPLAT1]])
; SINK-GATHER-NEXT: [[PREDPHI:%.*]] = select i1 [[C]], <8 x i32> [[TMP48]], <8 x i32> [[BROADCAST_SPLAT]]
; SINK-GATHER-NEXT: [[TMP49]] = add <8 x i32> [[VEC_PHI]], [[PREDPHI]]
; SINK-GATHER-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
index 348264c3c2fc4..b253d881fea05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int-inseltpoison.ll
@@ -435,124 +435,9 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
}
define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
-; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; SSE-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SSE-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SSE-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SSE-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SSE-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; SSE-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; SSE-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; SSE-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; SSE-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
-; SSE-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; SSE-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; SSE-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
-; SSE-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
-; SSE-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; SSE-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; SSE-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; SSE-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
-; SSE-NEXT: ret <8 x i32> [[R7]]
-;
-; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; SLM-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SLM-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SLM-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SLM-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SLM-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; SLM-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; SLM-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; SLM-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; SLM-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; SLM-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; SLM-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
-; SLM-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; SLM-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; SLM-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
-; SLM-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
-; SLM-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; SLM-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; SLM-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; SLM-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; SLM-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; SLM-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; SLM-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
-; SLM-NEXT: ret <8 x i32> [[R7]]
-;
-; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX1-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX1-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; AVX1-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX1-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX1-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
-; AVX1-NEXT: [[TMP2:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 8, i32 16, i32 4, i32 8>
-; AVX1-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
-; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX1-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX1-NEXT: [[R3:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 10, i32 11, i32 poison>
-; AVX1-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB7]], i32 7
-; AVX1-NEXT: ret <8 x i32> [[R7]]
-;
-; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX2-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX2-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX2-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX2-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX2-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; AVX2-NEXT: [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
-; AVX2-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX2-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
-; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
-; AVX2-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; AVX2-NEXT: ret <8 x i32> [[R71]]
-;
-; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX512-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX512-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX512-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX512-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX512-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; AVX512-NEXT: [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
-; AVX512-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX512-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; AVX512-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT: [[R0:%.*]] = insertelement <8 x i32> poison, i32 [[AB0]], i32 0
-; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
-; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX512-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT: [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; AVX512-NEXT: ret <8 x i32> [[R71]]
+; CHECK-LABEL: @sdiv_v8i32_undefs(
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv <8 x i32> [[A:%.*]], <i32 undef, i32 4, i32 8, i32 16, i32 undef, i32 4, i32 8, i32 16>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
;
%a0 = extractelement <8 x i32> %a, i32 0
%a1 = extractelement <8 x i32> %a, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index ab28d14d27b8d..5841c9580a755 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -435,124 +435,9 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
}
define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
-; SSE-LABEL: @sdiv_v8i32_undefs(
-; SSE-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; SSE-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SSE-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SSE-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SSE-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SSE-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; SSE-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; SSE-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; SSE-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; SSE-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; SSE-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; SSE-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
-; SSE-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; SSE-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; SSE-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
-; SSE-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SSE-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
-; SSE-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; SSE-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; SSE-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; SSE-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; SSE-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; SSE-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; SSE-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
-; SSE-NEXT: ret <8 x i32> [[R7]]
-;
-; SLM-LABEL: @sdiv_v8i32_undefs(
-; SLM-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; SLM-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SLM-NEXT: [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SLM-NEXT: [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SLM-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SLM-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; SLM-NEXT: [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; SLM-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; SLM-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; SLM-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; SLM-NEXT: [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; SLM-NEXT: [[AB3:%.*]] = sdiv i32 [[A3]], 16
-; SLM-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; SLM-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; SLM-NEXT: [[AB6:%.*]] = sdiv i32 [[A6]], 8
-; SLM-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; SLM-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
-; SLM-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; SLM-NEXT: [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; SLM-NEXT: [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; SLM-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; SLM-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; SLM-NEXT: [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; SLM-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
-; SLM-NEXT: ret <8 x i32> [[R7]]
-;
-; AVX1-LABEL: @sdiv_v8i32_undefs(
-; AVX1-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX1-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX1-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX1-NEXT: [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; AVX1-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX1-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX1-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX1-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <4 x i32> <i32 2, i32 3, i32 5, i32 6>
-; AVX1-NEXT: [[TMP2:%.*]] = sdiv <4 x i32> [[TMP1]], <i32 8, i32 16, i32 4, i32 8>
-; AVX1-NEXT: [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; AVX1-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
-; AVX1-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX1-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX1-NEXT: [[R3:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 10, i32 11, i32 7>
-; AVX1-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; AVX1-NEXT: [[R7:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB7]], i32 7
-; AVX1-NEXT: ret <8 x i32> [[R7]]
-;
-; AVX2-LABEL: @sdiv_v8i32_undefs(
-; AVX2-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX2-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX2-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX2-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX2-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX2-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX2-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; AVX2-NEXT: [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
-; AVX2-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX2-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; AVX2-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX2-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
-; AVX2-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX2-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
-; AVX2-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX2-NEXT: [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; AVX2-NEXT: ret <8 x i32> [[R71]]
-;
-; AVX512-LABEL: @sdiv_v8i32_undefs(
-; AVX512-NEXT: [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
-; AVX512-NEXT: [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; AVX512-NEXT: [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; AVX512-NEXT: [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; AVX512-NEXT: [[AB0:%.*]] = sdiv i32 [[A0]], undef
-; AVX512-NEXT: [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; AVX512-NEXT: [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; AVX512-NEXT: [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 8, i32 16>
-; AVX512-NEXT: [[AB4:%.*]] = sdiv i32 [[A4]], undef
-; AVX512-NEXT: [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
-; AVX512-NEXT: [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 8, i32 16>
-; AVX512-NEXT: [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
-; AVX512-NEXT: [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; AVX512-NEXT: [[TMP5:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT: [[R32:%.*]] = shufflevector <8 x i32> [[R1]], <8 x i32> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 4, i32 5, i32 6, i32 7>
-; AVX512-NEXT: [[R4:%.*]] = insertelement <8 x i32> [[R32]], i32 [[AB4]], i32 4
-; AVX512-NEXT: [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
-; AVX512-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; AVX512-NEXT: [[R71:%.*]] = shufflevector <8 x i32> [[R5]], <8 x i32> [[TMP6]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
-; AVX512-NEXT: ret <8 x i32> [[R71]]
+; CHECK-LABEL: @sdiv_v8i32_undefs(
+; CHECK-NEXT: [[TMP1:%.*]] = sdiv <8 x i32> [[A:%.*]], <i32 undef, i32 4, i32 8, i32 16, i32 undef, i32 4, i32 8, i32 16>
+; CHECK-NEXT: ret <8 x i32> [[TMP1]]
;
%a0 = extractelement <8 x i32> %a, i32 0
%a1 = extractelement <8 x i32> %a, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/div-possibly-extended-with-poisons.ll b/llvm/test/Transforms/SLPVectorizer/X86/div-possibly-extended-with-poisons.ll
index 1ce21a280b602..df4b25312a626 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/div-possibly-extended-with-poisons.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/div-possibly-extended-with-poisons.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-100 < %s | FileCheck %s
+; RUN: opt -S --passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -slp-threshold=-3300 < %s | FileCheck %s
define i8 @test(ptr %g_127, i32 %0, i16 %1) {
; CHECK-LABEL: define i8 @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
index 9d73d2a7d89d0..f52f3f78e12c2 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/multi-nodes-to-shuffle.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-127 | FileCheck %s
-; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-115 -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-3300 | FileCheck %s
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux -slp-threshold=-3300 -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
define void @test(i64 %p0, i64 %p1, i64 %p2, i64 %p3) {
; CHECK-LABEL: @test(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
index 7f914fb4497ad..a798fd603db06 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/no_alternate_divrem.ll
@@ -93,27 +93,27 @@ entry:
define void @test_urem_add(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_urem_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_1:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 1
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1]], i32 2
+; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[GEP2_1:%.*]] = getelementptr i32, ptr [[ARR2:%.*]], i32 1
-; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr i32, ptr [[ARR2]], i32 2
+; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr i32, ptr [[ARR2:%.*]], i32 2
; CHECK-NEXT: [[GEP2_3:%.*]] = getelementptr i32, ptr [[ARR2]], i32 3
-; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[GEP1_1]], align 4
; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[Y0:%.*]] = add nsw i32 [[A0:%.*]], 1146
-; CHECK-NEXT: [[Y1:%.*]] = add nsw i32 [[A1:%.*]], 146
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
-; CHECK-NEXT: [[RES0:%.*]] = urem i32 [[V0]], [[Y0]]
-; CHECK-NEXT: [[RES1:%.*]] = urem i32 [[V1]], [[Y1]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[A1:%.*]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1146, i32 146>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[A2:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[A3:%.*]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[V3]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[TMP4]], <i32 42, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
; CHECK-NEXT: [[RES2:%.*]] = urem i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[RES3:%.*]] = add nsw i32 [[V3]], [[Y3]]
-; CHECK-NEXT: store i32 [[RES0]], ptr [[ARR2]], align 4
-; CHECK-NEXT: store i32 [[RES1]], ptr [[GEP2_1]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = urem <2 x i32> [[TMP9]], [[TMP2]]
+; CHECK-NEXT: store <2 x i32> [[TMP10]], ptr [[ARR2]], align 4
; CHECK-NEXT: store i32 [[RES2]], ptr [[GEP2_2]], align 4
+; CHECK-NEXT: [[RES3:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
; CHECK-NEXT: store i32 [[RES3]], ptr [[GEP2_3]], align 4
; CHECK-NEXT: ret void
;
@@ -152,27 +152,27 @@ entry:
define void @test_srem_add(ptr %arr1, ptr %arr2, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
; CHECK-LABEL: @test_srem_add(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[GEP1_1:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 1
-; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1]], i32 2
+; CHECK-NEXT: [[GEP1_2:%.*]] = getelementptr i32, ptr [[ARR1:%.*]], i32 2
; CHECK-NEXT: [[GEP1_3:%.*]] = getelementptr i32, ptr [[ARR1]], i32 3
-; CHECK-NEXT: [[GEP2_1:%.*]] = getelementptr i32, ptr [[ARR2:%.*]], i32 1
-; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr i32, ptr [[ARR2]], i32 2
+; CHECK-NEXT: [[GEP2_2:%.*]] = getelementptr i32, ptr [[ARR2:%.*]], i32 2
; CHECK-NEXT: [[GEP2_3:%.*]] = getelementptr i32, ptr [[ARR2]], i32 3
-; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[ARR1]], align 4
-; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[GEP1_1]], align 4
; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[GEP1_2]], align 4
; CHECK-NEXT: [[V3:%.*]] = load i32, ptr [[GEP1_3]], align 4
-; CHECK-NEXT: [[Y0:%.*]] = add nsw i32 [[A0:%.*]], 1146
-; CHECK-NEXT: [[Y1:%.*]] = add nsw i32 [[A1:%.*]], 146
-; CHECK-NEXT: [[Y2:%.*]] = add nsw i32 [[A2:%.*]], 42
-; CHECK-NEXT: [[Y3:%.*]] = add nsw i32 [[A3:%.*]], 0
-; CHECK-NEXT: [[RES0:%.*]] = srem i32 [[V0]], [[Y0]]
-; CHECK-NEXT: [[RES1:%.*]] = srem i32 [[V1]], [[Y1]]
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> poison, i32 [[A0:%.*]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[A1:%.*]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 1146, i32 146>
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[A2:%.*]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[A3:%.*]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[V3]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = add nsw <2 x i32> [[TMP4]], <i32 42, i32 0>
+; CHECK-NEXT: [[TMP7:%.*]] = add <2 x i32> [[TMP6]], [[TMP5]]
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <2 x i32> [[TMP7]], i64 0
; CHECK-NEXT: [[RES2:%.*]] = srem i32 [[V2]], [[Y2]]
-; CHECK-NEXT: [[RES3:%.*]] = add nsw i32 [[V3]], [[Y3]]
-; CHECK-NEXT: store i32 [[RES0]], ptr [[ARR2]], align 4
-; CHECK-NEXT: store i32 [[RES1]], ptr [[GEP2_1]], align 4
+; CHECK-NEXT: [[TMP9:%.*]] = load <2 x i32>, ptr [[ARR1]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = srem <2 x i32> [[TMP9]], [[TMP2]]
+; CHECK-NEXT: store <2 x i32> [[TMP10]], ptr [[ARR2]], align 4
; CHECK-NEXT: store i32 [[RES2]], ptr [[GEP2_2]], align 4
+; CHECK-NEXT: [[RES3:%.*]] = extractelement <2 x i32> [[TMP7]], i64 1
; CHECK-NEXT: store i32 [[RES3]], ptr [[GEP2_3]], align 4
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
index 537443a78926b..7386b2bf45b17 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/parent-node-split-non-schedulable.ll
@@ -5,62 +5,65 @@ define i32 @main(ptr %c, i32 %0, i1 %tobool4.not, i16 %1) {
; CHECK-LABEL: define i32 @main(
; CHECK-SAME: ptr [[C:%.*]], i32 [[TMP0:%.*]], i1 [[TOBOOL4_NOT:%.*]], i16 [[TMP1:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*]]:
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> <i32 poison, i32 poison, i32 1, i32 0>, i32 [[TMP0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[TMP0]], i64 1
; CHECK-NEXT: br label %[[IF_END:.*]]
; CHECK: [[IF_END]]:
-; CHECK-NEXT: [[B_0_PH:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP32:%.*]], %[[WHILE_COND_PREHEADER:.*]] ]
-; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP33:%.*]], %[[WHILE_COND_PREHEADER]] ]
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B_0_PH]], i64 1
+; CHECK-NEXT: [[B_0_PH:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[TMP35:%.*]], %[[WHILE_COND_PREHEADER:.*]] ]
+; CHECK-NEXT: [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[TMP36:%.*]], %[[WHILE_COND_PREHEADER]] ]
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x i32> <i32 poison, i32 1>, i32 [[B_0_PH]], i64 0
; CHECK-NEXT: br i1 [[TOBOOL4_NOT]], label %[[R:.*]], label %[[IF_END9:.*]]
; CHECK: [[IF_END9]]:
; CHECK-NEXT: [[CONV11:%.*]] = sext i16 [[TMP1]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> <i32 0, i32 poison, i32 0, i32 1>, i32 [[CONV11]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> <i32 poison, i32 0>, i32 [[CONV11]], i64 0
; CHECK-NEXT: br label %[[R]]
; CHECK: [[R]]:
-; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ [[TMP5]], %[[IF_END9]] ], [ [[TMP4]], %[[IF_END]] ]
+; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ <i32 1, i32 0>, %[[IF_END9]] ], [ [[TMP2]], %[[IF_END]] ]
+; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ [[TMP5]], %[[IF_END9]] ], [ [[TMP4]], %[[IF_END]] ]
; CHECK-NEXT: [[TOBOOL12_NOT:%.*]] = icmp eq i32 [[B_0_PH]], 0
; CHECK-NEXT: br i1 [[TOBOOL12_NOT]], label %[[IF_END14:.*]], label %[[IF_THEN13:.*]]
; CHECK: [[IF_THEN13]]:
; CHECK-NEXT: br label %[[IF_END14]]
; CHECK: [[IF_END14]]:
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
+; CHECK-NEXT: [[AND17:%.*]] = and i32 [[TMP17]], 1
+; CHECK-NEXT: [[DIV20:%.*]] = sdiv i32 [[AND17]], [[TMP0]]
; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[C]], align 4
; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP3]], <2 x i32> poison, <2 x i32> <i32 1, i32 poison>
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP0]], i64 1
; CHECK-NEXT: [[TMP24:%.*]] = and <2 x i32> [[TMP13]], splat (i32 1)
; CHECK-NEXT: [[AND:%.*]] = extractelement <2 x i32> [[TMP24]], i64 0
; CHECK-NEXT: [[NOT:%.*]] = xor i32 [[AND]], 1
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i64 0
-; CHECK-NEXT: [[AND17:%.*]] = and i32 [[TMP9]], 1
-; CHECK-NEXT: [[DIV20:%.*]] = sdiv i32 [[AND17]], [[TMP0]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP7]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[AND17]], i64 1
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP10]], i64 2
; CHECK-NEXT: [[TMP27:%.*]] = shufflevector <2 x i32> [[TMP24]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> [[TMP27]], <4 x i32> <i32 0, i32 1, i32 2, i32 5>
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i32> <i32 0, i32 1, i32 poison, i32 1>, i32 [[DIV20]], i64 2
; CHECK-NEXT: [[TMP16:%.*]] = xor <4 x i32> [[TMP14]], [[TMP15]]
-; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> poison, <8 x i32> <i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <4 x i32> [[TMP16]], <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[TMP17:%.*]] = shufflevector <8 x i32> [[TMP22]], <8 x i32> [[TMP23]], <8 x i32> <i32 0, i32 1, i32 2, i32 8, i32 9, i32 10, i32 11, i32 poison>
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[NOT]], i64 7
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> <i32 poison, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 poison>, i32 [[B_0_PH]], i64 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP0]], i64 7
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i32> poison, i32 [[NOT]], i64 2
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x i32> [[TMP21]], <4 x i32> [[TMP6]], <4 x i32> <i32 poison, i32 poison, i32 2, i32 5>
+; CHECK-NEXT: [[TMP23:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP32:%.*]] = shufflevector <4 x i32> [[TMP22]], <4 x i32> [[TMP23]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x i32> <i32 poison, i32 0, i32 0, i32 0>, i32 [[B_0_PH]], i64 0
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x i32> <i32 0, i32 0, i32 poison, i32 0>, i32 [[TMP0]], i64 2
; CHECK-NEXT: br label %[[AH:.*]]
; CHECK: [[AH]]:
-; CHECK-NEXT: [[TMP21:%.*]] = phi <8 x i32> [ [[TMP20]], %[[AH]] ], [ [[TMP18]], %[[IF_END14]] ]
-; CHECK-NEXT: [[TMP25:%.*]] = extractelement <8 x i32> [[TMP21]], i64 5
-; CHECK-NEXT: [[TMP26:%.*]] = extractelement <8 x i32> [[TMP21]], i64 7
+; CHECK-NEXT: [[TMP37:%.*]] = phi <4 x i32> [ [[TMP33]], %[[AH]] ], [ [[TMP16]], %[[IF_END14]] ]
+; CHECK-NEXT: [[TMP38:%.*]] = phi <4 x i32> [ [[TMP34]], %[[AH]] ], [ [[TMP32]], %[[IF_END14]] ]
+; CHECK-NEXT: [[TMP25:%.*]] = extractelement <4 x i32> [[TMP37]], i64 2
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x i32> [[TMP38]], i64 2
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[TMP25]], [[TMP26]]
-; CHECK-NEXT: [[TMP28:%.*]] = extractelement <8 x i32> [[TMP21]], i64 4
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x i32> [[TMP37]], i64 1
; CHECK-NEXT: [[TMP29:%.*]] = or i32 [[ADD]], [[TMP28]]
-; CHECK-NEXT: [[TMP30:%.*]] = extractelement <8 x i32> [[TMP21]], i64 6
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i32> [[TMP37]], i64 3
; CHECK-NEXT: [[OR27:%.*]] = or i32 [[TMP29]], [[TMP30]]
; CHECK-NEXT: store i32 [[OR27]], ptr [[C]], align 4
; CHECK-NEXT: br i1 [[TOBOOL4_NOT]], label %[[WHILE_COND_PREHEADER]], label %[[AH]]
; CHECK: [[WHILE_COND_PREHEADER]]:
-; CHECK-NEXT: [[TMP31:%.*]] = extractelement <8 x i32> [[TMP21]], i64 1
+; CHECK-NEXT: [[TMP31:%.*]] = extractelement <4 x i32> [[TMP38]], i64 3
; CHECK-NEXT: [[CALL69:%.*]] = tail call i32 @s(i32 [[TMP31]])
-; CHECK-NEXT: [[TMP32]] = extractelement <8 x i32> [[TMP21]], i64 0
-; CHECK-NEXT: [[TMP33]] = shufflevector <8 x i32> [[TMP21]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP35]] = extractelement <4 x i32> [[TMP37]], i64 0
+; CHECK-NEXT: [[TMP36]] = shufflevector <4 x i32> [[TMP38]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
; CHECK-NEXT: br label %[[IF_END]]
;
entry:
>From 4e896605e31c7261d6e36a9b2f39b9b9e3844b16 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 15:58:36 +0000
Subject: [PATCH 2/3] Add runtime alias check test for int div
---
.../SLPVectorizer/X86/runtime-alias-checks.ll | 198 ++++++++++++++++++
1 file changed, 198 insertions(+)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
index 4cfebd254f4ad..1a72ae8a94bb6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/runtime-alias-checks.ll
@@ -1363,3 +1363,201 @@ bbl9:
store i8 %load53, ptr %gep54, align 1
ret void
}
+
+define void @test_udiv(ptr %dst, ptr %x, ptr %y) {
+; CHECK-LABEL: define void @test_udiv(
+; CHECK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[DST10:%.*]] = ptrtoaddr ptr [[DST]] to i64
+; CHECK-NEXT: [[Y9:%.*]] = ptrtoaddr ptr [[Y]] to i64
+; CHECK-NEXT: [[X8:%.*]] = ptrtoaddr ptr [[X]] to i64
+; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[X8]], 32
+; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[Y9]], 32
+; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[DST10]], 32
+; CHECK-NEXT: [[RT_BOUND0:%.*]] = icmp ult i64 [[DST10]], [[TMP0]]
+; CHECK-NEXT: [[RT_BOUND1:%.*]] = icmp ult i64 [[X8]], [[TMP2]]
+; CHECK-NEXT: [[RT_CONFLICT:%.*]] = and i1 [[RT_BOUND0]], [[RT_BOUND1]]
+; CHECK-NEXT: [[RT_BOUND011:%.*]] = icmp ult i64 [[DST10]], [[TMP1]]
+; CHECK-NEXT: [[RT_BOUND112:%.*]] = icmp ult i64 [[Y9]], [[TMP2]]
+; CHECK-NEXT: [[RT_CONFLICT13:%.*]] = and i1 [[RT_BOUND011]], [[RT_BOUND112]]
+; CHECK-NEXT: [[RT_CONFLICT_ALL:%.*]] = or i1 [[RT_CONFLICT]], [[RT_CONFLICT13]]
+; CHECK-NEXT: [[RT_GUARD:%.*]] = freeze i1 [[RT_CONFLICT_ALL]]
+; CHECK-NEXT: br i1 [[RT_GUARD]], label %[[ENTRY_RTSCALAR:.*]], label %[[ENTRY_RTVEC:.*]]
+; CHECK: [[ENTRY_RTVEC]]:
+; CHECK-NEXT: [[TMP3:%.*]] = load <8 x i32>, ptr [[X]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <8 x i32>, ptr [[Y]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = udiv <8 x i32> [[TMP3]], [[TMP4]]
+; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[DST]], align 4
+; CHECK-NEXT: br label %[[ENTRY_RTCONT:.*]]
+; CHECK: [[ENTRY_RTSCALAR]]:
+; CHECK-NEXT: [[X0_SCALAR:%.*]] = load i32, ptr [[X]], align 4
+; CHECK-NEXT: [[Y0_SCALAR:%.*]] = load i32, ptr [[Y]], align 4
+; CHECK-NEXT: [[D0_SCALAR:%.*]] = udiv i32 [[X0_SCALAR]], [[Y0_SCALAR]]
+; CHECK-NEXT: store i32 [[D0_SCALAR]], ptr [[DST]], align 4
+; CHECK-NEXT: [[X1P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 1
+; CHECK-NEXT: [[X1_SCALAR:%.*]] = load i32, ptr [[X1P_SCALAR]], align 4
+; CHECK-NEXT: [[Y1P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1
+; CHECK-NEXT: [[Y1_SCALAR:%.*]] = load i32, ptr [[Y1P_SCALAR]], align 4
+; CHECK-NEXT: [[D1_SCALAR:%.*]] = udiv i32 [[X1_SCALAR]], [[Y1_SCALAR]]
+; CHECK-NEXT: [[DST1_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; CHECK-NEXT: store i32 [[D1_SCALAR]], ptr [[DST1_SCALAR]], align 4
+; CHECK-NEXT: [[X2P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 2
+; CHECK-NEXT: [[X2_SCALAR:%.*]] = load i32, ptr [[X2P_SCALAR]], align 4
+; CHECK-NEXT: [[Y2P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2
+; CHECK-NEXT: [[Y2_SCALAR:%.*]] = load i32, ptr [[Y2P_SCALAR]], align 4
+; CHECK-NEXT: [[D2_SCALAR:%.*]] = udiv i32 [[X2_SCALAR]], [[Y2_SCALAR]]
+; CHECK-NEXT: [[DST2_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
+; CHECK-NEXT: store i32 [[D2_SCALAR]], ptr [[DST2_SCALAR]], align 4
+; CHECK-NEXT: [[X3P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 3
+; CHECK-NEXT: [[X3_SCALAR:%.*]] = load i32, ptr [[X3P_SCALAR]], align 4
+; CHECK-NEXT: [[Y3P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3
+; CHECK-NEXT: [[Y3_SCALAR:%.*]] = load i32, ptr [[Y3P_SCALAR]], align 4
+; CHECK-NEXT: [[D3_SCALAR:%.*]] = udiv i32 [[X3_SCALAR]], [[Y3_SCALAR]]
+; CHECK-NEXT: [[DST3_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
+; CHECK-NEXT: store i32 [[D3_SCALAR]], ptr [[DST3_SCALAR]], align 4
+; CHECK-NEXT: [[X4P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 4
+; CHECK-NEXT: [[X4_SCALAR:%.*]] = load i32, ptr [[X4P_SCALAR]], align 4
+; CHECK-NEXT: [[Y4P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 4
+; CHECK-NEXT: [[Y4_SCALAR:%.*]] = load i32, ptr [[Y4P_SCALAR]], align 4
+; CHECK-NEXT: [[D4_SCALAR:%.*]] = udiv i32 [[X4_SCALAR]], [[Y4_SCALAR]]
+; CHECK-NEXT: [[DST4_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 4
+; CHECK-NEXT: store i32 [[D4_SCALAR]], ptr [[DST4_SCALAR]], align 4
+; CHECK-NEXT: [[X5P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 5
+; CHECK-NEXT: [[X5_SCALAR:%.*]] = load i32, ptr [[X5P_SCALAR]], align 4
+; CHECK-NEXT: [[Y5P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 5
+; CHECK-NEXT: [[Y5_SCALAR:%.*]] = load i32, ptr [[Y5P_SCALAR]], align 4
+; CHECK-NEXT: [[D5_SCALAR:%.*]] = udiv i32 [[X5_SCALAR]], [[Y5_SCALAR]]
+; CHECK-NEXT: [[DST5_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 5
+; CHECK-NEXT: store i32 [[D5_SCALAR]], ptr [[DST5_SCALAR]], align 4
+; CHECK-NEXT: [[X6P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 6
+; CHECK-NEXT: [[X6_SCALAR:%.*]] = load i32, ptr [[X6P_SCALAR]], align 4
+; CHECK-NEXT: [[Y6P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 6
+; CHECK-NEXT: [[Y6_SCALAR:%.*]] = load i32, ptr [[Y6P_SCALAR]], align 4
+; CHECK-NEXT: [[D6_SCALAR:%.*]] = udiv i32 [[X6_SCALAR]], [[Y6_SCALAR]]
+; CHECK-NEXT: [[DST6_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 6
+; CHECK-NEXT: store i32 [[D6_SCALAR]], ptr [[DST6_SCALAR]], align 4
+; CHECK-NEXT: [[X7P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 7
+; CHECK-NEXT: [[X7_SCALAR:%.*]] = load i32, ptr [[X7P_SCALAR]], align 4
+; CHECK-NEXT: [[Y7P_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 7
+; CHECK-NEXT: [[Y7_SCALAR:%.*]] = load i32, ptr [[Y7P_SCALAR]], align 4
+; CHECK-NEXT: [[D7_SCALAR:%.*]] = udiv i32 [[X7_SCALAR]], [[Y7_SCALAR]]
+; CHECK-NEXT: [[DST7_SCALAR:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 7
+; CHECK-NEXT: store i32 [[D7_SCALAR]], ptr [[DST7_SCALAR]], align 4
+; CHECK-NEXT: br label %[[ENTRY_RTCONT]]
+; CHECK: [[ENTRY_RTCONT]]:
+; CHECK-NEXT: ret void
+;
+; NOCHK-LABEL: define void @test_udiv(
+; NOCHK-SAME: ptr [[DST:%.*]], ptr [[X:%.*]], ptr [[Y:%.*]]) #[[ATTR1]] {
+; NOCHK-NEXT: [[ENTRY:.*:]]
+; NOCHK-NEXT: [[X0:%.*]] = load i32, ptr [[X]], align 4
+; NOCHK-NEXT: [[Y0:%.*]] = load i32, ptr [[Y]], align 4
+; NOCHK-NEXT: [[D0:%.*]] = udiv i32 [[X0]], [[Y0]]
+; NOCHK-NEXT: store i32 [[D0]], ptr [[DST]], align 4
+; NOCHK-NEXT: [[X1P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 1
+; NOCHK-NEXT: [[X1:%.*]] = load i32, ptr [[X1P]], align 4
+; NOCHK-NEXT: [[Y1P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 1
+; NOCHK-NEXT: [[Y1:%.*]] = load i32, ptr [[Y1P]], align 4
+; NOCHK-NEXT: [[D1:%.*]] = udiv i32 [[X1]], [[Y1]]
+; NOCHK-NEXT: [[DST1:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 1
+; NOCHK-NEXT: store i32 [[D1]], ptr [[DST1]], align 4
+; NOCHK-NEXT: [[X2P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 2
+; NOCHK-NEXT: [[X2:%.*]] = load i32, ptr [[X2P]], align 4
+; NOCHK-NEXT: [[Y2P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 2
+; NOCHK-NEXT: [[Y2:%.*]] = load i32, ptr [[Y2P]], align 4
+; NOCHK-NEXT: [[D2:%.*]] = udiv i32 [[X2]], [[Y2]]
+; NOCHK-NEXT: [[DST2:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2
+; NOCHK-NEXT: store i32 [[D2]], ptr [[DST2]], align 4
+; NOCHK-NEXT: [[X3P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 3
+; NOCHK-NEXT: [[X3:%.*]] = load i32, ptr [[X3P]], align 4
+; NOCHK-NEXT: [[Y3P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 3
+; NOCHK-NEXT: [[Y3:%.*]] = load i32, ptr [[Y3P]], align 4
+; NOCHK-NEXT: [[D3:%.*]] = udiv i32 [[X3]], [[Y3]]
+; NOCHK-NEXT: [[DST3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3
+; NOCHK-NEXT: store i32 [[D3]], ptr [[DST3]], align 4
+; NOCHK-NEXT: [[X4P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 4
+; NOCHK-NEXT: [[X4:%.*]] = load i32, ptr [[X4P]], align 4
+; NOCHK-NEXT: [[Y4P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 4
+; NOCHK-NEXT: [[Y4:%.*]] = load i32, ptr [[Y4P]], align 4
+; NOCHK-NEXT: [[D4:%.*]] = udiv i32 [[X4]], [[Y4]]
+; NOCHK-NEXT: [[DST4:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 4
+; NOCHK-NEXT: store i32 [[D4]], ptr [[DST4]], align 4
+; NOCHK-NEXT: [[X5P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 5
+; NOCHK-NEXT: [[X5:%.*]] = load i32, ptr [[X5P]], align 4
+; NOCHK-NEXT: [[Y5P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 5
+; NOCHK-NEXT: [[Y5:%.*]] = load i32, ptr [[Y5P]], align 4
+; NOCHK-NEXT: [[D5:%.*]] = udiv i32 [[X5]], [[Y5]]
+; NOCHK-NEXT: [[DST5:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 5
+; NOCHK-NEXT: store i32 [[D5]], ptr [[DST5]], align 4
+; NOCHK-NEXT: [[X6P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 6
+; NOCHK-NEXT: [[X6:%.*]] = load i32, ptr [[X6P]], align 4
+; NOCHK-NEXT: [[Y6P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 6
+; NOCHK-NEXT: [[Y6:%.*]] = load i32, ptr [[Y6P]], align 4
+; NOCHK-NEXT: [[D6:%.*]] = udiv i32 [[X6]], [[Y6]]
+; NOCHK-NEXT: [[DST6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 6
+; NOCHK-NEXT: store i32 [[D6]], ptr [[DST6]], align 4
+; NOCHK-NEXT: [[X7P:%.*]] = getelementptr inbounds i32, ptr [[X]], i64 7
+; NOCHK-NEXT: [[X7:%.*]] = load i32, ptr [[X7P]], align 4
+; NOCHK-NEXT: [[Y7P:%.*]] = getelementptr inbounds i32, ptr [[Y]], i64 7
+; NOCHK-NEXT: [[Y7:%.*]] = load i32, ptr [[Y7P]], align 4
+; NOCHK-NEXT: [[D7:%.*]] = udiv i32 [[X7]], [[Y7]]
+; NOCHK-NEXT: [[DST7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 7
+; NOCHK-NEXT: store i32 [[D7]], ptr [[DST7]], align 4
+; NOCHK-NEXT: ret void
+;
+entry:
+ %x0 = load i32, ptr %x, align 4
+ %y0 = load i32, ptr %y, align 4
+ %d0 = udiv i32 %x0, %y0
+ store i32 %d0, ptr %dst, align 4
+ %x1p = getelementptr inbounds i32, ptr %x, i64 1
+ %x1 = load i32, ptr %x1p, align 4
+ %y1p = getelementptr inbounds i32, ptr %y, i64 1
+ %y1 = load i32, ptr %y1p, align 4
+ %d1 = udiv i32 %x1, %y1
+ %dst1 = getelementptr inbounds i32, ptr %dst, i64 1
+ store i32 %d1, ptr %dst1, align 4
+ %x2p = getelementptr inbounds i32, ptr %x, i64 2
+ %x2 = load i32, ptr %x2p, align 4
+ %y2p = getelementptr inbounds i32, ptr %y, i64 2
+ %y2 = load i32, ptr %y2p, align 4
+ %d2 = udiv i32 %x2, %y2
+ %dst2 = getelementptr inbounds i32, ptr %dst, i64 2
+ store i32 %d2, ptr %dst2, align 4
+ %x3p = getelementptr inbounds i32, ptr %x, i64 3
+ %x3 = load i32, ptr %x3p, align 4
+ %y3p = getelementptr inbounds i32, ptr %y, i64 3
+ %y3 = load i32, ptr %y3p, align 4
+ %d3 = udiv i32 %x3, %y3
+ %dst3 = getelementptr inbounds i32, ptr %dst, i64 3
+ store i32 %d3, ptr %dst3, align 4
+ %x4p = getelementptr inbounds i32, ptr %x, i64 4
+ %x4 = load i32, ptr %x4p, align 4
+ %y4p = getelementptr inbounds i32, ptr %y, i64 4
+ %y4 = load i32, ptr %y4p, align 4
+ %d4 = udiv i32 %x4, %y4
+ %dst4 = getelementptr inbounds i32, ptr %dst, i64 4
+ store i32 %d4, ptr %dst4, align 4
+ %x5p = getelementptr inbounds i32, ptr %x, i64 5
+ %x5 = load i32, ptr %x5p, align 4
+ %y5p = getelementptr inbounds i32, ptr %y, i64 5
+ %y5 = load i32, ptr %y5p, align 4
+ %d5 = udiv i32 %x5, %y5
+ %dst5 = getelementptr inbounds i32, ptr %dst, i64 5
+ store i32 %d5, ptr %dst5, align 4
+ %x6p = getelementptr inbounds i32, ptr %x, i64 6
+ %x6 = load i32, ptr %x6p, align 4
+ %y6p = getelementptr inbounds i32, ptr %y, i64 6
+ %y6 = load i32, ptr %y6p, align 4
+ %d6 = udiv i32 %x6, %y6
+ %dst6 = getelementptr inbounds i32, ptr %dst, i64 6
+ store i32 %d6, ptr %dst6, align 4
+ %x7p = getelementptr inbounds i32, ptr %x, i64 7
+ %x7 = load i32, ptr %x7p, align 4
+ %y7p = getelementptr inbounds i32, ptr %y, i64 7
+ %y7 = load i32, ptr %y7p, align 4
+ %d7 = udiv i32 %x7, %y7
+ %dst7 = getelementptr inbounds i32, ptr %dst, i64 7
+ store i32 %d7, ptr %dst7, align 4
+ ret void
+}
>From 1b016f868359afaaf48b93236549c8044169e722 Mon Sep 17 00:00:00 2001
From: Adam Scott <adamscott200322 at gmail.com>
Date: Sat, 15 Aug 2026 17:33:34 +0000
Subject: [PATCH 3/3] Add masked div/rem cost test coverage
---
.../Analysis/CostModel/X86/masked-divrem.ll | 509 ++++++++++++++++++
1 file changed, 509 insertions(+)
create mode 100644 llvm/test/Analysis/CostModel/X86/masked-divrem.ll
diff --git a/llvm/test/Analysis/CostModel/X86/masked-divrem.ll b/llvm/test/Analysis/CostModel/X86/masked-divrem.ll
new file mode 100644
index 0000000000000..fc4ae7eaa0f34
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-divrem.ll
@@ -0,0 +1,509 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+ssse3 | FileCheck %s --check-prefix=SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+sse4.2 | FileCheck %s --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx | FileCheck %s --check-prefix=AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=slm | FileCheck %s --check-prefix=SLM
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=goldmont | FileCheck %s --check-prefix=SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=btver2 | FileCheck %s --check-prefix=AVX1
+
+define void @udiv() {
+; SSE2-LABEL: 'udiv'
+; SSE2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:7 Lat:6 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:10 Lat:8 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:16 Lat:12 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:7 Lat:6 SizeLat:7 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:10 Lat:8 SizeLat:10 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:16 Lat:12 SizeLat:16 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:7 Lat:6 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:10 Lat:8 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:120 CodeSize:16 Lat:12 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:7 Lat:6 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:10 Lat:8 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:232 CodeSize:16 Lat:12 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'udiv'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:6 Lat:8 SizeLat:8 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:8 Lat:12 SizeLat:12 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:120 CodeSize:8 Lat:12 SizeLat:12 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:232 CodeSize:8 Lat:12 SizeLat:12 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'udiv'
+; AVX1-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3283 CodeSize:5 Lat:7 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6566 CodeSize:6 Lat:10 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4003 CodeSize:5 Lat:7 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:8006 CodeSize:6 Lat:10 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:59 CodeSize:7 Lat:7 SizeLat:7 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:118 CodeSize:10 Lat:10 SizeLat:10 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:115 CodeSize:7 Lat:7 SizeLat:7 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:230 CodeSize:10 Lat:10 SizeLat:10 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'udiv'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3282 CodeSize:5 Lat:6 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6564 CodeSize:6 Lat:8 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512F-LABEL: 'udiv'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:57 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:30 CodeSize:8 Lat:6 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:58 CodeSize:8 Lat:6 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'udiv'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:33 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:5 Lat:5 SizeLat:5 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SLM-LABEL: 'udiv'
+; SLM-NEXT: Cost Model: Found costs of RThru:1644 CodeSize:5 Lat:8 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.udiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:3288 CodeSize:6 Lat:12 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.udiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:6576 CodeSize:8 Lat:20 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.udiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.udiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.udiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.udiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:5 Lat:8 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.udiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:6 Lat:12 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.udiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:128 CodeSize:8 Lat:20 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.udiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:60 CodeSize:5 Lat:8 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.udiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:120 CodeSize:6 Lat:12 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.udiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:240 CodeSize:8 Lat:20 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.udiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V2i64 = call <2 x i64> @llvm.masked.udiv(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+ %V4i64 = call <4 x i64> @llvm.masked.udiv(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+ %V8i64 = call <8 x i64> @llvm.masked.udiv(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+
+ %V4i32 = call <4 x i32> @llvm.masked.udiv(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+ %V8i32 = call <8 x i32> @llvm.masked.udiv(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+ %V16i32 = call <16 x i32> @llvm.masked.udiv(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+
+ %V8i16 = call <8 x i16> @llvm.masked.udiv(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+ %V16i16 = call <16 x i16> @llvm.masked.udiv(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+ %V32i16 = call <32 x i16> @llvm.masked.udiv(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+
+ %V16i8 = call <16 x i8> @llvm.masked.udiv(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+ %V32i8 = call <32 x i8> @llvm.masked.udiv(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+ %V64i8 = call <64 x i8> @llvm.masked.udiv(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+ ret void
+}
+
+define void @sdiv() {
+; SSE2-LABEL: 'sdiv'
+; SSE2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:7 Lat:6 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:10 Lat:8 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:16 Lat:12 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:46 CodeSize:7 Lat:6 SizeLat:7 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:92 CodeSize:10 Lat:8 SizeLat:10 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:184 CodeSize:16 Lat:12 SizeLat:16 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:7 Lat:6 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:10 Lat:8 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:120 CodeSize:16 Lat:12 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:7 Lat:6 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:10 Lat:8 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:232 CodeSize:16 Lat:12 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'sdiv'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:6 Lat:8 SizeLat:8 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:8 Lat:12 SizeLat:12 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:92 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:184 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:120 CodeSize:8 Lat:12 SizeLat:12 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:232 CodeSize:8 Lat:12 SizeLat:12 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'sdiv'
+; AVX1-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3283 CodeSize:5 Lat:7 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6566 CodeSize:6 Lat:10 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:91 CodeSize:5 Lat:7 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:182 CodeSize:6 Lat:10 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:59 CodeSize:7 Lat:7 SizeLat:7 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:118 CodeSize:10 Lat:10 SizeLat:10 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:115 CodeSize:7 Lat:7 SizeLat:7 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:230 CodeSize:10 Lat:10 SizeLat:10 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'sdiv'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3282 CodeSize:5 Lat:6 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6564 CodeSize:6 Lat:8 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512F-LABEL: 'sdiv'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:57 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:30 CodeSize:8 Lat:6 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:58 CodeSize:8 Lat:6 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'sdiv'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:33 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:5 Lat:5 SizeLat:5 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SLM-LABEL: 'sdiv'
+; SLM-NEXT: Cost Model: Found costs of RThru:1644 CodeSize:5 Lat:8 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.sdiv.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:3288 CodeSize:6 Lat:12 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.sdiv.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:6576 CodeSize:8 Lat:20 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.sdiv.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.sdiv.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:92 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.sdiv.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:184 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.sdiv.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:5 Lat:8 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.sdiv.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:6 Lat:12 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.sdiv.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:128 CodeSize:8 Lat:20 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.sdiv.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:60 CodeSize:5 Lat:8 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.sdiv.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:120 CodeSize:6 Lat:12 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.sdiv.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:240 CodeSize:8 Lat:20 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.sdiv.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V2i64 = call <2 x i64> @llvm.masked.sdiv(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+ %V4i64 = call <4 x i64> @llvm.masked.sdiv(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+ %V8i64 = call <8 x i64> @llvm.masked.sdiv(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+
+ %V4i32 = call <4 x i32> @llvm.masked.sdiv(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+ %V8i32 = call <8 x i32> @llvm.masked.sdiv(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+ %V16i32 = call <16 x i32> @llvm.masked.sdiv(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+
+ %V8i16 = call <8 x i16> @llvm.masked.sdiv(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+ %V16i16 = call <16 x i16> @llvm.masked.sdiv(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+ %V32i16 = call <32 x i16> @llvm.masked.sdiv(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+
+ %V16i8 = call <16 x i8> @llvm.masked.sdiv(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+ %V32i8 = call <32 x i8> @llvm.masked.sdiv(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+ %V64i8 = call <64 x i8> @llvm.masked.sdiv(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+ ret void
+}
+
+define void @urem() {
+; SSE2-LABEL: 'urem'
+; SSE2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:7 Lat:6 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:10 Lat:8 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:16 Lat:12 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:7 Lat:6 SizeLat:7 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:10 Lat:8 SizeLat:10 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:16 Lat:12 SizeLat:16 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:7 Lat:6 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:10 Lat:8 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:120 CodeSize:16 Lat:12 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:7 Lat:6 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:10 Lat:8 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:232 CodeSize:16 Lat:12 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'urem'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:6 Lat:8 SizeLat:8 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:8 Lat:12 SizeLat:12 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:120 CodeSize:8 Lat:12 SizeLat:12 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:232 CodeSize:8 Lat:12 SizeLat:12 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'urem'
+; AVX1-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3283 CodeSize:5 Lat:7 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6566 CodeSize:6 Lat:10 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:4003 CodeSize:5 Lat:7 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:8006 CodeSize:6 Lat:10 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:59 CodeSize:7 Lat:7 SizeLat:7 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:118 CodeSize:10 Lat:10 SizeLat:10 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:115 CodeSize:7 Lat:7 SizeLat:7 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:230 CodeSize:10 Lat:10 SizeLat:10 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'urem'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3282 CodeSize:5 Lat:6 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6564 CodeSize:6 Lat:8 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512F-LABEL: 'urem'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:57 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:30 CodeSize:8 Lat:6 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:58 CodeSize:8 Lat:6 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'urem'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:33 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:5 Lat:5 SizeLat:5 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SLM-LABEL: 'urem'
+; SLM-NEXT: Cost Model: Found costs of RThru:1644 CodeSize:5 Lat:8 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.urem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:3288 CodeSize:6 Lat:12 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.urem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:6576 CodeSize:8 Lat:20 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.urem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:2002 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.urem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:4004 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.urem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:8008 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.urem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:5 Lat:8 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.urem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:6 Lat:12 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.urem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:128 CodeSize:8 Lat:20 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.urem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:60 CodeSize:5 Lat:8 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.urem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:120 CodeSize:6 Lat:12 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.urem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:240 CodeSize:8 Lat:20 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.urem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V2i64 = call <2 x i64> @llvm.masked.urem(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+ %V4i64 = call <4 x i64> @llvm.masked.urem(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+ %V8i64 = call <8 x i64> @llvm.masked.urem(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+
+ %V4i32 = call <4 x i32> @llvm.masked.urem(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+ %V8i32 = call <8 x i32> @llvm.masked.urem(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+ %V16i32 = call <16 x i32> @llvm.masked.urem(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+
+ %V8i16 = call <8 x i16> @llvm.masked.urem(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+ %V16i16 = call <16 x i16> @llvm.masked.urem(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+ %V32i16 = call <32 x i16> @llvm.masked.urem(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+
+ %V16i8 = call <16 x i8> @llvm.masked.urem(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+ %V32i8 = call <32 x i8> @llvm.masked.urem(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+ %V64i8 = call <64 x i8> @llvm.masked.urem(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+ ret void
+}
+
+define void @srem() {
+; SSE2-LABEL: 'srem'
+; SSE2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:7 Lat:6 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:10 Lat:8 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:16 Lat:12 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:46 CodeSize:7 Lat:6 SizeLat:7 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:92 CodeSize:10 Lat:8 SizeLat:10 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:184 CodeSize:16 Lat:12 SizeLat:16 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:7 Lat:6 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:10 Lat:8 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:120 CodeSize:16 Lat:12 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:7 Lat:6 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:10 Lat:8 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:232 CodeSize:16 Lat:12 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'srem'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:3284 CodeSize:6 Lat:8 SizeLat:8 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:6568 CodeSize:8 Lat:12 SizeLat:12 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:92 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:184 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:120 CodeSize:8 Lat:12 SizeLat:12 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:232 CodeSize:8 Lat:12 SizeLat:12 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'srem'
+; AVX1-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:3283 CodeSize:5 Lat:7 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:6566 CodeSize:6 Lat:10 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:91 CodeSize:5 Lat:7 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:182 CodeSize:6 Lat:10 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:59 CodeSize:7 Lat:7 SizeLat:7 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:118 CodeSize:10 Lat:10 SizeLat:10 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:115 CodeSize:7 Lat:7 SizeLat:7 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:230 CodeSize:10 Lat:10 SizeLat:10 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'srem'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1642 CodeSize:5 Lat:6 SizeLat:6 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:3282 CodeSize:5 Lat:6 SizeLat:6 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:6564 CodeSize:6 Lat:8 SizeLat:8 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:5 Lat:6 SizeLat:6 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:60 CodeSize:6 Lat:8 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:5 Lat:6 SizeLat:6 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:58 CodeSize:5 Lat:6 SizeLat:6 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:116 CodeSize:6 Lat:8 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512F-LABEL: 'srem'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:57 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:30 CodeSize:8 Lat:6 SizeLat:8 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:29 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:58 CodeSize:8 Lat:6 SizeLat:8 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'srem'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1641 CodeSize:5 Lat:5 SizeLat:5 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3281 CodeSize:5 Lat:5 SizeLat:5 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6561 CodeSize:5 Lat:5 SizeLat:5 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:5 Lat:5 SizeLat:5 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:17 CodeSize:5 Lat:5 SizeLat:5 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:33 CodeSize:5 Lat:5 SizeLat:5 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:6 CodeSize:5 Lat:5 SizeLat:5 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:11 CodeSize:5 Lat:5 SizeLat:5 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:5 Lat:5 SizeLat:5 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:41 CodeSize:5 Lat:5 SizeLat:5 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SLM-LABEL: 'srem'
+; SLM-NEXT: Cost Model: Found costs of RThru:1644 CodeSize:5 Lat:8 SizeLat:7 for: %V2i64 = call <2 x i64> @llvm.masked.srem.v2i64(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:3288 CodeSize:6 Lat:12 SizeLat:10 for: %V4i64 = call <4 x i64> @llvm.masked.srem.v4i64(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:6576 CodeSize:8 Lat:20 SizeLat:16 for: %V8i64 = call <8 x i64> @llvm.masked.srem.v8i64(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:46 CodeSize:5 Lat:6 SizeLat:6 for: %V4i32 = call <4 x i32> @llvm.masked.srem.v4i32(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:92 CodeSize:6 Lat:8 SizeLat:8 for: %V8i32 = call <8 x i32> @llvm.masked.srem.v8i32(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:184 CodeSize:8 Lat:12 SizeLat:12 for: %V16i32 = call <16 x i32> @llvm.masked.srem.v16i32(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:5 Lat:8 SizeLat:7 for: %V8i16 = call <8 x i16> @llvm.masked.srem.v8i16(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:6 Lat:12 SizeLat:10 for: %V16i16 = call <16 x i16> @llvm.masked.srem.v16i16(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:128 CodeSize:8 Lat:20 SizeLat:16 for: %V32i16 = call <32 x i16> @llvm.masked.srem.v32i16(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:60 CodeSize:5 Lat:8 SizeLat:7 for: %V16i8 = call <16 x i8> @llvm.masked.srem.v16i8(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:120 CodeSize:6 Lat:12 SizeLat:10 for: %V32i8 = call <32 x i8> @llvm.masked.srem.v32i8(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:240 CodeSize:8 Lat:20 SizeLat:16 for: %V64i8 = call <64 x i8> @llvm.masked.srem.v64i8(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V2i64 = call <2 x i64> @llvm.masked.srem(<2 x i64> poison, <2 x i64> poison, <2 x i1> poison)
+ %V4i64 = call <4 x i64> @llvm.masked.srem(<4 x i64> poison, <4 x i64> poison, <4 x i1> poison)
+ %V8i64 = call <8 x i64> @llvm.masked.srem(<8 x i64> poison, <8 x i64> poison, <8 x i1> poison)
+
+ %V4i32 = call <4 x i32> @llvm.masked.srem(<4 x i32> poison, <4 x i32> poison, <4 x i1> poison)
+ %V8i32 = call <8 x i32> @llvm.masked.srem(<8 x i32> poison, <8 x i32> poison, <8 x i1> poison)
+ %V16i32 = call <16 x i32> @llvm.masked.srem(<16 x i32> poison, <16 x i32> poison, <16 x i1> poison)
+
+ %V8i16 = call <8 x i16> @llvm.masked.srem(<8 x i16> poison, <8 x i16> poison, <8 x i1> poison)
+ %V16i16 = call <16 x i16> @llvm.masked.srem(<16 x i16> poison, <16 x i16> poison, <16 x i1> poison)
+ %V32i16 = call <32 x i16> @llvm.masked.srem(<32 x i16> poison, <32 x i16> poison, <32 x i1> poison)
+
+ %V16i8 = call <16 x i8> @llvm.masked.srem(<16 x i8> poison, <16 x i8> poison, <16 x i1> poison)
+ %V32i8 = call <32 x i8> @llvm.masked.srem(<32 x i8> poison, <32 x i8> poison, <32 x i1> poison)
+ %V64i8 = call <64 x i8> @llvm.masked.srem(<64 x i8> poison, <64 x i8> poison, <64 x i1> poison)
+ ret void
+}
+
More information about the llvm-commits
mailing list