[llvm] [X86][CostModel] Add vXi64 divide/remainder-by-constant costs (PR #208491)
Rito Takeuchi via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 13 06:19:39 PDT 2026
https://github.com/Licht-T updated https://github.com/llvm/llvm-project/pull/208491
>From d525e66791a74eab724f3a0c817fddf5bebeece0 Mon Sep 17 00:00:00 2001
From: Rito Takeuchi <licht-t at outlook.jp>
Date: Tue, 7 Jul 2026 07:33:16 +0900
Subject: [PATCH 1/4] [X86] Add vXi64 UDIV/UREM by-constant costs
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 12 +
llvm/test/Analysis/CostModel/X86/div.ll | 48 +-
llvm/test/Analysis/CostModel/X86/rem.ll | 48 +-
.../LoopVectorize/X86/idiv-by-const.ll | 758 ++++++++++++++++++
4 files changed, 818 insertions(+), 48 deletions(-)
create mode 100644 llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2993173d35edc..b4f86257dbec2 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -450,6 +450,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::SREM, MVT::v16i32, { 8 } }, // pmuludq+mul+sub sequence
{ ISD::UDIV, MVT::v16i32, { 5 } }, // pmuludq sequence
{ ISD::UREM, MVT::v16i32, { 7 } }, // pmuludq+mul+sub sequence
+
+ { ISD::UDIV, MVT::v8i64, { 15 } }, // pmuludq-based MULHU sequence
+ { ISD::UREM, MVT::v8i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
};
if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX512())
@@ -491,6 +494,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::SREM, MVT::v8i32, { 8 } }, // pmuludq+mul+sub sequence
{ ISD::UDIV, MVT::v8i32, { 5 } }, // pmuludq sequence
{ ISD::UREM, MVT::v8i32, { 7 } }, // pmuludq+mul+sub sequence
+
+ { ISD::UDIV, MVT::v4i64, { 15 } }, // pmuludq-based MULHU sequence
+ { ISD::UREM, MVT::v4i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
};
if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasAVX2())
@@ -606,6 +612,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::SREM, MVT::v16i32, { 17 } }, // vpmuldq+mul+sub sequence
{ ISD::UDIV, MVT::v16i32, { 15 } }, // vpmuludq sequence
{ ISD::UREM, MVT::v16i32, { 17 } }, // vpmuludq+mul+sub sequence
+
+ { ISD::UDIV, MVT::v8i64, { 22 } }, // vpmuludq-based MULHU sequence
+ { ISD::UREM, MVT::v8i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
};
if (Op2Info.isConstant() && ST->hasAVX512())
@@ -629,6 +638,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::SREM, MVT::v8i32, { 19 } }, // vpmuldq+mul+sub sequence
{ ISD::UDIV, MVT::v8i32, { 15 } }, // vpmuludq sequence
{ ISD::UREM, MVT::v8i32, { 19 } }, // vpmuludq+mul+sub sequence
+
+ { ISD::UDIV, MVT::v4i64, { 22 } }, // vpmuludq-based MULHU sequence
+ { ISD::UREM, MVT::v4i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
};
if (Op2Info.isConstant() && ST->hasAVX2())
diff --git a/llvm/test/Analysis/CostModel/X86/div.ll b/llvm/test/Analysis/CostModel/X86/div.ll
index 3af5b2dd217da..7bb05aa60e612 100644
--- a/llvm/test/Analysis/CostModel/X86/div.ll
+++ b/llvm/test/Analysis/CostModel/X86/div.ll
@@ -296,8 +296,8 @@ define i32 @udiv_const() {
; AVX2-LABEL: 'udiv_const'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX2-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX2-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -315,8 +315,8 @@ define i32 @udiv_const() {
; AVX512F-LABEL: 'udiv_const'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -334,8 +334,8 @@ define i32 @udiv_const() {
; AVX512BW-LABEL: 'udiv_const'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -534,8 +534,8 @@ define i32 @udiv_uniformconst() {
; AVX2-LABEL: 'udiv_uniformconst'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -553,8 +553,8 @@ define i32 @udiv_uniformconst() {
; AVX512F-LABEL: 'udiv_uniformconst'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -572,8 +572,8 @@ define i32 @udiv_uniformconst() {
; AVX512BW-LABEL: 'udiv_uniformconst'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -1400,8 +1400,8 @@ define i32 @udiv_constnegpow2() {
; AVX2-LABEL: 'udiv_constnegpow2'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX2-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX2-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1419,8 +1419,8 @@ define i32 @udiv_constnegpow2() {
; AVX512F-LABEL: 'udiv_constnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1438,8 +1438,8 @@ define i32 @udiv_constnegpow2() {
; AVX512BW-LABEL: 'udiv_constnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1638,8 +1638,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX2-LABEL: 'udiv_uniformconstnegpow2'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
@@ -1657,8 +1657,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512F-LABEL: 'udiv_uniformconstnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
@@ -1676,8 +1676,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512BW-LABEL: 'udiv_uniformconstnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:80 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
diff --git a/llvm/test/Analysis/CostModel/X86/rem.ll b/llvm/test/Analysis/CostModel/X86/rem.ll
index 06b23c0cf6428..3e9837e9c13a1 100644
--- a/llvm/test/Analysis/CostModel/X86/rem.ll
+++ b/llvm/test/Analysis/CostModel/X86/rem.ll
@@ -410,8 +410,8 @@ define i32 @urem_const() {
; AVX2-LABEL: 'urem_const'
; AVX2-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX2-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -429,8 +429,8 @@ define i32 @urem_const() {
; AVX512F-LABEL: 'urem_const'
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512F-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -448,8 +448,8 @@ define i32 @urem_const() {
; AVX512BW-LABEL: 'urem_const'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -648,8 +648,8 @@ define i32 @urem_uniformconst() {
; AVX2-LABEL: 'urem_uniformconst'
; AVX2-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:42 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 7)
@@ -667,8 +667,8 @@ define i32 @urem_uniformconst() {
; AVX512F-LABEL: 'urem_uniformconst'
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 7)
@@ -686,8 +686,8 @@ define i32 @urem_uniformconst() {
; AVX512BW-LABEL: 'urem_uniformconst'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 7)
@@ -1514,8 +1514,8 @@ define i32 @urem_constnegpow2() {
; AVX2-LABEL: 'urem_constnegpow2'
; AVX2-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX2-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX2-NEXT: Cost Model: Found costs of RThru:56 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX2-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1533,8 +1533,8 @@ define i32 @urem_constnegpow2() {
; AVX512F-LABEL: 'urem_constnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512F-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1552,8 +1552,8 @@ define i32 @urem_constnegpow2() {
; AVX512BW-LABEL: 'urem_constnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1752,8 +1752,8 @@ define i32 @urem_uniformconstnegpow2() {
; AVX2-LABEL: 'urem_uniformconstnegpow2'
; AVX2-NEXT: Cost Model: Found costs of 4 for: %I64 = urem i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:320 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:640 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:42 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 -16)
@@ -1771,8 +1771,8 @@ define i32 @urem_uniformconstnegpow2() {
; AVX512F-LABEL: 'urem_uniformconstnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 -16)
@@ -1790,8 +1790,8 @@ define i32 @urem_uniformconstnegpow2() {
; AVX512BW-LABEL: 'urem_uniformconstnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:240 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:480 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 -16)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
new file mode 100644
index 0000000000000..a7c2f8ec14024
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
@@ -0,0 +1,758 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=icelake-server | FileCheck %s --check-prefix=AVX512DQ256
+
+; Division of i64 by a loop-invariant constant becomes a magic multiply-high
+; sequence, so these loops should vectorize exactly where that sequence is
+; formed: MULHU (unsigned) needs AVX2. The signed cases still scalarize their
+; multiply everywhere here, so vectorizing them is a measured regression and
+; the cost model must decline.
+;
+; The last RUN line matters on its own: Intel AVX512 CPUs set
+; prefer-256-bit, so they read the v4i64 entries rather than the v8i64
+; ones the plain -mattr runs above exercise.
+
+define void @udiv_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
+;
+;
+; SSE-LABEL: define void @udiv_by_const(
+; SSE-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; SSE-NEXT: [[ENTRY:.*]]:
+; SSE-NEXT: br label %[[LOOP:.*]]
+; SSE: [[LOOP]]:
+; SSE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; SSE-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; SSE-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; SSE-NEXT: [[D:%.*]] = udiv i64 [[V]], 1000000007
+; SSE-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; SSE-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; SSE-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; SSE-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; SSE-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; SSE: [[EXIT]]:
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @udiv_by_const(
+; AVX2-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT: [[ITER_CHECK:.*]]:
+; AVX2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX2: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX2-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; AVX2-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX2: [[VECTOR_PH]]:
+; AVX2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; AVX2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 4
+; AVX2-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[TMP0]], i64 8
+; AVX2-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP0]], i64 12
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; AVX2-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; AVX2-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; AVX2-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; AVX2-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP6:%.*]] = udiv <4 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX2-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 4
+; AVX2-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX2-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 12
+; AVX2-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX2-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX2-NEXT: store <4 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX2-NEXT: store <4 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; AVX2-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX2-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX2: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX2-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; AVX2-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX2: [[VEC_EPILOG_PH]]:
+; AVX2-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX2-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 4
+; AVX2-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX2-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX2: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX2-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX2-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
+; AVX2-NEXT: [[TMP14:%.*]] = udiv <4 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX2-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX2-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 4
+; AVX2-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX2-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX2: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX2-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX2: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX2-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX2-NEXT: [[D:%.*]] = udiv i64 [[V]], 1000000007
+; AVX2-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX2-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX2-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX2-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX2-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @udiv_by_const(
+; AVX512F-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512F-NEXT: [[ITER_CHECK:.*]]:
+; AVX512F-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512F-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX512F: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX512F-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; AVX512F-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512F: [[VECTOR_PH]]:
+; AVX512F-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; AVX512F-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512F-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512F: [[VECTOR_BODY]]:
+; AVX512F-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512F-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512F-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 8
+; AVX512F-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[TMP0]], i64 16
+; AVX512F-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP0]], i64 24
+; AVX512F-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8
+; AVX512F-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; AVX512F-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
+; AVX512F-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i64>, ptr [[TMP3]], align 8
+; AVX512F-NEXT: [[TMP4:%.*]] = udiv <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP5:%.*]] = udiv <8 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP6:%.*]] = udiv <8 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP7:%.*]] = udiv <8 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512F-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX512F-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 16
+; AVX512F-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 24
+; AVX512F-NEXT: store <8 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX512F-NEXT: store <8 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX512F-NEXT: store <8 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX512F-NEXT: store <8 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX512F-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX512F-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512F-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX512F: [[MIDDLE_BLOCK]]:
+; AVX512F-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512F-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX512F: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX512F-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; AVX512F-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX512F: [[VEC_EPILOG_PH]]:
+; AVX512F-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512F-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 8
+; AVX512F-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX512F-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX512F: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX512F-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512F-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX512F-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i64>, ptr [[TMP13]], align 8
+; AVX512F-NEXT: [[TMP14:%.*]] = udiv <8 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX512F-NEXT: store <8 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX512F-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 8
+; AVX512F-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX512F-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX512F: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX512F-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX512F-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX512F: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX512F-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; AVX512F-NEXT: br label %[[LOOP:.*]]
+; AVX512F: [[LOOP]]:
+; AVX512F-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512F-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512F-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512F-NEXT: [[D:%.*]] = udiv i64 [[V]], 1000000007
+; AVX512F-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512F-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512F-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512F-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512F-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; AVX512F: [[EXIT]]:
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @udiv_by_const(
+; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512DQ-NEXT: [[ITER_CHECK:.*]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX512DQ: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ: [[VECTOR_PH]]:
+; AVX512DQ-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; AVX512DQ-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512DQ-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512DQ: [[VECTOR_BODY]]:
+; AVX512DQ-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512DQ-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512DQ-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[TMP0]], i64 16
+; AVX512DQ-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP0]], i64 24
+; AVX512DQ-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i64>, ptr [[TMP3]], align 8
+; AVX512DQ-NEXT: [[TMP4:%.*]] = udiv <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP5:%.*]] = udiv <8 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP6:%.*]] = udiv <8 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP7:%.*]] = udiv <8 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512DQ-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX512DQ-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 16
+; AVX512DQ-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 24
+; AVX512DQ-NEXT: store <8 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; AVX512DQ-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX512DQ: [[MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX512DQ: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX512DQ-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX512DQ: [[VEC_EPILOG_PH]]:
+; AVX512DQ-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512DQ-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 8
+; AVX512DQ-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX512DQ-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX512DQ: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX512DQ-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512DQ-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX512DQ-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i64>, ptr [[TMP13]], align 8
+; AVX512DQ-NEXT: [[TMP14:%.*]] = udiv <8 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX512DQ-NEXT: store <8 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 8
+; AVX512DQ-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX512DQ-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX512DQ: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX512DQ-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX512DQ: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX512DQ-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; AVX512DQ-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ: [[LOOP]]:
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[D:%.*]] = udiv i64 [[V]], 1000000007
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; AVX512DQ: [[EXIT]]:
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @udiv_by_const(
+; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512DQ256-NEXT: [[ITER_CHECK:.*]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX512DQ256: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ256: [[VECTOR_PH]]:
+; AVX512DQ256-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; AVX512DQ256-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512DQ256-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512DQ256: [[VECTOR_BODY]]:
+; AVX512DQ256-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512DQ256-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[TMP0]], i64 4
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[TMP0]], i64 8
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP0]], i64 12
+; AVX512DQ256-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP5:%.*]] = udiv <4 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP6:%.*]] = udiv <4 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP7:%.*]] = udiv <4 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512DQ256-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 4
+; AVX512DQ256-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX512DQ256-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 12
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
+; AVX512DQ256-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX512DQ256: [[MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX512DQ256: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX512DQ256-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3:![0-9]+]]
+; AVX512DQ256: [[VEC_EPILOG_PH]]:
+; AVX512DQ256-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512DQ256-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 4
+; AVX512DQ256-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX512DQ256-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX512DQ256: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX512DQ256-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512DQ256-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX512DQ256-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
+; AVX512DQ256-NEXT: [[TMP14:%.*]] = udiv <4 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 4
+; AVX512DQ256-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX512DQ256-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; AVX512DQ256: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX512DQ256: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX512DQ256-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ITER_CHECK]] ]
+; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ256: [[LOOP]]:
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[D:%.*]] = udiv i64 [[V]], 1000000007
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
+; AVX512DQ256: [[EXIT]]:
+; AVX512DQ256-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.n, %loop]
+ %p = getelementptr i64, ptr %i, i64 %iv
+ %v = load i64, ptr %p
+ %d = udiv i64 %v, 1000000007
+ %q = getelementptr i64, ptr %o, i64 %iv
+ store i64 %d, ptr %q
+ %iv.n = add i64 %iv, 1
+ %c = icmp eq i64 %iv.n, %n
+ br i1 %c, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @urem_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
+;
+;
+; SSE-LABEL: define void @urem_by_const(
+; SSE-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[ENTRY:.*]]:
+; SSE-NEXT: br label %[[LOOP:.*]]
+; SSE: [[LOOP]]:
+; SSE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; SSE-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; SSE-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; SSE-NEXT: [[D:%.*]] = urem i64 [[V]], 1000000007
+; SSE-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; SSE-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; SSE-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; SSE-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; SSE-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; SSE: [[EXIT]]:
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @urem_by_const(
+; AVX2-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX2-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX2: [[VECTOR_PH]]:
+; AVX2-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; AVX2-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; AVX2-NEXT: [[TMP1:%.*]] = urem <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX2-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX2-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; AVX2-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX2-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX2-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX2: [[SCALAR_PH]]:
+; AVX2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX2-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX2-NEXT: [[D:%.*]] = urem i64 [[V]], 1000000007
+; AVX2-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX2-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX2-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX2-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX2-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @urem_by_const(
+; AVX512F-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[ENTRY:.*]]:
+; AVX512F-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512F-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512F: [[VECTOR_PH]]:
+; AVX512F-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; AVX512F-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512F-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512F: [[VECTOR_BODY]]:
+; AVX512F-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512F-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512F-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8
+; AVX512F-NEXT: [[TMP1:%.*]] = urem <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512F-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512F-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX512F-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; AVX512F-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512F-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; AVX512F: [[MIDDLE_BLOCK]]:
+; AVX512F-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512F-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX512F: [[SCALAR_PH]]:
+; AVX512F-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX512F-NEXT: br label %[[LOOP:.*]]
+; AVX512F: [[LOOP]]:
+; AVX512F-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512F-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512F-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512F-NEXT: [[D:%.*]] = urem i64 [[V]], 1000000007
+; AVX512F-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512F-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512F-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512F-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512F-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; AVX512F: [[EXIT]]:
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @urem_by_const(
+; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[ENTRY:.*]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ: [[VECTOR_PH]]:
+; AVX512DQ-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; AVX512DQ-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512DQ-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512DQ: [[VECTOR_BODY]]:
+; AVX512DQ-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512DQ-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512DQ-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[TMP0]], align 8
+; AVX512DQ-NEXT: [[TMP1:%.*]] = urem <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512DQ-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; AVX512DQ-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; AVX512DQ: [[MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX512DQ: [[SCALAR_PH]]:
+; AVX512DQ-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ: [[LOOP]]:
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[D:%.*]] = urem i64 [[V]], 1000000007
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; AVX512DQ: [[EXIT]]:
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @urem_by_const(
+; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[ENTRY:.*]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ256: [[VECTOR_PH]]:
+; AVX512DQ256-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; AVX512DQ256-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; AVX512DQ256-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512DQ256: [[VECTOR_BODY]]:
+; AVX512DQ256-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512DQ256-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX]]
+; AVX512DQ256-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP0]], align 8
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = urem <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX]]
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; AVX512DQ256: [[MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX512DQ256: [[SCALAR_PH]]:
+; AVX512DQ256-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ256: [[LOOP]]:
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[D:%.*]] = urem i64 [[V]], 1000000007
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
+; AVX512DQ256: [[EXIT]]:
+; AVX512DQ256-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.n, %loop]
+ %p = getelementptr i64, ptr %i, i64 %iv
+ %v = load i64, ptr %p
+ %d = urem i64 %v, 1000000007
+ %q = getelementptr i64, ptr %o, i64 %iv
+ store i64 %d, ptr %q
+ %iv.n = add i64 %iv, 1
+ %c = icmp eq i64 %iv.n, %n
+ br i1 %c, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @sdiv_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
+;
+;
+; SSE-LABEL: define void @sdiv_by_const(
+; SSE-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[ENTRY:.*]]:
+; SSE-NEXT: br label %[[LOOP:.*]]
+; SSE: [[LOOP]]:
+; SSE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; SSE-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; SSE-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; SSE-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
+; SSE-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; SSE-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; SSE-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; SSE-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; SSE-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; SSE: [[EXIT]]:
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @sdiv_by_const(
+; AVX2-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX2-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX2-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
+; AVX2-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX2-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX2-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX2-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX2-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @sdiv_by_const(
+; AVX512F-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[ENTRY:.*]]:
+; AVX512F-NEXT: br label %[[LOOP:.*]]
+; AVX512F: [[LOOP]]:
+; AVX512F-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512F-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512F-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512F-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
+; AVX512F-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512F-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512F-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512F-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512F-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512F: [[EXIT]]:
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @sdiv_by_const(
+; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[ENTRY:.*]]:
+; AVX512DQ-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ: [[LOOP]]:
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ: [[EXIT]]:
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @sdiv_by_const(
+; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[ENTRY:.*]]:
+; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ256: [[LOOP]]:
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ256: [[EXIT]]:
+; AVX512DQ256-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.n, %loop]
+ %p = getelementptr i64, ptr %i, i64 %iv
+ %v = load i64, ptr %p
+ %d = sdiv i64 %v, 1000000007
+ %q = getelementptr i64, ptr %o, i64 %iv
+ store i64 %d, ptr %q
+ %iv.n = add i64 %iv, 1
+ %c = icmp eq i64 %iv.n, %n
+ br i1 %c, label %exit, label %loop
+exit:
+ ret void
+}
+
+define void @srem_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
+;
+;
+; SSE-LABEL: define void @srem_by_const(
+; SSE-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[ENTRY:.*]]:
+; SSE-NEXT: br label %[[LOOP:.*]]
+; SSE: [[LOOP]]:
+; SSE-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; SSE-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; SSE-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; SSE-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
+; SSE-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; SSE-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; SSE-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; SSE-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; SSE-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; SSE: [[EXIT]]:
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @srem_by_const(
+; AVX2-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[LOOP:.*]]
+; AVX2: [[LOOP]]:
+; AVX2-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX2-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX2-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
+; AVX2-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX2-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX2-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX2-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX2-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX2: [[EXIT]]:
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @srem_by_const(
+; AVX512F-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[ENTRY:.*]]:
+; AVX512F-NEXT: br label %[[LOOP:.*]]
+; AVX512F: [[LOOP]]:
+; AVX512F-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512F-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512F-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512F-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
+; AVX512F-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512F-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512F-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512F-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512F-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512F: [[EXIT]]:
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @srem_by_const(
+; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[ENTRY:.*]]:
+; AVX512DQ-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ: [[LOOP]]:
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ: [[EXIT]]:
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @srem_by_const(
+; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[ENTRY:.*]]:
+; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
+; AVX512DQ256: [[LOOP]]:
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ256: [[EXIT]]:
+; AVX512DQ256-NEXT: ret void
+;
+entry:
+ br label %loop
+loop:
+ %iv = phi i64 [0, %entry], [%iv.n, %loop]
+ %p = getelementptr i64, ptr %i, i64 %iv
+ %v = load i64, ptr %p
+ %d = srem i64 %v, 1000000007
+ %q = getelementptr i64, ptr %o, i64 %iv
+ store i64 %d, ptr %q
+ %iv.n = add i64 %iv, 1
+ %c = icmp eq i64 %iv.n, %n
+ br i1 %c, label %exit, label %loop
+exit:
+ ret void
+}
+
>From 6a10860e0f5a9296a2437dc9d98b0dba72c3edaa Mon Sep 17 00:00:00 2001
From: Rito Takeuchi <licht-t at outlook.jp>
Date: Sun, 12 Jul 2026 15:24:32 +0900
Subject: [PATCH 2/4] [X86] Add DQ-gated vXi64 SDIV/SREM by-constant costs
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 35 +++
llvm/test/Analysis/CostModel/X86/div.ll | 229 ++++++++++++++++++
llvm/test/Analysis/CostModel/X86/rem.ll | 191 +++++++++++++++
.../LoopVectorize/X86/idiv-by-const.ll | 202 +++++++++++++--
4 files changed, 637 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index b4f86257dbec2..6493cd5c40c1f 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -422,6 +422,24 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
if (auto KindCost = Entry->Cost[CostKind])
return LT.first * *KindCost;
+ static const CostKindTblEntry AVX512DQUniformConstCostTable[] = {
+ { ISD::SDIV, MVT::v4i64, { 15 } }, // vpmullq-based MULHS sequence
+ { ISD::SREM, MVT::v4i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
+ { ISD::SDIV, MVT::v8i64, { 15 } }, // vpmullq-based MULHS sequence
+ { ISD::SREM, MVT::v8i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
+ // The remainder's multiply-back is a single vpmullq with DQ, just like the
+ // pmulld the vXi32 entries above rely on. Without DQ it is another
+ // vpmuludq schoolbook, so the AVX512/AVX2 tables charge more.
+ { ISD::UREM, MVT::v4i64, { 17 } }, // MULHU + vpmullq + sub sequence
+ { ISD::UREM, MVT::v8i64, { 17 } }, // MULHU + vpmullq + sub sequence
+ };
+
+ if (Op2Info.isUniform() && Op2Info.isConstant() && ST->hasDQI())
+ if (const auto *Entry =
+ CostTableLookup(AVX512DQUniformConstCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
static const CostKindTblEntry AVX512UniformConstCostTable[] = {
{ ISD::SHL, MVT::v64i8, { 2, 12, 5, 6 } }, // psllw + pand.
{ ISD::SRL, MVT::v64i8, { 2, 12, 5, 6 } }, // psrlw + pand.
@@ -597,6 +615,23 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
if (auto KindCost = Entry->Cost[CostKind])
return LT.first * *KindCost;
+ static const CostKindTblEntry AVX512DQConstCostTable[] = {
+ { ISD::SDIV, MVT::v4i64, { 19 } }, // vpmullq-based MULHS sequence
+ { ISD::SREM, MVT::v4i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
+ { ISD::SDIV, MVT::v8i64, { 19 } }, // vpmullq-based MULHS sequence
+ { ISD::SREM, MVT::v8i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
+ // The remainder's multiply-back is a single vpmullq with DQ, whereas the
+ // AVX512/AVX2 tables have to charge for another vpmuludq schoolbook.
+ { ISD::UREM, MVT::v4i64, { 24 } }, // MULHU + vpmullq + sub sequence
+ { ISD::UREM, MVT::v8i64, { 24 } }, // MULHU + vpmullq + sub sequence
+ };
+
+ if (Op2Info.isConstant() && ST->hasDQI())
+ if (const auto *Entry =
+ CostTableLookup(AVX512DQConstCostTable, ISD, LT.second))
+ if (auto KindCost = Entry->Cost[CostKind])
+ return LT.first * *KindCost;
+
static const CostKindTblEntry AVX512ConstCostTable[] = {
{ ISD::SDIV, MVT::v64i8, { 28 } }, // 4*ext+4*pmulhw sequence
{ ISD::SREM, MVT::v64i8, { 32 } }, // 4*ext+4*pmulhw+mul+sub sequence
diff --git a/llvm/test/Analysis/CostModel/X86/div.ll b/llvm/test/Analysis/CostModel/X86/div.ll
index 7bb05aa60e612..78dd39ac292c3 100644
--- a/llvm/test/Analysis/CostModel/X86/div.ll
+++ b/llvm/test/Analysis/CostModel/X86/div.ll
@@ -6,6 +6,7 @@
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX2
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512DQ
;
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
@@ -212,6 +213,25 @@ define i32 @sdiv_const() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'sdiv_const'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = sdiv i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = sdiv i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'sdiv_const'
; SLM-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, 7
; SLM-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 6, i64 7>
@@ -349,6 +369,25 @@ define i32 @udiv_const() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv_const'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = udiv i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = udiv i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, 7
%V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
@@ -468,6 +507,25 @@ define i32 @sdiv_uniformconst() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sdiv_uniformconst'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = sdiv i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = sdiv i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = sdiv i64 undef, 7
%V2i64 = sdiv <2 x i64> undef, <i64 7, i64 7>
@@ -587,6 +645,25 @@ define i32 @udiv_uniformconst() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv_uniformconst'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = udiv i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = udiv i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, 7
%V2i64 = udiv <2 x i64> undef, <i64 7, i64 7>
@@ -726,6 +803,25 @@ define i32 @sdiv_constpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:38 CodeSize:76 Lat:58 SizeLat:79 for: %V64i8 = sdiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'sdiv_constpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I64 = sdiv i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V2i64 = sdiv <2 x i64> undef, <i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:4 SizeLat:5 for: %V4i64 = sdiv <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V8i64 = sdiv <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I32 = sdiv i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V4i32 = sdiv <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:4 SizeLat:5 for: %V8i32 = sdiv <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V16i32 = sdiv <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = sdiv i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:16 Lat:34 SizeLat:31 for: %V8i16 = sdiv <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:25 CodeSize:31 Lat:31 SizeLat:44 for: %V16i16 = sdiv <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:36 CodeSize:38 Lat:55 SizeLat:50 for: %V32i16 = sdiv <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I8 = sdiv i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:61 Lat:62 SizeLat:79 for: %V16i8 = sdiv <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:45 CodeSize:61 Lat:71 SizeLat:112 for: %V32i8 = sdiv <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:92 CodeSize:137 Lat:100 SizeLat:167 for: %V64i8 = sdiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'sdiv_constpow2'
; SLM-NEXT: Cost Model: Found costs of 4 for: %I64 = sdiv i64 undef, 16
; SLM-NEXT: Cost Model: Found costs of RThru:24 CodeSize:16 Lat:42 SizeLat:23 for: %V2i64 = sdiv <2 x i64> undef, <i64 8, i64 16>
@@ -883,6 +979,25 @@ define i32 @udiv_constpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:7 CodeSize:15 Lat:27 SizeLat:18 for: %V64i8 = udiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'udiv_constpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I64 = udiv i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V2i64 = udiv <2 x i64> undef, <i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V4i64 = udiv <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V8i64 = udiv <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I32 = udiv i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V4i32 = udiv <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V8i32 = udiv <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V16i32 = udiv <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I16 = udiv i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:11 SizeLat:10 for: %V8i16 = udiv <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:10 Lat:10 SizeLat:14 for: %V16i16 = udiv <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:11 CodeSize:11 Lat:16 SizeLat:15 for: %V32i16 = udiv <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I8 = udiv i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:12 Lat:27 SizeLat:18 for: %V16i8 = udiv <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:30 SizeLat:24 for: %V32i8 = udiv <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:30 Lat:19 SizeLat:36 for: %V64i8 = udiv <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'udiv_constpow2'
; SLM-NEXT: Cost Model: Found costs of 1 for: %I64 = udiv i64 undef, 16
; SLM-NEXT: Cost Model: Found costs of RThru:4 CodeSize:5 Lat:6 SizeLat:7 for: %V2i64 = udiv <2 x i64> undef, <i64 8, i64 16>
@@ -1040,6 +1155,25 @@ define i32 @sdiv_uniformconstpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:4 CodeSize:11 Lat:27 SizeLat:16 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'sdiv_uniformconstpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I64 = sdiv i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:4 SizeLat:5 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I32 = sdiv i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:4 CodeSize:4 Lat:4 SizeLat:5 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = sdiv i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:13 Lat:22 SizeLat:14 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:18 CodeSize:20 Lat:37 SizeLat:26 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I8 = sdiv i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:13 Lat:29 SizeLat:16 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:13 Lat:29 SizeLat:24 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:11 CodeSize:34 Lat:39 SizeLat:35 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'sdiv_uniformconstpow2'
; SLM-NEXT: Cost Model: Found costs of 4 for: %I64 = sdiv i64 undef, 16
; SLM-NEXT: Cost Model: Found costs of RThru:11 CodeSize:14 Lat:13 SizeLat:15 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 16)
@@ -1177,6 +1311,25 @@ define i32 @udiv_uniformconstpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:8 SizeLat:3 for: %V32i8 = udiv <32 x i8> undef, splat (i8 16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:8 SizeLat:3 for: %V64i8 = udiv <64 x i8> undef, splat (i8 16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv_uniformconstpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I64 = udiv i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V2i64 = udiv <2 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V4i64 = udiv <4 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V8i64 = udiv <8 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I32 = udiv i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V4i32 = udiv <4 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V8i32 = udiv <8 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V16i32 = udiv <16 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I16 = udiv i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %V8i16 = udiv <8 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:7 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:5 Lat:10 SizeLat:7 for: %V32i16 = udiv <32 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 1 for: %I8 = udiv i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:8 SizeLat:3 for: %V16i8 = udiv <16 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:2 Lat:8 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:12 SizeLat:6 for: %V64i8 = udiv <64 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, 16
%V2i64 = udiv <2 x i64> undef, <i64 16, i64 16>
@@ -1316,6 +1469,25 @@ define i32 @sdiv_constnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'sdiv_constnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = sdiv i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = sdiv i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'sdiv_constnegpow2'
; SLM-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, -16
; SLM-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 -8, i64 -16>
@@ -1453,6 +1625,25 @@ define i32 @udiv_constnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv_constnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = udiv i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = udiv i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, -16
%V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
@@ -1572,6 +1763,25 @@ define i32 @sdiv_uniformconstnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sdiv_uniformconstnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = sdiv <16 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = sdiv i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = sdiv <8 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = sdiv <16 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = sdiv <32 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = sdiv i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = sdiv <16 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = sdiv <32 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = sdiv <64 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = sdiv i64 undef, -16
%V2i64 = sdiv <2 x i64> undef, <i64 -16, i64 -16>
@@ -1691,6 +1901,25 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'udiv_uniformconstnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = udiv <16 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I16 = udiv i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = udiv <8 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = udiv <16 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = udiv <32 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = udiv i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = udiv <16 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = udiv <32 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = udiv <64 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = udiv i64 undef, -16
%V2i64 = udiv <2 x i64> undef, <i64 -16, i64 -16>
diff --git a/llvm/test/Analysis/CostModel/X86/rem.ll b/llvm/test/Analysis/CostModel/X86/rem.ll
index 3e9837e9c13a1..64a5f9931ac58 100644
--- a/llvm/test/Analysis/CostModel/X86/rem.ll
+++ b/llvm/test/Analysis/CostModel/X86/rem.ll
@@ -6,6 +6,7 @@
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512,AVX512DQ
;
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=slm | FileCheck %s --check-prefixes=SSE,SLM
; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,GLM
@@ -307,6 +308,25 @@ define i32 @srem_const() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'srem_const'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = srem i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, <i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'srem_const'
; SLM-NEXT: Cost Model: Found costs of 4 for: %I64 = srem i64 undef, 7
; SLM-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, <i64 6, i64 7>
@@ -463,6 +483,25 @@ define i32 @urem_const() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'urem_const'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, <i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 17, i16 18, i16 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, <i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 17, i8 18, i8 19>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = urem i64 undef, 7
%V2i64 = urem <2 x i64> undef, <i64 6, i64 7>
@@ -582,6 +621,25 @@ define i32 @srem_uniformconst() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'srem_uniformconst'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = srem i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = srem i64 undef, 7
%V2i64 = srem <2 x i64> undef, <i64 7, i64 7>
@@ -701,6 +759,25 @@ define i32 @urem_uniformconst() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, splat (i8 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'urem_uniformconst'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, splat (i32 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, splat (i16 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, 7
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, splat (i8 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = urem i64 undef, 7
%V2i64 = urem <2 x i64> undef, <i64 7, i64 7>
@@ -859,6 +936,25 @@ define i32 @srem_constpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:42 CodeSize:84 Lat:70 SizeLat:90 for: %V64i8 = srem <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'srem_constpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:11 SizeLat:7 for: %I64 = srem i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:20 SizeLat:8 for: %V2i64 = srem <2 x i64> undef, <i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:20 SizeLat:10 for: %V4i64 = srem <4 x i64> undef, <i64 2, i64 4, i64 8, i64 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:20 SizeLat:8 for: %V8i64 = srem <8 x i64> undef, <i64 2, i64 4, i64 8, i64 16, i64 32, i64 64, i64 128, i64 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:9 SizeLat:6 for: %I32 = srem i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:7 for: %V4i32 = srem <4 x i32> undef, <i32 2, i32 4, i32 8, i32 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:9 for: %V8i32 = srem <8 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:7 for: %V16i32 = srem <16 x i32> undef, <i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64, i32 128, i32 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:9 SizeLat:6 for: %I16 = srem i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:18 CodeSize:18 Lat:40 SizeLat:33 for: %V8i16 = srem <8 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:28 CodeSize:33 Lat:37 SizeLat:48 for: %V16i16 = srem <16 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:41 CodeSize:44 Lat:63 SizeLat:56 for: %V32i16 = srem <32 x i16> undef, <i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 128, i16 256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:9 SizeLat:6 for: %I8 = srem i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:47 CodeSize:68 Lat:81 SizeLat:92 for: %V16i8 = srem <16 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:50 CodeSize:70 Lat:80 SizeLat:130 for: %V32i8 = srem <32 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:97 CodeSize:143 Lat:108 SizeLat:173 for: %V64i8 = srem <64 x i8> undef, <i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16, i8 2, i8 4, i8 8, i8 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'srem_constpow2'
; SLM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:11 SizeLat:7 for: %I64 = srem i64 undef, 16
; SLM-NEXT: Cost Model: Found costs of RThru:45 CodeSize:26 Lat:66 SizeLat:34 for: %V2i64 = srem <2 x i64> undef, <i64 8, i64 16>
@@ -1135,6 +1231,25 @@ define i32 @srem_uniformconstpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:8 CodeSize:19 Lat:39 SizeLat:27 for: %V64i8 = srem <64 x i8> undef, splat (i8 16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'srem_uniformconstpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:11 SizeLat:7 for: %I64 = srem i64 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:20 SizeLat:8 for: %V2i64 = srem <2 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:20 SizeLat:10 for: %V4i64 = srem <4 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:20 SizeLat:8 for: %V8i64 = srem <8 x i64> undef, splat (i64 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:9 SizeLat:6 for: %I32 = srem i32 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:7 for: %V4i32 = srem <4 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:9 for: %V8i32 = srem <8 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:15 SizeLat:7 for: %V16i32 = srem <16 x i32> undef, splat (i32 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:9 SizeLat:6 for: %I16 = srem i16 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:6 Lat:10 SizeLat:6 for: %V8i16 = srem <8 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:10 CodeSize:15 Lat:28 SizeLat:18 for: %V16i16 = srem <16 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:23 CodeSize:26 Lat:45 SizeLat:32 for: %V32i16 = srem <32 x i16> undef, splat (i16 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:6 Lat:9 SizeLat:6 for: %I8 = srem i8 undef, 16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:12 CodeSize:20 Lat:48 SizeLat:29 for: %V16i8 = srem <16 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:14 CodeSize:22 Lat:38 SizeLat:42 for: %V32i8 = srem <32 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:40 Lat:47 SizeLat:41 for: %V64i8 = srem <64 x i8> undef, splat (i8 16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'srem_uniformconstpow2'
; SLM-NEXT: Cost Model: Found costs of RThru:7 CodeSize:6 Lat:11 SizeLat:7 for: %I64 = srem i64 undef, 16
; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:24 Lat:37 SizeLat:26 for: %V2i64 = srem <2 x i64> undef, splat (i64 16)
@@ -1411,6 +1526,25 @@ define i32 @srem_constnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
+; AVX512DQ-LABEL: 'srem_constnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = srem i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, <i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:21 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
; SLM-LABEL: 'srem_constnegpow2'
; SLM-NEXT: Cost Model: Found costs of 4 for: %I64 = srem i64 undef, -16
; SLM-NEXT: Cost Model: Found costs of RThru:160 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, <i64 -8, i64 -16>
@@ -1567,6 +1701,25 @@ define i32 @urem_constnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'urem_constnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:20 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256, i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, <i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256, i16 -2, i16 -4, i16 -8, i16 -16, i16 -32, i16 -64, i16 -128, i16 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, <i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16, i8 -2, i8 -4, i8 -8, i8 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = urem i64 undef, -16
%V2i64 = urem <2 x i64> undef, <i64 -8, i64 -16>
@@ -1686,6 +1839,25 @@ define i32 @srem_uniformconstnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'srem_uniformconstnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = srem i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = srem <2 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = srem <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = srem <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = srem i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = srem <4 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = srem <8 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = srem <16 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = srem i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = srem <8 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = srem <16 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = srem <32 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = srem i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = srem <16 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = srem <32 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = srem <64 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = srem i64 undef, -16
%V2i64 = srem <2 x i64> undef, <i64 -16, i64 -16>
@@ -1805,6 +1977,25 @@ define i32 @urem_uniformconstnegpow2() {
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, splat (i8 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512DQ-LABEL: 'urem_uniformconstnegpow2'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = urem i64 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:120 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = urem <2 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = urem <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:17 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = urem <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = urem i32 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = urem <4 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = urem <8 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:7 CodeSize:4 Lat:4 SizeLat:4 for: %V16i32 = urem <16 x i32> undef, splat (i32 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of 4 for: %I16 = urem i16 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V8i16 = urem <8 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:4 SizeLat:4 for: %V16i16 = urem <16 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i16 = urem <32 x i16> undef, splat (i16 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %I8 = urem i8 undef, -16
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V16i8 = urem <16 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:4 SizeLat:4 for: %V32i8 = urem <32 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:32 CodeSize:4 Lat:4 SizeLat:4 for: %V64i8 = urem <64 x i8> undef, splat (i8 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = urem i64 undef, -16
%V2i64 = urem <2 x i64> undef, <i64 -16, i64 -16>
diff --git a/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
index a7c2f8ec14024..a854b66294fc5 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
@@ -605,34 +605,154 @@ define void @sdiv_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
; AVX512DQ-LABEL: define void @sdiv_by_const(
; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; AVX512DQ-NEXT: [[ENTRY:.*]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX512DQ: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 32
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ: [[VECTOR_PH]]:
+; AVX512DQ-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; AVX512DQ-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; AVX512DQ-NEXT: br label %[[LOOP:.*]]
; AVX512DQ: [[LOOP]]:
-; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
-; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P]], i64 16
+; AVX512DQ-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 24
+; AVX512DQ-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD2:%.*]] = load <8 x i64>, ptr [[TMP1]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD3:%.*]] = load <8 x i64>, ptr [[TMP2]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD4:%.*]] = load <8 x i64>, ptr [[TMP3]], align 8
+; AVX512DQ-NEXT: [[TMP4:%.*]] = sdiv <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP5:%.*]] = sdiv <8 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP6:%.*]] = sdiv <8 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP7:%.*]] = sdiv <8 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX512DQ-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 16
+; AVX512DQ-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 24
+; AVX512DQ-NEXT: store <8 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX512DQ-NEXT: store <8 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 32
+; AVX512DQ-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; AVX512DQ: [[MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX512DQ: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX512DQ-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; AVX512DQ: [[VEC_EPILOG_PH]]:
+; AVX512DQ-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512DQ-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 8
+; AVX512DQ-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX512DQ-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX512DQ: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX512DQ-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512DQ-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX512DQ-NEXT: [[WIDE_LOAD8:%.*]] = load <8 x i64>, ptr [[TMP13]], align 8
+; AVX512DQ-NEXT: [[TMP14:%.*]] = sdiv <8 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX512DQ-NEXT: store <8 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 8
+; AVX512DQ-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX512DQ-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; AVX512DQ: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX512DQ-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX512DQ: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX512DQ-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ-NEXT: br label %[[LOOP1:.*]]
+; AVX512DQ: [[LOOP1]]:
+; AVX512DQ-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP1]] ]
+; AVX512DQ-NEXT: [[P1:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV1]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P1]], align 8
; AVX512DQ-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
-; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV1]]
; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
-; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV1]], 1
; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
-; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
; AVX512DQ: [[EXIT]]:
; AVX512DQ-NEXT: ret void
;
; AVX512DQ256-LABEL: define void @sdiv_by_const(
; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; AVX512DQ256-NEXT: [[ENTRY:.*]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
+; AVX512DQ256: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], 16
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ256: [[VECTOR_PH]]:
+; AVX512DQ256-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 16
+; AVX512DQ256-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
; AVX512DQ256: [[LOOP]]:
-; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
-; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = getelementptr i64, ptr [[P]], i64 4
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P]], i64 8
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[P]], i64 12
+; AVX512DQ256-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i64>, ptr [[TMP2]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD4:%.*]] = load <4 x i64>, ptr [[TMP3]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = sdiv <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP5:%.*]] = sdiv <4 x i64> [[WIDE_LOAD2]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP6:%.*]] = sdiv <4 x i64> [[WIDE_LOAD3]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP7:%.*]] = sdiv <4 x i64> [[WIDE_LOAD4]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i64 4
+; AVX512DQ256-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[TMP8]], i64 8
+; AVX512DQ256-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP8]], i64 12
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP8]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP5]], ptr [[TMP9]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP6]], ptr [[TMP10]], align 8
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP7]], ptr [[TMP11]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 16
+; AVX512DQ256-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[TMP12]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP8:![0-9]+]]
+; AVX512DQ256: [[MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; AVX512DQ256: [[VEC_EPILOG_ITER_CHECK]]:
+; AVX512DQ256-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; AVX512DQ256: [[VEC_EPILOG_PH]]:
+; AVX512DQ256-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; AVX512DQ256-NEXT: [[N_MOD_VF5:%.*]] = urem i64 [[N]], 4
+; AVX512DQ256-NEXT: [[N_VEC6:%.*]] = sub i64 [[N]], [[N_MOD_VF5]]
+; AVX512DQ256-NEXT: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; AVX512DQ256: [[VEC_EPILOG_VECTOR_BODY]]:
+; AVX512DQ256-NEXT: [[INDEX7:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], %[[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT9:%.*]], %[[VEC_EPILOG_VECTOR_BODY]] ]
+; AVX512DQ256-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[I]], i64 [[INDEX7]]
+; AVX512DQ256-NEXT: [[WIDE_LOAD8:%.*]] = load <4 x i64>, ptr [[TMP13]], align 8
+; AVX512DQ256-NEXT: [[TMP14:%.*]] = sdiv <4 x i64> [[WIDE_LOAD8]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[O]], i64 [[INDEX7]]
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP14]], ptr [[TMP15]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT9]] = add nuw i64 [[INDEX7]], 4
+; AVX512DQ256-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT9]], [[N_VEC6]]
+; AVX512DQ256-NEXT: br i1 [[TMP16]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]]
+; AVX512DQ256: [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N10:%.*]] = icmp eq i64 [[N]], [[N_VEC6]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N10]], label %[[EXIT]], label %[[VEC_EPILOG_SCALAR_PH]]
+; AVX512DQ256: [[VEC_EPILOG_SCALAR_PH]]:
+; AVX512DQ256-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC6]], %[[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ256-NEXT: br label %[[LOOP1:.*]]
+; AVX512DQ256: [[LOOP1]]:
+; AVX512DQ256-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP1]] ]
+; AVX512DQ256-NEXT: [[P1:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV1]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P1]], align 8
; AVX512DQ256-NEXT: [[D:%.*]] = sdiv i64 [[V]], 1000000007
-; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV1]]
; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
-; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV1]], 1
; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
-; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP10:![0-9]+]]
; AVX512DQ256: [[EXIT]]:
; AVX512DQ256-NEXT: ret void
;
@@ -709,34 +829,76 @@ define void @srem_by_const(ptr noalias %o, ptr noalias %i, i64 %n) {
; AVX512DQ-LABEL: define void @srem_by_const(
; AVX512DQ-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; AVX512DQ-NEXT: [[ENTRY:.*]]:
+; AVX512DQ-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8
+; AVX512DQ-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ: [[VECTOR_PH]]:
+; AVX512DQ-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 8
+; AVX512DQ-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; AVX512DQ-NEXT: br label %[[LOOP:.*]]
; AVX512DQ: [[LOOP]]:
-; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; AVX512DQ-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
-; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i64>, ptr [[P]], align 8
+; AVX512DQ-NEXT: [[TMP1:%.*]] = srem <8 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: store <8 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX512DQ-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 8
+; AVX512DQ-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; AVX512DQ: [[MIDDLE_BLOCK]]:
+; AVX512DQ-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX512DQ: [[SCALAR_PH]]:
+; AVX512DQ-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ-NEXT: br label %[[LOOP1:.*]]
+; AVX512DQ: [[LOOP1]]:
+; AVX512DQ-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP1]] ]
+; AVX512DQ-NEXT: [[P1:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV1]]
+; AVX512DQ-NEXT: [[V:%.*]] = load i64, ptr [[P1]], align 8
; AVX512DQ-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
-; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV1]]
; AVX512DQ-NEXT: store i64 [[D]], ptr [[Q]], align 8
-; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ-NEXT: [[IV_N]] = add i64 [[IV1]], 1
; AVX512DQ-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
-; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP12:![0-9]+]]
; AVX512DQ: [[EXIT]]:
; AVX512DQ-NEXT: ret void
;
; AVX512DQ256-LABEL: define void @srem_by_const(
; AVX512DQ256-SAME: ptr noalias [[O:%.*]], ptr noalias [[I:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
; AVX512DQ256-NEXT: [[ENTRY:.*]]:
+; AVX512DQ256-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4
+; AVX512DQ256-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; AVX512DQ256: [[VECTOR_PH]]:
+; AVX512DQ256-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4
+; AVX512DQ256-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
; AVX512DQ256-NEXT: br label %[[LOOP:.*]]
; AVX512DQ256: [[LOOP]]:
-; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_N:%.*]], %[[LOOP]] ]
+; AVX512DQ256-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[LOOP]] ]
; AVX512DQ256-NEXT: [[P:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV]]
-; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[P]], align 8
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = srem <4 x i64> [[WIDE_LOAD]], splat (i64 1000000007)
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP1]], ptr [[TMP2]], align 8
+; AVX512DQ256-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[TMP3]], label %[[MIDDLE_BLOCK:.*]], label %[[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
+; AVX512DQ256: [[MIDDLE_BLOCK]]:
+; AVX512DQ256-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; AVX512DQ256-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; AVX512DQ256: [[SCALAR_PH]]:
+; AVX512DQ256-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; AVX512DQ256-NEXT: br label %[[LOOP1:.*]]
+; AVX512DQ256: [[LOOP1]]:
+; AVX512DQ256-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_N:%.*]], %[[LOOP1]] ]
+; AVX512DQ256-NEXT: [[P1:%.*]] = getelementptr i64, ptr [[I]], i64 [[IV1]]
+; AVX512DQ256-NEXT: [[V:%.*]] = load i64, ptr [[P1]], align 8
; AVX512DQ256-NEXT: [[D:%.*]] = srem i64 [[V]], 1000000007
-; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV]]
+; AVX512DQ256-NEXT: [[Q:%.*]] = getelementptr i64, ptr [[O]], i64 [[IV1]]
; AVX512DQ256-NEXT: store i64 [[D]], ptr [[Q]], align 8
-; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV]], 1
+; AVX512DQ256-NEXT: [[IV_N]] = add i64 [[IV1]], 1
; AVX512DQ256-NEXT: [[C:%.*]] = icmp eq i64 [[IV_N]], [[N]]
-; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT:.*]], label %[[LOOP]]
+; AVX512DQ256-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP1]], !llvm.loop [[LOOP12:![0-9]+]]
; AVX512DQ256: [[EXIT]]:
; AVX512DQ256-NEXT: ret void
;
>From ebfcb26cb15d118db83c07cda0580be6d302ee95 Mon Sep 17 00:00:00 2001
From: Rito Takeuchi <licht-t at outlook.jp>
Date: Sun, 12 Jul 2026 15:25:21 +0900
Subject: [PATCH 3/4] [X86][CostModel] Lower vXi64 divide-by-constant costs so
SLP vectorizes
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 16 +-
llvm/test/Analysis/CostModel/X86/div.ll | 80 +-
.../SLPVectorizer/X86/idiv-by-const.ll | 2262 +++++++++++++++++
3 files changed, 2310 insertions(+), 48 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 6493cd5c40c1f..e80fd692dfd42 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -423,9 +423,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
return LT.first * *KindCost;
static const CostKindTblEntry AVX512DQUniformConstCostTable[] = {
- { ISD::SDIV, MVT::v4i64, { 15 } }, // vpmullq-based MULHS sequence
+ { ISD::SDIV, MVT::v4i64, { 9 } }, // vpmullq-based MULHS sequence
{ ISD::SREM, MVT::v4i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
- { ISD::SDIV, MVT::v8i64, { 15 } }, // vpmullq-based MULHS sequence
+ { ISD::SDIV, MVT::v8i64, { 9 } }, // vpmullq-based MULHS sequence
{ ISD::SREM, MVT::v8i64, { 17 } }, // vpmullq-based MULHS+mul+sub sequence
// The remainder's multiply-back is a single vpmullq with DQ, just like the
// pmulld the vXi32 entries above rely on. Without DQ it is another
@@ -469,7 +469,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::UDIV, MVT::v16i32, { 5 } }, // pmuludq sequence
{ ISD::UREM, MVT::v16i32, { 7 } }, // pmuludq+mul+sub sequence
- { ISD::UDIV, MVT::v8i64, { 15 } }, // pmuludq-based MULHU sequence
+ { ISD::UDIV, MVT::v8i64, { 9 } }, // pmuludq-based MULHU sequence
{ ISD::UREM, MVT::v8i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
};
@@ -513,7 +513,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::UDIV, MVT::v8i32, { 5 } }, // pmuludq sequence
{ ISD::UREM, MVT::v8i32, { 7 } }, // pmuludq+mul+sub sequence
- { ISD::UDIV, MVT::v4i64, { 15 } }, // pmuludq-based MULHU sequence
+ { ISD::UDIV, MVT::v4i64, { 9 } }, // pmuludq-based MULHU sequence
{ ISD::UREM, MVT::v4i64, { 21 } }, // pmuludq-based MULHU+mul+sub sequence
};
@@ -616,9 +616,9 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
return LT.first * *KindCost;
static const CostKindTblEntry AVX512DQConstCostTable[] = {
- { ISD::SDIV, MVT::v4i64, { 19 } }, // vpmullq-based MULHS sequence
+ { ISD::SDIV, MVT::v4i64, { 9 } }, // vpmullq-based MULHS sequence
{ ISD::SREM, MVT::v4i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
- { ISD::SDIV, MVT::v8i64, { 19 } }, // vpmullq-based MULHS sequence
+ { ISD::SDIV, MVT::v8i64, { 9 } }, // vpmullq-based MULHS sequence
{ ISD::SREM, MVT::v8i64, { 21 } }, // vpmullq-based MULHS+mul+sub sequence
// The remainder's multiply-back is a single vpmullq with DQ, whereas the
// AVX512/AVX2 tables have to charge for another vpmuludq schoolbook.
@@ -648,7 +648,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::UDIV, MVT::v16i32, { 15 } }, // vpmuludq sequence
{ ISD::UREM, MVT::v16i32, { 17 } }, // vpmuludq+mul+sub sequence
- { ISD::UDIV, MVT::v8i64, { 22 } }, // vpmuludq-based MULHU sequence
+ { ISD::UDIV, MVT::v8i64, { 9 } }, // vpmuludq-based MULHU sequence
{ ISD::UREM, MVT::v8i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
};
@@ -674,7 +674,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
{ ISD::UDIV, MVT::v8i32, { 15 } }, // vpmuludq sequence
{ ISD::UREM, MVT::v8i32, { 19 } }, // vpmuludq+mul+sub sequence
- { ISD::UDIV, MVT::v4i64, { 22 } }, // vpmuludq-based MULHU sequence
+ { ISD::UDIV, MVT::v4i64, { 9 } }, // vpmuludq-based MULHU sequence
{ ISD::UREM, MVT::v4i64, { 28 } }, // vpmuludq-based MULHU+mul+sub sequence
};
diff --git a/llvm/test/Analysis/CostModel/X86/div.ll b/llvm/test/Analysis/CostModel/X86/div.ll
index 78dd39ac292c3..59d232a2101c8 100644
--- a/llvm/test/Analysis/CostModel/X86/div.ll
+++ b/llvm/test/Analysis/CostModel/X86/div.ll
@@ -216,8 +216,8 @@ define i32 @sdiv_const() {
; AVX512DQ-LABEL: 'sdiv_const'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -316,8 +316,8 @@ define i32 @udiv_const() {
; AVX2-LABEL: 'udiv_const'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX2-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX2-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -335,8 +335,8 @@ define i32 @udiv_const() {
; AVX512F-LABEL: 'udiv_const'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -354,8 +354,8 @@ define i32 @udiv_const() {
; AVX512BW-LABEL: 'udiv_const'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -373,8 +373,8 @@ define i32 @udiv_const() {
; AVX512DQ-LABEL: 'udiv_const'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 6, i64 7>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 4, i64 5, i64 6, i64 7>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 4, i64 5, i64 6, i64 7, i64 8, i64 9, i64 10, i64 11>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 4, i32 5, i32 6, i32 7>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
@@ -511,8 +511,8 @@ define i32 @sdiv_uniformconst() {
; AVX512DQ-LABEL: 'sdiv_uniformconst'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 7)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 7)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 7)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 7)
@@ -592,8 +592,8 @@ define i32 @udiv_uniformconst() {
; AVX2-LABEL: 'udiv_uniformconst'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -611,8 +611,8 @@ define i32 @udiv_uniformconst() {
; AVX512F-LABEL: 'udiv_uniformconst'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -630,8 +630,8 @@ define i32 @udiv_uniformconst() {
; AVX512BW-LABEL: 'udiv_uniformconst'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -649,8 +649,8 @@ define i32 @udiv_uniformconst() {
; AVX512DQ-LABEL: 'udiv_uniformconst'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 7)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 7)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 7)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, 7
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 7)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 7)
@@ -1472,8 +1472,8 @@ define i32 @sdiv_constnegpow2() {
; AVX512DQ-LABEL: 'sdiv_constnegpow2'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:19 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1572,8 +1572,8 @@ define i32 @udiv_constnegpow2() {
; AVX2-LABEL: 'udiv_constnegpow2'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX2-NEXT: Cost Model: Found costs of RThru:44 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX2-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1591,8 +1591,8 @@ define i32 @udiv_constnegpow2() {
; AVX512F-LABEL: 'udiv_constnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512F-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1610,8 +1610,8 @@ define i32 @udiv_constnegpow2() {
; AVX512BW-LABEL: 'udiv_constnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1629,8 +1629,8 @@ define i32 @udiv_constnegpow2() {
; AVX512DQ-LABEL: 'udiv_constnegpow2'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, <i64 -8, i64 -16>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:22 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, <i64 -2, i64 -4, i64 -8, i64 -16, i64 -32, i64 -64, i64 -128, i64 -256>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16>
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, <i32 -2, i32 -4, i32 -8, i32 -16, i32 -32, i32 -64, i32 -128, i32 -256>
@@ -1767,8 +1767,8 @@ define i32 @sdiv_uniformconstnegpow2() {
; AVX512DQ-LABEL: 'sdiv_uniformconstnegpow2'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = sdiv i64 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = sdiv <2 x i64> undef, splat (i64 -16)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = sdiv <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = sdiv <8 x i64> undef, splat (i64 -16)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = sdiv i32 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = sdiv <4 x i32> undef, splat (i32 -16)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:6 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = sdiv <8 x i32> undef, splat (i32 -16)
@@ -1848,8 +1848,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX2-LABEL: 'udiv_uniformconstnegpow2'
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX2-NEXT: Cost Model: Found costs of RThru:30 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:18 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX2-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
@@ -1867,8 +1867,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512F-LABEL: 'udiv_uniformconstnegpow2'
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX512F-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512F-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX512F-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
@@ -1886,8 +1886,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512BW-LABEL: 'udiv_uniformconstnegpow2'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX512BW-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX512BW-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
@@ -1905,8 +1905,8 @@ define i32 @udiv_uniformconstnegpow2() {
; AVX512DQ-LABEL: 'udiv_uniformconstnegpow2'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I64 = udiv i64 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:4 SizeLat:4 for: %V2i64 = udiv <2 x i64> undef, splat (i64 -16)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
-; AVX512DQ-NEXT: Cost Model: Found costs of RThru:15 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V4i64 = udiv <4 x i64> undef, splat (i64 -16)
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:9 CodeSize:4 Lat:4 SizeLat:4 for: %V8i64 = udiv <8 x i64> undef, splat (i64 -16)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:4 SizeLat:4 for: %I32 = udiv i32 undef, -16
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V4i32 = udiv <4 x i32> undef, splat (i32 -16)
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:5 CodeSize:4 Lat:4 SizeLat:4 for: %V8i32 = udiv <8 x i32> undef, splat (i32 -16)
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll b/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
new file mode 100644
index 0000000000000..fb24b0ba3dce6
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
@@ -0,0 +1,2262 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 6
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=AVX512DQ
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=icelake-server | FileCheck %s --check-prefix=AVX512DQ256
+
+; Straight-line division/remainder of consecutive elements by a constant, as
+; produced by fully unrolling a small trip-count loop. These should vectorize
+; exactly where the sequence is formed: MULHU (unsigned) needs AVX2, MULHS
+; (signed) additionally needs AVX512DQ's vpmullq. Below AVX2, and for the
+; signed cases without DQ, the multiply scalarizes and the loops stay scalar.
+
+define void @v4i64_udiv(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_udiv(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = udiv i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = udiv i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = udiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = udiv i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_udiv(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_udiv(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512F-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_udiv(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_udiv(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = udiv i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = udiv i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = udiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = udiv i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v4i64_udiv_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_udiv_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = udiv i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = udiv i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = udiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = udiv i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_udiv_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_udiv_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512F-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_udiv_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_udiv_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = udiv i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = udiv i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = udiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = udiv i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v8i64_udiv(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_udiv(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = udiv i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = udiv i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = udiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = udiv i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = udiv i64 [[V4]], 7
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = udiv i64 [[V5]], 7
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = udiv i64 [[V6]], 7
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = udiv i64 [[V7]], 7
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_udiv(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX2-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[TMP3]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_udiv(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = udiv <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512F-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_udiv(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = udiv <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_udiv(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[TMP3]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = udiv i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = udiv i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = udiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = udiv i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = udiv i64 %v4, 7
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = udiv i64 %v5, 7
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = udiv i64 %v6, 7
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = udiv i64 %v7, 7
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v8i64_udiv_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_udiv_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = udiv i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = udiv i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = udiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = udiv i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = udiv i64 [[V4]], 13
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = udiv i64 [[V5]], 17
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = udiv i64 [[V6]], 19
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = udiv i64 [[V7]], 23
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_udiv_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX2-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[TMP3]], <i64 13, i64 17, i64 19, i64 23>
+; AVX2-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_udiv_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = udiv <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512F-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_udiv_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = udiv <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_udiv_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = udiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = udiv <4 x i64> [[TMP3]], <i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = udiv i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = udiv i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = udiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = udiv i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = udiv i64 %v4, 13
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = udiv i64 %v5, 17
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = udiv i64 %v6, 19
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = udiv i64 %v7, 23
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v4i64_sdiv(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_sdiv(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_sdiv(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_sdiv(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_sdiv(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_sdiv(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = sdiv i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = sdiv i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = sdiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = sdiv i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v4i64_sdiv_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_sdiv_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_sdiv_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_sdiv_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_sdiv_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_sdiv_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = sdiv i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = sdiv i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = sdiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = sdiv i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v8i64_sdiv(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_sdiv(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 7
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 7
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 7
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 7
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_sdiv(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX2-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 7
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX2-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX2-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX2-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 7
+; AVX2-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX2-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX2-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX2-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX2-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 7
+; AVX2-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX2-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX2-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX2-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX2-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 7
+; AVX2-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX2-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_sdiv(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 7
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 7
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 7
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512F-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX512F-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 7
+; AVX512F-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512F-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX512F-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX512F-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX512F-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 7
+; AVX512F-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX512F-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX512F-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX512F-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX512F-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 7
+; AVX512F-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX512F-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX512F-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX512F-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX512F-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 7
+; AVX512F-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX512F-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_sdiv(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = sdiv <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_sdiv(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = sdiv <4 x i64> [[TMP3]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = sdiv i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = sdiv i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = sdiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = sdiv i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = sdiv i64 %v4, 7
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = sdiv i64 %v5, 7
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = sdiv i64 %v6, 7
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = sdiv i64 %v7, 7
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v8i64_sdiv_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_sdiv_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 13
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 17
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 19
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 23
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_sdiv_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX2-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 13
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX2-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX2-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX2-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 17
+; AVX2-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX2-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX2-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX2-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX2-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 19
+; AVX2-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX2-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX2-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX2-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX2-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 23
+; AVX2-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX2-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_sdiv_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = sdiv i64 [[V0]], 3
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = sdiv i64 [[V1]], 5
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = sdiv i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = sdiv i64 [[V3]], 11
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512F-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX512F-NEXT: [[D4:%.*]] = sdiv i64 [[V4]], 13
+; AVX512F-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512F-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX512F-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX512F-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX512F-NEXT: [[D5:%.*]] = sdiv i64 [[V5]], 17
+; AVX512F-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX512F-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX512F-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX512F-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX512F-NEXT: [[D6:%.*]] = sdiv i64 [[V6]], 19
+; AVX512F-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX512F-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX512F-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX512F-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX512F-NEXT: [[D7:%.*]] = sdiv i64 [[V7]], 23
+; AVX512F-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX512F-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_sdiv_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = sdiv <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_sdiv_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = sdiv <4 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = sdiv <4 x i64> [[TMP3]], <i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = sdiv i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = sdiv i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = sdiv i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = sdiv i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = sdiv i64 %v4, 13
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = sdiv i64 %v5, 17
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = sdiv i64 %v6, 19
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = sdiv i64 %v7, 23
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v4i64_urem(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_urem(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = urem i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = urem i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = urem i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_urem(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = urem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_urem(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = urem i64 [[V0]], 7
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = urem i64 [[V1]], 7
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = urem i64 [[V3]], 7
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_urem(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = urem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_urem(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = urem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = urem i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = urem i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = urem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = urem i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v4i64_urem_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_urem_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_urem_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_urem_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_urem_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512DQ-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512DQ-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; AVX512DQ-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512DQ-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512DQ-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512DQ-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512DQ-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX512DQ-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512DQ-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512DQ-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512DQ-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512DQ-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; AVX512DQ-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512DQ-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_urem_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512DQ256-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512DQ256-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; AVX512DQ256-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512DQ256-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512DQ256-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512DQ256-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512DQ256-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX512DQ256-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512DQ256-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512DQ256-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512DQ256-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512DQ256-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; AVX512DQ256-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512DQ256-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = urem i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = urem i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = urem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = urem i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v8i64_urem(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_urem(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = urem i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = urem i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = urem i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = urem i64 [[V4]], 7
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = urem i64 [[V5]], 7
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = urem i64 [[V6]], 7
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = urem i64 [[V7]], 7
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_urem(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX2-NEXT: [[TMP2:%.*]] = urem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX2-NEXT: [[TMP4:%.*]] = urem <4 x i64> [[TMP3]], splat (i64 7)
+; AVX2-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_urem(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512F-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_urem(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_urem(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = urem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = urem <4 x i64> [[TMP3]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = urem i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = urem i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = urem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = urem i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = urem i64 %v4, 7
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = urem i64 %v5, 7
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = urem i64 %v6, 7
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = urem i64 %v7, 7
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v8i64_urem_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_urem_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = urem i64 [[V4]], 13
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = urem i64 [[V5]], 17
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = urem i64 [[V6]], 19
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = urem i64 [[V7]], 23
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_urem_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = urem i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = urem i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = urem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = urem i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX2-NEXT: [[D4:%.*]] = urem i64 [[V4]], 13
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX2-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX2-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX2-NEXT: [[D5:%.*]] = urem i64 [[V5]], 17
+; AVX2-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX2-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX2-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX2-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX2-NEXT: [[D6:%.*]] = urem i64 [[V6]], 19
+; AVX2-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX2-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX2-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX2-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX2-NEXT: [[D7:%.*]] = urem i64 [[V7]], 23
+; AVX2-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX2-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_urem_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512F-NEXT: [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512F-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_urem_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_urem_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ256-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = urem i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = urem i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = urem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = urem i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = urem i64 %v4, 13
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = urem i64 %v5, 17
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = urem i64 %v6, 19
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = urem i64 %v7, 23
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v4i64_srem(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_srem(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_srem(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_srem(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_srem(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = srem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_srem(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = srem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = srem i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = srem i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = srem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = srem i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v4i64_srem_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v4i64_srem_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v4i64_srem_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v4i64_srem_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v4i64_srem_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512DQ-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512DQ-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX512DQ-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512DQ-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512DQ-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512DQ-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512DQ-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512DQ-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512DQ-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512DQ-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512DQ-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512DQ-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX512DQ-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512DQ-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v4i64_srem_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512DQ256-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512DQ256-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX512DQ256-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512DQ256-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512DQ256-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512DQ256-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512DQ256-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512DQ256-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512DQ256-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512DQ256-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512DQ256-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512DQ256-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX512DQ256-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512DQ256-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = srem i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = srem i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = srem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = srem i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ ret void
+}
+
+define void @v8i64_srem(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_srem(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = srem i64 [[V4]], 7
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = srem i64 [[V5]], 7
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = srem i64 [[V6]], 7
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = srem i64 [[V7]], 7
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_srem(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX2-NEXT: [[D4:%.*]] = srem i64 [[V4]], 7
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX2-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX2-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX2-NEXT: [[D5:%.*]] = srem i64 [[V5]], 7
+; AVX2-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX2-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX2-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX2-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX2-NEXT: [[D6:%.*]] = srem i64 [[V6]], 7
+; AVX2-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX2-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX2-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX2-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX2-NEXT: [[D7:%.*]] = srem i64 [[V7]], 7
+; AVX2-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX2-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_srem(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = srem i64 [[V0]], 7
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = srem i64 [[V1]], 7
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = srem i64 [[V3]], 7
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512F-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX512F-NEXT: [[D4:%.*]] = srem i64 [[V4]], 7
+; AVX512F-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512F-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX512F-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX512F-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX512F-NEXT: [[D5:%.*]] = srem i64 [[V5]], 7
+; AVX512F-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX512F-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX512F-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX512F-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX512F-NEXT: [[D6:%.*]] = srem i64 [[V6]], 7
+; AVX512F-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX512F-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX512F-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX512F-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX512F-NEXT: [[D7:%.*]] = srem i64 [[V7]], 7
+; AVX512F-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX512F-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_srem(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = srem <8 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_srem(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = srem <4 x i64> [[TMP1]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512DQ256-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512DQ256-NEXT: [[TMP3:%.*]] = load <4 x i64>, ptr [[X4]], align 8
+; AVX512DQ256-NEXT: [[TMP4:%.*]] = srem <4 x i64> [[TMP3]], splat (i64 7)
+; AVX512DQ256-NEXT: store <4 x i64> [[TMP4]], ptr [[O4]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = srem i64 %v0, 7
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = srem i64 %v1, 7
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = srem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = srem i64 %v3, 7
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = srem i64 %v4, 7
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = srem i64 %v5, 7
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = srem i64 %v6, 7
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = srem i64 %v7, 7
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
+
+define void @v8i64_srem_nonuniform(ptr noalias %x, ptr noalias %dst) {
+; SSE-LABEL: define void @v8i64_srem_nonuniform(
+; SSE-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; SSE-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; SSE-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; SSE-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; SSE-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; SSE-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; SSE-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; SSE-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; SSE-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; SSE-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; SSE-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; SSE-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; SSE-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; SSE-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; SSE-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; SSE-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; SSE-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; SSE-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; SSE-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; SSE-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; SSE-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; SSE-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; SSE-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; SSE-NEXT: [[D4:%.*]] = srem i64 [[V4]], 13
+; SSE-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; SSE-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; SSE-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; SSE-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; SSE-NEXT: [[D5:%.*]] = srem i64 [[V5]], 17
+; SSE-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; SSE-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; SSE-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; SSE-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; SSE-NEXT: [[D6:%.*]] = srem i64 [[V6]], 19
+; SSE-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; SSE-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; SSE-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; SSE-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; SSE-NEXT: [[D7:%.*]] = srem i64 [[V7]], 23
+; SSE-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; SSE-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; SSE-NEXT: ret void
+;
+; AVX2-LABEL: define void @v8i64_srem_nonuniform(
+; AVX2-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX2-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX2-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX2-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX2-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX2-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX2-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX2-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX2-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX2-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX2-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX2-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX2-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX2-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX2-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX2-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX2-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX2-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX2-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX2-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX2-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX2-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX2-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX2-NEXT: [[D4:%.*]] = srem i64 [[V4]], 13
+; AVX2-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX2-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX2-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX2-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX2-NEXT: [[D5:%.*]] = srem i64 [[V5]], 17
+; AVX2-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX2-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX2-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX2-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX2-NEXT: [[D6:%.*]] = srem i64 [[V6]], 19
+; AVX2-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX2-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX2-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX2-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX2-NEXT: [[D7:%.*]] = srem i64 [[V7]], 23
+; AVX2-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX2-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX2-NEXT: ret void
+;
+; AVX512F-LABEL: define void @v8i64_srem_nonuniform(
+; AVX512F-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512F-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512F-NEXT: [[V0:%.*]] = load i64, ptr [[X0]], align 8
+; AVX512F-NEXT: [[D0:%.*]] = srem i64 [[V0]], 3
+; AVX512F-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512F-NEXT: store i64 [[D0]], ptr [[O0]], align 8
+; AVX512F-NEXT: [[X1:%.*]] = getelementptr i64, ptr [[X]], i64 1
+; AVX512F-NEXT: [[V1:%.*]] = load i64, ptr [[X1]], align 8
+; AVX512F-NEXT: [[D1:%.*]] = srem i64 [[V1]], 5
+; AVX512F-NEXT: [[O1:%.*]] = getelementptr i64, ptr [[DST]], i64 1
+; AVX512F-NEXT: store i64 [[D1]], ptr [[O1]], align 8
+; AVX512F-NEXT: [[X2:%.*]] = getelementptr i64, ptr [[X]], i64 2
+; AVX512F-NEXT: [[V2:%.*]] = load i64, ptr [[X2]], align 8
+; AVX512F-NEXT: [[D2:%.*]] = srem i64 [[V2]], 7
+; AVX512F-NEXT: [[O2:%.*]] = getelementptr i64, ptr [[DST]], i64 2
+; AVX512F-NEXT: store i64 [[D2]], ptr [[O2]], align 8
+; AVX512F-NEXT: [[X3:%.*]] = getelementptr i64, ptr [[X]], i64 3
+; AVX512F-NEXT: [[V3:%.*]] = load i64, ptr [[X3]], align 8
+; AVX512F-NEXT: [[D3:%.*]] = srem i64 [[V3]], 11
+; AVX512F-NEXT: [[O3:%.*]] = getelementptr i64, ptr [[DST]], i64 3
+; AVX512F-NEXT: store i64 [[D3]], ptr [[O3]], align 8
+; AVX512F-NEXT: [[X4:%.*]] = getelementptr i64, ptr [[X]], i64 4
+; AVX512F-NEXT: [[V4:%.*]] = load i64, ptr [[X4]], align 8
+; AVX512F-NEXT: [[D4:%.*]] = srem i64 [[V4]], 13
+; AVX512F-NEXT: [[O4:%.*]] = getelementptr i64, ptr [[DST]], i64 4
+; AVX512F-NEXT: store i64 [[D4]], ptr [[O4]], align 8
+; AVX512F-NEXT: [[X5:%.*]] = getelementptr i64, ptr [[X]], i64 5
+; AVX512F-NEXT: [[V5:%.*]] = load i64, ptr [[X5]], align 8
+; AVX512F-NEXT: [[D5:%.*]] = srem i64 [[V5]], 17
+; AVX512F-NEXT: [[O5:%.*]] = getelementptr i64, ptr [[DST]], i64 5
+; AVX512F-NEXT: store i64 [[D5]], ptr [[O5]], align 8
+; AVX512F-NEXT: [[X6:%.*]] = getelementptr i64, ptr [[X]], i64 6
+; AVX512F-NEXT: [[V6:%.*]] = load i64, ptr [[X6]], align 8
+; AVX512F-NEXT: [[D6:%.*]] = srem i64 [[V6]], 19
+; AVX512F-NEXT: [[O6:%.*]] = getelementptr i64, ptr [[DST]], i64 6
+; AVX512F-NEXT: store i64 [[D6]], ptr [[O6]], align 8
+; AVX512F-NEXT: [[X7:%.*]] = getelementptr i64, ptr [[X]], i64 7
+; AVX512F-NEXT: [[V7:%.*]] = load i64, ptr [[X7]], align 8
+; AVX512F-NEXT: [[D7:%.*]] = srem i64 [[V7]], 23
+; AVX512F-NEXT: [[O7:%.*]] = getelementptr i64, ptr [[DST]], i64 7
+; AVX512F-NEXT: store i64 [[D7]], ptr [[O7]], align 8
+; AVX512F-NEXT: ret void
+;
+; AVX512DQ-LABEL: define void @v8i64_srem_nonuniform(
+; AVX512DQ-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ-NEXT: [[TMP2:%.*]] = srem <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ-NEXT: ret void
+;
+; AVX512DQ256-LABEL: define void @v8i64_srem_nonuniform(
+; AVX512DQ256-SAME: ptr noalias [[X:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0]] {
+; AVX512DQ256-NEXT: [[X0:%.*]] = getelementptr i64, ptr [[X]], i64 0
+; AVX512DQ256-NEXT: [[O0:%.*]] = getelementptr i64, ptr [[DST]], i64 0
+; AVX512DQ256-NEXT: [[TMP1:%.*]] = load <8 x i64>, ptr [[X0]], align 8
+; AVX512DQ256-NEXT: [[TMP2:%.*]] = srem <8 x i64> [[TMP1]], <i64 3, i64 5, i64 7, i64 11, i64 13, i64 17, i64 19, i64 23>
+; AVX512DQ256-NEXT: store <8 x i64> [[TMP2]], ptr [[O0]], align 8
+; AVX512DQ256-NEXT: ret void
+;
+ %x0 = getelementptr i64, ptr %x, i64 0
+ %v0 = load i64, ptr %x0
+ %d0 = srem i64 %v0, 3
+ %o0 = getelementptr i64, ptr %dst, i64 0
+ store i64 %d0, ptr %o0
+ %x1 = getelementptr i64, ptr %x, i64 1
+ %v1 = load i64, ptr %x1
+ %d1 = srem i64 %v1, 5
+ %o1 = getelementptr i64, ptr %dst, i64 1
+ store i64 %d1, ptr %o1
+ %x2 = getelementptr i64, ptr %x, i64 2
+ %v2 = load i64, ptr %x2
+ %d2 = srem i64 %v2, 7
+ %o2 = getelementptr i64, ptr %dst, i64 2
+ store i64 %d2, ptr %o2
+ %x3 = getelementptr i64, ptr %x, i64 3
+ %v3 = load i64, ptr %x3
+ %d3 = srem i64 %v3, 11
+ %o3 = getelementptr i64, ptr %dst, i64 3
+ store i64 %d3, ptr %o3
+ %x4 = getelementptr i64, ptr %x, i64 4
+ %v4 = load i64, ptr %x4
+ %d4 = srem i64 %v4, 13
+ %o4 = getelementptr i64, ptr %dst, i64 4
+ store i64 %d4, ptr %o4
+ %x5 = getelementptr i64, ptr %x, i64 5
+ %v5 = load i64, ptr %x5
+ %d5 = srem i64 %v5, 17
+ %o5 = getelementptr i64, ptr %dst, i64 5
+ store i64 %d5, ptr %o5
+ %x6 = getelementptr i64, ptr %x, i64 6
+ %v6 = load i64, ptr %x6
+ %d6 = srem i64 %v6, 19
+ %o6 = getelementptr i64, ptr %dst, i64 6
+ store i64 %d6, ptr %o6
+ %x7 = getelementptr i64, ptr %x, i64 7
+ %v7 = load i64, ptr %x7
+ %d7 = srem i64 %v7, 23
+ %o7 = getelementptr i64, ptr %dst, i64 7
+ store i64 %d7, ptr %o7
+ ret void
+}
>From c35c28e2874cebbf477655125de59661f4b5982d Mon Sep 17 00:00:00 2001
From: Rito Takeuchi <licht-t at outlook.jp>
Date: Mon, 13 Jul 2026 22:17:21 +0900
Subject: [PATCH 4/4] [X86] Use explicit -mattr for the prefer-256-bit RUN line
in idiv-by-const tests
---
llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll | 2 +-
llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
index a854b66294fc5..f6489248b3166 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/idiv-by-const.ll
@@ -4,7 +4,7 @@
; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mcpu=icelake-server | FileCheck %s --check-prefix=AVX512DQ256
+; RUN: opt -passes=loop-vectorize -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq,+avx512vl,+prefer-256-bit | FileCheck %s --check-prefix=AVX512DQ256
; Division of i64 by a loop-invariant constant becomes a magic multiply-high
; sequence, so these loops should vectorize exactly where that sequence is
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll b/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
index fb24b0ba3dce6..08a33dcd119c7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/idiv-by-const.ll
@@ -4,7 +4,7 @@
; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefix=AVX2
; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mattr=+avx512f | FileCheck %s --check-prefix=AVX512F
; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefix=AVX512DQ
-; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mcpu=icelake-server | FileCheck %s --check-prefix=AVX512DQ256
+; RUN: opt -passes=slp-vectorizer -S < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512dq,+avx512vl,+prefer-256-bit | FileCheck %s --check-prefix=AVX512DQ256
; Straight-line division/remainder of consecutive elements by a constant, as
; produced by fully unrolling a small trip-count loop. These should vectorize
More information about the llvm-commits
mailing list