[llvm] [CostModel][X86] Improve vXi1 mask broadcast cost handling (PR #222467)
Dmitry Sidorov via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 04:50:46 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/222467
>From 45446a5bad2f14e71d0f0d6ecbbc4a6869b047de Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 21 Sep 2026 13:07:10 +0200
Subject: [PATCH 1/2] [NFC][CostModel][X86] Add cost tests for vXi1 mask
broadcasts
---
.../CostModel/X86/shuffle-broadcast.ll | 78 +++++++-
.../CostModel/X86/shuffle-mask-broadcast.ll | 75 +++++++
.../Analysis/CostModel/X86/shuffle-splat.ll | 186 +++++++++++++++++-
3 files changed, 333 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
index 7bfab69063d55..6bef388ead5fb 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
@@ -4,9 +4,10 @@
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx | FileCheck %s -check-prefixes=AVX1
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512DQ
;
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
@@ -464,3 +465,74 @@ define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a
%V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
ret void
}
+
+define void @test_load_vXi1(ptr %p16, ptr %p64) {
+; SSE2-LABEL: 'test_load_vXi1'
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; SSE2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSSE3-LABEL: 'test_load_vXi1'
+; SSSE3-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; SSSE3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'test_load_vXi1'
+; SSE42-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'test_load_vXi1'
+; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'test_load_vXi1'
+; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX2-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX2-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512F-LABEL: 'test_load_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_load_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_load_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX512VBMI-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX512VBMI-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_load_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
+; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
+; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %src16 = load <16 x i1>, ptr %p16
+ %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+ %src64 = load <64 x i1>, ptr %p64
+ %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll b/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
new file mode 100644
index 0000000000000..8b33d75e662ae
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vl,+fast-variable-crosslane-shuffle | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vl,+fast-variable-crosslane-shuffle | FileCheck %s --check-prefixes=CHECK,X64-NODQ
+; RUN: opt < %s -mtriple=i386-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vl,+fast-variable-crosslane-shuffle | FileCheck %s --check-prefixes=CHECK,X86
+
+define <16 x i1> @lane16_bw_0(<16 x i1> %src) #0 {
+; CHECK-LABEL: 'lane16_bw_0'
+; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+;
+ %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 0)
+ ret <16 x i1> %s
+}
+
+define <16 x i1> @lane16_bw_1(<16 x i1> %src) #0 {
+; CHECK-LABEL: 'lane16_bw_1'
+; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+;
+ %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 1)
+ ret <16 x i1> %s
+}
+
+define <16 x i1> @lane16_dword_0(<16 x i1> %src) #1 {
+; CHECK-LABEL: 'lane16_dword_0'
+; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+;
+ %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 0)
+ ret <16 x i1> %s
+}
+
+define <64 x i1> @lane64_no512_0(<64 x i1> %src) #2 {
+; CHECK-LABEL: 'lane64_no512_0'
+; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+ %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 0)
+ ret <64 x i1> %s
+}
+
+define <64 x i1> @lane64_no512_17(<64 x i1> %src) #2 {
+; CHECK-LABEL: 'lane64_no512_17'
+; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+ %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 17)
+ ret <64 x i1> %s
+}
+
+define <64 x i1> @lane64_512_0(<64 x i1> %src) #0 {
+; CHECK-LABEL: 'lane64_512_0'
+; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+ %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 0)
+ ret <64 x i1> %s
+}
+
+define <64 x i1> @lane64_512_17(<64 x i1> %src) #0 {
+; CHECK-LABEL: 'lane64_512_17'
+; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+ %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 17)
+ ret <64 x i1> %s
+}
+
+attributes #0 = { "prefer-vector-width"="256" "min-legal-vector-width"="512" }
+attributes #1 = { "prefer-vector-width"="512" "min-legal-vector-width"="512" }
+attributes #2 = { "prefer-vector-width"="256" "min-legal-vector-width"="0" }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; X64: {{.*}}
+; X64-NODQ: {{.*}}
+; X86: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
index b3cc220c0f3c3..70498a3ee0d8f 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
@@ -4,9 +4,12 @@
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+sse4.2 | FileCheck %s -check-prefixes=SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx | FileCheck %s -check-prefixes=AVX1
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx2 | FileCheck %s -check-prefixes=AVX2
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512
-; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512,AVX512VBMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512DQ
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512dq,+avx512vl,+fast-variable-crosslane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512DQ-FAST
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mattr=+avx512f,+avx512bw,+avx512vbmi,+fast-variable-crosslane-shuffle | FileCheck %s --check-prefixes=AVX512,AVX512VBMI-FAST
;
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=slm | FileCheck %s --check-prefixes=SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=all -mcpu=goldmont | FileCheck %s --check-prefixes=SSE,SSE42
@@ -425,6 +428,126 @@ define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1>
ret void
}
+;
+; Verify the cost model for splat shuffles of masks wider than one legal mask
+; register. The first lane of every part is a plain splat, any other lane is
+; not.
+;
+
+define void @test_multireg_vXi1(<128 x i1> %src128) {
+; SSE2-LABEL: 'test_multireg_vXi1'
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSSE3-LABEL: 'test_multireg_vXi1'
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'test_multireg_vXi1'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'test_multireg_vXi1'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'test_multireg_vXi1'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512-LABEL: 'test_multireg_vXi1'
+; AVX512-NEXT: Cost Model: Found costs of 129 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512-NEXT: Cost Model: Found costs of 129 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512-NEXT: Cost Model: Found costs of 129 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+ %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> splat (i32 64)
+ %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> splat (i32 65)
+ ret void
+}
+
+define void @test_sublane_vXi1(<32 x i1> %src32, <64 x i1> %src64) {
+; SSE2-LABEL: 'test_sublane_vXi1'
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSSE3-LABEL: 'test_sublane_vXi1'
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'test_sublane_vXi1'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'test_sublane_vXi1'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'test_sublane_vXi1'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512-LABEL: 'test_sublane_vXi1'
+; AVX512-NEXT: Cost Model: Found costs of 33 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512-NEXT: Cost Model: Found costs of 33 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512-NEXT: Cost Model: Found costs of 33 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> splat (i32 1)
+ %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> splat (i32 16)
+ %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> splat (i32 17)
+ %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> splat (i32 1)
+ %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> splat (i32 16)
+ %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> splat (i32 17)
+ %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> splat (i32 49)
+ ret void
+}
+
;
; Tests the cost model for splat shuffles of second operand.
;
@@ -464,3 +587,60 @@ define void @test_upper_vXf32(<2 x float> %a64, <2 x float> %b64, <4 x float> %a
%V512 = shufflevector <16 x float> %a512, <16 x float> %b512, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
ret void
}
+
+define void @test_upper_vXi1(<64 x i1> %a64, <64 x i1> %b64) {
+; SSE2-LABEL: 'test_upper_vXi1'
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:4 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSSE3-LABEL: 'test_upper_vXi1'
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; SSSE3-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; SSE42-LABEL: 'test_upper_vXi1'
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; SSE42-NEXT: Cost Model: Found costs of RThru:1 CodeSize:2 Lat:3 SizeLat:2 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX1-LABEL: 'test_upper_vXi1'
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:3 Lat:4 SizeLat:6 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX2-LABEL: 'test_upper_vXi1'
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512-LABEL: 'test_upper_vXi1'
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512-NEXT: Cost Model: Found costs of 65 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+ %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 17)
+ %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 64)
+ %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 80)
+ %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 81)
+ ret void
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX512BW: {{.*}}
+; AVX512DQ: {{.*}}
+; AVX512DQ-FAST: {{.*}}
+; AVX512F: {{.*}}
+; AVX512VBMI: {{.*}}
+; AVX512VBMI-FAST: {{.*}}
>From 665d55559f9c634466de76b1560361eac42d223b Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Mon, 21 Sep 2026 13:10:18 +0200
Subject: [PATCH 2/2] [CostModel][X86] Improve vXi1 mask broadcast cost
handling
The AVX512 shuffle table has no vXi1 entries, so a mask splat falls back to
scalarization and is priced at N+1, up to 65 for v64i1, while AVX2 prices the
same shuffle at 1.
A scalar splat moves the value across in a gpr. A mask lane splat sign extends
the mask to a vector, splats it there and converts it back, so it is priced by
lane and by subtarget. The index names a lane of the whole shuffle, so it is
folded onto the first operand and then onto the legalized register holding it.
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 71 ++++++
.../CostModel/X86/shuffle-broadcast.ll | 59 +++--
.../CostModel/X86/shuffle-mask-broadcast.ll | 54 +++--
.../Analysis/CostModel/X86/shuffle-splat.ll | 223 +++++++++++++++---
4 files changed, 339 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e41870a4ecbfa..ba31e1c84efc4 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1989,6 +1989,77 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
// registers are the same.
LT.first = 1;
+ if (LT.second.isVectorOf(MVT::i1)) {
+ // A scalar splat moves the value across in a gpr.
+ if (!Args.empty() && !Args.front()->getType()->isVectorTy()) {
+ static const CostKindCosts ScalarSplat = {1, 2, 2, 2};
+ static const CostKindCosts ScalarSplatUnpck = {2, 6, 3, 3};
+ const CostKindCosts &Costs = LT.second == MVT::v64i1 && !ST->is64Bit()
+ ? ScalarSplatUnpck
+ : ScalarSplat;
+ if (auto KindCost = Costs[CostKind])
+ return *KindCost;
+ }
+
+ // A mask lane splat sign extends the mask to a vector, splats it there
+ // and converts it back, so it is priced by lane and by subtarget.
+ static const CostKindCosts SmallLaneSplat = {1, 3, 3, 3};
+ static const CostKindCosts LaneSplat = {1, 5, 3, 3};
+ static const CostKindCosts LaneSplatVarPerm = {1, 5, 4, 4};
+ static const CostKindCosts LaneSplatPerm = {2, 8, 4, 5};
+ static const CostKindCosts LaneSplatCross = {3, 9, 5, 6};
+ static const CostKindCosts LaneSplatCrossPerm = {2, 6, 5, 7};
+
+ // The index names a lane of the whole shuffle, so fold it onto the
+ // first operand and then onto the legalized register holding it.
+ int NumSrcElts = SrcTy->getElementCount().getKnownMinValue();
+ int NumRegElts = LT.second.getVectorNumElements();
+ int Lane = Index < 0 ? Index : (Index % NumSrcElts) % NumRegElts;
+
+ if (LT.second == MVT::v64i1 && !ST->useBWIRegs()) {
+ // A legal mask register does not imply that we can use a 512-bit
+ // vector for the round trip. In this case lower1BitShuffle falls back
+ // to extracting the bit and splatting it through a GPR.
+ static const CostKindCosts ExtractAndSplat = {1, 7, 5, 5};
+ static const CostKindCosts ExtractAndSplatUnpck = {2, 10, 5, 5};
+ static const CostKindCosts ShiftMask = {1, 4, 1, 1};
+ const CostKindCosts &Costs =
+ ST->is64Bit() ? ExtractAndSplat : ExtractAndSplatUnpck;
+ InstructionCost Cost = *Costs[CostKind];
+ if (Lane != 0)
+ Cost += *ShiftMask[CostKind];
+ return Cost;
+ }
+
+ bool IsCrossSublane =
+ NumRegElts >= 32 && !ST->hasVBMI() && Lane >= 16 && Lane % 16 != 0;
+ bool IsVarPerm = ST->hasFastVariableCrossLaneShuffle() &&
+ (NumRegElts == 8 || (NumRegElts >= 32 && ST->hasVBMI()));
+ const CostKindCosts &CrossCosts = ST->hasFastVariableCrossLaneShuffle()
+ ? LaneSplatCrossPerm
+ : LaneSplatCross;
+ const CostKindCosts &Costs = NumRegElts <= 4 ? SmallLaneSplat
+ : Lane == 0 ? LaneSplat
+ : IsCrossSublane ? CrossCosts
+ : IsVarPerm ? LaneSplatVarPerm
+ : LaneSplatPerm;
+ if (auto KindCost = Costs[CostKind]) {
+ InstructionCost Cost = *KindCost;
+ // When avoiding 512-bit widening, lower1BitShuffle extends v16i1 to
+ // v16i16 instead. With BW, its vpmovm2w / vpmovw2m round trip does not
+ // require DQ.
+ bool UsesBWConversions =
+ NumRegElts == 16 && ST->hasBWI() && !ST->canExtendTo512DQ();
+ if (!ST->hasDQI() && NumRegElts <= 16 && !UsesBWConversions) {
+ if (CostKind == TTI::TCK_RecipThroughput)
+ Cost += 1;
+ else if (CostKind == TTI::TCK_Latency)
+ Cost += 3;
+ }
+ return Cost;
+ }
+ }
+
// If we're broadcasting a load then AVX/AVX2 can do this for free.
// If many-used-load whose every use is one of a small set of operations
// that SLP can rewrite into a single vector lane, codegen can fold it into
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
index 6bef388ead5fb..a31780f5c76c9 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-broadcast.ll
@@ -408,14 +408,41 @@ define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; AVX512-LABEL: 'test_vXi1'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 5 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 9 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 17 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 33 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX512F-LABEL: 'test_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> zeroinitializer
%V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> zeroinitializer
@@ -504,30 +531,30 @@ define void @test_load_vXi1(ptr %p16, ptr %p64) {
;
; AVX512F-LABEL: 'test_load_vXi1'
; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
-; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
; AVX512F-NEXT: Cost Model: Found costs of RThru:8 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
-; AVX512F-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX512BW-LABEL: 'test_load_vXi1'
; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
-; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
-; AVX512BW-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX512VBMI-LABEL: 'test_load_vXi1'
; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
-; AVX512VBMI-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
-; AVX512VBMI-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
; AVX512DQ-LABEL: 'test_load_vXi1'
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:4 SizeLat:1 for: %src16 = load <16 x i1>, ptr %p16, align 2
-; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> zeroinitializer
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:4 SizeLat:1 for: %src64 = load <64 x i1>, ptr %p64, align 8
-; AVX512DQ-NEXT: Cost Model: Found costs of 0 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> zeroinitializer
; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%src16 = load <16 x i1>, ptr %p16
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll b/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
index 8b33d75e662ae..46024ebc677ff 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-mask-broadcast.ll
@@ -5,7 +5,7 @@
define <16 x i1> @lane16_bw_0(<16 x i1> %src) #0 {
; CHECK-LABEL: 'lane16_bw_0'
-; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
;
%s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 0)
@@ -14,7 +14,7 @@ define <16 x i1> @lane16_bw_0(<16 x i1> %src) #0 {
define <16 x i1> @lane16_bw_1(<16 x i1> %src) #0 {
; CHECK-LABEL: 'lane16_bw_1'
-; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
;
%s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 1)
@@ -22,27 +22,51 @@ define <16 x i1> @lane16_bw_1(<16 x i1> %src) #0 {
}
define <16 x i1> @lane16_dword_0(<16 x i1> %src) #1 {
-; CHECK-LABEL: 'lane16_dword_0'
-; CHECK-NEXT: Cost Model: Found costs of 17 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+; X64-LABEL: 'lane16_dword_0'
+; X64-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; X64-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+;
+; X64-NODQ-LABEL: 'lane16_dword_0'
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
+;
+; X86-LABEL: 'lane16_dword_0'
+; X86-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> zeroinitializer
+; X86-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x i1> %s
;
%s = shufflevector <16 x i1> %src, <16 x i1> poison, <16 x i32> splat (i32 0)
ret <16 x i1> %s
}
define <64 x i1> @lane64_no512_0(<64 x i1> %src) #2 {
-; CHECK-LABEL: 'lane64_no512_0'
-; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+; X64-LABEL: 'lane64_no512_0'
+; X64-NEXT: Cost Model: Found costs of RThru:1 CodeSize:5 Lat:7 SizeLat:5 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; X64-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+; X64-NODQ-LABEL: 'lane64_no512_0'
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:5 Lat:7 SizeLat:5 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+; X86-LABEL: 'lane64_no512_0'
+; X86-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:10 SizeLat:5 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; X86-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
;
%s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 0)
ret <64 x i1> %s
}
define <64 x i1> @lane64_no512_17(<64 x i1> %src) #2 {
-; CHECK-LABEL: 'lane64_no512_17'
-; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
-; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+; X64-LABEL: 'lane64_no512_17'
+; X64-NEXT: Cost Model: Found costs of RThru:2 CodeSize:6 Lat:11 SizeLat:6 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; X64-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+; X64-NODQ-LABEL: 'lane64_no512_17'
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:6 Lat:11 SizeLat:6 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; X64-NODQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
+;
+; X86-LABEL: 'lane64_no512_17'
+; X86-NEXT: Cost Model: Found costs of RThru:3 CodeSize:6 Lat:14 SizeLat:6 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; X86-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
;
%s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 17)
ret <64 x i1> %s
@@ -50,7 +74,7 @@ define <64 x i1> @lane64_no512_17(<64 x i1> %src) #2 {
define <64 x i1> @lane64_512_0(<64 x i1> %src) #0 {
; CHECK-LABEL: 'lane64_512_0'
-; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
+; CHECK-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> zeroinitializer
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
;
%s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 0)
@@ -59,7 +83,7 @@ define <64 x i1> @lane64_512_0(<64 x i1> %src) #0 {
define <64 x i1> @lane64_512_17(<64 x i1> %src) #0 {
; CHECK-LABEL: 'lane64_512_17'
-; CHECK-NEXT: Cost Model: Found costs of 65 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; CHECK-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
; CHECK-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <64 x i1> %s
;
%s = shufflevector <64 x i1> %src, <64 x i1> poison, <64 x i32> splat (i32 17)
@@ -69,7 +93,3 @@ define <64 x i1> @lane64_512_17(<64 x i1> %src) #0 {
attributes #0 = { "prefer-vector-width"="256" "min-legal-vector-width"="512" }
attributes #1 = { "prefer-vector-width"="512" "min-legal-vector-width"="512" }
attributes #2 = { "prefer-vector-width"="256" "min-legal-vector-width"="0" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64: {{.*}}
-; X64-NODQ: {{.*}}
-; X86: {{.*}}
diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
index 70498a3ee0d8f..aef521bf5a2dd 100644
--- a/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
+++ b/llvm/test/Analysis/CostModel/X86/shuffle-splat.ll
@@ -410,14 +410,59 @@ define void @test_vXi1(<2 x i1> %src2, <4 x i1> %src4, <8 x i1> %src8, <16 x i1>
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; AVX512-LABEL: 'test_vXi1'
-; AVX512-NEXT: Cost Model: Found costs of 3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
-; AVX512-NEXT: Cost Model: Found costs of 5 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 9 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 17 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 33 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX512F-LABEL: 'test_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-FAST-LABEL: 'test_vXi1'
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:3 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-FAST-LABEL: 'test_vXi1'
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:4 for: %V8 = shufflevector <8 x i1> %src8, <8 x i1> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V16 = shufflevector <16 x i1> %src16, <16 x i1> poison, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V32 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V64 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V2 = shufflevector <2 x i1> %src2, <2 x i1> poison, <2 x i32> <i32 1, i32 1>
%V4 = shufflevector <4 x i1> %src4, <4 x i1> poison, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
@@ -465,11 +510,41 @@ define void @test_multireg_vXi1(<128 x i1> %src128) {
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; AVX512-LABEL: 'test_multireg_vXi1'
-; AVX512-NEXT: Cost Model: Found costs of 129 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
-; AVX512-NEXT: Cost Model: Found costs of 129 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
-; AVX512-NEXT: Cost Model: Found costs of 129 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX512F-LABEL: 'test_multireg_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_multireg_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_multireg_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_multireg_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-FAST-LABEL: 'test_multireg_vXi1'
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-FAST-LABEL: 'test_multireg_vXi1'
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V65 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> <i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65, i32 65>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V0 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> zeroinitializer
%V64 = shufflevector <128 x i1> %src128, <128 x i1> poison, <128 x i32> splat (i32 64)
@@ -528,15 +603,65 @@ define void @test_sublane_vXi1(<32 x i1> %src32, <64 x i1> %src64) {
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; AVX512-LABEL: 'test_sublane_vXi1'
-; AVX512-NEXT: Cost Model: Found costs of 33 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; AVX512-NEXT: Cost Model: Found costs of 33 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
-; AVX512-NEXT: Cost Model: Found costs of 33 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX512F-LABEL: 'test_sublane_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_sublane_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_sublane_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_sublane_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-FAST-LABEL: 'test_sublane_vXi1'
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-FAST-LABEL: 'test_sublane_vXi1'
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %W17 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V1 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V16 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V17 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V49 = shufflevector <64 x i1> %src64, <64 x i1> poison, <64 x i32> <i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49, i32 49>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%W1 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> splat (i32 1)
%W16 = shufflevector <32 x i1> %src32, <32 x i1> poison, <32 x i32> splat (i32 16)
@@ -624,12 +749,47 @@ define void @test_upper_vXi1(<64 x i1> %a64, <64 x i1> %b64) {
; AVX2-NEXT: Cost Model: Found costs of RThru:1 CodeSize:1 Lat:3 SizeLat:2 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
-; AVX512-LABEL: 'test_upper_vXi1'
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
-; AVX512-NEXT: Cost Model: Found costs of 65 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+; AVX512F-LABEL: 'test_upper_vXi1'
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:8 SizeLat:3 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:3 CodeSize:4 Lat:11 SizeLat:5 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512BW-LABEL: 'test_upper_vXi1'
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-LABEL: 'test_upper_vXi1'
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512VBMI-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-LABEL: 'test_upper_vXi1'
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:3 CodeSize:5 Lat:9 SizeLat:6 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512DQ-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512DQ-FAST-LABEL: 'test_upper_vXi1'
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:8 SizeLat:5 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:2 CodeSize:5 Lat:6 SizeLat:7 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512DQ-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
+;
+; AVX512VBMI-FAST-LABEL: 'test_upper_vXi1'
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17, i32 17>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:3 Lat:5 SizeLat:3 for: %V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64, i32 64>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V80 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80, i32 80>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:1 CodeSize:4 Lat:5 SizeLat:4 for: %V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> <i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81, i32 81>
+; AVX512VBMI-FAST-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret void
;
%V17 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 17)
%V64 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 64)
@@ -637,10 +797,3 @@ define void @test_upper_vXi1(<64 x i1> %a64, <64 x i1> %b64) {
%V81 = shufflevector <64 x i1> %a64, <64 x i1> %b64, <64 x i32> splat (i32 81)
ret void
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512BW: {{.*}}
-; AVX512DQ: {{.*}}
-; AVX512DQ-FAST: {{.*}}
-; AVX512F: {{.*}}
-; AVX512VBMI: {{.*}}
-; AVX512VBMI-FAST: {{.*}}
More information about the llvm-commits
mailing list