[llvm] [X86][CostModel] Price predicate mask-expansion fanout for wide predicated ops (PR #201666)
Sumukh J Bharadwaj via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 13:59:59 PDT 2026
https://github.com/amd-subharad updated https://github.com/llvm/llvm-project/pull/201666
>From c7d5ca28cb431a20a1d234510a0377fb12298c4f Mon Sep 17 00:00:00 2001
From: Sumukh Bharadwaj <Sumukh.Bharadwaj at amd.com>
Date: Sat, 3 Oct 2026 00:24:37 +0530
Subject: [PATCH 1/2] [X86][CostModel] Precommit tests for predicate
mask-expansion fanout (NFC)
Add wider masked load/store/gather/scatter (v16i64/v32i64/v16f64/v32f64)
and matching wide-data selects to the existing cost-model tests, where the
data type legalizes into more register parts than its <N x i1> predicate.
These pin the current (pre-fix) costs; the follow-up X86 TTI change updates
the CHECK lines to show the added mask-expansion fanout.
---
.../CostModel/X86/masked-intrinsic-cost.ll | 145 ++++++++++++++++++
llvm/test/Analysis/CostModel/X86/select.ll | 30 ++++
2 files changed, 175 insertions(+)
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index 587a70e54b37b..e697a84d598eb 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -69,6 +69,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE2-NEXT: Cost Model: Found costs of RThru:188 CodeSize:220 Lat:316 SizeLat:220 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SSE2-NEXT: Cost Model: Found costs of RThru:94 CodeSize:110 Lat:158 SizeLat:110 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SSE2-NEXT: Cost Model: Found costs of RThru:46 CodeSize:54 Lat:78 SizeLat:54 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT: Cost Model: Found costs of RThru:87 CodeSize:103 Lat:151 SizeLat:103 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:174 CodeSize:206 Lat:302 SizeLat:206 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:71 CodeSize:87 Lat:135 SizeLat:87 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:142 CodeSize:174 Lat:270 SizeLat:174 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SSE42-LABEL: 'masked_load'
@@ -128,6 +132,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:256 SizeLat:160 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SSE42-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SSE42-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:64 SizeLat:40 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:256 SizeLat:160 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:56 CodeSize:72 Lat:120 SizeLat:72 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:112 CodeSize:144 Lat:240 SizeLat:144 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_load'
@@ -187,6 +195,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:258 SizeLat:162 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:64 SizeLat:40 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; AVX-NEXT: Cost Model: Found costs of 16 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; AVX-NEXT: Cost Model: Found costs of 16 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_load'
@@ -246,6 +258,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; KNL-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:258 SizeLat:162 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:64 SizeLat:40 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; KNL-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_load'
@@ -305,6 +321,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SKX-NEXT: Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of 2 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SKX-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_load'
@@ -364,6 +384,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; VBMI2-NEXT: Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of 2 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; VBMI2-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
%V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -428,6 +452,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
%V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
%V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+ %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr poison, i32 1, <16 x i1> %m16, <16 x i64> poison)
+ %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr poison, i32 1, <32 x i1> %m32, <32 x i64> poison)
+ %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr poison, i32 1, <16 x i1> %m16, <16 x double> poison)
+ %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr poison, i32 1, <32 x i1> %m32, <32 x double> poison)
ret i32 0
}
@@ -489,6 +517,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE2-NEXT: Cost Model: Found costs of RThru:188 CodeSize:220 Lat:220 SizeLat:220 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; SSE2-NEXT: Cost Model: Found costs of RThru:94 CodeSize:110 Lat:110 SizeLat:110 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; SSE2-NEXT: Cost Model: Found costs of RThru:46 CodeSize:54 Lat:54 SizeLat:54 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; SSE2-NEXT: Cost Model: Found costs of RThru:87 CodeSize:103 Lat:103 SizeLat:103 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; SSE2-NEXT: Cost Model: Found costs of RThru:174 CodeSize:206 Lat:206 SizeLat:206 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; SSE2-NEXT: Cost Model: Found costs of RThru:71 CodeSize:87 Lat:87 SizeLat:87 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; SSE2-NEXT: Cost Model: Found costs of RThru:142 CodeSize:174 Lat:174 SizeLat:174 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SSE42-LABEL: 'masked_store'
@@ -548,6 +580,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:160 SizeLat:160 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; SSE42-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; SSE42-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:40 SizeLat:40 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; SSE42-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; SSE42-NEXT: Cost Model: Found costs of RThru:128 CodeSize:160 Lat:160 SizeLat:160 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; SSE42-NEXT: Cost Model: Found costs of RThru:56 CodeSize:72 Lat:72 SizeLat:72 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; SSE42-NEXT: Cost Model: Found costs of RThru:112 CodeSize:144 Lat:144 SizeLat:144 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_store'
@@ -607,6 +643,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:162 SizeLat:162 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; AVX-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; AVX-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:40 SizeLat:40 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; AVX-NEXT: Cost Model: Found costs of 32 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 64 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; AVX-NEXT: Cost Model: Found costs of 32 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 64 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_store'
@@ -666,6 +706,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; KNL-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:162 SizeLat:162 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; KNL-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:40 SizeLat:40 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; KNL-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_store'
@@ -725,6 +769,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SKX-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_store'
@@ -784,6 +832,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; VBMI2-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
+; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr undef, i32 1, <8 x i1> %m8)
@@ -848,6 +900,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr undef, i32 1, <16 x i1> %m16)
call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr undef, i32 1, <8 x i1> %m8)
+ call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr poison, i32 1, <16 x i1> %m16)
+ call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr poison, i32 1, <32 x i1> %m32)
+ call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr poison, i32 1, <16 x i1> %m16)
+ call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr poison, i32 1, <32 x i1> %m32)
ret i32 0
}
@@ -877,6 +933,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SSE2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:288 SizeLat:192 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SSE2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:144 SizeLat:96 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SSE2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:72 SizeLat:48 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE2-NEXT: Cost Model: Found costs of RThru:73 CodeSize:89 Lat:137 SizeLat:89 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:146 CodeSize:178 Lat:274 SizeLat:178 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:121 SizeLat:73 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SSE2-NEXT: Cost Model: Found costs of RThru:114 CodeSize:146 Lat:242 SizeLat:146 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SSE42-LABEL: 'masked_gather'
@@ -904,6 +964,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SSE42-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:226 SizeLat:130 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SSE42-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:113 SizeLat:65 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SSE42-NEXT: Cost Model: Found costs of RThru:25 CodeSize:33 Lat:57 SizeLat:33 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SSE42-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:113 SizeLat:65 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:226 SizeLat:130 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:41 CodeSize:57 Lat:105 SizeLat:57 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SSE42-NEXT: Cost Model: Found costs of RThru:82 CodeSize:114 Lat:210 SizeLat:114 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX1-LABEL: 'masked_gather'
@@ -931,6 +995,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; AVX1-NEXT: Cost Model: Found costs of RThru:107 CodeSize:139 Lat:235 SizeLat:139 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:117 SizeLat:69 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; AVX1-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:59 SizeLat:35 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX1-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:121 SizeLat:73 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:114 CodeSize:146 Lat:242 SizeLat:146 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:113 SizeLat:65 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; AVX1-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:226 SizeLat:130 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX2-LABEL: 'masked_gather'
@@ -958,6 +1026,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; AVX2-NEXT: Cost Model: Found costs of RThru:106 CodeSize:138 Lat:234 SizeLat:138 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:117 SizeLat:69 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; AVX2-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:59 SizeLat:35 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; AVX2-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:121 SizeLat:73 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:113 CodeSize:145 Lat:241 SizeLat:145 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:113 SizeLat:65 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; AVX2-NEXT: Cost Model: Found costs of RThru:97 CodeSize:129 Lat:225 SizeLat:129 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKL-LABEL: 'masked_gather'
@@ -985,6 +1057,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SKL-NEXT: Cost Model: Found costs of RThru:106 CodeSize:138 Lat:234 SizeLat:138 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:117 SizeLat:69 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:59 SizeLat:35 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:8 Lat:144 SizeLat:48 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:8 Lat:144 SizeLat:48 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_gather'
@@ -1012,6 +1088,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; KNL-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_gather'
@@ -1039,6 +1119,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SKX-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_gather'
@@ -1066,6 +1150,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; VBMI2-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
%V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -1098,6 +1186,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
%V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> undef, i32 1, <16 x i1> %m16, <16 x i8> undef)
%V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x i8> undef)
+ %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> poison, i32 1, <16 x i1> %m16, <16 x i64> poison)
+ %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> poison, i32 1, <32 x i1> %m32, <32 x i64> poison)
+ %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> poison, i32 1, <16 x i1> %m16, <16 x double> poison)
+ %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> poison, i32 1, <32 x i1> %m32, <32 x double> poison)
ret i32 0
}
@@ -1127,6 +1219,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; SSE2-NEXT: Cost Model: Found costs of RThru:160 CodeSize:192 Lat:192 SizeLat:192 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; SSE2-NEXT: Cost Model: Found costs of RThru:80 CodeSize:96 Lat:96 SizeLat:96 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; SSE2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:48 Lat:48 SizeLat:48 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; SSE2-NEXT: Cost Model: Found costs of RThru:73 CodeSize:89 Lat:89 SizeLat:89 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SSE2-NEXT: Cost Model: Found costs of RThru:146 CodeSize:178 Lat:178 SizeLat:178 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SSE2-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:73 SizeLat:73 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SSE2-NEXT: Cost Model: Found costs of RThru:114 CodeSize:146 Lat:146 SizeLat:146 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SSE42-LABEL: 'masked_scatter'
@@ -1154,6 +1250,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; SSE42-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:130 SizeLat:130 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; SSE42-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:65 SizeLat:65 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; SSE42-NEXT: Cost Model: Found costs of RThru:25 CodeSize:33 Lat:33 SizeLat:33 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; SSE42-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:65 SizeLat:65 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SSE42-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:130 SizeLat:130 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SSE42-NEXT: Cost Model: Found costs of RThru:41 CodeSize:57 Lat:57 SizeLat:57 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SSE42-NEXT: Cost Model: Found costs of RThru:82 CodeSize:114 Lat:114 SizeLat:114 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX1-LABEL: 'masked_scatter'
@@ -1181,6 +1281,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; AVX1-NEXT: Cost Model: Found costs of RThru:107 CodeSize:139 Lat:139 SizeLat:139 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; AVX1-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:69 SizeLat:69 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; AVX1-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:35 SizeLat:35 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; AVX1-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:73 SizeLat:73 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; AVX1-NEXT: Cost Model: Found costs of RThru:114 CodeSize:146 Lat:146 SizeLat:146 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; AVX1-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:65 SizeLat:65 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; AVX1-NEXT: Cost Model: Found costs of RThru:98 CodeSize:130 Lat:130 SizeLat:130 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX2-LABEL: 'masked_scatter'
@@ -1208,6 +1312,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; AVX2-NEXT: Cost Model: Found costs of RThru:106 CodeSize:138 Lat:138 SizeLat:138 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; AVX2-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:69 SizeLat:69 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; AVX2-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:35 SizeLat:35 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; AVX2-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:73 SizeLat:73 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:113 CodeSize:145 Lat:145 SizeLat:145 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; AVX2-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:65 SizeLat:65 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; AVX2-NEXT: Cost Model: Found costs of RThru:97 CodeSize:129 Lat:129 SizeLat:129 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKL-LABEL: 'masked_scatter'
@@ -1235,6 +1343,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; SKL-NEXT: Cost Model: Found costs of RThru:106 CodeSize:138 Lat:138 SizeLat:138 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; SKL-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:69 SizeLat:69 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; SKL-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:35 SizeLat:35 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; SKL-NEXT: Cost Model: Found costs of RThru:57 CodeSize:73 Lat:73 SizeLat:73 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKL-NEXT: Cost Model: Found costs of RThru:113 CodeSize:145 Lat:145 SizeLat:145 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SKL-NEXT: Cost Model: Found costs of RThru:49 CodeSize:65 Lat:65 SizeLat:65 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKL-NEXT: Cost Model: Found costs of RThru:97 CodeSize:129 Lat:129 SizeLat:129 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_scatter'
@@ -1262,6 +1374,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; KNL-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; KNL-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_scatter'
@@ -1289,6 +1405,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; SKX-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; SKX-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_scatter'
@@ -1316,6 +1436,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; VBMI2-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; VBMI2-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
@@ -1348,6 +1472,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> undef, i32 1, <16 x i1> %m16)
call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
+ call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> poison, i32 1, <16 x i1> %m16)
+ call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> poison, i32 1, <32 x i1> %m32)
+ call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> poison, i32 1, <16 x i1> %m16)
+ call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> poison, i32 1, <32 x i1> %m32)
ret i32 0
}
@@ -2745,3 +2873,20 @@ declare void @llvm.masked.compressstore.v64i8(<64 x i8>, ptr, <64 x i1>)
declare void @llvm.masked.compressstore.v32i8(<32 x i8>, ptr, <32 x i1>)
declare void @llvm.masked.compressstore.v16i8(<16 x i8>, ptr, <16 x i1>)
declare void @llvm.masked.compressstore.v8i8(<8 x i8>, ptr, <8 x i1>)
+
+declare <16 x i64> @llvm.masked.load.v16i64.p0(ptr, i32, <16 x i1>, <16 x i64>)
+declare <32 x i64> @llvm.masked.load.v32i64.p0(ptr, i32, <32 x i1>, <32 x i64>)
+declare <16 x double> @llvm.masked.load.v16f64.p0(ptr, i32, <16 x i1>, <16 x double>)
+declare <32 x double> @llvm.masked.load.v32f64.p0(ptr, i32, <32 x i1>, <32 x double>)
+declare void @llvm.masked.store.v16i64.p0(<16 x i64>, ptr, i32, <16 x i1>)
+declare void @llvm.masked.store.v32i64.p0(<32 x i64>, ptr, i32, <32 x i1>)
+declare void @llvm.masked.store.v16f64.p0(<16 x double>, ptr, i32, <16 x i1>)
+declare void @llvm.masked.store.v32f64.p0(<32 x double>, ptr, i32, <32 x i1>)
+declare <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x i64>)
+declare <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x i64>)
+declare <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr>, i32, <16 x i1>, <16 x double>)
+declare <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr>, i32, <32 x i1>, <32 x double>)
+declare void @llvm.masked.scatter.v16i64.v16p0(<16 x i64>, <16 x ptr>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v32i64.v32p0(<32 x i64>, <32 x ptr>, i32, <32 x i1>)
+declare void @llvm.masked.scatter.v16f64.v16p0(<16 x double>, <16 x ptr>, i32, <16 x i1>)
+declare void @llvm.masked.scatter.v32f64.v32p0(<32 x double>, <32 x ptr>, i32, <32 x i1>)
diff --git a/llvm/test/Analysis/CostModel/X86/select.ll b/llvm/test/Analysis/CostModel/X86/select.ll
index f2c2890a182fc..36072106c099c 100644
--- a/llvm/test/Analysis/CostModel/X86/select.ll
+++ b/llvm/test/Analysis/CostModel/X86/select.ll
@@ -30,6 +30,8 @@ define i32 @test_select() {
; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; SSE2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:4 SizeLat:6 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; SSE2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:12 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; SSE2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:24 Lat:16 SizeLat:24 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; SSE2-NEXT: Cost Model: Found costs of RThru:32 CodeSize:48 Lat:32 SizeLat:48 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SSE4-LABEL: 'test_select'
@@ -49,6 +51,8 @@ define i32 @test_select() {
; SSE4-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; SSE4-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; SSE4-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; SSE4-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; SSE4-NEXT: Cost Model: Found costs of RThru:32 CodeSize:16 Lat:32 SizeLat:32 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; SSE4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'test_select'
@@ -68,6 +72,8 @@ define i32 @test_select() {
; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX1-NEXT: Cost Model: Found costs of 3 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX1-NEXT: Cost Model: Found costs of 6 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:12 SizeLat:8 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX1-NEXT: Cost Model: Found costs of RThru:24 CodeSize:8 Lat:24 SizeLat:16 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'test_select'
@@ -87,6 +93,8 @@ define i32 @test_select() {
; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512F-LABEL: 'test_select'
@@ -106,6 +114,8 @@ define i32 @test_select() {
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:2 SizeLat:4 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX512F-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX512F-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512BW-LABEL: 'test_select'
@@ -125,6 +135,8 @@ define i32 @test_select() {
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SLM-LABEL: 'test_select'
@@ -144,6 +156,8 @@ define i32 @test_select() {
; SLM-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:3 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; SLM-NEXT: Cost Model: Found costs of RThru:8 CodeSize:2 Lat:8 SizeLat:6 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; SLM-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:16 SizeLat:12 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:8 Lat:32 SizeLat:24 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:16 Lat:64 SizeLat:48 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%I64 = select i1 undef, i64 undef, i64 undef
@@ -166,6 +180,8 @@ define i32 @test_select() {
%V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
%V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+ %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+ %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
ret i32 undef
}
@@ -179,6 +195,8 @@ define i32 @test_select_fp() {
; SSE2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:2 SizeLat:3 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; SSE2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:6 Lat:4 SizeLat:6 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; SSE2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:12 Lat:8 SizeLat:12 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; SSE2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:24 Lat:16 SizeLat:24 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; SSE2-NEXT: Cost Model: Found costs of RThru:32 CodeSize:48 Lat:32 SizeLat:48 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; SSE2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SSE4-LABEL: 'test_select_fp'
@@ -190,6 +208,8 @@ define i32 @test_select_fp() {
; SSE4-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; SSE4-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; SSE4-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; SSE4-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; SSE4-NEXT: Cost Model: Found costs of RThru:32 CodeSize:16 Lat:32 SizeLat:32 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; SSE4-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX1-LABEL: 'test_select_fp'
@@ -201,6 +221,8 @@ define i32 @test_select_fp() {
; AVX1-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; AVX1-NEXT: Cost Model: Found costs of RThru:3 CodeSize:1 Lat:3 SizeLat:2 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; AVX1-NEXT: Cost Model: Found costs of RThru:6 CodeSize:2 Lat:6 SizeLat:4 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; AVX1-NEXT: Cost Model: Found costs of RThru:12 CodeSize:4 Lat:12 SizeLat:8 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; AVX1-NEXT: Cost Model: Found costs of RThru:24 CodeSize:8 Lat:24 SizeLat:16 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; AVX1-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX2-LABEL: 'test_select_fp'
@@ -212,6 +234,8 @@ define i32 @test_select_fp() {
; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; AVX2-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; AVX2-NEXT: Cost Model: Found costs of RThru:4 CodeSize:2 Lat:4 SizeLat:4 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; AVX2-NEXT: Cost Model: Found costs of RThru:8 CodeSize:4 Lat:8 SizeLat:8 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512-LABEL: 'test_select_fp'
@@ -223,6 +247,8 @@ define i32 @test_select_fp() {
; AVX512-NEXT: Cost Model: Found costs of 1 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; AVX512-NEXT: Cost Model: Found costs of 1 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; AVX512-NEXT: Cost Model: Found costs of 1 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; AVX512-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; AVX512-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SLM-LABEL: 'test_select_fp'
@@ -234,6 +260,8 @@ define i32 @test_select_fp() {
; SLM-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:4 SizeLat:3 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
; SLM-NEXT: Cost Model: Found costs of RThru:8 CodeSize:2 Lat:8 SizeLat:6 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
; SLM-NEXT: Cost Model: Found costs of RThru:16 CodeSize:4 Lat:16 SizeLat:12 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; SLM-NEXT: Cost Model: Found costs of RThru:32 CodeSize:8 Lat:32 SizeLat:24 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; SLM-NEXT: Cost Model: Found costs of RThru:64 CodeSize:16 Lat:64 SizeLat:48 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; SLM-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
%F64 = select i1 undef, double undef, double undef
@@ -246,6 +274,8 @@ define i32 @test_select_fp() {
%V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
%V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+ %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+ %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
ret i32 undef
}
>From 2e361a5f471e697e8772f71783b26dd0ecfc6c87 Mon Sep 17 00:00:00 2001
From: Sumukh Bharadwaj <Sumukh.Bharadwaj at amd.com>
Date: Sat, 3 Oct 2026 00:28:59 +0530
Subject: [PATCH 2/2] [X86][CostModel] Price predicate mask-expansion fanout
for wide predicated ops
A predicated op (masked load/store, gather/scatter, select) can carry a
compact <N x i1> predicate while its data legalizes into more register
parts than the mask. CodeGen then materializes a sub-mask per extra data
part - a kshiftr from the k-register on AVX-512, or vector unpack/extend on
SSE/AVX - which the current per-part cost tables miss.
Charge this fanout via a helper getPredicateFanoutCost() in the masked
memory, gather/scatter and select cost paths. It fires only when the data
has more parts than the mask. The masked-memory and gather/scatter sites
gate on a variable predicate (a constant/all-ones mask is folded by CodeGen
and pays nothing); the select site is gated to AVX-512, where the k-register
-> wide-data kshiftr is a real cost the per-part tables miss, while on
SSE/AVX a select blend is already priced per part.
Tests for the new wide types were pre-committed; this patch updates their
CHECK lines to show the added cost.
---
.../lib/Target/X86/X86TargetTransformInfo.cpp | 73 ++++--
llvm/test/Analysis/CostModel/X86/fptoi_sat.ll | 12 +-
.../X86/masked-intrinsic-cost-inseltpoison.ll | 108 ++++-----
.../CostModel/X86/masked-intrinsic-cost.ll | 228 +++++++++---------
llvm/test/Analysis/CostModel/X86/select.ll | 45 ++--
.../masked-gather-i32-with-i8-index.ll | 22 +-
.../masked-gather-i64-with-i8-index.ll | 22 +-
.../X86/CostModel/masked-load-i16.ll | 2 +-
.../X86/CostModel/masked-load-i32.ll | 12 +-
.../X86/CostModel/masked-load-i64.ll | 18 +-
.../masked-scatter-i32-with-i8-index.ll | 8 +-
.../masked-scatter-i64-with-i8-index.ll | 8 +-
.../X86/CostModel/masked-store-i16.ll | 2 +-
.../X86/CostModel/masked-store-i32.ll | 12 +-
.../X86/CostModel/masked-store-i64.ll | 18 +-
.../X86/gep-nodes-with-non-gep-inst.ll | 26 +-
.../SLPVectorizer/X86/pr47629-inseltpoison.ll | 58 ++++-
.../Transforms/SLPVectorizer/X86/pr47629.ll | 58 ++++-
...masked-loads-consecutive-loads-same-ptr.ll | 11 +-
.../X86/scatter-vectorize-reused-pointer.ll | 12 +-
20 files changed, 455 insertions(+), 300 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index e42916fe8b5d5..367b3a8943184 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -3840,6 +3840,21 @@ InstructionCost X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
}
+// Additive cost for a predicated op's "predicate fanout": a select, masked
+// load/store or gather/scatter keeps a compact <N x i1> mask, but if its data
+// legalizes into more parts than the mask, codegen builds a sub-mask per extra
+// part (kshiftr on AVX-512, unpack/extend on SSE/AVX) the per-part tables miss.
+// FanoutCostPerPart is a tie-break: it beats the vectorizer's widest-VF bias.
+static InstructionCost getPredicateFanoutCost(const X86TTIImpl &TTI,
+ Type *DataVTy, Type *MaskVTy) {
+ constexpr unsigned FanoutCostPerPart = 4;
+ unsigned DataParts = TTI.getNumberOfParts(DataVTy);
+ unsigned MaskParts = TTI.getNumberOfParts(MaskVTy);
+ if (DataParts > MaskParts && MaskParts > 0)
+ return InstructionCost((DataParts - MaskParts) * FanoutCostPerPart);
+ return InstructionCost(0);
+}
+
InstructionCost X86TTIImpl::getCmpSelInstrCost(
unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
@@ -3857,6 +3872,13 @@ InstructionCost X86TTIImpl::getCmpSelInstrCost(
int ISD = TLI->InstructionOpcodeToISD(Opcode);
assert(ISD && "Invalid opcode");
+ // Predicate fanout (see getPredicateFanoutCost), AVX-512 only: on SSE/AVX a
+ // select blend is already priced per part, so charging it there shrinks VF.
+ InstructionCost MaskExpCost = 0;
+ if (Opcode == Instruction::Select && ST->hasAVX512() &&
+ isa<VectorType>(ValTy) && isa_and_nonnull<VectorType>(CondTy))
+ MaskExpCost = getPredicateFanoutCost(*this, ValTy, CondTy);
+
InstructionCost ExtraCost = 0;
if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
// Some vector comparison predicates cost extra instructions.
@@ -4083,52 +4105,52 @@ InstructionCost X86TTIImpl::getCmpSelInstrCost(
if (ST->useSLMArithCosts())
if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasBWI())
if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasAVX512())
if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasAVX2())
if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasXOP())
if (const auto *Entry = CostTableLookup(XOPCostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasAVX())
if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasSSE42())
if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasSSE41())
if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasSSE2())
if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
if (ST->hasSSE1())
if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
if (auto KindCost = Entry->Cost[CostKind])
- return LT.first * (ExtraCost + *KindCost);
+ return LT.first * (ExtraCost + *KindCost) + MaskExpCost;
// Assume a 3cy latency for fp select ops.
if (CostKind == TTI::TCK_Latency && Opcode == Instruction::Select)
@@ -4136,7 +4158,8 @@ InstructionCost X86TTIImpl::getCmpSelInstrCost(
return 3;
return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
- Op1Info, Op2Info, I);
+ Op1Info, Op2Info, I) +
+ MaskExpCost;
}
unsigned X86TTIImpl::getAtomicMemIntrinsicMaxElementSize() const { return 16; }
@@ -6111,12 +6134,21 @@ X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
CostKind, {}, 0, MaskTy);
}
+ // Predicate fanout (see getPredicateFanoutCost): only a variable mask needs
+ // a per-part sub-mask; a constant mask is folded by codegen and pays nothing.
+ InstructionCost MaskExpCost = 0;
+ if (MICA.getVariableMask()) {
+ auto *MaskVecTy =
+ FixedVectorType::get(Type::getInt1Ty(SrcVTy->getContext()), NumElem);
+ MaskExpCost = getPredicateFanoutCost(*this, SrcVTy, MaskVecTy);
+ }
+
// Pre-AVX512 - each maskmov load costs 2 + store costs ~8.
if (!ST->hasAVX512())
- return Cost + LT.first * (IsLoad ? 2 : 8);
+ return Cost + LT.first * (IsLoad ? 2 : 8) + MaskExpCost;
- // AVX-512 masked load/store is cheaper
- return Cost + LT.first;
+ // AVX-512 masked load/store is cheaper.
+ return Cost + LT.first + MaskExpCost;
}
InstructionCost X86TTIImpl::getPointersChainCost(
@@ -7195,8 +7227,17 @@ X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter");
unsigned AddressSpace = MICA.getAddressSpace();
- return getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment,
- AddressSpace);
+ InstructionCost Cost =
+ getGSVectorCost(Opcode, CostKind, SrcVTy, Ptr, Alignment, AddressSpace);
+
+ // Predicate fanout (see getPredicateFanoutCost); variable mask only.
+ if (MICA.getVariableMask()) {
+ auto *MaskVecTy =
+ FixedVectorType::get(Type::getInt1Ty(SrcVTy->getContext()),
+ cast<FixedVectorType>(SrcVTy)->getNumElements());
+ Cost += getPredicateFanoutCost(*this, SrcVTy, MaskVecTy);
+ }
+ return Cost;
}
bool X86TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
diff --git a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
index 41bf88b1ec316..e5df2badfeb34 100644
--- a/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
+++ b/llvm/test/Analysis/CostModel/X86/fptoi_sat.ll
@@ -512,7 +512,7 @@ define void @casts() {
; AVX512F-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16f32u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f32(<16 x float> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16f32s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f32(<16 x float> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f32(<16 x float> undef)
-; AVX512F-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
+; AVX512F-NEXT: Cost Model: Found an estimated cost of 76 for instruction: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 69 for instruction: %v16f32u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f32(<16 x float> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 49 for instruction: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16f64u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f64(<16 x double> undef)
@@ -522,7 +522,7 @@ define void @casts() {
; AVX512F-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v16f64u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f64(<16 x double> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16f64u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f64(<16 x double> undef)
-; AVX512F-NEXT: Cost Model: Found an estimated cost of 76 for instruction: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
+; AVX512F-NEXT: Cost Model: Found an estimated cost of 80 for instruction: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 72 for instruction: %v16f64u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f64(<16 x double> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -615,7 +615,7 @@ define void @casts() {
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16f32u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f32(<16 x float> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16f32s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f32(<16 x float> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %v16f32u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f32(<16 x float> undef)
-; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
+; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v16f32s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f32(<16 x float> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 11 for instruction: %v16f32u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f32(<16 x float> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 49 for instruction: %v16f64s1 = call <16 x i1> @llvm.fptosi.sat.v16i1.v16f64(<16 x double> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16f64u1 = call <16 x i1> @llvm.fptoui.sat.v16i1.v16f64(<16 x double> undef)
@@ -625,7 +625,7 @@ define void @casts() {
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %v16f64u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f64(<16 x double> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v16f64s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f64(<16 x double> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %v16f64u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f64(<16 x double> undef)
-; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
+; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 22 for instruction: %v16f64s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f64(<16 x double> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 14 for instruction: %v16f64u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f64(<16 x double> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -1054,7 +1054,7 @@ define void @fp16() {
; AVX512F-NEXT: Cost Model: Found an estimated cost of 178 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 178 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 176 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; AVX512F-NEXT: Cost Model: Found an estimated cost of 186 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; AVX512F-NEXT: Cost Model: Found an estimated cost of 190 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 183 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
@@ -1107,7 +1107,7 @@ define void @fp16() {
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 178 for instruction: %v16f16u16 = call <16 x i16> @llvm.fptoui.sat.v16i16.v16f16(<16 x half> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 178 for instruction: %v16f16s32 = call <16 x i32> @llvm.fptosi.sat.v16i32.v16f16(<16 x half> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 176 for instruction: %v16f16u32 = call <16 x i32> @llvm.fptoui.sat.v16i32.v16f16(<16 x half> undef)
-; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 186 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
+; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 190 for instruction: %v16f16s64 = call <16 x i64> @llvm.fptosi.sat.v16i64.v16f16(<16 x half> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 183 for instruction: %v16f16u64 = call <16 x i64> @llvm.fptoui.sat.v16i64.v16f16(<16 x half> undef)
; AVX512DQ-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void
;
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
index eec3812c75493..180459121bd0f 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost-inseltpoison.ll
@@ -131,22 +131,22 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_load'
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V7F64 = call <7 x double> @llvm.masked.load.v7f64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V6F64 = call <6 x double> @llvm.masked.load.v6f64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V5F64 = call <5 x double> @llvm.masked.load.v5f64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V7F64 = call <7 x double> @llvm.masked.load.v7f64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V6F64 = call <6 x double> @llvm.masked.load.v6f64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V5F64 = call <5 x double> @llvm.masked.load.v5f64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V4F64 = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 1 undef, <4 x i1> %m4, <4 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3F64 = call <3 x double> @llvm.masked.load.v3f64.p0(ptr align 1 undef, <3 x i1> %m3, <3 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V2F64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr align 1 undef, <2 x i1> %m2, <2 x double> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F64 = call <1 x double> @llvm.masked.load.v1f64.p0(ptr align 1 undef, <1 x i1> %m1, <1 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V16F32 = call <16 x float> @llvm.masked.load.v16f32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V15F32 = call <15 x float> @llvm.masked.load.v15f32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V14F32 = call <14 x float> @llvm.masked.load.v14f32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V13F32 = call <13 x float> @llvm.masked.load.v13f32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V12F32 = call <12 x float> @llvm.masked.load.v12f32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V11F32 = call <11 x float> @llvm.masked.load.v11f32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V10F32 = call <10 x float> @llvm.masked.load.v10f32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V9F32 = call <9 x float> @llvm.masked.load.v9f32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V16F32 = call <16 x float> @llvm.masked.load.v16f32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V15F32 = call <15 x float> @llvm.masked.load.v15f32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V14F32 = call <14 x float> @llvm.masked.load.v14f32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V13F32 = call <13 x float> @llvm.masked.load.v13f32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V12F32 = call <12 x float> @llvm.masked.load.v12f32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V11F32 = call <11 x float> @llvm.masked.load.v11f32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V10F32 = call <10 x float> @llvm.masked.load.v10f32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V9F32 = call <9 x float> @llvm.masked.load.v9f32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V8F32 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 1 undef, <8 x i1> %m8, <8 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V7F32 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 undef, <7 x i1> %m7, <7 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V6F32 = call <6 x float> @llvm.masked.load.v6f32.p0(ptr align 1 undef, <6 x i1> %m6, <6 x float> undef)
@@ -155,22 +155,22 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3F32 = call <3 x float> @llvm.masked.load.v3f32.p0(ptr align 1 undef, <3 x i1> %m3, <3 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V2F32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 1 undef, <2 x i1> %m2, <2 x float> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F32 = call <1 x float> @llvm.masked.load.v1f32.p0(ptr align 1 undef, <1 x i1> %m1, <1 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V7I64 = call <7 x i64> @llvm.masked.load.v7i64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V6I64 = call <6 x i64> @llvm.masked.load.v6i64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V5I64 = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V8I64 = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V7I64 = call <7 x i64> @llvm.masked.load.v7i64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V6I64 = call <6 x i64> @llvm.masked.load.v6i64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V5I64 = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 1 undef, <4 x i1> %m4, <4 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3I64 = call <3 x i64> @llvm.masked.load.v3i64.p0(ptr align 1 undef, <3 x i1> %m3, <3 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V2I64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 1 undef, <2 x i1> %m2, <2 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1I64 = call <1 x i64> @llvm.masked.load.v1i64.p0(ptr align 1 undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V15I32 = call <15 x i32> @llvm.masked.load.v15i32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V14I32 = call <14 x i32> @llvm.masked.load.v14i32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V13I32 = call <13 x i32> @llvm.masked.load.v13i32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V12I32 = call <12 x i32> @llvm.masked.load.v12i32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V11I32 = call <11 x i32> @llvm.masked.load.v11i32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V10I32 = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V9I32 = call <9 x i32> @llvm.masked.load.v9i32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V15I32 = call <15 x i32> @llvm.masked.load.v15i32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V14I32 = call <14 x i32> @llvm.masked.load.v14i32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V13I32 = call <13 x i32> @llvm.masked.load.v13i32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V12I32 = call <12 x i32> @llvm.masked.load.v12i32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V11I32 = call <11 x i32> @llvm.masked.load.v11i32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V10I32 = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V9I32 = call <9 x i32> @llvm.masked.load.v9i32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V7I32 = call <7 x i32> @llvm.masked.load.v7i32.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V6I32 = call <6 x i32> @llvm.masked.load.v6i32.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i32> undef)
@@ -551,22 +551,22 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_store'
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr align 1 undef, <8 x i1> %m8)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v7f64.p0(<7 x double> undef, ptr align 1 undef, <7 x i1> %m7)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v6f64.p0(<6 x double> undef, ptr align 1 undef, <6 x i1> %m6)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v5f64.p0(<5 x double> undef, ptr align 1 undef, <5 x i1> %m5)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr align 1 undef, <8 x i1> %m8)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v7f64.p0(<7 x double> undef, ptr align 1 undef, <7 x i1> %m7)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v6f64.p0(<6 x double> undef, ptr align 1 undef, <6 x i1> %m6)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v5f64.p0(<5 x double> undef, ptr align 1 undef, <5 x i1> %m5)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v4f64.p0(<4 x double> undef, ptr align 1 undef, <4 x i1> %m4)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3f64.p0(<3 x double> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v2f64.p0(<2 x double> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1f64.p0(<1 x double> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v16f32.p0(<16 x float> undef, ptr align 1 undef, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v15f32.p0(<15 x float> undef, ptr align 1 undef, <15 x i1> %m15)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v14f32.p0(<14 x float> undef, ptr align 1 undef, <14 x i1> %m14)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v13f32.p0(<13 x float> undef, ptr align 1 undef, <13 x i1> %m13)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v12f32.p0(<12 x float> undef, ptr align 1 undef, <12 x i1> %m12)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v11f32.p0(<11 x float> undef, ptr align 1 undef, <11 x i1> %m11)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v10f32.p0(<10 x float> undef, ptr align 1 undef, <10 x i1> %m10)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v9f32.p0(<9 x float> undef, ptr align 1 undef, <9 x i1> %m9)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v16f32.p0(<16 x float> undef, ptr align 1 undef, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v15f32.p0(<15 x float> undef, ptr align 1 undef, <15 x i1> %m15)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v14f32.p0(<14 x float> undef, ptr align 1 undef, <14 x i1> %m14)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v13f32.p0(<13 x float> undef, ptr align 1 undef, <13 x i1> %m13)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v12f32.p0(<12 x float> undef, ptr align 1 undef, <12 x i1> %m12)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v11f32.p0(<11 x float> undef, ptr align 1 undef, <11 x i1> %m11)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v10f32.p0(<10 x float> undef, ptr align 1 undef, <10 x i1> %m10)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v9f32.p0(<9 x float> undef, ptr align 1 undef, <9 x i1> %m9)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v8f32.p0(<8 x float> undef, ptr align 1 undef, <8 x i1> %m8)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v7f32.p0(<7 x float> undef, ptr align 1 undef, <7 x i1> %m7)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v6f32.p0(<6 x float> undef, ptr align 1 undef, <6 x i1> %m6)
@@ -575,22 +575,22 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3f32.p0(<3 x float> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v2f32.p0(<2 x float> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1f32.p0(<1 x float> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v8i64.p0(<8 x i64> undef, ptr align 1 undef, <8 x i1> %m8)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v7i64.p0(<7 x i64> undef, ptr align 1 undef, <7 x i1> %m7)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v6i64.p0(<6 x i64> undef, ptr align 1 undef, <6 x i1> %m6)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v5i64.p0(<5 x i64> undef, ptr align 1 undef, <5 x i1> %m5)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v8i64.p0(<8 x i64> undef, ptr align 1 undef, <8 x i1> %m8)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v7i64.p0(<7 x i64> undef, ptr align 1 undef, <7 x i1> %m7)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v6i64.p0(<6 x i64> undef, ptr align 1 undef, <6 x i1> %m6)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v5i64.p0(<5 x i64> undef, ptr align 1 undef, <5 x i1> %m5)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v4i64.p0(<4 x i64> undef, ptr align 1 undef, <4 x i1> %m4)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3i64.p0(<3 x i64> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v2i64.p0(<2 x i64> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1i64.p0(<1 x i64> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v16i32.p0(<16 x i32> undef, ptr align 1 undef, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v15i32.p0(<15 x i32> undef, ptr align 1 undef, <15 x i1> %m15)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v14i32.p0(<14 x i32> undef, ptr align 1 undef, <14 x i1> %m14)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v13i32.p0(<13 x i32> undef, ptr align 1 undef, <13 x i1> %m13)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v12i32.p0(<12 x i32> undef, ptr align 1 undef, <12 x i1> %m12)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v11i32.p0(<11 x i32> undef, ptr align 1 undef, <11 x i1> %m11)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v10i32.p0(<10 x i32> undef, ptr align 1 undef, <10 x i1> %m10)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v9i32.p0(<9 x i32> undef, ptr align 1 undef, <9 x i1> %m9)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v16i32.p0(<16 x i32> undef, ptr align 1 undef, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v15i32.p0(<15 x i32> undef, ptr align 1 undef, <15 x i1> %m15)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v14i32.p0(<14 x i32> undef, ptr align 1 undef, <14 x i1> %m14)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v13i32.p0(<13 x i32> undef, ptr align 1 undef, <13 x i1> %m13)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v12i32.p0(<12 x i32> undef, ptr align 1 undef, <12 x i1> %m12)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v11i32.p0(<11 x i32> undef, ptr align 1 undef, <11 x i1> %m11)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v10i32.p0(<10 x i32> undef, ptr align 1 undef, <10 x i1> %m10)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v9i32.p0(<9 x i32> undef, ptr align 1 undef, <9 x i1> %m9)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v8i32.p0(<8 x i32> undef, ptr align 1 undef, <8 x i1> %m8)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v7i32.p0(<7 x i32> undef, ptr align 1 undef, <7 x i1> %m7)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v6i32.p0(<6 x i32> undef, ptr align 1 undef, <6 x i1> %m6)
@@ -961,19 +961,19 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKL-LABEL: 'masked_gather'
-; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:16 CodeSize:6 Lat:40 SizeLat:16 for: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> align 1 undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:16 CodeSize:6 Lat:40 SizeLat:16 for: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> align 1 undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x i32> undef)
@@ -2163,7 +2163,7 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
; SKL-LABEL: 'test_gather_16f32_var_mask'
; SKL-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %sext_ind = sext <16 x i32> %ind to <16 x i64>
; SKL-NEXT: Cost Model: Found costs of 0 for: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x float> %res
;
; AVX512-LABEL: 'test_gather_16f32_var_mask'
@@ -2207,7 +2207,7 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
; SKL-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %sext_ind = sext <16 x i32> %ind to <16 x i64>
; SKL-NEXT: Cost Model: Found costs of 0 for: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x float> %res
;
; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
diff --git a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index e697a84d598eb..aa87fc3f33895 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -139,22 +139,22 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_load'
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V7F64 = call <7 x double> @llvm.masked.load.v7f64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V6F64 = call <6 x double> @llvm.masked.load.v6f64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V5F64 = call <5 x double> @llvm.masked.load.v5f64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V7F64 = call <7 x double> @llvm.masked.load.v7f64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V6F64 = call <6 x double> @llvm.masked.load.v6f64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x double> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V5F64 = call <5 x double> @llvm.masked.load.v5f64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V4F64 = call <4 x double> @llvm.masked.load.v4f64.p0(ptr align 1 undef, <4 x i1> %m4, <4 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3F64 = call <3 x double> @llvm.masked.load.v3f64.p0(ptr align 1 undef, <3 x i1> %m3, <3 x double> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V2F64 = call <2 x double> @llvm.masked.load.v2f64.p0(ptr align 1 undef, <2 x i1> %m2, <2 x double> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F64 = call <1 x double> @llvm.masked.load.v1f64.p0(ptr align 1 undef, <1 x i1> %m1, <1 x double> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V16F32 = call <16 x float> @llvm.masked.load.v16f32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V15F32 = call <15 x float> @llvm.masked.load.v15f32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V14F32 = call <14 x float> @llvm.masked.load.v14f32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V13F32 = call <13 x float> @llvm.masked.load.v13f32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V12F32 = call <12 x float> @llvm.masked.load.v12f32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V11F32 = call <11 x float> @llvm.masked.load.v11f32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V10F32 = call <10 x float> @llvm.masked.load.v10f32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V9F32 = call <9 x float> @llvm.masked.load.v9f32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V16F32 = call <16 x float> @llvm.masked.load.v16f32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V15F32 = call <15 x float> @llvm.masked.load.v15f32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V14F32 = call <14 x float> @llvm.masked.load.v14f32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V13F32 = call <13 x float> @llvm.masked.load.v13f32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V12F32 = call <12 x float> @llvm.masked.load.v12f32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V11F32 = call <11 x float> @llvm.masked.load.v11f32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V10F32 = call <10 x float> @llvm.masked.load.v10f32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x float> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V9F32 = call <9 x float> @llvm.masked.load.v9f32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V8F32 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr align 1 undef, <8 x i1> %m8, <8 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V7F32 = call <7 x float> @llvm.masked.load.v7f32.p0(ptr align 1 undef, <7 x i1> %m7, <7 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V6F32 = call <6 x float> @llvm.masked.load.v6f32.p0(ptr align 1 undef, <6 x i1> %m6, <6 x float> undef)
@@ -163,22 +163,22 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3F32 = call <3 x float> @llvm.masked.load.v3f32.p0(ptr align 1 undef, <3 x i1> %m3, <3 x float> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V2F32 = call <2 x float> @llvm.masked.load.v2f32.p0(ptr align 1 undef, <2 x i1> %m2, <2 x float> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F32 = call <1 x float> @llvm.masked.load.v1f32.p0(ptr align 1 undef, <1 x i1> %m1, <1 x float> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V8I64 = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V7I64 = call <7 x i64> @llvm.masked.load.v7i64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V6I64 = call <6 x i64> @llvm.masked.load.v6i64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V5I64 = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V8I64 = call <8 x i64> @llvm.masked.load.v8i64.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V7I64 = call <7 x i64> @llvm.masked.load.v7i64.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V6I64 = call <6 x i64> @llvm.masked.load.v6i64.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i64> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V5I64 = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 1 undef, <5 x i1> %m5, <5 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V4I64 = call <4 x i64> @llvm.masked.load.v4i64.p0(ptr align 1 undef, <4 x i1> %m4, <4 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V3I64 = call <3 x i64> @llvm.masked.load.v3i64.p0(ptr align 1 undef, <3 x i1> %m3, <3 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V2I64 = call <2 x i64> @llvm.masked.load.v2i64.p0(ptr align 1 undef, <2 x i1> %m2, <2 x i64> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1I64 = call <1 x i64> @llvm.masked.load.v1i64.p0(ptr align 1 undef, <1 x i1> %m1, <1 x i64> undef)
-; AVX-NEXT: Cost Model: Found costs of 4 for: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V15I32 = call <15 x i32> @llvm.masked.load.v15i32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V14I32 = call <14 x i32> @llvm.masked.load.v14i32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V13I32 = call <13 x i32> @llvm.masked.load.v13i32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V12I32 = call <12 x i32> @llvm.masked.load.v12i32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V11I32 = call <11 x i32> @llvm.masked.load.v11i32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V10I32 = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x i32> undef)
-; AVX-NEXT: Cost Model: Found costs of 5 for: %V9I32 = call <9 x i32> @llvm.masked.load.v9i32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 8 for: %V16I32 = call <16 x i32> @llvm.masked.load.v16i32.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V15I32 = call <15 x i32> @llvm.masked.load.v15i32.p0(ptr align 1 undef, <15 x i1> %m15, <15 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V14I32 = call <14 x i32> @llvm.masked.load.v14i32.p0(ptr align 1 undef, <14 x i1> %m14, <14 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V13I32 = call <13 x i32> @llvm.masked.load.v13i32.p0(ptr align 1 undef, <13 x i1> %m13, <13 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V12I32 = call <12 x i32> @llvm.masked.load.v12i32.p0(ptr align 1 undef, <12 x i1> %m12, <12 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V11I32 = call <11 x i32> @llvm.masked.load.v11i32.p0(ptr align 1 undef, <11 x i1> %m11, <11 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V10I32 = call <10 x i32> @llvm.masked.load.v10i32.p0(ptr align 1 undef, <10 x i1> %m10, <10 x i32> undef)
+; AVX-NEXT: Cost Model: Found costs of 9 for: %V9I32 = call <9 x i32> @llvm.masked.load.v9i32.p0(ptr align 1 undef, <9 x i1> %m9, <9 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 2 for: %V8I32 = call <8 x i32> @llvm.masked.load.v8i32.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V7I32 = call <7 x i32> @llvm.masked.load.v7i32.p0(ptr align 1 undef, <7 x i1> %m7, <7 x i32> undef)
; AVX-NEXT: Cost Model: Found costs of 3 for: %V6I32 = call <6 x i32> @llvm.masked.load.v6i32.p0(ptr align 1 undef, <6 x i1> %m6, <6 x i32> undef)
@@ -195,10 +195,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:258 SizeLat:162 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; AVX-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:64 SizeLat:40 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; AVX-NEXT: Cost Model: Found costs of 8 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; AVX-NEXT: Cost Model: Found costs of 16 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; AVX-NEXT: Cost Model: Found costs of 8 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; AVX-NEXT: Cost Model: Found costs of 16 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; AVX-NEXT: Cost Model: Found costs of 20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; AVX-NEXT: Cost Model: Found costs of 44 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; AVX-NEXT: Cost Model: Found costs of 20 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; AVX-NEXT: Cost Model: Found costs of 44 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_load'
@@ -258,10 +258,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; KNL-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:258 SizeLat:162 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:128 SizeLat:80 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:64 SizeLat:40 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; KNL-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; KNL-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; KNL-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; KNL-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of 6 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of 12 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of 6 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of 12 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_load'
@@ -321,10 +321,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SKX-NEXT: Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of 2 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; SKX-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; SKX-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; SKX-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; SKX-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of 6 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of 16 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of 6 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of 16 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_load'
@@ -384,10 +384,10 @@ define i32 @masked_load(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; VBMI2-NEXT: Cost Model: Found costs of 1 for: %V32I8 = call <32 x i8> @llvm.masked.load.v32i8.p0(ptr align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of 1 for: %V16I8 = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of 2 for: %V8I8 = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; VBMI2-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; VBMI2-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; VBMI2-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; VBMI2-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 6 for: %ME_V16I64 = call <16 x i64> @llvm.masked.load.v16i64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 16 for: %ME_V32I64 = call <32 x i64> @llvm.masked.load.v32i64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 6 for: %ME_V16F64 = call <16 x double> @llvm.masked.load.v16f64.p0(ptr align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of 16 for: %ME_V32F64 = call <32 x double> @llvm.masked.load.v32f64.p0(ptr align 1 poison, <32 x i1> %m32, <32 x double> poison)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
%V8F64 = call <8 x double> @llvm.masked.load.v8f64.p0(ptr undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -587,22 +587,22 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SSE42-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; AVX-LABEL: 'masked_store'
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr align 1 undef, <8 x i1> %m8)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v7f64.p0(<7 x double> undef, ptr align 1 undef, <7 x i1> %m7)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v6f64.p0(<6 x double> undef, ptr align 1 undef, <6 x i1> %m6)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v5f64.p0(<5 x double> undef, ptr align 1 undef, <5 x i1> %m5)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr align 1 undef, <8 x i1> %m8)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v7f64.p0(<7 x double> undef, ptr align 1 undef, <7 x i1> %m7)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v6f64.p0(<6 x double> undef, ptr align 1 undef, <6 x i1> %m6)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v5f64.p0(<5 x double> undef, ptr align 1 undef, <5 x i1> %m5)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v4f64.p0(<4 x double> undef, ptr align 1 undef, <4 x i1> %m4)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3f64.p0(<3 x double> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v2f64.p0(<2 x double> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1f64.p0(<1 x double> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v16f32.p0(<16 x float> undef, ptr align 1 undef, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v15f32.p0(<15 x float> undef, ptr align 1 undef, <15 x i1> %m15)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v14f32.p0(<14 x float> undef, ptr align 1 undef, <14 x i1> %m14)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v13f32.p0(<13 x float> undef, ptr align 1 undef, <13 x i1> %m13)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v12f32.p0(<12 x float> undef, ptr align 1 undef, <12 x i1> %m12)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v11f32.p0(<11 x float> undef, ptr align 1 undef, <11 x i1> %m11)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v10f32.p0(<10 x float> undef, ptr align 1 undef, <10 x i1> %m10)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v9f32.p0(<9 x float> undef, ptr align 1 undef, <9 x i1> %m9)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v16f32.p0(<16 x float> undef, ptr align 1 undef, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v15f32.p0(<15 x float> undef, ptr align 1 undef, <15 x i1> %m15)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v14f32.p0(<14 x float> undef, ptr align 1 undef, <14 x i1> %m14)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v13f32.p0(<13 x float> undef, ptr align 1 undef, <13 x i1> %m13)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v12f32.p0(<12 x float> undef, ptr align 1 undef, <12 x i1> %m12)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v11f32.p0(<11 x float> undef, ptr align 1 undef, <11 x i1> %m11)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v10f32.p0(<10 x float> undef, ptr align 1 undef, <10 x i1> %m10)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v9f32.p0(<9 x float> undef, ptr align 1 undef, <9 x i1> %m9)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v8f32.p0(<8 x float> undef, ptr align 1 undef, <8 x i1> %m8)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v7f32.p0(<7 x float> undef, ptr align 1 undef, <7 x i1> %m7)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v6f32.p0(<6 x float> undef, ptr align 1 undef, <6 x i1> %m6)
@@ -611,22 +611,22 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3f32.p0(<3 x float> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v2f32.p0(<2 x float> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1f32.p0(<1 x float> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v8i64.p0(<8 x i64> undef, ptr align 1 undef, <8 x i1> %m8)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v7i64.p0(<7 x i64> undef, ptr align 1 undef, <7 x i1> %m7)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v6i64.p0(<6 x i64> undef, ptr align 1 undef, <6 x i1> %m6)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v5i64.p0(<5 x i64> undef, ptr align 1 undef, <5 x i1> %m5)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v8i64.p0(<8 x i64> undef, ptr align 1 undef, <8 x i1> %m8)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v7i64.p0(<7 x i64> undef, ptr align 1 undef, <7 x i1> %m7)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v6i64.p0(<6 x i64> undef, ptr align 1 undef, <6 x i1> %m6)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v5i64.p0(<5 x i64> undef, ptr align 1 undef, <5 x i1> %m5)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v4i64.p0(<4 x i64> undef, ptr align 1 undef, <4 x i1> %m4)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v3i64.p0(<3 x i64> undef, ptr align 1 undef, <3 x i1> %m3)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v2i64.p0(<2 x i64> undef, ptr align 1 undef, <2 x i1> %m2)
; AVX-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:3 SizeLat:3 for: call void @llvm.masked.store.v1i64.p0(<1 x i64> undef, ptr align 1 undef, <1 x i1> %m1)
-; AVX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v16i32.p0(<16 x i32> undef, ptr align 1 undef, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v15i32.p0(<15 x i32> undef, ptr align 1 undef, <15 x i1> %m15)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v14i32.p0(<14 x i32> undef, ptr align 1 undef, <14 x i1> %m14)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v13i32.p0(<13 x i32> undef, ptr align 1 undef, <13 x i1> %m13)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v12i32.p0(<12 x i32> undef, ptr align 1 undef, <12 x i1> %m12)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v11i32.p0(<11 x i32> undef, ptr align 1 undef, <11 x i1> %m11)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v10i32.p0(<10 x i32> undef, ptr align 1 undef, <10 x i1> %m10)
-; AVX-NEXT: Cost Model: Found costs of 17 for: call void @llvm.masked.store.v9i32.p0(<9 x i32> undef, ptr align 1 undef, <9 x i1> %m9)
+; AVX-NEXT: Cost Model: Found costs of 20 for: call void @llvm.masked.store.v16i32.p0(<16 x i32> undef, ptr align 1 undef, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v15i32.p0(<15 x i32> undef, ptr align 1 undef, <15 x i1> %m15)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v14i32.p0(<14 x i32> undef, ptr align 1 undef, <14 x i1> %m14)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v13i32.p0(<13 x i32> undef, ptr align 1 undef, <13 x i1> %m13)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v12i32.p0(<12 x i32> undef, ptr align 1 undef, <12 x i1> %m12)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v11i32.p0(<11 x i32> undef, ptr align 1 undef, <11 x i1> %m11)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v10i32.p0(<10 x i32> undef, ptr align 1 undef, <10 x i1> %m10)
+; AVX-NEXT: Cost Model: Found costs of 21 for: call void @llvm.masked.store.v9i32.p0(<9 x i32> undef, ptr align 1 undef, <9 x i1> %m9)
; AVX-NEXT: Cost Model: Found costs of 8 for: call void @llvm.masked.store.v8i32.p0(<8 x i32> undef, ptr align 1 undef, <8 x i1> %m8)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v7i32.p0(<7 x i32> undef, ptr align 1 undef, <7 x i1> %m7)
; AVX-NEXT: Cost Model: Found costs of 9 for: call void @llvm.masked.store.v6i32.p0(<6 x i32> undef, ptr align 1 undef, <6 x i1> %m6)
@@ -643,10 +643,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; AVX-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:162 SizeLat:162 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; AVX-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; AVX-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:40 SizeLat:40 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
-; AVX-NEXT: Cost Model: Found costs of 32 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 64 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
-; AVX-NEXT: Cost Model: Found costs of 32 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
-; AVX-NEXT: Cost Model: Found costs of 64 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
+; AVX-NEXT: Cost Model: Found costs of 44 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 92 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; AVX-NEXT: Cost Model: Found costs of 44 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; AVX-NEXT: Cost Model: Found costs of 92 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; AVX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_store'
@@ -706,10 +706,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; KNL-NEXT: Cost Model: Found costs of RThru:130 CodeSize:162 Lat:162 SizeLat:162 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:64 CodeSize:80 Lat:80 SizeLat:80 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; KNL-NEXT: Cost Model: Found costs of RThru:32 CodeSize:40 Lat:40 SizeLat:40 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
-; KNL-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
-; KNL-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
-; KNL-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
-; KNL-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of 12 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of 12 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_store'
@@ -769,10 +769,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; SKX-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
-; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
-; SKX-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
-; SKX-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
-; SKX-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_store'
@@ -832,10 +832,10 @@ define i32 @masked_store(<1 x i1> %m1, <2 x i1> %m2, <3 x i1> %m3, <4 x i1> %m4,
; VBMI2-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v32i8.p0(<32 x i8> undef, ptr align 1 undef, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of 1 for: call void @llvm.masked.store.v16i8.p0(<16 x i8> undef, ptr align 1 undef, <16 x i1> %m16)
; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v8i8.p0(<8 x i8> undef, ptr align 1 undef, <8 x i1> %m8)
-; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
-; VBMI2-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
-; VBMI2-NEXT: Cost Model: Found costs of 2 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
-; VBMI2-NEXT: Cost Model: Found costs of 4 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16i64.p0(<16 x i64> poison, ptr align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v32i64.p0(<32 x i64> poison, ptr align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of 6 for: call void @llvm.masked.store.v16f64.p0(<16 x double> poison, ptr align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of 16 for: call void @llvm.masked.store.v32f64.p0(<32 x double> poison, ptr align 1 poison, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
call void @llvm.masked.store.v8f64.p0(<8 x double> undef, ptr undef, i32 1, <8 x i1> %m8)
@@ -1033,19 +1033,19 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKL-LABEL: 'masked_gather'
-; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x double> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:16 CodeSize:6 Lat:40 SizeLat:16 for: %V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4F64 = call <4 x double> @llvm.masked.gather.v4f64.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2F64 = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x double> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1F64 = call <1 x double> @llvm.masked.gather.v1f64.v1p0(<1 x ptr> align 1 undef, <1 x i1> %m1, <1 x double> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %V16F32 = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8F32 = call <8 x float> @llvm.masked.gather.v8f32.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4F32 = call <4 x float> @llvm.masked.gather.v4f32.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2F32 = call <2 x float> @llvm.masked.gather.v2f32.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x float> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i64> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:16 CodeSize:6 Lat:40 SizeLat:16 for: %V8I64 = call <8 x i64> @llvm.masked.gather.v8i64.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4I64 = call <4 x i64> @llvm.masked.gather.v4i64.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2I64 = call <2 x i64> @llvm.masked.gather.v2i64.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x i64> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:2 CodeSize:3 Lat:6 SizeLat:3 for: %V1I64 = call <1 x i64> @llvm.masked.gather.v1i64.v1p0(<1 x ptr> align 1 undef, <1 x i1> %m1, <1 x i64> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i32> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %V16I32 = call <16 x i32> @llvm.masked.gather.v16i32.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:12 CodeSize:2 Lat:36 SizeLat:12 for: %V8I32 = call <8 x i32> @llvm.masked.gather.v8i32.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:6 CodeSize:1 Lat:18 SizeLat:6 for: %V4I32 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> align 1 undef, <4 x i1> %m4, <4 x i32> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:4 CodeSize:1 Lat:10 SizeLat:4 for: %V2I32 = call <2 x i32> @llvm.masked.gather.v2i32.v2p0(<2 x ptr> align 1 undef, <2 x i1> %m2, <2 x i32> undef)
@@ -1057,10 +1057,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SKL-NEXT: Cost Model: Found costs of RThru:106 CodeSize:138 Lat:234 SizeLat:138 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:53 CodeSize:69 Lat:117 SizeLat:69 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:27 CodeSize:35 Lat:59 SizeLat:35 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; SKL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:8 Lat:144 SizeLat:48 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; SKL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:8 Lat:144 SizeLat:48 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:36 CodeSize:16 Lat:84 SizeLat:36 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:76 CodeSize:36 Lat:172 SizeLat:76 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:36 CodeSize:16 Lat:84 SizeLat:36 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKL-NEXT: Cost Model: Found costs of RThru:76 CodeSize:36 Lat:172 SizeLat:76 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; KNL-LABEL: 'masked_gather'
@@ -1088,10 +1088,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; KNL-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; KNL-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:12 Lat:144 SizeLat:48 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; KNL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:12 Lat:144 SizeLat:48 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_gather'
@@ -1119,10 +1119,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; SKX-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; SKX-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:148 SizeLat:52 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; SKX-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:148 SizeLat:52 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_gather'
@@ -1150,10 +1150,10 @@ define i32 @masked_gather(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m8
; VBMI2-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:269 SizeLat:173 for: %V32I8 = call <32 x i8> @llvm.masked.gather.v32i8.v32p0(<32 x ptr> align 1 undef, <32 x i1> %m32, <32 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:134 SizeLat:86 for: %V16I8 = call <16 x i8> @llvm.masked.gather.v16i8.v16p0(<16 x ptr> align 1 undef, <16 x i1> %m16, <16 x i8> undef)
; VBMI2-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:67 SizeLat:43 for: %V8I8 = call <8 x i8> @llvm.masked.gather.v8i8.v8p0(<8 x ptr> align 1 undef, <8 x i1> %m8, <8 x i8> undef)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:68 SizeLat:20 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:136 SizeLat:40 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16I64 = call <16 x i64> @llvm.masked.gather.v16i64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:148 SizeLat:52 for: %ME_V32I64 = call <32 x i64> @llvm.masked.gather.v32i64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x i64> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:72 SizeLat:24 for: %ME_V16F64 = call <16 x double> @llvm.masked.gather.v16f64.v16p0(<16 x ptr> align 1 poison, <16 x i1> %m16, <16 x double> poison)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:148 SizeLat:52 for: %ME_V32F64 = call <32 x double> @llvm.masked.gather.v32f64.v32p0(<32 x ptr> align 1 poison, <32 x i1> %m32, <32 x double> poison)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
%V8F64 = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> undef, i32 1, <8 x i1> %m8, <8 x double> undef)
@@ -1374,10 +1374,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; KNL-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; KNL-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
-; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
-; KNL-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; KNL-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:12 Lat:48 SizeLat:48 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; KNL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; KNL-NEXT: Cost Model: Found costs of RThru:48 CodeSize:12 Lat:48 SizeLat:48 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; KNL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; SKX-LABEL: 'masked_scatter'
@@ -1405,10 +1405,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; SKX-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; SKX-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
-; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
-; SKX-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; SKX-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:52 SizeLat:52 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; SKX-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; SKX-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:52 SizeLat:52 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; SKX-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
; VBMI2-LABEL: 'masked_scatter'
@@ -1436,10 +1436,10 @@ define i32 @masked_scatter(<1 x i1> %m1, <2 x i1> %m2, <4 x i1> %m4, <8 x i1> %m
; VBMI2-NEXT: Cost Model: Found costs of RThru:141 CodeSize:173 Lat:173 SizeLat:173 for: call void @llvm.masked.scatter.v32i8.v32p0(<32 x i8> undef, <32 x ptr> align 1 undef, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:70 CodeSize:86 Lat:86 SizeLat:86 for: call void @llvm.masked.scatter.v16i8.v16p0(<16 x i8> undef, <16 x ptr> align 1 undef, <16 x i1> %m16)
; VBMI2-NEXT: Cost Model: Found costs of RThru:35 CodeSize:43 Lat:43 SizeLat:43 for: call void @llvm.masked.scatter.v8i8.v8p0(<8 x i8> undef, <8 x ptr> align 1 undef, <8 x i1> %m8)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:20 CodeSize:2 Lat:20 SizeLat:20 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
-; VBMI2-NEXT: Cost Model: Found costs of RThru:40 CodeSize:4 Lat:40 SizeLat:40 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16i64.v16p0(<16 x i64> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:52 SizeLat:52 for: call void @llvm.masked.scatter.v32i64.v32p0(<32 x i64> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:24 CodeSize:6 Lat:24 SizeLat:24 for: call void @llvm.masked.scatter.v16f64.v16p0(<16 x double> poison, <16 x ptr> align 1 poison, <16 x i1> %m16)
+; VBMI2-NEXT: Cost Model: Found costs of RThru:52 CodeSize:16 Lat:52 SizeLat:52 for: call void @llvm.masked.scatter.v32f64.v32p0(<32 x double> poison, <32 x ptr> align 1 poison, <32 x i1> %m32)
; VBMI2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 0
;
call void @llvm.masked.scatter.v8f64.v8p0(<8 x double> undef, <8 x ptr> undef, i32 1, <8 x i1> %m8)
@@ -2291,7 +2291,7 @@ define <16 x float> @test_gather_16f32_var_mask(ptr %base, <16 x i32> %ind, <16
; SKL-LABEL: 'test_gather_16f32_var_mask'
; SKL-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %sext_ind = sext <16 x i32> %ind to <16 x i64>
; SKL-NEXT: Cost Model: Found costs of 0 for: %gep.v = getelementptr float, ptr %base, <16 x i64> %sext_ind
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x float> %res
;
; AVX512-LABEL: 'test_gather_16f32_var_mask'
@@ -2335,7 +2335,7 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x ptr> %ptrs, <16 x i32>
; SKL-LABEL: 'test_gather_16f32_ra_var_mask'
; SKL-NEXT: Cost Model: Found costs of RThru:10 CodeSize:1 Lat:1 SizeLat:1 for: %sext_ind = sext <16 x i32> %ind to <16 x i64>
; SKL-NEXT: Cost Model: Found costs of 0 for: %gep.v = getelementptr float, <16 x ptr> %ptrs, <16 x i64> %sext_ind
-; SKL-NEXT: Cost Model: Found costs of RThru:24 CodeSize:4 Lat:72 SizeLat:24 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
+; SKL-NEXT: Cost Model: Found costs of RThru:28 CodeSize:8 Lat:76 SizeLat:28 for: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0(<16 x ptr> align 4 %gep.v, <16 x i1> %mask, <16 x float> undef)
; SKL-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret <16 x float> %res
;
; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
diff --git a/llvm/test/Analysis/CostModel/X86/select.ll b/llvm/test/Analysis/CostModel/X86/select.ll
index 36072106c099c..486b7432ff02b 100644
--- a/llvm/test/Analysis/CostModel/X86/select.ll
+++ b/llvm/test/Analysis/CostModel/X86/select.ll
@@ -114,8 +114,8 @@ define i32 @test_select() {
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX512F-NEXT: Cost Model: Found costs of RThru:2 CodeSize:4 Lat:2 SizeLat:4 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
-; AVX512F-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
-; AVX512F-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
+; AVX512F-NEXT: Cost Model: Found costs of 6 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX512F-NEXT: Cost Model: Found costs of 12 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; AVX512BW-LABEL: 'test_select'
@@ -135,8 +135,8 @@ define i32 @test_select() {
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
-; AVX512BW-NEXT: Cost Model: Found costs of 2 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
-; AVX512BW-NEXT: Cost Model: Found costs of 4 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
+; AVX512BW-NEXT: Cost Model: Found costs of 6 for: %ME_V16I64 = select <16 x i1> poison, <16 x i64> poison, <16 x i64> poison
+; AVX512BW-NEXT: Cost Model: Found costs of 16 for: %ME_V32I64 = select <32 x i1> poison, <32 x i64> poison, <32 x i64> poison
; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SLM-LABEL: 'test_select'
@@ -238,18 +238,31 @@ define i32 @test_select_fp() {
; AVX2-NEXT: Cost Model: Found costs of RThru:16 CodeSize:8 Lat:16 SizeLat:16 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
; AVX2-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
-; AVX512-LABEL: 'test_select_fp'
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %F64 = select i1 undef, double undef, double undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V2F64 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V4F64 = select <4 x i1> undef, <4 x double> undef, <4 x double> undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V8F64 = select <8 x i1> undef, <8 x double> undef, <8 x double> undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %F32 = select i1 undef, float undef, float undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
-; AVX512-NEXT: Cost Model: Found costs of 1 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
-; AVX512-NEXT: Cost Model: Found costs of 2 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
-; AVX512-NEXT: Cost Model: Found costs of 4 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
-; AVX512-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+; AVX512F-LABEL: 'test_select_fp'
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %F64 = select i1 undef, double undef, double undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V2F64 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V4F64 = select <4 x i1> undef, <4 x double> undef, <4 x double> undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V8F64 = select <8 x i1> undef, <8 x double> undef, <8 x double> undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %F32 = select i1 undef, float undef, float undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
+; AVX512F-NEXT: Cost Model: Found costs of 1 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; AVX512F-NEXT: Cost Model: Found costs of 6 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; AVX512F-NEXT: Cost Model: Found costs of 12 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
+; AVX512F-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
+;
+; AVX512BW-LABEL: 'test_select_fp'
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %F64 = select i1 undef, double undef, double undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V2F64 = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V4F64 = select <4 x i1> undef, <4 x double> undef, <4 x double> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V8F64 = select <8 x i1> undef, <8 x double> undef, <8 x double> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %F32 = select i1 undef, float undef, float undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V4F32 = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V8F32 = select <8 x i1> undef, <8 x float> undef, <8 x float> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 1 for: %V16F32 = select <16 x i1> undef, <16 x float> undef, <16 x float> undef
+; AVX512BW-NEXT: Cost Model: Found costs of 6 for: %ME_V16F64 = select <16 x i1> poison, <16 x double> poison, <16 x double> poison
+; AVX512BW-NEXT: Cost Model: Found costs of 16 for: %ME_V32F64 = select <32 x i1> poison, <32 x double> poison, <32 x double> poison
+; AVX512BW-NEXT: Cost Model: Found costs of RThru:0 CodeSize:1 Lat:1 SizeLat:1 for: ret i32 undef
;
; SLM-LABEL: 'test_select_fp'
; SLM-NEXT: Cost Model: Found costs of RThru:2 CodeSize:1 Lat:2 SizeLat:2 for: %F64 = select i1 undef, double undef, double undef
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
index 8f5da77027970..35413ef34c7da 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i32-with-i8-index.ll
@@ -41,20 +41,20 @@ define void @test() {
;
; AVX2-FASTGATHER-LABEL: 'test'
; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, ptr %inB, align 4
-; AVX2-FASTGATHER: Cost of 4 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 6 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 12 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 24 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 48 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
+; AVX2-FASTGATHER: Cost of 4 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 6 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 12 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 28 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 60 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, ptr %inB, align 4
-; AVX512: Cost of 8 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 17 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 10 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 18 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 36 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 72 for VF 64: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
+; AVX512: Cost of 8 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 17 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 10 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 18 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 40 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 84 for VF 64: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
index 6801a549e19c4..3c0c960b91d06 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-gather-i64-with-i8-index.ll
@@ -41,20 +41,20 @@ define void @test() {
;
; AVX2-FASTGATHER-LABEL: 'test'
; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
-; AVX2-FASTGATHER: Cost of 4 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 6 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 12 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 24 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX2-FASTGATHER: Cost of 48 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
+; AVX2-FASTGATHER: Cost of 4 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 6 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 16 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 36 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2-FASTGATHER: Cost of 76 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
-; AVX512: Cost of 8 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 18 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 10 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 20 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 40 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
-; AVX512: Cost of 80 for VF 64: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad>
+; AVX512: Cost of 8 for VF 2: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 18 for VF 4: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 10 for VF 8: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 24 for VF 16: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 52 for VF 32: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 108 for VF 64: WIDEN ir<%valB.loaded> = load ir<%inB>, ir<%canLoad> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
index 73b83ab686802..0c8608fef07ad 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i16.ll
@@ -45,7 +45,7 @@ define void @test(ptr %B) {
; AVX512: Cost of 1 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
index c41a0a25b3d26..be00ac690601d 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i32.ll
@@ -27,16 +27,16 @@ define void @test(ptr %B) {
; AVX1: Cost of 3 for VF 2: WIDEN ir<%valB.loaded> = load vp<[[VP6:%[0-9]+]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 2 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 2 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 4 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 8 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 8 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 20 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, ptr %inB, align 4
; AVX2: Cost of 3 for VF 2: WIDEN ir<%valB.loaded> = load vp<[[VP6:%[0-9]+]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 2 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 2 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 4 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 8 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 8 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 20 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, ptr %inB, align 4
@@ -44,8 +44,8 @@ define void @test(ptr %B) {
; AVX512: Cost of 1 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 4 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 16 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
index 8a08b731bc848..1d1bd1451fdce 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-load-i64.ll
@@ -26,26 +26,26 @@ define void @test(ptr %B) {
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
; AVX1: Cost of 2 for VF 2: WIDEN ir<%valB.loaded> = load vp<[[VP6:%[0-9]+]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 2 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 4 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 8 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 16 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 8 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 20 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 44 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
; AVX2: Cost of 2 for VF 2: WIDEN ir<%valB.loaded> = load vp<[[VP6:%[0-9]+]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 2 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 4 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 8 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 16 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 8 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 20 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 44 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, ptr %inB, align 8
; AVX512: Cost of 1 for VF 2: WIDEN ir<%valB.loaded> = load vp<[[VP6:%[0-9]+]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 4: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 8: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 4 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 8 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 16: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 16 for VF 32: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 36 for VF 64: WIDEN ir<%valB.loaded> = load vp<[[VP6]]>, ir<%canLoad> (!alias.scope !7, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
index 6959fea2d512b..e3702ba9c1617 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i32-with-i8-index.ll
@@ -50,10 +50,10 @@ define void @test() {
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
; AVX512: Cost of 5 for VF 2: REPLICATE store ir<%valB>, ir<%out>
; AVX512: Cost of 10.5 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX512: Cost of 10 for VF 8: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 18 for VF 16: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 36 for VF 32: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 72 for VF 64: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
+; AVX512: Cost of 10 for VF 8: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 18 for VF 16: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 40 for VF 32: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 84 for VF 64: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
index 41ae89933204e..112a0d9376234 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-scatter-i64-with-i8-index.ll
@@ -50,10 +50,10 @@ define void @test() {
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
; AVX512: Cost of 5 for VF 2: REPLICATE store ir<%valB>, ir<%out>
; AVX512: Cost of 11 for VF 4: REPLICATE store ir<%valB>, ir<%out>
-; AVX512: Cost of 10 for VF 8: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 20 for VF 16: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 40 for VF 32: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
-; AVX512: Cost of 80 for VF 64: WIDEN store ir<%out>, ir<%valB>, ir<%canStore>
+; AVX512: Cost of 10 for VF 8: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 24 for VF 16: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 52 for VF 32: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 108 for VF 64: WIDEN store ir<%out>, ir<%valB>, ir<%canStore> (!vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
index 800c9eae5e2bb..8ccb0994c63c0 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i16.ll
@@ -45,7 +45,7 @@ define void @test(ptr %C) {
; AVX512: Cost of 1 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
index 3e6b1f09aca06..1711b0177b6fe 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i32.ll
@@ -34,16 +34,16 @@ define void @test(ptr %C) {
; AVX1: Cost of 9 for VF 2: WIDEN store vp<[[VP7:%[0-9]+]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 8 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 8 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 16 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 32 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 20 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 44 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
; AVX2: Cost of 9 for VF 2: WIDEN store vp<[[VP7:%[0-9]+]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 8 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 8 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 16 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 32 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 20 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 44 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, ptr %out, align 4
@@ -51,8 +51,8 @@ define void @test(ptr %C) {
; AVX512: Cost of 1 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 4 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 16 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
index 5447e15b3da13..25bd3bf7d9a30 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/CostModel/masked-store-i64.ll
@@ -33,26 +33,26 @@ define void @test(ptr %C) {
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
; AVX1: Cost of 8 for VF 2: WIDEN store vp<[[VP7:%[0-9]+]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX1: Cost of 8 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 16 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 32 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX1: Cost of 64 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 20 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 44 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX1: Cost of 92 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX2-LABEL: 'test'
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
; AVX2: Cost of 8 for VF 2: WIDEN store vp<[[VP7:%[0-9]+]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX2: Cost of 8 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 16 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 32 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX2: Cost of 64 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 20 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 44 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX2: Cost of 92 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
; AVX512-LABEL: 'test'
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, ptr %out, align 8
; AVX512: Cost of 1 for VF 2: WIDEN store vp<[[VP7:%[0-9]+]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 4: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
; AVX512: Cost of 1 for VF 8: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 2 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 4 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
-; AVX512: Cost of 8 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 6 for VF 16: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 16 for VF 32: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
+; AVX512: Cost of 36 for VF 64: WIDEN store vp<[[VP7]]>, ir<%valB>, ir<%canStore> (!alias.scope !8, !noalias !4, !vplan.execution.frequency 11529215046068469760 (62.5%, estimated))
;
entry:
br label %for.body
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll b/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll
index dfd2c4a217dd7..ab56ca7591bfe 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/gep-nodes-with-non-gep-inst.ll
@@ -9,9 +9,17 @@ define void @test() {
; CHECK-NEXT: [[COND_IN_V:%.*]] = select i1 false, ptr null, ptr null
; CHECK-NEXT: br label [[BB:%.*]]
; CHECK: bb:
-; CHECK-NEXT: [[TMP0:%.*]] = call <13 x i64> @llvm.masked.load.v13i64.p0(ptr align 8 [[COND_IN_V]], <13 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <13 x i64> poison)
-; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <13 x i64> [[TMP0]], <13 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[COND_IN_V]], align 8
+; CHECK-NEXT: [[BV:%.*]] = icmp eq i64 [[V]], 0
+; CHECK-NEXT: [[IN_1:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 4
+; CHECK-NEXT: [[V_1:%.*]] = load i64, ptr [[IN_1]], align 8
+; CHECK-NEXT: [[BV_1:%.*]] = icmp eq i64 [[V_1]], 0
+; CHECK-NEXT: [[IN_2:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 8
+; CHECK-NEXT: [[V_2:%.*]] = load i64, ptr [[IN_2]], align 8
+; CHECK-NEXT: [[BV_2:%.*]] = icmp eq i64 [[V_2]], 0
+; CHECK-NEXT: [[IN_3:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 12
+; CHECK-NEXT: [[V_3:%.*]] = load i64, ptr [[IN_3]], align 8
+; CHECK-NEXT: [[BV_3:%.*]] = icmp eq i64 [[V_3]], 0
; CHECK-NEXT: ret void
;
; CHECK-SLP-THRESHOLD-LABEL: define void @test
@@ -20,9 +28,15 @@ define void @test() {
; CHECK-SLP-THRESHOLD-NEXT: [[COND_IN_V:%.*]] = select i1 false, ptr null, ptr null
; CHECK-SLP-THRESHOLD-NEXT: br label [[BB:%.*]]
; CHECK-SLP-THRESHOLD: bb:
-; CHECK-SLP-THRESHOLD-NEXT: [[TMP0:%.*]] = call <13 x i64> @llvm.masked.load.v13i64.p0(ptr align 8 [[COND_IN_V]], <13 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true>, <13 x i64> poison)
-; CHECK-SLP-THRESHOLD-NEXT: [[TMP1:%.*]] = shufflevector <13 x i64> [[TMP0]], <13 x i64> poison, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
-; CHECK-SLP-THRESHOLD-NEXT: [[TMP2:%.*]] = icmp eq <4 x i64> [[TMP1]], zeroinitializer
+; CHECK-SLP-THRESHOLD-NEXT: [[IN_2:%.*]] = getelementptr i64, ptr [[COND_IN_V]], i64 8
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP0:%.*]] = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 8 [[COND_IN_V]], <5 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true>, <5 x i64> poison)
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP1:%.*]] = shufflevector <5 x i64> [[TMP0]], <5 x i64> poison, <2 x i32> <i32 4, i32 0>
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP2:%.*]] = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 8 [[IN_2]], <5 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true>, <5 x i64> poison)
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP3:%.*]] = shufflevector <5 x i64> [[TMP2]], <5 x i64> poison, <2 x i32> <i32 4, i32 0>
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP3]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP5:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP6:%.*]] = shufflevector <5 x i64> [[TMP2]], <5 x i64> [[TMP0]], <4 x i32> <i32 4, i32 0, i32 9, i32 5>
+; CHECK-SLP-THRESHOLD-NEXT: [[TMP7:%.*]] = icmp eq <4 x i64> [[TMP6]], zeroinitializer
; CHECK-SLP-THRESHOLD-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
index 02d7698fc040e..01ec3f173ce49 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629-inseltpoison.ll
@@ -286,11 +286,30 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
;
; AVX2-LABEL: define void @gather_load_3(
; AVX2-SAME: ptr noalias captures(none) [[TMP0:%.*]], ptr noalias readonly captures(none) [[TMP1:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT: [[TMP3:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr align 4 [[TMP1]], <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[SHORT_TBAA0]]
-; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <22 x i32> [[TMP3]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
-; AVX2-NEXT: [[TMP5:%.*]] = add <8 x i32> [[TMP4]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
-; AVX2-NEXT: store <8 x i32> [[TMP6]], ptr [[TMP0]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
+; AVX2-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
+; AVX2-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
+; AVX2-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP12]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison), !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
+; AVX2-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
+; AVX2-NEXT: [[TMP18:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP5]], i64 1
+; AVX2-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP7]], i64 2
+; AVX2-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP9]], i64 3
+; AVX2-NEXT: [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP11]], i64 4
+; AVX2-NEXT: [[TMP22:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> <i32 1, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[TMP23:%.*]] = shufflevector <8 x i32> [[TMP21]], <8 x i32> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 poison>
+; AVX2-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP16]], i64 7
+; AVX2-NEXT: [[TMP25:%.*]] = add <8 x i32> [[TMP24]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT: store <8 x i32> [[TMP25]], ptr [[TMP0]], align 4, !tbaa [[SHORT_TBAA0]]
; AVX2-NEXT: ret void
;
; AVX512F-LABEL: define void @gather_load_3(
@@ -426,11 +445,30 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
;
; AVX2-LABEL: define void @gather_load_4(
; AVX2-SAME: ptr noalias captures(none) [[T0:%.*]], ptr noalias readonly captures(none) [[T1:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT: [[TMP1:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr align 4 [[T1]], <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[SHORT_TBAA0]]
-; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <22 x i32> [[TMP1]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
-; AVX2-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
-; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
-; AVX2-NEXT: store <8 x i32> [[TMP4]], ptr [[T0]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 11
+; AVX2-NEXT: [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX2-NEXT: [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX2-NEXT: [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX2-NEXT: [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX2-NEXT: [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
+; AVX2-NEXT: [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T15:%.*]] = load i32, ptr [[T14]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T19:%.*]] = load i32, ptr [[T18]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[T26]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison), !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
+; AVX2-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T7]], i64 1
+; AVX2-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T11]], i64 2
+; AVX2-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T15]], i64 3
+; AVX2-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T19]], i64 4
+; AVX2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 1, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 poison>
+; AVX2-NEXT: [[TMP10:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[T31]], i64 7
+; AVX2-NEXT: [[TMP11:%.*]] = add <8 x i32> [[TMP10]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT: store <8 x i32> [[TMP11]], ptr [[T0]], align 4, !tbaa [[SHORT_TBAA0]]
; AVX2-NEXT: ret void
;
; AVX512F-LABEL: define void @gather_load_4(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
index d22e056b4bba4..2a8480baf0755 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/pr47629.ll
@@ -286,11 +286,30 @@ define void @gather_load_3(ptr noalias nocapture %0, ptr noalias nocapture reado
;
; AVX2-LABEL: define void @gather_load_3(
; AVX2-SAME: ptr noalias captures(none) [[TMP0:%.*]], ptr noalias readonly captures(none) [[TMP1:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT: [[TMP3:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr align 4 [[TMP1]], <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[SHORT_TBAA0]]
-; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <22 x i32> [[TMP3]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
-; AVX2-NEXT: [[TMP5:%.*]] = add <8 x i32> [[TMP4]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
-; AVX2-NEXT: [[TMP6:%.*]] = shufflevector <8 x i32> [[TMP5]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
-; AVX2-NEXT: store <8 x i32> [[TMP6]], ptr [[TMP0]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP3:%.*]] = load i32, ptr [[TMP1]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 11
+; AVX2-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 4
+; AVX2-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 15
+; AVX2-NEXT: [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 18
+; AVX2-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 6
+; AVX2-NEXT: [[TMP13:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[TMP12]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison), !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i64 21
+; AVX2-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP17:%.*]] = insertelement <8 x i32> poison, i32 [[TMP3]], i64 0
+; AVX2-NEXT: [[TMP18:%.*]] = insertelement <8 x i32> [[TMP17]], i32 [[TMP5]], i64 1
+; AVX2-NEXT: [[TMP19:%.*]] = insertelement <8 x i32> [[TMP18]], i32 [[TMP7]], i64 2
+; AVX2-NEXT: [[TMP20:%.*]] = insertelement <8 x i32> [[TMP19]], i32 [[TMP9]], i64 3
+; AVX2-NEXT: [[TMP21:%.*]] = insertelement <8 x i32> [[TMP20]], i32 [[TMP11]], i64 4
+; AVX2-NEXT: [[TMP22:%.*]] = shufflevector <2 x i32> [[TMP14]], <2 x i32> poison, <8 x i32> <i32 1, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[TMP23:%.*]] = shufflevector <8 x i32> [[TMP21]], <8 x i32> [[TMP22]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 poison>
+; AVX2-NEXT: [[TMP24:%.*]] = insertelement <8 x i32> [[TMP23]], i32 [[TMP16]], i64 7
+; AVX2-NEXT: [[TMP25:%.*]] = add <8 x i32> [[TMP24]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT: store <8 x i32> [[TMP25]], ptr [[TMP0]], align 4, !tbaa [[SHORT_TBAA0]]
; AVX2-NEXT: ret void
;
; AVX512F-LABEL: define void @gather_load_3(
@@ -426,11 +445,30 @@ define void @gather_load_4(ptr noalias nocapture %t0, ptr noalias nocapture read
;
; AVX2-LABEL: define void @gather_load_4(
; AVX2-SAME: ptr noalias captures(none) [[T0:%.*]], ptr noalias readonly captures(none) [[T1:%.*]]) #[[ATTR0]] {
-; AVX2-NEXT: [[TMP1:%.*]] = call <22 x i32> @llvm.masked.load.v22i32.p0(ptr align 4 [[T1]], <22 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true>, <22 x i32> poison), !tbaa [[SHORT_TBAA0]]
-; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <22 x i32> [[TMP1]], <22 x i32> poison, <8 x i32> <i32 0, i32 4, i32 6, i32 9, i32 11, i32 15, i32 18, i32 21>
-; AVX2-NEXT: [[TMP3:%.*]] = add <8 x i32> [[TMP2]], <i32 1, i32 3, i32 3, i32 2, i32 2, i32 4, i32 1, i32 4>
-; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 6, i32 3, i32 2, i32 7>
-; AVX2-NEXT: store <8 x i32> [[TMP4]], ptr [[T0]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T6:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 11
+; AVX2-NEXT: [[T10:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 4
+; AVX2-NEXT: [[T14:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 15
+; AVX2-NEXT: [[T18:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 18
+; AVX2-NEXT: [[T26:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 6
+; AVX2-NEXT: [[T30:%.*]] = getelementptr inbounds i32, ptr [[T1]], i64 21
+; AVX2-NEXT: [[T3:%.*]] = load i32, ptr [[T1]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T7:%.*]] = load i32, ptr [[T6]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T11:%.*]] = load i32, ptr [[T10]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T15:%.*]] = load i32, ptr [[T14]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[T19:%.*]] = load i32, ptr [[T18]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 4 [[T26]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison), !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 0, i32 3>
+; AVX2-NEXT: [[T31:%.*]] = load i32, ptr [[T30]], align 4, !tbaa [[SHORT_TBAA0]]
+; AVX2-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[T3]], i64 0
+; AVX2-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[T7]], i64 1
+; AVX2-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[T11]], i64 2
+; AVX2-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[T15]], i64 3
+; AVX2-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[T19]], i64 4
+; AVX2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <8 x i32> <i32 1, i32 0, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[TMP9:%.*]] = shufflevector <8 x i32> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 9, i32 poison>
+; AVX2-NEXT: [[TMP10:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[T31]], i64 7
+; AVX2-NEXT: [[TMP11:%.*]] = add <8 x i32> [[TMP10]], <i32 1, i32 2, i32 3, i32 4, i32 1, i32 2, i32 3, i32 4>
+; AVX2-NEXT: store <8 x i32> [[TMP11]], ptr [[T0]], align 4, !tbaa [[SHORT_TBAA0]]
; AVX2-NEXT: ret void
;
; AVX512F-LABEL: define void @gather_load_4(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
index 23a901fd768b3..0a34004b77fa3 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark-masked-loads-consecutive-loads-same-ptr.ll
@@ -15,9 +15,16 @@
define void @test(ptr noalias %p, ptr noalias %p1) {
; CHECK-LABEL: @test(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP0:%.*]] = call <35 x i32> @llvm.masked.load.v35i32.p0(ptr align 4 [[P:%.*]], <35 x i1> <i1 true, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true>, <35 x i32> poison)
-; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <35 x i32> [[TMP0]], <35 x i32> poison, <4 x i32> <i32 0, i32 32, i32 33, i32 34>
+; CHECK-NEXT: [[I:%.*]] = load i32, ptr [[P:%.*]], align 4
+; CHECK-NEXT: [[ARRAYIDX4:%.*]] = getelementptr i32, ptr [[P]], i64 32
+; CHECK-NEXT: [[I2:%.*]] = load i32, ptr [[ARRAYIDX4]], align 4
+; CHECK-NEXT: [[ARRAYIDX11:%.*]] = getelementptr i32, ptr [[P]], i64 33
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[ARRAYIDX11]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[I]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[I2]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP6]], <4 x i32> <i32 0, i32 1, i32 4, i32 5>
; CHECK-NEXT: [[TMP5:%.*]] = add nsw <4 x i32> [[TMP4]], [[TMP1]]
; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr [[P1:%.*]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll b/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
index 77084f5b97e7d..39acbabf94ac0 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/scatter-vectorize-reused-pointer.ll
@@ -5,12 +5,16 @@ define void @test(i1 %c, ptr %arg) {
; CHECK-LABEL: @test(
; CHECK-NEXT: br i1 [[C:%.*]], label [[IF:%.*]], label [[ELSE:%.*]]
; CHECK: if:
-; CHECK-NEXT: [[TMP1:%.*]] = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 8 [[ARG:%.*]], <5 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true>, <5 x i64> poison)
-; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <5 x i64> [[TMP1]], <5 x i64> poison, <4 x i32> <i32 0, i32 1, i32 3, i32 4>
+; CHECK-NEXT: [[ARG2_2:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 24
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[ARG]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i64>, ptr [[ARG2_2]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> [[TMP1]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
; CHECK-NEXT: br label [[JOIN:%.*]]
; CHECK: else:
-; CHECK-NEXT: [[TMP3:%.*]] = call <5 x i64> @llvm.masked.load.v5i64.p0(ptr align 8 [[ARG]], <5 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true>, <5 x i64> poison)
-; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <5 x i64> [[TMP3]], <5 x i64> poison, <4 x i32> <i32 0, i32 1, i32 3, i32 4>
+; CHECK-NEXT: [[ARG_2:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 24
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i64>, ptr [[ARG]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = load <2 x i64>, ptr [[ARG_2]], align 8
+; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP4]], <4 x i32> <i32 1, i32 0, i32 3, i32 2>
; CHECK-NEXT: br label [[JOIN]]
; CHECK: join:
; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i64> [ [[TMP6]], [[IF]] ], [ [[TMP12]], [[ELSE]] ]
More information about the llvm-commits
mailing list