[llvm] db848fb - [NFC][LV][X86] Improve test coverage for masked mem ops
Roman Lebedev via llvm-commits
llvm-commits at lists.llvm.org
Wed Oct 27 03:37:39 PDT 2021
Author: Roman Lebedev
Date: 2021-10-27T13:36:04+03:00
New Revision: db848fbf671d99256b2840e19ed3b0b349069e48
URL: https://github.com/llvm/llvm-project/commit/db848fbf671d99256b2840e19ed3b0b349069e48
DIFF: https://github.com/llvm/llvm-project/commit/db848fbf671d99256b2840e19ed3b0b349069e48.diff
LOG: [NFC][LV][X86] Improve test coverage for masked mem ops
Added:
llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/masked-load-i16.ll
llvm/test/Analysis/CostModel/X86/masked-load-i32.ll
llvm/test/Analysis/CostModel/X86/masked-load-i64.ll
llvm/test/Analysis/CostModel/X86/masked-load-i8.ll
llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/masked-store-i16.ll
llvm/test/Analysis/CostModel/X86/masked-store-i32.ll
llvm/test/Analysis/CostModel/X86/masked-store-i64.ll
llvm/test/Analysis/CostModel/X86/masked-store-i8.ll
Modified:
llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll
llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll
Removed:
################################################################################
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll
index 942bad54eb3d9..fb1b8523eca30 100644
--- a/llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/gather-i16-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 444 for VF 32 For instruction: %valB = load i16, i16* %inB, align 2
; AVX512: LV: Found an estimated cost of 888 for VF 64 For instruction: %valB = load i16, i16* %inB, align 2
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i16, i16* %inB, align 4
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i16, i16* %inB, align 2
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll
index 6cca9cf32fc82..98ba241b743e3 100644
--- a/llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/gather-i64-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 40 for VF 32 For instruction: %valB = load i64, i64* %inB, align 8
; AVX512: LV: Found an estimated cost of 80 for VF 64 For instruction: %valB = load i64, i64* %inB, align 8
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %v0 = load i8, i8* %in0, align 2
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i64, i64* %inB, align 8
define void @test() {
entry:
diff --git a/llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll
index d7a21f5880c10..6c1cf6192f7a3 100644
--- a/llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/gather-i8-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 442 for VF 32 For instruction: %valB = load i8, i8* %inB, align 1
; AVX512: LV: Found an estimated cost of 884 for VF 64 For instruction: %valB = load i8, i8* %inB, align 1
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i8, i8* %inB, align 4
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i8, i8* %inB, align 1
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
index 70c375182d262..62412a5d1af0b 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i32-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 36 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4
; AVX512: LV: Found an estimated cost of 72 for VF 64 For instruction: %valB.loaded = load i32, i32* %inB, align 4
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i32, i32* %inB, align 4
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i32, i32* %inB, align 4
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
new file mode 100644
index 0000000000000..b8eba8b0327bb
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-gather-i64-with-i8-index.ll
@@ -0,0 +1,93 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i64] zeroinitializer, align 128
+ at C = global [1024 x i64] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 6 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 12 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 24 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 48 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 10 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 24 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 20 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 40 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 80 for VF 64 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i64, i64* %inB, align 8
+define void @test() {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canLoad = icmp ne i8 %valA, 0
+ br i1 %canLoad, label %load, label %mask
+
+load:
+ %valA.ext = sext i8 %valA to i64
+ %inB = getelementptr inbounds [1024 x i64], [1024 x i64]* @B, i64 0, i64 %valA.ext
+ %valB.loaded = load i64, i64* %inB
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %valB = phi i64 [ %valB.loaded, %load ], [ 0, %mask ]
+ %out = getelementptr inbounds [1024 x i64], [1024 x i64]* @C, i64 0, i64 %iv
+ store i64 %valB, i64* %out
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll
new file mode 100644
index 0000000000000..c8c3078f1625c
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-load-i16.ll
@@ -0,0 +1,91 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at C = global [1024 x i16] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE2: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE2: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE2: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE2: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE42: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE42: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE42: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; SSE42: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX1: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX1: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX1: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX1: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 32 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 64 For instruction: %valB.loaded = load i16, i16* %inB, align 2
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i16, i16* %inB, align 2
+define void @test([1024 x i16]* %B) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canLoad = icmp ne i8 %valA, 0
+ br i1 %canLoad, label %load, label %mask
+
+load:
+ %inB = getelementptr inbounds [1024 x i16], [1024 x i16]* %B, i64 0, i64 %iv
+ %valB.loaded = load i16, i16* %inB
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %valB = phi i16 [ %valB.loaded, %load ], [ 0, %mask ]
+ %out = getelementptr inbounds [1024 x i16], [1024 x i16]* @C, i64 0, i64 %iv
+ store i16 %valB, i16* %out
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i32.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i32.ll
new file mode 100644
index 0000000000000..f74c9f044d0ba
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-load-i32.ll
@@ -0,0 +1,91 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at C = global [1024 x i32] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE2: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE2: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE2: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE2: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE42: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE42: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE42: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; SSE42: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX1: LV: Found an estimated cost of 3 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX1: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX1: LV: Found an estimated cost of 2 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX1: LV: Found an estimated cost of 4 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX1: LV: Found an estimated cost of 8 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 4 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 2 for VF 32 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+; AVX512: LV: Found an estimated cost of 4 for VF 64 For instruction: %valB.loaded = load i32, i32* %inB, align 4
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i32, i32* %inB, align 4
+define void @test([1024 x i32]* %B) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canLoad = icmp ne i8 %valA, 0
+ br i1 %canLoad, label %load, label %mask
+
+load:
+ %inB = getelementptr inbounds [1024 x i32], [1024 x i32]* %B, i64 0, i64 %iv
+ %valB.loaded = load i32, i32* %inB
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %valB = phi i32 [ %valB.loaded, %load ], [ 0, %mask ]
+ %out = getelementptr inbounds [1024 x i32], [1024 x i32]* @C, i64 0, i64 %iv
+ store i32 %valB, i32* %out
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i64.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i64.ll
new file mode 100644
index 0000000000000..c5a7825348e98
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-load-i64.ll
@@ -0,0 +1,91 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at C = global [1024 x i64] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE2: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; SSE42: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 4 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 8 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX1: LV: Found an estimated cost of 16 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 16 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 16 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 2 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 4 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 2 for VF 16 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 4 for VF 32 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+; AVX512: LV: Found an estimated cost of 8 for VF 64 For instruction: %valB.loaded = load i64, i64* %inB, align 8
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i64, i64* %inB, align 8
+define void @test([1024 x i64]* %B) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canLoad = icmp ne i8 %valA, 0
+ br i1 %canLoad, label %load, label %mask
+
+load:
+ %inB = getelementptr inbounds [1024 x i64], [1024 x i64]* %B, i64 0, i64 %iv
+ %valB.loaded = load i64, i64* %inB
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %valB = phi i64 [ %valB.loaded, %load ], [ 0, %mask ]
+ %out = getelementptr inbounds [1024 x i64], [1024 x i64]* @C, i64 0, i64 %iv
+ store i64 %valB, i64* %out
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll b/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll
new file mode 100644
index 0000000000000..fc540da587008
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-load-i8.ll
@@ -0,0 +1,91 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at C = global [1024 x i8] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE2: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE2: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE2: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE2: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE42: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE42: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE42: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; SSE42: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX1: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX1: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX1: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX1: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX1: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 3000000 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 4 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 8 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 32 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 64 For instruction: %valB.loaded = load i8, i8* %inB, align 1
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB.loaded = load i8, i8* %inB, align 1
+define void @test([1024 x i8]* %B) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canLoad = icmp ne i8 %valA, 0
+ br i1 %canLoad, label %load, label %mask
+
+load:
+ %inB = getelementptr inbounds [1024 x i8], [1024 x i8]* %B, i64 0, i64 %iv
+ %valB.loaded = load i8, i8* %inB
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %valB = phi i8 [ %valB.loaded, %load ], [ 0, %mask ]
+ %out = getelementptr inbounds [1024 x i8], [1024 x i8]* @C, i64 0, i64 %iv
+ store i8 %valB, i8* %out
+
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll
index 8718e709d1c3f..9a06888bcdac7 100644
--- a/llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/masked-scatter-i32-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 36 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
; AVX512: LV: Found an estimated cost of 72 for VF 64 For instruction: store i32 %valB, i32* %out, align 4
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: %valB = load i32, i32* %inB, align 4
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i32 %valB, i32* %out, align 4
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll
new file mode 100644
index 0000000000000..2e40fe52603c1
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-scatter-i64-with-i8-index.ll
@@ -0,0 +1,92 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i64] zeroinitializer, align 128
+ at C = global [1024 x i64] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 40 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 10 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 24 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 40 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 80 for VF 64 For instruction: store i64 %valB, i64* %out, align 8
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i64 %valB, i64* %out, align 8
+define void @test() {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inB = getelementptr inbounds [1024 x i64], [1024 x i64]* @B, i64 0, i64 %iv
+ %valB = load i64, i64* %inB
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canStore = icmp ne i8 %valA, 0
+ br i1 %canStore, label %store, label %mask
+
+store:
+ %valA.ext = sext i8 %valA to i64
+ %out = getelementptr inbounds [1024 x i64], [1024 x i64]* @C, i64 0, i64 %valA.ext
+ store i64 %valB, i64* %out
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i16.ll b/llvm/test/Analysis/CostModel/X86/masked-store-i16.ll
new file mode 100644
index 0000000000000..ca6805b4682d4
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-store-i16.ll
@@ -0,0 +1,90 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i16] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; SSE2: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; SSE2: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; SSE2: LV: Found an estimated cost of 16 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; SSE42: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; SSE42: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; SSE42: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; SSE42: LV: Found an estimated cost of 16 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; AVX1: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; AVX1: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; AVX1: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+; AVX1: LV: Found an estimated cost of 40 for VF 32 For instruction: store i16 %valB, i16* %out, align 2
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i16 %valB, i16* %out, align 2
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 20 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+; AVX2-FASTGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i16 %valB, i16* %out, align 2
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 4 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 1 for VF 32 For instruction: store i16 %valB, i16* %out, align 2
+; AVX512: LV: Found an estimated cost of 2 for VF 64 For instruction: store i16 %valB, i16* %out, align 2
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i16 %valB, i16* %out, align 2
+define void @test([1024 x i16]* %C) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inB = getelementptr inbounds [1024 x i16], [1024 x i16]* @B, i64 0, i64 %iv
+ %valB = load i16, i16* %inB
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canStore = icmp ne i8 %valA, 0
+ br i1 %canStore, label %store, label %mask
+
+store:
+ %out = getelementptr inbounds [1024 x i16], [1024 x i16]* %C, i64 0, i64 %iv
+ store i16 %valB, i16* %out
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i32.ll b/llvm/test/Analysis/CostModel/X86/masked-store-i32.ll
new file mode 100644
index 0000000000000..b9ac46f3bae3b
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-store-i32.ll
@@ -0,0 +1,90 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i32] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; SSE2: LV: Found an estimated cost of 5 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; SSE2: LV: Found an estimated cost of 11 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; SSE2: LV: Found an estimated cost of 22 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; SSE42: LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; SSE42: LV: Found an estimated cost of 5 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; SSE42: LV: Found an estimated cost of 11 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; SSE42: LV: Found an estimated cost of 22 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; AVX1: LV: Found an estimated cost of 8 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; AVX1: LV: Found an estimated cost of 8 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; AVX1: LV: Found an estimated cost of 16 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+; AVX1: LV: Found an estimated cost of 32 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 9 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 32 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 9 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+; AVX2-FASTGATHER: LV: Found an estimated cost of 32 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 2 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 4 for VF 64 For instruction: store i32 %valB, i32* %out, align 4
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i32 %valB, i32* %out, align 4
+define void @test([1024 x i32]* %C) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inB = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %iv
+ %valB = load i32, i32* %inB
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canStore = icmp ne i8 %valA, 0
+ br i1 %canStore, label %store, label %mask
+
+store:
+ %out = getelementptr inbounds [1024 x i32], [1024 x i32]* %C, i64 0, i64 %iv
+ store i32 %valB, i32* %out
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i64.ll b/llvm/test/Analysis/CostModel/X86/masked-store-i64.ll
new file mode 100644
index 0000000000000..4ac91c756a535
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-store-i64.ll
@@ -0,0 +1,90 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i64] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; SSE2: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 2 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 5 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 10 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; SSE42: LV: Found an estimated cost of 20 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 8 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 8 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 16 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 32 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX1: LV: Found an estimated cost of 64 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 16 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 32 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 64 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 16 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 32 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX2-FASTGATHER: LV: Found an estimated cost of 64 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 2 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 4 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 1 for VF 8 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 2 for VF 16 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 4 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
+; AVX512: LV: Found an estimated cost of 8 for VF 64 For instruction: store i64 %valB, i64* %out, align 8
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i64 %valB, i64* %out, align 8
+define void @test([1024 x i64]* %C) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inB = getelementptr inbounds [1024 x i64], [1024 x i64]* @B, i64 0, i64 %iv
+ %valB = load i64, i64* %inB
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canStore = icmp ne i8 %valA, 0
+ br i1 %canStore, label %store, label %mask
+
+store:
+ %out = getelementptr inbounds [1024 x i64], [1024 x i64]* %C, i64 0, i64 %iv
+ store i64 %valB, i64* %out
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/masked-store-i8.ll b/llvm/test/Analysis/CostModel/X86/masked-store-i8.ll
new file mode 100644
index 0000000000000..bdcd8058e92ec
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/masked-store-i8.ll
@@ -0,0 +1,90 @@
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse42 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE42
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,-fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-SLOWGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2,+fast-gather --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2-FASTGATHER
+; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+ at A = global [1024 x i8] zeroinitializer, align 128
+ at B = global [1024 x i8] zeroinitializer, align 128
+
+; CHECK: LV: Checking a loop in "test"
+;
+; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; SSE2: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; SSE2: LV: Found an estimated cost of 5 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; SSE2: LV: Found an estimated cost of 11 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; SSE2: LV: Found an estimated cost of 23 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+;
+; SSE42: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; SSE42: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; SSE42: LV: Found an estimated cost of 5 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; SSE42: LV: Found an estimated cost of 11 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; SSE42: LV: Found an estimated cost of 23 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+;
+; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; AVX1: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; AVX1: LV: Found an estimated cost of 4 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; AVX1: LV: Found an estimated cost of 8 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; AVX1: LV: Found an estimated cost of 16 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+; AVX1: LV: Found an estimated cost of 40 for VF 32 For instruction: store i8 %valB, i8* %out, align 1
+;
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-SLOWGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i8 %valB, i8* %out, align 1
+;
+; AVX2-FASTGATHER: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 4 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 8 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 16 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+; AVX2-FASTGATHER: LV: Found an estimated cost of 40 for VF 32 For instruction: store i8 %valB, i8* %out, align 1
+;
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 2 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 4 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 2 for VF 8 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 16 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 32 For instruction: store i8 %valB, i8* %out, align 1
+; AVX512: LV: Found an estimated cost of 1 for VF 64 For instruction: store i8 %valB, i8* %out, align 1
+;
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i8 %valB, i8* %out, align 1
+define void @test([1024 x i8]* %C) {
+entry:
+ br label %for.body
+
+for.body:
+ %iv = phi i64 [ 0, %entry ], [ %iv.next, %end ]
+
+ %inB = getelementptr inbounds [1024 x i8], [1024 x i8]* @B, i64 0, i64 %iv
+ %valB = load i8, i8* %inB
+
+ %inA = getelementptr inbounds [1024 x i8], [1024 x i8]* @A, i64 0, i64 %iv
+ %valA = load i8, i8* %inA
+ %canStore = icmp ne i8 %valA, 0
+ br i1 %canStore, label %store, label %mask
+
+store:
+ %out = getelementptr inbounds [1024 x i8], [1024 x i8]* %C, i64 0, i64 %iv
+ store i8 %valB, i8* %out
+ br label %end
+
+mask:
+ br label %end
+
+end:
+ %iv.next = add nuw nsw i64 %iv, 1
+ %cmp = icmp ult i64 %iv.next, 1024
+ br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll
index e9cefedafd7e7..d1b999d0c9278 100644
--- a/llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/scatter-i16-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 464 for VF 32 For instruction: store i16 %valB, i16* %out, align 2
; AVX512: LV: Found an estimated cost of 928 for VF 64 For instruction: store i16 %valB, i16* %out, align 2
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i16 %valB, i16* %out
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i16 %valB, i16* %out, align 2
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll
index 7bdc72d640f43..1fc3215190260 100644
--- a/llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/scatter-i32-with-i8-index.ll
@@ -49,15 +49,15 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX2-FASTGATHER: LV: Found an estimated cost of 224 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
; AVX2-FASTGATHER: LV: Found an estimated cost of 448 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
;
-; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 6 for VF 2 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 14 for VF 4 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 18 for VF 16 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 36 for VF 32 For instruction: store i32 %valB, i32* %out
-; AVX512: LV: Found an estimated cost of 72 for VF 64 For instruction: store i32 %valB, i32* %out
+; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 6 for VF 2 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 14 for VF 4 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 10 for VF 8 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 18 for VF 16 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 36 for VF 32 For instruction: store i32 %valB, i32* %out, align 4
+; AVX512: LV: Found an estimated cost of 72 for VF 64 For instruction: store i32 %valB, i32* %out, align 4
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i32 %valB, i32* %out
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i32 %valB, i32* %out, align 4
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll
index 86a4cd1ec3914..5eb697212a556 100644
--- a/llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/scatter-i64-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 40 for VF 32 For instruction: store i64 %valB, i64* %out, align 8
; AVX512: LV: Found an estimated cost of 80 for VF 64 For instruction: store i64 %valB, i64* %out, align 8
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i64 %valB, i64* %out
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i64 %valB, i64* %out, align 8
define void @test() {
entry:
br label %for.body
diff --git a/llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll b/llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll
index 61039e71bbe38..69c1c0305490d 100644
--- a/llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll
+++ b/llvm/test/Analysis/CostModel/X86/scatter-i8-with-i8-index.ll
@@ -57,7 +57,7 @@ target triple = "x86_64-unknown-linux-gnu"
; AVX512: LV: Found an estimated cost of 456 for VF 32 For instruction: store i8 %valB, i8* %out, align 1
; AVX512: LV: Found an estimated cost of 928 for VF 64 For instruction: store i8 %valB, i8* %out, align 1
;
-; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i8 %valB, i8* %out
+; CHECK-NOT: LV: Found an estimated cost of {{[0-9]+}} for VF {{[0-9]+}} For instruction: store i8 %valB, i8* %out, align 1
define void @test() {
entry:
br label %for.body
More information about the llvm-commits
mailing list