[llvm] ca5fcc7 - [CostModel][X86] Add CostKinds test coverage for cttz intrinsics

Tue Sep 6 06:39:42 PDT 2022

Author: Simon Pilgrim
Date: 2022-09-06T14:39:24+01:00
New Revision: ca5fcc7027fd74027660146060b2d7ba9787e7cd

URL: https://github.com/llvm/llvm-project/commit/ca5fcc7027fd74027660146060b2d7ba9787e7cd
DIFF: https://github.com/llvm/llvm-project/commit/ca5fcc7027fd74027660146060b2d7ba9787e7cd.diff

LOG: [CostModel][X86] Add CostKinds test coverage for cttz intrinsics

Added: 
    llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
    llvm/test/Analysis/CostModel/X86/cttz-latency.ll
    llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll

Modified: 
    

Removed: 
    


################################################################################
diff  --git a/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll b/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
new file mode 100644
index 000000000000..02a0e23288e2

--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/cttz-codesize.ll
@@ -0,0 +1,691 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=-bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,NOBMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,BMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+sse4.2 | FileCheck %s -check-prefixes=CHECK,BMI,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx | FileCheck %s -check-prefixes=CHECK,BMI,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx2 | FileCheck %s -check-prefixes=CHECK,BMI,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx512f | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx512vl,+avx512vpopcntdq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=code-size -mattr=+bmi,+avx512vl,+avx512bitalg | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+
+; Verify the cost of scalar trailing zero count instructions.
+
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare  i8 @llvm.cttz.i8(i8, i1)
+
+define i64 @var_cttz_i64(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 0)
+  ret i64 %cttz
+}
+
+define i64 @var_cttz_i64u(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 1)
+  ret i64 %cttz
+}
+
+define i32 @var_cttz_i32(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 0)
+  ret i32 %cttz
+}
+
+define i32 @var_cttz_i32u(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 1)
+  ret i32 %cttz
+}
+
+define i16 @var_cttz_i16(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 0)
+  ret i16 %cttz
+}
+
+define i16 @var_cttz_i16u(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 1)
+  ret i16 %cttz
+}
+
+define i8 @var_cttz_i8(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 0)
+  ret i8 %cttz
+}
+
+define i8 @var_cttz_i8u(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 1)
+  ret i8 %cttz
+}
+
+; Verify the cost of vector trailing zero count instructions.
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
+
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
+
+declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
+declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
+declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)
+declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)
+
+define <2 x i64> @var_cttz_v2i64(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 0)
+  ret <2 x i64> %cttz
+}
+
+define <2 x i64> @var_cttz_v2i64u(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 1)
+  ret <2 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 0)
+  ret <4 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64u(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 1)
+  ret <4 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 0)
+  ret <8 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64u(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 1)
+  ret <8 x i64> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 0)
+  ret <4 x i32> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32u(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 1)
+  ret <4 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 0)
+  ret <8 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32u(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 1)
+  ret <8 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 0)
+  ret <16 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32u(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 1)
+  ret <16 x i32> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 0)
+  ret <8 x i16> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16u(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 1)
+  ret <8 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 0)
+  ret <16 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16u(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 1)
+  ret <16 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 0)
+  ret <32 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16u(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 1)
+  ret <32 x i16> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 0)
+  ret <16 x i8> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8u(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 1)
+  ret <16 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 0)
+  ret <32 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8u(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 1)
+  ret <32 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 0)
+  ret <64 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8u(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 1)
+  ret <64 x i8> %cttz
+}

diff  --git a/llvm/test/Analysis/CostModel/X86/cttz-latency.ll b/llvm/test/Analysis/CostModel/X86/cttz-latency.ll
new file mode 100644
index 000000000000..06ec07a2eb9b
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/cttz-latency.ll
@@ -0,0 +1,691 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=-bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,NOBMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,BMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+sse4.2 | FileCheck %s -check-prefixes=CHECK,BMI,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx | FileCheck %s -check-prefixes=CHECK,BMI,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx2 | FileCheck %s -check-prefixes=CHECK,BMI,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx512f | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx512vl,+avx512vpopcntdq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=latency -mattr=+bmi,+avx512vl,+avx512bitalg | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+
+; Verify the cost of scalar trailing zero count instructions.
+
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare  i8 @llvm.cttz.i8(i8, i1)
+
+define i64 @var_cttz_i64(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 0)
+  ret i64 %cttz
+}
+
+define i64 @var_cttz_i64u(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 1)
+  ret i64 %cttz
+}
+
+define i32 @var_cttz_i32(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 0)
+  ret i32 %cttz
+}
+
+define i32 @var_cttz_i32u(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 1)
+  ret i32 %cttz
+}
+
+define i16 @var_cttz_i16(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 0)
+  ret i16 %cttz
+}
+
+define i16 @var_cttz_i16u(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 1)
+  ret i16 %cttz
+}
+
+define i8 @var_cttz_i8(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 0)
+  ret i8 %cttz
+}
+
+define i8 @var_cttz_i8u(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 1)
+  ret i8 %cttz
+}
+
+; Verify the cost of vector trailing zero count instructions.
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
+
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
+
+declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
+declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
+declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)
+declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)
+
+define <2 x i64> @var_cttz_v2i64(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 0)
+  ret <2 x i64> %cttz
+}
+
+define <2 x i64> @var_cttz_v2i64u(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 1)
+  ret <2 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 0)
+  ret <4 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64u(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 1)
+  ret <4 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 0)
+  ret <8 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64u(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 1)
+  ret <8 x i64> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 0)
+  ret <4 x i32> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32u(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 1)
+  ret <4 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 0)
+  ret <8 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32u(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 1)
+  ret <8 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 0)
+  ret <16 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32u(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 1)
+  ret <16 x i32> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 0)
+  ret <8 x i16> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16u(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 1)
+  ret <8 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 0)
+  ret <16 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16u(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 1)
+  ret <16 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 0)
+  ret <32 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16u(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 1)
+  ret <32 x i16> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 0)
+  ret <16 x i8> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8u(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 1)
+  ret <16 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 0)
+  ret <32 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8u(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 1)
+  ret <32 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 0)
+  ret <64 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8u(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 1)
+  ret <64 x i8> %cttz
+}

diff  --git a/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll b/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll
new file mode 100644
index 000000000000..1dae02478282
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/X86/cttz-sizelatency.ll
@@ -0,0 +1,691 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=-bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,NOBMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE2,BMI
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+sse4.2 | FileCheck %s -check-prefixes=CHECK,BMI,SSE42
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx | FileCheck %s -check-prefixes=CHECK,BMI,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx2 | FileCheck %s -check-prefixes=CHECK,BMI,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx512f | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx512vl,+avx512vpopcntdq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes="print<cost-model>" 2>&1 -disable-output -cost-kind=size-latency -mattr=+bmi,+avx512vl,+avx512bitalg | FileCheck %s -check-prefixes=CHECK,BMI,AVX512
+
+; Verify the cost of scalar trailing zero count instructions.
+
+declare i64 @llvm.cttz.i64(i64, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i16 @llvm.cttz.i16(i16, i1)
+declare  i8 @llvm.cttz.i8(i8, i1)
+
+define i64 @var_cttz_i64(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 0)
+  ret i64 %cttz
+}
+
+define i64 @var_cttz_i64u(i64 %a) {
+; NOBMI-LABEL: 'var_cttz_i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+; BMI-LABEL: 'var_cttz_i64u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i64 %cttz
+;
+  %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 1)
+  ret i64 %cttz
+}
+
+define i32 @var_cttz_i32(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 0)
+  ret i32 %cttz
+}
+
+define i32 @var_cttz_i32u(i32 %a) {
+; NOBMI-LABEL: 'var_cttz_i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+; BMI-LABEL: 'var_cttz_i32u'
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
+; BMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i32 %cttz
+;
+  %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 1)
+  ret i32 %cttz
+}
+
+define i16 @var_cttz_i16(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 0)
+  ret i16 %cttz
+}
+
+define i16 @var_cttz_i16u(i16 %a) {
+; CHECK-LABEL: 'var_cttz_i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i16 %cttz
+;
+  %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 1)
+  ret i16 %cttz
+}
+
+define i8 @var_cttz_i8(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 0)
+  ret i8 %cttz
+}
+
+define i8 @var_cttz_i8u(i8 %a) {
+; CHECK-LABEL: 'var_cttz_i8u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret i8 %cttz
+;
+  %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 1)
+  ret i8 %cttz
+}
+
+; Verify the cost of vector trailing zero count instructions.
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
+
+declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>, i1)
+declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>, i1)
+declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>, i1)
+declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>, i1)
+
+declare <8 x i64> @llvm.cttz.v8i64(<8 x i64>, i1)
+declare <16 x i32> @llvm.cttz.v16i32(<16 x i32>, i1)
+declare <32 x i16> @llvm.cttz.v32i16(<32 x i16>, i1)
+declare <64 x i8> @llvm.cttz.v64i8(<64 x i8>, i1)
+
+define <2 x i64> @var_cttz_v2i64(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 0)
+  ret <2 x i64> %cttz
+}
+
+define <2 x i64> @var_cttz_v2i64u(<2 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v2i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v2i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v2i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v2i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v2i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <2 x i64> %cttz
+;
+  %cttz = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 1)
+  ret <2 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 0)
+  ret <4 x i64> %cttz
+}
+
+define <4 x i64> @var_cttz_v4i64u(<4 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i64> %cttz
+;
+  %cttz = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %a, i1 1)
+  ret <4 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 0)
+  ret <8 x i64> %cttz
+}
+
+define <8 x i64> @var_cttz_v8i64u(<8 x i64> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i64u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i64u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i64u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i64u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i64u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i64> %cttz
+;
+  %cttz = call <8 x i64> @llvm.cttz.v8i64(<8 x i64> %a, i1 1)
+  ret <8 x i64> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 0)
+  ret <4 x i32> %cttz
+}
+
+define <4 x i32> @var_cttz_v4i32u(<4 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v4i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v4i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v4i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v4i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v4i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <4 x i32> %cttz
+;
+  %cttz = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 1)
+  ret <4 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 0)
+  ret <8 x i32> %cttz
+}
+
+define <8 x i32> @var_cttz_v8i32u(<8 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v8i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v8i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v8i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v8i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v8i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i32> %cttz
+;
+  %cttz = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %a, i1 1)
+  ret <8 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 0)
+  ret <16 x i32> %cttz
+}
+
+define <16 x i32> @var_cttz_v16i32u(<16 x i32> %a) {
+; NOBMI-LABEL: 'var_cttz_v16i32u'
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; NOBMI-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i32u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i32u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i32u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i32u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i32> %cttz
+;
+  %cttz = call <16 x i32> @llvm.cttz.v16i32(<16 x i32> %a, i1 1)
+  ret <16 x i32> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 false)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 0)
+  ret <8 x i16> %cttz
+}
+
+define <8 x i16> @var_cttz_v8i16u(<8 x i16> %a) {
+; CHECK-LABEL: 'var_cttz_v8i16u'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <8 x i16> %cttz
+;
+  %cttz = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 1)
+  ret <8 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 0)
+  ret <16 x i16> %cttz
+}
+
+define <16 x i16> @var_cttz_v16i16u(<16 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v16i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i16> %cttz
+;
+  %cttz = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %a, i1 1)
+  ret <16 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 0)
+  ret <32 x i16> %cttz
+}
+
+define <32 x i16> @var_cttz_v32i16u(<32 x i16> %a) {
+; SSE2-LABEL: 'var_cttz_v32i16u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i16u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i16u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i16u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 102 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i16u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i16> %cttz
+;
+  %cttz = call <32 x i16> @llvm.cttz.v32i16(<32 x i16> %a, i1 1)
+  ret <32 x i16> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 0)
+  ret <16 x i8> %cttz
+}
+
+define <16 x i8> @var_cttz_v16i8u(<16 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v16i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v16i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v16i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v16i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v16i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <16 x i8> %cttz
+;
+  %cttz = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 1)
+  ret <16 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 0)
+  ret <32 x i8> %cttz
+}
+
+define <32 x i8> @var_cttz_v32i8u(<32 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v32i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v32i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v32i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v32i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v32i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 99 for instruction: %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <32 x i8> %cttz
+;
+  %cttz = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %a, i1 1)
+  ret <32 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 false)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 0)
+  ret <64 x i8> %cttz
+}
+
+define <64 x i8> @var_cttz_v64i8u(<64 x i8> %a) {
+; SSE2-LABEL: 'var_cttz_v64i8u'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 312 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; SSE42-LABEL: 'var_cttz_v64i8u'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 192 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX1-LABEL: 'var_cttz_v64i8u'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX2-LABEL: 'var_cttz_v64i8u'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+; AVX512-LABEL: 'var_cttz_v64i8u'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 199 for instruction: %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 true)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret <64 x i8> %cttz
+;
+  %cttz = call <64 x i8> @llvm.cttz.v64i8(<64 x i8> %a, i1 1)
+  ret <64 x i8> %cttz
+}


        
